forest_admin_datasource_intercom 1.49.0

data/lib/forest_admin_datasource_intercom/collections/offset_collection.rb

@@ -0,0 +1,311 @@
+module ForestAdminDatasourceIntercom
+  module Collections
+    # `POST /companies/list` paginates by offset. Companies have no search endpoint
+    # (`/companies/scroll` allows one open scroll per app, expiring in a minute), so filters are
+    # limited to exact lookups and everything else is refused. Long by line count only: refusals.
+    class OffsetCollection < BaseCollection # rubocop:disable Metrics/ClassLength
+      include RecordsById
+
+      # What one page holds when the read names no window: a relation resolving
+      # its target, a segment, a customizer. A list view always names one.
+      UNBOUNDED_PAGE_SIZE = Client::MAX_PER_PAGE
+
+      # And how many such pages are read before the answer is cut short. The
+      # figure only ever applies to a read with no window of its own.
+      MAX_COLLECTED_PAGES = 10
+
+      # What a read that *does* name a window may collect before it is cut
+      # short. A window is its own bound, so the page count above must not apply
+      # to it: a window needing eleven pages would come back one page short
+      # while the warning blamed it for naming no window -- which it did name.
+      # This is the backstop for a window nothing sane asked for, and it is the
+      # cursor walker's own record budget, that tier being bounded the same way.
+      MAX_COLLECTED_RECORDS = Pagination::CursorWalker::MAX_RECORDS
+
+      # How many records of this collection one relation read may resolve.
+      # Higher than `MAX_ID_READS`, which bounds a single request batch, and
+      # deliberately above every page size a list view offers: a page naming one
+      # account per row resolves, and a read naming more -- an export, a
+      # segment resolved whole -- is refused by name rather than answered with
+      # a nil where an account exists. There is no bulk read for a company, so
+      # each one costs a request and the figure is what an operator waits for.
+      MAX_RELATION_READS = 100
+
+      # Search is declared only to be refused by name: left to the customizer,
+      # it would be emulated as equalities on columns no lookup answers.
+      def initialize(datasource, name)
+        super
+        enable_count
+        enable_search
+      end
+
+      def ids_per_read = MAX_ID_READS
+      def max_resolvable_ids = MAX_RELATION_READS
+
+      def list(caller, filter, projection)
+        warn_ignored_sort(filter&.sort)
+
+        client.with_read_scope do
+          records = fetch_records(filter)
+          serialized = records.map { |record| serialize(record) }
+          rows = serialized.map { |record| project(record, projection) }
+
+          embed_relations(caller, serialized, rows, projection)
+          rows
+        end
+      end
+
+      # Count only, and never a group: `total_count` is exact on every listing,
+      # while grouping the page in hand would answer a fraction as if it were
+      # the whole.
+      def aggregate(_caller, filter, aggregation, _limit = nil)
+        refuse_unsupported_aggregation!(aggregation)
+
+        [{ 'group' => {}, 'value' => count_records(filter) }]
+      end
+
+      protected
+
+      # The endpoint that lists the collection by offset, the one that reads a
+      # record, and the lookups Intercom answers on the listing path.
+      def list_path = raise(NotImplementedError, "#{self.class} did not implement list_path")
+      def record_endpoint = raise(NotImplementedError, "#{self.class} did not implement record_endpoint")
+      def lookup_path = record_endpoint
+      def lookups = {}
+
+      def serialize(_entity) = raise(NotImplementedError, "#{self.class} did not implement serialize")
+
+      # A column advertises a filter only where Intercom looks the collection up
+      # by it, so a column cannot offer a filter this tier would then refuse.
+      # The primary key is the exception and it is not a filter: `id equals X`
+      # and `id in [...]` are answered by the record endpoint.
+      #
+      # Nothing is sortable: the listing takes no order and ordering a page in
+      # hand would order a fraction of the collection.
+      def add_column(name, type, is_primary_key: false)
+        add_field(name, ColumnSchema.new(column_type: type,
+                                         filter_operators: column_operators(name, is_primary_key),
+                                         is_primary_key: is_primary_key,
+                                         is_read_only: true,
+                                         is_sortable: false,
+                                         is_groupable: false))
+      end
+
+      private
+
+      def column_operators(name, is_primary_key)
+        return [Operators::EQUAL, Operators::IN] if is_primary_key
+
+        lookups.key?(name) ? [Operators::EQUAL] : []
+      end
+
+      def fetch_records(filter)
+        refuse_search! unless blank_search?(filter)
+        return [] if matches_nothing?(filter)
+
+        ids = id_lookup(filter)
+        return records_by_ids(page_window(ids, filter)) if ids
+
+        lookup = lookup_condition(filter)
+        return page_window(looked_up_records(lookup), filter) if lookup
+
+        refuse_condition!(filter.condition_tree) unless filter&.condition_tree.nil?
+
+        listed_records(filter)
+      end
+
+      def count_records(filter)
+        refuse_search! unless blank_search?(filter)
+        return 0 if matches_nothing?(filter)
+
+        ids = id_lookup(filter)
+        return count_by_ids(ids) if ids
+
+        lookup = lookup_condition(filter)
+        return looked_up_count(lookup) if lookup
+
+        refuse_condition!(filter.condition_tree) unless filter&.condition_tree.nil?
+
+        exact_count(read_offset_page(page: 1, per_page: 1))
+      end
+
+      # The window a list view asked for, read as the page Intercom counts from
+      # 1. An offset that does not fall on a page boundary is served by reading
+      # the page it lands in and the ones after it until the window is filled --
+      # exactly, rather than by rounding the offset to something the API likes.
+      def listed_records(filter)
+        offset, limit = window(filter&.page)
+        return [] if limit&.zero?
+
+        per_page = Client.bounded_per_page(limit || UNBOUNDED_PAGE_SIZE)
+        skip = offset % per_page
+
+        collected = collect_pages(first_page: (offset / per_page) + 1, per_page: per_page,
+                                  wanted: limit && (skip + limit))
+
+        limit ? (collected[skip, limit] || []) : collected.drop(skip)
+      end
+
+      def collect_pages(first_page:, per_page:, wanted:)
+        records = []
+        page = first_page
+        read = 0
+
+        loop do
+          answer = read_offset_page(page: page, per_page: per_page)
+          records.concat(answer.records)
+          read += 1
+          break if last_page?(answer, page) || (wanted && records.size >= wanted)
+          break if cap_reached?(read, records.size, wanted)
+
+          page += 1
+        end
+
+        # Intercom numbers its pages by `per_page`, so the last one read cannot
+        # be asked smaller: what it brings past the budget is cut here instead.
+        wanted ? records.first(MAX_COLLECTED_RECORDS) : records
+      end
+
+      def read_offset_page(page:, per_page:)
+        client.offset_page(list_path, page: page, per_page: per_page)
+      end
+
+      def last_page?(answer, page)
+        answer.records.empty? || (answer.total_pages && page >= answer.total_pages)
+      end
+
+      # Which of the two caps applies, and the reason that goes with it: a read
+      # with no window of its own is stopped by the page count, a read that
+      # named one by the record budget. Telling a list view it named no window
+      # is exactly the confusion these two keep apart.
+      def cap_reached?(read, collected, wanted)
+        if wanted
+          return false if collected < MAX_COLLECTED_RECORDS
+
+          warn_capped("#{collected} record(s)",
+                      'this read named a window larger than one answer may hold')
+        else
+          return false if read < MAX_COLLECTED_PAGES
+
+          warn_capped("#{read} page(s) / #{collected} record(s)",
+                      'this read named no window of its own, and a list view always does')
+        end
+
+        true
+      end
+
+      def warn_capped(reached, reason)
+        ForestAdminDatasourceIntercom.logger.warn(
+          "[forest_admin_datasource_intercom] Stopped reading #{name} after #{reached}; the rest is left out: " \
+          "#{reason}."
+        )
+      end
+
+      # A filter with no page asks for every record it matched; nil is how the
+      # window says so.
+      def window(page)
+        return [0, nil] if page.nil?
+
+        [page.offset.to_i.clamp(0, nil), page_limit(page)]
+      end
+
+      # `Or []`, which `ConditionTreeFactory.match_ids` builds from no ids at all
+      # -- a live-query segment that returned none.
+      def matches_nothing?(filter)
+        tree = filter&.condition_tree
+        tree.is_a?(Branch) && tree.aggregator.to_s.casecmp('or').zero? && Array(tree.conditions).empty?
+      end
+
+      def lookup_condition(filter)
+        tree = filter&.condition_tree
+        return nil unless tree.is_a?(Leaf) && tree.operator == Operators::EQUAL
+
+        parameter = lookups[tree.field.to_s]
+        parameter && { parameter => tree.value.to_s }
+      end
+
+      # Read as a single page; overflow is reported. Intercom answers a lookup matching nothing
+      # with a 404, read here as an empty page.
+      def looked_up_records(params)
+        answer = lookup(params)
+        return [] unless answer
+
+        warn_truncated_lookup(params) if answer.next_cursor
+        answer.records
+      end
+
+      # Intercom's own count when it gave one. A lookup spread over several
+      # pages without one cannot be counted from the first page alone.
+      def looked_up_count(params)
+        answer = lookup(params)
+        return 0 unless answer
+        return answer.total_count if answer.total_count
+        return exact_count(answer) if answer.next_cursor
+
+        answer.records.size
+      end
+
+      def lookup(params)
+        client.lookup_page(lookup_path, params: params)
+      rescue APIError => e
+        raise unless e.status == 404
+
+        nil
+      end
+
+      def refuse_search!
+        raise UnsupportedOperatorError,
+              "#{name} cannot answer a free-text search: Intercom exposes no search endpoint for this collection " \
+              "and looks a record up by id, #{lookups.keys.join(", ")} alone, one exact value at a time. Filter " \
+              'on one of those instead of searching.'
+      end
+
+      def refuse_condition!(tree)
+        leaves = []
+        tree.some_leaf do |leaf|
+          leaves << leaf
+          false
+        end
+        offender = leaves.find { |leaf| !published?(leaf) }
+        refused = if offender
+                    "filter #{offender.field.inspect}"
+                  else
+                    "combine conditions on #{leaves.map(&:field).uniq.join(", ")}"
+                  end
+
+        raise UnsupportedOperatorError,
+              "#{name} cannot #{refused}: Intercom exposes no search endpoint for this " \
+              "collection and looks a record up by #{lookups.keys.join(", ")} alone -- one exact value at a time, " \
+              'with no combination and no other operator. Filter on one of those, or reach the record from the ' \
+              'collection next door.'
+      end
+
+      def published?(leaf)
+        schema = fields[leaf.field]
+        schema.respond_to?(:filter_operators) && schema.filter_operators.include?(leaf.operator)
+      end
+
+      # Intercom takes no order on this listing at all -- there is no parameter
+      # for one -- so an order asked for and not applied is reported here or
+      # nowhere. The ascending primary-key sort the agent injects when a request
+      # names none is not one of those.
+      def warn_ignored_sort(sort)
+        clauses = Array(sort)
+        return if clauses.empty? || default_pk_sort?(clauses)
+
+        ForestAdminDatasourceIntercom.logger.warn(
+          "[forest_admin_datasource_intercom] #{name} was asked to sort on " \
+          "#{clauses.map { |clause| sort_field(clause) }.join(", ")}, and Intercom takes no order " \
+          'on this listing. The rows come back in the order the API imposes.'
+        )
+      end
+
+      def warn_truncated_lookup(params)
+        ForestAdminDatasourceIntercom.logger.warn(
+          "[forest_admin_datasource_intercom] #{name} looked up #{params.inspect} and Intercom advertised more " \
+          'records than one page holds; the rest is left out. This lookup is meant for a key that names one record.'
+        )
+      end
+    end
+  end
+end

data/lib/forest_admin_datasource_intercom/collections/records_by_id.rb

@@ -0,0 +1,97 @@
+module ForestAdminDatasourceIntercom
+  module Collections
+    # `id equals X` / `id in [...]` answered by the record endpoint, plus exact counting -- shared
+    # by the two paginated tiers, where a count over collected pages would answer a fraction.
+    module RecordsById
+      Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators
+      Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf
+      Aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation
+
+      # How many records a bulk read by id may fetch. Intercom exposes no "read
+      # these records" endpoint, so a set of ids is one request each -- the
+      # fan-out is bounded rather than turned into a rate limit halfway through
+      # a page. The collection that reads its ids in bulk raises the figure.
+      MAX_ID_READS = 25
+
+      protected
+
+      # The endpoint one record is read from.
+      def record_endpoint = raise(NotImplementedError, "#{self.class} did not implement record_endpoint")
+
+      # What rides along on that read. `display_as=plaintext` for the tiers
+      # whose records carry bodies written by end customers; nothing elsewhere.
+      def record_read_params = {}
+
+      def max_id_reads = MAX_ID_READS
+
+      # A bare primary-key leaf only: answering an `and` carrying a scope, or a search, with the
+      # ids alone would serve records it excludes. Deduplicated so an id is not read twice.
+      def id_lookup(filter)
+        tree = filter&.condition_tree
+        return nil unless tree.is_a?(Leaf) && tree.field.to_s == primary_key
+        return nil unless blank_search?(filter)
+
+        case tree.operator
+        when Operators::EQUAL then [tree.value].compact.map(&:to_s).uniq
+        when Operators::IN then Array(tree.value).compact.map(&:to_s).uniq
+        end
+      end
+
+      # A record the operator can no longer reach -- deleted, or outside the
+      # token's scope -- reads as "no record" rather than as a failed page.
+      def records_by_ids(ids)
+        refuse_id_reads!(ids.size, 'list') if ids.size > max_id_reads
+
+        client.fetch_records(record_endpoint, ids, params: record_read_params).compact
+      end
+
+      # How many of a set of ids name a record, which means reading them:
+      # Intercom has no "how many of these exist" endpoint. Past what a bulk
+      # read may fetch the count is refused rather than answered with the number
+      # the truncation left -- both paginated tiers advertise an exact count,
+      # and "25" where the question named forty records is not one.
+      def count_by_ids(ids)
+        refuse_id_reads!(ids.size, 'count') if ids.size > max_id_reads
+
+        records_by_ids(ids).size
+      end
+
+      # The count Intercom answered, or nothing at all. Counting the pages a
+      # read collected would answer a fraction of the collection as if it were
+      # the whole of it, which is the one thing the paginated tiers do not do.
+      def exact_count(page)
+        return page.total_count if page.total_count
+
+        raise UnsupportedOperatorError,
+              "#{name} cannot be counted: Intercom answered this listing without a total_count, and counting the " \
+              'pages the agent read would answer a fraction of the collection as if it were the whole of it.'
+      end
+
+      def refuse_unsupported_aggregation!(aggregation)
+        return if aggregation.is_a?(Aggregation) && aggregation.operation.to_s.casecmp('count').zero? &&
+                  Array(aggregation.groups).empty? && aggregation.field.nil?
+
+        raise UnsupportedOperatorError,
+              "#{name} can only be counted: Intercom exposes no aggregate endpoint, and grouping or summing the " \
+              'pages the agent read would answer a fraction of the collection as if it were the whole of it. ' \
+              'Chart it on a collection read whole, or wait for the bounded group-by of the reporting lot.'
+      end
+
+      # The same refusal for a list as for a count: the first `max_id_reads`
+      # rows of a longer set of ids look like the whole of it.
+      def refuse_id_reads!(asked, verb)
+        raise UnsupportedOperatorError,
+              "#{name} cannot #{verb} #{asked} records by id: a set of ids is read from Intercom record by " \
+              "record or in small batches, and this reads #{max_id_reads} at most. Answering with the ones it read " \
+              'would leave the rest out of an answer that looks whole. Ask for fewer records at a time, or use a ' \
+              'filter this collection answers instead of a list of ids.'
+      end
+
+      def blank_search?(filter)
+        search = filter.respond_to?(:search) ? filter.search : nil
+
+        search.nil? || search.to_s.strip.empty?
+      end
+    end
+  end
+end