forest_admin_datasource_intercom 1.49.0

data/lib/forest_admin_datasource_intercom/rate_limiter.rb

@@ -0,0 +1,148 @@
+module ForestAdminDatasourceIntercom
+  # Paces on Intercom's x-ratelimit-* headers. Workspace budget is 25 000/min but allocated in 10s windows
+  # (measured x-ratelimit-limit 1667), so the instantaneous rate matters. Paces, does not queue: callers
+  # woken at a reset can burst unmetered until the next response; the 429 retry behind this absorbs it.
+  class RateLimiter
+    # Intercom's allocation window. Only used as the ceiling below: the reset
+    # instant itself always comes from the response.
+    WINDOW = 10.0
+
+    # How far past the reset a request may be held. A little over one window, so
+    # a full window can be waited out, and no more: past this the wait is not
+    # Intercom's window emptying but a clock disagreeing.
+    DEFAULT_MAX_WAIT = 12.0
+
+    attr_reader :max_wait
+
+    def initialize(max_wait: DEFAULT_MAX_WAIT, now: nil, sleeper: nil)
+      @max_wait = max_wait.to_f
+      # Wall clock rather than monotonic on purpose: `X-RateLimit-Reset` is an
+      # absolute epoch second on Intercom's clock, so the two have to be
+      # comparable. `clamp_wait` is what keeps a skewed clock from turning that
+      # comparison into a long sleep.
+      @now      = now || -> { Time.now.to_f }
+      @sleeper  = sleeper || ->(seconds) { sleep(seconds) }
+      @mutex    = Mutex.new
+      @remaining = nil
+      @reset_at  = nil
+      @limit     = nil
+      @warned_at = nil
+    end
+
+    # Blocks until the current window has room, then returns. Called once per
+    # attempt, retries included: a replayed request spends the budget a first
+    # one did.
+    def acquire
+      wait, declined = @mutex.synchronize { plan_wait }
+
+      warn_saturated(declined) if declined
+      return if wait <= 0
+
+      @sleeper.call(wait)
+    end
+
+    # What a response says about the window it was answered in. Called on every
+    # response, the 429 included -- that one carries the most useful reset of
+    # all.
+    def observe(headers)
+      limit     = integer_header(headers, 'x-ratelimit-limit')
+      remaining = integer_header(headers, 'x-ratelimit-remaining')
+      reset_at  = integer_header(headers, 'x-ratelimit-reset')
+      return if remaining.nil? && reset_at.nil?
+
+      @mutex.synchronize { record(limit, remaining, reset_at) }
+    end
+
+    private
+
+    # Decrements locally so in-flight requests do not all pass on the same stale `remaining`; a
+    # `remaining` of 1 still allows this request. The declined wait is returned only once per window.
+    def plan_wait
+      unless exhausted?
+        @remaining -= 1 if @remaining
+        return [0, nil]
+      end
+
+      wait = clamp_wait(@reset_at - @now.call)
+      return [0, nil] if wait <= 0
+      return [0, first_warning? ? wait : nil] if wait > @max_wait
+
+      [wait, nil]
+    end
+
+    # Nothing left in a window that has not refilled yet. An unknown state --
+    # before the first response -- is not exhaustion: the first request is what
+    # discovers the budget.
+    def exhausted?
+      !@remaining.nil? && @remaining <= 0 && !@reset_at.nil?
+    end
+
+    # Intercom's reset is a timestamp from its clock, and the two clocks can
+    # disagree by more than the window is long. A wait longer than a window plus
+    # its own slack is that disagreement rather than a window emptying, so it is
+    # cut back to something a caller can afford to wait.
+    def clamp_wait(seconds)
+      return 0.0 if seconds <= 0
+
+      [seconds, WINDOW + @max_wait].min
+    end
+
+    # A window is adopted whole. A response answered in an older window than the
+    # one already recorded is ignored: replies come back out of order, and one
+    # from the previous window would otherwise resurrect a budget already spent.
+    #
+    # Within the same window the smaller `remaining` wins, so the local
+    # decrements of in-flight requests are not undone by a response that left
+    # Intercom before they were made.
+    def record(limit, remaining, reset_at)
+      return if reset_at && @reset_at && reset_at < @reset_at
+
+      new_window = reset_at && (@reset_at.nil? || reset_at > @reset_at)
+      @reset_at  = reset_at if reset_at
+      @limit     = limit if limit
+      return if remaining.nil?
+
+      @remaining = new_window || @remaining.nil? ? remaining : [remaining, @remaining].min
+    end
+
+    # One line per window. What the warning reports is a saturation that lasts,
+    # so a line per request puts one on every request it describes -- hundreds
+    # of them, burying the first, which is the only one the operator needed.
+    def first_warning?
+      now = @now.call
+      return false if @warned_at && now - @warned_at < WINDOW
+
+      @warned_at = now
+      true
+    end
+
+    def warn_saturated(wait)
+      ForestAdminDatasourceIntercom.logger.warn(
+        "[forest_admin_datasource_intercom] the Intercom rate-limit window is spent (limit #{@limit || "unknown"} " \
+        "per #{WINDOW.round}s) and its reset is #{wait.round(1)}s out, past the #{@max_wait.round(1)}s this waits. " \
+        'Letting the request through -- Intercom may answer 429, which the client retries. A reset this far out ' \
+        "usually means this host's clock disagrees with Intercom's. Reported once per #{WINDOW.round}s."
+      )
+    end
+
+    def integer_header(headers, name)
+      value = header_value(headers, name)
+      return nil if value.nil? || value.to_s.strip.empty?
+
+      Integer(value.to_s.strip, exception: false)
+    end
+
+    # Faraday hands over headers that look themselves up case-insensitively, but
+    # what reaches here is whatever the middleware was given -- a plain hash
+    # included -- and HTTP header names are case-insensitive on the wire.
+    def header_value(headers, name)
+      return nil if headers.nil?
+
+      direct = headers[name]
+      return direct unless direct.nil?
+      return nil unless headers.respond_to?(:find)
+
+      headers.find { |key, _value| key.to_s.downcase == name }&.last
+    end
+  end
+end

data/lib/forest_admin_datasource_intercom/retry_policy.rb

@@ -0,0 +1,95 @@
+module ForestAdminDatasourceIntercom
+  # Everything governing how the client reacts to a failed request, in one
+  # place: which statuses and exceptions are worth another attempt, on which
+  # verbs, and how long to wait.
+  class RetryPolicy
+    # Intercom allocates its quota in 10-second windows, so a 429 is recovered
+    # from within one of them -- unlike an API metering by the minute. The cap
+    # still has to cover a whole window: faraday-retry gives up outright when
+    # Retry-After exceeds max_interval, which would turn the 429 retry into an
+    # immediate give-up exactly when it matters.
+    DEFAULT_MAX_INTERVAL = 12
+
+    STATUSES = [429, 500, 502, 503, 504].freeze
+
+    # Named rather than referenced: the persistent adapter is a gem only loaded
+    # when it is the configured adapter -- `adapter: :net_http` never requires
+    # it -- so this file cannot name the class outright. Both places that need
+    # it resolve it at the moment they are asked, and answer that it cannot have
+    # been raised when nothing defined it.
+    PERSISTENT_ADAPTER_ERROR = 'Net::HTTP::Persistent::Error'.freeze
+
+    # faraday-retry's defaults plus ConnectionFailed, and the persistent adapter's own error, which it
+    # re-raises unwrapped for messages it does not translate.
+    EXCEPTIONS = [
+      Errno::ETIMEDOUT, 'Timeout::Error', Faraday::TimeoutError,
+      Faraday::RetriableResponse, Faraday::ConnectionFailed,
+      PERSISTENT_ADAPTER_ERROR
+    ].freeze
+
+    # The verbs that change nothing, so any transient failure is worth another
+    # attempt. Narrower than faraday-retry's idempotent default: a 502 or a
+    # dropped connection on the way back from a POST Intercom did perform would
+    # be replayed into a second reply on the conversation, or a second ticket.
+    RETRYABLE_METHODS = %i[get head options].freeze
+
+    # The paths Intercom answers a *read* on through POST: its three search
+    # endpoints, and the one listing that paginates by offset. Nothing that
+    # writes is spelled this way -- a reply is `POST /conversations/{id}/reply`,
+    # a ticket `POST /tickets` -- so the exemption below cannot reach one.
+    READ_ONLY_POST_PATHS = %r{/(search|list)\z}
+
+    # Widens `methods` (ORed by faraday-retry) to a 429 on any verb, and to a dropped connection on a
+    # read-only POST -- keep-alive makes an idle socket closed by the server an ordinary failure.
+    RETRY_IF = lambda do |env, exception|
+      env[:status] == 429 || (RetryPolicy.connection_failure?(exception) &&
+                              READ_ONLY_POST_PATHS.match?(env[:url]&.path.to_s))
+    end
+
+    # Whether another attempt would be dialling again rather than re-sending.
+    # See `PERSISTENT_ADAPTER_ERROR` for why one of the two is named.
+    def self.connection_failure?(exception)
+      return true if exception.is_a?(Faraday::ConnectionFailed)
+      return false unless Object.const_defined?(PERSISTENT_ADAPTER_ERROR)
+
+      exception.is_a?(Object.const_get(PERSISTENT_ADAPTER_ERROR))
+    end
+
+    # The cap for a call that must not hold the boot, deliberately below a
+    # rate-limit window where DEFAULT_MAX_INTERVAL sits above it: a Retry-After
+    # past the cap makes faraday-retry abandon outright, which is what turns a
+    # 429 at boot into an immediate give-up rather than a window of waiting per
+    # attempt.
+    BOOT_MAX_INTERVAL = 2
+
+    BACKOFF_FACTOR = 2
+
+    attr_reader :max_retries, :interval, :max_interval
+
+    # One retry rather than none, for what is read once and never revisited: a
+    # transient failure there costs its result for the whole life of the
+    # process, and half a second absorbs the hiccup without waiting a 429 out.
+    def self.boot
+      new(max_retries: 1, interval: 0.5, max_interval: BOOT_MAX_INTERVAL)
+    end
+
+    def initialize(max_retries: 3, interval: 0.5, max_interval: DEFAULT_MAX_INTERVAL)
+      @max_retries  = max_retries
+      @interval     = interval
+      @max_interval = max_interval
+    end
+
+    def to_faraday_options
+      {
+        max: @max_retries,
+        interval: @interval,
+        max_interval: @max_interval,
+        backoff_factor: BACKOFF_FACTOR,
+        retry_statuses: STATUSES,
+        exceptions: EXCEPTIONS,
+        methods: RETRYABLE_METHODS,
+        retry_if: RETRY_IF
+      }
+    end
+  end
+end

data/lib/forest_admin_datasource_intercom/schema/attribute_naming.rb

@@ -0,0 +1,52 @@
+module ForestAdminDatasourceIntercom
+  module Schema
+    # What the two introspectors do identically: turn the name a workspace gave
+    # an attribute into a name a Forest schema can carry, and say which
+    # attribute was left out when two of them land on one column.
+    #
+    # They differ in where the attributes come from -- `/ticket_types` per
+    # ticket type, `/data_attributes` per model -- and in the data types they
+    # map, which is why `COLUMN_TYPES` stays with each of them.
+    module AttributeNaming
+      # What a column name may not contain, and it has nothing to do with
+      # Intercom: Forest lists the fields of a request in a **comma-separated**
+      # query parameter, and uses a colon to name a field through a relation.
+      # A workspace names its attributes in free text -- measured, one is called
+      # `ID de l'objet en question (immo, facture, user)` -- and a comma in
+      # there splits the projection into fields no collection has, which the
+      # agent rejects as a 400 before the page is ever read.
+      UNSAFE_IN_A_COLUMN_NAME = /[,:]/
+
+      private
+
+      # Intercom hands these back HTML-escaped -- `Ce que j'ai vérifié` --
+      # which is an artefact of where they were typed, not part of the name.
+      def column_name_for(name)
+        CGI.unescapeHTML(name).gsub(UNSAFE_IN_A_COLUMN_NAME, ' ').squeeze(' ').strip
+      end
+
+      # An unknown data type reads as a string rather than being dropped:
+      # showing the value Intercom sent beats hiding a column because its type
+      # is new.
+      def column_type_for(definition)
+        self.class::COLUMN_TYPES.fetch(definition['data_type'].to_s, self.class::DEFAULT_COLUMN_TYPE)
+      end
+
+      # Two attributes reading as one column: the second is left out rather than
+      # sharing the first's entry, which would show its values under the first's
+      # name -- wrong values rather than missing ones, and worse.
+      #
+      # `attribute_kind` is the workspace's own vocabulary for these, which is
+      # what the operator has to go and rename.
+      def warn_collision(name, kept, column)
+        ForestAdminDatasourceIntercom.logger.warn(
+          "[forest_admin_datasource_intercom] the #{attribute_kind} attribute #{name.inspect} is left out: it " \
+          "reads as the column #{column.inspect}, which #{kept.inspect} already carries. Rename one of them in " \
+          'Intercom to publish both.'
+        )
+      end
+
+      def attribute_kind = raise(NotImplementedError, "#{self.class} did not implement attribute_kind")
+    end
+  end
+end

data/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb

@@ -0,0 +1,88 @@
+module ForestAdminDatasourceIntercom
+  module Schema
+    # Declared per model, so each column maps to exactly one attribute. Still display-only: which operators
+    # Intercom answers on `custom_attributes.{name}` is unmeasured. `api_writable` is carried for lot 4b.
+    class DataAttributesIntrospector
+      include AttributeNaming
+
+      # Intercom's attribute data types, mapped onto what Forest can render.
+      COLUMN_TYPES = {
+        'string' => 'String', 'integer' => 'Number', 'float' => 'Number', 'decimal' => 'Number',
+        'boolean' => 'Boolean', 'date' => 'Date', 'datetime' => 'Date'
+      }.freeze
+
+      DEFAULT_COLUMN_TYPE = 'String'.freeze
+
+      # `name` is the key `custom_attributes` uses, `column_name` the one the
+      # schema publishes; they differ whenever the workspace's own name cannot
+      # travel through Forest's query string.
+      Attribute = Struct.new(:name, :column_name, :column_type, :data_type, :api_writable, keyword_init: true)
+
+      def initialize(client, model:)
+        @client = client
+        @model = model
+      end
+
+      # A token without the permission on this model costs the custom-attribute
+      # columns, never the boot of the agent. Any other failure fails the boot:
+      # degraded on a timeout or a 5xx, this pod would publish a schema missing
+      # columns the next one has, and keep it for the life of the process.
+      def attributes
+        @attributes ||= build
+      rescue APIError => e
+        raise unless e.status == 403
+
+        ForestAdminDatasourceIntercom.logger.warn(
+          "[forest_admin_datasource_intercom] could not read the #{@model} attributes (#{e.message}); " \
+          'the collection boots without its custom-attribute columns.'
+        )
+        @attributes = []
+      end
+
+      private
+
+      def build
+        # Read on the boot connection: this happens while Rails is starting, and
+        # a slow Intercom must not turn that into minutes the operator sits
+        # through.
+        @client.fetch_all('data_attributes', params: { 'model' => @model }, boot: true)
+               .each_with_object({}) { |definition, union| collect(definition, union) }
+               .values
+      end
+
+      # The standard attributes are left out: they are columns this datasource
+      # declares by hand, with the filters the search table measured, and
+      # publishing them a second time under their `custom_attributes` name would
+      # show one fact twice -- the unfilterable copy winning nothing.
+      def collect(definition, union)
+        return unless definition.is_a?(Hash) && definition['custom'] && !definition['archived']
+
+        name = definition['name'].to_s
+        return if name.empty?
+
+        column = column_name_for(name)
+        return if column.empty?
+
+        add(union, name, column, definition)
+      end
+
+      def add(union, name, column, definition)
+        entry = union[column]
+        return union[column] = attribute_from(name, column, definition) if entry.nil?
+        return if entry.name == name
+
+        warn_collision(name, entry.name, column)
+      end
+
+      def attribute_from(name, column, definition)
+        Attribute.new(name: name, column_name: column, column_type: column_type_for(definition),
+                      data_type: definition['data_type'], api_writable: definition['api_writable'] == true)
+      end
+
+      # These are declared per model, so that is what the log calls them: it is
+      # the workspace's own vocabulary, and where the operator goes to rename
+      # the one that was left out.
+      def attribute_kind = @model
+    end
+  end
+end

data/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb

@@ -0,0 +1,133 @@
+module ForestAdminDatasourceIntercom
+  module Schema
+    # Declared per ticket type: one name has a different id on each type (measured: 14162161 vs 14162165) and
+    # Intercom filters by id only, so the union columns ship unfilterable (R7). Ids are kept per type for a
+    # future per-type filter translation, sparing a second boot-time round trip.
+    class TicketAttributesIntrospector
+      include AttributeNaming
+
+      # Intercom's attribute data types, mapped onto what Forest can render. A
+      # `list` is a single choice among values the workspace defined, so it reads
+      # as a string rather than as a Json blob; `files` is a list of attachments
+      # and has no scalar form at all.
+      COLUMN_TYPES = {
+        'string' => 'String', 'list' => 'String', 'integer' => 'Number', 'decimal' => 'Number',
+        'boolean' => 'Boolean', 'datetime' => 'Date', 'date' => 'Date', 'files' => 'Json'
+      }.freeze
+
+      DEFAULT_COLUMN_TYPE = 'String'.freeze
+
+      # `name` is the key the payload uses, `column_name` the one the schema
+      # publishes; they differ whenever the workspace's own name cannot travel
+      # through Forest's query string.
+      #
+      # `conflicting_ticket_types` names the ticket types declaring the same name
+      # with another type: their value would not be what the column holds.
+      Attribute = Struct.new(:name, :column_name, :column_type, :data_type, :ids_by_ticket_type,
+                             :conflicting_ticket_types, keyword_init: true)
+
+      def initialize(client)
+        @client = client
+      end
+
+      # The union, one entry per attribute name. A token without the
+      # ticket-types permission costs the attribute columns, never the boot of
+      # the agent; any other failure fails the boot, for the reason
+      # `DataAttributesIntrospector#attributes` gives.
+      def attributes
+        @attributes ||= build
+      rescue APIError => e
+        raise unless e.status == 403
+
+        ForestAdminDatasourceIntercom.logger.warn(
+          "[forest_admin_datasource_intercom] could not read the ticket types (#{e.message}); " \
+          'the Tickets collection boots without its attribute columns.'
+        )
+        @attributes = []
+      end
+
+      private
+
+      def build
+        # Read on the boot connection: this happens while Rails is starting, and
+        # a slow Intercom must not turn that into minutes the operator sits
+        # through.
+        @client.fetch_all('ticket_types', boot: true)
+               .each_with_object({}) { |ticket_type, union| collect(ticket_type, union) }
+               .values
+      end
+
+      def collect(ticket_type, union)
+        return unless ticket_type.is_a?(Hash)
+
+        type_id = ticket_type['id'].to_s
+        definitions(ticket_type).each do |definition|
+          entry = entry_for(definition, union, type_id)
+          next if entry.nil?
+
+          entry.ids_by_ticket_type[type_id] = definition['id'].to_s
+        end
+      end
+
+      # The union is keyed by column name rather than by the workspace's own,
+      # since that is what has to be unique in a schema. Two different attributes
+      # landing on one column would otherwise share an entry, and the second's
+      # values would be read under the first's name -- wrong values rather than
+      # missing ones, which is worse.
+      def entry_for(definition, union, type_id)
+        name = definition['name'].to_s
+        # An archived attribute is not offered any more, and a nameless one has
+        # nothing to be a column of.
+        return nil if name.empty? || definition['archived']
+
+        column = column_name_for(name)
+        return nil if column.empty?
+
+        entry = union[column]
+        return union[column] = attribute_from(name, column, definition) if entry.nil?
+        return same_name(entry, definition, type_id) if entry.name == name
+
+        warn_collision(name, entry.name, column)
+        nil
+      end
+
+      # The same name declared with another type by another ticket type: that
+      # ticket type's value is left out of the column rather than shown under a
+      # type it does not have.
+      def same_name(entry, definition, type_id)
+        column_type = column_type_for(definition)
+        return entry if column_type == entry.column_type
+
+        entry.conflicting_ticket_types << type_id
+        warn_type_conflict(entry, column_type, type_id)
+        nil
+      end
+
+      def warn_type_conflict(entry, column_type, type_id)
+        ForestAdminDatasourceIntercom.logger.warn(
+          "[forest_admin_datasource_intercom] the ticket attribute #{entry.name.inspect} is a #{column_type} on " \
+          "ticket type #{type_id} and a #{entry.column_type} elsewhere; the column stays a #{entry.column_type} " \
+          'and reads as empty on that ticket type. Give the two attributes different names to publish both.'
+        )
+      end
+
+      def attribute_from(name, column, definition)
+        Attribute.new(name: name, column_name: column, column_type: column_type_for(definition),
+                      data_type: definition['data_type'], ids_by_ticket_type: {}, conflicting_ticket_types: [])
+      end
+
+      def definitions(ticket_type)
+        container = ticket_type['ticket_type_attributes']
+        return [] unless container.is_a?(Hash)
+
+        list = container['data']
+        list.is_a?(Array) ? list.grep(Hash) : []
+      end
+
+      # These are declared per ticket type, and that is what the log calls
+      # them: the workspace's own vocabulary, and where the operator goes to
+      # rename the one that was left out.
+      def attribute_kind = 'ticket'
+    end
+  end
+end

data/lib/forest_admin_datasource_intercom/throttle.rb

@@ -0,0 +1,19 @@
+module ForestAdminDatasourceIntercom
+  # Holds a request until the rate-limit window has room, and feeds the window
+  # back what the response says about it. A middleware rather than a call in
+  # each client method: there is one code path for every request here, where the
+  # client has one per endpoint, and this one also covers the requests the client
+  # never issues itself -- the replays `retry` performs.
+  class Throttle < Faraday::Middleware
+    def initialize(app, limiter:)
+      super(app)
+      @limiter = limiter
+    end
+
+    def call(env)
+      @limiter.acquire
+
+      @app.call(env).on_complete { |response_env| @limiter.observe(response_env[:response_headers]) }
+    end
+  end
+end

data/lib/forest_admin_datasource_intercom/version.rb

@@ -0,0 +1,3 @@
+module ForestAdminDatasourceIntercom
+  VERSION = "1.49.0"
+end

data/lib/forest_admin_datasource_intercom.rb

@@ -0,0 +1,57 @@
+require_relative 'forest_admin_datasource_intercom/version'
+require 'cgi/escape'
+require 'json'
+require 'logger'
+require 'set'
+require 'time'
+require 'uri'
+require 'yaml'
+require 'zeitwerk'
+# Required explicitly rather than relied on via the toolkit: `active_support` brings the autoload
+# `Time.use_zone` needs, `active_support/time` brings `Time.zone` (see `Query::CallerZone`).
+require 'active_support'
+require 'active_support/time'
+require 'faraday'
+require 'faraday/retry'
+require 'forest_admin_datasource_toolkit'
+
+loader = Zeitwerk::Loader.for_gem
+loader.setup
+
+module ForestAdminDatasourceIntercom
+  class Error < StandardError; end
+  class ConfigurationError < Error; end
+
+  # Descends from the toolkit's ValidationError so the agent answers 400 with the message, not a 500:
+  # the datasource refuses a filter Intercom cannot express exactly rather than approximating it.
+  class UnsupportedOperatorError < ForestAdminDatasourceToolkit::Exceptions::ValidationError; end
+
+  # Raised when an Intercom API call fails. Carries the HTTP status and the
+  # parsed response body so callers can surface Intercom's own error message
+  # instead of a generic string.
+  class APIError < Error
+    attr_reader :status, :body
+
+    def initialize(message, status: nil, body: nil)
+      super(message)
+      @status = status
+      @body = body
+    end
+  end
+
+  class << self
+    attr_writer :logger
+
+    def logger
+      @logger ||= default_logger
+    end
+
+    private
+
+    def default_logger
+      return Rails.logger if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+
+      Logger.new($stderr).tap { |l| l.progname = 'forest_admin_datasource_intercom' }
+    end
+  end
+end