forest_admin_datasource_intercom 1.49.0

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/exe/forest_admin_intercom_probe

@@ -0,0 +1,287 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Enumerates what Intercom's search endpoints really filter, and with which
+# operators, against a real workspace. The answer is what `search_fields.yml`
+# holds and what the schema derives its filters from -- and it cannot be read
+# off the documentation: measured during lot 1, `/tickets/search` refuses
+# `company_id` with `invalid_field` although a ticket carries one.
+#
+#   INTERCOM_ACCESS_TOKEN=... forest_admin_intercom_probe --endpoint tickets
+#   INTERCOM_ACCESS_TOKEN=... forest_admin_intercom_probe --region eu --out measured.yml
+#
+# Shipped with the gem rather than left in the repository: what it measures is
+# the customer's workspace, and the operator running it has the gem installed
+# and not a clone of the agent.
+#
+# It sends one search per (field, operator) cell, asking for a single record,
+# and reads Intercom's refusal codes: `invalid_field` for a field the endpoint
+# does not filter at all -- the rest of its row is then skipped -- `data_invalid`
+# for an operator it refuses on that field, `invalid_value` for a value shape it
+# refuses. Read-only: a search changes nothing, and no payload is printed or
+# written, only field names, operators and error codes.
+#
+# What it writes with `--out` is *evidence*, deliberately not the table itself:
+# `search_fields.yml` carries the prose that says why a column stays refused,
+# and a generated file would drop it. Merge the measurements into the table by
+# hand, which is also where deciding to expose a newly discovered field belongs.
+
+$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
+
+require 'forest_admin_datasource_intercom'
+require 'optparse'
+
+module ProbeSearchFields
+  SearchFields = ForestAdminDatasourceIntercom::Query::SearchFields
+
+  # Whether this file is being run as the command rather than loaded by a
+  # spec. The basenames and not the paths: RubyGems installs an executable
+  # behind a stub of its own and `load`s this file from it, so `$PROGRAM_NAME`
+  # is the stub while `__FILE__` is this file -- comparing the two would make
+  # the shipped command exit without doing anything, which is the one thing
+  # shipping it was for. A spec loading it runs under `rspec`, whose basename
+  # is not this one.
+  def self.invoked_as_command?(program_name)
+    File.basename(program_name.to_s) == File.basename(__FILE__)
+  end
+
+  # A cell is probed with the value shapes its field plausibly takes, most
+  # likely first: a wrong shape is refused with `data_invalid` just like an
+  # unsupported operator, so a single attempt would report an operator as
+  # refused when only the value was wrong.
+  VALUES = {
+    'date' => ['1', 1, '2026-01-01'],
+    'number' => [0, '0'],
+    'boolean' => [true, 'true'],
+    'string' => ['forest-probe', 0],
+    'text' => ['forest-probe']
+  }.freeze
+
+  LIST_OPERATORS = %w[IN NIN].freeze
+
+  # What Intercom answers a search it refuses: `invalid_field`, `data_invalid`
+  # and `invalid_value` all travel on a 400.
+  REFUSAL_STATUSES = [400, 422].freeze
+
+  # Raised when an answer measures nothing, so the run stops before writing.
+  class Inconclusive < StandardError; end
+
+  # A candidate carries no declared type -- discovering it is the point -- so
+  # one is guessed from its name, and the fallbacks above cover a wrong guess.
+  def self.guess_type(field)
+    case field
+    when /(_at|_since|_until)\z/ then 'date'
+    when /\Acount_|_count\z|\Atime_to_|_time\z/ then 'number'
+    when /\A(open|read|is_|has_|.*_participated)/ then 'boolean'
+    else 'string'
+    end
+  end
+
+  class Probe
+    def initialize(client, endpoint)
+      @client = client
+      @endpoint = endpoint
+    end
+
+    # Every operator of the alphabet on every field of the row: what the table
+    # already declares, plus the candidates. Nothing is assumed from the
+    # declared operators -- an operator missing from the table is exactly what
+    # this is here to find.
+    def run(field, type)
+      results = {}
+
+      SearchFields::KNOWN_OPERATORS.each do |operator|
+        outcome = probe(field, operator, type)
+        return { unfilterable: outcome } if outcome[:code] == 'invalid_field'
+
+        results[operator] = outcome
+      end
+
+      { operators: results }
+    end
+
+    private
+
+    def probe(field, operator, type)
+      last = nil
+
+      VALUES.fetch(type, VALUES['string']).each do |value|
+        outcome = attempt(field, operator, operator_value(operator, value))
+        # `invalid_field` is about the field, not about what was sent with it:
+        # another value shape cannot make a field the endpoint does not filter
+        # appear, so the row ends here rather than paying a request per shape.
+        return outcome if outcome[:ok] || outcome[:code] == 'invalid_field'
+
+        last = outcome
+      end
+
+      last
+    end
+
+    # Only an answer about the search itself is a measurement. A 401, a 429
+    # that outlasted the retries, a 5xx or a timeout says nothing about the
+    # field, and recorded as a refusal it would be wrong evidence.
+    def attempt(field, operator, value)
+      @client.search_page(@endpoint.path, query: { 'field' => field, 'operator' => operator, 'value' => value },
+                                          per_page: 1, list_key: list_key)
+      { ok: true }
+    rescue ForestAdminDatasourceIntercom::APIError => e
+      raise Inconclusive, inconclusive_message(field, operator, e) unless REFUSAL_STATUSES.include?(e.status)
+
+      { ok: false, code: error_code(e), status: e.status }
+    end
+
+    def inconclusive_message(field, operator, error)
+      "#{@endpoint.path} answered #{field} #{operator} with #{error.status ? "HTTP #{error.status}" : "no status"}, " \
+        'which says nothing about the field. Nothing was written; check the token and the region, then run ' \
+        'the probe again.'
+    end
+
+    def operator_value(operator, value)
+      LIST_OPERATORS.include?(operator) ? [value] : value
+    end
+
+    # `/tickets/search` answers under `tickets` and `/conversations/search`
+    # under `conversations`; a body read under the wrong key raises before the
+    # outcome is classified.
+    def list_key = @endpoint.name
+
+    def error_code(error)
+      errors = error.body.is_a?(Hash) ? error.body['errors'] : nil
+      first = Array(errors).first
+
+      (first.is_a?(Hash) ? first['code'] : nil) || "http_#{error.status}"
+    end
+  end
+
+  class Report
+    def initialize(endpoint)
+      @endpoint = endpoint
+      @rows = {}
+    end
+
+    def record(field, column, result)
+      @rows[field] = { 'column' => column, 'result' => result }
+    end
+
+    # What the run says about the committed table, which is the only reason to
+    # run it: an operator the table promises and Intercom refuses is a filter
+    # the interface offers and the read cannot honour.
+    def print_diff
+      @rows.each do |field, row|
+        column = row['column']
+        result = row['result']
+
+        if result[:unfilterable]
+          declared = column ? "  <- table declares it on column '#{column}'" : ''
+          puts "  #{field.ljust(42)} NOT FILTERABLE (#{result[:unfilterable][:code]})#{declared}"
+          next
+        end
+
+        print_operators(field, column, result[:operators])
+      end
+    end
+
+    def to_yaml_document
+      { 'endpoint' => @endpoint.name, 'path' => @endpoint.path, 'measured_at' => Time.now.utc.strftime('%Y-%m-%d'),
+        'fields' => @rows.to_h { |field, row| [field, measured_row(row)] } }.to_yaml
+    end
+
+    private
+
+    def print_operators(field, column, outcomes)
+      supported = outcomes.select { |_, outcome| outcome[:ok] }.keys
+      declared = column ? Array(@endpoint.field(column)&.operators) : []
+      puts "  #{field.ljust(42)} #{supported.empty? ? "(no operator answered)" : supported.join(" ")}"
+
+      report_drift(declared - supported, supported - declared)
+    end
+
+    def report_drift(promised, discovered)
+      puts "      ! the table promises #{promised.join(", ")} here, Intercom refuses it" unless promised.empty?
+      puts "      + Intercom also accepts #{discovered.join(", ")}" unless discovered.empty?
+    end
+
+    def measured_row(row)
+      result = row['result']
+      return { 'filterable' => false, 'code' => result[:unfilterable][:code] } if result[:unfilterable]
+
+      { 'filterable' => true,
+        'operators' => result[:operators].select { |_, outcome| outcome[:ok] }.keys,
+        'refused' => result[:operators].reject { |_, outcome| outcome[:ok] }
+                                       .transform_values { |outcome| outcome[:code] } }
+    end
+  end
+
+  class CLI
+    def self.call(argv)
+      new(options(argv)).run
+    end
+
+    def self.options(argv)
+      options = { endpoints: SearchFields.endpoints, region: 'us', token: ENV.fetch('INTERCOM_ACCESS_TOKEN', nil) }
+
+      OptionParser.new do |parser|
+        parser.banner = 'Usage: INTERCOM_ACCESS_TOKEN=... forest_admin_intercom_probe [options]'
+        parser.on('--endpoint NAME', "one of #{SearchFields.endpoints.join(", ")}") { |v| options[:endpoints] = [v] }
+        parser.on('--region NAME', 'us (default), eu or au') { |v| options[:region] = v }
+        parser.on('--token TOKEN', 'defaults to $INTERCOM_ACCESS_TOKEN') { |v| options[:token] = v }
+        parser.on('--out PATH', 'write the measurements as YAML evidence') { |v| options[:out] = v }
+      end.parse!(argv)
+
+      options
+    end
+
+    def initialize(options)
+      @options = options
+    end
+
+    def run
+      abort('No token: set INTERCOM_ACCESS_TOKEN or pass --token.') if @options[:token].to_s.empty?
+
+      documents = @options[:endpoints].map { |name| probe_endpoint(SearchFields.fetch(name)) }
+      write(documents) if @options[:out]
+      0
+    rescue Inconclusive => e
+      abort("Probe stopped: #{e.message}")
+    end
+
+    private
+
+    def probe_endpoint(endpoint)
+      puts "\n#{endpoint.path} -- #{endpoint.fields.size} declared field(s), #{endpoint.candidates.size} candidate(s)"
+      probe = Probe.new(client, endpoint)
+      report = Report.new(endpoint)
+
+      targets(endpoint).each { |field, column, type| report.record(field, column, probe.run(field, type)) }
+      report.print_diff
+      report.to_yaml_document
+    end
+
+    # The declared fields first, with the type the table gives them; then the
+    # candidates, whose type is guessed. A candidate already covered by a
+    # declared field is not probed twice.
+    def targets(endpoint)
+      declared = endpoint.fields.values.map { |field| [field.field, field.column, field.type] }
+      candidates = (endpoint.candidates - declared.map(&:first)).map { |field| [field, nil, guess_type(field)] }
+
+      declared + candidates
+    end
+
+    def guess_type(field) = ProbeSearchFields.guess_type(field)
+
+    def client
+      @client ||= ForestAdminDatasourceIntercom::Client.new(
+        ForestAdminDatasourceIntercom::Configuration.new(access_token: @options[:token], region: @options[:region])
+      )
+    end
+
+    def write(documents)
+      File.write(@options[:out], documents.join("\n"))
+      puts "\nWritten to #{@options[:out]}. Merge the measurements into search_fields.yml by hand: " \
+           'the table carries the prose a generated file would drop.'
+    end
+  end
+end
+
+exit(ProbeSearchFields::CLI.call(ARGV)) if ProbeSearchFields.invoked_as_command?($PROGRAM_NAME)

data/forest_admin_datasource_intercom.gemspec

@@ -0,0 +1,38 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift lib unless $LOAD_PATH.include?(lib)
+
+require_relative 'lib/forest_admin_datasource_intercom/version'
+
+Gem::Specification.new do |spec|
+  spec.name        = 'forest_admin_datasource_intercom'
+  spec.version     = ForestAdminDatasourceIntercom::VERSION
+  spec.authors     = ['Forest Admin']
+  spec.email       = ['contact@forestadmin.com']
+  spec.homepage    = 'https://www.forestadmin.com'
+  spec.summary     = 'Intercom datasource for Forest Admin Ruby agent.'
+  spec.description = 'Surface Intercom conversations, tickets, contacts and companies as Forest Admin collections.'
+  spec.license     = 'Apache-2.0'
+  spec.required_ruby_version = '>= 3.0.0'
+
+  spec.metadata['homepage_uri']    = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/ForestAdmin/agent-ruby'
+  spec.metadata['changelog_uri']   = 'https://github.com/ForestAdmin/agent-ruby/blob/main/CHANGELOG.md'
+  spec.metadata['rubygems_mfa_required'] = 'false'
+
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) ||
+        f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile])
+    end
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activesupport', '>= 6.1'
+  spec.add_dependency 'faraday', '~> 2.0'
+  spec.add_dependency 'faraday-net_http_persistent', '~> 2.0'
+  spec.add_dependency 'faraday-retry', '~> 2.0'
+  spec.add_dependency 'forest_admin_datasource_toolkit', ">= #{ForestAdminDatasourceIntercom::VERSION}"
+  spec.add_dependency 'zeitwerk', '~> 2.3'
+end

data/lib/forest_admin_datasource_intercom/cache.rb

@@ -0,0 +1,84 @@
+module ForestAdminDatasourceIntercom
+  # TTL store for workspace reference reads (teammates, teams, ticket types, ticket states), re-read
+  # on every page otherwise. Records are never cached here; see `Client#with_read_scope` for per-page dedup.
+  class Cache
+    Entry = Struct.new(:value, :expires_at)
+
+    # Monotonic rather than wall clock: this measures an elapsed span, and a
+    # host whose clock steps -- an NTP correction, a suspend -- would otherwise
+    # expire every entry at once or hold them all far past the window.
+    def initialize(ttl:, now: nil)
+      @ttl = ttl.to_f
+      @now = now || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+      @mutex = Mutex.new
+      @entries = {}
+      # Bumped by `clear`, and carried by every read that is about to fetch, so
+      # a response already in flight when the store was dropped cannot land in
+      # it afterwards. See `fetch`.
+      @generation = 0
+    end
+
+    def enabled? = @ttl.positive?
+
+    # The block runs outside the mutex (it is an HTTP request): two concurrent misses both read, which is
+    # cheaper than serializing every thread. A raising block writes nothing, and the generation is read
+    # before the block so a response in flight across `clear` is not stored, nor one `store` declines.
+    def fetch(key, store: -> { true })
+      return yield unless enabled?
+
+      hit = read(key)
+      return hit.value if hit
+
+      generation = @mutex.synchronize { @generation }
+      value = yield
+      write(key, value, generation) if store.call
+      value
+    end
+
+    # Drops everything held, without waiting the window out. Nothing inside this
+    # package calls it: the window is what expires an entry, and no read here
+    # writes. It is reachable through `Configuration#reference_cache` for the
+    # one case the window cannot serve -- a customizer that has just written a
+    # team or a ticket type into the workspace and wants the next page to show
+    # it. Documented in the README beside the ttl.
+    def clear
+      @mutex.synchronize do
+        @entries.clear
+        @generation += 1
+      end
+    end
+
+    private
+
+    def read(key)
+      @mutex.synchronize do
+        entry = @entries[key]
+        next nil if entry.nil?
+
+        if entry.expires_at <= @now.call
+          @entries.delete(key)
+          next nil
+        end
+
+        entry
+      end
+    end
+
+    # Expired entries are dropped on the way past rather than by a sweeper: the
+    # keys are a handful of endpoints, so the walk is short and there is no
+    # thread to own.
+    #
+    # A write from before a `clear` is dropped rather than stored: the store it
+    # was going to land in no longer exists.
+    def write(key, value, generation)
+      now = @now.call
+
+      @mutex.synchronize do
+        next if generation != @generation
+
+        @entries.delete_if { |_, entry| entry.expires_at <= now }
+        @entries[key] = Entry.new(value, now + @ttl)
+      end
+    end
+  end
+end