active_sanction 1.0.0

data/lib/active_sanction.rb

@@ -0,0 +1,331 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/error"
+require "active_sanction/version"
+require "active_sanction/deprecation"
+require "active_sanction/configuration"
+require "active_sanction/name"
+require "active_sanction/address"
+require "active_sanction/identifier"
+require "active_sanction/partial_date"
+require "active_sanction/entity"
+require "active_sanction/snapshot"
+require "active_sanction/snapshot/bundle"
+require "active_sanction/http_client"
+require "active_sanction/validators"
+require "active_sanction/validator_store"
+require "active_sanction/fetcher"
+require "active_sanction/payload_cache"
+require "active_sanction/parsers"
+require "active_sanction/sources"
+require "active_sanction/storage"
+require "active_sanction/normalizer"
+require "active_sanction/similarity"
+require "active_sanction/phonetics"
+require "active_sanction/index"
+require "active_sanction/country"
+require "active_sanction/scorer"
+require "active_sanction/query"
+require "active_sanction/match_result"
+require "active_sanction/matcher"
+require "active_sanction/subject"
+require "active_sanction/sync"
+require "active_sanction/diff"
+require "active_sanction/rescreen"
+require "active_sanction/doctor"
+require "active_sanction/client"
+require "active_sanction/sources/ofac_sdn"
+require "active_sanction/sources/ofac_consolidated"
+require "active_sanction/sources/un_consolidated"
+require "active_sanction/sources/canada_sema"
+require "active_sanction/sources/eu_fsf"
+require "active_sanction/sources/uk_sanctions_list"
+require "active_sanction/sources/australia_dfat"
+
+module ActiveSanction
+  # Guards the default client. Building one is cheap, but replacing it drops
+  # a matcher that indexed every stored list, and two threads racing to
+  # `configure` at boot should not each get a different one. A constant rather
+  # than a memoized ivar, because a lazily created lock is not one.
+  #
+  # @api private
+  CLIENT_LOCK = T.let(Mutex.new, Mutex)
+  private_constant :CLIENT_LOCK
+
+  # Where .with_configuration keeps the settings in force. Fiber-local, which
+  # is what `Thread#[]` means: two threads screening through two clients read
+  # two configurations, and neither can see the other's.
+  #
+  # @api private
+  CONFIGURATION_KEY = :active_sanction_configuration
+  private_constant :CONFIGURATION_KEY
+
+  class << self
+    extend T::Sig
+
+    # The client the module-level calls answer through, built from the
+    # defaults on first use so that nothing has to remember to initialize it.
+    #
+    #   ActiveSanction.client.screen("Bosco Ntaganda")   # same as ActiveSanction.screen(...)
+    #
+    # Everything below is sugar over this object. A process that needs two
+    # configurations at once -- a pinned list version for an audit re-run
+    # beside the current one for live traffic, one tenant's sources beside
+    # another's -- builds its own with `Client.new` and holds them itself;
+    # this one is what a script and the README quickstart use. See Client.
+    sig { returns(Client) }
+    def client
+      CLIENT_LOCK.synchronize { @client ||= T.let(Client.new, T.nilable(Client)) }
+    end
+
+    # The settings in force: whatever .with_configuration has installed on
+    # this fiber, or the default client's.
+    #
+    # Frozen, because it belongs to a built client. `configure` is how it is
+    # changed, and it changes it by building a new client rather than by
+    # editing this one -- a settings object that could move underneath a
+    # running index is the thing Client exists to remove.
+    sig { returns(Configuration) }
+    def config = Thread.current[CONFIGURATION_KEY] || client.configuration
+
+    # The one entry point an application is expected to call at boot:
+    #
+    #   ActiveSanction.configure do |c|
+    #     c.user_agent = "my-app/1.0 (compliance@example.com)"
+    #   end
+    #
+    # The block is handed a mutable copy of what is configured now, so
+    # settings accumulate across calls, and the copy is frozen into a new
+    # default client when the block returns. That replaces the held matcher,
+    # which is the behaviour a changed store or source list needs: an
+    # initializer that names a store must not leave a matcher behind that
+    # indexed a different one.
+    #
+    # Configure at boot, before anything screens. Later is honoured from the
+    # next call and does not reach what has already happened -- names folded
+    # under the old dictionary are already in an index, and scores recorded
+    # under the old weights were recorded under the old weights.
+    sig { params(block: T.proc.params(config: Configuration).void).returns(Configuration) }
+    def configure(&block)
+      settings = client.configuration.dup
+      block.call(settings)
+      CLIENT_LOCK.synchronize { @client = T.let(Client.new(configuration: settings), T.nilable(Client)) }
+      settings
+    end
+
+    # Runs a block with `configuration` in force, so that everything reached
+    # from inside it reads those settings rather than the default client's.
+    # This is how a Client makes its own User-Agent, dictionary, XML backend
+    # and query defaults reach code that was written against the module --
+    # the fetch layer, the normalizer, Query -- without every one of them
+    # having to be handed a configuration it does not otherwise want.
+    #
+    #   ActiveSanction.with_configuration(audit_client.configuration) { ... }
+    #
+    # The one limit is the one every fiber-local has: **a thread started
+    # inside the block does not inherit it**, and starts from the default
+    # client's settings. Code that fans out has to reinstall the
+    # configuration in each worker, which is what Sync does.
+    sig { params(configuration: Configuration, block: T.proc.returns(T.untyped)).returns(T.untyped) }
+    def with_configuration(configuration, &block)
+      previous = Thread.current[CONFIGURATION_KEY]
+      Thread.current[CONFIGURATION_KEY] = configuration
+      block.call
+    ensure
+      Thread.current[CONFIGURATION_KEY] = previous
+    end
+
+    # Drops the default client and anything this fiber had installed, so the
+    # next call builds one from the defaults. What a suite runs between
+    # examples, and the reason a spec that configures a store does not leak it
+    # into the next one:
+    #
+    #   config.after { ActiveSanction.reset! }
+    #
+    # It clears the fiber-local on the calling thread only; a thread that
+    # exited holding one has already taken it with it.
+    sig { returns(T.self_type) }
+    def reset!
+      CLIENT_LOCK.synchronize { @client = T.let(nil, T.nilable(Client)) }
+      Thread.current[CONFIGURATION_KEY] = nil
+      self
+    end
+
+    # Where the default client's synced lists are read from. Gzipped JSON
+    # under `storage_dir` unless the application named its own -- see
+    # Configuration#storage.
+    sig { returns(Storage::Base) }
+    def storage = client.storage
+
+    # The default client's matcher, built from its store on first use.
+    # See Client#matcher, which is where all of it is documented.
+    sig { returns(Matcher) }
+    def matcher = client.matcher
+
+    # Screens one name against every configured list:
+    #
+    #   ActiveSanction.screen(name: "Bosco Ntaganda", type: :individual, threshold: 75)
+    #
+    # Sugar over .matcher, which is where everything this does is documented.
+    sig { params(query: T.untyped, overrides: T.untyped).returns(T::Array[MatchResult]) }
+    def screen(query = nil, **overrides) = client.screen(query, **overrides)
+
+    # Screens a list of names, returning one array of results per query, in
+    # the order they were given. See Matcher#screen_all.
+    sig { params(queries: T.untyped, overrides: T.untyped).returns(T::Array[T::Array[MatchResult]]) }
+    def screen_all(queries, **overrides) = client.screen_all(queries, **overrides)
+
+    # Fetches, parses and stores every configured list, and returns what each
+    # one did:
+    #
+    #   report = ActiveSanction.sync!                   # every configured source
+    #   report = ActiveSanction.sync!(:ofac_sdn)        # one
+    #   report = ActiveSanction.sync!(force: true)      # bypass conditional GET
+    #   report = ActiveSanction.sync!(concurrency: 3)   # fetch three publishers at once
+    #
+    #   report.failed?                                  # => false
+    #   report[:ofac_sdn].status                        # => :updated
+    #   exit report.exit_code                           # 1 if any source failed
+    #
+    # A failing source does not raise and does not stop the others: it is
+    # captured into the report and **its previous snapshot is kept**, because
+    # yesterday's list with a visible age is safer than no list. See Sync,
+    # which is where all of that is documented, and Sync::Report.
+    #
+    # The block, if given, is called with each Sync::Result as that source
+    # finishes -- the progress hook for a run that takes minutes.
+    #
+    # Drops the shared matcher when any list changed, so the next screening
+    # call is answered by what was just synced. Not concurrent-safe against
+    # another sync of the same storage; see Client.
+    sig do
+      params(sources: T.untyped, options: T.untyped,
+             block: T.nilable(T.proc.params(result: Sync::Result).void)).returns(Sync::Report)
+    end
+    def sync!(*sources, **options, &block) = T.unsafe(client).sync!(*sources, **options, &block)
+
+    # What changed between two snapshots of one source:
+    #
+    #   diff = ActiveSanction.diff(:ofac_sdn, from: last_months_snapshot, to: todays_snapshot)
+    #   diff = ActiveSanction.diff(:ofac_sdn, from: last_months_snapshot)  # `to:` is what is stored now
+    #
+    #   diff.added     # => [Entity], newly listed
+    #   diff.removed   # => [Entity], delisted
+    #   diff.modified  # => [Diff::Change], amended, with the fields that moved
+    #   diff.changed   # => [Entity], what to re-screen a book of business against
+    #
+    # So that re-screening runs against the eleven records that moved rather
+    # than against the whole list. A first sync -- `from: nil` -- is a baseline
+    # rather than a list of additions, and an amended record reports as one
+    # modification rather than as a delisting and a new listing. See Diff,
+    # which is where all of that is documented.
+    sig { params(source: T.untyped, options: T.untyped).returns(Diff) }
+    def diff(source = nil, **options) = T.unsafe(client).diff(source, **options)
+
+    # Applies a snapshot diff to a book of subjects, and returns the alerts:
+    #
+    #   book = [
+    #     ActiveSanction::Subject.new(id: "cust_1", name: "Bosco Ntaganda", date_of_birth: "1973"),
+    #     ActiveSanction::Subject.new(id: "cust_2", name: "Jane Miller")
+    #   ]
+    #
+    #   alerts = ActiveSanction.rescreen(book, diff: diff, threshold: 75)
+    #
+    #   alerts.first.subject_id      # => "cust_1"
+    #   alerts.first.change          # => :newly_listed | :delisted | :details_changed
+    #   alerts.first.result          # => a full MatchResult, with its explanation
+    #   alerts.first.previous_score  # => what it scored against the old list version
+    #
+    # Screening answers about a name; this answers about a book of business,
+    # and it is the step that turns a diff into an alert. The cost is the book
+    # times the handful of records that moved rather than the book times the
+    # whole corpus, which is what makes rescreening after every sync
+    # affordable -- an empty diff scores nothing at all.
+    #
+    # A delisting raises an alert too: it is a change of status a compliance
+    # team has to record, and it is the one that lets a customer back through
+    # the door. The block, if given, is called with each alert as it is
+    # raised, so a large book streams past a small diff without accumulating
+    # anything. See Rescreen, which is where all of that is documented.
+    sig do
+      params(subjects: T.untyped, diff: T.untyped, options: T.untyped,
+             block: T.nilable(T.proc.params(alert: Rescreen::Alert).void)).returns(T::Array[Rescreen::Alert])
+    end
+    def rescreen(subjects, diff:, **options, &block)
+      T.unsafe(client).rescreen(subjects, diff: diff, **options, &block)
+    end
+
+    # Diagnoses whether a source's format has drifted -- fetching each list,
+    # parsing it, and comparing what it measures against the version that was
+    # stored at the last sync:
+    #
+    #   report = ActiveSanction.doctor                    # every configured source
+    #   report = ActiveSanction.doctor(:ofac_sdn)         # one
+    #   report = ActiveSanction.doctor(tolerance: 0.05)   # report smaller movements
+    #
+    #   report.ok?        # => false
+    #   report.findings   # => [Doctor::Finding, ...]
+    #   puts report
+    #   exit report.exit_code
+    #
+    # The failure this exists for is the one a sync cannot see: a file that
+    # still parses cleanly and means something different. 19,321 entities
+    # carrying zero passports looks exactly as healthy as 19,321 carrying
+    # 23,429 if all anyone counts is records, and screening a passport number
+    # against the first returns a clean result for somebody who is on the list.
+    #
+    # Nothing is written -- not the snapshot, not the payload cache, not the
+    # conditional-GET validators -- so a diagnosis can never be the reason a
+    # sync skipped a list that changed, and nothing here repairs anything.
+    # Deciding that a 40% drop in record count is a delisting wave rather than
+    # a broken parse is a judgment call this library does not make. See Doctor,
+    # which is where all of that is documented.
+    #
+    # The block, if given, is called with each Doctor::Diagnosis as that source
+    # finishes.
+    sig do
+      params(sources: T.untyped, options: T.untyped,
+             block: T.nilable(T.proc.params(diagnosis: Doctor::Diagnosis).void)).returns(Doctor::Report)
+    end
+    def doctor(*sources, **options, &block) = T.unsafe(client).doctor(*sources, **options, &block)
+
+    # Writes one stored list to a portable, optionally signed bundle file:
+    #
+    #   ActiveSanction.export(:ofac_sdn, to: "ofac_sdn.asb")
+    #   ActiveSanction.export(:ofac_sdn, to: "ofac_sdn.asb", sign_with: private_key)
+    #
+    # One file, produced once, that another machine loads and screens against
+    # without reaching the publisher at all -- which is what an air-gapped
+    # installation needs, and what a deploy needs on the afternoon OFAC is
+    # down. See Snapshot::Bundle, and docs/bundle_format.md, which specifies
+    # the format well enough to be implemented outside Ruby.
+    sig { params(source: T.untyped, options: T.untyped).returns(Snapshot::Bundle::Header) }
+    def export(source, **options) = T.unsafe(client).export(source, **options)
+
+    # Loads a bundle into the configured store and returns the snapshot it
+    # held:
+    #
+    #   snapshot = ActiveSanction.import("ofac_sdn.asb", verify_with: public_key)
+    #   snapshot.trusted?   # => true
+    #
+    # A tampered bundle raises, a bundle signed by an unknown key raises
+    # something different, and both happen before anything is stored. See
+    # Client#import, which is where the one subtlety -- verification does not
+    # survive a write to disk -- is documented.
+    sig { params(path: T.untyped, options: T.untyped).returns(Snapshot) }
+    def import(path, **options) = T.unsafe(client).import(path, **options)
+
+    # Drops the shared matcher so the next screening call builds one over
+    # what is stored now. What a process calls after a sync -- a matcher is
+    # built once and never updated, which is what lets it be screened from
+    # many threads without a lock.
+    sig { returns(T.self_type) }
+    def reload!
+      client.reload!
+      self
+    end
+  end
+end

data/lib/generators/active_sanction/install/install_generator.rb

@@ -0,0 +1,60 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module ActiveSanction
+  # Rails generators, loaded only by Rails' own generator lookup. Nothing here
+  # is required by `require "active_sanction"`, which is what keeps Rails and
+  # ActiveRecord out of this gem's runtime dependencies.
+  #
+  # @api private
+  module Generators
+    # The migration that creates the tables Storage::ActiveRecord reads and
+    # writes:
+    #
+    #     $ rails generate active_sanction:install
+    #     $ rails db:migrate
+    #
+    # Only ever loaded by Rails' generator lookup, which is what keeps
+    # ActiveRecord and Rails out of this gem's runtime dependencies: nothing
+    # under lib/generators is required by `require "active_sanction"`.
+    #
+    # The migration it copies is ordinary `create_table` calls rather than a
+    # call into the gem. A migration is immutable history -- `rails db:migrate`
+    # on a fresh checkout has to build the schema the existing one was built
+    # from -- and a migration whose body lived in gem code would silently mean
+    # something different after a `bundle update`. The cost is that a schema
+    # change ships as a second migration, which is the same cost every other
+    # table in the host application pays.
+    #
+    # @api private
+    class InstallGenerator < ::Rails::Generators::Base
+      extend T::Sig
+      include ::ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates the migration for the ActiveSanction storage tables."
+
+      sig { void }
+      def create_migration_file
+        migration_template "create_active_sanction_tables.rb.tt",
+                           File.join(db_migrate_path, "create_active_sanction_tables.rb")
+      end
+
+      private
+
+      # Stamped into the generated class so the migration keeps running under
+      # the Rails compatibility layer it was written against, which is what
+      # `ActiveRecord::Migration[7.1]` means.
+      sig { returns(String) }
+      def migration_version
+        "[#{::ActiveRecord::VERSION::MAJOR}.#{::ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+# Storage for ActiveSanction::Storage::ActiveRecord.
+#
+# One row per synced list, one per record on it, and one per name, address and
+# identifier hanging off that record. A full OFAC SDN sync is one snapshot row,
+# 19,015 entity rows and some 65,000 child rows -- OFAC ships 39,468 names and
+# 25,078 addresses -- written with `insert_all` inside a single transaction.
+#
+# ### Why the children carry `snapshot_id` as well as `entity_id`
+#
+# So that reading a list is four indexed range scans rather than an `IN` clause
+# holding 19,015 entity ids, so that replacing one is four `delete_all`s, and
+# so that a host can scope a lookup to the list it means to screen against.
+#
+# ### Why `normalized_value` is indexed
+#
+# It is the candidate-generation path: an equality probe that narrows 19,015
+# records to the few hundred worth scoring in Ruby, before any of them are
+# loaded. Write the key with
+# `ActiveSanction::Storage::ActiveRecord.prefilter_key` at query time -- it is
+# the same fold this adapter applied on the way in, and a key built any other
+# way will not find the rows.
+#
+# ### Why there are no foreign keys
+#
+# These tables are owned wholly by the adapter, which deletes children before
+# parents inside the transaction that replaces a list, so nothing here depends
+# on the database enforcing the order. Adding the constraints costs a check per
+# row on a write that inserts tens of thousands of them. A host that wants them
+# anyway can add them: the delete order above is already the one they require.
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :active_sanction_snapshots do |t|
+      t.string   :source, null: false
+      t.datetime :fetched_at, null: false
+      # sha256:<64 hex>, and what a match result cites: "which list version
+      # cleared this customer" is answered by comparing an audit record's
+      # checksum against this column, without loading either list.
+      t.string   :checksum, null: false
+      t.integer  :record_count, null: false
+      t.integer  :schema_version, null: false
+      t.string   :source_version
+
+      # One stored list per source, enforced by the database rather than by
+      # the adapter alone: a sync replaces a list, and two rows for one source
+      # would mean screening against whichever the query planner returned.
+      t.index :source, unique: true
+    end
+
+    create_table :active_sanction_entities do |t|
+      t.bigint  :snapshot_id, null: false
+      # The order the publisher emitted the record in. A snapshot checksums its
+      # entities order-independently, so nothing else would catch a store
+      # handing rows back in whatever order the planner chose -- and a report
+      # whose rows move between two runs against an unchanged list is a report
+      # an examiner cannot cite.
+      t.integer :position, null: false
+      # ActiveSanction::Entity#id: the publisher's own reference, namespaced by
+      # source, e.g. "ofac_sdn:2674". Not unique across the table -- two
+      # snapshots of the same list both hold it.
+      t.string  :external_id, null: false
+      t.string  :source, null: false
+      # `type` is Rails' single-table-inheritance column, so the canonical
+      # record's `type` (individual, organization, vessel, aircraft) is stored
+      # under a name that will not make ActiveRecord try to instantiate it.
+      t.string  :entity_type, null: false
+      t.string  :source_ref
+      # JSON, as the canonical value objects serialize themselves. These are
+      # small, repeated, and nothing screens on them structurally, so they do
+      # not earn tables of their own the way names and identifiers do. `text`
+      # rather than a native JSON column so one migration runs everywhere; a
+      # host on PostgreSQL that wants to query them can change the type.
+      t.text    :dates_of_birth
+      t.text    :nationalities
+      t.text    :programs
+      t.text    :listed_on
+      t.text    :remarks
+
+      t.index %i[snapshot_id position], unique: true
+      t.index %i[snapshot_id external_id]
+    end
+
+    create_table :active_sanction_names do |t|
+      t.bigint  :entity_id, null: false
+      t.bigint  :snapshot_id, null: false
+      t.integer :position, null: false
+      # `text`, because what the publisher wrote has to come back byte for byte
+      # -- the snapshot's checksum is computed over it, so a column that
+      # truncated a long organization name would fail the read rather than
+      # quietly shorten a name.
+      t.text    :value, null: false
+      # Bounded, because it is indexed and MySQL will not index an unbounded
+      # column. 512 is comfortably past the longest name any of the launch
+      # lists publishes, and the key builder truncates to it so that a
+      # pathological name still produces a key a query can reproduce.
+      t.string  :normalized_value, null: false, limit: 512
+      t.string  :kind, null: false
+      t.string  :quality
+      t.string  :script
+
+      t.index %i[entity_id position], unique: true
+      t.index :snapshot_id
+      t.index :normalized_value
+    end
+
+    create_table :active_sanction_addresses do |t|
+      t.bigint  :entity_id, null: false
+      t.bigint  :snapshot_id, null: false
+      t.integer :position, null: false
+      t.text    :street
+      t.string  :city
+      t.string  :state_province
+      t.string  :postal_code
+      t.string  :country
+      t.text    :note
+
+      t.index %i[entity_id position], unique: true
+      t.index :snapshot_id
+    end
+
+    create_table :active_sanction_identifiers do |t|
+      t.bigint  :entity_id, null: false
+      t.bigint  :snapshot_id, null: false
+      t.integer :position, null: false
+      t.string  :kind, null: false
+      t.text    :value, null: false
+      # ActiveSanction::Identifier#normalized_value: the number with everything
+      # that is not alphanumeric removed, which is what lets OFAC's
+      # `AB-123 456` and the UN's `AB123456` find each other.
+      t.string  :normalized_value, null: false, limit: 512
+      t.string  :country
+      t.text    :issued_on
+      t.text    :expires_on
+      t.text    :note
+
+      t.index %i[entity_id position], unique: true
+      t.index :snapshot_id
+      t.index :normalized_value
+    end
+  end
+end