active_sanction 1.0.0

data/lib/active_sanction/storage/active_record/reader.rb

@@ -0,0 +1,124 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "json"
+require "active_sanction/entity"
+require "active_sanction/snapshot"
+require "active_sanction/storage/base"
+require "active_sanction/storage/active_record/row"
+
+module ActiveSanction
+  module Storage
+    class ActiveRecord < Base
+      # Turns the rows of one stored list back into a Snapshot.
+      #
+      # Four queries and no models. Every one of them is a `pluck` against an
+      # indexed `snapshot_id`, because the alternative -- instantiating 19,015
+      # Row::Entity objects and letting their associations load, or worse, not
+      # letting them -- spends the whole cost of ActiveRecord's object graph on
+      # data that is about to be turned into different objects anyway.
+      #
+      # The Snapshot is rebuilt with the checksum that was stored beside it,
+      # which makes construction the verification: Snapshot recomputes the
+      # digest over the records that actually came back and refuses to build if
+      # it does not match. A row deleted by hand, a write that half landed, a
+      # column edited in a console -- all of them raise rather than screen a
+      # customer against a list that is quietly missing people.
+      #
+      # @api private
+      class Reader
+        extend T::Sig
+
+        # @api private
+        ENTITY_COLUMNS = T.let(%i[
+          id external_id source source_ref entity_type dates_of_birth nationalities programs listed_on remarks
+        ].freeze, T::Array[Symbol])
+
+        # @api private
+        NAME_COLUMNS = T.let(%i[entity_id value kind quality script].freeze, T::Array[Symbol])
+        # @api private
+        ADDRESS_COLUMNS = T.let(
+          %i[entity_id street city state_province postal_code country note].freeze, T::Array[Symbol]
+        )
+        # @api private
+        IDENTIFIER_COLUMNS = T.let(
+          %i[entity_id kind value country issued_on expires_on note].freeze, T::Array[Symbol]
+        )
+
+        # Identifier members that are stored as JSON and rebuilt as dates.
+        #
+        # @api private
+        IDENTIFIER_DATES = T.let(%i[issued_on expires_on].freeze, T::Array[Symbol])
+
+        sig { params(row: T.untyped).void }
+        def initialize(row)
+          @row = T.let(row, T.untyped)
+        end
+
+        sig { returns(Snapshot) }
+        def call
+          Snapshot.new(
+            source: @row.source.to_sym, entities: entities, fetched_at: @row.fetched_at.to_time,
+            checksum: @row.checksum, record_count: @row.record_count, schema_version: @row.schema_version,
+            source_version: @row.source_version
+          )
+        end
+
+        private
+
+        sig { returns(T::Array[Entity]) }
+        def entities
+          names = children(Row::Name, NAME_COLUMNS)
+          addresses = children(Row::Address, ADDRESS_COLUMNS)
+          identifiers = identifier_children
+          rows(Row::Entity, ENTITY_COLUMNS).map do |cells|
+            entity(cells, names, addresses, identifiers)
+          end
+        end
+
+        sig do
+          params(cells: T::Hash[Symbol, T.untyped], names: T.untyped, addresses: T.untyped,
+                 identifiers: T.untyped).returns(Entity)
+        end
+        def entity(cells, names, addresses, identifiers)
+          key = cells[:id]
+          Entity.from_h(
+            id: cells[:external_id], source: cells[:source], source_ref: cells[:source_ref],
+            type: cells[:entity_type], names: names[key], addresses: addresses[key], identifiers: identifiers[key],
+            dates_of_birth: parse(cells[:dates_of_birth]), nationalities: parse(cells[:nationalities]),
+            programs: parse(cells[:programs]), listed_on: parse(cells[:listed_on]), remarks: cells[:remarks]
+          )
+        end
+
+        sig { returns(T::Hash[T.untyped, T::Array[T::Hash[Symbol, T.untyped]]]) }
+        def identifier_children
+          children(Row::Identifier, IDENTIFIER_COLUMNS).transform_values do |list|
+            list.map { |member| member.merge(IDENTIFIER_DATES.to_h { |date| [date, parse(member[date])] }) }
+          end
+        end
+
+        # Every child of every entity on this list, in one query, grouped by
+        # the entity they hang off and left in the order they were written.
+        sig do
+          params(model: T.untyped, columns: T::Array[Symbol])
+            .returns(T::Hash[T.untyped, T::Array[T::Hash[Symbol, T.untyped]]])
+        end
+        def children(model, columns)
+          model.where(snapshot_id: @row.id).order(:entity_id, :position).pluck(*columns)
+               .group_by(&:first)
+               .transform_values { |group| group.map { |cells| columns.drop(1).zip(cells.drop(1)).to_h } }
+        end
+
+        sig { params(model: T.untyped, columns: T::Array[Symbol]).returns(T::Array[T::Hash[Symbol, T.untyped]]) }
+        def rows(model, columns)
+          model.where(snapshot_id: @row.id).order(:position).pluck(*columns).map { |cells| columns.zip(cells).to_h }
+        end
+
+        sig { params(json: T.untyped).returns(T.untyped) }
+        def parse(json) = json.nil? ? nil : JSON.parse(json)
+      end
+    end
+  end
+end

data/lib/active_sanction/storage/active_record/row.rb

@@ -0,0 +1,175 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_record"
+require "active_sanction/identifier"
+require "active_sanction/storage/base"
+
+module ActiveSanction
+  module Storage
+    class ActiveRecord < Base
+      # The five tables, as ActiveRecord models.
+      #
+      #   Row::Snapshot     active_sanction_snapshots     one row per synced list
+      #   Row::Entity       active_sanction_entities      one row per record on it
+      #   Row::Name         active_sanction_names         one row per name variant
+      #   Row::Address      active_sanction_addresses
+      #   Row::Identifier   active_sanction_identifiers
+      #
+      # They are namespaced under `Row` rather than named `Snapshot`, `Entity`
+      # and so on directly, for a reason that is not stylistic: this file sits
+      # inside ActiveSanction, where `Snapshot` and `Entity` already mean the
+      # canonical value objects. A model shadowing either would make every
+      # unqualified reference in this adapter mean whichever one Ruby's lexical
+      # lookup reached first. `Row::Snapshot` is the table; `Snapshot` is the
+      # record.
+      #
+      # ### These are public
+      #
+      # Unlike Storage::FileSystem's directory layout, which is `@api private`
+      # under #62, the schema here is the point of the adapter. An installation
+      # that already has a database wants to *query* its lists -- to narrow
+      # 19,015 OFAC records to the few hundred worth scoring in Ruby, before
+      # any of them are loaded:
+      #
+      #   Row::Name.matching("Aiman al-Zawahiri").pluck(:entity_id)
+      #
+      # So the table names, the columns and the associations are part of what
+      # this adapter promises, and change under SemVer like anything else
+      # public.
+      #
+      # ### What they deliberately do not do
+      #
+      # No validations, no callbacks, no `dependent: :destroy`. Every write
+      # goes through `insert_all` in batches inside one transaction, which
+      # bypasses all three by design: a sync writes 19,015 entities and some
+      # 65,000 rows hanging off them, and row-at-a-time saves with callbacks
+      # turn seconds into minutes. Deletion is Row::Snapshot#discard!, which
+      # drops the children in bulk and then the parent.
+      #
+      # @api private
+      module Row
+        # Abstract, so a host can point the sanctions tables at a database
+        # other than its application's without touching ActiveRecord::Base:
+        #
+        #   ActiveSanction::Storage::ActiveRecord::Row::Base.connects_to(database: { writing: :sanctions })
+        #
+        # Left unconnected here, so it inherits whatever ActiveRecord::Base is
+        # connected to -- the right default, because the common case is a Rails
+        # application with one database.
+        #
+        # @api private
+        class Base < ::ActiveRecord::Base
+          self.abstract_class = true
+        end
+
+        # `active_sanction_snapshots` -- one row per synced list, carrying the
+        # checksum the whole list is rebuilt against.
+        #
+        # @api private
+        class Snapshot < Base
+          extend T::Sig
+
+          self.table_name = "active_sanction_snapshots"
+
+          has_many :entities, class_name: "ActiveSanction::Storage::ActiveRecord::Row::Entity",
+                              foreign_key: :snapshot_id, inverse_of: :snapshot, dependent: nil
+
+          # Everything stored for this list, including the row itself, dropped
+          # in bulk. `delete_all` rather than `destroy_all`: these rows have no
+          # callbacks to run, and instantiating 19,015 of them to throw them
+          # away is exactly the cost this adapter exists to avoid.
+          #
+          # Children first, parent last, so the sequence is one a host that has
+          # added foreign keys of its own can also execute.
+          sig { void }
+          def discard!
+            [Name, Address, Identifier, Entity].each { |model| model.where(snapshot_id: id).delete_all }
+            self.class.where(id: id).delete_all
+          end
+        end
+
+        # `active_sanction_entities` -- one row per record on a list, with its
+        # names, addresses and identifiers hanging off it.
+        #
+        # @api private
+        class Entity < Base
+          self.table_name = "active_sanction_entities"
+
+          belongs_to :snapshot, class_name: "ActiveSanction::Storage::ActiveRecord::Row::Snapshot",
+                                inverse_of: :entities
+
+          has_many :names, class_name: "ActiveSanction::Storage::ActiveRecord::Row::Name",
+                           foreign_key: :entity_id, inverse_of: :entity, dependent: nil
+          has_many :addresses, class_name: "ActiveSanction::Storage::ActiveRecord::Row::Address",
+                               foreign_key: :entity_id, inverse_of: :entity, dependent: nil
+          has_many :identifiers, class_name: "ActiveSanction::Storage::ActiveRecord::Row::Identifier",
+                                 foreign_key: :entity_id, inverse_of: :entity, dependent: nil
+        end
+
+        # `active_sanction_names` -- one row per name variant, primary or alias.
+        # `normalized_value` is the indexed column the prefilter probes.
+        #
+        # @api private
+        class Name < Base
+          self.table_name = "active_sanction_names"
+
+          belongs_to :entity, class_name: "ActiveSanction::Storage::ActiveRecord::Row::Entity",
+                              inverse_of: :names
+
+          # The names whose prefilter key is the one this string folds to --
+          # the candidate-generation path, and the reason the index exists:
+          #
+          #   Row::Name.matching("Aiman AL-ZAWAHIRI").pluck(:entity_id)
+          #
+          # An equality lookup on an indexed column, so it stays a b-tree probe
+          # rather than the table scan a `LIKE '%...%'` would be. It prefilters
+          # and does not match: it answers "which records are worth scoring in
+          # Ruby", never "is this the same person".
+          scope :matching, lambda { |value|
+            where(normalized_value: ActiveSanction::Storage::ActiveRecord.prefilter_key(value))
+          }
+        end
+
+        # `active_sanction_addresses` -- one row per published address. Not a
+        # prefilter path: addresses on these lists are too partial to probe on.
+        #
+        # @api private
+        class Address < Base
+          self.table_name = "active_sanction_addresses"
+
+          belongs_to :entity, class_name: "ActiveSanction::Storage::ActiveRecord::Row::Entity",
+                              inverse_of: :addresses
+        end
+
+        # `active_sanction_identifiers` -- one row per document number, keyed on
+        # ActiveSanction::Identifier#normalized_value so that two publishers'
+        # punctuation of the same passport finds each other.
+        #
+        # @api private
+        class Identifier < Base
+          self.table_name = "active_sanction_identifiers"
+
+          belongs_to :entity, class_name: "ActiveSanction::Storage::ActiveRecord::Row::Entity",
+                              inverse_of: :identifiers
+
+          # The other prefilter path, and the stronger one: an exact document
+          # match is near-decisive where a name match never is. Keyed on
+          # ActiveSanction::Identifier#normalized_value, which is why OFAC's
+          # `AB-123 456` and the UN's `AB123456` find each other.
+          scope :matching, lambda { |value|
+            where(normalized_value: ActiveSanction::Identifier.new(value: value).normalized_value)
+          }
+        end
+
+        # Parents before children: the order a write inserts in, and the
+        # reverse of the order a delete removes in.
+        #
+        # @api private
+        ALL = T.let([Snapshot, Entity, Name, Address, Identifier].freeze, T::Array[T.untyped])
+      end
+    end
+  end
+end

data/lib/active_sanction/storage/active_record/writer.rb

@@ -0,0 +1,141 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "json"
+require "active_sanction/storage/base"
+require "active_sanction/storage/active_record/row"
+
+module ActiveSanction
+  module Storage
+    class ActiveRecord < Base
+      # Turns one Snapshot into rows, in batches, with `insert_all`.
+      #
+      # Called with the `active_sanction_snapshots` row already created and
+      # inside the transaction that created it, which is the arrangement the
+      # acceptance criterion asks for: a sync that dies partway through 19,015
+      # entities rolls back to the list that was there before it, rather than
+      # leaving a snapshot row advertising records that are not underneath it.
+      #
+      # ### Why the entity ids are read back rather than returned
+      #
+      # A child row needs its parent's primary key, and `insert_all` can return
+      # the keys it assigned -- on PostgreSQL and on recent SQLite. MySQL has
+      # no `RETURNING`, and an adapter that worked on two of the three
+      # databases Rails ships support for would not be worth the word
+      # "optional" in the issue. So the entities go in first, and one ordered
+      # `pluck` of their ids follows: one extra query per sync, against a
+      # method that works everywhere.
+      #
+      # @api private
+      class Writer
+        extend T::Sig
+
+        sig { params(row: T.untyped, snapshot: Snapshot, batch_size: Integer).void }
+        def initialize(row, snapshot, batch_size:)
+          @row = T.let(row, T.untyped)
+          @snapshot = T.let(snapshot, Snapshot)
+          @batch_size = T.let(batch_size, Integer)
+        end
+
+        sig { returns(T.untyped) }
+        def call
+          write_entities
+          write_children
+          @row
+        end
+
+        private
+
+        sig { void }
+        def write_entities
+          each_batch { |pairs| insert(Row::Entity, pairs.map { |entity, position| entity_row(entity, position) }) }
+        end
+
+        sig { void }
+        def write_children
+          ids = entity_ids
+          each_batch { |pairs| write_batch(pairs, ids) }
+        end
+
+        sig { params(pairs: T::Array[T.untyped], ids: T::Array[T.untyped]).void }
+        def write_batch(pairs, ids)
+          insert(Row::Name, pairs.flat_map { |entity, index| name_rows(entity, ids.fetch(index)) })
+          insert(Row::Address, pairs.flat_map { |entity, index| address_rows(entity, ids.fetch(index)) })
+          insert(Row::Identifier, pairs.flat_map { |entity, index| identifier_rows(entity, ids.fetch(index)) })
+        end
+
+        # The list in slices, each entity paired with the position it was
+        # written at -- which is the index its primary key sits at in `ids`.
+        sig { params(block: T.proc.params(pairs: T::Array[T.untyped]).void).void }
+        def each_batch(&block) = @snapshot.entities.each_with_index.each_slice(@batch_size, &block)
+
+        # Ordered by the column the writer just filled in, so the id at index
+        # `n` belongs to the entity written at position `n`.
+        sig { returns(T::Array[T.untyped]) }
+        def entity_ids = Row::Entity.where(snapshot_id: @row.id).order(:position).pluck(:id)
+
+        sig { params(model: T.untyped, rows: T::Array[T::Hash[Symbol, T.untyped]]).void }
+        def insert(model, rows)
+          rows.each_slice(@batch_size) { |slice| model.insert_all(slice) }
+        end
+
+        sig { params(entity: Entity, position: Integer).returns(T::Hash[Symbol, T.untyped]) }
+        def entity_row(entity, position)
+          {
+            snapshot_id: @row.id, position: position, external_id: entity.id, source: entity.source.to_s,
+            entity_type: entity.type.to_s, source_ref: entity.source_ref,
+            dates_of_birth: json(entity.dates_of_birth.map(&:to_h)), nationalities: json(entity.nationalities),
+            programs: json(entity.programs), listed_on: json(entity.listed_on&.to_h), remarks: entity.remarks
+          }
+        end
+
+        sig { params(entity: Entity, entity_id: T.untyped).returns(T::Array[T::Hash[Symbol, T.untyped]]) }
+        def name_rows(entity, entity_id)
+          entity.names.each_with_index.map do |name, position|
+            {
+              entity_id: entity_id, snapshot_id: @row.id, position: position, value: name.value,
+              normalized_value: ActiveSanction::Storage::ActiveRecord.prefilter_key(name.value),
+              kind: name.kind.to_s, quality: name.quality&.to_s, script: name.script&.to_s
+            }
+          end
+        end
+
+        sig { params(entity: Entity, entity_id: T.untyped).returns(T::Array[T::Hash[Symbol, T.untyped]]) }
+        def address_rows(entity, entity_id)
+          entity.addresses.each_with_index.map do |address, position|
+            {
+              entity_id: entity_id, snapshot_id: @row.id, position: position, street: address.street,
+              city: address.city, state_province: address.state_province, postal_code: address.postal_code,
+              country: address.country, note: address.note
+            }
+          end
+        end
+
+        sig { params(entity: Entity, entity_id: T.untyped).returns(T::Array[T::Hash[Symbol, T.untyped]]) }
+        def identifier_rows(entity, entity_id)
+          entity.identifiers.each_with_index.map do |identifier, position|
+            {
+              entity_id: entity_id, snapshot_id: @row.id, position: position, kind: identifier.kind.to_s,
+              value: identifier.value, normalized_value: identifier.normalized_value, country: identifier.country,
+              issued_on: json(identifier.issued_on&.to_h), expires_on: json(identifier.expires_on&.to_h),
+              note: identifier.note
+            }
+          end
+        end
+
+        # Nil rather than "[]" or "null" for anything the publisher did not
+        # give, which keeps the column readable in a database console and
+        # matches what the canonical record means by an absent member: Entity
+        # reads a nil collection back as an empty one.
+        sig { params(value: T.untyped).returns(T.nilable(String)) }
+        def json(value)
+          return nil if value.nil? || (value.respond_to?(:empty?) && value.empty?)
+
+          JSON.generate(value)
+        end
+      end
+    end
+  end
+end