active_sanction 1.0.0

data/lib/active_sanction/storage.rb

@@ -0,0 +1,74 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "active_sanction/error"
+
+module ActiveSanction
+  # Where a synced list lives between the sync that fetched it and the
+  # screening run that reads it.
+  #
+  #   store = ActiveSanction::Storage::Memory.new
+  #   store.write_snapshot(ActiveSanction::Sources[:ofac_sdn].new.sync)
+  #   store.read_snapshot(:ofac_sdn)   # => Snapshot
+  #
+  # The namespace holds one interface (Storage::Base), the in-memory
+  # implementation of it, and the metadata object a store can answer with
+  # without loading a list. The adapters that persist anywhere else -- gzipped
+  # JSON on disk (#24), ActiveRecord (#25), whatever a host writes privately --
+  # are subclasses of Base and nothing here has to know they exist.
+  module Storage
+    # A source that has never been synced, or whose snapshot has been deleted,
+    # asked for by name. Raised rather than answered with nil wherever a caller
+    # named the source itself: screening against a list that turns out not to
+    # be there has to fail loudly, because the result of screening against
+    # nothing is a clean report.
+    class MissingSnapshot < StorageError; end
+
+    # A stored snapshot that cannot be trusted to be what it says it is: a
+    # truncated file, bytes that no longer hash to the checksum recorded beside
+    # them, a sidecar that is not JSON, a list filed under one source that
+    # claims to be another.
+    #
+    # Raised rather than repaired, and rather than returning whatever could
+    # still be read. A store that hands back the 8,000 records it managed to
+    # parse out of 19,015 produces a report that looks exactly like a clean
+    # one, which is the most expensive thing this library can get wrong. An
+    # operator can always delete the list and re-sync; nobody can recover a
+    # screening decision made against a list that was quietly half there.
+    class CorruptSnapshot < StorageError; end
+
+    # A stored snapshot written under a Snapshot::SCHEMA_VERSION this code does
+    # not know how to read -- almost always because the directory was written
+    # by a newer active_sanction than the one now reading it.
+    #
+    # Separate from CorruptSnapshot because the file is fine and the fix is
+    # different: upgrade the gem, or discard the list and re-sync under this
+    # one. It has to be caught before the list is parsed, because a newer
+    # schema will usually still deserialize -- into records missing whatever
+    # the new version added, with a checksum that verifies, and with no
+    # symptom other than names that stop matching.
+    class UnsupportedSchema < StorageError; end
+  end
+end
+
+require "active_sanction/storage/meta"
+require "active_sanction/storage/base"
+require "active_sanction/storage/memory"
+require "active_sanction/storage/file_system"
+
+# Storage::ActiveRecord (#25) is optional in the strong sense: ActiveRecord is
+# not a dependency of this gem and must not become one, so the adapter is
+# loaded only where it can be, and the gem is fully usable without it.
+#
+# Both orders have to work, which is why this is two clauses rather than one.
+# A script that requires ActiveRecord itself has already defined the constant
+# by the time this file runs, and the first clause loads the adapter now. A
+# Rails application loads ActiveSupport long before ActiveRecord::Base -- the
+# framework is deliberately lazy about it -- so the second clause books the
+# adapter onto the hook Rails runs when Base is finally loaded. An application
+# that wants it unconditionally can always require it by name.
+if defined?(ActiveRecord::Base)
+  require "active_sanction/storage/active_record"
+elsif defined?(ActiveSupport) && ActiveSupport.respond_to?(:on_load)
+  ActiveSupport.on_load(:active_record) { require "active_sanction/storage/active_record" }
+end

data/lib/active_sanction/subject.rb

@@ -0,0 +1,271 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/error"
+require "active_sanction/query"
+require "active_sanction/scorer/subject"
+
+module ActiveSanction
+  # One entry in a book of business: an application's own id, and everything
+  # it knows about the person or company behind it.
+  #
+  #   subject = ActiveSanction::Subject.new(
+  #     id:            "cust_1",
+  #     name:          "Bosco Ntaganda",
+  #     type:          :individual,
+  #     date_of_birth: "1973",
+  #     country:       "CD"
+  #   )
+  #
+  #   subject.id     # => "cust_1"
+  #   subject.name   # => "Bosco Ntaganda"
+  #
+  # ### Why an id is the whole of what this adds
+  #
+  # `screen` answers about a name. Rescreening answers about *a customer*, and
+  # the two are not the same question: an alert has to name the row in the
+  # host's database that a compliance team is going to open, hold and
+  # eventually dispose of. A book screened as bare names comes back as an
+  # array somebody has to re-join by position, which works exactly until a
+  # book is filtered, streamed in batches, or contains the same name twice --
+  # and two customers called Jane Miller is not a corner case, it is Tuesday.
+  #
+  # So the id is required, it is the caller's own, and nothing here interprets
+  # it. It travels onto every alert the subject produces (Rescreen::Alert), so
+  # a run's output joins back to the host's records without the host having
+  # kept the order it sent them in.
+  #
+  # ### It takes what `screen` takes
+  #
+  # Every evidence field a screening call accepts is accepted here, in every
+  # spelling Query accepts it in -- `date_of_birth:` and `dates_of_birth:`,
+  # `country:`, `countries:` and `nationalities:`, `identifier:` and
+  # `identifiers:`. A subject is a query about a customer, and a caller should
+  # not have to learn a second vocabulary to write one.
+  #
+  # The two things it refuses are the search *options* that a rescreen decides
+  # for itself. `sources:` is settled by the diff being applied -- a rescreen
+  # runs against one list version pair and nothing else -- and `limit:` would
+  # cap the alerts a subject can raise, which is not a thing this library is
+  # willing to do: an alert dropped for being eleventh is a sanctions hit
+  # nobody sees. Both are refused rather than ignored, because a search option
+  # that is silently dropped is a caller screening under a rule they think
+  # they set.
+  #
+  # `threshold:` is accepted, and is the one policy knob a subject carries.
+  # Risk-based screening is ordinary -- a correspondent bank at 70, a retail
+  # customer at 85 -- and a book that could not express it would force a host
+  # into one call per tier. Left unset, the subject is screened at whatever
+  # threshold the run names. See Rescreen.
+  #
+  # ### Where the evidence lives
+  #
+  # In a Scorer::Subject, built once here, which is the same object a Query
+  # holds and hands the scorer. The name is folded once, at construction, and
+  # a rescreening run compares that one folded form against every changed
+  # record rather than re-folding per comparison -- which is what makes
+  # streaming a large book past a small diff cheap.
+  #
+  # Instances are frozen on construction and compare by value.
+  class Subject
+    extend T::Sig
+
+    # The canonical spelling of every field, and the shape `#to_h` emits.
+    #
+    # @api private
+    MEMBERS = T.let((%i[id] + Scorer::Subject::MEMBERS + %i[threshold]).freeze, T::Array[Symbol])
+
+    # Search options a rescreen settles for itself. See the class comment.
+    #
+    # @api private
+    REFUSED = T.let(
+      {
+        sources: "the diff being applied names the list -- a rescreen runs against one pair of list versions",
+        limit: "a rescreen reports every changed record a subject matches -- an alert dropped for being " \
+               "eleventh is a hit nobody sees"
+      }.freeze,
+      T::Hash[Symbol, String]
+    )
+
+    # The caller's own id for this subject, carried onto every alert.
+    sig { returns(String) }
+    attr_reader :id
+
+    # The evidence, folded once. What the scorer compares against a record.
+    sig { returns(Scorer::Subject) }
+    attr_reader :evidence
+
+    # The lowest score worth an alert for this subject, or nil to take the
+    # run's. See the class comment.
+    sig { returns(T.nilable(Float)) }
+    attr_reader :threshold
+
+    class << self
+      extend T::Sig
+
+      # Whatever a caller had, as a Subject:
+      #
+      #   Subject.build(subject)                                  # itself
+      #   Subject.build(id: "cust_1", name: "Bosco Ntaganda")     # a Hash, string keys or symbol
+      #
+      # What a book of business is streamed through, so a host can hand this
+      # library the rows it already has rather than mapping them first.
+      sig { params(value: T.untyped).returns(Subject) }
+      def build(value)
+        case value
+        when Subject then value
+        when Hash then from_h(value)
+        else
+          raise InvalidArgument,
+                "a book holds ActiveSanction::Subject or Hash entries, got #{value.class}. A rescreen reports " \
+                "alerts against a caller's own id, so a bare name is not enough to raise one"
+        end
+      end
+
+      # Rebuilds a subject from #to_h output, accepting string keys so a book
+      # read out of a database or a JSON payload needs no translation.
+      sig { params(hash: T.untyped).returns(Subject) }
+      def from_h(hash)
+        # `new(**hash)` past a required keyword parameter is one of the few
+        # things Sorbet cannot check statically. #initialize validates what
+        # arrives, which is where a bad round-trip is caught.
+        T.unsafe(self).new(**hash.to_h.transform_keys { |key| key.to_s.to_sym })
+      end
+    end
+
+    # `id` and `name` are required; everything else is optional, for the
+    # reason Scorer::Subject gives -- most callers have a name and little
+    # else, and a field absent on either side is neutral rather than a
+    # conflict.
+    #
+    # The evidence fields are taken in any spelling Query accepts them in, and
+    # resolved here rather than in a separate builder: unlike a Query, which a
+    # screening call constructs on a caller's behalf, this is the object a
+    # host writes out by hand, so `.new` is the door everything comes through.
+    sig { params(id: T.untyped, fields: T.untyped).void }
+    def initialize(id:, **fields)
+      attributes = normalize(fields)
+      @id = T.let(id!(id), String)
+      @evidence = T.let(
+        Scorer::Subject.new(name: attributes[:name], type: attributes[:type],
+                            dates_of_birth: attributes[:dates_of_birth] || [],
+                            nationalities: attributes[:nationalities] || [],
+                            identifiers: attributes[:identifiers] || []),
+        Scorer::Subject
+      )
+      @threshold = T.let(threshold!(attributes[:threshold]), T.nilable(Float))
+      freeze
+    end
+
+    # The name as the caller wrote it, which is what an alert quotes back.
+    sig { returns(String) }
+    def name = evidence.name
+
+    sig { returns(T.nilable(Symbol)) }
+    def type = evidence.type
+
+    sig { returns(T::Array[PartialDate]) }
+    def dates_of_birth = evidence.dates_of_birth
+
+    sig { returns(T::Array[String]) }
+    def nationalities = evidence.nationalities
+
+    sig { returns(T::Array[Identifier]) }
+    def identifiers = evidence.identifiers
+
+    # The folded name every comparison runs against, folded once at
+    # construction. What a rescreening run hands the index rather than the
+    # string it came from.
+    sig { returns(Normalizer::Form) }
+    def form = evidence.form
+
+    # This subject as the screening call it is, at the threshold given or its
+    # own -- which is what a MatchResult records as the question that was
+    # asked. `sources:` is the list the run covers.
+    sig { params(threshold: T.untyped, sources: T.untyped).returns(Query) }
+    def query(threshold: nil, sources: nil)
+      T.unsafe(Query).new(**evidence.to_h, sources: sources, threshold: threshold || self.threshold)
+    end
+
+    sig { returns(T::Hash[Symbol, T.untyped]) }
+    def to_h = { id: id }.merge(evidence.to_h).merge(threshold: threshold)
+
+    sig { params(other: T.untyped).returns(T::Boolean) }
+    def ==(other)
+      return false unless other.instance_of?(self.class)
+
+      to_h == other.to_h
+    end
+    alias eql? ==
+
+    sig { returns(Integer) }
+    def hash = [self.class, to_h].hash
+
+    sig { returns(String) }
+    def inspect = "#<#{self.class} #{id} #{name.inspect}>"
+
+    private
+
+    # Symbol keys, with every spelling Query accepts resolved to its canonical
+    # one -- and the two search options a rescreen settles for itself refused
+    # by name rather than ignored.
+    sig { params(fields: T::Hash[Symbol, T.untyped]).returns(T::Hash[Symbol, T.untyped]) }
+    def normalize(fields)
+      attributes = fields.each_with_object({}) do |(key, value), resolved|
+        member = Query::ALIASES.fetch(key, key)
+        reason = REFUSED[member]
+        raise InvalidArgument, "a Subject does not take #{member}: #{reason}" if reason
+
+        resolved[member] = wrap(member, value)
+      end
+      unknown = attributes.keys - MEMBERS
+      raise InvalidArgument, "unknown Subject attribute(s): #{unknown.join(", ")}" if unknown.any?
+
+      attributes
+    end
+
+    # A single value where a collection is expected is a collection of one,
+    # for the reason Query gives: `dates_of_birth: "1973"` is what a
+    # caller with one date writes, and `Array(hash)` would read one identifier
+    # as a list of its pairs.
+    sig { params(member: Symbol, value: T.untyped).returns(T.untyped) }
+    def wrap(member, value)
+      return value unless Query::COLLECTIONS.include?(member)
+      return value if value.nil? || value.is_a?(Array)
+
+      [value]
+    end
+
+    # Required, and taken as the caller wrote it: an id this library edited
+    # would not join back to the row it came from.
+    sig { params(value: T.untyped).returns(String) }
+    def id!(value)
+      string = value.to_s.strip
+      if string.empty?
+        raise InvalidArgument,
+              "a subject needs an id -- an alert has to name the record a compliance team will open, and a " \
+              "book screened by position cannot survive being filtered or streamed"
+      end
+
+      -string
+    end
+
+    # nil means "whatever the run says", which is the common case and the
+    # reason this is not defaulted from configuration here: a subject that
+    # quietly carried the configured threshold could not be told apart from
+    # one that asked for it, and the run could never override it.
+    sig { params(value: T.untyped).returns(T.nilable(Float)) }
+    def threshold!(value)
+      return nil if value.nil?
+
+      number = begin
+        Float(value)
+      rescue TypeError, ArgumentError
+        raise InvalidArgument, "threshold must be a number between 0 and 100, got #{value.inspect}"
+      end
+      Scorer.threshold!(number)
+    end
+  end
+end

data/lib/active_sanction/sync/report.rb

@@ -0,0 +1,245 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "time"
+require "active_sanction/sync/result"
+
+module ActiveSanction
+  class Sync
+    # What a whole sync run did, one Result per source.
+    #
+    #   report = ActiveSanction.sync!
+    #
+    #   report.failed?                  # => true
+    #   report[:un_consolidated].error  # => "Net::ReadTimeout: ..."
+    #   report.updated.map(&:source)    # => [:ofac_sdn]
+    #   puts report                     # => the table below
+    #
+    #   4 sources in 13.08s: 1 updated, 2 unchanged, 1 failed
+    #     ofac_sdn           updated    19015 records  just fetched   12.41s
+    #     ofac_consolidated  unchanged   1203 records  2h old          0.28s
+    #     canada_sema        unchanged    684 records  2h old          0.19s
+    #     un_consolidated    failed       612 records  3d old          1.11s  Net::ReadTimeout: execution expired
+    #
+    # ### It is an object, not console output
+    #
+    # This is the operational surface of a sync: it is what a host application
+    # alerts on, what a scheduled job exits with, and what the instrumentation
+    # hooks (#59) emit. So it serializes to a documented shape and `.from_h`
+    # rebuilds it -- a summary that only existed as printed text would mean
+    # every host that wants to notice a degrading source has to scrape a log.
+    #
+    # Note what the table prints beside a failure: the record count and age of
+    # the snapshot that source is *still* being screened against. A failed sync
+    # keeps its previous snapshot, which is the right call and is only safe
+    # while the age of what is being screened against is visible.
+    #
+    # Instances are frozen on construction and compare by value.
+    class Report
+      extend T::Sig
+      extend T::Generic
+      include Enumerable
+
+      # @api private
+      Elem = type_member { { fixed: Result } }
+
+      # @api private
+      MEMBERS = T.let(%i[results started_at duration].freeze, T::Array[Symbol])
+
+      sig { returns(T::Array[Result]) }
+      attr_reader :results
+
+      # When the run began, UTC.
+      sig { returns(Time) }
+      attr_reader :started_at
+
+      # Wall-clock seconds for the whole run, which is less than the sum of the
+      # per-source durations when sources ran in parallel.
+      sig { returns(Float) }
+      attr_reader :duration
+
+      # Rebuilds from #to_h output, accepting string keys so a report survives
+      # the trip through JSON.
+      sig { params(hash: T.untyped).returns(T.attached_class) }
+      def self.from_h(hash)
+        attributes = hash.to_h.transform_keys(&:to_sym)
+        unknown = attributes.keys - MEMBERS
+        raise InvalidArgument, "unknown Sync::Report attribute(s): #{unknown.join(", ")}" if unknown.any?
+
+        T.unsafe(self).new(**attributes)
+      end
+
+      sig { params(results: T.untyped, started_at: T.untyped, duration: T.untyped).void }
+      def initialize(results:, started_at: nil, duration: 0.0)
+        @results = T.let(results!(results), T::Array[Result])
+        @started_at = T.let(time!(started_at), Time)
+        @duration = T.let(duration.to_f, Float)
+        freeze
+      end
+
+      sig { override.params(block: T.nilable(T.proc.params(result: Result).void)).returns(T.untyped) }
+      def each(&block)
+        return enum_for(:each) unless block
+
+        results.each(&block)
+        self
+      end
+
+      # One source's result, or nil if the run did not cover it. A run that did
+      # not cover a source is not the same as one where it succeeded, which is
+      # why this does not raise: a caller asking about a source it did not sync
+      # is asking a question with an answer.
+      sig { params(source: T.untyped).returns(T.nilable(Result)) }
+      def [](source)
+        key = source.to_sym
+        results.find { |result| result.source == key }
+      end
+
+      sig { returns(T::Array[Symbol]) }
+      def sources = results.map(&:source)
+
+      sig { returns(T::Array[Result]) }
+      def updated = results.select(&:updated?)
+
+      sig { returns(T::Array[Result]) }
+      def unchanged = results.select(&:unchanged?)
+
+      sig { returns(T::Array[Result]) }
+      def failed = results.select(&:failed?)
+
+      # Sources that came out of this run with no snapshot stored at all, and
+      # so are not covered by screening. Louder than `failed` and rarer: a
+      # source that failed but kept its previous list is stale, one that has
+      # nothing stored is missing.
+      sig { returns(T::Array[Result]) }
+      def unscreenable = results.reject(&:stored?)
+
+      sig { returns(T::Boolean) }
+      def failed? = results.any?(&:failed?)
+
+      sig { returns(T::Boolean) }
+      def success? = !failed?
+
+      sig { returns(Integer) }
+      def size = results.size
+
+      sig { returns(T::Boolean) }
+      def empty? = results.empty?
+
+      # Records stored across every source the run covered, failures included:
+      # what is screenable now, rather than what was downloaded.
+      sig { returns(Integer) }
+      def record_count = results.sum { |result| result.record_count || 0 }
+
+      # The age of the stalest list this run left behind, in seconds. The one
+      # number to alert on if a host only wants one.
+      sig { returns(T.nilable(Integer)) }
+      def oldest_age = results.filter_map(&:age).max
+
+      # What a scheduled job should exit with, so that cron mails somebody and
+      # CI goes red when a source is failing. Deliberately here rather than
+      # left to each caller to derive: "one source failed" has to mean the same
+      # thing to every wrapper anyone writes around a sync.
+      #
+      #   exit ActiveSanction.sync!.exit_code
+      sig { returns(Integer) }
+      def exit_code = failed? ? 1 : 0
+
+      # For a caller that wants any failure to be fatal, in the manner of
+      # Fetcher::Result#success!. Note that this is not what `sync!` does: the
+      # run has already finished and every other source has already been
+      # stored, so raising here reports a failure rather than causing one.
+      sig { returns(T.self_type) }
+      def success!
+        return self if success?
+
+        raise Failed, self
+      end
+
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      def to_h
+        {
+          results: results.map(&:to_h),
+          started_at: started_at.iso8601,
+          duration: duration
+        }
+      end
+
+      # The sentence a failing run should put in front of a human: which
+      # sources failed, out of how many, and why.
+      sig { params(result: Result).returns(String) }
+      def records(result) = result.record_count&.to_s || "-"
+
+      sig { params(block: T.proc.params(result: Result).returns(String)).returns(Integer) }
+      def width(&block) = results.map { |result| block.call(result).length }.max.to_i
+
+      sig { returns(String) }
+      def failure_message
+        "#{failed.size} of #{size} source(s) failed to sync: " +
+          failed.map { |result| "#{result.source} (#{result.error})" }.join("; ")
+      end
+
+      sig { returns(String) }
+      def summary
+        counts = { updated: updated.size, unchanged: unchanged.size, failed: failed.size }
+                 .reject { |_status, count| count.zero? }
+                 .map { |status, count| "#{count} #{status}" }
+        "#{size} #{size == 1 ? "source" : "sources"} in #{format("%.2f", duration)}s" \
+          "#{": #{counts.join(", ")}" unless counts.empty?}"
+      end
+
+      sig { returns(String) }
+      def to_s = ([summary] + rows).join("\n")
+
+      sig { params(other: T.untyped).returns(T::Boolean) }
+      def ==(other)
+        return false unless other.instance_of?(self.class)
+
+        to_h == other.to_h
+      end
+      alias eql? ==
+
+      sig { returns(Integer) }
+      def hash = [self.class, to_h].hash
+
+      sig { returns(String) }
+      def inspect = "#<#{self.class} #{summary}>"
+
+      private
+
+      # One padded line per source. Columns rather than sentences because the
+      # thing an operator is doing with this is scanning down it for the row
+      # that is not like the others.
+      sig { returns(T::Array[String]) }
+      def rows
+        name = width { |result| result.source.to_s }
+        count = width { |result| records(result) }
+        age = width(&:age_in_words)
+        results.map do |result|
+          "  #{result.source.to_s.ljust(name)}  #{result.status.to_s.ljust(9)}  " \
+            "#{records(result).rjust(count)} records  #{result.age_in_words.ljust(age)}  " \
+            "#{format("%.2f", result.duration).rjust(6)}s#{"  #{result.error}" if result.failed?}"
+        end
+      end
+
+      sig { params(value: T.untyped).returns(T::Array[Result]) }
+      def results!(value)
+        list = Array(value).map { |result| result.is_a?(Result) ? result : Result.from_h(result) }
+        list.freeze
+      end
+
+      sig { params(value: T.untyped).returns(Time) }
+      def time!(value)
+        time = case value
+               when nil then Time.now
+               when Time then value
+               when String then Time.parse(value)
+               else raise InvalidArgument, "started_at is not a time: #{value.inspect}"
+               end
+        Time.at(time.to_i).utc
+      end
+    end
+  end
+end