active_sanction 1.0.0

data/lib/active_sanction/scorer/subject.rb

@@ -0,0 +1,235 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/country"
+require "active_sanction/entity"
+require "active_sanction/identifier"
+require "active_sanction/normalizer"
+require "active_sanction/partial_date"
+
+module ActiveSanction
+  module Scorer
+    # What the caller knows about the person or company being screened.
+    #
+    #   subject = ActiveSanction::Scorer::Subject.new(
+    #     name:           "Bosco Ntaganda",
+    #     type:           :individual,
+    #     dates_of_birth: "1973",
+    #     nationalities:  %w[CD],
+    #     identifiers:    [{ kind: :passport, value: "AB-123 456" }]
+    #   )
+    #
+    #   subject.form.value   # => "bosco ntaganda"
+    #
+    # One side of every comparison the scorer makes, and the mirror image of
+    # the Entity on the other side: the same four kinds of evidence, arriving
+    # from an application's own customer record rather than from a government.
+    #
+    # ### Only `name` is required, and that is the shape of the problem
+    #
+    # Most callers have a name and little else, and most records carry less
+    # than that -- Canada supplies no aliases and often no date of birth. So
+    # every field but the name is optional on both sides, and a field absent
+    # on either side is neutral rather than a conflict. See Adjustments, where
+    # that rule is the difference between a screening tool and a tool that
+    # systematically under-scores the sparser lists.
+    #
+    # ### `type` decides two things
+    #
+    # It is the entity type the caller is asking about, and it does two jobs
+    # that are easy to confuse. It selects the normalizer's stoplists, so a
+    # company name is folded with its legal form stripped -- and both sides of
+    # a comparison have to be folded the same way, which is why the scorer
+    # folds each candidate name under its own entity's type. And it filters:
+    # a subject that says `:individual` is never scored against a vessel, at
+    # any name similarity. See Scorer.
+    #
+    # Passing no type is a legitimate answer and a different question -- no
+    # stoplist, no filter -- rather than a worse one.
+    #
+    # ### The fold happens once, here
+    #
+    # `form` is the folded name, produced by the one `Normalizer.call` every
+    # other stage uses, and held for the life of the subject. A screening call
+    # compares one subject against a few hundred candidates, and folding the
+    # query per candidate would be the same string folded a few hundred times.
+    #
+    # A caller that has already folded a name passes the Form, which is what
+    # screening one name against several indexes should do.
+    #
+    # ### Where Query fits
+    #
+    # This is the scorer's input, not the library's public screening API. The
+    # `Query` object (#33) validates what a host application sends -- a
+    # threshold, a limit, a source filter -- and builds one of these for the
+    # matcher to score with. Everything on this class is evidence about a
+    # subject; nothing on it is a search option.
+    #
+    # Instances are frozen on construction and compare by value.
+    class Subject
+      extend T::Sig
+
+      # @api private
+      MEMBERS = T.let(%i[name type dates_of_birth nationalities identifiers].freeze, T::Array[Symbol])
+
+      # The folded name every comparison runs against.
+      sig { returns(Normalizer::Form) }
+      attr_reader :form
+
+      sig { returns(T.nilable(Symbol)) }
+      attr_reader :type
+
+      sig { returns(T::Array[PartialDate]) }
+      attr_reader :dates_of_birth
+
+      # As published by the caller, in the caller's own vocabulary: `RU`,
+      # `Russia` and `Russian Federation` are all fine here. See #countries
+      # for the resolved form the scorer compares on.
+      sig { returns(T::Array[String]) }
+      attr_reader :nationalities
+
+      sig { returns(T::Array[Identifier]) }
+      attr_reader :identifiers
+
+      sig { params(hash: T.untyped).returns(T.attached_class) }
+      def self.from_h(hash)
+        attributes = hash.to_h.transform_keys(&:to_sym)
+        unknown = attributes.keys - MEMBERS
+        raise InvalidArgument, "unknown Subject attribute(s): #{unknown.join(", ")}" if unknown.any?
+
+        T.unsafe(self).new(**attributes)
+      end
+
+      # `name` is a String, a Name or an already-folded Form. The three
+      # collections each accept a single value as a collection of one, since
+      # `dates_of_birth: "1973"` is what a caller with one date writes.
+      #
+      # Dates accept anything PartialDate reads, including the free text these
+      # lists publish; identifiers accept an Identifier, its hash, or a bare
+      # document number, which becomes an identifier of unstated kind.
+      sig do
+        params(name: T.untyped, type: T.untyped, dates_of_birth: T.untyped, nationalities: T.untyped,
+               identifiers: T.untyped).void
+      end
+      def initialize(name:, type: nil, dates_of_birth: [], nationalities: [], identifiers: [])
+        @type = T.let(type!(type), T.nilable(Symbol))
+        @form = T.let(form!(name), Normalizer::Form)
+        @dates_of_birth = T.let(Array(dates_of_birth).map { |value| date!(value) }.freeze, T::Array[PartialDate])
+        @nationalities = T.let(strings(nationalities), T::Array[String])
+        @identifiers = T.let(Array(identifiers).map { |value| identifier!(value) }.freeze, T::Array[Identifier])
+        @countries = T.let(resolve(@nationalities), T::Array[String])
+        freeze
+      end
+
+      # The name as the caller wrote it, which is what a report quotes back.
+      sig { returns(String) }
+      def name = form.original
+
+      # The alpha-2 codes #nationalities resolved to, which may be shorter
+      # than the list it came from: a value Country does not recognize is
+      # dropped here rather than guessed at, and the scorer treats a subject
+      # whose countries did not all resolve as one that cannot contradict a
+      # record. See Adjustments.
+      sig { returns(T::Array[String]) }
+      attr_reader :countries
+
+      # True when every nationality the caller gave resolved to a country.
+      # A conflict penalty is only applied when both sides can say this.
+      sig { returns(T::Boolean) }
+      def countries? = nationalities.any? && countries.size == nationalities.uniq.size
+
+      sig { returns(T::Boolean) }
+      def dates_of_birth? = dates_of_birth.any?
+
+      sig { returns(T::Boolean) }
+      def identifiers? = identifiers.any?
+
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      def to_h
+        {
+          name: name,
+          type: type,
+          dates_of_birth: dates_of_birth.map(&:to_h),
+          nationalities: nationalities,
+          identifiers: identifiers.map(&:to_h)
+        }
+      end
+
+      sig { params(other: T.untyped).returns(T::Boolean) }
+      def ==(other)
+        return false unless other.instance_of?(self.class)
+
+        to_h == other.to_h
+      end
+      alias eql? ==
+
+      sig { returns(Integer) }
+      def hash = [self.class, to_h].hash
+
+      sig { returns(String) }
+      def inspect = "#<#{self.class} #{name.inspect}#{" type=#{type.inspect}" if type}>"
+
+      private
+
+      # A Form is taken as it stands, and it is the caller's job to have
+      # folded it under the same type -- there is no way to check, since a
+      # Form does not record which stoplist produced it, and re-folding it
+      # here would silently discard the caller's intent. See Normalizer on
+      # why one fold, once, is the whole point.
+      sig { params(value: T.untyped).returns(Normalizer::Form) }
+      def form!(value)
+        return value if value.is_a?(Normalizer::Form)
+
+        string = value.to_s.strip
+        raise InvalidArgument, "name is required -- there is nothing to screen without one" if string.empty?
+
+        folded = Normalizer.call(string, type: type)
+        raise InvalidArgument, "name folds away to nothing: #{string.inspect}" if folded.empty?
+
+        folded
+      end
+
+      sig { params(value: T.untyped).returns(T.nilable(Symbol)) }
+      def type!(value)
+        return nil if value.nil? || value.to_s.empty?
+
+        symbol = value.to_s.downcase.to_sym
+        return symbol if Entity::TYPES.include?(symbol)
+
+        raise InvalidArgument, "unknown type #{symbol.inspect}, expected one of #{Entity::TYPES.join(", ")} or nil"
+      end
+
+      sig { params(value: T.untyped).returns(PartialDate) }
+      def date!(value)
+        case value
+        when PartialDate then value
+        when Hash then PartialDate.from_h(value)
+        else PartialDate.parse(value) || raise(InvalidArgument, "not a date of birth: #{value.inspect}")
+        end
+      end
+
+      sig { params(value: T.untyped).returns(Identifier) }
+      def identifier!(value)
+        case value
+        when Identifier then value
+        when Hash then Identifier.from_h(value)
+        else Identifier.new(value: value)
+        end
+      end
+
+      # A value Country does not recognize is dropped rather than guessed at.
+      # #countries? is how a caller tells a fully resolved list from a partly
+      # resolved one, which is the difference between a nationality that may
+      # contradict a record and one that may only agree with it.
+      sig { params(values: T::Array[String]).returns(T::Array[String]) }
+      def resolve(values) = values.filter_map { |value| Country.code(value) }.uniq.freeze
+
+      sig { params(value: T.untyped).returns(T::Array[String]) }
+      def strings(value)
+        Array(value).map { |entry| -entry.to_s.strip }.reject(&:empty?).uniq.freeze
+      end
+    end
+  end
+end

data/lib/active_sanction/scorer/weights.rb

@@ -0,0 +1,306 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module ActiveSanction
+  module Scorer
+    # Every number the scorer uses, with a default and a reason for it.
+    #
+    #   ActiveSanction::Scorer::Weights.default.token_set        # => 0.45
+    #   ActiveSanction::Scorer::Weights.default.dob_conflict     # => -35.0
+    #
+    #   ActiveSanction.configure do |c|
+    #     c.scorer_weights = { dob_conflict: -20.0 }
+    #   end
+    #
+    # There are two kinds of number here and they are in different units,
+    # which is the thing to know before changing one.
+    #
+    # ### The name shares, which are fractions of the name score
+    #
+    # `jaro_winkler`, `levenshtein`, `token_sort`, `token_set` and `phonetic`
+    # are shares of one blended similarity. They are each between 0 and 1 and
+    # they must sum to exactly 1, which is what makes the name score a
+    # percentage rather than an arbitrary total: two identical names score
+    # 100 because every share agreed, and a share that agreed on nothing
+    # contributes nothing.
+    #
+    # The defaults are set from what the four algorithms disagree about --
+    # Similarity's own table is the argument, and it is worth reading beside
+    # this:
+    #
+    #                                        JW     Lev    Sort    Set   Phon   blend
+    #     abbas abu / abu abbas            0.805  0.333  1.000  1.000  1.000    90.4
+    #     putin vladimir vladimirovich /
+    #       vladimir putin                 0.679  0.357  0.500  1.000  1.000    76.2
+    #     gazprom / gazprom neft           0.917  0.583  0.583  1.000  1.000    84.2
+    #     kim jong un / kim yong chol      0.869  0.615  0.462  0.462  0.667    54.8
+    #
+    # **The token ratios carry most of the weight**, because the query shapes
+    # this corpus actually produces are rearrangements. An individual is
+    # published surname-first and typed given-name-first; a patronymic is on
+    # the record and not in the query. Row one and row two are both true
+    # matches that the character algorithms score in the sixties and the
+    # seventies, and a blend that let them decide would miss the two most
+    # common true positives there are.
+    #
+    # **`token_set` is the largest single share** because a 1.0 from it means
+    # something specific and strong: every word of the shorter name appears in
+    # the longer one. That is the shape of nearly every honest partial query.
+    #
+    # **The character algorithms are the brake.** Row four is the case they
+    # exist for -- two names already written in the same order, where sorting
+    # loses the information that they are, and the token ratios happily score
+    # a different person at 0.462. Levenshtein's share is the smallest because
+    # it is the harshest measure in the set: on a name that is one token
+    # longer it is already down in the fifties, and giving it more would pull
+    # every true partial match down with it.
+    #
+    # **The phonetic share is small and it is a share, not a bonus.** A shared
+    # Double Metaphone key is real evidence -- it is what puts `QADHAFI` and
+    # `GADDAFI` together -- and it is weak evidence, because `HSN` is the key
+    # for `HUSSEIN` and equally for `HASSAN`. Five points is what it is worth
+    # on its own; the reason it is inside the sum rather than added on top is
+    # that a bonus would put an identical pair over 100 and need clamping to
+    # get back, and a score that reaches its ceiling by two different routes
+    # is one nobody can reason about.
+    #
+    # ### The adjustments, which are points on the 0..100 score
+    #
+    # These are added to the name score, not multiplied into it, because they
+    # are separate evidence rather than a re-reading of the name. A passport
+    # number is not "more name"; it is the thing that makes a mediocre name
+    # match decisive, and a boost that scaled with the name score could not do
+    # that.
+    #
+    # `identifier_match` is 40 and is meant to be decisive: a name in the
+    # fifties plus the right passport number clears any sane threshold, which
+    # is the entire reason a screening tool asks for document numbers.
+    #
+    # `dob_conflict` at -35 is the one number the acceptance criteria pin
+    # down. A name-identical pair scores 100, and a genuine date-of-birth
+    # conflict has to put it under the threshold rather than merely rank it
+    # lower -- 65 is under every default this library ships. The exact match
+    # is worth less than the conflict costs on purpose: sharing a birthday
+    # with a listed person is a coincidence a few thousand people have, and
+    # not having theirs is not.
+    #
+    # `dob_overlap` at 6 is what a year-only date is worth. Most of these
+    # records carry one -- see PartialDate on why the type exists -- and
+    # `1948` against `1948-12-10` is agreement worth noting and not worth much.
+    #
+    # Nationality moves the score least in both directions, because it is the
+    # softest of the three. People hold two passports, lists record the
+    # country a person was born in as often as the one they are a citizen of,
+    # and a conflict there is weaker evidence than a date conflict by some way.
+    #
+    # ### Penalties are stored negative
+    #
+    # `dob_conflict` is `-35.0` rather than `35.0` subtracted somewhere else,
+    # so that a Reason's contribution is the number in this object and a host
+    # reading a configuration can see which way each one pushes. A boost
+    # written negative, or a penalty written positive, is refused.
+    #
+    # Instances are frozen on construction and compare by value.
+    class Weights
+      extend T::Sig
+
+      # The five shares of the blended name score. They sum to 1.
+      #
+      # @api private
+      NAME_SHARES = T.let(%i[jaro_winkler levenshtein token_sort token_set phonetic].freeze, T::Array[Symbol])
+
+      # Points added to the name score. Anything listed in PENALTIES must be
+      # zero or negative; everything else here must be zero or positive.
+      #
+      # @api private
+      ADJUSTMENTS = T.let(%i[
+        low_quality_alias identifier_match dob_exact dob_overlap dob_conflict
+        nationality_match nationality_conflict
+      ].freeze, T::Array[Symbol])
+
+      # @api private
+      PENALTIES = T.let(%i[low_quality_alias dob_conflict nationality_conflict].freeze, T::Array[Symbol])
+
+      # @api private
+      MEMBERS = T.let((NAME_SHARES + ADJUSTMENTS).freeze, T::Array[Symbol])
+
+      # Every weight this library ships, and the numbers the committed
+      # accuracy report was measured with. A host tuning one starts from
+      # here, changes what it means to and leaves the rest alone -- the
+      # shares are held to summing to 1.0, so they move in pairs.
+      DEFAULTS = T.let({
+        jaro_winkler: 0.15,
+        levenshtein: 0.10,
+        token_sort: 0.25,
+        token_set: 0.45,
+        phonetic: 0.05,
+        low_quality_alias: -10.0,
+        identifier_match: 40.0,
+        dob_exact: 15.0,
+        dob_overlap: 6.0,
+        dob_conflict: -35.0,
+        nationality_match: 6.0,
+        nationality_conflict: -12.0
+      }.freeze, T::Hash[Symbol, Float])
+
+      # Floating point addition of five decimal fractions does not land on 1.0
+      # exactly, and refusing a set of shares over the last bit of a Float
+      # would be refusing arithmetic rather than a misconfiguration.
+      #
+      # @api private
+      SHARE_TOLERANCE = T.let(1e-9, Float)
+
+      # Spelled out rather than defined from MEMBERS in a loop, because a
+      # reader Sorbet cannot see is a reader every call site has to be
+      # `T.unsafe` to reach. Each one reads the hash; nothing here is stored
+      # twice.
+      sig { returns(Float) }
+      def jaro_winkler = fetch(:jaro_winkler)
+
+      sig { returns(Float) }
+      def levenshtein = fetch(:levenshtein)
+
+      sig { returns(Float) }
+      def token_sort = fetch(:token_sort)
+
+      sig { returns(Float) }
+      def token_set = fetch(:token_set)
+
+      sig { returns(Float) }
+      def phonetic = fetch(:phonetic)
+
+      sig { returns(Float) }
+      def low_quality_alias = fetch(:low_quality_alias)
+
+      sig { returns(Float) }
+      def identifier_match = fetch(:identifier_match)
+
+      sig { returns(Float) }
+      def dob_exact = fetch(:dob_exact)
+
+      sig { returns(Float) }
+      def dob_overlap = fetch(:dob_overlap)
+
+      sig { returns(Float) }
+      def dob_conflict = fetch(:dob_conflict)
+
+      sig { returns(Float) }
+      def nationality_match = fetch(:nationality_match)
+
+      sig { returns(Float) }
+      def nationality_conflict = fetch(:nationality_conflict)
+
+      class << self
+        extend T::Sig
+
+        # The shipped numbers. Built at load, so nothing has to synchronize
+        # its construction.
+        sig { returns(Weights) }
+        def default = DEFAULT
+
+        # The shipped numbers with some replaced, which is what a host almost
+        # always wants:
+        #
+        #   Weights.build(dob_conflict: -20.0)
+        #
+        # A Weights passes through, so a caller holding either can hand this
+        # whatever it has.
+        sig { params(value: T.untyped).returns(Weights) }
+        def build(value)
+          return default if value.nil?
+          return value if value.is_a?(Weights)
+          raise InvalidArgument, "expected a #{self} or a Hash of weights, got #{value.class}" unless value.is_a?(Hash)
+
+          T.unsafe(default).merge(**value.to_h { |member, weight| [member.to_s.to_sym, weight] })
+        end
+      end
+
+      # Every member defaults, so `new` and `new(dob_conflict: -20.0)` are
+      # both a complete set. Unlike the normalizer's dictionaries there is no
+      # danger in a partial replacement here: a number left out is the shipped
+      # one, and the shares are checked to sum to 1 whatever a caller passed.
+      sig { params(overrides: T.untyped).void }
+      def initialize(**overrides)
+        unknown = overrides.keys - MEMBERS
+        raise InvalidArgument, "unknown weight(s): #{unknown.join(", ")}" if unknown.any?
+
+        @weights = T.let(DEFAULTS.merge(overrides).to_h { |member, weight| [member, number!(member, weight)] }.freeze,
+                         T::Hash[Symbol, Float])
+        validate_shares!
+        freeze
+      end
+
+      # These weights with some replaced.
+      sig { params(overrides: T.untyped).returns(Weights) }
+      def merge(**overrides) = T.unsafe(self.class).new(**@weights, **overrides)
+
+      sig { params(member: Symbol).returns(Float) }
+      def fetch(member) = @weights.fetch(member)
+
+      sig { returns(T::Hash[Symbol, Float]) }
+      def to_h = @weights.dup
+
+      sig { params(other: T.untyped).returns(T::Boolean) }
+      def ==(other)
+        return false unless other.instance_of?(self.class)
+
+        to_h == other.to_h
+      end
+      alias eql? ==
+
+      sig { returns(Integer) }
+      def hash = [self.class, to_h].hash
+
+      sig { returns(String) }
+      def inspect = "#<#{self.class} #{to_h.map { |member, weight| "#{member}=#{weight}" }.join(" ")}>"
+
+      private
+
+      sig { params(member: Symbol, value: T.untyped).returns(Float) }
+      def number!(member, value)
+        number = begin
+          Float(value)
+        rescue TypeError, ArgumentError
+          raise InvalidArgument, "#{member} must be a number, got #{value.inspect}"
+        end
+        direction!(member, number)
+        number
+      end
+
+      # A boost written negative is a configuration that quietly inverts a
+      # signal -- a passport match that lowers a score -- and it would look
+      # exactly like a scorer bug from outside.
+      sig { params(member: Symbol, number: Float).void }
+      def direction!(member, number)
+        if PENALTIES.include?(member)
+          raise InvalidArgument, "#{member} is a penalty and cannot be positive, got #{number}" if number.positive?
+        elsif number.negative?
+          raise InvalidArgument, "#{member} is a boost and cannot be negative, got #{number}"
+        end
+      end
+
+      sig { void }
+      def validate_shares!
+        NAME_SHARES.each do |share|
+          weight = @weights.fetch(share)
+          raise InvalidArgument, "#{share} must be between 0 and 1, got #{weight}" unless weight.between?(0.0, 1.0)
+        end
+        total = NAME_SHARES.sum { |share| @weights.fetch(share) }
+        return if (total - 1.0).abs <= SHARE_TOLERANCE
+
+        raise InvalidArgument,
+              "the name shares must sum to 1.0, got #{total.round(6)} -- " \
+              "#{NAME_SHARES.map { |share| "#{share}=#{@weights.fetch(share)}" }.join(", ")}"
+      end
+
+      # Last, because building it runs #initialize, which calls every private
+      # method above.
+      #
+      # @api private
+      DEFAULT = T.let(new, Weights)
+      private_constant :DEFAULT
+    end
+  end
+end