active_sanction 1.0.0

data/lib/active_sanction/similarity/token_set.rb

@@ -0,0 +1,159 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module ActiveSanction
+  module Similarity
+    # Token set ratio: compare what two names share against what each of them
+    # is, and take the best of it. 0..1.
+    #
+    #   TokenSet.call("putin vladimir vladimirovich", "vladimir putin")  # => 1.0
+    #   TokenSet.call("vladimir putin", "vladimir zhirinovsky")          # => 0.5714
+    #   TokenSet.call("gazprom neft", "gazprombank")                     # => 0.5833
+    #
+    # ### The shape it exists for
+    #
+    # A token one side has and the other does not, which is the ordinary
+    # condition of a real query rather than an edge case. Lists carry the full
+    # legal name and a query carries what somebody typed: a patronymic that
+    # only Russian records bother with, a middle name a payment message
+    # dropped, a `bin` chain written out in full on one side and abbreviated on
+    # the other. Sorting does nothing for any of it -- TokenSort scores
+    # `PUTIN, Vladimir Vladimirovich` against `Vladimir Putin` at 0.5, because
+    # half the characters on the longer side have no partner and the edit
+    # distance charges for every one of them.
+    #
+    # ### The three strings
+    #
+    # Take both names as sets of tokens and split them three ways -- what they
+    # share, what only the left has, what only the right has -- then build:
+    #
+    #   t0  the shared tokens, alphabetically
+    #   t1  t0, then the left's own tokens, alphabetically
+    #   t2  t0, then the right's own tokens, alphabetically
+    #
+    # and score the pair as the best of `t0/t1`, `t0/t2` and `t1/t2`. For
+    # `putin vladimir vladimirovich` against `vladimir putin` that is:
+    #
+    #   t0 = "putin vladimir"
+    #   t1 = "putin vladimir vladimirovich"
+    #   t2 = "putin vladimir"
+    #
+    # `t0/t2` is a perfect match, and the answer is 1.0. Every one of the three
+    # comparisons runs on Levenshtein, for the reason TokenSort gives.
+    #
+    # The construction is Seatgeek's, from `fuzzywuzzy`'s `token_set_ratio`,
+    # and it is kept as published rather than adjusted. A screening score that
+    # nobody outside this repository can reproduce is a score that has to be
+    # argued from scratch every time an examiner asks about it.
+    #
+    # ### What that tolerance costs, and who pays it
+    #
+    # A name whose tokens are all present in the other scores 1.0, however
+    # much else the other one says:
+    #
+    #   TokenSet.call("gazprom", "gazprom neft")  # => 1.0
+    #
+    # Those are two different companies, and this ratio cannot tell them
+    # apart -- `t0/t2` is a string against itself whenever one side's tokens
+    # are a subset of the other's, and no length difference is large enough to
+    # change that. The same generosity finds `Vladimir Putin` inside `PUTIN,
+    # Vladimir Vladimirovich`, so it is not a bug to be fixed here; it is the
+    # single property this algorithm has, and it points in both directions.
+    #
+    # Recorded rather than lamented, and recorded in the specs as well: what
+    # keeps it from deciding a hit is that the scorer (#32) blends four
+    # numbers, and the other three all charge for the extra word. Levenshtein
+    # scores that pair 0.583 and TokenSort 0.583. A scorer that let this
+    # column vote alone would put every subsidiary of every listed parent in
+    # front of an analyst at 100.
+    #
+    # ### Sets, so a repeated token is one token
+    #
+    # `ALI, Ali Hassan` is `ali ali hassan` folded, and the second `ali` is not
+    # a second piece of evidence. Deduplicating is what the name of the
+    # algorithm says and what these lists want: a repeated given name is a
+    # naming convention, not a stronger signal, and counting it twice would
+    # make an entity's score depend on how many times its own name repeats.
+    module TokenSet
+      extend T::Sig
+
+      module_function
+
+      # The similarity of two already-folded names -- see Similarity for what
+      # `threshold:` does and what it promises, and for why either side may be
+      # a string or the tokens it splits into.
+      #
+      # The threshold is passed down to all three comparisons rather than
+      # applied to their maximum, which keeps the promise exactly: the largest
+      # of three numbers is at or above the cutoff precisely when one of them
+      # is, and that one comes back exact.
+      sig { params(left: Value, right: Value, threshold: Numeric).returns(Float).checked(:tests) }
+      def call(left, right, threshold: 0.0)
+        cutoff = Similarity.threshold!(threshold)
+        left_tokens = Similarity.tokens(left)
+        right_tokens = Similarity.tokens(right)
+        # A name with no tokens cannot be scored, and the three strings would
+        # all be empty and compare as identical if this fell through. Both
+        # empty is the answer the other algorithms give for two empty strings.
+        return 1.0 if left_tokens.empty? && right_tokens.empty?
+        return 0.0 if left_tokens.empty? || right_tokens.empty?
+
+        shared, left_full, right_full = strings(left_tokens, right_tokens)
+        best(shared, left_full, right_full, cutoff)
+      end
+
+      # The three strings this actually compares, in the order `t0, t1, t2`.
+      #
+      # Public for the reason TokenSort.sorted is: this is the whole of the
+      # difference between what a caller passed and what was scored, and a hit
+      # a compliance user cannot account for is a hit they cannot clear.
+      #
+      # Where the sets are made: `&` and `-` both deduplicate what they keep,
+      # apart from the repeats inside a name's own tokens, which `uniq`
+      # removes.
+      sig { params(left: T::Array[String], right: T::Array[String]).returns([String, String, String]).checked(:tests) }
+      def strings(left, right)
+        shared = (left & right).sort
+        [shared.join(" "),
+         (shared + (left - right).uniq.sort).join(" "),
+         (shared + (right - left).uniq.sort).join(" ")]
+      end
+
+      # 1.0 for any two non-empty names, which is not much of a bound and is
+      # the true one.
+      #
+      # The other three algorithms can rule a perfect score out from a length
+      # difference alone, because a character one name has and the other does
+      # not costs something wherever it falls. This one cannot: a subset scores
+      # 1.0 at any length, which is the whole of what it is for. A tighter
+      # number here would be a wrong one, and wrong in the direction that
+      # silently discards true matches -- a caller cannot tell a pair rejected
+      # by a ceiling from one that scored badly.
+      #
+      # So `threshold:` buys this ratio nothing before the comparison and only
+      # what the three Levenshtein calls can find inside it. That is a real
+      # cost, paid where the tolerance is, and it is the reason the scorer
+      # runs the cheap columns first.
+      sig { params(left_length: Integer, right_length: Integer).returns(Float).checked(:tests) }
+      def ceiling(left_length, right_length)
+        return left_length == right_length ? 1.0 : 0.0 if left_length.zero? || right_length.zero?
+
+        1.0
+      end
+
+      # The best of the three, each computed against the caller's cutoff.
+      sig do
+        params(shared: String, left_full: String, right_full: String, cutoff: Float).returns(Float).checked(:tests)
+      end
+      def best(shared, left_full, right_full, cutoff)
+        [Levenshtein.call(shared, left_full, threshold: cutoff),
+         Levenshtein.call(shared, right_full, threshold: cutoff),
+         Levenshtein.call(left_full, right_full, threshold: cutoff)].max
+      end
+
+      private_class_method :best
+    end
+  end
+end

data/lib/active_sanction/similarity/token_sort.rb

@@ -0,0 +1,120 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module ActiveSanction
+  module Similarity
+    # Token sort ratio: put both names' tokens in alphabetical order, then
+    # compare what comes out. 0..1.
+    #
+    #   ActiveSanction::Similarity::TokenSort.call("abbas abu", "abu abbas")    # => 1.0
+    #   ActiveSanction::Similarity::TokenSort.call("smith john", "john smith")  # => 1.0
+    #   ActiveSanction::Similarity::TokenSort.call("abbas abu", "abbas abd")    # => 0.8889
+    #
+    # ### The shape it exists for
+    #
+    # Sanctions lists store a personal name inverted and a query almost never
+    # is. OFAC publishes `ABBAS, Abu` and `ZAYDAN, Muhammad`; a customer
+    # record says `Abu Abbas` and a payment message says `Muhammad Zaydan`.
+    # The comma does not survive to be read here -- Normalizer turns
+    # punctuation into a space, so both sides arrive as the same tokens in
+    # opposite orders -- and character-level comparison is helpless in front of
+    # that:
+    #
+    #   JaroWinkler.call("abbas abu", "abu abbas")   # => 0.8053
+    #   Levenshtein.call("abbas abu", "abu abbas")   # => 0.3333
+    #   TokenSort.call("abbas abu", "abu abbas")     # => 1.0
+    #
+    # Those are two names that are not merely similar but identical, scored as
+    # a miss at the 85 this industry screens on. Sorting is what makes word
+    # order stop mattering, and word order is the difference between how these
+    # lists are written and how anybody types.
+    #
+    # It is worth being plain that this is not clever. Sorting is a blunt
+    # instrument that answers one question exactly -- are these the same words
+    # in some order -- and the reason it is the right instrument is that the
+    # inversion it defeats is a publishing convention rather than a
+    # coincidence, applied to essentially every individual on every list.
+    #
+    # ### What it costs, and why Levenshtein is still here
+    #
+    # Sorting destroys the information that two names were *already* in the
+    # same order, which for a pair that shares its leading tokens is
+    # information worth having:
+    #
+    #   Levenshtein.call("kim jong un", "kim yong chol")  # => 0.6154
+    #   TokenSort.call("kim jong un", "kim yong chol")    # => 0.4615
+    #
+    # `jong` and `yong` line up as written and are pulled apart by the sort,
+    # which puts `jong` next to `chol` and `un` next to `yong`. So this ratio
+    # is an additional question rather than a better one, and the scorer (#32)
+    # is where the two answers meet. A pipeline that sorted first and compared
+    # once would be strictly worse than one that does neither.
+    #
+    # ### Why Levenshtein underneath and not Jaro-Winkler
+    #
+    # Winkler's premise is that people get the beginning of a name right and
+    # drift later, which is true of a name as it is written and false of one
+    # whose words have just been put in alphabetical order: the front of a
+    # sorted string is whichever token happened to sort first, so the prefix
+    # bonus would be paying for a property of the alphabet. Sorted `abbas abd`
+    # and `abbas abu` share six characters of prefix for no reason anybody
+    # typed.
+    #
+    # Levenshtein also charges honestly for a token the other side does not
+    # have, which after sorting is most of what is left to measure. Being
+    # generous about that is the other ratio's job, and two generous
+    # algorithms stacked on each other is how a token ratio starts saying yes
+    # to everything.
+    #
+    # ### What it does not handle, which is why TokenSet exists
+    #
+    # A token on one side and not the other. Sorting lines the shared words up
+    # but does nothing about the ones with nowhere to go, and the edit
+    # distance charges for every character of them:
+    #
+    #   TokenSort.call("putin vladimir vladimirovich", "vladimir putin")  # => 0.5
+    #
+    # Half the name is a patronymic the query did not carry, and half is what
+    # this scores. That is the whole of TokenSet's job.
+    module TokenSort
+      extend T::Sig
+
+      module_function
+
+      # The similarity of two already-folded names -- see Similarity for what
+      # `threshold:` does and what it promises, and for why either side may be
+      # a string or the tokens it splits into.
+      sig { params(left: Value, right: Value, threshold: Numeric).returns(Float).checked(:tests) }
+      def call(left, right, threshold: 0.0)
+        cutoff = Similarity.threshold!(threshold)
+
+        Levenshtein.call(sorted(left), sorted(right), threshold: cutoff)
+      end
+
+      # The string this actually compares: the name's tokens in alphabetical
+      # order, single-spaced.
+      #
+      # Public because a score nobody can account for is a score nobody can
+      # defend, and `sorted` is the whole of the difference between what a
+      # caller passed and what was compared. An analyst asking why `kim jong
+      # un` scored what it did against `kim yong chol` gets the answer by
+      # printing this.
+      sig { params(value: Value).returns(String).checked(:tests) }
+      def sorted(value) = Similarity.tokens(value).sort.join(" ")
+
+      # Levenshtein's ceiling, on the lengths of the two folded names, because
+      # sorting a folded name's tokens does not change its length -- the same
+      # characters and the same single spaces come out in a different order.
+      # That equality is the one thing this delegation rests on, and it is the
+      # reason the contract says *folded* names: a string carrying double
+      # spaces or a leading one is shorter after the sort than the length it
+      # reports, and a ceiling computed from the longer length would come in
+      # under the real score, which is the one direction a bound may never
+      # err in.
+      sig { params(left_length: Integer, right_length: Integer).returns(Float).checked(:tests) }
+      def ceiling(left_length, right_length) = Levenshtein.ceiling(left_length, right_length)
+    end
+  end
+end

data/lib/active_sanction/similarity.rb

@@ -0,0 +1,183 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module ActiveSanction
+  # How close two folded names are, as a number between 0 and 1. Stage 3 of
+  # the matching pipeline.
+  #
+  #   left  = ActiveSanction::Normalizer.call("ABBAS, Abu")
+  #   right = ActiveSanction::Normalizer.call("Abu Abbas")
+  #
+  #   ActiveSanction::Similarity::JaroWinkler.call(left.value, right.value)  # => 0.8053
+  #   ActiveSanction::Similarity::Levenshtein.call(left.value, right.value)  # => 0.3333
+  #   ActiveSanction::Similarity::TokenSort.call(left.tokens, right.tokens)  # => 1.0
+  #   ActiveSanction::Similarity::TokenSet.call(left.tokens, right.tokens)   # => 1.0
+  #
+  # Four algorithms and nothing else. Two compare characters -- JaroWinkler
+  # and Levenshtein, stage 3a -- and two compare tokens -- TokenSort and
+  # TokenSet, stage 3b. The token ratios are rearrangements of the same two
+  # names with Levenshtein run over the result, so there is still exactly one
+  # place in this library that knows how to walk two strings. The scorer (#32)
+  # blends all four, and nothing in the library has any other way to ask how
+  # close two names are.
+  #
+  # ### Why four
+  #
+  # Each of them is generous about something, and their disagreements are the
+  # point:
+  #
+  #                                        JW     Lev    Sort    Set
+  #     abbas abu / abu abbas            0.805  0.333   1.000  1.000
+  #     putin vladimir vladimirovich /
+  #       vladimir putin                 0.679  0.357   0.500  1.000
+  #     gazprom / gazprom neft           0.917  0.583   0.583  1.000
+  #     kim jong un / kim yong chol      0.869  0.615   0.462  0.462
+  #
+  # Row one is an inverted name, which is the single most common query shape
+  # against these lists, which character comparison misses and sorting makes
+  # trivial. Row two adds a patronymic on one side, which sorting cannot
+  # absorb either and the set ratio is built for. Row three is two different
+  # companies, and the 1.000 is what that tolerance costs. Row four is a pair
+  # already written in the same order, where sorting *loses* the information
+  # that they are, and the character algorithms are the honest ones.
+  #
+  # No single column is the score. #32 blends them, and rows three and four
+  # are why it has to.
+  #
+  # Stage 3c is next door, in Phonetics, and not in here: Double Metaphone
+  # answers what a name *sounds* like with a key rather than a number, and a
+  # key cannot be blended with four scores or held to the contract below.
+  #
+  # ### Why pure Ruby
+  #
+  # `fuzzy_string_match` and the other C extensions are faster per call, and
+  # they would put a native build in the path of every application that
+  # installs this gem and in every container image the future service is
+  # deployed from. That is a poor trade at the volume involved: the corpus is
+  # roughly 46,000 name strings, and the inverted index (#31) narrows a query
+  # to a few hundred candidates before any of this runs.
+  #
+  # `rake benchmark:similarity` is what that costs, and the numbers are the
+  # argument. On name-length strings without a JIT, Jaro-Winkler runs about
+  # 20 us per pair and Levenshtein about 60. The token ratios cost what they
+  # are: the sort ratio is one Levenshtein call on a rearranged string and
+  # prices like one, and the set ratio is three and costs about 110. The whole
+  # of a 500-candidate query, all four on every name, is about 25 ms with a
+  # threshold passed and six times that without one; YJIT takes a factor of
+  # two and a half off both and puts a thresholded query near 10 ms. That is
+  # the budget a screening call has, and it is the reason a threshold is worth
+  # passing -- the exits below are most of the difference between those two
+  # numbers.
+  #
+  # ### The contract all four keep
+  #
+  # **Folded names in.** Nothing here normalizes anything -- see Normalizer
+  # for why the fold happens once, at one entry point, for both sides of a
+  # comparison. A caller passes `form.value`, or `form.tokens` to either of
+  # the token ratios; passing raw publisher text instead scores the case and
+  # the punctuation rather than the name.
+  #
+  # **A similarity out, not a distance.** 1.0 is identical, 0.0 is nothing in
+  # common, and the number is a Float that is never rounded here. The scorer
+  # works in 0..100 and does its own rounding; rounding twice is how a
+  # threshold comparison starts disagreeing with the number printed beside it.
+  #
+  # **`threshold:` is an optimization, not a filter.** Passing one lets the
+  # algorithm stop as soon as the score provably cannot reach it, and any
+  # score below it is reported as 0.0 rather than computed exactly:
+  #
+  #   Similarity::Levenshtein.call("gazprom", "gazprom neft")   # => 0.5833...
+  #   Similarity::Levenshtein.call("gazprom", "gazprom neft", threshold: 0.8)
+  #   # => 0.0
+  #
+  # A score at or above the threshold is exactly the score the same call
+  # without a threshold returns -- the early exits are bounds on what a pair
+  # can reach, never approximations of what it did reach. `ceiling` on each
+  # algorithm is that bound, exposed so it can be held to that promise. How
+  # much it is worth differs: Levenshtein's is tight, Jaro-Winkler's is loose
+  # because the prefix bonus can add 0.4 to anything, and TokenSet has none at
+  # all, because a name that is a subset of another scores 1.0 at any length.
+  #
+  # @api private
+  module Similarity
+    extend T::Sig
+    extend T::Helpers
+
+    # Called as `Similarity.threshold!`, which is where `raise` comes from.
+    requires_ancestor { Kernel }
+
+    # A folded name, as either the string or the tokens it splits into. The
+    # token ratios take either on either side, because their two callers hold
+    # different things: a Form already carries `tokens` and should not pay for
+    # a split per comparison, while a spec, a console and the benchmark are
+    # written in strings.
+    Value = T.type_alias { T.any(String, T::Array[String]) }
+
+    module_function
+
+    # A threshold as a Float, or an InvalidArgument.
+    #
+    # The 0..1 range is checked rather than assumed because the surrounding
+    # library speaks in 0..100 -- the scorer's weights, its thresholds and
+    # everything a compliance user reads are percentages -- and a `85` that
+    # arrives here unchecked does not fail. It silently rejects every pair,
+    # which reads as "nothing matched" and is the one failure this domain
+    # cannot afford.
+    sig { params(threshold: Numeric).returns(Float).checked(:tests) }
+    def threshold!(threshold)
+      cutoff = threshold.to_f
+      return cutoff if cutoff.between?(0.0, 1.0)
+
+      raise InvalidArgument,
+            "threshold must be between 0.0 and 1.0, got #{threshold.inspect} -- " \
+            "these are similarities on a 0..1 scale, not percentages"
+    end
+
+    # The form both algorithms work in: an Array of codepoints.
+    #
+    # Codepoints rather than characters because comparing Integers is cheaper
+    # than comparing one-character Strings, and both algorithms compare in
+    # their innermost loop. Codepoints rather than bytes because a byte is not
+    # a character outside ASCII, and half of these names are not ASCII: `ж` is
+    # two bytes, and a byte-wise edit distance would charge two edits for
+    # changing one letter and score Cyrillic pairs against a different scale
+    # than Latin ones.
+    #
+    # A string that is not valid UTF-8 is scrubbed rather than raising, for
+    # the reason Form repairs one: a single stray byte in a government file
+    # must not take a whole index build down with it. Normalizer has already
+    # done this to anything that came through it.
+    sig { params(string: String).returns(T::Array[Integer]).checked(:tests) }
+    def codepoints(string) = string.valid_encoding? ? string.codepoints : string.scrub.codepoints
+
+    # The form both token ratios work in: an Array of tokens.
+    #
+    # `String#split` with no argument, which is all the splitting a folded
+    # value needs: Normalizer's stage 4 turned every punctuation mark into a
+    # space and stage 5 collapsed the runs, so whitespace is the only boundary
+    # left in the string and there are no empty tokens at either end.
+    #
+    # An Array is taken as it stands. That is the path that matters -- the
+    # index (#31) hands the scorer a few hundred candidates, each carrying
+    # several names, and every one of those names has already been split once
+    # by the Form it lives in.
+    #
+    # Scrubbed first when it has to be, for the reason `codepoints` scrubs and
+    # Form repairs: `String#split` raises on a byte sequence that is not valid
+    # UTF-8, and one stray byte in a government file must not take an index
+    # build down with it.
+    sig { params(value: Value).returns(T::Array[String]).checked(:tests) }
+    def tokens(value)
+      return value unless value.is_a?(String)
+
+      (value.valid_encoding? ? value : value.scrub).split
+    end
+  end
+end
+
+require "active_sanction/similarity/jaro_winkler"
+require "active_sanction/similarity/levenshtein"
+require "active_sanction/similarity/token_sort"
+require "active_sanction/similarity/token_set"