active_sanction 1.0.0

data/lib/active_sanction/sources/ofac/record.rb

@@ -0,0 +1,251 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/entity"
+require "active_sanction/name"
+require "active_sanction/address"
+require "active_sanction/identifier"
+require "active_sanction/sources/remarks"
+require "active_sanction/sources/ofac/remarks_parser"
+
+module ActiveSanction
+  module Sources
+    class Ofac < Base
+      # One joined OFAC record -- a row of the primary file plus the ALT and
+      # ADD rows that share its `ent_num` -- turned into an Entity.
+      #
+      # Separate from the adapter because they are two jobs: the adapter says
+      # what the list is and where it lives, and this says what OFAC's columns
+      # mean. The mapping is where all the judgment sits, so it is worth being
+      # able to read it on its own.
+      #
+      # Shared by both OFAC adapters, because SDN.CSV and CONS_PRIM.CSV are
+      # the same twelve columns with the same conventions -- the same is true
+      # of ALT and ADD -- and the only thing that differs between them is
+      # which list a row is on. `source` is passed in rather than hard-coded
+      # for that reason, and #remark_fields is the hook a subclass overrides
+      # to record anything its own list publishes on top.
+      class Record
+        extend T::Sig
+
+        # OFAC's `SDN_Type` as published, and what each maps to. Blank is the
+        # one that matters: 9,923 of 19,321 rows leave it empty and every one
+        # of them is an organization. Defaulting blank to "unknown" would
+        # mis-type the largest group in the list.
+        TYPES = T.let(
+          { "individual" => :individual, "vessel" => :vessel, "aircraft" => :aircraft }.freeze,
+          T::Hash[String, Symbol]
+        )
+
+        # The SDN_Type values this record class knows how to map, as a method
+        # rather than as the constant itself: a subclass reading a list that
+        # publishes a fourth one declares its own TYPES, and the caller asking
+        # -- Ofac#column_assertions -- has only the class in hand.
+        sig { returns(T::Array[String]) }
+        def self.published_types = TYPES.keys
+
+        # ALT.CSV's `alt_type`, which OFAC publishes as exactly these three.
+        ALIAS_KINDS = T.let({ "aka" => :aka, "fka" => :fka, "nka" => :nka }.freeze, T::Hash[String, Symbol])
+
+        # Multiple programs arrive in one field separated by `] [`:
+        # `"IRAQ2] [IRGC] [SDGT"` is three sanctions programs, not one.
+        PROGRAM_SEPARATOR = T.let(/\]\s*\[/, Regexp)
+
+        # Columns OFAC publishes outside its Remarks field that the canonical
+        # model has no home for. Appended to remarks rather than dropped: a
+        # vessel's flag and owner are real screening signal, and losing them to
+        # keep a schema tidy is the wrong trade.
+        COLUMNS_IN_REMARKS = T.let({
+          title: "Title", vessel_type: "Vessel type", tonnage: "Tonnage",
+          gross_registered_tonnage: "GRT", vessel_flag: "Vessel flag", vessel_owner: "Vessel owner"
+        }.freeze, T::Hash[Symbol, String])
+
+        # A name matches one already on the record when the letters and digits
+        # agree; OFAC's own punctuation does not have to. 24 of the 4,349
+        # inline aliases repeat an ALT.CSV row, and the rest are names the list
+        # publishes nowhere else.
+        INSIGNIFICANT = T.let(/[^[:alnum:]]+/, Regexp)
+
+        sig { returns(Parsers::DelimitedTable::Row) }
+        attr_reader :row
+
+        # The list this row is on, passed in rather than hard-coded: SDN.CSV
+        # and CONS_PRIM.CSV are read by this same class.
+        sig { returns(Symbol) }
+        attr_reader :source
+
+        sig { returns(T::Array[Parsers::DelimitedTable::Row]) }
+        attr_reader :aliases
+
+        sig { returns(T::Array[Parsers::DelimitedTable::Row]) }
+        attr_reader :addresses
+
+        sig do
+          params(row: Parsers::DelimitedTable::Row, source: Symbol,
+                 aliases: T::Array[Parsers::DelimitedTable::Row],
+                 addresses: T::Array[Parsers::DelimitedTable::Row]).void
+        end
+        def initialize(row:, source:, aliases: [], addresses: [])
+          @row = T.let(row, Parsers::DelimitedTable::Row)
+          @source = T.let(source, Symbol)
+          @aliases = T.let(aliases, T::Array[Parsers::DelimitedTable::Row])
+          @addresses = T.let(addresses, T::Array[Parsers::DelimitedTable::Row])
+          @parsed_remarks = T.let(nil, T.nilable(RemarksParser))
+        end
+
+        # The entity, or nil for a row with no name -- which cannot be screened
+        # against and is never what OFAC meant to publish.
+        sig { returns(T.nilable(Entity)) }
+        def entity
+          return nil if row.null?(:sdn_name)
+
+          # `new(**hash)` past required keyword parameters is one of the few
+          # things Sorbet cannot check statically. #from_remarks below is the
+          # hash, and it carries the two members it names and nothing else.
+          T.unsafe(Entity).new(source: source, source_ref: row[:ent_num], type: type,
+                               names: names, addresses: places, identifiers: identifiers,
+                               programs: programs, remarks: remarks, **from_remarks)
+        end
+
+        # The members no OFAC column feeds. De-duplicated because one entity's
+        # remark can report the same nationality twice -- "nationality Iran;
+        # alt. nationality Iran" -- and a record that claims one thing twice
+        # is not a record that claims it more strongly.
+        sig { returns(T::Hash[Symbol, T.untyped]) }
+        def from_remarks
+          { dates_of_birth: parsed_remarks.dates_of_birth.uniq,
+            nationalities: parsed_remarks.nationalities.uniq }
+        end
+
+        # OFAC's free text, read for the fields it has no columns for. Exposed
+        # rather than kept private because the adapter folds every record's
+        # into one coverage figure, which is how drift in a heuristic parser
+        # gets noticed at all.
+        sig { returns(RemarksParser) }
+        def parsed_remarks
+          @parsed_remarks ||= RemarksParser.new(row[:remarks])
+        end
+
+        sig { returns(Symbol) }
+        def type
+          published = row[:sdn_type]
+          return :organization if published.nil?
+
+          TYPES.fetch(published.downcase, :organization)
+        end
+
+        # True when OFAC published a type this adapter does not know. Worth
+        # surfacing rather than silently absorbing: a new value here means the
+        # list grew a category, and everything in it is currently being called
+        # an organization.
+        sig { returns(T::Boolean) }
+        def unknown_type?
+          published = row[:sdn_type]
+          !published.nil? && !TYPES.key?(published.downcase)
+        end
+
+        # The primary name first, then every alias in the order OFAC filed it.
+        # `alt_num` ordering is the closest thing these aliases have to a
+        # priority, so it is preserved rather than sorted away. Last come the
+        # aliases that appear only inside the remark -- 4,325 names that are in
+        # no other column of any of the three files.
+        sig { returns(T::Array[Name]) }
+        def names
+          published = [Name.new(value: row[:sdn_name], kind: :primary)] + alias_names
+          published + new_names(published, parsed_remarks.aliases)
+        end
+
+        sig { returns(T::Array[Address]) }
+        def places
+          addresses.filter_map { |address| place(address) }
+        end
+
+        # The call sign, then every document number the remark named. Compared
+        # through Identifier's own equality, which already treats `AB-123 456`
+        # and `ab123456` as one document, so a number OFAC wrote twice does not
+        # become two.
+        sig { returns(T::Array[Identifier]) }
+        def identifiers
+          (call_sign + parsed_remarks.identifiers).uniq
+        end
+
+        # A vessel's call sign is a registered, near-unique string, which makes
+        # it far more like a document number than like a name -- and an
+        # Identifier is matchable where a line of remarks is not. Filed as
+        # :other because it is not any of the document kinds the model names.
+        sig { returns(T::Array[Identifier]) }
+        def call_sign
+          return [] if row.null?(:call_sign)
+
+          [Identifier.new(kind: :other, value: row[:call_sign], note: "call sign")]
+        rescue ArgumentError
+          []
+        end
+
+        sig { returns(T::Array[String]) }
+        def programs
+          return [] if row.null?(:program)
+
+          T.must(row[:program]).split(PROGRAM_SEPARATOR).map { |program| program.strip.delete("[]") }.reject(&:empty?)
+        end
+
+        # OFAC's remark verbatim, then the columns that have nowhere else to
+        # go, behind the marker that makes them trivial to strip again.
+        sig { returns(T.nilable(String)) }
+        def remarks
+          Remarks.build(row[:remarks], remark_fields)
+        end
+
+        # The label/value pairs appended behind the marker. A subclass reading
+        # a list that publishes something more -- which sub-list of the
+        # consolidated file a row is on -- prepends to this rather than
+        # rewriting #remarks.
+        sig { returns(T::Array[T.untyped]) }
+        def remark_fields
+          COLUMNS_IN_REMARKS.map { |column, label| [label, row[column]] }
+        end
+
+        private
+
+        sig { params(published: T::Array[Name], candidates: T::Array[Name]).returns(T::Array[Name]) }
+        def new_names(published, candidates)
+          seen = published.map { |name| key(name) }
+          candidates.reject { |name| seen.include?(key(name)) }
+        end
+
+        sig { params(name: Name).returns(String) }
+        def key(name) = name.value.upcase.gsub(INSIGNIFICANT, "")
+
+        sig { returns(T::Array[Name]) }
+        def alias_names
+          aliases.filter_map do |alt|
+            next nil if alt.null?(:alt_name)
+
+            Name.new(value: alt[:alt_name], kind: ALIAS_KINDS.fetch(alt.fetch(:alt_type).to_s.downcase, :aka))
+          end
+        end
+
+        # ADD.CSV combines city, state, province and postal code into one
+        # column, so "London EC3N 1DY" arrives undivided. It is filed under
+        # `city` whole rather than split on a guess: a rule that turns
+        # "London EC3N 1DY" into a city and a postcode also turns "Dubai" into
+        # a city and turns half of Latin America into nonsense.
+        # 3,211 of ADD.CSV's 25,078 rows carry an `ent_num` and an `add_num`
+        # and then nothing at all -- no street, no city, no country, no
+        # remark. They are dropped rather than kept as empty addresses, which
+        # is why a full sync yields ~21.9k addresses from ~25.1k rows. An
+        # Address that locates nothing cannot be screened on and would only
+        # inflate the count.
+        sig { params(address: Parsers::DelimitedTable::Row).returns(T.nilable(Address)) }
+        def place(address)
+          Address.new(street: address[:address], city: address[:city_state_province_postal_code],
+                      country: address[:country], note: address[:add_remarks])
+        rescue ArgumentError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb

@@ -0,0 +1,120 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module ActiveSanction
+  module Sources
+    class Ofac < Base
+      class RemarksParser
+        # How much of OFAC's free text the parser understood, accumulated over
+        # a whole sync.
+        #
+        #   coverage = source.remarks_coverage
+        #   coverage.percentage   # => 93.4
+        #   coverage.to_s         # => "recognized 82943 of 88827 segments (93.4%), 41216 extracted"
+        #   coverage.top(3)
+        #   # => [["Member of the", 615], ["ICTY indictee.", 45], ["all offices worldwide.", 43]]
+        #
+        # ### Why a number and not a pass/fail
+        #
+        # RemarksParser is heuristic against text a government writes for
+        # people, and OFAC changes how it writes things without telling
+        # anybody. The failure that matters is not a crash -- nothing here
+        # raises, and an unread segment is still in the remark -- it is the
+        # quiet one, where a re-spelled label stops producing passports for six
+        # months and no one notices because the import still succeeds.
+        #
+        # A percentage recorded on every sync makes that visible: it moves when
+        # the file's vocabulary moves. #top is what turns the movement into
+        # work, since the shapes that suddenly appear in the unrecognized
+        # histogram are the new spellings, ranked by how many records they cost.
+        #
+        # `recognized` counts a segment matched as prose as well as one that
+        # produced a value, because "we know this citation carries no fields"
+        # and "we have never seen this" are different states and only the
+        # second is actionable. `extracted` counts the second kind alone.
+        class Coverage
+          extend T::Sig
+
+          # A shape, not a segment: enough leading words to recognize the
+          # pattern, with digits masked so that 4,000 distinct tax numbers
+          # collapse into one line rather than flooding the histogram.
+          SHAPE_WORDS = T.let(3, Integer)
+
+          sig { returns(Integer) }
+          attr_reader :segments
+
+          sig { returns(Integer) }
+          attr_reader :extracted
+
+          # Extracted plus prose: a citation known to carry no fields is
+          # recognized, and only what is neither is actionable.
+          sig { returns(Integer) }
+          attr_reader :recognized
+
+          # Unrecognized shapes to how many segments each cost.
+          sig { returns(T::Hash[String, Integer]) }
+          attr_reader :unknown
+
+          sig { void }
+          def initialize
+            @segments = T.let(0, Integer)
+            @extracted = T.let(0, Integer)
+            @recognized = T.let(0, Integer)
+            @unknown = T.let(Hash.new(0), T::Hash[String, Integer])
+          end
+
+          # Folds one parsed remark in. Returns self, so a caller can chain it
+          # into a fold over the file.
+          sig { params(parsed: RemarksParser).returns(T.self_type) }
+          def record(parsed)
+            @segments += parsed.segments.size
+            @extracted += parsed.extracted.size
+            @recognized += parsed.extracted.size + parsed.prose.size
+            parsed.unrecognized.each do |segment|
+              shape = shape(segment)
+              @unknown[shape] = @unknown.fetch(shape, 0) + 1
+            end
+            self
+          end
+
+          # 0.0 for an empty run rather than a division by zero: a sync that
+          # read no remarks has no coverage to report, and neither perfect nor
+          # nil is an honest way to say so.
+          sig { returns(Float) }
+          def ratio = segments.zero? ? 0.0 : recognized.fdiv(segments)
+
+          sig { returns(Float) }
+          def percentage = (ratio * 100).round(1).to_f
+
+          # The unrecognized shapes that cost the most segments, which is where
+          # a new label shows up first.
+          sig { params(count: Integer).returns(T::Array[T.untyped]) }
+          def top(count = 10) = unknown.sort_by { |shape, tally| [-tally, shape] }.first(count)
+
+          sig { returns(T::Hash[Symbol, T.untyped]) }
+          def to_h
+            { segments: segments, extracted: extracted, recognized: recognized,
+              ratio: ratio, unknown: unknown.size }
+          end
+
+          sig { returns(String) }
+          def to_s
+            "recognized #{recognized} of #{segments} segments (#{percentage}%), #{extracted} extracted"
+          end
+
+          sig { returns(String) }
+          def inspect = "#<#{self.class} #{self}>"
+
+          private
+
+          sig { params(segment: String).returns(String) }
+          def shape(segment)
+            -segment.split(/\s+/).first(SHAPE_WORDS).join(" ").gsub(/\d/, "#")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb

@@ -0,0 +1,121 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module ActiveSanction
+  module Sources
+    class Ofac < Base
+      class RemarksParser
+        # What OFAC calls things: every label the parser knows how to find at
+        # the front of a remark segment, and the patterns compiled from them.
+        #
+        # Kept apart from the parsing because they change for different
+        # reasons. How a segment is read is settled; which words OFAC opens one
+        # with is not, and is what a maintainer edits when Coverage#top shows a
+        # spelling nobody has seen before. The tail of that list is long and
+        # thin -- a dozen country-specific labels appear fewer than 30 times
+        # each -- so this table is deliberately the high-volume ones rather
+        # than an attempt at all of them.
+        module Vocabulary
+          extend T::Sig
+
+          # The labels that map onto a canonical field. `citizen` and
+          # `nationality` are one field written two ways, and OFAC uses both.
+          FIELDS = T.let({
+            "DOB" => :date_of_birth,
+            "POB" => :place_of_birth,
+            "nationality" => :nationality,
+            "citizen" => :nationality,
+            "Gender" => :gender,
+            "a.k.a." => :aka,
+            "f.k.a." => :fka,
+            "n.k.a." => :nka
+          }.freeze, T::Hash[String, Symbol])
+
+          # Every document label OFAC writes, grouped by the Identifier kind it
+          # means. One kind covers many labels because governments name the
+          # same document differently -- an R.F.C. is Mexico's tax number and a
+          # NIT is Colombia's -- and a matcher comparing numbers should not
+          # have to know whose vocabulary it is holding. The published label is
+          # not lost: it becomes the identifier's `note`, because :tax_id is
+          # our word for it and "R.F.C." is theirs, and a user justifying a hit
+          # needs to see theirs.
+          DOCUMENTS = T.let({
+            passport: ["Passport", "Diplomatic Passport"],
+            national_id: [
+              "National ID No.", "Identification Number", "Cedula No.", "C.U.R.P.", "C.U.I.P.",
+              "D.N.I.", "C.I.N.", "Driver's License No.", "Birth Certificate Number", "Residency Number"
+            ],
+            tax_id: [
+              "Tax ID No.", "R.F.C.", "V.A.T. Number", "NIT", "RUC", "RIF",
+              "Unified Social Credit Code (USCC)"
+            ],
+            registration_number: [
+              "Registration Number", "Business Registration Number", "Business Registration Document",
+              "Company Number", "Commercial Registry Number", "Registration ID", "Government Gazette Number",
+              "Folio Mercantil No.", "Matricula Mercantil No", "Legal Entity Number", "Chamber of Commerce Number",
+              "Trade License No.", "License", "C.R. No.", "Economic Register Number (CBLS)",
+              "Central Registration System Number", "D-U-N-S Number", "Enterprise Number", "Business Number",
+              "Entity Code", "Public Registration Number"
+            ],
+            # Registered strings that are not documents: a vessel's IMO number,
+            # a bank's SWIFT code, an aircraft's serial. :other is a real
+            # answer -- a number we cannot classify still matches on its number.
+            other: [
+              "MMSI", "Vessel Registration Identification IMO", "Vessel Registration Identification",
+              "Identification Number IMO", "Company Number IMO", "SWIFT/BIC",
+              "Aircraft Manufacturer's Serial Number (MSN)", "Aircraft Tail Number",
+              # Published with its own gloss attached, every time, all 52 of them.
+              "Aircraft Construction Number (also called L/N or S/N or F/N)", "Aircraft Construction Number"
+            ]
+          }.freeze, T::Hash[Symbol, T::Array[String]])
+
+          # Shapes that are recognized and carry nothing to extract: statutory
+          # citations, relationship notes, contact details, the date a company
+          # rather than a person was established. Naming them is what makes the
+          # coverage statistic mean something -- without this list the number
+          # would sit near half forever and real drift would hide in the noise.
+          PROSE = T.let([
+            "Secondary sanctions risk", "Additional Sanctions Information", "Linked To",
+            "Transactions Prohibited For Persons Owned or Controlled By U.S. Financial Institutions",
+            "Organization Established Date", "Organization Type", "Target Type", "Executive Order",
+            "CAATSA Section", "For more information", "Website", "Email Address", "Phone Number",
+            "Telephone", "Fax", "Vessel Year of Build", "Former Vessel Flag", "Aircraft Manufacture Date",
+            "Aircraft Model", "Aircraft Operator"
+          ].freeze, T::Array[String])
+
+          # A label is followed by whitespace, a `#` (`NIT # 123`) or a colon,
+          # and may carry one full stop this table does not spell ("Matricula
+          # Mercantil No." against "Matricula Mercantil No"). The lookahead is
+          # what stops "Passport" from matching the first word of a longer
+          # label, and the longest-first ordering is what stops "Business
+          # Registration Number" from being read as a stray word and then
+          # "Registration Number".
+          #
+          # The alternation is escaped and joined by hand rather than built
+          # with Regexp.union, which embeds its own `(?-mix:...)` and would
+          # switch case-insensitivity back off for the labels inside it.
+          sig { params(labels: T::Array[String], tail: String).returns(Regexp) }
+          def self.pattern(labels, tail)
+            alternation = labels.sort_by { |label| -label.length }.map { |label| Regexp.escape(label) }.join("|")
+            /\A(?<label>#{alternation})\.?(?=[\s#:]|\z)[\s#:]*#{tail}\z/i
+          end
+
+          DOCUMENT_KINDS = T.let(
+            DOCUMENTS.each_with_object({}) do |(kind, labels), lookup|
+              labels.each { |label| lookup[label.downcase] = kind }
+            end.freeze,
+            T::Hash[String, Symbol]
+          )
+
+          FIELD_KINDS = T.let(FIELDS.transform_keys(&:downcase).freeze, T::Hash[String, Symbol])
+
+          FIELD_PATTERN = T.let(pattern(FIELDS.keys, "(?<value>.*)"), Regexp)
+          DOCUMENT_PATTERN = T.let(pattern(DOCUMENTS.values.flatten, "(?<rest>.*)"), Regexp)
+          PROSE_PATTERN = T.let(pattern(PROSE, ".*"), Regexp)
+        end
+      end
+    end
+  end
+end