active_sanction 1.0.0

data/lib/active_sanction/sources/ofac_consolidated.rb

@@ -0,0 +1,203 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/sources"
+require "active_sanction/sources/ofac"
+
+module ActiveSanction
+  module Sources
+    # OFAC's Consolidated (non-SDN) list: everything the US sanctions that is
+    # not a Specially Designated National.
+    #
+    #   snapshot = ActiveSanction::Sources[:ofac_consolidated].new.sync
+    #
+    # It is small -- 481 entities against the SDN list's 19,321 -- and it is
+    # screened anyway, because being on it still carries real legal weight. A
+    # Chinese semiconductor firm on the CMIC list cannot be invested in; a
+    # Russian bank on the SSI list can be transacted with but not lent to
+    # beyond a tenor the directive sets. Neither is a blocking sanction, and
+    # both are prohibitions a compliance team has to act on.
+    #
+    # ### One file, several lists
+    #
+    # OFAC ships the non-SDN lists as one set of three CSVs in the same shape
+    # as the SDN files -- the reading is Ofac's, and this adapter adds no
+    # parsing of its own -- but the rows in it belong to six different lists:
+    #
+    #   Sectoral Sanctions Identifications (SSI)   the Russia/Ukraine directives
+    #   Non-SDN CMIC                               Chinese military-industrial firms
+    #   Non-SDN Palestinian Legislative Council    NS-PLC
+    #   Non-SDN Menu-Based Sanctions (NS-MBS)      HKAA, CAATSA, EO 14024 directives
+    #   CAPTA                                      foreign financial institutions
+    #   FSE                                        foreign sanctions evaders
+    #
+    # A hit on CMIC and a hit on NS-PLC are different findings with different
+    # consequences, so which one a record is on has to survive the parse.
+    # #lists answers it for any entity this adapter produced:
+    #
+    #   ActiveSanction::Sources::OfacConsolidated.lists(entity)   # => [:cmic]
+    #   ActiveSanction::Sources::OfacConsolidated.names(entity)   # => ["Non-SDN CMIC List"]
+    #
+    # It is a function of `entity.programs`, which is a canonical Entity member
+    # -- so it keeps working on a record that has been stored, serialized and
+    # read back, with no per-source column anywhere downstream. The names are
+    # also appended to `remarks` behind the `[source fields]` marker, which is
+    # where a compliance report reads them from.
+    #
+    # ### Why the programs and not a column
+    #
+    # There is no list column. CONS_PRIM.CSV has the same twelve columns as
+    # SDN.CSV, and the only thing in it that names a list is the program code
+    # -- `CMIC-EO13959`, `NS-PLC`, `HKAA`. OFAC states the membership
+    # explicitly in CONS_ADVANCED.XML, a 4.5 MB re-publication of the same 481
+    # records, and downloading the list twice to read one attribute off the
+    # second copy is a poor trade. So the program code is what LISTS maps.
+    #
+    # Checked against that XML's own attribution, the mapping is exact for 478
+    # of the 481 published records. The three it is not are all the same
+    # ambiguity, and SHARED is where it lives.
+    class OfacConsolidated < Ofac
+      extend T::Sig
+
+      key :ofac_consolidated
+
+      url :prim, "https://sanctionslistservice.ofac.treas.gov/api/download/CONS_PRIM.CSV"
+      url :alt, "https://sanctionslistservice.ofac.treas.gov/api/download/CONS_ALT.CSV"
+      url :add, "https://sanctionslistservice.ofac.treas.gov/api/download/CONS_ADD.CSV"
+
+      # Ofac declares 0.90, calibrated on the SDN file, where RemarksParser
+      # reads about 97% of the segments. This list reads at 80.3%, and it is
+      # not a parse that has gone wrong: the CMIC rows publish a vocabulary the
+      # SDN file has no equivalent of -- `Effective Date (CMIC)`,
+      # `Listing Date (CMIC)`, `Purchase/Sales For Divestment`, `HKAA
+      # Section 5`, an equity ticker -- none of which is a name, a date of
+      # birth, a nationality or a document number, so none of it is anything
+      # this parser has a home for.
+      #
+      # Inheriting the SDN's floor made every first `ActiveSanction.doctor` run
+      # against a deployment with no stored snapshot warn about this list
+      # forever, with no fix available to whoever read the warning. 0.75 sits
+      # the same distance below what the file does as 0.90 does for the SDN,
+      # and it is a floor rather than a target: it exists for the run with
+      # nothing to compare against, and is not consulted once there is a
+      # snapshot or a committed baseline. Found by the canary (#69) on its
+      # first run against the published lists.
+      floor :remarks_coverage, 0.75
+
+      # The sub-lists, spelled the way OFAC's own `/sanctions-lists` endpoint
+      # spells them -- which is what a report has to print beside a hit.
+      #
+      # @api private
+      NAMES = T.let({
+        ssi: "Sectoral Sanctions Identifications List",
+        cmic: "Non-SDN CMIC List",
+        ns_plc: "Non-SDN Palestinian Legislative Council List",
+        ns_mbs: "Non-SDN Menu-Based Sanctions List",
+        capta: "CAPTA List",
+        fse: "FSE List"
+      }.freeze, T::Hash[Symbol, String])
+
+      # Program code to sub-list, for every program that names exactly one.
+      #
+      # A program absent from here is not an error: `SDGT` appears on three
+      # consolidated rows because those people are on the SDN list as well, and
+      # it says nothing about which non-SDN list they are on. FSE-IR and FSE-SY
+      # are carried because the FSE list is one OFAC still publishes and can
+      # refill, though nothing is on it today.
+      #
+      # @api private
+      LISTS = T.let({
+        "UKRAINE-EO13662" => :ssi,
+        "UKRAINE-EO13685" => :ssi,
+        "VENEZUELA-EO13850" => :ssi,
+        "IRAN-CON-ARMS-EO" => :ssi,
+        "CMIC-EO13959" => :cmic,
+        "NS-PLC" => :ns_plc,
+        "HKAA" => :ns_mbs,
+        "CAATSA - RUSSIA" => :ns_mbs,
+        "BURMA-EO14014" => :ns_mbs,
+        "ILLICIT-DRUGS-EO14059" => :ns_mbs,
+        "561-Related" => :capta,
+        "CAPTA" => :capta,
+        "FSE-IR" => :fse,
+        "FSE-SY" => :fse
+      }.freeze, T::Hash[String, Symbol])
+
+      # The one program OFAC uses for two lists, and the rule that reads it.
+      #
+      # EO 14024 is the Russia authority behind both the SSI directives and
+      # several menu-based determinations, so `RUSSIA-EO14024` alone means
+      # NS-MBS -- the Central Bank of Russia, the Ministry of Finance -- while
+      # `RUSSIA-EO14024` beside a program that is unambiguously SSI means the
+      # entity is on SSI under both authorities. That reads 92 of the 95 rows
+      # carrying it the way OFAC's own XML does.
+      #
+      # The three it does not are Gazprom, Transneft and Rosselkhozbank, which
+      # are on SSI *and* NS-MBS and come out marked SSI only. It is the safer
+      # direction of the two errors -- the record is still returned, still
+      # matched, still flagged as a non-SDN sanctions hit, and the program code
+      # OFAC published is on the entity verbatim for anyone who needs to look
+      # closer -- but it is an error, and nothing in the CSVs distinguishes
+      # those three from the 89 rows carrying the identical program pair.
+      #
+      # @api private
+      SHARED = T.let(
+        { "RUSSIA-EO14024" => { with: :ssi, alone: :ns_mbs } }.freeze,
+        T::Hash[String, T::Hash[Symbol, Symbol]]
+      )
+
+      # Which sub-lists a record is on, as an Array of the keys NAMES uses.
+      # Takes an Entity, or the programs themselves.
+      #
+      #   OfacConsolidated.lists(entity)                             # => [:ssi]
+      #   OfacConsolidated.lists(%w[UKRAINE-EO13662 RUSSIA-EO14024])  # => [:ssi]
+      #   OfacConsolidated.lists(%w[RUSSIA-EO14024])                  # => [:ns_mbs]
+      #
+      # Empty for a record whose programs name no list this adapter knows,
+      # which is what #parse warns about.
+      sig { params(programs: T.untyped).returns(T::Array[Symbol]) }
+      def self.lists(programs)
+        codes = programs.respond_to?(:programs) ? programs.programs : Array(programs)
+        certain = codes.filter_map { |code| LISTS[code] }.uniq
+        shared = codes.filter_map { |code| SHARED[code] }
+                      .map { |rule| certain.include?(rule[:with]) ? rule[:with] : rule[:alone] }
+        (certain + shared).uniq
+      end
+
+      # The same answer as OFAC spells it, which is what goes in a report.
+      sig { params(programs: T.untyped).returns(T::Array[String]) }
+      def self.names(programs) = lists(programs).map { |list| NAMES.fetch(list) }
+
+      private
+
+      sig { override.returns(T.untyped) }
+      def record_class = Record
+
+      # A row whose programs name no sub-list is the signal that OFAC has
+      # added an authority: the entity is still returned, with its programs
+      # intact, but nothing downstream can say which list it puts it on until
+      # LISTS learns the code. Every other adapter's drift shows up as a
+      # parse warning, and so does this.
+      sig { override.params(record: T.untyped).void }
+      def note(record)
+        super
+        note_unattributed(record) if record.lists.empty?
+      end
+
+      sig { params(record: T.untyped).void }
+      def note_unattributed(record)
+        @unmapped << Parsers::Warning.new(
+          line: record.row.line,
+          message: "row #{record.row[:ent_num].inspect} carries no program naming a consolidated sub-list " \
+                   "(#{record.programs.join(", ")}); it is on none this adapter knows"
+        )
+      end
+    end
+  end
+end
+
+require "active_sanction/sources/ofac_consolidated/record"
+
+ActiveSanction::Sources.register(ActiveSanction::Sources::OfacConsolidated)

data/lib/active_sanction/sources/ofac_sdn.rb

@@ -0,0 +1,43 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/sources"
+require "active_sanction/sources/ofac"
+
+module ActiveSanction
+  module Sources
+    # The US Specially Designated Nationals list: the largest and most
+    # frequently screened sanctions list there is, and the one whose format
+    # dictates most of what the parsing toolkits have to handle.
+    #
+    #   snapshot = ActiveSanction::Sources[:ofac_sdn].new.sync
+    #
+    # ### Three files, one list
+    #
+    # OFAC publishes the SDN list as three headerless CSVs joined on `ent_num`:
+    #
+    #   SDN.CSV   19,321 rows   primary names, type, programs, remarks
+    #   ALT.CSV   20,147 rows   aliases -- more of them than there are entities
+    #   ADD.CSV   25,078 rows   addresses
+    #
+    # Each is fetched and cached independently by Base, because they change
+    # independently; the join happens in Ofac#parse, which the consolidated
+    # list uses too -- the two files are the same twelve columns and are read
+    # by the same code.
+    #
+    # What is left here is the declaration: which list this is, and where its
+    # three files live. Everything else, including the free-text remarks
+    # parsing that gives the list its secondary identifiers, is in Ofac.
+    class OfacSdn < Ofac
+      key :ofac_sdn
+
+      url :sdn, "https://sanctionslistservice.ofac.treas.gov/api/download/SDN.CSV"
+      url :alt, "https://sanctionslistservice.ofac.treas.gov/api/download/ALT.CSV"
+      url :add, "https://sanctionslistservice.ofac.treas.gov/api/download/ADD.CSV"
+    end
+  end
+end
+
+ActiveSanction::Sources.register(ActiveSanction::Sources::OfacSdn)

data/lib/active_sanction/sources/remarks.rb

@@ -0,0 +1,84 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module ActiveSanction
+  module Sources
+    # The publisher's own free text, plus the fields the canonical model has no
+    # home for, kept together in one string.
+    #
+    #   Remarks.build(row[:remarks], [["Vessel flag", "Panama"], ["Tonnage", "8000"]])
+    #   # => "Registered in Panama [source fields] Vessel flag: Panama; Tonnage: 8000"
+    #
+    #   Remarks.published(entity.remarks)   # => "Registered in Panama"
+    #
+    # ### Why an adapter appends rather than drops
+    #
+    # Every list publishes something the canonical Entity has nowhere to put --
+    # OFAC a vessel's flag and owner, the UN a place of birth and a gender.
+    # Dropping it keeps the schema tidy at the cost of real screening signal,
+    # which is the wrong trade; putting it in remarks keeps it, at the cost of
+    # mixing it with the publisher's own prose.
+    #
+    # The marker is what pays that cost back. Anything reading the remark for
+    # what the publisher actually wrote -- OFAC's remarks parser (#19) is the
+    # first, and it must never see a vessel flag and read it as a nationality
+    # -- takes the text before the marker, which is what .published does.
+    #
+    # ### One marker, not one per source
+    #
+    # Both launch adapters invented this independently and gave it different
+    # spellings, which would have meant every consumer of a remark learning
+    # which source it came from before it could strip anything. There is one
+    # marker now, and one format, and adding a third source does not add a
+    # third convention.
+    module Remarks
+      extend T::Sig
+
+      # @api private
+      MARKER = T.let("[source fields]", String)
+
+      # @api private
+      SEPARATOR = T.let("; ", String)
+
+      # `fields` is a list of label/value pairs. A value may be an Array -- the
+      # UN files three designations under one element -- and a label whose
+      # value is missing or blank is left out entirely rather than printed
+      # against an empty string.
+      sig { params(published: T.untyped, fields: T.untyped).returns(T.nilable(String)) }
+      def self.build(published, fields = [])
+        appended = Array(fields).filter_map { |label, value| entry(label, value) }
+        text = string_or_nil(published)
+        return text if appended.empty?
+
+        [text, MARKER, appended.join(SEPARATOR)].compact.join(" ")
+      end
+
+      # The publisher's own text, with anything an adapter appended stripped
+      # back off. nil when the publisher wrote nothing and every word in the
+      # remark was put there by us, which is the honest answer: a consumer
+      # asking what the publisher said should not be handed a vessel's tonnage.
+      sig { params(remarks: T.untyped).returns(T.nilable(String)) }
+      def self.published(remarks)
+        string_or_nil(remarks.to_s.split(MARKER, 2).first)
+      end
+
+      sig { params(label: T.untyped, value: T.untyped).returns(T.nilable(String)) }
+      def self.entry(label, value)
+        values = Array(value).filter_map { |item| string_or_nil(item) }
+        "#{label}: #{values.join(", ")}" if values.any?
+      end
+      private_class_method :entry
+
+      sig { params(value: T.untyped).returns(T.nilable(String)) }
+      def self.string_or_nil(value)
+        return nil if value.nil?
+
+        string = value.to_s.strip
+        string.empty? ? nil : string
+      end
+      private_class_method :string_or_nil
+    end
+  end
+end

data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb

@@ -0,0 +1,104 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/partial_date"
+
+module ActiveSanction
+  module Sources
+    class UkSanctionsList < Base
+      # The UK Sanctions List's own date convention, which no general date
+      # parser reads and which is the single most valuable thing this adapter
+      # gets right.
+      #
+      #   PublishedDate.call("04/08/2026")   # => 2026-08-04, day precision
+      #   PublishedDate.call("dd/mm/1945")   # => 1945, year precision
+      #   PublishedDate.call("dd/06/1945")   # => 1945-06, month precision
+      #   PublishedDate.call("00/00/1975")   # => 1975, year precision
+      #   PublishedDate.call("1945")         # => 1945, year precision
+      #   PublishedDate.call("15/08/19yy")   # => nil
+      #
+      # ### The placeholders are the point
+      #
+      # The FCDO writes every date `DD/MM/YYYY`, and where it does not know a
+      # component it writes the component's own letters in its place: 800 of
+      # the 3,788 published birth dates read `dd/mm/1962`, and 23 read
+      # `dd/07/1978`. One record uses `00` for the same purpose --
+      # `00/00/1975`. That is a publisher stating precision, and it maps onto
+      # PartialDate exactly.
+      #
+      # Read with an ordinary parser those become nil, and 824 of 3,788 birth
+      # dates -- 22% of everything the list says about when a person was born
+      # -- silently disappear. Read *credulously*, as a zeroth day of a zeroth
+      # month, they become invalid dates or, worse, dates the scorer would
+      # compare as though the FCDO had been precise.
+      #
+      # ### `19yy` gets no date, on purpose
+      #
+      # One record publishes `15/08/19yy`: a day and a month, and a century
+      # where the year should be. PartialDate has no shape for a date with no
+      # year, and inventing one -- a 1900-1999 span, which would throw the day
+      # and month away, or a year of 1900, which would be a fact nobody
+      # published -- is exactly the false precision PartialDate exists to
+      # prevent. It reads as nil and the adapter keeps the published string in
+      # `remarks` instead.
+      #
+      # @api private
+      module PublishedDate
+        extend T::Sig
+
+        SEPARATOR = T.let("/", String)
+
+        # A component the FCDO filled in. Anything else -- `dd`, `mm`, `yy`,
+        # `00`, an empty field -- is the publisher saying it does not know,
+        # which is not the same as a zero and must never be read as one.
+        KNOWN = T.let(/\A0*[1-9]\d*\z/, Regexp)
+
+        YEAR_ONLY = T.let(/\A\d{4}\z/, Regexp)
+
+        module_function
+
+        # The PartialDate a UKSL date string states, at the precision it
+        # states it, or nil for one that states no year at all.
+        #
+        # Returns nil rather than raising, for the reason PartialDate::Parser
+        # does: a birth date is one field on a record, and a string this does
+        # not read should not abort the import of the entity around it. The
+        # adapter records what it could not read.
+        sig { params(text: T.untyped).returns(T.nilable(PartialDate)) }
+        def call(text)
+          string = text.to_s.strip
+          return nil if string.empty?
+          return build(year: string) if string.match?(YEAR_ONLY)
+
+          day, month, year = string.split(SEPARATOR, 3).map { |part| component(part) }
+          return nil if year.nil?
+
+          # A day the FCDO gave under a month it did not is a day of an unknown
+          # month, which PartialDate rightly refuses. The month is the part
+          # that is missing, so the day goes with it.
+          build(year: year, month: month, day: (day if month))
+        end
+
+        # nil for a placeholder, so that "the FCDO did not say" and "the FCDO
+        # said zero" cannot be confused downstream.
+        sig { params(part: T.untyped).returns(T.nilable(Integer)) }
+        def component(part)
+          string = part.to_s.strip
+          string.match?(KNOWN) ? string.to_i : nil
+        end
+
+        # A shape these regexes accept can still be an impossible date -- the
+        # list publishes no 31 February today, but a screening tool should not
+        # be the thing that breaks on the day it does.
+        sig { params(attributes: T.untyped).returns(T.nilable(PartialDate)) }
+        def build(**attributes)
+          PartialDate.new(**attributes)
+        rescue ArgumentError
+          nil
+        end
+      end
+    end
+  end
+end