active_sanction 1.0.0

data/lib/active_sanction/parsers/delimited_table/reader.rb

@@ -0,0 +1,194 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "csv"
+require "stringio"
+
+module ActiveSanction
+  module Parsers
+    class DelimitedTable
+      # One pass over one payload. Enumerable, and lazy: rows are yielded as
+      # they are read rather than collected, so a 5.6 MB list costs one row of
+      # memory plus whatever the caller keeps.
+      #
+      #   reader = table.read(bytes)
+      #   reader.each { |row| ... }
+      #   reader.warnings          # => rows that could not be read
+      #
+      # Re-enumerating rewinds and starts over, which also resets #warnings --
+      # so `reader.count` followed by `reader.warnings` reports the warnings
+      # from the counting pass, not from two passes appended together.
+      #
+      # @api private
+      class Reader
+        extend T::Sig
+        extend T::Generic
+        include Enumerable
+
+        Elem = type_member { { fixed: Row } }
+
+        # A parser that raises on every row is not isolating failures, it is
+        # failing -- most often because the payload is not the format the table
+        # was told to expect (an HTML error page saved as .csv is the classic).
+        # Collecting 19,321 warnings to say so helps nobody.
+        MAX_CONSECUTIVE_FAILURES = T.let(100, Integer)
+
+        EOF = T.let(Object.new.freeze, Object)
+        private_constant :EOF
+
+        sig { returns(DelimitedTable) }
+        attr_reader :table
+
+        # The rows this pass could not read. Reset by each pass -- see the
+        # class comment.
+        sig { returns(T::Array[Warning]) }
+        attr_reader :warnings
+
+        sig { params(table: DelimitedTable, payload: T.untyped).void }
+        def initialize(table:, payload:)
+          @table = T.let(table, DelimitedTable)
+          @payload = T.let(payload, T.untyped)
+          @warnings = T.let([], T::Array[Warning])
+        end
+
+        sig { override.params(block: T.nilable(T.proc.params(row: Row).void)).returns(T.untyped) }
+        def each(&block)
+          return enum_for(:each) unless block
+
+          csv = start
+          columns = table.columns || header!(csv)
+          consecutive = 0
+          loop do
+            values = shift(csv)
+            break if values.equal?(EOF)
+
+            consecutive = advance(consecutive, values)
+            block.call(build(columns, values, csv.lineno)) unless values.nil?
+          end
+          self
+        end
+
+        # Every row, in memory. The convenience the small files get to use;
+        # anything list-sized should stay with #each.
+        sig { returns(T::Array[Row]) }
+        def to_a = each.to_a
+
+        private
+
+        # Counts consecutive unreadable rows, and stops the pass once there
+        # have been too many to be explained by anything but the wrong format.
+        sig { params(consecutive: Integer, values: T.untyped).returns(Integer) }
+        def advance(consecutive, values)
+          return 0 unless values.nil?
+
+          count = consecutive + 1
+          give_up!(count) if count >= MAX_CONSECUTIVE_FAILURES
+          count
+        end
+
+        sig { returns(CSV) }
+        def start
+          @warnings = []
+          CSV.new(StringIO.new(payload!), **table.csv_options)
+        end
+
+        # No sanctions list has ever been published empty, so a payload with
+        # nothing in it is a failed download, a moved URL or an outage -- never
+        # a day on which nobody is sanctioned. Yielding no rows would let a
+        # sync succeed at screening against nothing, which is the most
+        # expensive way this library can fail, so it raises instead. The XML
+        # reader refuses the same payload for the same reason.
+        sig { returns(String) }
+        def payload!
+          string = decoded
+          raise ParseError, "expected #{table.col_sep_name} rows, got an empty payload" if string.strip.empty?
+
+          string
+        end
+
+        # Decoding happens once per pass rather than per row, and never raises.
+        # Format#decode says why, and strips the BOM; what is left here is the
+        # marker only a delimited file carries.
+        sig { returns(String) }
+        def decoded
+          string, replaced = table.decode(@payload)
+          record(0, table.invalid_bytes_message) if replaced
+          trim(string)
+        end
+
+        # SUB (0x1A) is CP/M's end-of-file character, and DOS-lineage export
+        # tooling still writes it: all three OFAC files end with `\r\n\x1A`.
+        # Left alone it parses as a final one-column row, so every sync reports
+        # a malformed row it can do nothing about -- and a warning that fires
+        # every single time is a warning nobody reads.
+        sig { params(string: String).returns(String) }
+        def trim(string) = string.sub(/\r?\n?\x1A\s*\z/, "")
+
+        # Column names taken from the file's own first row, lowercased and
+        # snake_cased so that `City/State/Province/ZIP/Postal Code` and
+        # `city_state_province_zip_postal_code` are the same column to an
+        # adapter regardless of how the publisher capitalized it this quarter.
+        # An empty payload has already been refused, so what is left to fail on
+        # here is a first row that could not be read at all -- and a file whose
+        # header is unreadable has no columns to name anything by.
+        sig { params(csv: CSV).returns(T::Array[Symbol]) }
+        def header!(csv)
+          values = shift(csv)
+          if values.nil? || values.equal?(EOF)
+            raise ParseError.new("expected a header row, read nothing usable as one", line: 1)
+          end
+
+          values.map { |value| normalize_header(value) }
+        end
+
+        sig { params(value: T.untyped).returns(Symbol) }
+        def normalize_header(value)
+          value.to_s.strip.downcase.gsub(/[^a-z0-9]+/, "_").delete_prefix("_").delete_suffix("_").to_sym
+        end
+
+        # Returns the row's values, EOF at the end of the payload, or nil for a
+        # row that could not be parsed -- already recorded as a warning.
+        sig { params(csv: CSV).returns(T.untyped) }
+        def shift(csv)
+          row = csv.shift
+          row.nil? ? EOF : row
+        rescue CSV::MalformedCSVError => e
+          record(csv.lineno, "malformed #{table.col_sep_name}: #{e.message}")
+          nil
+        end
+
+        sig { params(columns: T::Array[Symbol], values: T::Array[T.untyped], line: Integer).returns(Row) }
+        def build(columns, values, line)
+          record_arity(columns, values, line) unless values.size == columns.size
+          Row.new(values: table.coerce(columns, values), line: line)
+        end
+
+        # A row of the wrong width is kept, not dropped. Short rows are padded
+        # with nil and long ones keep their extra values under no name, because
+        # a publisher appending a column mid-year should degrade the fields
+        # nobody has mapped yet rather than the whole list.
+        sig { params(columns: T::Array[Symbol], values: T::Array[T.untyped], line: Integer).void }
+        def record_arity(columns, values, line)
+          shape = values.size < columns.size ? "only #{values.size}" : values.size.to_s
+          record(line, "expected #{columns.size} columns, got #{shape}", values.join(table.col_sep))
+        end
+
+        sig { params(line: T.nilable(Integer), message: String, snippet: T.untyped).void }
+        def record(line, message, snippet = nil)
+          @warnings << Warning.new(line: line, message: message, snippet: snippet)
+        end
+
+        sig { params(consecutive: Integer).void }
+        def give_up!(consecutive)
+          raise ParseError.new(
+            "#{consecutive} consecutive rows could not be parsed. This payload is almost certainly not the " \
+            "#{table.col_sep_name} it was read as -- check the URL, and whether the publisher served an " \
+            "error page. First complaint: #{warnings.first}", line: warnings.first&.line
+          )
+        end
+      end
+    end
+  end
+end

data/lib/active_sanction/parsers/delimited_table/row.rb

@@ -0,0 +1,81 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module ActiveSanction
+  module Parsers
+    class DelimitedTable
+      # One parsed record: column name to value, with the publisher's null
+      # sentinel already resolved to nil.
+      #
+      #   row[:sdn_name]   # => "AEROCARIBBEAN AIRLINES"
+      #   row[:title]      # => nil, because OFAC wrote "-0- " there
+      #   row.line         # => 2
+      #
+      # Deliberately not a Hash. A Hash would answer `row[:sdn_nme]` with nil
+      # and let a typo in an adapter look like an empty column all the way into
+      # a snapshot; #[] here raises on a name the table never declared, and
+      # #fetch is available for the genuinely optional case.
+      class Row
+        extend T::Sig
+
+        # Column name to value, with the publisher's null sentinel already
+        # resolved to nil.
+        sig { returns(T::Hash[Symbol, T.nilable(String)]) }
+        attr_reader :values
+
+        sig { returns(Integer) }
+        attr_reader :line
+
+        sig { params(values: T::Hash[Symbol, T.nilable(String)], line: Integer).void }
+        def initialize(values:, line:)
+          @values = T.let(values.freeze, T::Hash[Symbol, T.nilable(String)])
+          @line = T.let(line, Integer)
+          freeze
+        end
+
+        # Raises on an undeclared column, which is almost always a typo in an
+        # adapter rather than a question about the data.
+        sig { params(column: Symbol).returns(T.nilable(String)) }
+        def [](column)
+          values.fetch(column) do
+            raise MissingKey, "no column #{column.inspect} in this table. Declared: #{columns.join(", ")}"
+          end
+        end
+
+        sig { params(column: Symbol, default: T.untyped).returns(T.untyped) }
+        def fetch(column, default = nil) = values.fetch(column, default)
+
+        # True when the publisher left the column blank or wrote its null
+        # sentinel there. Both arrive as nil, because "-0- " and "" mean the
+        # same thing in a file that uses both.
+        sig { params(column: Symbol).returns(T::Boolean) }
+        def null?(column) = self[column].nil?
+
+        sig { returns(T::Array[Symbol]) }
+        def columns = values.keys
+
+        sig { returns(T::Hash[Symbol, T.nilable(String)]) }
+        def to_h = values
+
+        sig { params(other: T.untyped).returns(T::Boolean) }
+        def ==(other)
+          return false unless other.instance_of?(self.class)
+
+          values == other.values && line == other.line
+        end
+        alias eql? ==
+
+        sig { returns(Integer) }
+        def hash = [self.class, values, line].hash
+
+        sig { returns(String) }
+        def inspect
+          filled = values.compact
+          "#<#{self.class} line=#{line} #{filled.map { |k, v| "#{k}=#{v.inspect}" }.join(" ")}>"
+        end
+      end
+    end
+  end
+end

data/lib/active_sanction/parsers/delimited_table.rb

@@ -0,0 +1,151 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "csv"
+require "active_sanction/parsers/format"
+require "active_sanction/parsers/delimited_table/row"
+require "active_sanction/parsers/delimited_table/reader"
+
+module ActiveSanction
+  module Parsers
+    # Reads a record-oriented delimited list -- CSV, TSV, anything stdlib CSV
+    # can be told about -- into rows an adapter can map onto Entities.
+    #
+    # A table is a description of the file, built once and reused for every
+    # sync; a Reader is one pass over one payload.
+    #
+    #   SDN = ActiveSanction::Parsers::DelimitedTable.new(
+    #     columns:  %i[ent_num sdn_name sdn_type program title remarks],
+    #     null:     "-0-",
+    #     encoding: Encoding::WINDOWS_1252
+    #   )
+    #
+    #   SDN.read(bytes).each { |row| row[:sdn_name] }
+    #
+    # ### Headerless files, and why columns are declared
+    #
+    # OFAC ships all three of its files with no header row, so the names have
+    # to come from somewhere. Declaring them in the adapter also pins the
+    # file's shape: if OFAC inserts a column, rows arrive the wrong width and
+    # every one of them says so in #warnings, which is a far better failure
+    # than 19,321 entities quietly built from shifted fields.
+    #
+    # A file that does carry a header is read with `columns: nil`, and the
+    # names come from its first row.
+    #
+    # ### The null sentinel
+    #
+    # OFAC does not leave a field empty; it writes `-0- `, with a trailing
+    # space, and it does this roughly a quarter of a million times:
+    #
+    #     36,"AEROCARIBBEAN AIRLINES",-0- ,"CUBA",-0- ,-0- ,...
+    #
+    # Any declared sentinel is matched after stripping surrounding whitespace,
+    # and a field that is empty or all whitespace is nil as well -- a list that
+    # uses both conventions in one file (they all do) should not make an
+    # adapter check for both.
+    class DelimitedTable
+      extend T::Sig
+      include Format
+
+      # @api private
+      SEPARATOR_NAMES = T.let(
+        { "," => "CSV", "\t" => "TSV", "|" => "pipe-delimited text",
+          ";" => "semicolon-delimited text" }.freeze,
+        T::Hash[String, String]
+      )
+
+      # nil where the file names its own columns -- see #headers?.
+      sig { returns(T.nilable(T::Array[Symbol])) }
+      attr_reader :columns
+
+      sig { returns(String) }
+      attr_reader :col_sep
+
+      sig { returns(String) }
+      attr_reader :quote_char
+
+      sig { override.returns(T::Array[String]) }
+      attr_reader :nulls
+
+      sig { override.returns(Encoding) }
+      attr_reader :encoding
+
+      # `liberal_parsing` is on by default because these files are published,
+      # not validated: an unescaped quote inside a company name is common
+      # enough in OFAC and UK OFSI data that failing the row is the wrong
+      # default. Turn it off for a source where a stray quote should be loud.
+      sig do
+        params(columns: T.untyped, null: T.untyped, col_sep: T.untyped, quote_char: T.untyped,
+               encoding: T.untyped, liberal_parsing: T::Boolean).void
+      end
+      def initialize(columns: nil, null: nil, col_sep: ",", quote_char: '"',
+                     encoding: DEFAULT_ENCODING, liberal_parsing: true)
+        @columns = T.let(columns!(columns), T.nilable(T::Array[Symbol]))
+        @nulls = T.let(nulls!(null), T::Array[String])
+        @col_sep = T.let(col_sep.to_s, String)
+        @quote_char = T.let(quote_char.to_s, String)
+        @encoding = T.let(encoding!(encoding), Encoding)
+        @liberal_parsing = T.let(liberal_parsing, T::Boolean)
+        freeze
+      end
+
+      # A pass over one payload. Takes the bytes as a String, which is what
+      # Sources::Base hands #parse.
+      sig { params(payload: T.untyped).returns(Reader) }
+      def read(payload) = Reader.new(table: self, payload: payload)
+
+      # Whether the file names its own columns.
+      sig { returns(T::Boolean) }
+      def headers? = columns.nil?
+
+      # Zips a row's values against the column names. Extra values are dropped
+      # and missing ones are nil; the Reader has already warned about both.
+      sig { params(names: T::Array[Symbol], values: T::Array[T.untyped]).returns(T::Hash[Symbol, T.nilable(String)]) }
+      def coerce(names, values)
+        names.each_with_index.to_h { |name, index| [name, value(values[index])] }.freeze
+      end
+
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      def csv_options
+        { col_sep: col_sep, quote_char: quote_char, headers: false,
+          liberal_parsing: @liberal_parsing, skip_blanks: true }
+      end
+
+      # What to call this format in an error message, so a complaint about
+      # OFAC's file says "CSV" rather than "delimited text".
+      sig { returns(String) }
+      def col_sep_name = SEPARATOR_NAMES.fetch(col_sep, "delimited text")
+
+      sig { returns(String) }
+      def inspect
+        declared = columns
+        shape = declared.nil? ? "headers from file" : "#{declared.size} columns"
+        "#<#{self.class} #{col_sep_name} #{shape}#{" null=#{nulls.first.inspect}" if nulls.any?}>"
+      end
+
+      private
+
+      sig { params(value: T.untyped).returns(T.nilable(T::Array[Symbol])) }
+      def columns!(value)
+        return nil if value.nil?
+
+        names = column_names!(value)
+        duplicated = names.tally.select { |_, count| count > 1 }.keys
+        raise InvalidArgument, "duplicate column name(s): #{duplicated.join(", ")}" if duplicated.any?
+
+        names.freeze
+      end
+
+      sig { params(value: T.untyped).returns(T::Array[Symbol]) }
+      def column_names!(value)
+        raise InvalidArgument, "columns must be an Array of names, got #{value.inspect}" unless value.is_a?(Array)
+        raise InvalidArgument, "columns cannot be empty; pass nil to read them from the file's header" if value.empty?
+
+        value.map { |name| name.to_s.strip.to_sym }
+      end
+    end
+  end
+end

data/lib/active_sanction/parsers/format.rb

@@ -0,0 +1,107 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module ActiveSanction
+  module Parsers
+    # The two questions every format toolkit has to answer about a publisher's
+    # bytes, regardless of whether they arrive as rows or as elements: what
+    # encoding they are in, and what the publisher writes where it means
+    # nothing.
+    #
+    # Included into the description objects -- DelimitedTable, XmlRecords --
+    # rather than into the readers, because both are properties of the *file*
+    # that an adapter declares once and reuses for every sync.
+    #
+    # (Not to be confused with `Sources::Definition#format`, which is the
+    # publisher-facing label -- :csv, :xml -- that a CLI prints. This is the
+    # machinery behind reading either one.)
+    #
+    # @api private
+    module Format
+      extend T::Sig
+      extend T::Helpers
+
+      abstract!
+
+      # Included into DelimitedTable and XmlRecords, both of which are ordinary
+      # objects; this is what lets `raise` and `Array()` resolve here.
+      requires_ancestor { Kernel }
+
+      DEFAULT_ENCODING = T.let(Encoding::UTF_8, Encoding)
+
+      # A UTF-8 BOM left in place becomes part of the first thing parsed: the
+      # first column name of a headered CSV, or the `<?xml` of a document that
+      # then does not start with `<?xml`. Neither publisher meant to send it.
+      BOM = T.let("", String)
+
+      # Both are the including class's to hold, because both are assigned by
+      # its constructor from what an adapter declared about the file. Declaring
+      # them abstract here rather than reading the instance variables directly
+      # is what lets this module say what it needs of an includer instead of
+      # reaching into one.
+      sig { abstract.returns(T::Array[String]) }
+      def nulls; end
+
+      sig { abstract.returns(Encoding) }
+      def encoding; end
+
+      # What a caller declared as null, resolved. Public because an adapter
+      # joining files by hand needs the same rule the reader applies.
+      #
+      #   table.value("-0- ")   # => nil
+      #   table.value("  ")     # => nil
+      #   table.value(" CUBA")  # => "CUBA"
+      sig { params(raw: T.untyped).returns(T.nilable(String)) }
+      def value(raw)
+        string = raw.to_s.strip
+        return nil if string.empty? || nulls.include?(string)
+
+        -string
+      end
+
+      # The payload as a String in UTF-8, and whether anything had to be
+      # replaced to get there.
+      #
+      # Decoding never raises: a byte that is not valid in the declared
+      # encoding becomes U+FFFD and the caller reports it, because losing one
+      # character of one address is a far better outcome than refusing to load
+      # the list. OFAC serves Windows-1252 and the UN serves UTF-8, and neither
+      # declares it in a header we can trust, which is why the encoding is
+      # something the adapter states.
+      sig { params(payload: T.untyped).returns([String, T::Boolean]) }
+      def decode(payload)
+        string = payload.to_s.dup.force_encoding(encoding)
+        return [string.delete_prefix(BOM), false] if string.valid_encoding? && encoding == DEFAULT_ENCODING
+
+        replaced = !string.valid_encoding?
+        utf8 = string.encode(DEFAULT_ENCODING, invalid: :replace, undef: :replace, replace: "�")
+        [utf8.delete_prefix(BOM), replaced]
+      end
+
+      sig { returns(String) }
+      def invalid_bytes_message
+        "payload contains bytes that are not valid #{encoding}; they were replaced with U+FFFD"
+      end
+
+      private
+
+      # Accepts one sentinel or several: a publisher that writes both "-0-" and
+      # "N/A" is not unusual, and an adapter should be able to say so once.
+      sig { params(value: T.untyped).returns(T::Array[String]) }
+      def nulls!(value)
+        Array(value).map { |sentinel| -sentinel.to_s.strip }.reject(&:empty?).uniq.freeze
+      end
+
+      sig { params(value: T.untyped).returns(Encoding) }
+      def encoding!(value)
+        return value if value.is_a?(Encoding)
+
+        Encoding.find(value.to_s)
+      rescue ArgumentError
+        raise InvalidArgument, "unknown encoding #{value.inspect}"
+      end
+    end
+  end
+end

data/lib/active_sanction/parsers/join.rb

@@ -0,0 +1,149 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "set"
+
+module ActiveSanction
+  module Parsers
+    # Joins one primary file to any number of child files on a shared column.
+    #
+    # Sanctions publishers routinely split one logical record across several
+    # files. OFAC is the extreme case: an entity's name is in SDN.CSV, its
+    # aliases are in ALT.CSV and its addresses are in ADD.CSV, and all three
+    # are keyed on `ent_num`. None of the three means anything alone.
+    #
+    #   join = ActiveSanction::Parsers::Join.new(
+    #     on: :ent_num, aliases: ALT.read(raw[:alt]), addresses: ADD.read(raw[:add])
+    #   )
+    #
+    #   join.each(SDN.read(raw[:sdn])) do |row, related|
+    #     related[:aliases]     # => [Row, ...] -- always an Array, never nil
+    #     related[:addresses]   # => [Row, ...]
+    #   end
+    #
+    # ### What streams and what does not
+    #
+    # The primary file streams: one row at a time, and the caller decides what
+    # to keep. The child files are indexed, which means they are held in memory
+    # for the length of the join.
+    #
+    # That asymmetry is not a shortcut, it is the only honest option. A join
+    # can stream both sides only if both are sorted on the key, and a
+    # publisher's sort order is not something to bet a parse on -- OFAC's files
+    # happen to arrive sorted today, and nothing says they will next quarter. So
+    # the smaller side is indexed and the larger side streams: for OFAC that is
+    # ~45k child rows resident while 19,321 primary rows pass through, which is
+    # a few tens of megabytes and entirely affordable. A source whose child
+    # files are genuinely too large for that wants a different strategy, and
+    # should say so rather than discovering it here.
+    #
+    # ### Orphans
+    #
+    # A child row whose key matches no primary row is dropped and counted. It
+    # is worth counting: a nonzero orphan count after a sync usually means the
+    # three files were downloaded at different moments and do not describe the
+    # same version of the list, which is a data problem no amount of careful
+    # parsing fixes.
+    class Join
+      extend T::Sig
+
+      # `warnings` gathers every complaint from every file in the join, primary
+      # first, and `orphans` counts the child rows that matched nothing. Both
+      # are populated by #each, since that is when the files are actually read.
+      sig { returns(Symbol) }
+      attr_reader :on
+
+      # The child readers, by the name the caller gave each one; that name is
+      # what #each yields them back under.
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      attr_reader :children
+
+      sig { returns(T::Hash[Symbol, Integer]) }
+      attr_reader :orphans
+
+      sig { returns(T::Array[Warning]) }
+      attr_reader :warnings
+
+      sig { params(on: T.untyped, children: T.untyped).void }
+      def initialize(on:, **children)
+        raise InvalidArgument, "a join needs at least one child reader" if children.empty?
+
+        @on = T.let(on.to_sym, Symbol)
+        @children = T.let(children, T::Hash[Symbol, T.untyped])
+        @orphans = T.let({}, T::Hash[Symbol, Integer])
+        @warnings = T.let([], T::Array[Warning])
+      end
+
+      # Yields each primary row with its related child rows. Returns an
+      # Enumerator without a block, so `join.each(rows).lazy` works.
+      #
+      # Re-running rebuilds the indexes rather than reusing them, because the
+      # readers reset their own warnings on re-enumeration and a join that kept
+      # a stale index would report a first pass's problems against a second
+      # pass's rows.
+      sig { params(primary: T.untyped, block: T.untyped).returns(T.untyped) }
+      def each(primary, &block)
+        return enum_for(:each, primary) unless block
+
+        indexes = build_indexes
+        matched = Hash.new { |hash, name| hash[name] = Set.new }
+        primary.each { |row| block.call(row, related(indexes, matched, row.fetch(on))) }
+        @warnings = collect_warnings(primary)
+        count_orphans(indexes, matched)
+        self
+      end
+
+      sig { returns(String) }
+      def inspect = "#<#{self.class} on=#{on.inspect} children=#{children.keys.join(", ")}>"
+
+      private
+
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      def build_indexes
+        children.transform_values { |reader| index(reader) }
+      end
+
+      # Rows filed under their key, in the order the publisher wrote them --
+      # OFAC's `alt_num` ordering is the closest thing its aliases have to a
+      # priority, so it must survive the join.
+      sig { params(reader: T.untyped).returns(T::Hash[T.untyped, T::Array[T.untyped]]) }
+      def index(reader)
+        table = Hash.new { |hash, key| hash[key] = [] }
+        reader.each do |row|
+          key = row.fetch(on)
+          next if key.nil?
+
+          table[key] << row
+        end
+        table
+      end
+
+      # Also records that the key was seen, which is what makes a child row
+      # left over at the end of the pass an orphan rather than just unvisited.
+      sig do
+        params(indexes: T::Hash[Symbol, T.untyped], matched: T.untyped, key: T.untyped)
+          .returns(T::Hash[Symbol, T::Array[T.untyped]])
+      end
+      def related(indexes, matched, key)
+        indexes.to_h do |name, table|
+          matched[name] << key unless key.nil?
+          [name, key.nil? ? [] : table.fetch(key, [])]
+        end
+      end
+
+      sig { params(indexes: T::Hash[Symbol, T.untyped], matched: T.untyped).void }
+      def count_orphans(indexes, matched)
+        @orphans = indexes.to_h do |name, table|
+          [name, table.except(*matched[name]).values.sum(&:size)]
+        end
+      end
+
+      sig { params(primary: T.untyped).returns(T::Array[Warning]) }
+      def collect_warnings(primary)
+        (primary.warnings + children.values.flat_map(&:warnings)).freeze
+      end
+    end
+  end
+end