active_sanction 1.0.0

data/lib/active_sanction/parsers/xml_records/backends/rexml.rb

@@ -0,0 +1,157 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "rexml/parsers/pullparser"
+require "rexml/text"
+require "active_sanction/parsers/xml_records/backends"
+require "active_sanction/parsers/xml_records/builder"
+
+module ActiveSanction
+  module Parsers
+    class XmlRecords
+      module Backends
+        # The default backend: stdlib REXML, pulled one event at a time.
+        #
+        # Chosen as the default because it is everywhere Ruby is, needs no
+        # build step, and gives every installation the same answer -- see
+        # Backends for why that last property is the one that matters for a
+        # list whose parsed content gets checksummed into an audit trail.
+        #
+        # It is the slower of the two. On the UN's 2.2 MB consolidated list
+        # that is a fraction of a second on a job that already spent longer
+        # downloading the file; a host parsing OFAC's 126 MB advanced XML on a
+        # schedule is the case that should switch to libxml2, and can.
+        class Rexml
+          extend T::Sig
+
+          # Nothing to check: `rexml` is a declared dependency of this gem.
+          sig { returns(T::Boolean) }
+          def self.available? = true
+
+          sig { returns(T.nilable(String)) }
+          def self.unavailable_reason = nil
+
+          # The document element's attributes, which is where these publishers
+          # put the version of the list.
+          sig { returns(T.nilable(T::Hash[String, String])) }
+          attr_reader :root
+
+          sig { params(table: XmlRecords, xml: String).void }
+          def initialize(table:, xml:)
+            @table = T.let(table, XmlRecords)
+            @xml = T.let(xml, String)
+            @root = T.let(nil, T.nilable(T::Hash[String, String]))
+            @depth = T.let(0, Integer)
+            @builder = T.let(Builder.new(table: table), Builder)
+            @newlines = T.let(nil, T.nilable(T::Array[Integer]))
+          end
+
+          sig { params(block: T.proc.params(record: Record).void).void }
+          def each_record(&block)
+            parser = ::REXML::Parsers::PullParser.new(@xml)
+            pull(parser, &block)
+            truncated! if @builder.open? || @depth.positive?
+          rescue ::REXML::ParseException => e
+            raise malformed(e, parser)
+          end
+
+          private
+
+          sig { params(parser: T.untyped, block: T.proc.params(record: Record).void).void }
+          def pull(parser, &block)
+            handle(parser.pull, parser, &block) while parser.has_next?
+          end
+
+          sig { params(event: T.untyped, parser: T.untyped, block: T.proc.params(record: Record).void).void }
+          def handle(event, parser, &block)
+            case event.event_type
+            when :start_element then start(event[0], event[1], parser)
+            when :end_element then close(&block)
+            when :text then text(::REXML::Text.unnormalize(event[0]))
+            when :cdata then text(event[0])
+            end
+          end
+
+          sig { params(string: String).void }
+          def text(string)
+            @builder.text(string) if @builder.open?
+          end
+
+          # The first element in the document is its root, whose attributes are
+          # where these publishers put the version of the list -- the UN's
+          # `dateGenerated`. Captured whether or not anything else parses.
+          sig { params(name: T.untyped, attributes: T.untyped, parser: T.untyped).void }
+          def start(name, attributes, parser)
+            local = Backends.local_name(name)
+            attrs = Backends.local_attributes(attributes)
+            @root ||= attrs
+            @depth += 1
+            return unless @builder.open? || @table.record?(local)
+
+            @builder.enter(local, attrs, line_at(parser))
+          end
+
+          sig { params(block: T.proc.params(record: Record).void).void }
+          def close(&block)
+            @depth -= 1
+            return unless @builder.open?
+
+            record = @builder.leave
+            block.call(record) if record
+          end
+
+          # REXML stops at the end of the payload without complaint when a tag
+          # was never closed, so a download cut in half looks like a short list
+          # rather than like a problem. Elements still open at the end are the
+          # evidence, and it is worth raising on: a truncated list that loads
+          # quietly is how a screening run silently stops covering people.
+          #
+          # The depth counter is what catches the commoner half of it. A
+          # download truncated between two records leaves the builder closed
+          # and only the document element unfinished, so counting starts and
+          # ends is the only thing that can tell that list from a complete one.
+          sig { void }
+          def truncated!
+            raise MalformedDocument.new("the document ended inside an unclosed element", line: nil)
+          end
+
+          sig { params(error: T.untyped, parser: T.untyped).returns(MalformedDocument) }
+          def malformed(error, parser)
+            MalformedDocument.new(error.message.lines.first.to_s.strip, line: line_at(parser))
+          end
+
+          # REXML reports a byte offset rather than a line, so the offsets of
+          # every newline are indexed once per pass and binary-searched. The
+          # index is built over the bytes, not the characters, because that is
+          # what the offset counts and the two part company on the first
+          # accented name -- of which these lists have thousands.
+          sig { params(parser: T.untyped).returns(T.nilable(Integer)) }
+          def line_at(parser)
+            offset = parser&.source&.position
+            return nil unless offset.is_a?(Integer)
+
+            (newlines.bsearch_index { |at| at >= offset } || newlines.size) + 1
+          end
+
+          sig { returns(T::Array[Integer]) }
+          def newlines
+            @newlines ||= begin
+              bytes = @xml.b
+              offsets = T.let([], T::Array[Integer])
+              at = T.let(bytes.index("\n"), T.nilable(Integer))
+              while at
+                offsets << at
+                at = bytes.index("\n", at + 1)
+              end
+              offsets
+            end
+          end
+        end
+
+        register(:rexml, Rexml)
+      end
+    end
+  end
+end

data/lib/active_sanction/parsers/xml_records/backends.rb

@@ -0,0 +1,103 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module ActiveSanction
+  module Parsers
+    class XmlRecords
+      # The XML libraries this toolkit knows how to drive, and the seam a host
+      # application swaps one for another through.
+      #
+      #   ActiveSanction.configure { |c| c.xml_backend = :nokogiri }
+      #
+      # ### The contract
+      #
+      # A backend is a class that answers `.available?` and, per pass, is built
+      # with `table:` and the decoded `xml:` and answers two things:
+      #
+      #   backend.each_record { |record| ... }   # yields XmlRecords::Record
+      #   backend.root                           # the document element's attributes
+      #
+      # It raises XmlRecords::MalformedDocument, with a line where its library
+      # supplies one, when the payload stops being XML. Everything else --
+      # paths, null resolution, warnings, Enumerable -- lives above it and is
+      # written once, so a backend is only ever event translation. Registering
+      # another is a public act:
+      #
+      #   ActiveSanction::Parsers::XmlRecords::Backends.register(:ox, MyOxBackend)
+      #
+      # ### Why the default is not chosen by what happens to be loaded
+      #
+      # It would be easy to prefer Nokogiri whenever a host has it, and #15
+      # first asked for exactly that. It is the wrong default here. Snapshot
+      # checksums a list's parsed content, and a screening decision is supposed
+      # to be re-derivable months later in front of an examiner. If the parser
+      # is picked by whether Rails happened to load Nokogiri, then two installs
+      # of the same gem screening the same file can checksum apart, and the
+      # thing that changed is invisible in every artifact either one keeps.
+      #
+      # So the default is REXML on every installation -- stdlib, no build step,
+      # and the same answer everywhere -- and a host that wants libxml2's speed
+      # says so out loud, in one line, where a reviewer can see it.
+      #
+      # @api private
+      module Backends
+        DEFAULT = T.let(:rexml, Symbol)
+
+        class << self
+          extend T::Sig
+
+          # A backend is a class answering the four methods the comment above
+          # names, which is why these signatures say `T.untyped` where one
+          # goes: registering another is a public act, and an out-of-repo
+          # backend is not a subclass of anything here.
+
+          sig { params(name: T.untyped, backend: T.untyped).returns(T.untyped) }
+          def register(name, backend)
+            registry[name.to_sym] = backend
+          end
+
+          sig { params(name: T.untyped).returns(T.untyped) }
+          def resolve(name)
+            backend = registry.fetch(name.to_sym) do
+              raise InvalidArgument,
+                    "unknown XML backend #{name.inspect}. Registered: #{registry.keys.join(", ")}"
+            end
+            return backend if backend.available?
+
+            raise InvalidArgument, "the #{name.inspect} XML backend cannot run here: #{backend.unavailable_reason}"
+          end
+
+          # Every registered backend that could actually run in this process.
+          sig { returns(T::Array[Symbol]) }
+          def available = registry.select { |_, backend| backend.available? }.keys
+
+          sig { returns(T::Hash[Symbol, T.untyped]) }
+          def registry
+            @registry ||= T.let({}, T.nilable(T::Hash[Symbol, T.untyped]))
+          end
+
+          # An element name with any namespace prefix removed, so a list that
+          # grows an `xmlns` next quarter does not stop parsing. The prefix is
+          # discarded rather than resolved: none of these publishers uses two
+          # namespaces in one document, and an adapter written against
+          # `INDIVIDUAL` should not have to be rewritten as `un:INDIVIDUAL`.
+          sig { params(name: T.untyped).returns(String) }
+          def local_name(name)
+            string = name.to_s
+            index = string.rindex(":")
+            index.nil? ? string : string[(index + 1)..]
+          end
+
+          # Attribute keys are stripped the same way, so `xsi:type` is read as
+          # `type` and a default-namespaced document reads like a plain one.
+          sig { params(attributes: T.untyped).returns(T::Hash[String, String]) }
+          def local_attributes(attributes)
+            (attributes || {}).to_h { |key, value| [local_name(key), value.to_s] }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_sanction/parsers/xml_records/builder.rb

@@ -0,0 +1,90 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/parsers/xml_records/record"
+
+module ActiveSanction
+  module Parsers
+    class XmlRecords
+      # Turns a stream of parser events into one Record at a time.
+      #
+      # This is the part of XML parsing that is the same whichever library is
+      # doing it: enter an element, collect its text, leave it into its parent,
+      # and hand back a finished Record when the record element itself closes.
+      # A backend translates its library's events into these four calls and
+      # writes no tree-building code of its own, which is what keeps a second
+      # backend small enough to be worth having.
+      #
+      # It is also what makes the toolkit streaming rather than DOM-based: the
+      # stack only ever holds one record's depth, so a 126 MB file costs the
+      # memory of its largest single record and not of itself.
+      #
+      # @api private
+      class Builder
+        extend T::Sig
+
+        Frame = Struct.new(:name, :attributes, :text, :children, :line)
+        private_constant :Frame
+
+        sig { params(table: XmlRecords).void }
+        def initialize(table:)
+          @table = T.let(table, XmlRecords)
+          @stack = T.let([], T::Array[T.untyped])
+        end
+
+        # Whether we are inside a record. A backend asks this to decide whether
+        # an element is part of a record or is still the file's scaffolding.
+        sig { returns(T::Boolean) }
+        def open? = !@stack.empty?
+
+        sig { params(name: T.untyped, attributes: T.untyped, line: T.nilable(Integer)).void }
+        def enter(name, attributes = {}, line = nil)
+          @stack << Frame.new(name, attributes, nil, [], line)
+        end
+
+        # Appended rather than replaced: a parser is free to split one run of
+        # text across several events, and libxml2 does exactly that around
+        # entity references.
+        sig { params(string: String).void }
+        def text(string)
+          frame = @stack.last
+          return if frame.nil?
+
+          frame.text = frame.text ? frame.text + string : +string
+        end
+
+        # Closes the innermost element. Returns the finished Record when that
+        # was the record element itself, and nil while still inside one.
+        sig { returns(T.nilable(Record)) }
+        def leave
+          frame = @stack.pop
+          return nil if frame.nil?
+
+          record = build(frame)
+          return record if @stack.empty?
+
+          @stack.last.children << record
+          nil
+        end
+
+        # A void element -- `<QUALITY/>` -- which libxml2 reports as a start
+        # with no matching end.
+        sig { params(name: T.untyped, attributes: T.untyped, line: T.nilable(Integer)).returns(T.nilable(Record)) }
+        def void(name, attributes = {}, line = nil)
+          enter(name, attributes, line)
+          leave
+        end
+
+        private
+
+        sig { params(frame: T.untyped).returns(Record) }
+        def build(frame)
+          Record.new(table: @table, name: frame.name, attributes: frame.attributes,
+                     text: frame.text, children: frame.children, line: frame.line)
+        end
+      end
+    end
+  end
+end

data/lib/active_sanction/parsers/xml_records/reader.rb

@@ -0,0 +1,134 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module ActiveSanction
+  module Parsers
+    class XmlRecords
+      # One pass over one payload. Enumerable, and lazy: records are yielded as
+      # they are parsed rather than collected, so the memory cost of a pass is
+      # one record plus whatever the caller keeps -- which is the whole point
+      # of the toolkit, and the reason OFAC's 126 MB advanced XML can drop in
+      # later without a redesign.
+      #
+      #   reader = table.read(bytes)
+      #   reader.each { |record| record["DATAID"] }
+      #   reader.root       # => {"dateGenerated" => "2026-08-28T00:00:00"}
+      #   reader.warnings   # => what could not be read
+      #
+      # Re-enumerating re-parses from the start, which also resets #warnings --
+      # so `reader.count` followed by `reader.warnings` reports the warnings
+      # from the counting pass, not from two passes appended together.
+      #
+      # @api private
+      class Reader
+        extend T::Sig
+        extend T::Generic
+        include Enumerable
+
+        Elem = type_member { { fixed: Record } }
+
+        sig { returns(XmlRecords) }
+        attr_reader :table
+
+        # The records this pass could not read. Reset by each pass -- see the
+        # class comment.
+        sig { returns(T::Array[Warning]) }
+        attr_reader :warnings
+
+        sig { params(table: XmlRecords, payload: T.untyped).void }
+        def initialize(table:, payload:)
+          @table = T.let(table, XmlRecords)
+          @payload = T.let(payload, T.untyped)
+          @warnings = T.let([], T::Array[Warning])
+          @backend = T.let(nil, T.untyped)
+          @count = T.let(0, Integer)
+        end
+
+        sig { override.params(block: T.nilable(T.proc.params(record: Record).void)).returns(T.untyped) }
+        def each(&block)
+          return enum_for(:each) unless block
+
+          @warnings = []
+          @backend = table.backend.new(table: table, xml: decoded)
+          read(&block)
+          self
+        end
+
+        # The document element's attributes. The UN puts the list's generation
+        # date there and nowhere else, and it is the version string an examiner
+        # recognises, so it has to be reachable without the adapter reaching
+        # around the toolkit for it.
+        #
+        # Parsing far enough to answer costs only the bytes up to the first
+        # record, so asking before a pass is cheap; asking after one is free.
+        sig { returns(T::Hash[String, T.untyped]) }
+        def root
+          each.first if @backend.nil?
+          @backend&.root || {}
+        end
+
+        # Every record, in memory. The convenience the small lists get to use;
+        # anything list-sized should stay with #each.
+        sig { returns(T::Array[Record]) }
+        def to_a = each.to_a
+
+        sig { returns(String) }
+        def inspect = "#<#{self.class} #{table.record_names.join(", ")} via #{table.backend}>"
+
+        private
+
+        # A document that stops being XML part-way is not treated the way a
+        # malformed CSV row is, because it cannot be: XML has no row boundary
+        # to resynchronise on, so everything after the break is unreadable no
+        # matter how it is handled. What can be saved is everything before it,
+        # and that is what happens -- the records already yielded stand, and
+        # the break is recorded as a warning naming the line.
+        #
+        # Failing before the first record is the different case, and raises:
+        # nothing was salvaged, and the overwhelmingly likely cause is that the
+        # payload was never XML. An HTML error page saved under a .xml URL is
+        # the classic, and reporting that as "0 records, one warning" would let
+        # a sync succeed at screening against nothing.
+        sig { params(block: T.proc.params(record: Record).void).void }
+        def read(&block)
+          @count = 0
+          @backend.each_record do |record|
+            @count += 1
+            block.call(record)
+          end
+        rescue MalformedDocument => e
+          give_up!(e) if @count.zero?
+          record(e.line, "the document ended after #{@count} record(s): #{e.message}")
+        end
+
+        sig { returns(String) }
+        def decoded
+          string, replaced = table.decode(@payload)
+          record(nil, table.invalid_bytes_message) if replaced
+          raise ParseError, "expected an XML document, got an empty payload" if string.strip.empty?
+
+          string
+        end
+
+        sig { params(line: T.nilable(Integer), message: String).void }
+        def record(line, message)
+          @warnings << Warning.new(line: line, message: message)
+        end
+
+        # The line the backend stopped on is carried through, because that is
+        # the whole difference between "this 25 MB file is not XML" and a
+        # complaint somebody can open an editor to.
+        sig { params(error: MalformedDocument).void }
+        def give_up!(error)
+          raise ParseError.new(
+            "no <#{table.record_names.join("> or <")}> element could be read before the document stopped " \
+            "being XML. This payload is almost certainly not the XML it was read as -- check the URL, " \
+            "and whether the publisher served an error page. #{error.message}", line: error.line
+          )
+        end
+      end
+    end
+  end
+end

data/lib/active_sanction/parsers/xml_records/record.rb

@@ -0,0 +1,174 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module ActiveSanction
+  module Parsers
+    class XmlRecords
+      # One record element and everything under it: the UN's `<INDIVIDUAL>`,
+      # Canada's `<record>`. Fields are read by path relative to the record,
+      # and a nested element is itself a Record, which is what lets an adapter
+      # walk repeated children without knowing how the file was parsed.
+      #
+      #   record.name                       # => "INDIVIDUAL"
+      #   record["DATAID"]                  # => "6907993"
+      #   record["INDIVIDUAL_DATE_OF_BIRTH/YEAR"]
+      #   record.values("NATIONALITY/VALUE")     # => ["Chad", "Sudan"]
+      #   record.nodes("INDIVIDUAL_ALIAS")       # => [Record, Record]
+      #   record["@dateGenerated"]               # an attribute, XPath-style
+      #
+      # ### Absent, empty, and blank are one answer
+      #
+      # An element that is missing, self-closing, or holds only whitespace all
+      # read as nil. That is not laziness about the difference; it is the only
+      # reading that survives the UN, which files placeholder aliases as
+      # `<INDIVIDUAL_ALIAS><QUALITY/><ALIAS_NAME/></INDIVIDUAL_ALIAS>` and means
+      # nothing at all by them. An adapter that had to distinguish the three
+      # would produce blank-valued Names for every one of those placeholders.
+      #
+      # ### Why #[] does not raise the way a CSV Row does
+      #
+      # DelimitedTable::Row raises on a column its table never declared, since
+      # a table's shape is fixed and an unknown name there is a typo. XML has
+      # no such shape: one `<INDIVIDUAL>` carries elements the next one omits,
+      # so an absent path is ordinary and #[] answers nil. #fetch is there for
+      # the field an adapter considers mandatory, and it names the record and
+      # what the record does carry when the field is missing.
+      class Record
+        extend T::Sig
+
+        # @api private
+        SEPARATOR = T.let("/", String)
+        # @api private
+        ATTRIBUTE = T.let("@", String)
+
+        # @api private
+        UNSET = T.let(Object.new.freeze, Object)
+        private_constant :UNSET
+
+        # The element's own name, with any namespace prefix already removed by
+        # the backend -- see Backends.local_name.
+        sig { returns(String) }
+        attr_reader :name
+
+        sig { returns(T::Hash[String, T.untyped]) }
+        attr_reader :attributes
+
+        sig { returns(T::Array[Record]) }
+        attr_reader :children
+
+        # nil where the backend reports no position.
+        sig { returns(T.nilable(Integer)) }
+        attr_reader :line
+
+        sig do
+          params(table: XmlRecords, name: T.untyped, attributes: T::Hash[String, T.untyped], text: T.untyped,
+                 children: T::Array[Record], line: T.nilable(Integer)).void
+        end
+        def initialize(table:, name:, attributes: {}, text: nil, children: [], line: nil)
+          @table = T.let(table, XmlRecords)
+          @name = T.let(-name.to_s, String)
+          @attributes = T.let(attributes.freeze, T::Hash[String, T.untyped])
+          @raw_text = T.let(text, T.untyped)
+          @children = T.let(children.freeze, T::Array[Record])
+          @line = T.let(line, T.nilable(Integer))
+          freeze
+        end
+
+        # This element's own text, with blanks and any declared null sentinel
+        # resolved to nil. Text belonging to child elements is not included.
+        sig { returns(T.nilable(String)) }
+        def text = table.value(@raw_text)
+
+        # The first value at `path`, or nil if nothing is there.
+        sig { params(path: T.untyped).returns(T.nilable(String)) }
+        def [](path) = values(path).first
+
+        # Every value at `path`, in document order, with blanks dropped. The
+        # answer to a repeated element: the UN files each nationality as its
+        # own `<NATIONALITY><VALUE>`.
+        sig { params(path: T.untyped).returns(T::Array[String]) }
+        def values(path)
+          steps, attribute = split(path)
+          nodes = descend(steps)
+          return nodes.filter_map { |node| node.attribute(attribute) } if attribute
+
+          nodes.filter_map(&:text)
+        end
+
+        # The elements at `path`, as Records, whether or not they hold text --
+        # an adapter reading `<INDIVIDUAL_ADDRESS>` wants the node, not a value.
+        sig { params(path: T.untyped).returns(T::Array[Record]) }
+        def nodes(path)
+          steps, attribute = split(path)
+          raise InvalidArgument, "#nodes reads elements, not the attribute #{path.inspect}" if attribute
+
+          descend(steps)
+        end
+
+        sig { params(key: T.untyped).returns(T.nilable(String)) }
+        def attribute(key) = table.value(attributes[key.to_s])
+
+        # For a field the adapter treats as mandatory. Raises rather than
+        # letting a renamed element arrive downstream as a nil nobody notices.
+        sig { params(path: T.untyped, default: T.untyped).returns(T.untyped) }
+        def fetch(path, default = UNSET)
+          value = self[path]
+          return value unless value.nil?
+          return default unless default.equal?(UNSET)
+
+          raise MissingKey, "no value at #{path.inspect} in <#{name}>#{" on line #{line}" if line}. " \
+                            "It carries: #{present.join(", ")}"
+        end
+
+        sig { params(path: T.untyped).returns(T::Boolean) }
+        def null?(path) = self[path].nil?
+
+        # The child element names that actually carry something, which is what
+        # a #fetch failure has to print and what makes an unfamiliar list
+        # explorable from a console.
+        sig { returns(T::Array[String]) }
+        def present
+          names = children.select { |child| !child.text.nil? || child.children.any? }.map(&:name)
+          names.uniq
+        end
+
+        sig { returns(String) }
+        def to_s = text.to_s
+
+        sig { returns(String) }
+        def inspect = "#<#{self.class} <#{name}>#{" line=#{line}" if line} #{present.join(" ")}>"
+
+        protected
+
+        sig { returns(XmlRecords) }
+        attr_reader :table
+
+        sig { params(wanted: String).returns(T::Array[Record]) }
+        def children_named(wanted) = children.select { |child| child.name == wanted }
+
+        private
+
+        # Splits "INDIVIDUAL_ALIAS/ALIAS_NAME" into its steps, and peels off a
+        # trailing "@attr" as the attribute to read instead of the text.
+        sig { params(path: T.untyped).returns([T::Array[String], T.nilable(String)]) }
+        def split(path)
+          steps = path.to_s.split(SEPARATOR).map(&:strip).reject(&:empty?)
+          return [steps, nil] unless steps.last&.start_with?(ATTRIBUTE)
+
+          [steps[0..-2], steps.last.delete_prefix(ATTRIBUTE)]
+        end
+
+        # An empty path is the record itself, which is what makes `record["@id"]`
+        # and `record[""]` mean the obvious things.
+        sig { params(steps: T::Array[String]).returns(T::Array[Record]) }
+        def descend(steps)
+          steps.inject([self]) do |nodes, wanted|
+            nodes.flat_map { |node| node.children_named(wanted) }
+          end
+        end
+      end
+    end
+  end
+end