active_sanction 1.0.0

data/lib/active_sanction/parsers/spreadsheet/archive.rb

@@ -0,0 +1,258 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "zlib"
+
+module ActiveSanction
+  module Parsers
+    class Spreadsheet
+      # The ZIP container an .xlsx workbook arrives in, read with nothing but
+      # `zlib`.
+      #
+      #   archive = Archive.new(bytes)
+      #   archive.names             # => ["[Content_Types].xml", "xl/workbook.xml", ...]
+      #   archive.fetch("xl/workbook.xml")   # => "<?xml version=..."
+      #
+      # ### Why this is here rather than in a gem
+      #
+      # An .xlsx file is a ZIP of XML parts, and the gem already reads XML. The
+      # only thing standing between it and a spreadsheet is the container, and
+      # the container is a 1990 format with a fixed-width header -- a hundred
+      # lines of unpacking, against a spreadsheet gem that would be the first
+      # third-party dependency this library has taken for one publisher's
+      # choice of file format. The gemspec's rule is that a compliance library
+      # should not be the reason a deployment installs something; that rule is
+      # worth more than the hundred lines.
+      #
+      # What is deliberately not implemented is everything a *general* ZIP
+      # reader needs and a published spreadsheet never uses: encryption, spanned
+      # archives, and ZIP64. Each of them raises by name rather than being
+      # half-read, because a container this cannot read is a payload that must
+      # not be parsed as though it were empty.
+      #
+      # ### Read from the central directory, not from the local headers
+      #
+      # A ZIP entry's sizes appear twice -- once in the central directory at the
+      # end of the file and once in the local header in front of the bytes --
+      # and the local copy is allowed to be zeroes, with the real sizes written
+      # in a data descriptor *after* the compressed data. Excel does exactly
+      # that on some writes. So sizes and offsets are taken from the central
+      # directory, which is authoritative, and the local header is read only for
+      # the two lengths that say where the entry's bytes actually begin.
+      #
+      # @api private
+      class Archive
+        extend T::Sig
+
+        EOCD_SIGNATURE = T.let("PK\x05\x06".b, String)
+        CENTRAL_SIGNATURE = T.let("PK\x01\x02".b, String)
+        LOCAL_SIGNATURE = T.let("PK\x03\x04".b, String)
+
+        # 22 bytes of fixed header plus a comment that the format caps at 64 KB.
+        EOCD_FIXED = T.let(22, Integer)
+        MAX_COMMENT = T.let(0xFFFF, Integer)
+
+        CENTRAL_FIXED = T.let(46, Integer)
+        LOCAL_FIXED = T.let(30, Integer)
+
+        STORED = T.let(0, Integer)
+        DEFLATED = T.let(8, Integer)
+
+        # The value every ZIP64 field is replaced by in the 32-bit record that
+        # cannot hold it. Seeing one means the real number is in an extra field
+        # this does not read -- see the class comment.
+        OVERFLOW_32 = T.let(0xFFFFFFFF, Integer)
+        OVERFLOW_16 = T.let(0xFFFF, Integer)
+
+        # General purpose bit 0. Set on an entry whose bytes are encrypted,
+        # which inflates to noise rather than failing, so it is refused here.
+        ENCRYPTED = T.let(0x0001, Integer)
+
+        # A raw deflate stream -- no zlib header, no adler checksum -- which is
+        # what a ZIP entry holds and what a negative window size selects.
+        RAW_DEFLATE = T.let(-Zlib::MAX_WBITS, Integer)
+
+        sig { params(payload: T.untyped).void }
+        def initialize(payload)
+          @bytes = T.let(binary(payload), String)
+          @entries = T.let(read_central_directory, T::Hash[String, T::Hash[Symbol, Integer]])
+        end
+
+        # The part names in the archive, in central-directory order.
+        sig { returns(T::Array[String]) }
+        def names = @entries.keys
+
+        sig { params(name: T.untyped).returns(T::Boolean) }
+        def include?(name) = @entries.key?(name.to_s)
+
+        # One part's bytes, decompressed, or nil when the archive has no such
+        # part. The parts of a workbook are all XML, and the XML reader decodes
+        # them, so what comes back here is binary.
+        sig { params(name: T.untyped).returns(T.nilable(String)) }
+        def [](name)
+          entry = @entries[name.to_s]
+          entry.nil? ? nil : extract(name.to_s, entry)
+        end
+
+        # For a part the caller cannot proceed without. Names what the archive
+        # does hold, because the usual cause of a missing part is that the
+        # payload is not the workbook it was taken for.
+        sig { params(name: T.untyped).returns(String) }
+        def fetch(name)
+          self[name] || raise(ParseError, "#{name.to_s.inspect} is not in this archive. It holds: #{names.join(", ")}")
+        end
+
+        sig { returns(String) }
+        def inspect = "#<#{self.class} #{@entries.size} part(s)>"
+
+        private
+
+        sig { params(payload: T.untyped).returns(String) }
+        def binary(payload)
+          string = payload.to_s
+          string.encoding == Encoding::BINARY ? string : string.dup.force_encoding(Encoding::BINARY)
+        end
+
+        # The end-of-central-directory record is the only fixed landmark in a
+        # ZIP, and it is at the end -- behind a comment of unknown length, so it
+        # is searched for backwards. A payload with no such record is not a
+        # truncated spreadsheet that could be salvaged; it is a file whose index
+        # was never received, and none of the entries can be located without it.
+        sig { returns(Integer) }
+        def eocd_offset
+          last = @bytes.bytesize - EOCD_FIXED
+          offset = last.negative? ? nil : @bytes.rindex(EOCD_SIGNATURE, last)
+          return offset if offset && offset >= last - MAX_COMMENT
+
+          raise ParseError,
+                "expected a ZIP archive (an .xlsx workbook is one), and found no end-of-central-directory " \
+                "record in #{@bytes.bytesize} byte(s) -- the payload is truncated, or is not a workbook"
+        end
+
+        sig { returns(T::Hash[String, T::Hash[Symbol, Integer]]) }
+        def read_central_directory
+          eocd = eocd_offset
+          count, size, start = @bytes[eocd + 10, 12].to_s.unpack("vVV")
+          refuse_zip64!(count, start)
+          walk_central_directory(Integer(start), Integer(size), Integer(count))
+        end
+
+        sig { params(count: T.untyped, start: T.untyped).void }
+        def refuse_zip64!(count, start)
+          return unless count == OVERFLOW_16 || start == OVERFLOW_32
+
+          raise ParseError,
+                "this is a ZIP64 archive, which this reader does not implement. No published sanctions list is " \
+                "anywhere near the 4 GB that requires one, so the payload is almost certainly not a workbook"
+        end
+
+        sig { params(start: Integer, size: Integer, count: Integer).returns(T::Hash[String, T::Hash[Symbol, Integer]]) }
+        def walk_central_directory(start, size, count)
+          entries = {}
+          offset = start
+          finish = start + size
+          count.times do
+            break if offset + CENTRAL_FIXED > finish
+
+            name, entry, offset = read_central_entry(offset)
+            entries[name] = entry
+          end
+          entries
+        end
+
+        sig { params(offset: Integer).returns([String, T::Hash[Symbol, Integer], Integer]) }
+        def read_central_entry(offset)
+          header = @bytes[offset, CENTRAL_FIXED].to_s
+          unless header.start_with?(CENTRAL_SIGNATURE)
+            raise ParseError.new("the central directory ends inside an entry header", offset: offset)
+          end
+
+          entry = central_entry(header)
+          lengths = header[28, 6].to_s.unpack("vvv").map { |length| Integer(length) }
+          name = @bytes[offset + CENTRAL_FIXED, T.must(lengths.first)].to_s.force_encoding(Encoding::UTF_8)
+          [name, entry, offset + CENTRAL_FIXED + lengths.sum]
+        end
+
+        # Flags, compression method, both sizes and where the entry's local
+        # header sits. The four bytes skipped are the modification timestamp,
+        # and the four after them the CRC, which the size check below stands in
+        # for -- a part that inflates to the stated length has not been
+        # truncated, and a part whose deflate stream is damaged raises out of
+        # zlib before it can be measured.
+        sig { params(header: String).returns(T::Hash[Symbol, Integer]) }
+        def central_entry(header)
+          flags, method, compressed, uncompressed = header[8, 20].to_s.unpack("vvx8VV")
+          { flags: Integer(flags), method: Integer(method), compressed: Integer(compressed),
+            uncompressed: Integer(uncompressed), local: Integer(header[42, 4].to_s.unpack1("V")) }
+        end
+
+        sig { params(name: String, entry: T::Hash[Symbol, Integer]).returns(String) }
+        def extract(name, entry)
+          refuse_unreadable!(name, entry)
+          data = @bytes[data_offset(name, entry), Integer(entry.fetch(:compressed))].to_s
+          bytes = entry.fetch(:method) == STORED ? data : inflate(name, data)
+          verify_size!(name, bytes, Integer(entry.fetch(:uncompressed)))
+          bytes
+        end
+
+        sig { params(name: String, entry: T::Hash[Symbol, Integer]).void }
+        def refuse_unreadable!(name, entry)
+          if entry.fetch(:flags).anybits?(ENCRYPTED)
+            raise ParseError, "#{name.inspect} is encrypted, and this reader holds no password"
+          end
+          if entry.values_at(:compressed, :uncompressed, :local).include?(OVERFLOW_32)
+            raise ParseError, "#{name.inspect} states its size in a ZIP64 extra field, which this reader does not read"
+          end
+          return if [STORED, DEFLATED].include?(entry.fetch(:method))
+
+          raise ParseError,
+                "#{name.inspect} is stored with compression method #{entry.fetch(:method)}, and this reader " \
+                "implements only stored (0) and deflate (8), which is everything a spreadsheet writer emits"
+        end
+
+        # The local header repeats the name and carries its own extra field,
+        # and the two lengths are the only thing read from it -- see the class
+        # comment on why the sizes are not.
+        sig { params(name: String, entry: T::Hash[Symbol, Integer]).returns(Integer) }
+        def data_offset(name, entry)
+          offset = Integer(entry.fetch(:local))
+          header = @bytes[offset, LOCAL_FIXED].to_s
+          unless header.start_with?(LOCAL_SIGNATURE)
+            raise ParseError.new("the central directory points at #{name.inspect}, which is not a local header",
+                                 offset: offset)
+          end
+
+          name_length, extra_length = header[26, 4].to_s.unpack("vv")
+          offset + LOCAL_FIXED + Integer(name_length) + Integer(extra_length)
+        end
+
+        sig { params(name: String, data: String).returns(String) }
+        def inflate(name, data)
+          stream = Zlib::Inflate.new(RAW_DEFLATE)
+          begin
+            stream.inflate(data) << stream.finish
+          ensure
+            stream.close
+          end
+        rescue Zlib::Error => e
+          raise ParseError, "#{name.inspect} could not be decompressed: #{e.class} #{e.message}"
+        end
+
+        # The central directory states what the part should weigh, and a part
+        # that arrives short is a truncated download rather than a shorter
+        # spreadsheet. Saying so here is what stops half a sanctions list being
+        # parsed as though it were the whole one.
+        sig { params(name: String, bytes: String, expected: Integer).void }
+        def verify_size!(name, bytes, expected)
+          return if expected.zero? || bytes.bytesize == expected
+
+          raise ParseError,
+                "#{name.inspect} unpacked to #{bytes.bytesize} byte(s) where the archive's own index says " \
+                "#{expected} -- the payload is truncated or corrupt"
+        end
+      end
+    end
+  end
+end

data/lib/active_sanction/parsers/spreadsheet/reader.rb

@@ -0,0 +1,213 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/parsers/xml_records"
+
+module ActiveSanction
+  module Parsers
+    class Spreadsheet
+      # One pass over one workbook. Enumerable, and lazy in the same sense the
+      # XML reader is: the sheet is streamed row by row, so what is held is one
+      # row plus the shared string table -- which is not optional, a cell being
+      # an offset into it.
+      #
+      #   reader = table.read(bytes)
+      #   reader.each { |row| row[:reference] }
+      #   reader.warnings          # => rows that could not be read
+      #   reader.sheet_names       # => every sheet, whether or not it was read
+      #
+      # Re-enumerating re-reads from the start, which also resets #warnings --
+      # so `reader.count` followed by `reader.warnings` reports the warnings
+      # from the counting pass, not from two passes appended together. The ZIP
+      # is unpacked once and reused across passes.
+      #
+      # @api private
+      class Reader
+        extend T::Sig
+        extend T::Generic
+        include Enumerable
+
+        Elem = type_member { { fixed: Row } }
+
+        ROWS = T.let(Parsers::XmlRecords.new(records: %w[row]), Parsers::XmlRecords)
+
+        # `A`, `AB`, `XFD`: the letters in a cell's `r="C7"` address, which is
+        # the only thing that says which column a cell is in. A row states only
+        # the cells that hold something, so position in the row means nothing.
+        ADDRESS = T.let(/\A([A-Z]+)/, Regexp)
+
+        LETTERS = T.let(26, Integer)
+
+        DATE_FORMATS = T.let({ day: "%Y-%m-%d", month: "%Y-%m", year: "%Y" }.freeze, T::Hash[Symbol, String])
+
+        sig { returns(Spreadsheet) }
+        attr_reader :table
+
+        sig { returns(T::Array[Warning]) }
+        attr_reader :warnings
+
+        sig { params(table: Spreadsheet, payload: T.untyped).void }
+        def initialize(table:, payload:)
+          @table = T.let(table, Spreadsheet)
+          @payload = T.let(payload, T.untyped)
+          @warnings = T.let([], T::Array[Warning])
+          @workbook = T.let(nil, T.nilable(Workbook))
+        end
+
+        # The workbook's sheets, in the order it lists them. Worth printing when
+        # a publisher adds a second tab: only one of them is being read.
+        sig { returns(T::Array[String]) }
+        def sheet_names = workbook.sheet_names
+
+        # When the workbook says it was last saved -- see Workbook#modified.
+        # Costs one part of the archive and no pass over the sheet.
+        sig { returns(T.nilable(String)) }
+        def modified = workbook.modified
+
+        sig { override.params(block: T.nilable(T.proc.params(row: Row).void)).returns(T.untyped) }
+        def each(&block)
+          return enum_for(:each) unless block
+
+          @warnings = []
+          rows = ROWS.read(workbook.sheet(table.sheet))
+          columns = T.let(nil, T.nilable(T::Array[Symbol]))
+          rows.each do |node|
+            cells = read_row(node)
+            columns = table.columns || header!(cells, node) and next if columns.nil?
+
+            # A `<row>` with no cell holding anything is spacing, not a record.
+            block.call(build(columns, cells, number(node))) if cells.any?
+          end
+          @warnings.concat(rows.warnings)
+          self
+        end
+
+        # Every row, in memory. The convenience the small sheets get to use.
+        sig { returns(T::Array[Row]) }
+        def to_a = each.to_a
+
+        sig { returns(String) }
+        def inspect = "#<#{self.class} #{table.sheet_name} of #{sheet_names.size} sheet(s)>"
+
+        private
+
+        sig { returns(Workbook) }
+        def workbook
+          @workbook ||= Workbook.new(table: table, archive: Archive.new(@payload))
+        end
+
+        # Column index to value, indexed from the cell's own address so that a
+        # row which states only the cells it filled still lines up with the
+        # header. Nothing here assumes the row is contiguous, because it is not:
+        # a row whose first two cells are empty starts at `C`.
+        sig { params(node: Parsers::XmlRecords::Record).returns(T::Hash[Integer, String]) }
+        def read_row(node)
+          node.nodes("c").each_with_object({}) do |cell, values|
+            index = column_index(cell.attribute("r"))
+            value = table.value(cell_value(cell))
+            values[index] = value if index && value
+          end
+        end
+
+        # `C` is 2, `AA` is 26, in the bijective base-26 a spreadsheet numbers
+        # its columns with.
+        sig { params(address: T.nilable(String)).returns(T.nilable(Integer)) }
+        def column_index(address)
+          letters = ADDRESS.match(address.to_s)&.captures&.first
+          return nil if letters.nil?
+
+          letters.each_char.inject(0) { |index, letter| (index * LETTERS) + (letter.ord - 64) } - 1
+        end
+
+        # A cell states its type in `t` and its value in `<v>` -- except an
+        # inline string, which is the one shape that carries its text where a
+        # shared string's offset would be.
+        sig { params(cell: Parsers::XmlRecords::Record).returns(T.nilable(String)) }
+        def cell_value(cell)
+          case cell.attribute("t")
+          when "s" then shared(cell["v"])
+          when "inlineStr" then table.unescape(cell.values("is/t").join)
+          when "b" then cell["v"] == "1" ? "TRUE" : "FALSE"
+          when "str", "e", "d" then cell["v"]
+          else numeric(cell)
+          end
+        end
+
+        sig { params(offset: T.nilable(String)).returns(T.nilable(String)) }
+        def shared(offset) = offset.nil? ? nil : workbook.strings[offset.to_i]
+
+        # A number is a date when the cell's style says it is displayed as one,
+        # and is otherwise passed through as the publisher wrote it -- `1963`
+        # stays `1963`, because a year is not a date and rendering it as one
+        # would invent a January the 1st nobody published.
+        sig { params(cell: Parsers::XmlRecords::Record).returns(T.nilable(String)) }
+        def numeric(cell)
+          raw = cell["v"]
+          precision = workbook.precision(cell.attribute("s")&.to_i)
+          return raw if raw.nil? || precision.nil?
+
+          formatted(raw, precision) || raw
+        end
+
+        # ISO 8601, truncated to what the cell's format actually displays: a
+        # `mmm-yy` cell says a month and its serial's day is whatever Excel
+        # needed to store one. PartialDate::Parser reads all three shapes, so an
+        # adapter gets the publisher's own precision without a format of its own.
+        sig { params(raw: String, precision: Symbol).returns(T.nilable(String)) }
+        def formatted(raw, precision)
+          workbook.date(raw)&.strftime(DATE_FORMATS.fetch(precision))
+        end
+
+        # Column names from the sheet's own first row, normalized the way the
+        # delimited reader normalizes a CSV header, so that `Name of Individual
+        # or Entity` and `name_of_individual_or_entity` are the same column
+        # however the publisher capitalized it this quarter.
+        sig { params(cells: T::Hash[Integer, String], node: Parsers::XmlRecords::Record).returns(T::Array[Symbol]) }
+        def header!(cells, node)
+          raise ParseError.new("expected a header row, and it is empty", line: number(node)) if cells.empty?
+
+          (0..T.must(cells.keys.max)).map { |index| normalize_header(cells[index], index) }
+        end
+
+        # A header cell the publisher left blank still names a column, because
+        # the columns after it have to keep their positions.
+        sig { params(value: T.nilable(String), index: Integer).returns(Symbol) }
+        def normalize_header(value, index)
+          name = value.to_s.strip.downcase.gsub(/[^a-z0-9]+/, "_").delete_prefix("_").delete_suffix("_")
+          name.empty? ? :"column_#{index + 1}" : name.to_sym
+        end
+
+        sig { params(node: Parsers::XmlRecords::Record).returns(Integer) }
+        def number(node) = node.attribute("r")&.to_i || node.line || 0
+
+        sig do
+          params(columns: T::Array[Symbol], cells: T::Hash[Integer, String], number: Integer).returns(Row)
+        end
+        def build(columns, cells, number)
+          record_width(columns, cells, number)
+          Row.new(values: table.coerce(columns, cells), number: number)
+        end
+
+        # A row with a value in a column the header never named is kept, and the
+        # extra value is dropped rather than the row. A publisher appending a
+        # column mid-year should degrade the fields nobody has mapped yet, not
+        # the whole list -- but it must say so, because a column silently
+        # ignored is how a new sanctions measure stops being read.
+        sig { params(columns: T::Array[Symbol], cells: T::Hash[Integer, String], number: Integer).void }
+        def record_width(columns, cells, number)
+          beyond = cells.keys.select { |index| index >= columns.size }
+          return if beyond.empty?
+
+          @warnings << Warning.new(
+            line: number,
+            message: "row #{number} holds #{beyond.size} value(s) past the #{columns.size} column(s) " \
+                     "this sheet named",
+            snippet: beyond.map { |index| cells[index] }.join(" | ")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/active_sanction/parsers/spreadsheet/row.rb

@@ -0,0 +1,88 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module ActiveSanction
+  module Parsers
+    class Spreadsheet
+      # One row of a sheet: column name to value, with every cell already
+      # resolved to the string it displays as -- a shared string looked up, a
+      # date serial rendered, a blank cell nil.
+      #
+      #   row[:name_of_individual_or_entity]   # => "MOHAMMAD HASSAN AKHUND"
+      #   row[:date_of_birth]                  # => "1963-01-30"
+      #   row[:alias_strength]                 # => nil, the cell being empty
+      #   row.number                           # => 2, the sheet's own row number
+      #
+      # Deliberately not a Hash, for the reason DelimitedTable::Row is not one:
+      # a Hash would answer `row[:date_of_brith]` with nil and let a typo in an
+      # adapter look like an empty column all the way into a snapshot. #[]
+      # raises on a name the sheet never had, and #fetch is there for the
+      # genuinely optional case.
+      #
+      # It is a separate class from the delimited table's row rather than a
+      # shared one because the two disagree about what locates a record. A CSV
+      # row is at a line of a file; a sheet row is at a row number the
+      # spreadsheet itself assigns, which is what is showing in the corner of
+      # the window when somebody opens the published file to check a record --
+      # and which survives the workbook being re-saved with different XML.
+      class Row
+        extend T::Sig
+
+        sig { returns(T::Hash[Symbol, T.nilable(String)]) }
+        attr_reader :values
+
+        # The sheet's own row number, 1-based and counting the header row --
+        # the number a reader of the published file will see beside the record.
+        sig { returns(Integer) }
+        attr_reader :number
+
+        sig { params(values: T::Hash[Symbol, T.nilable(String)], number: Integer).void }
+        def initialize(values:, number:)
+          @values = T.let(values.freeze, T::Hash[Symbol, T.nilable(String)])
+          @number = T.let(number, Integer)
+          freeze
+        end
+
+        # Raises on a column the sheet never had, which is almost always a typo
+        # in an adapter rather than a question about the data.
+        sig { params(column: Symbol).returns(T.nilable(String)) }
+        def [](column)
+          values.fetch(column) do
+            raise MissingKey, "no column #{column.inspect} in this sheet. Read: #{columns.join(", ")}"
+          end
+        end
+
+        sig { params(column: Symbol, default: T.untyped).returns(T.untyped) }
+        def fetch(column, default = nil) = values.fetch(column, default)
+
+        sig { params(column: Symbol).returns(T::Boolean) }
+        def null?(column) = self[column].nil?
+
+        sig { returns(T::Array[Symbol]) }
+        def columns = values.keys
+
+        sig { returns(T::Hash[Symbol, T.nilable(String)]) }
+        def to_h = values
+
+        sig { params(other: T.untyped).returns(T::Boolean) }
+        def ==(other)
+          return false unless other.instance_of?(self.class)
+
+          values == other.values && number == other.number
+        end
+        alias eql? ==
+
+        sig { returns(Integer) }
+        def hash = [self.class, values, number].hash
+
+        sig { returns(String) }
+        def inspect
+          filled = values.compact
+          "#<#{self.class} row=#{number} #{filled.map { |name, value| "#{name}=#{value.inspect}" }.join(" ")}>"
+        end
+      end
+    end
+  end
+end