active_sanction 1.0.0

data/lib/active_sanction/sources/australia_dfat/published_date.rb

@@ -0,0 +1,166 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/partial_date"
+
+module ActiveSanction
+  module Sources
+    class AustraliaDfat < Base
+      # The dates DFAT writes, which are not one format and were never meant to
+      # be read by a machine.
+      #
+      #   PublishedDate.dates("13/06/1961, Approximately 1963")
+      #   # => [[#<PartialDate 1961-06-13>, #<PartialDate ~1963>], []]
+      #
+      #   PublishedDate.listing("Listed on 25 Jan. 2001 (amended on 3 Sep. 2003)")
+      #   # => #<PartialDate 2001-01-25>
+      #
+      # ### One cell, any number of birth dates, in any number of spellings
+      #
+      # The Date of Birth column holds every date the listing carries, and a
+      # committee that received four reports of when somebody was born publishes
+      # four. `1945, 1946, 1947, 1948, 1949, 1950, 1955, 1956, 1957, 1958` is one
+      # cell of this list, and so are `a) 5/12/1970 b) 1969`, `Approximately:
+      # Between 1972 and 1975`, `12 April 1965` and `03/1955`. The separators
+      # are commas, the letters of an enumeration DFAT inherits from the UN's
+      # own text on 17 rows, and -- on 28 more -- nothing but a space between
+      # two dates.
+      #
+      # Four records out of the 3,906 carry a fragment this cannot read, and all
+      # four are typed wrong at the source: `1980.1981`, `/02/1961`,
+      # `7/02/1950/11/1950`, `10/061962`. Each is kept verbatim in the record's
+      # remarks rather than dropped, so a date this version cannot parse is
+      # still in front of whoever reads the hit.
+      #
+      # ### Day first, because Australia writes day first
+      #
+      # `05/12/1970` is the fifth of December. Nothing in the file says so, and
+      # reading it the American way would move some of the 2,013 dates written
+      # this way by up to eleven months -- silently, and only for the days below
+      # the thirteenth. What settles it is the publisher: DFAT writes Australian
+      # dates, and not one of the 2,013 has a middle component above twelve,
+      # which a file in the other order would produce on any date after the
+      # twelfth of a month.
+      #
+      # Excel serial dates do not come through here at all: they arrive as
+      # ISO 8601 from the spreadsheet reader, which knows the cell's format,
+      # and PartialDate::Parser reads those directly.
+      #
+      # @api private
+      module PublishedDate
+        extend T::Sig
+
+        # `a)`, `b)`: the enumerators, which separate values rather than being
+        # part of one. Also removed from the addresses, for the same reason.
+        ENUMERATOR = T.let(/(?:\A|[[:space:]])[a-z]\)[[:space:]]*/, Regexp)
+
+        # Two dates with nothing between them but space. A digit on each side is
+        # what distinguishes `14/04/1970 14/04/1971`, which is two dates, from
+        # `12 April 1965`, which is one.
+        ADJACENT = T.let(/(?<=\d)[[:space:]]+(?=\d)/, Regexp)
+
+        # DFAT writes `Approximately:` before a span and `Approximately` before
+        # a year, and PartialDate::Parser knows the second spelling only.
+        APPROXIMATELY = T.let(/\Aapproximately:/i, Regexp)
+
+        # Every kind of space, because the cells carry more than one: 203 of the
+        # 6,823 birth dates end in a non-breaking space, which `String#strip`
+        # does not remove and which leaves an otherwise perfectly good
+        # `24/08/1962` matching no pattern at all.
+        SPACE = T.let(/[[:space:]]+/, Regexp)
+
+        DAY_MONTH_YEAR = T.let(%r{\A(\d{1,2})/(\d{1,2})/(\d{4})\z}, Regexp)
+        MONTH_YEAR = T.let(%r{\A(\d{1,2})/(\d{4})\z}, Regexp)
+
+        # A listing date is the one that follows the word "on", which is how
+        # every row that states one states it: `Listed on 25 January 2001`,
+        # `Designated under the Autonomous Sanctions Regulations 2011 on 2 March
+        # 2012`. Matching the bare year in a legislative title -- which is what
+        # the other 63% of the column consists of -- would report the year an
+        # instrument was made as the day a person was sanctioned.
+        LISTED_ON = T.let(/(\w[\w-]*)[[:space:]]+on:?[[:space:]]+(\d{1,2}[[:space:]]+[A-Za-z]+\.?,?[[:space:]]+\d{4})/,
+                          Regexp)
+
+        # A row that opens with a date and explains it afterwards: `28 Jan. 2003
+        # (amended on 2 Jul. 2007, ...)`.
+        LISTED_FIRST = T.let(/\A(\d{1,2}[[:space:]]+[A-Za-z]+\.?,?[[:space:]]+\d{4})/, Regexp)
+
+        # The words that make the date after them the wrong one. 17 rows read
+        # `... List 2001 (updated on 5 Aug. 2004)`, where the first date in the
+        # cell is when the listing was last touched rather than when it was
+        # made, and a designation dated by its own last amendment is worse than
+        # one left undated.
+        AMENDMENT = T.let(/\A(?:amend|relist|re-list|updat|expir)/i, Regexp)
+
+        module_function
+
+        # Every date in one cell, and the fragments that could not be read --
+        # which the adapter keeps in remarks rather than losing.
+        sig { params(text: T.untyped).returns([T::Array[PartialDate], T::Array[String]]) }
+        def dates(text)
+          dates = []
+          unread = []
+          fragments(text).each do |fragment|
+            date = one(fragment)
+            date.nil? ? unread << fragment : dates << date
+          end
+          [dates.uniq, unread]
+        end
+
+        # The date the designation was made, from the Listing Information prose,
+        # or nil where DFAT names only the instrument -- which is 63% of the
+        # list, and is the publisher saying nothing rather than this failing to
+        # read it.
+        sig { params(text: T.untyped).returns(T.nilable(PartialDate)) }
+        def listing(text)
+          string = text.to_s
+          opening = LISTED_FIRST.match(string)
+          return PartialDate::Parser.call(opening[1]) if opening
+
+          match = LISTED_ON.match(string)
+          return nil if match.nil? || AMENDMENT.match?(match[1].to_s)
+
+          PartialDate::Parser.call(match[2])
+        end
+
+        # One cell split into the values it holds. Commas first, because they
+        # are the separator DFAT means; then the enumerators and the bare space,
+        # which are what the UN's own prose left behind.
+        sig { params(text: T.untyped).returns(T::Array[String]) }
+        def fragments(text)
+          text.to_s.gsub(ENUMERATOR, ",").split(",").flat_map { |part| part.split(ADJACENT) }
+              .map { |part| part.gsub(SPACE, " ").strip }.reject(&:empty?)
+        end
+
+        sig { params(fragment: String).returns(T.nilable(PartialDate)) }
+        def one(fragment)
+          slashed(fragment) || PartialDate::Parser.call(fragment.sub(APPROXIMATELY, "approximately"))
+        end
+
+        # The two shapes with slashes in them, which PartialDate::Parser does
+        # not read and must not guess at: `05/12/1970` is a date in one order
+        # and a different date in the other, and only the publisher settles it.
+        sig { params(fragment: String).returns(T.nilable(PartialDate)) }
+        def slashed(fragment)
+          if (match = DAY_MONTH_YEAR.match(fragment))
+            build(year: match[3].to_i, month: match[2].to_i, day: match[1].to_i)
+          elsif (match = MONTH_YEAR.match(fragment))
+            build(year: match[2].to_i, month: match[1].to_i)
+          end
+        end
+
+        # A shape the regexes accept can still be an impossible date -- 31
+        # February -- and that is PartialDate's judgment, not ours. An
+        # unreadable fragment is reported as unread rather than dropped.
+        sig { params(attributes: T.untyped).returns(T.nilable(PartialDate)) }
+        def build(**attributes)
+          PartialDate.new(**attributes)
+        rescue ArgumentError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/active_sanction/sources/australia_dfat/record.rb

@@ -0,0 +1,319 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/entity"
+require "active_sanction/name"
+require "active_sanction/address"
+require "active_sanction/identifier"
+require "active_sanction/partial_date"
+require "active_sanction/sources/remarks"
+
+module ActiveSanction
+  module Sources
+    class AustraliaDfat < Base
+      # The rows sharing one reference, turned into an Entity.
+      #
+      # Separate from the adapter because they are two jobs: the adapter says
+      # what the list is and where it lives, and this says what DFAT's columns
+      # mean. The mapping is where all the judgment sits, so it is worth being
+      # able to read it on its own.
+      #
+      # ### A record is several rows, and the reference says which
+      #
+      # DFAT publishes one row per *name*, not per person. `1000` is the primary
+      # name and `1000a`, `1000b`, `1000c` are its aliases, and every other
+      # column is repeated on each of them. The 11,163 rows of the published
+      # list are 3,906 records: 3,906 primary names, 6,802 aliases and 455 names
+      # in original script.
+      #
+      # The repetition is not quite exact. 80 groups disagree with themselves
+      # about the additional information, 39 about the birth dates and 23 about
+      # the address -- an alias row carrying a detail the primary row does not,
+      # usually because the two came from different UN listings of the same
+      # person. So every column is unioned across the group rather than read off
+      # the primary row: two of the 3,906 records have a place of birth, and one
+      # an address, only because an alias row carried it.
+      #
+      # ### The trap in this list is the Control Date
+      #
+      # It is on all 11,163 rows, it is a real date, and it is not the day
+      # anybody was sanctioned. DFAT's own guide defines it as "the last date
+      # the sanction entry was updated or edited on the Consolidated List",
+      # which for the Taliban listings of January 2001 is a date in 2026. An
+      # adapter that mapped it to `listed_on` would report the whole list as
+      # having been sanctioned within the last few months, and would do it
+      # without anything looking wrong.
+      #
+      # The listing date is in the Listing Information prose instead, on the
+      # 1,438 records that state one; see PublishedDate.listing. The Control
+      # Date is kept in remarks, labelled as what it is.
+      #
+      # @api private
+      class Record
+        extend T::Sig
+
+        # The `Type` column, which is populated on every row and is the only
+        # place the list says what a listing is about. 346 rows are vessels:
+        # without the distinct type a search for a person can rank a tanker.
+        TYPES = T.let(
+          { "individual" => :individual, "entity" => :organization, "vessel" => :vessel }.freeze,
+          T::Hash[String, Symbol]
+        )
+
+        # `Name Type`, case-folded. A name in original script is the same name
+        # written another way rather than the designation proper, so it is filed
+        # as an alias -- which is what leaves `Entity#primary_name` answering
+        # with the one name DFAT calls primary.
+        NAME_KINDS = T.let(
+          { "primary name" => :primary, "alias" => :aka, "original script" => :aka }.freeze,
+          T::Hash[String, Symbol]
+        )
+
+        # `Alias Strength`, which DFAT publishes on all 6,802 aliases -- 6,346
+        # strong, 456 weak -- and on nothing else. Its guide defines strong as an
+        # alias assessed to be closely associated with the listing and weak as
+        # one that is not, which is the distinction the UN grades Good and Low,
+        # so it maps onto the grade the scorer already knows how to discount.
+        ALIAS_QUALITIES = T.let({ "strong" => :good, "weak" => :low }.freeze, T::Hash[String, Symbol])
+
+        # The four measures, which DFAT publishes as 1 and 0 rather than as the
+        # TRUE and FALSE its guide describes. Real screening context -- an arms
+        # embargo and an asset freeze are not the same finding -- with nowhere
+        # in the canonical model to live, so they are named in remarks.
+        MEASURES = T.let(
+          { targeted_financial_sanction: "targeted financial sanction", travel_ban: "travel ban",
+            arms_embargo: "arms embargo", maritime_restriction: "maritime restriction" }.freeze,
+          T::Hash[Symbol, String]
+        )
+
+        TRUTHY = T.let(%w[1 true yes y].freeze, T::Array[String])
+
+        # Columns kept verbatim in remarks, in the order they are printed. Place
+        # of birth is real screening signal with no member of its own; the
+        # instrument is the legislative vehicle, which is narrower than the
+        # framework in `programs` and changes every time a list is amended.
+        EXTRA_FIELDS = T.let(
+          [[:place_of_birth, "Place of birth"], [:instrument_of_designation, "Instrument of designation"],
+           [:control_date, "Control date (last edited, not listed)"]].freeze,
+          T::Array[[Symbol, String]]
+        )
+
+        # DFAT separates citizenships with a semicolon and nothing else, on all
+        # 399 rows that publish more than one.
+        SEMICOLON = T.let(";", String)
+
+        # What separates the two prose columns once they are joined.
+        PARAGRAPH = T.let("\n\n", String)
+
+        # Trailing spaces, and the carriage returns Excel escapes as `_x000D_`
+        # and the spreadsheet reader unescapes. A name that keeps either is a
+        # name nothing will ever match.
+        WHITESPACE = T.let(/[[:space:]]+/, Regexp)
+
+        sig { returns(T::Array[Parsers::Spreadsheet::Row]) }
+        attr_reader :rows
+
+        sig { params(rows: T::Array[Parsers::Spreadsheet::Row]).void }
+        def initialize(rows)
+          @rows = T.let(rows, T::Array[Parsers::Spreadsheet::Row])
+          @dates_of_birth = T.let(nil, T.nilable(T::Array[PartialDate]))
+          @unread_dates = T.let([], T::Array[String])
+        end
+
+        # The entity, or nil for a group whose rows are all nameless -- which
+        # cannot be screened against and is never what DFAT meant to publish.
+        sig { returns(T.nilable(Entity)) }
+        def entity
+          return nil if names.empty?
+
+          Entity.new(source: :australia_dfat, source_ref: source_ref, type: type, names: names,
+                     addresses: addresses, identifiers: identifiers, dates_of_birth: dates_of_birth,
+                     nationalities: nationalities, programs: programs, listed_on: listed_on,
+                     remarks: remarks)
+        end
+
+        # DFAT's own reference with the alias suffix removed: the `1000` that
+        # `1000`, `1000a` and `1000b` are all part of. Stable across syncs, and
+        # the number DFAT will quote back in a permit application.
+        sig { returns(T.nilable(String)) }
+        def source_ref = AustraliaDfat.group(primary[:reference])
+
+        sig { returns(Symbol) }
+        def type = TYPES.fetch(primary[:type].to_s.downcase, :organization)
+
+        # In published order, which puts the primary name first. A spelling
+        # DFAT files twice -- once as an alias and once in original script --
+        # keeps the first kind it was given.
+        sig { returns(T::Array[Name]) }
+        def names
+          @names ||= T.let(dedupe(rows.filter_map { |row| name(row) }), T.nilable(T::Array[Name]))
+        end
+
+        # Memoized, because reading them is also what fills the list of dates
+        # this could not use, which #remarks then keeps rather than losing.
+        sig { returns(T::Array[PartialDate]) }
+        def dates_of_birth
+          @dates_of_birth ||= begin
+            @unread_dates = []
+            column(:date_of_birth).flat_map { |cell| read_dates(cell) }.uniq
+          end
+        end
+
+        # Prose, the way the UN and the UK publish it: `Russia`, `Democratic
+        # People's Republic of Korea (North Korea)`. Country resolves it to an
+        # ISO code at scoring time and treats one it cannot resolve as absent
+        # rather than as a conflict.
+        sig { returns(T::Array[String]) }
+        def nationalities
+          column(:citizenship).flat_map { |cell| cell.split(SEMICOLON) }.filter_map { |value| collapse(value) }.uniq
+        end
+
+        # One column of free text, which DFAT does not decompose and this does
+        # not guess at. The whole published string is the street, because it is
+        # the address rather than an annotation about one -- and 859 rows hold
+        # more than one address, enumerated `a) ... b) ...` the way the birth
+        # dates are.
+        sig { returns(T::Array[Address]) }
+        def addresses
+          column(:address).flat_map { |cell| AustraliaDfat.enumerated(cell) }.uniq.filter_map { |line| address(line) }
+        end
+
+        # IMO numbers, on the 344 vessel rows that carry one. Filed the way the
+        # UK adapter files them: a registry number that is not any of the four
+        # document kinds, labelled in its note.
+        sig { returns(T::Array[Identifier]) }
+        def identifiers
+          column(:imo_number).filter_map { |number| identifier(number) }.uniq
+        end
+
+        # The sanctions framework, which is the closest thing DFAT publishes to
+        # OFAC's programme codes: `1267 (ISIL (Da'esh) and Al-Qaida)`,
+        # `Autonomous (Russia)`. Populated on every row, and 29 values cover the
+        # whole list.
+        sig { returns(T::Array[String]) }
+        def programs = column(:committees).uniq
+
+        sig { returns(T.nilable(PartialDate)) }
+        def listed_on = column(:listing_information).filter_map { |cell| PublishedDate.listing(cell) }.first
+
+        # DFAT's own prose verbatim -- the identifying detail and the listing
+        # narrative, which are two columns and one voice -- then the columns
+        # that have nowhere else to go, behind the marker that makes them
+        # trivial to strip again.
+        sig { returns(T.nilable(String)) }
+        def remarks
+          dates_of_birth # for its side effect: it is what fills @unread_dates, which #extras keeps
+          Remarks.build(published_prose, extras)
+        end
+
+        sig { returns(String) }
+        def inspect = "#<#{self.class} #{source_ref} #{rows.size} row(s)>"
+
+        private
+
+        # The row DFAT calls the primary name, falling back to the first row of
+        # the group: a group with no primary name is not published today, and if
+        # one ever is, its aliases still describe a real designation.
+        sig { returns(Parsers::Spreadsheet::Row) }
+        def primary
+          @primary ||= T.let(rows.find { |row| row[:name_type].to_s.downcase == "primary name" } || T.must(rows.first),
+                             T.nilable(Parsers::Spreadsheet::Row))
+        end
+
+        # One column across every row of the group, blanks dropped, order kept.
+        # Unioned rather than read off the primary row -- see the class comment
+        # on the 79 groups that disagree with themselves.
+        #
+        # De-duplicated on the collapsed text rather than on the string, because
+        # DFAT's repetition is not always byte-for-byte: an alias row routinely
+        # repeats the primary row's address or place of birth with one more
+        # trailing space, and keeping both would print the same sentence twice
+        # in the remark.
+        sig { params(name: Symbol).returns(T::Array[String]) }
+        def column(name)
+          seen = T.let({}, T::Hash[String, String])
+          rows.each do |row|
+            text = row[name]
+            key = collapse(text)
+            seen[key] ||= T.must(text).strip if key
+          end
+          seen.values
+        end
+
+        sig { params(row: Parsers::Spreadsheet::Row).returns(T.nilable(Name)) }
+        def name(row)
+          value = collapse(row[:name_of_individual_or_entity])
+          return nil if value.nil?
+
+          Name.new(value: value, kind: NAME_KINDS.fetch(row[:name_type].to_s.downcase, :aka),
+                   quality: ALIAS_QUALITIES[row[:alias_strength].to_s.downcase])
+        end
+
+        # A spelling published twice is two index entries that can only ever
+        # fire together. The first wins, so a name published as primary keeps
+        # its kind.
+        sig { params(published: T::Array[Name]).returns(T::Array[Name]) }
+        def dedupe(published)
+          seen = T.let({}, T::Hash[String, Name])
+          published.each { |name| seen[name.value] ||= name }
+          seen.values
+        end
+
+        sig { params(cell: String).returns(T::Array[PartialDate]) }
+        def read_dates(cell)
+          dates, unread = PublishedDate.dates(cell)
+          @unread_dates.concat(unread - dates.map(&:to_s))
+          dates
+        end
+
+        sig { params(line: String).returns(T.nilable(Address)) }
+        def address(line)
+          Address.new(street: collapse(line))
+        rescue ArgumentError
+          nil
+        end
+
+        sig { params(number: String).returns(T.nilable(Identifier)) }
+        def identifier(number)
+          Identifier.new(kind: :other, value: number, note: "IMO number")
+        rescue ArgumentError
+          nil
+        end
+
+        # The two columns DFAT writes prose in: the detail that identifies the
+        # person, and the account of how the listing came about. Both are the
+        # publisher's own words, so both are kept ahead of the marker.
+        sig { returns(T.nilable(String)) }
+        def published_prose
+          prose = (column(:additional_information) + column(:listing_information)).filter_map { |text| collapse(text) }
+          prose.empty? ? nil : prose.uniq.join(PARAGRAPH)
+        end
+
+        # Label/value pairs for Remarks.build, which drops the ones the record
+        # left blank.
+        sig { returns(T::Array[T.untyped]) }
+        def extras
+          EXTRA_FIELDS.map { |name, label| [label, column(name)] } +
+            [["Measures", measures], ["Date of birth, as published", @unread_dates.uniq]]
+        end
+
+        # The measures actually imposed, named rather than left as the four ones
+        # and zeroes DFAT publishes them as.
+        sig { returns(T::Array[String]) }
+        def measures
+          MEASURES.filter_map { |name, label| label if column(name).any? { |value| TRUTHY.include?(value.downcase) } }
+        end
+
+        sig { params(value: T.untyped).returns(T.nilable(String)) }
+        def collapse(value)
+          return nil if value.nil?
+
+          text = value.to_s.gsub(WHITESPACE, " ").strip
+          text.empty? ? nil : text
+        end
+      end
+    end
+  end
+end