active_sanction 1.0.0

data/lib/active_sanction/sources/canada_sema/record.rb

@@ -0,0 +1,282 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/entity"
+require "active_sanction/name"
+require "active_sanction/identifier"
+require "active_sanction/partial_date"
+require "active_sanction/parsers"
+require "active_sanction/sources/remarks"
+require "active_sanction/sources/canada_sema/source_ref"
+
+module ActiveSanction
+  module Sources
+    class CanadaSema < Base
+      # One `<record>` turned into an Entity.
+      #
+      # Separate from the adapter because they are two jobs: the adapter says
+      # what the list is and where it lives, and this says what Global Affairs'
+      # elements mean. The mapping is where all the judgment sits, so it is
+      # worth being able to read it on its own.
+      #
+      # Every element is optional and the file has no nesting, so which of the
+      # three record shapes a `<record>` is has to be read off which elements
+      # it happens to carry. See #type.
+      #
+      # @api private
+      class Record
+        extend T::Sig
+
+        COUNTRY = T.let("Country-Pays", String)
+        LAST_NAME = T.let("LastName-NomDeFamille", String)
+        GIVEN_NAME = T.let("GivenName-Prenom", String)
+        ENTITY_OR_SHIP = T.let("EntityOrShip-EntiteOuNavire", String)
+        ALIASES = T.let("Aliases-Alias", String)
+        TITLE_OR_SHIP_TYPE = T.let("TitleOrShipType-TitreOuTypeDeNavire", String)
+        IMO = T.let("ShipIMONumber-NumeroOMIDuNavire", String)
+        BORN_OR_BUILT = T.let("DateOfBirthOrShipBuildDate-DateDeNaissanceOuDateDeConstructionDuNavire", String)
+        SCHEDULE = T.let("Schedule-Annexe", String)
+        ITEM = T.let("Item-NumeroDarticle", String)
+        LISTED_ON = T.let("DateOfListing-DateDinscription", String)
+
+        # The bilingual separator, which is not one string. Countries pair on
+        # ` / ` and vessel types and titles pair on `|`, which the publisher
+        # writes with a space on neither, either or both sides and sometimes
+        # with a line break inside the half either side of it. The English half
+        # is the one before the first separator; whichever separator comes
+        # first in the string is the one that split it.
+        BILINGUAL = T.let(%r{\s*\|\s*|\s+/\s+}, Regexp)
+
+        # The one separator in the alias field that means what it looks like.
+        # See CanadaSema's class comment for why the comma is left alone.
+        ALIAS_SEPARATOR = T.let(";", String)
+
+        # Nine of the 2,598 published dates are two candidate dates rather than
+        # one -- `1972-08-26 or 1974-05-31`, `1987/1988` -- which is exactly
+        # what `Entity#dates_of_birth` is plural for. Only consulted after the
+        # whole string has failed to parse, so `1963-1964` stays the span the
+        # publisher wrote and no ISO date is ever split on its own separator.
+        ALTERNATIVES = T.let(%r{\s+or\s+|\s*/\s*}i, Regexp)
+
+        # Every element carrying a name or a name-like string arrives padded
+        # with whitespace the publisher did not mean, and 764 of those pads are
+        # U+00A0 rather than a space -- which `String#strip` does not touch, so
+        # `Premier ` would reach the matcher as a name nothing types.
+        WHITESPACE = T.let(/[[:space:]]+/, Regexp)
+
+        # For comparing two names the publisher wrote with different
+        # punctuation, which is the only thing that makes an alias a duplicate
+        # of the primary name rather than a second name.
+        INSIGNIFICANT = T.let(/[^[:alnum:]]+/, Regexp)
+
+        sig { returns(Parsers::XmlRecords::Record) }
+        attr_reader :node
+
+        # Fields of this record that could not be read. Collected by the
+        # adapter after #entity -- see CanadaSema#build.
+        sig { returns(T::Array[Parsers::Warning]) }
+        attr_reader :warnings
+
+        sig { params(node: Parsers::XmlRecords::Record).void }
+        def initialize(node)
+          @node = T.let(node, Parsers::XmlRecords::Record)
+          @warnings = T.let([], T::Array[Parsers::Warning])
+          @source_ref = T.let(nil, T.nilable(String))
+          @names = T.let(nil, T.nilable(T::Array[Name]))
+          @primary_name = T.let(nil, T.nilable(String))
+          @dates_of_birth = T.let(nil, T.nilable(T::Array[PartialDate]))
+        end
+
+        # The entity, or nil for a record with no name in any of its three name
+        # elements -- which cannot be screened against and is never what Global
+        # Affairs meant to publish.
+        sig { returns(T.nilable(Entity)) }
+        def entity
+          return nil if names.empty?
+
+          Entity.new(source: :canada_sema, source_ref: source_ref, type: type, names: names,
+                     identifiers: identifiers, dates_of_birth: dates_of_birth,
+                     programs: programs, listed_on: listed_on, remarks: remarks)
+        end
+
+        # An IMO number is only ever published for a ship, and a ship's name is
+        # published in the same element an organization's is, so the number is
+        # what separates them. The 1,445 records naming an entity with no IMO
+        # number are called organizations: some of them may be ships the
+        # publisher gave no number for, and nothing in the file says which.
+        sig { returns(Symbol) }
+        def type
+          return :vessel unless node.null?(IMO)
+
+          node.null?(ENTITY_OR_SHIP) ? :individual : :organization
+        end
+
+        # Derived, because Canada publishes no id at all. See SourceRef.
+        sig { returns(String) }
+        def source_ref
+          @source_ref ||= SourceRef.for(country: node[COUNTRY], schedule: node[SCHEDULE],
+                                        item: node[ITEM], name: primary_name)
+        end
+
+        # The primary name, then every alias the publisher separated
+        # unambiguously, minus any that is only the primary name repunctuated.
+        sig { returns(T::Array[Name]) }
+        def names
+          @names ||= build_names
+        end
+
+        # `Country-Pays` is not a nationality and must never be read as one:
+        # a Ukrainian official listed under the Special Economic Measures
+        # (Russia) Regulations is published under `Russia / Russie`. What the
+        # element names is the regulation the person is listed by, which is a
+        # sanctions program -- and for 80 of the records it says so outright,
+        # naming the Justice for Victims of Corrupt Foreign Officials
+        # Regulations rather than a country at all.
+        sig { returns(T::Array[String]) }
+        def programs = [english(node[COUNTRY])].compact
+
+        sig { returns(T.nilable(PartialDate)) }
+        def listed_on = PartialDate.parse(node[LISTED_ON])
+
+        # Empty for a vessel: the element it would come from is the ship's
+        # build date, and a hull laid down in 1980 has not got a date of birth.
+        sig { returns(T::Array[PartialDate]) }
+        def dates_of_birth
+          return [] if type == :vessel
+
+          @dates_of_birth ||= published_dates
+        end
+
+        # A ship's IMO number, which is permanent, unique and assigned by the
+        # IMO rather than by an owner -- so it is by far the most decisive
+        # thing this list publishes about anything. Filed as a registration
+        # number, which is what it is, with the note saying whose.
+        sig { returns(T::Array[Identifier]) }
+        def identifiers
+          return [] if node.null?(IMO)
+
+          [Identifier.new(kind: :registration_number, value: collapse(node[IMO]), note: "IMO number")]
+        rescue ArgumentError
+          []
+        end
+
+        # Canada publishes no free text of its own -- there is no comment or
+        # remarks element anywhere in the file -- so everything here is behind
+        # the marker, and `Remarks.published` on a Canadian record is correctly
+        # nil. What is kept is the citation the id was derived from, which is
+        # what an examiner needs to look a listing up in the Gazette, and both
+        # halves of every bilingual value verbatim.
+        sig { returns(T.nilable(String)) }
+        def remarks = Remarks.build(nil, extras)
+
+        private
+
+        sig { returns(T::Array[Name]) }
+        def build_names
+          primary = primary_name
+          return [] if primary.nil?
+
+          names = [Name.new(value: primary, kind: :primary)]
+          alias_values.each do |value|
+            next if names.any? { |name| comparable(name.value) == comparable(value) }
+
+            names << Name.new(value: value, kind: :aka)
+          end
+          names
+        end
+
+        # Canada files a person under surname then given names, which joined in
+        # that order reads "Balaba Dmitry Vladimirovich" -- not how anyone types
+        # a name into a screening form. The parts are joined the way they are
+        # spoken instead; the publisher's filing order is not lost, because the
+        # two elements are what the remark records the citation against.
+        sig { returns(T.nilable(String)) }
+        def primary_name
+          @primary_name ||= collapse(node[ENTITY_OR_SHIP]) ||
+                            collapse([node[GIVEN_NAME], node[LAST_NAME]].compact.join(" "))
+        end
+
+        sig { returns(T::Array[String]) }
+        def alias_values
+          collapse(node[ALIASES]).to_s.split(ALIAS_SEPARATOR).filter_map { |value| collapse(value) }
+        end
+
+        sig { params(value: String).returns(String) }
+        def comparable(value) = value.gsub(INSIGNIFICANT, "").downcase
+
+        # A date the whole string reads as, or the two candidates it reads as
+        # when split. Anything else is kept verbatim in the remark and warned
+        # about rather than dropped: `born in the early 1970s` is real signal
+        # that this class has no shape for.
+        sig { returns(T::Array[PartialDate]) }
+        def published_dates
+          raw = collapse(node[BORN_OR_BUILT])
+          return [] if raw.nil?
+
+          whole = PartialDate.parse(raw)
+          return [whole] unless whole.nil?
+
+          alternatives(raw)
+        end
+
+        sig { params(raw: String).returns(T::Array[PartialDate]) }
+        def alternatives(raw)
+          dates = raw.split(ALTERNATIVES).filter_map { |part| PartialDate.parse(part) }
+          note_unreadable(raw) if dates.empty?
+          dates
+        end
+
+        sig { params(raw: String).void }
+        def note_unreadable(raw)
+          @warnings << Parsers::Warning.new(
+            line: node.line,
+            message: "#{primary_name.inspect} has a date this parser does not read (#{raw.inspect}); " \
+                     "it was kept in remarks instead"
+          )
+        end
+
+        # Label/value pairs for Remarks.build, which drops the ones the record
+        # left blank. The bilingual values go in whole: the English half is
+        # what the canonical fields carry, and the remark is where the French
+        # the publisher wrote survives. Only the publisher's own line wrapping
+        # is taken out -- sixteen vessel types are published with a newline
+        # mid-phrase, and a remark is a line in a report.
+        sig { returns(T::Array[T.untyped]) }
+        def extras
+          [["Country", node[COUNTRY]], ["Schedule", node[SCHEDULE]], ["Item", node[ITEM]],
+           [title_label, node[TITLE_OR_SHIP_TYPE]], *date_extras]
+            .map { |label, value| [label, collapse(value)] }
+        end
+
+        # One element, two meanings, split by what the record is -- the same
+        # split the date element needs, for the same reason.
+        sig { returns(String) }
+        def title_label = type == :vessel ? "Vessel type" : "Title"
+
+        # A build year for a ship, and for anyone else the date string only if
+        # nothing could be read from it, so a date already in `dates_of_birth`
+        # is not also printed here.
+        sig { returns(T::Array[T.untyped]) }
+        def date_extras
+          return [["Built", node[BORN_OR_BUILT]]] if type == :vessel
+          return [] if dates_of_birth.any?
+
+          [["Date of birth", node[BORN_OR_BUILT]]]
+        end
+
+        # The English half of a value the publisher wrote in both languages,
+        # and the whole of one it wrote in only one.
+        sig { params(value: T.untyped).returns(T.nilable(String)) }
+        def english(value) = collapse(value.to_s.split(BILINGUAL, 2).first)
+
+        sig { params(value: T.untyped).returns(T.nilable(String)) }
+        def collapse(value)
+          string = value.to_s.split(WHITESPACE).join(" ")
+          string.empty? ? nil : -string
+        end
+      end
+    end
+  end
+end

data/lib/active_sanction/sources/canada_sema/source_ref.rb

@@ -0,0 +1,89 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "digest"
+
+module ActiveSanction
+  module Sources
+    class CanadaSema < Base
+      # The identity Canada does not publish, derived so that the same record
+      # in two syncs is the same record.
+      #
+      #   SourceRef.for(country: "Russia / Russie", schedule: "1, Part 1",
+      #                 item: "1114", name: "Kirill Alekseevich MORDASHOV")
+      #
+      # ### Why anything has to be derived at all
+      #
+      # Global Affairs publishes no id. What it publishes is a citation --
+      # which regulation, which schedule, which item -- and #35 diffs two syncs
+      # by comparing records under their ids, so without one every sync reports
+      # the entire list as removed and re-added, and a diff that says
+      # everything changed says nothing at all.
+      #
+      # ### What goes into it, and the one part that is not obvious
+      #
+      # The citation alone -- country, schedule, item -- is already unique
+      # across all 5,690 published records, so on the face of it the name is
+      # redundant. It is in the hash anyway, because of what happens when a
+      # schedule is amended.
+      #
+      # Item numbers are positions in a list, not identifiers: delete item 5
+      # from a schedule and everything after it moves up one. Hashing the
+      # citation alone would then hand item 6's old id to the person who used
+      # to be item 7, and the diff would report that one person quietly changed
+      # their name -- which is the same shape as a correction and reads as one.
+      # With the name in the hash, that amendment reports as a removal and an
+      # addition, which is noisier and true.
+      #
+      # The cost runs the other way: correcting a typo in a published name
+      # re-ids that record, so a spelling fix reads as one person leaving and
+      # another arriving. Churn in a diff is a nuisance; one id covering two
+      # different people is a screening failure, so the trade goes this way.
+      #
+      # ### Stability
+      #
+      # The digest is taken over the normalized parts joined by a separator
+      # that cannot occur in any of them, so an id depends on nothing but the
+      # record's own bytes -- not on iteration order, not on position in the
+      # file, not on the run. Changing NORMALIZE, SEPARATOR or LENGTH re-ids
+      # every Canadian record ever stored, which makes each of them a versioned
+      # decision rather than a cleanup.
+      #
+      # @api private
+      module SourceRef
+        extend T::Sig
+
+        # Case and the publisher's stray padding are noise: `Venezuela ` and
+        # `1, Part 1 ` are published both with and without their trailing
+        # space, and a record must not change id when a space does. Nothing
+        # further is folded -- not punctuation, not diacritics -- because every
+        # additional fold is another way for two genuinely different records to
+        # collide into one id.
+        NORMALIZE = T.let(/[[:space:]]+/, Regexp)
+
+        # A NUL cannot appear in XML character data at all, so no two different
+        # sets of field values can be re-parenthesized into each other:
+        # ("a", "bc") and ("ab", "c") hash apart.
+        SEPARATOR = T.let("\u0000", String)
+
+        # 64 bits of SHA-256. Across 5,690 records the chance of any collision
+        # at all is about one in a trillion, and an id this length stays
+        # readable in a log line and in the report that quotes it.
+        LENGTH = T.let(16, Integer)
+
+        module_function
+
+        sig { params(country: T.untyped, schedule: T.untyped, item: T.untyped, name: T.untyped).returns(String) }
+        def for(country:, schedule:, item:, name:)
+          parts = [country, schedule, item, name].map { |part| normalize(part) }
+          -T.must(Digest::SHA256.hexdigest(parts.join(SEPARATOR))[0, LENGTH])
+        end
+
+        sig { params(value: T.untyped).returns(String) }
+        def normalize(value) = value.to_s.split(NORMALIZE).join(" ").downcase
+      end
+    end
+  end
+end

data/lib/active_sanction/sources/canada_sema.rb

@@ -0,0 +1,153 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/parsers"
+require "active_sanction/sources"
+require "active_sanction/sources/base"
+
+module ActiveSanction
+  module Sources
+    # Canada's consolidated sanctions list: everyone named in a schedule to a
+    # regulation made under the Special Economic Measures Act, plus the people
+    # listed under the Justice for Victims of Corrupt Foreign Officials Act.
+    #
+    #   snapshot = ActiveSanction::Sources[:canada_sema].new.sync
+    #
+    # ### One file, one flat record shape
+    #
+    #   https://www.international.gc.ca/.../sanctions/sema-lmes.xml   2.9 MB
+    #
+    #     <data-set><record>   5,690 records
+    #
+    # Every record is the same eleven optional elements with no nesting, which
+    # makes this the simplest list the gem reads and the hardest one to give a
+    # stable identity to.
+    #
+    # ### The three things that make this source awkward
+    #
+    # **No stable identifier.** Global Affairs publishes no id of any kind.
+    # What it publishes is where in the law a person appears -- the schedule
+    # and the item number within it -- which is scoped per regulation and is
+    # renumbered whenever a schedule is amended. Record derives a deterministic
+    # synthetic id from it; see Record::SOURCE_REF_PARTS for what goes in and
+    # why the name is part of it.
+    #
+    # **Bilingual tags and values.** Element names pair English and French
+    # (`Country-Pays`), and so do the values in three of them. The separator is
+    # not one string: countries use ` / ` (`Belarus / Bélarus`) and vessel
+    # types and titles use `|` (`Oil Tanker | Navire-citerne`), sometimes with
+    # no space and sometimes wrapped across a line. Only those three elements
+    # are split; a name is never split, because `Islamic Revolutionary Guard
+    # Corps/Corps des Gardiens de la Révolution islamique` and `Victory/Pobeda
+    # Political Bloc` are the same punctuation meaning two different things and
+    # nothing in the file separates them.
+    #
+    # **The date element doubles as a ship's build date.** It is named
+    # `DateOfBirthOrShipBuildDate-...` and means whichever the record is. A
+    # build year is not a date of birth, so a vessel's goes to remarks and its
+    # `dates_of_birth` stays empty.
+    #
+    # ### What a clean Canadian result is worth
+    #
+    # Less than a clean OFAC one, and a screening policy should know it. Canada
+    # publishes no nationality, no address, no place of birth and no document
+    # number for any of the 5,690 records: an individual is a surname, given
+    # names, a date of birth roughly half the time, and a free-text alias
+    # field. There is nothing here to make a name match decisive with, which is
+    # the opposite of OFAC, where a passport number usually settles it.
+    #
+    # ### Aliases, and the comma this adapter refuses to split on
+    #
+    # 3,195 records carry an `Aliases-Alias` element, which is free text with
+    # no declared separator. Semicolons are unambiguous and are split on. Commas
+    # are not, and splitting on them manufactures names that match far too much:
+    # `Завод "Дагдизель", АО` and `М Инвест, ООО` would each yield a bare
+    # Russian legal form as an alias, and `Министерство образования, науки и
+    # молодежи Республики Крым` is one ministry, not two. So a comma-joined
+    # alias field stays one alias. That costs recall on roughly 300 records
+    # whose primary name is published anyway, and it is the cheaper of the two
+    # mistakes.
+    class CanadaSema < Base
+      extend T::Sig
+
+      key :canada_sema
+      jurisdiction :ca
+      authority "Global Affairs Canada"
+      format :xml
+
+      # The one that most often surprises people, and the reason this field
+      # exists at all. Global Affairs permits reproduction for
+      # non-commercial purposes without charge or further permission, and
+      # requires prior written permission for commercial redistribution --
+      # so a hosted service reselling this list is in a different position
+      # from an application screening against it in its own process.
+      licence_notice "Reproducible for non-commercial purposes without " \
+                     "charge or further permission, with the source " \
+                     "identified. Commercial redistribution requires prior " \
+                     "written permission from Global Affairs Canada. " \
+                     "Verified 2026-09-11."
+      licence_url "https://www.international.gc.ca/gac-amc/notices-avis.aspx?lang=eng"
+
+      url :main,
+          "https://www.international.gc.ca/world-monde/assets/office_docs/" \
+          "international_relations-relations_internationales/sanctions/sema-lmes.xml"
+
+      # @api private
+      RECORD = T.let("record", String)
+
+      # @api private
+      LIST = T.let(Parsers::XmlRecords.new(records: RECORD), Parsers::XmlRecords)
+
+      # Records that could not be used, and fields that could not be read.
+      # Read after #parse; sync orchestration (#34) reports them.
+      sig { returns(T::Array[Parsers::Warning]) }
+      attr_reader :warnings
+
+      sig { params(args: T.untyped, options: T.untyped).void }
+      def initialize(*args, **options)
+        super
+        @warnings = T.let([], T::Array[Parsers::Warning])
+        @unmapped = T.let([], T::Array[Parsers::Warning])
+      end
+
+      sig { override.params(raw: T.untyped).returns(T::Array[Entity]) }
+      def parse(raw)
+        reader = LIST.read(raw)
+        entities = build(reader)
+        @warnings = reader.warnings + @unmapped
+        entities
+      end
+
+      private
+
+      sig { params(reader: Parsers::XmlRecords::Reader).returns(T::Array[Entity]) }
+      def build(reader)
+        @unmapped = []
+        reader.filter_map do |node|
+          record = Record.new(node)
+          entity = record.entity
+          @unmapped.concat(record.warnings)
+          entity.nil? ? note_nameless(node) : entity
+        end
+      end
+
+      # A record with no name in any of its three name elements cannot be
+      # screened against. None of the 5,690 published today is nameless; the
+      # warning exists so that the day one is, it is visible rather than absent.
+      sig { params(node: Parsers::XmlRecords::Record).returns(NilClass) }
+      def note_nameless(node)
+        @unmapped << Parsers::Warning.new(
+          line: node.line,
+          message: "<#{node.name}> at item #{node[Record::ITEM].inspect} has no name and was skipped"
+        )
+        nil
+      end
+    end
+  end
+end
+
+require "active_sanction/sources/canada_sema/record"
+
+ActiveSanction::Sources.register(ActiveSanction::Sources::CanadaSema)