active_sanction 1.0.0

data/lib/active_sanction/normalizer/form.rb

@@ -0,0 +1,279 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/normalizer/dictionary/stoplist"
+
+module ActiveSanction
+  class Normalizer
+    # A name in both of the forms a screening decision needs: the string the
+    # publisher wrote, and the folded string a comparison actually runs
+    # against.
+    #
+    #   form = ActiveSanction::Normalizer.call("O'Brien, Seán")
+    #   form.original   # => "O'Brien, Seán"
+    #   form.value      # => "o brien sean"
+    #   form.tokens     # => ["o", "brien", "sean"]
+    #
+    #   org = ActiveSanction::Normalizer.call("Rosneft Oil Company", type: :organization)
+    #   org.value       # => "rosneft oil"
+    #   org.type        # => :organization
+    #
+    # Both halves travel together because both are needed at different ends of
+    # the same query. The scorers (#28, #29) compare `value`; the index (#31)
+    # keys on `tokens`; and what a compliance user reads in a hit is
+    # `original`, in the government's own capitalization and punctuation. A
+    # report that quotes the folded string instead is quoting this library
+    # rather than the list, which is not something anyone can take to an
+    # examiner.
+    #
+    # Instances are frozen on construction and compare by value.
+    #
+    # ### The pipeline
+    #
+    # Five stages, applied in this order to indexed names and query names
+    # alike -- see Normalizer for why that sameness is the whole point -- and
+    # a sixth that runs only for a caller who said what kind of entity the
+    # name belongs to:
+    #
+    # 1. **Unicode NFKD.** Decomposes `é` into `e` + combining acute, and folds
+    #    the compatibility forms a publisher's export tooling emits: full-width
+    #    `ＡＢＣ` becomes `ABC`, `Ⅳ` becomes `IV`, `①` becomes `1`, and the
+    #    no-break spaces scattered through the delimited lists become ordinary
+    #    ones.
+    # 2. **Strip combining marks.** What stage 1 separated is dropped, so
+    #    `Bélarus` is `belarus` and `Müller` is `muller`. Arabic gets the same
+    #    treatment for free and wants it: the harakat are optional in writing,
+    #    so one publisher's `مُحَمَّد` and another's `محمد` have to fold together, and
+    #    `أ` decomposes to a bare alef rather than staying a third spelling of
+    #    the same first letter.
+    # 3. **Casefold**, Unicode-aware: `String#downcase(:fold)` rather than
+    #    `String#downcase`, which is what turns `Straße` into `strasse` instead
+    #    of leaving a `ß` that no query will ever be typed with.
+    # 3b. **Transliterate the letters NFKD cannot help with** -- see
+    #    TRANSLITERATIONS.
+    # 4. **Punctuation to spaces**, not to nothing: `Al-Qaida` is `al qaida`
+    #    and `O'Brien` is `o brien`. Splitting is the conservative direction.
+    #    A hyphen and a space are written interchangeably across these lists,
+    #    so joining `Al-Qaida` into `alqaida` would make it unreachable from
+    #    the `al qaida` a caller types, while splitting it leaves both sides as
+    #    the same two tokens for the token ratios (#29) to work on.
+    # 5. **Collapse whitespace and strip**, which is what `tokens` is: the
+    #    folded string split on whitespace, with `value` its single-spaced
+    #    join.
+    # 6. **Drop the tokens that carry no identifying information**, given a
+    #    Stoplist: `LTD` and `COMPANY` from an organization, `SHAYKH` from a
+    #    person, nothing at all from either without one. Stage 6 is the only
+    #    one that depends on something outside the string, which is why it
+    #    arrives as an argument -- see Dictionary for what is on the lists and
+    #    for the particles they may never touch.
+    #
+    #    A name that folds away entirely keeps its unstripped tokens. An
+    #    organization called "The Company" is a poor name to screen on and a
+    #    worse one to index as the empty string, which matches everything or
+    #    nothing depending on which scorer sees it first.
+    #
+    # ### What it deliberately does not do
+    #
+    # **Non-Latin script is not transliterated.** Cyrillic, Arabic, Han, Kana
+    # and Hangul come out of here casefolded and stripped of marks, in their
+    # own script. `Путин` does not become `putin`, so a Cyrillic name matches a
+    # Cyrillic query and nothing else. That is a real recall limitation, and it
+    # is stated rather than papered over.
+    #
+    # What makes it survivable is that these lists publish a non-Latin name as
+    # an additional variant rather than instead of a Latin one -- the UN's
+    # ORIGINAL_SCRIPT aliases and Canada's Cyrillic ones both sit on records
+    # that carry a romanized name too, which is the one an English-language
+    # query finds. Romanization itself is a per-script problem with several
+    # competing standards for Cyrillic alone, and guessing at it costs
+    # precision everywhere, so v1 does not. Double Metaphone (#30) covers the
+    # case this actually leaves open, which is one name romanized two ways.
+    #
+    # One consequence worth knowing: NFKD decomposes Hangul syllables into
+    # jamo, so `김정은` folds to its letters rather than its syllable blocks.
+    # Nothing downstream cares -- both sides of a comparison are folded the
+    # same way -- but the value is not the string a Korean reader would type.
+    class Form
+      extend T::Sig
+
+      # Nonspacing and enclosing marks: the diacritics stage 1 separated from
+      # their letters. Spacing marks (`Mc`) are deliberately left alone --
+      # those are the Indic vowel signs, which are letters in every sense that
+      # matters here, and dropping them would fold `का` and `कि` onto the same
+      # consonant.
+      #
+      # @api private
+      MARKS = T.let(/[\p{Mn}\p{Me}]/, Regexp)
+
+      # The Latin letters NFKD leaves untouched, because they are letters in
+      # their own right rather than a letter wearing a mark: nothing decomposes
+      # `ø` into an `o`. Without this table `Bjørn` and `Bjorn` are two
+      # different names, which is the same false negative diacritic stripping
+      # exists to prevent -- it just happens to affect Scandinavian, Polish,
+      # Turkish, Icelandic, Vietnamese and Azerbaijani names rather than French
+      # ones.
+      #
+      # Applied after casefolding, so the table only has to carry lowercase
+      # keys. `ß` is absent because casefolding already turns it into `ss`.
+      #
+      # @api private
+      TRANSLITERATIONS = T.let(
+        {
+          "æ" => "ae", "œ" => "oe", "ø" => "o", "ð" => "d", "þ" => "th",
+          "đ" => "d", "ħ" => "h", "ı" => "i", "ł" => "l", "ŀ" => "l",
+          "ŋ" => "n", "ŧ" => "t", "ĸ" => "k", "ə" => "e"
+        }.freeze,
+        T::Hash[String, String]
+      )
+
+      # @api private
+      TRANSLITERABLE = T.let(Regexp.union(TRANSLITERATIONS.keys), Regexp)
+
+      # Everything that is not a letter, a digit or whitespace, which covers
+      # the punctuation and the symbols in one rule and needs no list of
+      # dashes and quotation marks to be kept in step with reality.
+      #
+      # Plus the characters Unicode calls letters and a transliterator uses as
+      # punctuation: the spacing modifier letters, U+02B0 to U+02FF (`ʻ ʼ ʹ ʾ
+      # ʿ`), and the two glottal stop letters (`ʔ ʕ`). The UN list writes
+      # `Sanʻa` and `Qurʼan` with these where OFAC writes a plain apostrophe or
+      # nothing at all, and a mark that survives here is a token no query will
+      # ever be typed with.
+      #
+      # @api private
+      PUNCTUATION = T.let(/[^[:alnum:][:space:]]|[ʔʕʰ-˿]/, Regexp)
+
+      # The publisher's string, untouched. This is what a hit is reported in.
+      sig { returns(String).checked(:tests) }
+      attr_reader :original
+
+      # The folded form: lowercase, unmarked, punctuation-free, single-spaced.
+      # Empty when the original carried nothing a comparison can use -- see
+      # #empty?.
+      sig { returns(String).checked(:tests) }
+      attr_reader :value
+
+      # `value` split on whitespace. Frozen, and the array the token ratios
+      # (#29) and the inverted index (#31) read rather than splitting again per
+      # comparison.
+      sig { returns(T::Array[String]).checked(:tests) }
+      attr_reader :tokens
+
+      # The entity type this name was folded for, or nil when the caller did
+      # not say. It is what decides stage 6, and it travels with the form
+      # because two folds of the same string under different types are two
+      # different answers -- which is also why `value` is part of #==.
+      sig { returns(T.nilable(Symbol)).checked(:tests) }
+      attr_reader :type
+
+      # Untyped for the reason the rest of the model is: what arrives here is a
+      # publisher's text as whatever the parser made of it. Anything that
+      # responds to `to_s` works, which includes Name -- `Name#to_s` is its
+      # value -- so an indexer can hand over the object it already has.
+      #
+      # `stoplist` is stage 6, and comes from a Dictionary rather than from
+      # here: which tokens carry no information is a property of the entity
+      # type and of the lists in force, neither of which the string knows.
+      #
+      # Building a Form directly skips the cache; Normalizer.call is the entry
+      # point everything in the library goes through, and the one that resolves
+      # a type into the stoplist for it.
+      sig { params(original: T.untyped, stoplist: T.nilable(Dictionary::Stoplist)).void.checked(:tests) }
+      def initialize(original, stoplist: nil)
+        @original = T.let(-original.to_s, String)
+        @type = T.let(stoplist&.type, T.nilable(Symbol))
+        @tokens = T.let(fold(@original, stoplist), T::Array[String])
+        @value = T.let(-@tokens.join(" "), String)
+        freeze
+      end
+
+      # True when nothing survived the fold: a name of `"---"`, of punctuation,
+      # of an emoji, or of whitespace alone. It happens in real data, and it
+      # matters because such a name cannot be indexed and cannot be scored --
+      # every comparison against it is meaningless rather than merely bad. The
+      # index (#31) skips these; the alternative is a record that matches
+      # everything or nothing depending on which scorer sees it first.
+      sig { returns(T::Boolean).checked(:tests) }
+      def empty? = value.empty?
+
+      sig { returns(String).checked(:tests) }
+      def to_s = value
+
+      # Class is part of the comparison to keep #== and #hash agreeing, which
+      # is what Hash and Set rely on -- and the index is built out of both.
+      #
+      # Two forms are equal when they came from the same original and folded
+      # to the same value. The second half is not redundant now that stage 6
+      # exists: `value` is a pure function of the original *and* the stoplist,
+      # so "Rosneft Oil Company" folded as an organization and the same string
+      # folded as nothing in particular are two different answers rather than
+      # one.
+      #
+      # Note that this makes two differently-written names that fold to the
+      # same string unequal *as forms* while comparing as identical *for
+      # matching*, which is the distinction the whole pipeline rests on.
+      sig { params(other: T.untyped).returns(T::Boolean).checked(:tests) }
+      def ==(other)
+        return false unless other.instance_of?(self.class)
+
+        original == other.original && value == other.value
+      end
+      alias eql? ==
+
+      sig { returns(Integer).checked(:tests) }
+      def hash
+        [self.class, original, value].hash
+      end
+
+      sig { returns(String) }
+      def inspect
+        "#<#{self.class} #{original.inspect} => #{value.inspect}>"
+      end
+
+      private
+
+      # The stages, in the order they have to run in: marks cannot be stripped
+      # before NFKD has separated them, the transliteration table only carries
+      # the lowercase keys casefolding produces, and the stoplist is written in
+      # the tokens the first five stages produce.
+      #
+      # Stripping every token is treated as stripping none. A name of nothing
+      # but legal forms is rare and real -- an organization called "The
+      # Company", a vessel whose only alias is its owner's suffix -- and the
+      # empty string is the one fold that cannot be scored at all.
+      sig do
+        params(string: String, stoplist: T.nilable(Dictionary::Stoplist))
+          .returns(T::Array[String]).checked(:tests)
+      end
+      def fold(string, stoplist)
+        decomposed = utf8(string).unicode_normalize(:nfkd).gsub(MARKS, "")
+        folded = decomposed.downcase(:fold).gsub(TRANSLITERABLE, TRANSLITERATIONS)
+        # `&:-@` is String#-@, the deduplicating freeze: 46,000 names share
+        # a few thousand distinct tokens between them, and the index holds
+        # onto every one of them.
+        tokens = folded.gsub(PUNCTUATION, " ").split.map(&:-@).freeze
+        return tokens if stoplist.nil?
+
+        kept = stoplist.reject(tokens)
+        kept.empty? ? tokens : kept.freeze
+      end
+
+      # `unicode_normalize` raises on a string that is not valid UTF-8, and one
+      # record's stray byte is not worth failing an entire index build over --
+      # a list that will not build is a list nobody is screened against. The
+      # bad bytes become U+FFFD, which stage 4 turns into a space.
+      #
+      # Only the folded form is repaired. `original` keeps whatever arrived,
+      # because the point of keeping it is to show a compliance user exactly
+      # what the publisher's file said.
+      sig { params(string: String).returns(String).checked(:tests) }
+      def utf8(string)
+        return string if string.encoding == Encoding::UTF_8 && string.valid_encoding?
+
+        string.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+      end
+    end
+  end
+end

data/lib/active_sanction/normalizer.rb

@@ -0,0 +1,137 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/normalizer/cache"
+require "active_sanction/normalizer/dictionary"
+require "active_sanction/normalizer/form"
+
+module ActiveSanction
+  # The one place a name is folded into the form a comparison runs against.
+  #
+  #   form = ActiveSanction::Normalizer.call("Bélarus")
+  #   form.value       # => "belarus"
+  #   form.original    # => "Bélarus"
+  #
+  #   ActiveSanction::Normalizer.call("CO., LTD.").tokens   # => ["co", "ltd"]
+  #   ActiveSanction::Normalizer.call(name)                 # a Name works too
+  #
+  #   ActiveSanction::Normalizer.call("PJSC Gazprom", type: :organization).value
+  #   # => "gazprom"
+  #
+  # Stage 1 of the matching pipeline: everything a query is compared against
+  # has been through here, and so has the query. Form documents what the fold
+  # does and why each stage is there; this class is the entry point and the
+  # cache in front of it.
+  #
+  # ### Why one entry point rather than a method on each side
+  #
+  # Because a matcher whose index and query fold differently does not fail --
+  # it silently stops matching, on exactly the records the difference touches.
+  # If the indexer strips `'` and the query path does not, `O'Brien` is
+  # unreachable from `O'Brien`, the suite still passes, and the symptom is a
+  # sanctioned person reported clean. That is the most expensive bug this
+  # library can have and it is invisible from either side alone, so there is
+  # one code path and both sides call it. `Normalizer.call` is that path.
+  #
+  # It also means normalization is a versioned decision. Changing anything in
+  # Form changes every folded string in the library at once, which is what a
+  # stored index (#31) has to be rebuilt against and what a screening decision
+  # recorded under an older gem was made under -- see MatchResult's
+  # reproducibility stamp (#33).
+  #
+  # ### Instances
+  #
+  # `Normalizer.call` runs against DEFAULT, a process-wide instance whose cache
+  # is shared and internally synchronized. An instance exists as a seam rather
+  # than for configuration: a caller that wants its own cache -- a smaller one,
+  # or one it can discard after a batch -- builds `Normalizer.new`, and the
+  # fold it gets is identical.
+  #
+  #   normalizer = ActiveSanction::Normalizer.new(cache_limit: 1_000)
+  #   normalizer.call("Al-Qaida").value    # => "al qaida"
+  #
+  # ### The dictionaries
+  #
+  # `type:` is what turns on stage 6, the pass that drops the tokens carrying
+  # no identifying information: `LTD` and `COMPANY` from an organization,
+  # `SHAYKH` from a person. It attaches here rather than to Form because the
+  # lists apply per entity type and are configurable, neither of which a string
+  # knows anything about; Dictionary is what is on them and why.
+  #
+  # Passing no type is not a lesser answer, it is a different question:
+  # "AERO-CARIBBEAN" as a bare string folds to `aero caribbean` whatever a
+  # dictionary says. A caller that has an Entity in hand should pass
+  # `entity.type`, and both sides of a comparison have to pass the same one --
+  # a query folded as an organization against an index folded as nothing is
+  # the same silent mismatch this class exists to prevent, one stage further
+  # down.
+  #
+  # A host's own lists reach the process-wide instance through configuration:
+  #
+  #   ActiveSanction.configure do |c|
+  #     c.normalizer_dictionary = { legal_forms: %w[OYJ TBK] }
+  #   end
+  #
+  # An instance can pin one instead -- `Normalizer.new(dictionary:)` -- which
+  # is what makes a fold reproducible against a dictionary that is not the one
+  # the host configured.
+  class Normalizer
+    extend T::Sig
+
+    sig { returns(Cache) }
+    attr_reader :cache
+
+    sig { params(cache_limit: Integer, dictionary: T.nilable(Dictionary)).void }
+    def initialize(cache_limit: Cache::DEFAULT_LIMIT, dictionary: nil)
+      @cache = T.let(Cache.new(limit: cache_limit), Cache)
+      @dictionary = T.let(dictionary, T.nilable(Dictionary))
+    end
+
+    # The pinned dictionary, or the configured one. Resolved per call rather
+    # than captured at construction because DEFAULT is built at load, which is
+    # before an application's initializer has run.
+    sig { returns(Dictionary) }
+    def dictionary = @dictionary || ActiveSanction.config.normalizer_dictionary
+
+    # The folded form of anything that responds to `to_s`, which is a String or
+    # a Name, optionally for a given entity type -- `:individual`,
+    # `:organization`, `:vessel`, `:aircraft`, or none.
+    #
+    # Idempotent in the sense the acceptance criterion asks for --
+    # `call(call(x).value, type:).value == call(x, type:).value` -- because the
+    # fold's output is already lowercase, unmarked, punctuation-free and
+    # single-spaced, and a stripped token cannot come back to be stripped
+    # again.
+    sig { params(value: T.untyped, type: T.nilable(Symbol)).returns(Form).checked(:tests) }
+    def call(value, type: nil)
+      string = value.to_s
+      stoplist = dictionary.stoplist(type)
+      return cache.fetch(string) { Form.new(string) } if stoplist.nil?
+
+      # The type and the lists in force are both part of the answer, so both
+      # are part of the key. The leading NUL is what keeps a composite key from
+      # colliding with the bare string of an untyped call; no name a publisher
+      # writes begins with one.
+      cache.fetch("\u0000#{stoplist.key}\u0000#{string}") { Form.new(string, stoplist: stoplist) }
+    end
+
+    class << self
+      extend T::Sig
+
+      # Delegates to DEFAULT. This is the call site everything in the library
+      # uses; see the class comment for why there is only one.
+      sig { params(value: T.untyped, type: T.nilable(Symbol)).returns(Form).checked(:tests) }
+      def call(value, type: nil) = DEFAULT.call(value, type: type)
+    end
+
+    # Built at load rather than memoized on first use, so nothing has to
+    # synchronize its construction. It holds a cache and a mutex, and reads its
+    # dictionary from the configuration on each call rather than holding one,
+    # which is why one process-wide instance is enough.
+    #
+    # @api private
+    DEFAULT = T.let(new, Normalizer)
+  end
+end

data/lib/active_sanction/parsers/column_shape.rb

@@ -0,0 +1,274 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/error"
+
+module ActiveSanction
+  module Parsers
+    # An assertion about what one column of a positional file contains,
+    # measured over the whole file.
+    #
+    #   ENT_NUM = ColumnShape.new(name: :ent_num, matches: /\A\d+\z/, at_least: 0.99)
+    #
+    #   tally = ENT_NUM.tally(rows.map { |row| row[:ent_num] })
+    #   tally.ok?       # => false
+    #   tally.ratio     # => 0.0
+    #   tally.sample    # => ["AEROCARIBBEAN AIRLINES", "AEROTAXI EJECUTIVO"]
+    #   tally.to_s      # => "ent_num numeric on 0.0% of 19321 rows (expected 99%)"
+    #
+    # ### Why a declared width is not enough
+    #
+    # A file that names its own columns cannot have them quietly swapped: the
+    # header moves with the data and an adapter reading `sdn_type` still gets
+    # the type. A headerless file has no such protection, and OFAC ships three
+    # of them. Declaring the column names pins the *width*, so a column
+    # inserted upstream arrives as a wrong-width row and every row says so --
+    # but a column *reordered* upstream keeps the width, parses cleanly, and
+    # produces 19,321 entities built from shifted fields. Nothing raises,
+    # nothing warns, and the list means something different.
+    #
+    # So the shape of the values is asserted separately from the shape of the
+    # row. `ent_num` is a number on essentially every row of OFAC's file, and
+    # a version of that file where it is a company name is not a version this
+    # library should screen against.
+    #
+    # ### `at_least`, rather than "every row"
+    #
+    # These are published files, not validated ones. A single row where a
+    # publisher typed a letter into a numeric column is a curiosity; a file
+    # where a third of them are is a format change. The threshold is what
+    # separates the two, and it defaults to 99% -- high enough that a real
+    # swap cannot hide under it, loose enough that one bad row does not stop a
+    # sync being diagnosed as healthy.
+    #
+    # Blank values are not counted at all. A column the publisher leaves empty
+    # is saying nothing about its shape, and OFAC leaves most of its columns
+    # empty most of the time -- `-0- ` roughly a quarter of a million times
+    # across the six files. Counting those as failures would make every
+    # assertion about an optional column fail on the day it was written.
+    #
+    # Instances are frozen on construction.
+    class ColumnShape
+      extend T::Sig
+
+      # @api private
+      DEFAULT_AT_LEAST = T.let(0.99, Float)
+
+      # Unmatched values kept as evidence. Enough to recognize what is in the
+      # column instead, and few enough that a whole shifted file does not
+      # arrive in a report someone has to read.
+      #
+      # @api private
+      SAMPLE_SIZE = T.let(3, Integer)
+
+      sig { returns(Symbol) }
+      attr_reader :name
+
+      # The share of non-blank values that must satisfy the rule.
+      sig { returns(Float) }
+      attr_reader :at_least
+
+      # What the column is supposed to hold, in words, for the message a
+      # failure prints: "numeric", "a known SDN_Type".
+      sig { returns(String) }
+      attr_reader :description
+
+      # One of `matches:` (a Regexp), `allowing:` (the values the column may
+      # hold, compared case-insensitively after stripping), or `satisfying:`
+      # (a callable taking the value and returning truthy). `description:`
+      # names the expectation in the failure message; the first two derive a
+      # readable one when it is not given.
+      sig do
+        params(name: T.untyped, matches: T.untyped, allowing: T.untyped, satisfying: T.untyped,
+               at_least: T.untyped, description: T.untyped).void
+      end
+      def initialize(name:, matches: nil, allowing: nil, satisfying: nil, at_least: DEFAULT_AT_LEAST,
+                     description: nil)
+        @name = T.let(name!(name), Symbol)
+        @allowed = T.let(allowing.nil? ? nil : allowed!(allowing), T.nilable(T::Array[String]))
+        @rule = T.let(rule!(matches, satisfying), T.nilable(T.proc.params(value: String).returns(T.untyped)))
+        @at_least = T.let(at_least!(at_least), Float)
+        @description = T.let(description!(description, matches), String)
+        freeze
+      end
+
+      # Whether one value satisfies the assertion. Blanks never reach here --
+      # see #tally.
+      sig { params(value: String).returns(T::Boolean) }
+      def satisfied_by?(value)
+        allowed = @allowed
+        return allowed.include?(value.strip.downcase) unless allowed.nil?
+
+        !!T.must(@rule).call(value)
+      end
+
+      # Measures the assertion over one file's worth of values, in one pass.
+      # Takes anything enumerable, so a caller can hand it a lazy reader
+      # rather than materializing 19,321 rows.
+      sig { params(values: T.untyped).returns(Tally) }
+      def tally(values)
+        checked = 0
+        matched = 0
+        blank = 0
+        sample = T.let([], T::Array[String])
+        values.each do |value|
+          string = value.nil? ? "" : value.to_s.strip
+          next blank += 1 if string.empty?
+
+          checked += 1
+          next matched += 1 if satisfied_by?(string)
+
+          sample << string if sample.size < SAMPLE_SIZE
+        end
+        Tally.new(shape: self, checked: checked, matched: matched, blank: blank, sample: sample)
+      end
+
+      sig { returns(String) }
+      def to_s = "#{name} #{description} on at least #{percentage(at_least)} of rows"
+
+      sig { returns(String) }
+      def inspect = "#<#{self.class} #{self}>"
+
+      # A percentage as a report prints one: no decimal where there is nothing
+      # after the point, since "99%" is what was declared and "99.0%" is not.
+      sig { params(ratio: Float).returns(String) }
+      def self.percentage(ratio)
+        value = (ratio * 100).round(1)
+        value == value.to_i ? "#{value.to_i}%" : "#{value}%"
+      end
+
+      sig { params(ratio: Float).returns(String) }
+      def percentage(ratio) = ColumnShape.percentage(ratio)
+
+      private
+
+      sig { params(value: T.untyped).returns(Symbol) }
+      def name!(value)
+        string = value.to_s.strip
+        raise InvalidArgument, "a column shape needs a column name" if string.empty?
+
+        string.to_sym
+      end
+
+      sig { params(value: T.untyped).returns(T::Array[String]) }
+      def allowed!(value)
+        list = Array(value).map { |entry| entry.to_s.strip.downcase }.reject(&:empty?)
+        raise InvalidArgument, "allowing: needs at least one value" if list.empty?
+
+        list.uniq.freeze
+      end
+
+      sig do
+        params(matches: T.untyped, satisfying: T.untyped)
+          .returns(T.nilable(T.proc.params(value: String).returns(T.untyped)))
+      end
+      def rule!(matches, satisfying)
+        return nil unless @allowed.nil?
+        return ->(value) { matches.match?(value) } if matches.is_a?(Regexp)
+        return ->(value) { satisfying.call(value) } if satisfying.respond_to?(:call)
+
+        raise InvalidArgument,
+              "a column shape needs one of matches: (a Regexp), allowing: (the values it may hold) " \
+              "or satisfying: (a callable)"
+      end
+
+      sig { params(value: T.untyped).returns(Float) }
+      def at_least!(value)
+        ratio = Float(value)
+        return ratio if ratio.between?(0.0, 1.0)
+
+        raise InvalidArgument, "at_least must be a share between 0 and 1, got #{value.inspect}"
+      end
+
+      sig { params(value: T.untyped, matches: T.untyped).returns(String) }
+      def description!(value, matches)
+        string = value.to_s.strip
+        return -string unless string.empty?
+        return -"one of #{@allowed.join(", ")}" unless @allowed.nil?
+
+        matches.is_a?(Regexp) ? -"matching #{matches.inspect}" : "as declared"
+      end
+
+      # What a ColumnShape measured over one file. Frozen, serializable, and
+      # deliberately keeping the values it rejected: "ent_num is numeric on 0%
+      # of rows" says a column moved, and the sample says which one moved into
+      # it.
+      #
+      # @api private
+      class Tally
+        extend T::Sig
+
+        sig { returns(ColumnShape) }
+        attr_reader :shape
+
+        # Non-blank values measured.
+        sig { returns(Integer) }
+        attr_reader :checked
+
+        sig { returns(Integer) }
+        attr_reader :matched
+
+        # Values the publisher left empty, which are not measured -- see the
+        # ColumnShape comment.
+        sig { returns(Integer) }
+        attr_reader :blank
+
+        # Up to ColumnShape::SAMPLE_SIZE of the values that did not satisfy the
+        # assertion, as evidence.
+        sig { returns(T::Array[String]) }
+        attr_reader :sample
+
+        sig do
+          params(shape: ColumnShape, checked: Integer, matched: Integer, blank: Integer,
+                 sample: T::Array[String]).void
+        end
+        def initialize(shape:, checked:, matched:, blank: 0, sample: [])
+          @shape = T.let(shape, ColumnShape)
+          @checked = T.let(checked, Integer)
+          @matched = T.let(matched, Integer)
+          @blank = T.let(blank, Integer)
+          @sample = T.let(sample.dup.freeze, T::Array[String])
+          freeze
+        end
+
+        sig { returns(Symbol) }
+        def name = shape.name
+
+        # 1.0 for a column with nothing in it to measure. A file where the
+        # column is blank on every row is a different complaint -- a fill rate
+        # that fell to zero -- and reporting it here as a shape violation would
+        # say the wrong thing about it twice.
+        sig { returns(Float) }
+        def ratio = checked.zero? ? 1.0 : matched.fdiv(checked)
+
+        sig { returns(T::Boolean) }
+        def ok? = ratio >= shape.at_least
+
+        sig { returns(T::Boolean) }
+        def failed? = !ok?
+
+        sig { returns(T::Hash[Symbol, T.untyped]) }
+        def to_h
+          { column: name, description: shape.description, checked: checked, matched: matched,
+            blank: blank, ratio: ratio.round(4), at_least: shape.at_least, sample: sample }
+        end
+
+        sig { returns(String) }
+        def to_s
+          "#{name} #{shape.description} on #{shape.percentage(ratio)} of #{checked} rows " \
+            "(expected #{shape.percentage(shape.at_least)})#{evidence}"
+        end
+
+        sig { returns(String) }
+        def inspect = "#<#{self.class} #{self}>"
+
+        private
+
+        sig { returns(String) }
+        def evidence = sample.empty? ? "" : ": #{sample.map(&:inspect).join(", ")}"
+      end
+    end
+  end
+end