vicary 0.2.0

data/lib/vicary/lexicon.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "set"
+
+module Vicary
+  # Read a language-neutral word list that ships beside the gazetteer.
+  #
+  # The stoplist used to be a literal in Python's `name_candidates.py`, which was
+  # fine while there was one front door. With three, a hand-transliterated word
+  # list is a second detector wearing the first one's name: the divergence shows
+  # up as prose corruption in one language and not the others, and no parity
+  # check on *masked output* would catch it, because a stop word going missing
+  # changes what gets masked in essays nobody put in a fixture.
+  #
+  # So the list is authored once, language-neutrally, under `asset/lexicon/` in
+  # the repository, and vendored into each package the same way the gazetteer is.
+  # This is the reader; it owns nothing.
+  #
+  # Format
+  # ------
+  # `#!` lines are directives, `#` lines are comments, blank lines are skipped,
+  # and every other line contributes whitespace-separated words. One directive is
+  # required:
+  #
+  #     #!lexicon 1               format version
+  #     #!list <name> <count>     the list's name, and its DISTINCT word count
+  #
+  # The count is asserted, not trusted. A short read here makes the redactor
+  # **more** aggressive — fewer stop words means more capitalised ordinary words
+  # become name candidates — which looks privacy-safe, corrupts prose, and passes
+  # any check that only asks whether something was masked. Same reasoning as the
+  # gazetteer's per-tier counts; same failure mode if it is skipped.
+  #
+  # This is the fourth reader of this format, after `asset/vicary_build/lexicon.py`,
+  # `python/src/vicary/lexicon.py` and `typescript/src/lexicon.ts`. The duplication
+  # is deliberate — the build tool must not import one of the implementations it
+  # feeds — and it is only honest because something compares the results.
+  # `test/lexicon_test.rb` pins this reader's output against the count and the
+  # spot-checks the other three are pinned against.
+  module Lexicon
+    # On-disk format version for a lexicon file. Bump when the parse changes, so
+    # a stale vendored copy fails loudly rather than parsing to a different list.
+    LEXICON_FORMAT = 1
+
+    # Filename suffix. Named so a second list costs a file rather than a refactor.
+    SUFFIX = ".txt"
+
+    # A lexicon is absent, unreadable, or not the shape this reader understands.
+    #
+    # Its own class rather than a bare RuntimeError for the same reason
+    # {Asset::FormatError} has one: a caller that wants to tell "the install is
+    # incomplete" apart from "something else raised" cannot do it on a message.
+    class LexiconError < StandardError; end
+
+    class << self
+      # Where the vendored copy of +name+ lives, whether or not it exists.
+      #
+      # Searched along the same path as the gazetteer, in the same order, so a
+      # package cannot end up reading its stoplist from one cut and its
+      # gazetteer from another.
+      def path(name)
+        filename = "#{name}#{SUFFIX}"
+        tried = Asset.search_path
+        found = tried.find { |dir| dir.join(filename).file? }
+        return found.join(filename) if found
+
+        # Return the first location rather than raising, so `load` reports the
+        # same "missing" error whether the directory is absent or the file
+        # inside it is.
+        (tried.first || Pathname.new(".")).join(filename)
+      end
+
+      # Parse lexicon text into its case-folded distinct words.
+      #
+      # Public so a test can feed it a deliberately malformed document. A parser
+      # reachable only through a vendored file on disk is a parser whose failure
+      # paths are never exercised — and every one of this parser's failure paths
+      # exists to turn a silent short read into a loud one.
+      #
+      # @param where [String] a path, used only in error messages, so a failure
+      #   names a file.
+      def parse(name, text, where)
+        declared = nil
+        saw_format = false
+        words = Set.new
+
+        text.split(/\r?\n/, -1).each_with_index do |line, index|
+          lineno = index + 1
+          stripped = line.strip
+
+          if stripped.start_with?("#!")
+            parts = stripped[2..].to_s.split(/\s+/).reject(&:empty?)
+            raise LexiconError, "#{where}:#{lineno}: empty directive" if parts.empty?
+
+            case parts[0]
+            when "lexicon"
+              saw_format = true
+              unless parts.length == 2 && parts[1] == LEXICON_FORMAT.to_s
+                raise LexiconError,
+                      "#{where}:#{lineno}: lexicon format " \
+                      "#{parts[1..].join(' ').inspect}, this build reads " \
+                      "#{LEXICON_FORMAT}"
+              end
+            when "list"
+              unless parts.length == 3 && parts[1] == name
+                raise LexiconError,
+                      "#{where}:#{lineno}: expected `\#!list #{name} <count>`, " \
+                      "got #{stripped.inspect}"
+              end
+              # `to_i` would read "3x" as 3, where Python's `int()` raises. The
+              # difference is the whole guard: a count this reader silently
+              # repaired is a count that no longer proves anything about the parse.
+              unless /\A\d+\z/.match?(parts[2])
+                raise LexiconError,
+                      "#{where}:#{lineno}: count #{parts[2].inspect} is not an integer"
+              end
+
+              declared = parts[2].to_i
+            else
+              # Refused rather than ignored: an unrecognised directive means the
+              # file was written by something that knows more than this reader,
+              # and guessing which lines are still words is how a partial list
+              # loads as a whole one.
+              raise LexiconError, "#{where}:#{lineno}: unknown directive #{parts[0].inspect}"
+            end
+            next
+          end
+
+          next if stripped.empty? || stripped.start_with?("#")
+
+          stripped.split(/\s+/).each do |word|
+            words << word.downcase unless word.empty?
+          end
+        end
+
+        unless saw_format
+          raise LexiconError, "#{where}: no `\#!lexicon` directive; not a lexicon file"
+        end
+        raise LexiconError, "#{where}: no `\#!list #{name} <count>` directive" if declared.nil?
+
+        unless words.size == declared
+          raise LexiconError,
+                "#{where}: declares #{declared} distinct words, parsed " \
+                "#{words.size}. A short read makes the redactor more aggressive, " \
+                "which is why this is an error and not a warning."
+        end
+
+        words
+      end
+
+      # The case-folded distinct words of lexicon +name+.
+      #
+      # Raises {LexiconError} rather than returning a partial list. An empty or
+      # truncated stoplist is the quiet failure this whole module exists to
+      # prevent.
+      def load(name, path: nil)
+        location = Pathname.new(path || self.path(name))
+        begin
+          text = location.read(encoding: "UTF-8")
+        rescue Errno::ENOENT
+          raise LexiconError,
+                "lexicon #{name.inspect} missing at #{location}. The installed " \
+                "vicary gem is incomplete — reinstall it, or vendor the asset " \
+                "with `rake sync_assets` from a checkout."
+        rescue SystemCallError => e
+          raise LexiconError, "cannot read lexicon at #{location}: #{e.message}"
+        end
+
+        parse(name, text, location.to_s)
+      end
+    end
+  end
+end

data/lib/vicary/minter.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Vicary
+  # Hands out `{KIND_n}` placeholders, stable per distinct original.
+  #
+  # The Ruby port of `python/src/vicary/redaction.py`'s minter.
+  #
+  # Why numbering, stated as a measurement rather than a preference: a bare
+  # `{NAME}` standing for every person in a document is **not reversible**. On 25
+  # injected essays the unnumbered masker produced 37 not-restorable violations
+  # and only 36% of essays round-tripped — one token meant `Marisol` in one
+  # paragraph and `Terrence Okonkwo` in the next, so no map keyed on the token
+  # can put either back.
+  #
+  # Two properties, and the second is the one that needs care:
+  #
+  # * **Injective** — distinct originals never share a placeholder, which is what
+  #   makes restore well-defined.
+  # * **Stable within a document** — the *same* original always gets the same
+  #   index, so a name written five times masks to one placeholder rather than
+  #   five. That matters beyond restorability: a scoring model reading
+  #   `{NAME_1} argued … {NAME_1} concluded` can still see one person doing two
+  #   things, where `{NAME_1} … {NAME_5}` reads as two strangers.
+  #
+  # Keyed on the exact original text, because restore must return the exact
+  # bytes. `Terrence` and `Terrence's` are therefore different keys — correct but
+  # unsatisfying, and the reason surname-folding does NOT belong here: folding
+  # them together would make the mapping non-injective again.
+  #
+  # **Indices follow mint order, which is discovery order, not position in the
+  # text.** One minter serves the whole document precisely so that holds;
+  # per-pass minters would restart each counter and emit `{NAME_1}` twice for two
+  # different people, which is the bug numbering exists to remove.
+  class PlaceholderMinter
+    # Off reproduces the unnumbered output byte for byte, so the two arms stay
+    # separately measurable.
+    attr_reader :number
+
+    def initialize(number: true)
+      @number = number
+      # `[kind, original]` -> index. A tuple key rather than the joined string
+      # the other two ports use: Ruby hashes take array keys directly, so there
+      # is no separator to pick and no way for a name containing one to collide.
+      # Insertion-ordered, which is what makes {#assigned} come back in discovery
+      # order.
+      @assigned = {}
+      @high = Hash.new(0)
+    end
+
+    # The placeholder `original` should be replaced by.
+    def mint(kind, original)
+      return "{#{kind}}" unless @number
+
+      key = [kind, original]
+      index = @assigned[key]
+      if index.nil?
+        index = @high[kind] + 1
+        @high[kind] = index
+        @assigned[key] = index
+      end
+      "{#{kind}_#{index}}"
+    end
+
+    # Replace every match with a minted placeholder; returns the text and count.
+    def substitute(kind, pattern, text)
+      count = 0
+      replaced = text.gsub(pattern) do |match|
+        count += 1
+        mint(kind, match)
+      end
+      [replaced, count]
+    end
+
+    # `{placeholder => original}` — the restore map, for free.
+    #
+    # Insertion-ordered, so the map reads in the order the document discovered
+    # each span rather than in the order the placeholders sort.
+    def assigned
+      @assigned.keys.to_h { |kind, original| [mint(kind, original), original] }
+    end
+  end
+
+  # Put the originals back.
+  #
+  # Longest placeholder first, so `{NAME_1}` cannot be partially consumed while
+  # `{NAME_11}` is still pending.
+  def self.restore(text, map)
+    map.keys.sort_by { |k| -k.length }.reduce(text) do |out, placeholder|
+      # Block form, not the two-argument one: a replacement *string* interprets
+      # `\1`, `\&` and `\\`, so a restored name containing a backslash would come
+      # back altered. The block returns the original bytes untouched.
+      out.gsub(placeholder) { map[placeholder] }
+    end
+  end
+end

data/lib/vicary/redact.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Vicary
+  # Retained, and no longer raised by anything.
+  #
+  # The reason it existed still holds: the alternative to an error is a silent
+  # no-op redactor, which satisfies every caller and redacts nothing. Kept so a
+  # future incomplete arm can raise it rather than inventing a new type.
+  class NotPortedError < StandardError; end
+
+  # Whether this build detects third-party names.
+  #
+  # Exported so a host can assert on it rather than infer it from a version
+  # number. True since candidate generation landed; it is still meaningful,
+  # because {NAMES_IDENTITY} turns the whole route back off at runtime.
+  DETECTS_NAMES = true
+
+  # Only the identity the caller handed over, plus the structured entities. No
+  # gazetteer is loaded and no candidate is generated — 0% recall on third-party
+  # names, which is a defensible choice only when it is a chosen one.
+  NAMES_IDENTITY = "identity"
+  # Generation plus the offline notability oracle: the shippable arm.
+  NAMES_GAZETTEER = "gazetteer"
+  # ...and the lowercase route, the only one that reaches a student who writes
+  # without capitals.
+  NAMES_LOWERCASE = "gazetteer-lowercase"
+
+  # The default, matching the reference. Recall is what to buy inbound.
+  DEFAULT_NAME_DETECTION = NAMES_LOWERCASE
+
+  NAME_DETECTION_ENV_VAR = "VICARY_NAME_DETECTION"
+
+  IDENTITY_ALIASES = Set.new(%w[identity off none 0 false no]).freeze
+  GAZETTEER_ALIASES = Set.new(%w[gazetteer on 1 true yes names]).freeze
+  LOWERCASE_ALIASES = Set.new(%w[gazetteer-lowercase gazetteer_lowercase lowercase full max]).freeze
+
+  # The redaction entry point — the whole detector, wired end to end.
+  #
+  # Two passes over one document, in this order and for this reason:
+  #
+  # 1. **The identity and structured pass** ({Vicary::Structured}) — the
+  #    student's own name, school and school acronym, then every syntactic
+  #    entity: email, URL, SSN, IP, phone, street address, date of birth,
+  #    `@handle`, payment card behind a Luhn gate, ZIP and age. These are exact
+  #    patterns, and they run first so that no looser match can consume part of
+  #    one.
+  # 2. **Candidate generation** ({Vicary::Candidates}) — the third-party names
+  #    nothing hands over: the classmate, the teacher, the relative, the
+  #    neighbour. High recall by construction, filtered by the offline notability
+  #    oracle so the public figures a student writes *about* survive.
+  #
+  # Generation runs LAST, for the same reason it does in the reference: a broad
+  # capitalised-word match run early would swallow the first token of an address
+  # or the local part of an email, and a name half-eaten by another pattern leaks
+  # the remainder.
+  #
+  # **One minter for the whole document.** Placeholder indices follow mint order
+  # across both passes, so `{NAME_1}` means one person from the first line to the
+  # last. Two minters would restart each counter and hand the same token to two
+  # different people, which is the defect numbering exists to remove.
+  #
+  # The arm this reproduces is `local-gazetteer-lowercase` — generation, plus the
+  # gazetteer notability oracle, plus the lowercase route. That is the arm the
+  # conformance golden was produced by, and a port comparing against those bytes
+  # while implementing a different arm is measuring two changes at once.
+  class << self
+    # Resolve how hard the detector looks for names it was not handed.
+    #
+    # Explicit argument, then `VICARY_NAME_DETECTION`, then the code default.
+    #
+    # An unrecognized non-empty value resolves to the **default**, not to
+    # `identity`. Dropping silently to `identity` would leave redaction on and
+    # reporting spans while finding none of the names a reader would call PII — a
+    # failure that looks exactly like success from every log line and metric.
+    def name_detection(value = nil)
+      raw = (value || ENV[NAME_DETECTION_ENV_VAR] || "").strip.downcase
+      return NAMES_IDENTITY if !raw.empty? && IDENTITY_ALIASES.include?(raw)
+      return NAMES_GAZETTEER if GAZETTEER_ALIASES.include?(raw)
+      return NAMES_LOWERCASE if LOWERCASE_ALIASES.include?(raw)
+
+      DEFAULT_NAME_DETECTION
+    end
+
+    # Wire the bundled gazetteer into a detection level.
+    #
+    # Generation and the oracle are ONE decision, not two: generation alone masks
+    # every public figure a student writes about, and the oracle alone has
+    # nothing to judge. There is deliberately no supported way to ask for half of
+    # it.
+    #
+    # At {NAMES_IDENTITY} this returns nothing and the 2.1 MB asset is never
+    # touched. At the other two levels the first lookup pays the decompression;
+    # call `Vicary::Gazetteer.load` at process start to move that off the first
+    # request.
+    def gazetteer_oracles(level)
+      return { candidates: false } if level == NAMES_IDENTITY
+
+      oracles = {
+        candidates: true,
+        notable: ->(name) { Gazetteer.notable?(name) },
+        notability_tier: ->(name) { Gazetteer.notability(name) },
+        title: ->(name) { Gazetteer.title?(name) },
+        title_prefix: ->(key) { Gazetteer.title_prefix?(key) },
+        # Wired at BOTH gazetteer levels, unlike `given_name` below. This one
+        # decides a placeholder's type, not a verdict, so it has nothing to do
+        # with which candidate routes are on.
+        settlement: ->(name) { Gazetteer.settlement?(name) },
+      }
+      # The one difference between the two gazetteer levels. Absent rather than
+      # nil, so `gazetteer` and `gazetteer-lowercase` differ by the presence of a
+      # key rather than by a value the merge would have to strip.
+      oracles[:given_name] = ->(token) { Gazetteer.common_given_name?(token) } if level == NAMES_LOWERCASE
+      oracles
+    end
+
+    # Redact `text`, returning the masked bytes and everything needed to undo it.
+    #
+    # One minter for the whole document, because placeholder indices follow mint
+    # order across every pass.
+    #
+    # Returns `[masked_text, n_masked, restore_map]`.
+    def redact_with_report(text, identity, options = {})
+      names = options[:names]
+      keep = options[:keep] || Set.new
+      number_placeholders = options.fetch(:number_placeholders, true)
+      headings_are_orthographic = options.fetch(:headings_are_orthographic, true)
+      corroborate = options.fetch(:corroborate, true)
+      relation_refusal = options.fetch(:relation_refusal, true)
+      title_relation_refusal = options.fetch(:title_relation_refusal, true)
+
+      minter = PlaceholderMinter.new(number: number_placeholders)
+      return [text, 0, minter.assigned] if text.nil? || text.empty?
+
+      masked, n = Structured.mask(text, identity, minter)
+
+      # Candidate generation runs LAST, so every exact pattern has already
+      # claimed its span.
+      oracles = gazetteer_oracles(name_detection(names))
+      if oracles.delete(:candidates)
+        masked, count = Candidates.mask_candidates(
+          masked,
+          oracles.merge(
+            keep: keep,
+            corroborate: corroborate,
+            minter: minter,
+            headings_are_orthographic: headings_are_orthographic,
+            relation_refusal: relation_refusal,
+            title_relation_refusal: title_relation_refusal,
+          ),
+        )
+        n += count
+      end
+
+      [masked, n, minter.assigned]
+    end
+
+    # Redact personal names and structured PII from `text`.
+    #
+    # @param text [String] the composition to redact.
+    # @param identity [Object] the student the detector is told about — anything
+    #   answering `first_name`, `last_name` and `school_name`. Every reference arm
+    #   interpolates these strings, so a caller that omits them is measuring a
+    #   different system and misses the easiest spans in the fixture.
+    # @param options [Hash] the defaults are the reference arm; every flag exists
+    #   so its arm stays separately measurable.
+    def redact(text, identity, options = {})
+      redact_with_report(text, identity, options)[0]
+    end
+  end
+end