vicary 0.2.15 → 0.2.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b9a6c30ad9946f8263d26b94e52b3d5d05db3d2ac0629425d992d5cebb1a7981
-  data.tar.gz: 8102ad087d3e7d1e3e4ce63a459de3d8598efb96f584e6c5ba02c815ac663611
+  metadata.gz: 381ed16c17b2aa427a72b173af92825ad08db621b13e562fdc7977d41d760a26
+  data.tar.gz: 9e7a68508af7f405888c9a0e5ebc9b7993b3f0e7de6149c91156cf7c23b9fe29
 SHA512:
-  metadata.gz: 1956a3e47dc351c62ac4a58266ed48fbf936745ed6d6cf72b2abfc35df0f50b19ae2b4f603b80248e330153c866312ade9017777cbdf0f46c211ab2abeccb1fb
-  data.tar.gz: 42fa8950725aa8c58efa07462aec67843e671eef013c211a9025655df09970ddba05c54d92c524a3f244241d8bc1d147ee8acea9bd62f01b6b3e1c6d5eed8e15
+  metadata.gz: 87ffdbf4edb505a3c1d5faca88c817a5a1ad44b093fbfc329f99958c05a528b078c3fdcc1d97c9e6ffca2d255be617cb3912d6d6e2a9735f02861600aa0f0ce6
+  data.tar.gz: ec871699f441d07106e5f32981eb9a34f35ce2935f75a7f78163fc0ae277dc1afbe8fdd042e3dbf9676b8a4493dc9fb8078c36a0e3ba9cb65804cad79b17c93b

data/README.md

@@ -45,6 +45,19 @@ replaced. A map that is absent, or that covers only some of the placeholders in
 the text, yields **no** spans and translates as the identity — a caller can
 handle "no spans" and cannot detect a wrong offset.
 
+Before a reader sees it, check that no placeholder the host did not restore
+survived — echoed by a model, or paraphrased as `[NAME_1]`, `{Name_1}` or a bare
+`NAME_1`:
+
+```ruby
+show(Vicary.assert_no_placeholders(feedback))    # raises PlaceholderLeakError
+show(Vicary.scrub_placeholders(feedback).text)   # or drop each sentence carrying one
+```
+
+The error names the kinds and the count, never the text.
+`Vicary.find_placeholders` and `Vicary.has_placeholders?` answer without raising;
+`scrub_placeholders(text, quote: true)` elides instead, for a verbatim quote.
+
 ## Checking it
 
 Three layers, because each catches what the one above it cannot.

data/lib/vicary/candidates.rb

@@ -300,6 +300,39 @@ module Vicary
       most much many few several enough
     ]).freeze
 
+    # Phonetic spellings of English stop words, each with the stop word it
+    # spells. Young writers, and young Spanish-L1 writers most of all,
+    # spell the closed-class words by ear — "dey sai ai smel", "so ai dont get
+    # mai clos dirtier" — and several of those spellings are given names the
+    # tier knows, so the lowercase route masked "sai ai" and "mai clos" and the
+    # sentence-initial rule let the tier vouch for "Ai", "Mi" and "De". A
+    # spelling of a stop word is the same word, so in a document that writes
+    # this way it is a stop word too. Only the spellings are read; the pairing
+    # is documentation. See `phonetic_function_words` for when a document
+    # qualifies.
+    PHONETIC_FUNCTION_WORDS = {
+      "ai" => "i", "becuz" => "because", "bicuz" => "because", "bot" => "but",
+      "cud" => "could", "da" => "the", "dan" => "than", "dat" => "that",
+      "de" => "the", "deir" => "their", "dem" => "them", "den" => "then",
+      "dere" => "there", "dese" => "these", "dey" => "they", "dis" => "this",
+      "dose" => "those", "frum" => "from", "hiz" => "his", "iz" => "is",
+      "mai" => "my", "mi" => "me", "sai" => "say", "sed" => "said",
+      "shud" => "should", "sum" => "some", "tu" => "to", "wat" => "what",
+      "wen" => "when", "wud" => "would", "wuz" => "was", "yu" => "you",
+      "yur" => "your",
+    }.freeze
+
+    # Distinct phonetic spellings a document must write in lower case before
+    # any of them is read as a word rather than a name. One is a coincidence —
+    # a friend called Mai — two different ones is a writer spelling by ear.
+    PHONETIC_REGISTER_MIN = 2
+
+    PHONETIC_TOKEN = Regexp.new(
+      "(?<![A-Za-z'’-])(?:" +
+        PHONETIC_FUNCTION_WORDS.keys.sort_by { |w| [-w.length, w] }.join("|") +
+        ")(?![A-Za-z'’-])"
+    )
+
     HONORIFIC_SET = Set.new(HONORIFICS.map(&:downcase)).freeze
 
     # One capitalised word, hyphens and apostrophes included so
@@ -742,6 +775,20 @@ module Vicary
         strip(without_clitic(strip(token.downcase, ".,")), "'’")
       end
 
+      # The phonetic spellings this document uses as words, to be read as
+      # stops. Empty unless the document writes at least PHONETIC_REGISTER_MIN
+      # distinct spellings in lower case. When it does, the spellings it writes
+      # in lower case — the writer's own testimony that the token is a word to
+      # them — less any it also capitalises mid-sentence, which is the opposite
+      # testimony and wins.
+      def phonetic_function_words(text, written_as_a_capital = Set.new)
+        seen = Set.new
+        each_match(text, PHONETIC_TOKEN) { |m| seen << m[0] }
+        return Set.new if seen.size < PHONETIC_REGISTER_MIN
+
+        Set.new(seen.reject { |w| written_as_a_capital.include?(w) })
+      end
+
       # Whether a capital on `token` mid-sentence is a mistake rather than a
       # construction — the narrow half of the stoplist.
       #
@@ -846,7 +893,11 @@ module Vicary
       # the stoplist so that a bare "Mrs." cannot become a candidate on its own,
       # but "Mrs. Okonkwo" has to stay whole: masking only the surname leaves the
       # relationship and the surname's position in the text.
-      def trim(tokens)
+      #
+      # `phonetic` is `phonetic_function_words` for the document: spellings read
+      # as stop words here. Except a particle *inside* a run — "Maria de Cruz"
+      # keeps its `de` however the rest of the document spells "the".
+      def trim(tokens, phonetic = Set.new)
         runs = []
         current = []
         tokens.each_with_index do |token, index|
@@ -854,7 +905,10 @@ module Vicary
             HONORIFIC_SET.include?(strip(token.downcase, ".,")) &&
             index + 1 < tokens.length &&
             !stop?(tokens[index + 1])
-          if stop?(token) && !introduces_a_name
+          spelled_stop = !phonetic.empty? && phonetic.include?(fold(token)) &&
+                         !(index.positive? && index < tokens.length - 1 &&
+                           PARTICLES.include?(token))
+          if (stop?(token) || spelled_stop) && !introduces_a_name
             unless current.empty?
               runs << current
               current = []
@@ -1849,7 +1903,7 @@ module Vicary
       #   capitalisation signal, and the seed stands on the given-name tier
       #   alone.
       def find_lowercase_candidates(text, is_given, protected_span, corroborate = nil,
-                                    settlement = nil)
+                                    settlement = nil, phonetic = Set.new)
         tokens = each_match(text, LOWER_TOKEN).map do |m|
           [m[0], m.begin(0), m.begin(0) + m[0].length]
         end
@@ -1857,7 +1911,10 @@ module Vicary
         index = 0
         while index < tokens.length
           word, start, = tokens[index]
-          if stop?(word) || !is_given.call(word)
+          # A phonetic spelling may not SEED a span ("sai ai smel"), but may
+          # still be the token a real seed reaches ("jeni sai") — see the
+          # reference.
+          if stop?(word) || phonetic.include?(word) || !is_given.call(word)
             index += 1
             next
           end
@@ -1996,6 +2053,10 @@ module Vicary
         )
         written_as_a_capital = casing.written_as_a_capital
         lower_cased = casing.written_in_lower_case
+        # Read off the same casing pass, so a spelling the writer capitalises
+        # mid-sentence is never one of them; and only where a given-name oracle
+        # exists, because it only withdraws what the tier proposed or vouched for.
+        phonetic = given_name.nil? ? Set.new : phonetic_function_words(text, written_as_a_capital)
         stray_capitals = mid_sentence_corroboration && !given_name.nil? &&
                          capitalises_ordinary_words?(text, headings)
 
@@ -2007,7 +2068,7 @@ module Vicary
           tokens = words(span)
           # A long all-caps run means the capitalisation told us nothing, so the
           # stoplist is carrying the whole decision.
-          trim(tokens).each do |run|
+          trim(tokens, phonetic).each do |run|
             next if run.empty?
 
             joined = run.join(" ")
@@ -2078,7 +2139,7 @@ module Vicary
           # evidence to offer and is better served by it.
           find_lowercase_candidates(
             text, given_name, is_protected,
-            habit == LOWERCASE ? nil : written_as_a_capital, settlement
+            habit == LOWERCASE ? nil : written_as_a_capital, settlement, phonetic
           ).each do |candidate|
             next if claimed.any? { |s, e| candidate.start < e && candidate.end > s }
 

data/lib/vicary/conformance.rb

@@ -181,6 +181,23 @@ module Vicary
         raw
       end
 
+      # The placeholder guard's contract: the pattern source, the cases and the
+      # reference's answers. Absent is an error, as for `spans.json`: a guard
+      # checked against nothing reports the same green as one that agrees.
+      def load_placeholders(dir = nil)
+        dir = Pathname.new(dir || directory)
+        path = dir.join("placeholders.json")
+        unless path.file?
+          raise SpecError,
+                "no placeholders.json at #{path}. The ports would check their " \
+                "placeholder guard against nothing."
+        end
+
+        raw = JSON.parse(path.read)
+        require_version(raw["document_version"], "placeholders.json")
+        raw
+      end
+
       # Score an implementation against every frame.
       #
       # The block receives (sentence, identity) — the same input every Python arm

data/lib/vicary/placeholders.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module Vicary
+  # Text bound for a reader still carries a redaction placeholder.
+  #
+  # Names the kinds and the count, never the text around them: the message is
+  # going to a log, and the text is exactly what was not supposed to leave.
+  class PlaceholderLeakError < ArgumentError
+    attr_reader :kinds, :count
+
+    def initialize(kinds, count)
+      @kinds = kinds.freeze
+      @count = count
+      noun = count == 1 ? "placeholder" : "placeholders"
+      super("#{count} redaction #{noun} would reach a reader " \
+            "(kinds: #{kinds.join(', ')})")
+    end
+  end
+
+  # The host boundary: assert that no placeholder reaches a reader — the Ruby
+  # port of `python/src/vicary/placeholders.py`.
+  #
+  # vicary masks what it redacts as typed placeholders (`{NAME_1}`,
+  # `{LOCATION_2}`, `{EMAIL}`) and a host restores the ones a reader may see.
+  # Everything else must never be displayed, and two ways one gets out anyway
+  # are invisible to the masker: a model echoes a placeholder the host's restore
+  # map does not cover, or paraphrases one — `[NAME_1]`, `{Name_1}`,
+  # `{ NAME_1 }`, a bare `NAME_1`.
+  #
+  # {assert_no_placeholders} raises before the text is shown;
+  # {scrub_placeholders} removes them (a prose sentence carrying one is dropped,
+  # a quote elides it with `…`). Nothing is rewritten back into a word here.
+  #
+  # The pattern source is byte-identical in all three front doors, written in
+  # the dialect the engines share — ASCII classes rather than `\b`/`\w`/`\s`,
+  # and `\A`/`\z` rather than `^`/`$`, which anchor per line in Ruby.
+  # `test/placeholders_test.rb` compares it with `conformance/placeholders.json`.
+  module Placeholders
+    KINDS = %w[
+      ADDRESS AGE AWS_ACCESS_KEY AWS_SECRET_KEY
+      CREDIT_DEBIT_CARD_CVV CREDIT_DEBIT_CARD_EXPIRY
+      CREDIT_DEBIT_CARD_NUMBER DATE_OF_BIRTH DRIVER_ID EMAIL
+      IP_ADDRESS LICENSE_PLATE LOCATION MAC_ADDRESS NAME
+      ORGANIZATION PASSWORD PERSON PHONE PIN SCHOOL URL
+      US_BANK_ACCOUNT_NUMBER US_BANK_ROUTING_NUMBER US_PASSPORT_NUMBER
+      US_SOCIAL_SECURITY_NUMBER USERNAME ZIP_CODE
+    ].freeze
+    private_constant :KINDS
+
+    # Every placeholder kind vicary or its Guardrail arm can mint, plus
+    # `PERSON`, which is what a model writes when it paraphrases `NAME`.
+    # Longest first, so a match reports the whole kind.
+    PLACEHOLDER_KINDS = KINDS.sort_by { |kind| [-kind.length, kind] }.freeze
+
+    # The kind reported for a `{UPPER_CASE}` token that names no known kind.
+    OTHER_KIND = "OTHER"
+
+    WORD = "[A-Za-z0-9_]"
+    private_constant :WORD
+
+    # `NAME` -> `[Nn][Aa][Mm][Ee]`: case-insensitive in every dialect.
+    def self.fold(kind)
+      kind.chars.map { |c| c.match?(/[A-Z]/) ? "[#{c}#{c.downcase}]" : c }.join
+    end
+    private_class_method :fold
+
+    # Anything placeholder-shaped: vicary's own `{KIND}` / `{KIND_n}` for any
+    # upper-case kind; a known kind in any case inside `{}`, `[]` or `<>` with
+    # stray spaces or tabs; or a bare upper-case `KIND_n` as a whole word.
+    PLACEHOLDER_LIKE = Regexp.new(
+      "\\{[A-Z][A-Z_]*(?:_[0-9]+)?\\}" \
+      "|[{\\[<][ \\t]*(?:#{PLACEHOLDER_KINDS.map { |k| fold(k) }.join('|')})" \
+      "(?:_[0-9]+)?[ \\t]*[}\\]>]" \
+      "|(?<!#{WORD})(?:#{PLACEHOLDER_KINDS.join('|')})_[0-9]+(?!#{WORD})"
+    )
+
+    # A sentence ends at `.`, `!` or `?` followed by whitespace — but not after
+    # an honorific, an initial, `e.g.`, `i.e.` or `vs.`.
+    SENTENCE_END = Regexp.new(
+      "(?<![A-Z][a-z]\\.)" \
+      "(?<!(?<!#{WORD})[A-Z][a-z][a-z]\\.)" \
+      "(?<!(?<!#{WORD})[A-Z]\\.)" \
+      "(?<!(?<!#{WORD})e\\.g\\.)" \
+      "(?<!(?<!#{WORD})i\\.e\\.)" \
+      "(?<!(?<!#{WORD})vs\\.)" \
+      "(?<=[.!?])[ \\t\\n\\r\\f\\v]+"
+    )
+
+    # What stands in for a placeholder when every sentence of a prose field
+    # carries one. Any other kind reads "this".
+    GENERIC_STAND_INS = {
+      "NAME" => "someone",
+      "PERSON" => "someone",
+      "LOCATION" => "a place",
+      "SCHOOL" => "the school",
+      "ORGANIZATION" => "an organization"
+    }.freeze
+
+    TRIM = /\A[ \t\n\r\f\v]+|[ \t\n\r\f\v]+\z/
+    TRIM_TOKEN = /\A[{}\[\]<> \t\n\r\f\v]+|[{}\[\]<> \t\n\r\f\v]+\z/
+    ELISION = /[ \t\n\r\f\v]*…[ \t\n\r\f\v]*/
+    ELISION_BEFORE_PUNCT = / … ([.,;:!?])/
+    NUMBER_SUFFIX = /_[0-9]+\z/
+    private_constant :TRIM, :TRIM_TOKEN, :ELISION, :ELISION_BEFORE_PUNCT,
+                     :NUMBER_SUFFIX
+
+    # One placeholder found in text. `start`/`end` count characters.
+    Match = Struct.new(:text, :kind, :start, :end) do
+      def to_h
+        { "text" => text, "kind" => kind, "start" => start, "end" => self.end }
+      end
+    end
+
+    # Text with every placeholder removed, and how many were.
+    ScrubResult = Struct.new(:text, :removed)
+
+    class << self
+      # The kind a placeholder token names: `[ Name_2 ]` -> `NAME`; else
+      # {OTHER_KIND}.
+      def kind_of(token)
+        kind = token.gsub(TRIM_TOKEN, "").upcase.sub(NUMBER_SUFFIX, "")
+        PLACEHOLDER_KINDS.include?(kind) ? kind : OTHER_KIND
+      end
+
+      # Every placeholder in `text`, in order.
+      def find(text)
+        return [] if text.nil? || text.empty?
+
+        text.to_enum(:scan, PLACEHOLDER_LIKE).map do
+          m = Regexp.last_match
+          Match.new(m[0], kind_of(m[0]), m.begin(0), m.end(0))
+        end
+      end
+
+      # Whether `text` carries any placeholder.
+      def any?(text)
+        !text.nil? && !text.empty? && PLACEHOLDER_LIKE.match?(text)
+      end
+
+      # Return `text` unchanged, or raise {Vicary::PlaceholderLeakError}. Call
+      # it on the last string a host builds before a reader sees it.
+      def assert_none(text)
+        found = find(text)
+        unless found.empty?
+          raise PlaceholderLeakError.new(found.map(&:kind).uniq.sort, found.length)
+        end
+
+        text
+      end
+
+      # Remove every placeholder from `text`. Prose (the default) drops each
+      # sentence carrying one, or — when every sentence does — replaces each
+      # with its {GENERIC_STAND_INS} entry; `quote: true` elides each as `…`.
+      def scrub(text, quote: false)
+        return ScrubResult.new(text, 0) unless any?(text)
+
+        removed = text.scan(PLACEHOLDER_LIKE).length
+        if quote
+          out = text.gsub(PLACEHOLDER_LIKE, "…").gsub(ELISION, " … ")
+                    .gsub(ELISION_BEFORE_PUNCT, ' …\1').gsub(TRIM, "")
+          return ScrubResult.new(out, removed)
+        end
+
+        kept = text.gsub(TRIM, "").split(SENTENCE_END).reject { |s| any?(s) }
+        return ScrubResult.new(kept.join(" "), removed) unless kept.empty?
+
+        out = text.gsub(PLACEHOLDER_LIKE) do |token|
+          GENERIC_STAND_INS.fetch(kind_of(token), "this")
+        end
+        ScrubResult.new(out, removed)
+      end
+    end
+  end
+
+  class << self
+    # Every placeholder in `text` — see {Vicary::Placeholders}.
+    def find_placeholders(text)
+      Placeholders.find(text)
+    end
+
+    # Whether `text` carries any placeholder.
+    def has_placeholders?(text)
+      Placeholders.any?(text)
+    end
+
+    # Return `text`, or raise {Vicary::PlaceholderLeakError}.
+    def assert_no_placeholders(text)
+      Placeholders.assert_none(text)
+    end
+
+    # Remove every placeholder from `text`.
+    def scrub_placeholders(text, quote: false)
+      Placeholders.scrub(text, quote: quote)
+    end
+  end
+end

data/lib/vicary/version.rb

@@ -6,5 +6,5 @@ module Vicary
   # Shared across all three front doors on purpose: one detector, one number. A
   # gem 0.3.0 that corresponds to nothing on PyPI cannot be reasoned about, and
   # the parity claim is between *versions*, not between package names.
-  VERSION = "0.2.15"
+  VERSION = "0.2.17"
 end

data/lib/vicary.rb

@@ -44,3 +44,4 @@ require_relative "vicary/corpus"
 require_relative "vicary/latency_baseline"
 require_relative "vicary/redact"
 require_relative "vicary/spans"
+require_relative "vicary/placeholders"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vicary
 version: !ruby/object:Gem::Version
-  version: 0.2.15
+  version: 0.2.17
 platform: ruby
 authors:
 - Blake Thomas
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-13 00:00:00.000000000 Z
+date: 2026-10-21 00:00:00.000000000 Z
 dependencies: []
 description: |
   Finds the names a student writes about — classmates, teachers, relatives — and
@@ -38,6 +38,7 @@ files:
 - lib/vicary/latency_baseline.rb
 - lib/vicary/lexicon.rb
 - lib/vicary/minter.rb
+- lib/vicary/placeholders.rb
 - lib/vicary/redact.rb
 - lib/vicary/spans.rb
 - lib/vicary/structured.rb