vicary 0.2.0 → 0.2.4

data/lib/vicary/latency_baseline.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require "json"
+require "pathname"
+
+module Vicary
+  # Is this build slower than the last release, and is that a fair question here?
+  #
+  # The latency gate used to hold an absolute number — 10 ms — which is a claim
+  # about the machine as much as about the code. It passed on a laptop and failed
+  # on the CI runner enforcing it, so v0.2.3 published to PyPI and npm and was
+  # refused by RubyGems on the same commit. This gem is the one that caught it.
+  #
+  # What replaced it asks a relative question: is this port slower than it was at
+  # the last release, by more than the tolerance. That only means something
+  # between measurements taken on comparable hardware, so this module's real work
+  # is REFUSING to compare when they are not — a machine difference reported as a
+  # code regression is worse than no gate, because it trains the reader to ignore
+  # it.
+  #
+  # This port reaches its own verdict from the shared file. It does not read
+  # Python's answer.
+  module LatencyBaseline
+    BASELINE_FILENAME = "latency_baseline.json"
+
+    # Set by CI on the one matrix entry whose language version matches the
+    # recorded profile. Absent everywhere else on purpose: a developer's laptop
+    # measures the same commit two to three times faster than the runner, and
+    # comparing that against a runner baseline reports a phantom improvement.
+    PROFILE_ENV_VAR = "VICARY_LATENCY_PROFILE"
+
+    IMPLEMENTATION = "ruby"
+
+    DEFAULT_TOLERANCE_PCT = 8.0
+
+    # The gate's answer, and — when it declines — why.
+    Comparison = Struct.new(
+      :measured_ms, :baseline_ms, :regression_pct, :tolerance_pct,
+      :comparable, :reason,
+      keyword_init: true
+    ) do
+      def holds?
+        return false unless comparable && !regression_pct.nil?
+
+        regression_pct <= tolerance_pct
+      end
+    end
+
+    class << self
+      def baseline_path(dir = nil)
+        root = dir || Conformance.directory
+        return nil if root.nil?
+
+        path = Pathname.new(root).join(BASELINE_FILENAME)
+        path.exist? ? path : nil
+      end
+
+      def load(dir = nil)
+        path = baseline_path(dir)
+        return nil if path.nil?
+
+        JSON.parse(path.read)
+      end
+
+      # `major.minor` of the running Ruby, matching how the profile records it.
+      def language_version
+        RUBY_VERSION.split(".").first(2).join(".")
+      end
+
+      # Compare +measured_ms+ against the recorded baseline for this port.
+      #
+      # Every reason below is a refusal to compare, not a failure to measure: the
+      # number was measured either way and is reported either way. What is
+      # withheld is the verdict, because the two sides would not be like for like.
+      def compare(measured_ms, corpus_id, dir: nil, implementation: IMPLEMENTATION,
+                  observed_language_version: nil, profile_env: nil)
+        doc = load(dir)
+        tolerance = (doc && doc["tolerance_pct"] || DEFAULT_TOLERANCE_PCT).to_f
+        lang = observed_language_version || language_version
+
+        declined = lambda do |reason, baseline_ms = nil|
+          Comparison.new(measured_ms: measured_ms, baseline_ms: baseline_ms,
+                         regression_pct: nil, tolerance_pct: tolerance,
+                         comparable: false, reason: reason)
+        end
+
+        return declined.call("no #{BASELINE_FILENAME} in this checkout") if doc.nil?
+
+        profile = doc["profile"] || {}
+        want_profile = profile["id"]
+        have_profile = (profile_env || ENV[PROFILE_ENV_VAR] || "").strip
+        if have_profile.empty?
+          return declined.call(
+            "#{PROFILE_ENV_VAR} is unset, so this machine does not claim to be " \
+            "#{want_profile.inspect}; the baseline was recorded there"
+          )
+        end
+        unless have_profile == want_profile
+          return declined.call(
+            "#{PROFILE_ENV_VAR}=#{have_profile.inspect} but the baseline was " \
+            "recorded on #{want_profile.inspect}"
+          )
+        end
+
+        want_lang = (profile["language_versions"] || {})[implementation]
+        if !want_lang.nil? && want_lang.to_s != lang
+          return declined.call(
+            "#{implementation} #{lang} is not the #{want_lang} the baseline was " \
+            "recorded on; interpreter versions differ by more than the bar"
+          )
+        end
+
+        want_corpus = doc["corpus"]
+        if !want_corpus.nil? && want_corpus != corpus_id
+          return declined.call(
+            "corpus #{corpus_id.inspect} is not the #{want_corpus.inspect} the " \
+            "baseline was recorded on; latency scales with essay length"
+          )
+        end
+
+        entry = (doc["implementations"] || {})[implementation] || {}
+        recorded = entry["pooled_median_ms"]
+        if recorded.nil?
+          return declined.call(
+            "no baseline recorded for #{implementation} yet — the next release " \
+            "records one"
+          )
+        end
+
+        recorded = recorded.to_f
+        if recorded <= 0
+          return declined.call(
+            "recorded baseline for #{implementation} is not positive", recorded
+          )
+        end
+
+        Comparison.new(
+          measured_ms: measured_ms, baseline_ms: recorded,
+          regression_pct: (measured_ms / recorded - 1.0) * 100.0,
+          tolerance_pct: tolerance, comparable: true, reason: nil
+        )
+      end
+
+      def render(comparison)
+        c = comparison
+        unless c.comparable
+          return format("latency %.3f ms — NOT COMPARED against the last release: %s",
+                        c.measured_ms, c.reason)
+        end
+
+        sign = c.regression_pct >= 0 ? "+" : ""
+        format("latency %.3f ms vs %.3f ms at the last release — %s%.2f%% " \
+               "against a %d%% bar",
+               c.measured_ms, c.baseline_ms, sign, c.regression_pct, c.tolerance_pct)
+      end
+
+      # The keyword arguments Gates.measure wants. Returns the *detail* rather
+      # than a value when the comparison was declined, so the gate reports NOT
+      # MEASURED with the reason attached instead of quietly passing.
+      def gate_fields(measured_ms, corpus_id, **opts)
+        c = compare(measured_ms, corpus_id, **opts)
+        if c.comparable
+          { latency_regression_pct: c.regression_pct }
+        else
+          { latency_regression_detail: render(c) }
+        end
+      end
+    end
+  end
+end

data/lib/vicary/structured.rb

@@ -19,11 +19,12 @@ module Vicary
   #
   # **Order is the contract, not an optimisation.** The first pattern to claim a
   # span wins, and placeholder indices follow mint order, so reordering these
-  # tables changes the output bytes even when it changes no verdict. Identity
-  # runs first (an address line can otherwise swallow a surname); EMAIL before
-  # PHONE; SSN and CARD before the generic digit runs; ZIP and AGE last, because
-  # both are bare digits and would claim characters belonging to a phone, card or
-  # address.
+  # tables changes the output bytes even when it changes no verdict. EMAIL and URL
+  # run first, because a school-issued address and a profile URL *contain* the
+  # writer's name and both are anchored too tightly to take one out of prose;
+  # identity next (an address line can otherwise swallow a surname); SSN and CARD
+  # before the generic digit runs; ZIP and AGE last, because both are bare digits
+  # and would claim characters belonging to a phone, card or address.
   #
   # ## Regex dialect
   #
@@ -141,13 +142,34 @@ module Vicary
     # Date of birth, explicitly labelled.
     DOB = %r{\b(?:date\s+of\s+birth|d\.?o\.?b\.?|born\s+on)\s*:?\s*\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b}i
 
-    # (placeholder kind, pattern) in application order.
+    # The two structured patterns that run BEFORE identity interpolation, because
+    # their match text can legitimately *contain* the writer's own name:
+    # `first.last@district.org` and a profile URL ending in a name slug. Identity
+    # interpolation is a literal-name substitution, so running it first left these
+    # shredded rather than masked — `{NAME_2}.{NAME_1}{USERNAME_1}.k12.oh.us`
+    # instead of `{EMAIL_1}`, with the domain tail surviving in the clear and the
+    # span unrestorable on the round trip.
+    #
+    # Putting them first is safe in the direction that matters, and that asymmetry
+    # is the whole argument. Both are anchored on structure a name cannot supply —
+    # EMAIL needs an `@` and a dotted TLD, URL needs a scheme or a `www.` — so
+    # neither can reach into prose and take a bare surname out of it. Patterns
+    # that *could* still run after identity.
+    STRUCTURED_BEFORE_IDENTITY = [
+      ["EMAIL", EMAIL],
+      ["URL", URL_PATTERN],
+    ].freeze
+
+    # (placeholder kind, pattern) in application order, running AFTER identity
+    # interpolation.
     #
     # CARD is handled separately because it needs the Luhn gate; ZIP and AGE run
     # after it for the reason in the module docstring.
+    #
+    # Numbering is unaffected by which of these two tables a pattern sits in: the
+    # minter counts per kind, so `{EMAIL_1}` is the first email whether emails are
+    # matched before or after names.
     STRUCTURED = [
-      ["EMAIL", EMAIL],
-      ["URL", URL_PATTERN],
       ["US_SOCIAL_SECURITY_NUMBER", SSN],
       ["IP_ADDRESS", IP],
       ["PHONE", PHONE],
@@ -182,6 +204,29 @@ module Vicary
     # Delacroix-Whitfields' house").
     POSSESSIVE_TAIL = "(?:['’]s|s['’])?"
 
+    # Is this single character a word character? Used to decide whether a literal
+    # needs a boundary lookaround on each end.
+    #
+    # A constant because it was previously built inside {literal_boundaries},
+    # which runs five times per redaction: at 50 redactions of one identity that
+    # was 250 regex compilations of a pattern that never varies.
+    #
+    # `Regexp#initialize` was 26% of this port's redaction CPU before this and
+    # the identity-pattern cache below; the fold cache in `gazetteer.rb` took the
+    # allocation half. Over ten runs of the 25-essay corpus gate, median p50 went
+    # 5.79 -> 3.10 ms and median p95 8.46 -> 4.43 ms.
+    #
+    # The number that mattered is the worst run, not the median: p95 ranged
+    # 7.75-11.35 ms before, so the 10 ms latency gate was failing outright about
+    # one run in ten and being read as a busy machine. It ranges 4.13-8.63 ms now.
+    # Three samples could not see that — the tail is one essay plus a GC pause,
+    # and it took ten runs per arm to separate the fix from the noise.
+    WORD_CHARACTER = /\A[#{W}]\z/.freeze
+
+    # How many identities' compiled patterns to keep. Small on purpose: the shape
+    # this serves is one student's essays in a row, not a working set.
+    IDENTITY_CACHE_MAX = 64
+
     class << self
       # Luhn checksum. Cuts the card pattern's false positives on long numbers.
       def luhn_ok?(digits)
@@ -210,10 +255,9 @@ module Vicary
       # ASCII-only where Python's is Unicode-aware; these agree with Python for
       # an accented name.
       def literal_boundaries(literal)
-        word = /\A[#{W}]\z/
         [
-          literal[0].to_s.match?(word) ? "(?<![#{W}])" : "",
-          literal[-1].to_s.match?(word) ? "(?![#{W}])" : "",
+          literal[0].to_s.match?(WORD_CHARACTER) ? "(?<![#{W}])" : "",
+          literal[-1].to_s.match?(WORD_CHARACTER) ? "(?![#{W}])" : "",
         ]
       end
 
@@ -242,10 +286,39 @@ module Vicary
       # of it, so "Jane Quincy-Adams" becomes one `{NAME}` rather than two
       # adjacent placeholders.
       def identity_patterns(identity)
-        out = []
         first = identity_field(identity, :first_name)
         last = identity_field(identity, :last_name)
         school = identity_field(identity, :school_name)
+        extras = extra_names(identity).map { |raw| raw.to_s.strip }
+
+        # Keyed on the field VALUES, never on the identity object: a host that
+        # reuses one mutable struct per request would otherwise get the previous
+        # student's patterns, which is a privacy failure rather than a stale
+        # cache. Two identities with the same fields produce the same patterns by
+        # construction, so sharing an entry between them is exact.
+        key = [first, last, school, extras].freeze
+        cached = @identity_patterns_cache&.[](key)
+        return cached if cached
+
+        patterns = build_identity_patterns(first, last, school, extras)
+
+        # Bounded, and cleared wholesale rather than evicted one at a time. The
+        # win is a batch redacting many essays for ONE student, where the cache
+        # holds a single entry; a long-running host cycling through thousands
+        # gets the bound instead of a leak, and refilling it costs what building
+        # the patterns cost before this existed.
+        @identity_patterns_cache ||= {}
+        @identity_patterns_cache.clear if @identity_patterns_cache.size >= IDENTITY_CACHE_MAX
+        @identity_patterns_cache[key] = patterns
+      end
+
+      # Drop the memoized identity patterns. For tests that measure the build.
+      def reset_identity_cache
+        @identity_patterns_cache = nil
+      end
+
+      def build_identity_patterns(first, last, school, extras)
+        out = []
 
         if !first.empty? && !last.empty?
           out << ["NAME", word_pattern("#{first} #{last}")]
@@ -255,8 +328,7 @@ module Vicary
         out << ["NAME", word_pattern(last)] if !last.empty? && !AMBIGUOUS_SURNAMES.include?(last.downcase)
         out << ["NAME", word_pattern(first)] if !first.empty? && !AMBIGUOUS_GIVEN_NAMES.include?(first.downcase)
 
-        extra_names(identity).each do |raw|
-          extra = raw.to_s.strip
+        extras.each do |extra|
           out << ["NAME", word_pattern(extra)] unless extra.empty?
         end
 
@@ -286,10 +358,15 @@ module Vicary
         masked = text
         n = 0
 
-        # Identity patterns run FIRST: a name is the span most likely to be
+        # Identity patterns run early: a name is the span most likely to be
         # partially consumed by a looser pattern (an address line can swallow a
         # surname), and masking it first makes that impossible.
-        (identity_patterns(identity) + STRUCTURED).each do |kind, pattern|
+        #
+        # Early, not first. Email and URL precede it, because those two are the
+        # patterns whose own match text contains a name — see
+        # STRUCTURED_BEFORE_IDENTITY for why that direction is the safe one.
+        patterns = STRUCTURED_BEFORE_IDENTITY + identity_patterns(identity) + STRUCTURED
+        patterns.each do |kind, pattern|
           masked, count = minter.substitute(kind, pattern, masked)
           n += count
         end

data/lib/vicary/version.rb

@@ -6,5 +6,5 @@ module Vicary
   # Shared across all three front doors on purpose: one detector, one number. A
   # gem 0.3.0 that corresponds to nothing on PyPI cannot be reasoned about, and
   # the parity claim is between *versions*, not between package names.
-  VERSION = "0.2.0"
+  VERSION = "0.2.4"
 end

data/lib/vicary.rb

@@ -17,7 +17,7 @@
 #
 # Three layers check that claim, and each catches what the one above it cannot:
 #
-# * `rake conformance` scores the 52 frames — the final bar, and a coarse first
+# * `rake conformance` scores the 54 frames — the final bar, and a coarse first
 #   one;
 # * `rake test` runs `test/primitives_test.rb`, forty-odd primitives over the
 #   shared `primitives.json` corpus, which says *which brick is crooked*;
@@ -38,4 +38,8 @@ require_relative "vicary/minter"
 require_relative "vicary/structured"
 require_relative "vicary/candidates"
 require_relative "vicary/conformance"
+require_relative "vicary/gates"
+require_relative "vicary/census"
+require_relative "vicary/corpus"
+require_relative "vicary/latency_baseline"
 require_relative "vicary/redact"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vicary
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.4
 platform: ruby
 authors:
 - Blake Thomas
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-08-11 00:00:00.000000000 Z
+date: 2026-08-13 00:00:00.000000000 Z
 dependencies: []
 description: |
   Finds the names a student writes about — classmates, teachers, relatives — and
@@ -29,8 +29,12 @@ files:
 - lib/vicary.rb
 - lib/vicary/asset.rb
 - lib/vicary/candidates.rb
+- lib/vicary/census.rb
 - lib/vicary/conformance.rb
+- lib/vicary/corpus.rb
+- lib/vicary/gates.rb
 - lib/vicary/gazetteer.rb
+- lib/vicary/latency_baseline.rb
 - lib/vicary/lexicon.rb
 - lib/vicary/minter.rb
 - lib/vicary/redact.rb
@@ -41,7 +45,6 @@ licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/bwthomas/vicary
-  source_code_uri: https://github.com/bwthomas/vicary
   changelog_uri: https://github.com/bwthomas/vicary/blob/main/CHANGELOG.md
   bug_tracker_uri: https://github.com/bwthomas/vicary/issues
 post_install_message: