vicary 0.2.11 → 0.2.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2417f5e9aa64949da91c1ca4cf5cc1dac1b8a4bdc70929576bbc7b64d8e629f9
-  data.tar.gz: 67cd930a6714621ed7872054fac572b87a210940b1644032a845b16d2b2bbfb3
+  metadata.gz: 861bcafb7f5671834b082aea4da0edc1bdd050d71aab888d6b1944f58c70b36f
+  data.tar.gz: 31fbd6f351e16c1ff1cb0acea2f5422b94decfbeb8b0f8021e20386561cd309c
 SHA512:
-  metadata.gz: dfe4bc4a8eb71174d969e41f270eadf8c7170a97e54870fc27b8f2610a8ce2e6b976e39cba2ed87435652d8ca366ca4fb9d90795f773bd8b20db6017a3bce99b
-  data.tar.gz: 3bef064a891171f7668d8773e813c6efedea6f25385c5da72a4141279662838b623bcefe4a06de28866efdf7f53b04051fa74a02ad6f9c35acc70b9eba1b0f9a
+  metadata.gz: e5da98338e5c69f9687c1d6daf0b937837dd8d8f074768781f73f47fe789f6e99cebed77b8449b79d32118e16e738e6e45903e79b0b9ab7333acd64575de8827
+  data.tar.gz: '0148acecb49a386d9cec1b5e3832a01385820572b32f8b30f4bfbbf49c3293af70a2ceba44b93990ffc85056ddb555e80630b0e4e3da786c95aec9c8500ae4a3'

data/assets/MANIFEST.json

@@ -1,11 +1,11 @@
 {
   "assets": {
     "notability.txt.gz": {
-      "bytes": 2234918,
+      "bytes": 2291959,
       "cut_date": "2026-08-07",
-      "format": 5,
-      "min_package_version": "0.1.0",
-      "sha256": "ea882c2ab1a80bd5f3b56e32a41e9c32a314ae44389f9596ea5e7ec013d92d69",
+      "format": 6,
+      "min_package_version": "0.2.13",
+      "sha256": "e26ad0a046c5935d8ba10bfde9fd073275d7bda490cf202ea61e9f606e16e916",
       "sources": [
         "https://qlever.dev/api/wikidata",
         "https://www2.census.gov/topics/genealogy/2010surnames/names.zip"
@@ -14,6 +14,7 @@
         "demonym": 1047,
         "full": 295049,
         "given": 8138,
+        "given_corroboration": 21900,
         "place": 25444,
         "settlement": 23234,
         "short": 1229,
@@ -36,5 +37,5 @@
     }
   },
   "manifest_version": 1,
-  "written_by": "vicary 0.2.10"
+  "written_by": "vicary 0.2.13"
 }

data/lib/vicary/asset.rb

@@ -31,7 +31,7 @@ module Vicary
   # her name removed.
   module Asset
     # Asset format this reader understands. Refuse anything else.
-    SUPPORTED_FORMAT = 5
+    SUPPORTED_FORMAT = 6
 
     ASSET_FILENAME = "notability.txt.gz"
     MANIFEST_FILENAME = "MANIFEST.json"

data/lib/vicary/candidates.rb

@@ -167,6 +167,61 @@ module Vicary
     # Any word token, used to find all-caps runs and mid-sentence capitals.
     WORD_TOKEN = /[A-Za-z][A-Za-z'’-]*/
 
+    # Word-initial particles that legitimately carry an interior capital, so an
+    # interior capital on one of them is a name shape rather than orthographic
+    # noise. Closed and small, which is the whole of this signal's safety
+    # argument: `McDonald`, `MacArthur`, `DeShawn`, `DiCaprio`, `LaGrange`,
+    # `VanHalen`.
+    #
+    # The exemption is paid for in misses and the price is named here rather
+    # than discovered later — `LaTer` and `DeCide` are indistinguishable from
+    # `LaGrange` and `DeShawn` by orthography alone. It allows the particle
+    # exactly ONE capital, at the position right after it, so a second interior
+    # capital still fires; `dePenDs` is caught that way.
+    INTERIOR_CAPITAL_PREFIXES = %w[mc mac de di la le van von du da del san st].freeze
+
+    # Splits a word into the pieces an apostrophe or hyphen makes. `O'Brien` and
+    # `Jean-Luc` are two initial capitals rather than one interior capital, and
+    # without this split both read as orthographic noise — which would veto two
+    # of the commonest surname shapes there are.
+    PIECE = /[^'’\-‐-―]+/
+
+    # The word tokens {Candidates.interior_capital?} can possibly say yes to,
+    # and the only ones {Candidates.capitalises_inside_a_word?} needs to look
+    # at.
+    #
+    # It is {WORD_TOKEN} with one letter of the continuation class removed: a
+    # capital somewhere after the first character. That is a *necessary*
+    # condition for an interior capital — every position the exemptions in
+    # {Candidates.interior_capital?} still allow to fire is at piece-index 1 or
+    # later, so it is at word-index 1 or later too — which makes this a filter
+    # and never a second opinion. The answer stays that method's.
+    #
+    # The match is the same substring {WORD_TOKEN} would have produced, not a
+    # fragment of one. A token whose first capital after position 0 sits at k
+    # has every earlier character in `[a-z'’-]` by construction, so the engine
+    # matches from the token's own start, and the trailing class then runs
+    # greedily to the token's own end.
+    #
+    # Why this is worth a second pattern: the scan is per-document and the words
+    # it is looking for are rare. On the twenty-essay gate corpus exactly ONE
+    # document contains an interior capital, so the other nineteen used to be
+    # tokenised to the last word — and a regex spent on each — to return false.
+    INTERIOR_CAP_WORD = /[A-Za-z][a-z'’-]*[A-Z][A-Za-z'’-]*/
+
+    # The two-character shape {INTERIOR_CAP_WORD} cannot match without: a
+    # capital with a word character in front of it. Every capital that pattern
+    # fires on is preceded either by its own first character or by one of
+    # `[a-z'’-]`, and both are inside this class, so a document that fails
+    # this has no interior capital and needs no word scan at all.
+    #
+    # It exists because the precise pattern backtracks. `[a-z'’-]*[A-Z]` is
+    # retried at every letter of every word that has no capital after it, which
+    # is most words in most documents; this is a two-character scan that cannot
+    # backtrack. On the twenty-essay gate corpus it clears thirteen documents
+    # outright at a twentieth of the cost.
+    INTERIOR_CAP_HINT = /[A-Za-z'’-][A-Z]/
+
     # Where a sentence begins: start of text, after terminal punctuation and any
     # closing quote, after a line break, or immediately inside an *opening*
     # quote. A capital in one of these positions is required by orthography, so
@@ -1079,13 +1134,26 @@ module Vicary
       # given-name tier. `is_given` is passed in rather than defaulted so this is
       # only reachable on the path where an oracle exists.
       #
+      # **The tier is read at a lower floor here than it is for generation**,
+      # when the caller supplies `is_given_for_corroboration`. Absent, it
+      # defaults to `is_given` and this behaves exactly as it did before the dial
+      # existed. The two roles cost differently: a generation hit invents a span
+      # out of lower-case prose, a corroboration hit can only restore one the
+      # writer's own capital already proposed, and the shipped 1,800-birth knee
+      # was measured against the first. **This is the only rule that reads the
+      # lower floor** — the mid-sentence stray rule and the case-variance rule
+      # both keep the generation floor, because the corpus that priced this
+      # channel has zero true catches in the first one's suppressed set.
+      #
       # ANY token counts, not just the first, and the heading rule is what made
       # that distinction load-bearing. Before it, this was only ever reached for
       # single-token spans, so "first token" and "any token" were the same thing.
       # A heading is title-cased, so a multi-token span inside one also arrives
       # here — and "My Brother Terrence Okonkwo" leads with an honorific, so
       # checking only the first token consulted "Brother" and leaked the name.
-      def corroborated?(tokens, written_as_a_capital, is_given)
+      def corroborated?(tokens, written_as_a_capital, is_given,
+                        is_given_for_corroboration = nil)
+        vouches = is_given_for_corroboration || is_given
         # Both channels see the same stripped token, and the strip set is `.,'’`
         # rather than the `'’` {.mid_sentence_capitals} folds with. That
         # asymmetry is deliberate and was a defect once: the capital channel
@@ -1095,7 +1163,7 @@ module Vicary
         # character — asked the tier about `Terrence'` and was told no.
         tokens.each do |token|
           stripped = strip(token.downcase, ".,'’")
-          return true if written_as_a_capital.include?(stripped) || is_given.call(stripped)
+          return true if written_as_a_capital.include?(stripped) || vouches.call(stripped)
 
           # ...and again with the possessive off. "Terrence's" at a sentence
           # start is the shape this is for: the writer capitalised "Terrence"
@@ -1114,13 +1182,44 @@ module Vicary
           # reverse, so it can only reduce suppression, never increase it.
           folded = without_clitic(stripped)
           if folded != stripped &&
-             (written_as_a_capital.include?(folded) || is_given.call(folded))
+             (written_as_a_capital.include?(folded) || vouches.call(folded))
             return true
           end
         end
         false
       end
 
+      # Drop a lone capital on a word this same document also writes lower-case.
+      #
+      # The third of the capital-discounting rules, and the only one that needs
+      # neither a position nor a list. {.suppressed_as_an_unevidenced_capital?}
+      # asks whether orthography required the capital;
+      # {.suppressed_as_a_stray_mid_sentence_capital?} asks whether the writer
+      # is a sloppy capitaliser in general. This asks the narrowest question of
+      # the three and the one with the best evidence behind it: did the writer,
+      # in this document, write this exact word as a word? If they did, a
+      # capital elsewhere on the same letters is not testimony about a name.
+      #
+      # The given-name tier still rescues, exactly as it does in
+      # {.corroborated?}, and it is the reason this is safe to run outside the
+      # stray-capital gate. `Bill` in a document that also writes "bill"
+      # survives on the tier; `Summer`, `Space`, `Love` and `Fight` do not.
+      #
+      # Measured on the 56-paper NWP corpus as a post-hoc arm over the recorded
+      # spans: +7 false positives recovered, 0 public entities, 0 PII lost, and
+      # it is the only free arm that moves papers-damaged-for-nothing on its own
+      # (23 -> 20).
+      def suppressed_as_a_word_the_writer_also_writes_lower_case?(tokens, lower_cased, is_given)
+        return false unless tokens.length == 1
+
+        stripped = strip(tokens[0].downcase, ".,'’")
+        return false unless lower_cased.include?(stripped)
+        return false if is_given.call(stripped)
+
+        folded = without_clitic(stripped)
+        !(folded != stripped && is_given.call(folded))
+      end
+
       # The sentence-initial guard: drop a span whose only evidence is a capital
       # that orthography required, unless a second channel vouches for it.
       #
@@ -1143,9 +1242,11 @@ module Vicary
       # **Do not "fix" it**; the tier feeding it was the defect, and that was
       # addressed in 0.1.0 by adding SSA births to the given-name tier.
       def suppressed_as_an_unevidenced_capital?(tokens, start, starts, emphasis, headings,
-                                                written_as_a_capital, is_given)
+                                                written_as_a_capital, is_given,
+                                                is_given_for_corroboration = nil)
         capital_is_the_only_evidence?(tokens, start, starts, emphasis, headings) &&
-          !corroborated?(tokens, written_as_a_capital, is_given)
+          !corroborated?(tokens, written_as_a_capital, is_given,
+                         is_given_for_corroboration)
       end
 
       # Whether this document capitalises words that cannot be names.
@@ -1166,6 +1267,17 @@ module Vicary
       # therefore counts the names too, this cannot be satisfied by a document
       # that simply names a lot of people. Headings are excluded because title
       # case capitalises every word in one.
+      #
+      # Two channels. The stray-capital channel needs a
+      # mid-sentence capital to land on a hand-curated list, so it can only ever
+      # speak about words someone thought to curate. The second channel —
+      # {.capitalises_inside_a_word?} — needs no list, because a capital in the
+      # middle of a word is not a shape English produces under any rule.
+      #
+      # Measured on the 56-paper NWP corpus: the curated channel fires on 10
+      # papers, the interior channel on 12 (14 counting headings, which it does
+      # count), they overlap on 7, and 5 papers are reached by the interior
+      # channel alone.
       def capitalises_ordinary_words?(text, headings = [])
         count = 0
         each_match(text, MID_SENTENCE_CAP) do |m|
@@ -1178,7 +1290,98 @@ module Vicary
 
           count += 1
         end
-        count >= STRAY_CAPITALS_MIN
+        return true if count >= STRAY_CAPITALS_MIN
+
+        capitalises_inside_a_word?(text)
+      end
+
+      # A capital in a non-initial position that no name form explains.
+      #
+      # Four exemptions, and each one is a real name shape rather than a hedge:
+      # all-caps pieces (`BILL` is a writer who stopped using case, a different
+      # defect with a different rule), apostrophes and hyphens split (`O'Brien`
+      # and `Jean-Luc` are two initial capitals — see {PIECE}), particles (see
+      # {INTERIOR_CAPITAL_PREFIXES}), and single characters (`T.V` splits to `T`
+      # and `V`; an initial has no interior).
+      #
+      # What survives is `ChoaCh`, `PoSitive`, `grandParints`, `surPise` —
+      # orthographic noise, and unlike a mid-sentence capital it cannot be
+      # confused with correct English, because correct English has no such form.
+      def interior_capital?(word)
+        word.scan(PIECE).each do |piece|
+          next if piece.length < 2 || !piece.match?(/\A[A-Za-z]+\z/) || upper?(piece)
+
+          lowered = piece.downcase
+          start = 1
+          INTERIOR_CAPITAL_PREFIXES.each do |prefix|
+            next unless lowered.start_with?(prefix) && piece.length > prefix.length &&
+                        piece[prefix.length].match?(/[A-Z]/)
+
+            start = [start, prefix.length + 1].max
+          end
+          return true if piece[start..].to_s.match?(/[A-Z]/)
+        end
+        false
+      end
+
+      # Whether this document puts a capital in the middle of a word.
+      #
+      # Headings are NOT excluded here, and that is a departure with a reason.
+      # Everywhere else a heading's capitals are discounted because title case
+      # put them there. Title case does not put a capital in the *middle* of a
+      # word, so the argument does not transfer — and excluding headings anyway
+      # costs signal that is measured rather than hypothetical: on the 56-paper
+      # NWP corpus 14 papers carry an interior capital and 2 of them (`SPecial`,
+      # `AFter`) carry it only inside a heading.
+      #
+      # The scan is over {INTERIOR_CAP_WORD} rather than every word token, which
+      # is a filter and not a change of answer — see that pattern.
+      def capitalises_inside_a_word?(text)
+        return false unless INTERIOR_CAP_HINT.match?(text)
+
+        text.scan(INTERIOR_CAP_WORD) do |word|
+          return true if interior_capital?(word)
+        end
+        false
+      end
+
+      # Lower-cased forms of every word this document writes with a lower-case
+      # initial.
+      #
+      # The mirror of {.mid_sentence_capitals}: the same scan read for the
+      # opposite testimony. That method records the words a document capitalises
+      # where orthography would not have and reads them as evidence those words
+      # are names; this records the words the writer themself also wrote as
+      # words.
+      #
+      # Case-insensitively *equal*, not merely similar — no plural fold, no edit
+      # distance, no stem. The evidence is the writer's own hand on the same
+      # letters, which is what keeps the rule free of any imported collision: it
+      # consults no list, so it cannot inherit one's mistakes.
+      def written_in_lower_case(text)
+        out = Set.new
+        # `scan` rather than `each_match`, and a byte comparison rather than
+        # `token[0].match?(/[a-z]/)` — which allocated a one-character string
+        # and ran a regex for every word of every document. Exact, not an
+        # approximation: {WORD_TOKEN}'s first character is `[A-Za-z]`, so it is
+        # ASCII and a single byte by construction.
+        text.scan(WORD_TOKEN) do |token|
+          first = token.getbyte(0)
+          next unless first >= 97 && first <= 122
+
+          # {.strip} copies the string whether or not it takes anything off,
+          # and almost no word ends in an apostrophe. Same value either way — a
+          # strip that removes nothing returns an equal string, and the set
+          # holds values.
+          #
+          # Only the tail is worth testing. {.strip} works in from both ends,
+          # and the head was just established to be a lower-case ASCII letter,
+          # so the leading pass stops on the first character every time.
+          lowered = token.downcase
+          lowered = strip(lowered, "'’") if lowered.end_with?("'", "’")
+          out << lowered
+        end
+        out
       end
 
       # The mid-sentence guard: drop a lone capital a sloppy capitaliser chose.
@@ -1685,6 +1888,9 @@ module Vicary
       # Options, all optional:
       # * `:given_name` — turns on the lowercase route. Absent, this keys on
       #   capitalisation alone and misses lowercase writing by construction.
+      # * `:given_name_corroboration` — the same tier at its permissive floor,
+      #   read by the sentence-initial rule ONLY. Absent, that rule reads
+      #   `:given_name` and nothing changes. Inert without `:given_name`.
       # * `:title`, `:title_prefix` — protect work titles and fictional-character
       #   names from generation entirely. Absent, a student writing about a book
       #   has the book redacted.
@@ -1700,12 +1906,14 @@ module Vicary
       #   second half of the same rule and neither half works alone.
       def find_candidates(text, options = {})
         given_name = options[:given_name]
+        given_name_corroboration = options[:given_name_corroboration]
         title = options[:title]
         title_prefix = options[:title_prefix]
         settlement = options[:settlement]
         headings_are_orthographic = options.fetch(:headings_are_orthographic, true)
         title_relation_refusal = options.fetch(:title_relation_refusal, true)
         mid_sentence_corroboration = options.fetch(:mid_sentence_corroboration, true)
+        case_variance = options.fetch(:case_variance, true)
 
         blocked = each_match(text, PROTECTED).map { |m| [m.begin(0), m.begin(0) + m[0].length] }
         starts = sentence_starts(text)
@@ -1744,6 +1952,16 @@ module Vicary
         # `habit` is: two call sites computing it separately could disagree.
         stray_capitals = mid_sentence_corroboration && !given_name.nil? &&
                          capitalises_ordinary_words?(text, headings)
+        # The mirror of `written_as_a_capital`, read once for the same reason.
+        # Empty unless the writer marks proper nouns at all: this rule reads the
+        # ABSENCE of a capital as testimony, and the habit states in as many
+        # words that an absence means nothing in a LOWERCASE or SILENT document.
+        # Running it there would suppress every capital the writer did manage.
+        lower_cased = if case_variance && !given_name.nil? && marks_proper_nouns?(habit)
+                        written_in_lower_case(text)
+                      else
+                        Set.new
+                      end
 
         out = []
         each_match(text, CANDIDATE_RE) do |m|
@@ -1763,12 +1981,22 @@ module Vicary
 
             start = m.begin(0) + offset
             next if is_protected.call(start, start + joined.length)
+
+            # Cheapest of the three capital-discounting rules and the only one
+            # that reads neither position nor list, so it goes first.
+            if !lower_cased.empty? && !given_name.nil? &&
+               suppressed_as_a_word_the_writer_also_writes_lower_case?(run, lower_cased,
+                                                                       given_name)
+              next
+            end
+
             # Requiring a second signal is only sound when there is a second
             # signal to require, which is why this is reached only where an
             # oracle exists.
             if !given_name.nil? &&
                suppressed_as_an_unevidenced_capital?(run, start, starts, emphasis, headings,
-                                                     written_as_a_capital, given_name)
+                                                     written_as_a_capital, given_name,
+                                                     given_name_corroboration)
               next
             end
 

data/lib/vicary/gazetteer.rb

@@ -52,7 +52,8 @@ module Vicary
     # forgotten here would read back as an empty set, and an empty KEEP tier
     # redacts everything it was built to protect while presenting as
     # over-aggressive tuning.
-    TIER_NAMES = %w[full short place given title demonym settlement].freeze
+    TIER_NAMES = %w[full short place given given_corroboration title
+                    demonym settlement].freeze
 
     # Name particles that may lead a two- or three-token *partial* surname.
     #
@@ -179,7 +180,8 @@ module Vicary
     # use rather than taken as constructor arguments, because they are functions
     # of +title+ and must never be able to disagree with it.
     class Index
-      attr_reader :full, :short, :place, :given, :title, :demonym, :settlement, :meta
+      attr_reader :full, :short, :place, :given, :given_corroboration, :title,
+                  :demonym, :settlement, :meta
 
       def initialize(asset)
         asset.tiers.each_key do |name|
@@ -198,6 +200,10 @@ module Vicary
         @place = asset.tiers.fetch("place", EMPTY)
         # Common given names. The INVERSE signal — see #common_given_name?.
         @given = asset.tiers.fetch("given", EMPTY)
+        # Given names between the corroboration floor and the generation floor —
+        # the INCREMENT over +given+, never the whole set. Readers union the two;
+        # see #vouches_for_a_given_name_in_corroboration?.
+        @given_corroboration = asset.tiers.fetch("given_corroboration", EMPTY)
         # Works and fictional characters — multi-token only. See #title?.
         @title = asset.tiers.fetch("title", EMPTY)
         # English demonyms — `cuban`, `nigerian`. A KEEP, see DEMONYM.
@@ -297,6 +303,28 @@ module Vicary
         !key.empty? && !key.include?(" ") && given.include?(key)
       end
 
+      # #common_given_name?, at the permissive corroboration floor.
+      #
+      # **Strictly wider than #common_given_name? and never narrower**, because
+      # +given_corroboration+ holds the increment and this unions it with
+      # +given+ rather than replacing it. A caller reaching for this one cannot
+      # accidentally get a smaller answer than the generation tier would give,
+      # which is the failure a second independently-built list would allow.
+      #
+      # The two floors exist because the two roles cost differently. A
+      # generation hit creates a span out of lower-case prose; a corroboration
+      # hit can only un-suppress a span the writer's own capital already
+      # proposed. The shipped 1,800-birth knee was measured against the first
+      # and does not transfer to the second.
+      #
+      # Consulted by Candidates.corroborated? and by nothing else.
+      def vouches_for_a_given_name_in_corroboration?(token)
+        key = Gazetteer.normalize(token)
+        return false if key.empty? || key.include?(" ")
+
+        given.include?(key) || given_corroboration.include?(key)
+      end
+
       # True when +name+ is a town, city or village.
       #
       # **Not part of the notability decision, and deliberately not consulted by
@@ -391,6 +419,11 @@ module Vicary
         load.common_given_name?(token)
       end
 
+      # The given-name tier at its permissive, corroboration-only floor.
+      def vouches_for_a_given_name_in_corroboration?(token)
+        load.vouches_for_a_given_name_in_corroboration?(token)
+      end
+
       # True when +name+ is a town or city — a TYPING signal, not a keep.
       def settlement?(name)
         load.settlement?(name)

data/lib/vicary/redact.rb

@@ -120,7 +120,16 @@ module Vicary
       # The one difference between the two gazetteer levels. Absent rather than
       # nil, so `gazetteer` and `gazetteer-lowercase` differ by the presence of a
       # key rather than by a value the merge would have to strip.
-      oracles[:given_name] = ->(token) { Gazetteer.common_given_name?(token) } if level == NAMES_LOWERCASE
+      if level == NAMES_LOWERCASE
+        oracles[:given_name] = ->(token) { Gazetteer.common_given_name?(token) }
+        # Paired deliberately: the corroboration floor is inert without
+        # `given_name` — every rule that could read it is gated on an oracle
+        # being supplied — so wiring it at the bare level too would be a dial
+        # that looks set and does nothing, which is worse than an absent one.
+        oracles[:given_name_corroboration] = lambda { |token|
+          Gazetteer.vouches_for_a_given_name_in_corroboration?(token)
+        }
+      end
       oracles
     end
 

data/lib/vicary/version.rb

@@ -6,5 +6,5 @@ module Vicary
   # Shared across all three front doors on purpose: one detector, one number. A
   # gem 0.3.0 that corresponds to nothing on PyPI cannot be reasoned about, and
   # the parity claim is between *versions*, not between package names.
-  VERSION = "0.2.11"
+  VERSION = "0.2.14"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vicary
 version: !ruby/object:Gem::Version
-  version: 0.2.11
+  version: 0.2.14
 platform: ruby
 authors:
 - Blake Thomas
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-10 00:00:00.000000000 Z
+date: 2026-09-13 00:00:00.000000000 Z
 dependencies: []
 description: |
   Finds the names a student writes about — classmates, teachers, relatives — and