vicary 0.2.16 → 0.2.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e78fbd85fa4dde2e481bbf8b5349ceda35e4ab98fbf9ad0b336761be3f6379d
-  data.tar.gz: b9ad9cd2ebeaffa8b438c4b051d378edb47cfc86f1117cce68732f6cd3f2cdd3
+  metadata.gz: 96894f65cf1e88e86b89131843ae808215e94c6df4ce8e62b47286fd8ba60431
+  data.tar.gz: 3054de303e6ba58b370a27d34d96b046ba27878f2bcbe8e40efa0c421c09f1b3
 SHA512:
-  metadata.gz: 9ef2a213c75f9cf539620a393b20e7f2521173fb14a2bfbc2d6753654a8a19f19875b143a6b4240cc4ce09b01667cccc4d1da8d2e4c52d774bc9678c52eb13cb
-  data.tar.gz: 7102716c3687292aff85bc8191e0fbbe65ef457930f9fcce2c5b78e58ed3134a8e69d5b07f728a09438f0aeda84f97753a82419e91478ebd401ba99df5c2dd47
+  metadata.gz: 80246adb714cb8a2d32794187fafdd80cc58b573bf633cf5f6c89528f89cc71624e1bba013eea26f6b0c7fdf238f8d7e033b25391b60fd7338c7439ad8e823fd
+  data.tar.gz: 9c1b2a572734f144b49cc770be0d731f4b3559d88d020b9a6be77d78c9dd24faa1a25219a5a1274e46c1e3697c87c569e36d7b9228acd84769e24fcb1faed1ce

data/assets/MANIFEST.json

@@ -22,11 +22,11 @@
       }
     },
     "stop_words_never_capitalised.txt": {
-      "bytes": 8017,
-      "entries": 499,
+      "bytes": 8108,
+      "entries": 511,
       "format": 1,
       "min_package_version": "0.2.10",
-      "sha256": "23798cb88672c126d71a9d0988b9adf0aa044bdc291847012d76deae71fe1d33"
+      "sha256": "809e92aa74f001533be8bd33bfe119f5edab4bef2fbde687140f16c998c1039f"
     },
     "stop_words_sometimes_capitalised.txt": {
       "bytes": 6351,
@@ -37,5 +37,5 @@
     }
   },
   "manifest_version": 1,
-  "written_by": "vicary 0.2.13"
+  "written_by": "vicary 0.2.17"
 }

data/assets/stop_words_never_capitalised.txt

@@ -1,5 +1,5 @@
 #!lexicon 1
-#!list stop_words_never_capitalised 499
+#!list stop_words_never_capitalised 511
 # Ordinary words that must never become name candidates, and that English never
 # capitalises in the middle of a sentence.
 #
@@ -58,6 +58,7 @@
   a an the this that these those there here it its it's
   i me my mine myself we us our ours ourselves you your yours
   he him his she her hers they them their theirs who whom whose which what
+  how why where wherever whatever whoever
   and or but so because although though however therefore thus hence yet
   if then else when while until since before after during once whenever
   for from to into onto out off over under above below between among across
@@ -109,18 +110,19 @@
   cants coulds didnts dids doeses doings donts durings eithers elses etcs
   everies everybodies everyones everythings everywheres feels fews finallies
   finds froms gaves gets gives gos gots growings hads hases havings helpeds
-  hences hims hises howevers ids ifs ills ins insteads intos ises isnts justs
-  keeps kepts knews knows learneds learnings learns leaves lefts lets lookeds
-  lookings looks makes maybes mights mosts movings muches musts myselfs
-  neededs neithers nobodies nos nothings nots nowheres nows offs oftens okays
-  onces onlies ons ontos ourselveses outs perhapses playings pluses puts
-  quites rarelies rathers reads reallies remembereds remembers runnings saids
-  saws says sets severals shalls shes shoulds sinces sittings somebodies
-  someones somethings sometimeses somewheres standings starteds stoppeds
-  suches talkings taughts teaches tells thans thems thens therefores thes
-  theses thinkings thinks thises thoses thoughs thoughts throughs thuses
+  hences hims hises howevers hows ids ifs ills ins insteads intos ises isnts
+  justs keeps kepts knews knows learneds learnings learns leaves lefts lets
+  lookeds lookings looks makes maybes mights mosts movings muches musts
+  myselfs neededs neithers nobodies nos nothings nots nowheres nows offs
+  oftens okays onces onlies ons ontos ourselveses outs perhapses playings
+  pluses puts quites rarelies rathers reads reallies remembereds remembers
+  runnings saids saws says sets severals shalls shes shoulds sinces sittings
+  somebodies someones somethings sometimeses somewheres standings starteds
+  stoppeds suches talkings taughts teaches tells thans thems thens therefores
+  thes theses thinkings thinks thises thoses thoughs thoughts throughs thuses
   todays togethers tolds tomorrows toos trieds tries tryings unders unlesses
   untils ups uses usings usuallies veries versuses walkings wanteds wants
-  wases wents weres whenevers whens whethers whiches whoms whos whoses withins
-  withouts wonts workeds woulds writes wrotes yeahs yeses yesterdays yets yous
+  wases wents weres whatevers whenevers whens wheres wherevers whethers
+  whiches whies whoevers whoms whos whoses withins withouts wonts workeds
+  woulds writes wrotes yeahs yeses yesterdays yets yous
 # <<< end generated inflections

data/lib/vicary/candidates.rb

@@ -257,6 +257,16 @@ module Vicary
     # Matches the abbreviation a break candidate sits directly behind.
     TRAILING_WORD = /([A-Za-z]+)\.\s*\z/
 
+    # Words that name the letter after them as a letter: "a capital C", "a big M".
+    # Feedback on spelling and capitals says this all the time, and the letter
+    # then stood alone mid-sentence with nothing to tell it from an initial.
+    # Measured on 696 generated replies: 4 of the 39 placeholders a host's guard
+    # had to remove.
+    LETTER_NAMING_WORDS = Set.new(%w[capital big small lowercase uppercase letter]).freeze
+
+    # The word directly before a span, read back over whitespace and an opening quote.
+    PRECEDING_WORD = /([A-Za-z]+)[\s'"‘“]*\z/
+
     # {SENTENCE_BREAK} with the `\A` arm removed, so it cannot match empty.
     #
     # A dialect difference, and it moved a number. `SENTENCE_BREAK` matches the
@@ -300,6 +310,39 @@ module Vicary
       most much many few several enough
     ]).freeze
 
+    # Phonetic spellings of English stop words, each with the stop word it
+    # spells. Young writers, and young Spanish-L1 writers most of all,
+    # spell the closed-class words by ear — "dey sai ai smel", "so ai dont get
+    # mai clos dirtier" — and several of those spellings are given names the
+    # tier knows, so the lowercase route masked "sai ai" and "mai clos" and the
+    # sentence-initial rule let the tier vouch for "Ai", "Mi" and "De". A
+    # spelling of a stop word is the same word, so in a document that writes
+    # this way it is a stop word too. Only the spellings are read; the pairing
+    # is documentation. See `phonetic_function_words` for when a document
+    # qualifies.
+    PHONETIC_FUNCTION_WORDS = {
+      "ai" => "i", "becuz" => "because", "bicuz" => "because", "bot" => "but",
+      "cud" => "could", "da" => "the", "dan" => "than", "dat" => "that",
+      "de" => "the", "deir" => "their", "dem" => "them", "den" => "then",
+      "dere" => "there", "dese" => "these", "dey" => "they", "dis" => "this",
+      "dose" => "those", "frum" => "from", "hiz" => "his", "iz" => "is",
+      "mai" => "my", "mi" => "me", "sai" => "say", "sed" => "said",
+      "shud" => "should", "sum" => "some", "tu" => "to", "wat" => "what",
+      "wen" => "when", "wud" => "would", "wuz" => "was", "yu" => "you",
+      "yur" => "your",
+    }.freeze
+
+    # Distinct phonetic spellings a document must write in lower case before
+    # any of them is read as a word rather than a name. One is a coincidence —
+    # a friend called Mai — two different ones is a writer spelling by ear.
+    PHONETIC_REGISTER_MIN = 2
+
+    PHONETIC_TOKEN = Regexp.new(
+      "(?<![A-Za-z'’-])(?:" +
+        PHONETIC_FUNCTION_WORDS.keys.sort_by { |w| [-w.length, w] }.join("|") +
+        ")(?![A-Za-z'’-])"
+    )
+
     HONORIFIC_SET = Set.new(HONORIFICS.map(&:downcase)).freeze
 
     # One capitalised word, hyphens and apostrophes included so
@@ -742,6 +785,20 @@ module Vicary
         strip(without_clitic(strip(token.downcase, ".,")), "'’")
       end
 
+      # The phonetic spellings this document uses as words, to be read as
+      # stops. Empty unless the document writes at least PHONETIC_REGISTER_MIN
+      # distinct spellings in lower case. When it does, the spellings it writes
+      # in lower case — the writer's own testimony that the token is a word to
+      # them — less any it also capitalises mid-sentence, which is the opposite
+      # testimony and wins.
+      def phonetic_function_words(text, written_as_a_capital = Set.new)
+        seen = Set.new
+        each_match(text, PHONETIC_TOKEN) { |m| seen << m[0] }
+        return Set.new if seen.size < PHONETIC_REGISTER_MIN
+
+        Set.new(seen.reject { |w| written_as_a_capital.include?(w) })
+      end
+
       # Whether a capital on `token` mid-sentence is a mistake rather than a
       # construction — the narrow half of the stoplist.
       #
@@ -846,7 +903,11 @@ module Vicary
       # the stoplist so that a bare "Mrs." cannot become a candidate on its own,
       # but "Mrs. Okonkwo" has to stay whole: masking only the surname leaves the
       # relationship and the surname's position in the text.
-      def trim(tokens)
+      #
+      # `phonetic` is `phonetic_function_words` for the document: spellings read
+      # as stop words here. Except a particle *inside* a run — "Maria de Cruz"
+      # keeps its `de` however the rest of the document spells "the".
+      def trim(tokens, phonetic = Set.new)
         runs = []
         current = []
         tokens.each_with_index do |token, index|
@@ -854,7 +915,10 @@ module Vicary
             HONORIFIC_SET.include?(strip(token.downcase, ".,")) &&
             index + 1 < tokens.length &&
             !stop?(tokens[index + 1])
-          if stop?(token) && !introduces_a_name
+          spelled_stop = !phonetic.empty? && phonetic.include?(fold(token)) &&
+                         !(index.positive? && index < tokens.length - 1 &&
+                           PARTICLES.include?(token))
+          if (stop?(token) || spelled_stop) && !introduces_a_name
             unless current.empty?
               runs << current
               current = []
@@ -871,6 +935,18 @@ module Vicary
       # Reading the document's own orthography
       # ---------------------------------------------------------------------
 
+      # A lone capital that is a label or a letter, never an initial: glued to a
+      # digit (`E9`, a quote label the word pattern used to cut at the digit) or
+      # named as a letter by the word before it ({LETTER_NAMING_WORDS}). Only a
+      # single character is ever released; `Jake2009` stays a candidate.
+      def letter_not_a_name?(text, start, finish)
+        return false unless finish - start == 1
+        return true if finish < text.length && text[finish].match?(/[0-9]/)
+
+        preceding = PRECEDING_WORD.match(text[0...start])
+        !preceding.nil? && LETTER_NAMING_WORDS.include?(preceding[1].downcase)
+      end
+
       # Offsets at which a sentence begins.
       #
       # A break directly behind a title abbreviation is not one — see
@@ -1849,7 +1925,7 @@ module Vicary
       #   capitalisation signal, and the seed stands on the given-name tier
       #   alone.
       def find_lowercase_candidates(text, is_given, protected_span, corroborate = nil,
-                                    settlement = nil)
+                                    settlement = nil, phonetic = Set.new)
         tokens = each_match(text, LOWER_TOKEN).map do |m|
           [m[0], m.begin(0), m.begin(0) + m[0].length]
         end
@@ -1857,7 +1933,10 @@ module Vicary
         index = 0
         while index < tokens.length
           word, start, = tokens[index]
-          if stop?(word) || !is_given.call(word)
+          # A phonetic spelling may not SEED a span ("sai ai smel"), but may
+          # still be the token a real seed reaches ("jeni sai") — see the
+          # reference.
+          if stop?(word) || phonetic.include?(word) || !is_given.call(word)
             index += 1
             next
           end
@@ -1996,6 +2075,10 @@ module Vicary
         )
         written_as_a_capital = casing.written_as_a_capital
         lower_cased = casing.written_in_lower_case
+        # Read off the same casing pass, so a spelling the writer capitalises
+        # mid-sentence is never one of them; and only where a given-name oracle
+        # exists, because it only withdraws what the tier proposed or vouched for.
+        phonetic = given_name.nil? ? Set.new : phonetic_function_words(text, written_as_a_capital)
         stray_capitals = mid_sentence_corroboration && !given_name.nil? &&
                          capitalises_ordinary_words?(text, headings)
 
@@ -2007,7 +2090,7 @@ module Vicary
           tokens = words(span)
           # A long all-caps run means the capitalisation told us nothing, so the
           # stoplist is carrying the whole decision.
-          trim(tokens).each do |run|
+          trim(tokens, phonetic).each do |run|
             next if run.empty?
 
             joined = run.join(" ")
@@ -2017,6 +2100,7 @@ module Vicary
 
             start = m.begin(0) + offset
             next if is_protected.call(start, start + joined.length)
+            next if letter_not_a_name?(text, start, start + joined.length)
 
             # Cheapest of the three capital-discounting rules and the only one
             # that reads neither position nor list, so it goes first.
@@ -2078,7 +2162,7 @@ module Vicary
           # evidence to offer and is better served by it.
           find_lowercase_candidates(
             text, given_name, is_protected,
-            habit == LOWERCASE ? nil : written_as_a_capital, settlement
+            habit == LOWERCASE ? nil : written_as_a_capital, settlement, phonetic
           ).each do |candidate|
             next if claimed.any? { |s, e| candidate.start < e && candidate.end > s }
 

data/lib/vicary/version.rb

@@ -6,5 +6,5 @@ module Vicary
   # Shared across all three front doors on purpose: one detector, one number. A
   # gem 0.3.0 that corresponds to nothing on PyPI cannot be reasoned about, and
   # the parity claim is between *versions*, not between package names.
-  VERSION = "0.2.16"
+  VERSION = "0.2.18"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vicary
 version: !ruby/object:Gem::Version
-  version: 0.2.16
+  version: 0.2.18
 platform: ruby
 authors:
 - Blake Thomas
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-19 00:00:00.000000000 Z
+date: 2026-10-22 00:00:00.000000000 Z
 dependencies: []
 description: |
   Finds the names a student writes about — classmates, teachers, relatives — and