vicary 0.2.12 → 0.2.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/assets/MANIFEST.json +6 -5
- data/assets/notability.txt.gz +0 -0
- data/lib/vicary/asset.rb +1 -1
- data/lib/vicary/candidates.rb +278 -14
- data/lib/vicary/gazetteer.rb +35 -2
- data/lib/vicary/redact.rb +10 -1
- data/lib/vicary/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b9a6c30ad9946f8263d26b94e52b3d5d05db3d2ac0629425d992d5cebb1a7981
|
|
4
|
+
data.tar.gz: 8102ad087d3e7d1e3e4ce63a459de3d8598efb96f584e6c5ba02c815ac663611
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1956a3e47dc351c62ac4a58266ed48fbf936745ed6d6cf72b2abfc35df0f50b19ae2b4f603b80248e330153c866312ade9017777cbdf0f46c211ab2abeccb1fb
|
|
7
|
+
data.tar.gz: 42fa8950725aa8c58efa07462aec67843e671eef013c211a9025655df09970ddba05c54d92c524a3f244241d8bc1d147ee8acea9bd62f01b6b3e1c6d5eed8e15
|
data/assets/MANIFEST.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"assets": {
|
|
3
3
|
"notability.txt.gz": {
|
|
4
|
-
"bytes":
|
|
4
|
+
"bytes": 2291959,
|
|
5
5
|
"cut_date": "2026-08-07",
|
|
6
|
-
"format":
|
|
7
|
-
"min_package_version": "0.
|
|
8
|
-
"sha256": "
|
|
6
|
+
"format": 6,
|
|
7
|
+
"min_package_version": "0.2.13",
|
|
8
|
+
"sha256": "e26ad0a046c5935d8ba10bfde9fd073275d7bda490cf202ea61e9f606e16e916",
|
|
9
9
|
"sources": [
|
|
10
10
|
"https://qlever.dev/api/wikidata",
|
|
11
11
|
"https://www2.census.gov/topics/genealogy/2010surnames/names.zip"
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
"demonym": 1047,
|
|
15
15
|
"full": 295049,
|
|
16
16
|
"given": 8138,
|
|
17
|
+
"given_corroboration": 21900,
|
|
17
18
|
"place": 25444,
|
|
18
19
|
"settlement": 23234,
|
|
19
20
|
"short": 1229,
|
|
@@ -36,5 +37,5 @@
|
|
|
36
37
|
}
|
|
37
38
|
},
|
|
38
39
|
"manifest_version": 1,
|
|
39
|
-
"written_by": "vicary 0.2.
|
|
40
|
+
"written_by": "vicary 0.2.13"
|
|
40
41
|
}
|
data/assets/notability.txt.gz
CHANGED
|
Binary file
|
data/lib/vicary/asset.rb
CHANGED
data/lib/vicary/candidates.rb
CHANGED
|
@@ -167,6 +167,51 @@ module Vicary
|
|
|
167
167
|
# Any word token, used to find all-caps runs and mid-sentence capitals.
|
|
168
168
|
WORD_TOKEN = /[A-Za-z][A-Za-z'’-]*/
|
|
169
169
|
|
|
170
|
+
# Word-initial particles that legitimately carry an interior capital, so an
|
|
171
|
+
# interior capital on one of them is a name shape rather than orthographic
|
|
172
|
+
# noise. Closed and small, which is the whole of this signal's safety
|
|
173
|
+
# argument: `McDonald`, `MacArthur`, `DeShawn`, `DiCaprio`, `LaGrange`,
|
|
174
|
+
# `VanHalen`.
|
|
175
|
+
#
|
|
176
|
+
# The exemption is paid for in misses and the price is named here rather
|
|
177
|
+
# than discovered later — `LaTer` and `DeCide` are indistinguishable from
|
|
178
|
+
# `LaGrange` and `DeShawn` by orthography alone. It allows the particle
|
|
179
|
+
# exactly ONE capital, at the position right after it, so a second interior
|
|
180
|
+
# capital still fires; `dePenDs` is caught that way.
|
|
181
|
+
INTERIOR_CAPITAL_PREFIXES = %w[mc mac de di la le van von du da del san st].freeze
|
|
182
|
+
|
|
183
|
+
# Splits a word into the pieces an apostrophe or hyphen makes. `O'Brien` and
|
|
184
|
+
# `Jean-Luc` are two initial capitals rather than one interior capital, and
|
|
185
|
+
# without this split both read as orthographic noise — which would veto two
|
|
186
|
+
# of the commonest surname shapes there are.
|
|
187
|
+
PIECE = /[^'’\-‐-―]+/
|
|
188
|
+
|
|
189
|
+
# The only shape a word with an interior capital can carry: a capital with a
|
|
190
|
+
# word character in front of it. Every capital {Candidates.interior_capital?}
|
|
191
|
+
# is allowed to fire on sits at word-index 1 or later, so it is preceded
|
|
192
|
+
# either by the word's own first character or by one of `[A-Za-z'’-]`, and
|
|
193
|
+
# both are inside this class. A document that fails this has no interior
|
|
194
|
+
# capital and needs no word scan at all; a document that passes has told the
|
|
195
|
+
# scan exactly where to look. That makes this a filter and never a second
|
|
196
|
+
# opinion — the answer stays that method's.
|
|
197
|
+
#
|
|
198
|
+
# Two characters, and it cannot backtrack. It replaced a precise word
|
|
199
|
+
# pattern (`[A-Za-z][a-z'’-]*[A-Z][A-Za-z'’-]*`) that did: `[a-z'’-]*[A-Z]`
|
|
200
|
+
# is retried at every letter of every word that has no capital after it,
|
|
201
|
+
# which is most words in most documents, and running it over the whole text
|
|
202
|
+
# cost threefold what running this one and walking out from its hits costs —
|
|
203
|
+
# 57 µs per document against 17.5 on the twenty-essay gate corpus. The walk
|
|
204
|
+
# is in {Candidates.capitalises_inside_a_word?}; the word it reaches is the
|
|
205
|
+
# same substring {WORD_TOKEN} would have produced, because the walk uses
|
|
206
|
+
# that token's own continuation class.
|
|
207
|
+
INTERIOR_CAP_HINT = /[A-Za-z'’-][A-Z]/
|
|
208
|
+
|
|
209
|
+
# The complement of {WORD_TOKEN}'s continuation class, and its first-character
|
|
210
|
+
# class. The interior-capital walk reads outwards from a hint rather than
|
|
211
|
+
# forwards from a word, so it needs the boundary rather than the token.
|
|
212
|
+
NOT_WORD_CHARACTER = /[^A-Za-z'’-]/
|
|
213
|
+
WORD_FIRST_CHARACTER = /[A-Za-z]/
|
|
214
|
+
|
|
170
215
|
# Where a sentence begins: start of text, after terminal punctuation and any
|
|
171
216
|
# closing quote, after a line break, or immediately inside an *opening*
|
|
172
217
|
# quote. A capital in one of these positions is required by orthography, so
|
|
@@ -328,6 +373,11 @@ module Vicary
|
|
|
328
373
|
# One row of the precedence table: a tag, and what it decides.
|
|
329
374
|
PrecedenceRow = Struct.new(:tag, :mask, :kind)
|
|
330
375
|
|
|
376
|
+
# What one pass over a document's words says about how the writer cases
|
|
377
|
+
# them. See {Candidates.document_casing}; either half is empty when the
|
|
378
|
+
# caller did not ask for it.
|
|
379
|
+
DocumentCasing = Struct.new(:written_as_a_capital, :written_in_lower_case)
|
|
380
|
+
|
|
331
381
|
# The precedence table. The first row whose tag the span carries decides both
|
|
332
382
|
# the mask/keep verdict and the placeholder, and that is the whole
|
|
333
383
|
# classification policy.
|
|
@@ -1030,16 +1080,63 @@ module Vicary
|
|
|
1030
1080
|
# Horses" vouch for "Horses" as a name — the heading corroborating itself,
|
|
1031
1081
|
# one line removed.
|
|
1032
1082
|
def mid_sentence_capitals(text, starts, headings = [])
|
|
1033
|
-
|
|
1083
|
+
document_casing(text, starts, headings, lower_case: false).written_as_a_capital
|
|
1084
|
+
end
|
|
1085
|
+
|
|
1086
|
+
# Both halves of the document's casing testimony, from ONE pass over its
|
|
1087
|
+
# words.
|
|
1088
|
+
#
|
|
1089
|
+
# {.mid_sentence_capitals} and {.written_in_lower_case} ask opposite
|
|
1090
|
+
# questions of the same tokens — one reads the words a writer capitalised
|
|
1091
|
+
# where orthography would not have, the other the words they left
|
|
1092
|
+
# lower-case — and a token answers exactly one of them, because a word's
|
|
1093
|
+
# first character is either a capital or it is not. Run separately they
|
|
1094
|
+
# tokenised every document twice.
|
|
1095
|
+
#
|
|
1096
|
+
# That second tokenisation is what this exists to remove, and it is worth
|
|
1097
|
+
# a method rather than a comment: the pair was most of what the two
|
|
1098
|
+
# orthography channels added to the release latency gate.
|
|
1099
|
+
#
|
|
1100
|
+
# Each half is opt-in because each has a caller that does not want it. The
|
|
1101
|
+
# lower-case set is evidence only where the writer marks proper nouns at
|
|
1102
|
+
# all (see `lower_cased` in {.find_candidates}), and asking for a set that
|
|
1103
|
+
# will be discarded is the cost this method was written to avoid.
|
|
1104
|
+
def document_casing(text, starts = Set.new, headings = [], capitals: true, lower_case: true)
|
|
1105
|
+
capitalised = Set.new
|
|
1106
|
+
lowered = Set.new
|
|
1034
1107
|
each_match(text, WORD_TOKEN) do |m|
|
|
1035
1108
|
token = m[0]
|
|
1036
|
-
|
|
1109
|
+
# A byte comparison rather than `token[0].match?(/[A-Z]/)`, which
|
|
1110
|
+
# allocated a one-character string and ran a regex for every word of
|
|
1111
|
+
# every document. Exact, not an approximation: {WORD_TOKEN}'s first
|
|
1112
|
+
# character is `[A-Za-z]`, so it is ASCII, a single byte, and
|
|
1113
|
+
# lower-case exactly when it is not upper-case — which is what splits
|
|
1114
|
+
# the two halves, at one test rather than two.
|
|
1115
|
+
first = token.getbyte(0)
|
|
1116
|
+
if first >= 97 && first <= 122
|
|
1117
|
+
next unless lower_case
|
|
1118
|
+
|
|
1119
|
+
# {.strip} copies the string whether or not it takes anything off,
|
|
1120
|
+
# and almost no word ends in an apostrophe. Same value either way —
|
|
1121
|
+
# a strip that removes nothing returns an equal string, and the set
|
|
1122
|
+
# holds values.
|
|
1123
|
+
#
|
|
1124
|
+
# Only the tail is worth testing. {.strip} works in from both ends,
|
|
1125
|
+
# and the head was just established to be a lower-case ASCII letter,
|
|
1126
|
+
# so the leading pass stops on the first character every time.
|
|
1127
|
+
lowered_token = token.downcase
|
|
1128
|
+
lowered_token = strip(lowered_token, "'’") if lowered_token.end_with?("'", "’")
|
|
1129
|
+
lowered << lowered_token
|
|
1130
|
+
next
|
|
1131
|
+
end
|
|
1132
|
+
next unless capitals
|
|
1133
|
+
next if starts.include?(m.begin(0))
|
|
1037
1134
|
next if token.length > 1 && upper?(token)
|
|
1038
1135
|
next if overlaps?(headings, m.begin(0), m.begin(0) + token.length)
|
|
1039
1136
|
|
|
1040
|
-
|
|
1137
|
+
capitalised << strip(token.downcase, "'’")
|
|
1041
1138
|
end
|
|
1042
|
-
|
|
1139
|
+
DocumentCasing.new(capitalised, lowered)
|
|
1043
1140
|
end
|
|
1044
1141
|
|
|
1045
1142
|
# ---------------------------------------------------------------------
|
|
@@ -1079,13 +1176,26 @@ module Vicary
|
|
|
1079
1176
|
# given-name tier. `is_given` is passed in rather than defaulted so this is
|
|
1080
1177
|
# only reachable on the path where an oracle exists.
|
|
1081
1178
|
#
|
|
1179
|
+
# **The tier is read at a lower floor here than it is for generation**,
|
|
1180
|
+
# when the caller supplies `is_given_for_corroboration`. Absent, it
|
|
1181
|
+
# defaults to `is_given` and this behaves exactly as it did before the dial
|
|
1182
|
+
# existed. The two roles cost differently: a generation hit invents a span
|
|
1183
|
+
# out of lower-case prose, a corroboration hit can only restore one the
|
|
1184
|
+
# writer's own capital already proposed, and the shipped 1,800-birth knee
|
|
1185
|
+
# was measured against the first. **This is the only rule that reads the
|
|
1186
|
+
# lower floor** — the mid-sentence stray rule and the case-variance rule
|
|
1187
|
+
# both keep the generation floor, because the corpus that priced this
|
|
1188
|
+
# channel has zero true catches in the first one's suppressed set.
|
|
1189
|
+
#
|
|
1082
1190
|
# ANY token counts, not just the first, and the heading rule is what made
|
|
1083
1191
|
# that distinction load-bearing. Before it, this was only ever reached for
|
|
1084
1192
|
# single-token spans, so "first token" and "any token" were the same thing.
|
|
1085
1193
|
# A heading is title-cased, so a multi-token span inside one also arrives
|
|
1086
1194
|
# here — and "My Brother Terrence Okonkwo" leads with an honorific, so
|
|
1087
1195
|
# checking only the first token consulted "Brother" and leaked the name.
|
|
1088
|
-
def corroborated?(tokens, written_as_a_capital, is_given
|
|
1196
|
+
def corroborated?(tokens, written_as_a_capital, is_given,
|
|
1197
|
+
is_given_for_corroboration = nil)
|
|
1198
|
+
vouches = is_given_for_corroboration || is_given
|
|
1089
1199
|
# Both channels see the same stripped token, and the strip set is `.,'’`
|
|
1090
1200
|
# rather than the `'’` {.mid_sentence_capitals} folds with. That
|
|
1091
1201
|
# asymmetry is deliberate and was a defect once: the capital channel
|
|
@@ -1095,7 +1205,7 @@ module Vicary
|
|
|
1095
1205
|
# character — asked the tier about `Terrence'` and was told no.
|
|
1096
1206
|
tokens.each do |token|
|
|
1097
1207
|
stripped = strip(token.downcase, ".,'’")
|
|
1098
|
-
return true if written_as_a_capital.include?(stripped) ||
|
|
1208
|
+
return true if written_as_a_capital.include?(stripped) || vouches.call(stripped)
|
|
1099
1209
|
|
|
1100
1210
|
# ...and again with the possessive off. "Terrence's" at a sentence
|
|
1101
1211
|
# start is the shape this is for: the writer capitalised "Terrence"
|
|
@@ -1114,13 +1224,44 @@ module Vicary
|
|
|
1114
1224
|
# reverse, so it can only reduce suppression, never increase it.
|
|
1115
1225
|
folded = without_clitic(stripped)
|
|
1116
1226
|
if folded != stripped &&
|
|
1117
|
-
(written_as_a_capital.include?(folded) ||
|
|
1227
|
+
(written_as_a_capital.include?(folded) || vouches.call(folded))
|
|
1118
1228
|
return true
|
|
1119
1229
|
end
|
|
1120
1230
|
end
|
|
1121
1231
|
false
|
|
1122
1232
|
end
|
|
1123
1233
|
|
|
1234
|
+
# Drop a lone capital on a word this same document also writes lower-case.
|
|
1235
|
+
#
|
|
1236
|
+
# The third of the capital-discounting rules, and the only one that needs
|
|
1237
|
+
# neither a position nor a list. {.suppressed_as_an_unevidenced_capital?}
|
|
1238
|
+
# asks whether orthography required the capital;
|
|
1239
|
+
# {.suppressed_as_a_stray_mid_sentence_capital?} asks whether the writer
|
|
1240
|
+
# is a sloppy capitaliser in general. This asks the narrowest question of
|
|
1241
|
+
# the three and the one with the best evidence behind it: did the writer,
|
|
1242
|
+
# in this document, write this exact word as a word? If they did, a
|
|
1243
|
+
# capital elsewhere on the same letters is not testimony about a name.
|
|
1244
|
+
#
|
|
1245
|
+
# The given-name tier still rescues, exactly as it does in
|
|
1246
|
+
# {.corroborated?}, and it is the reason this is safe to run outside the
|
|
1247
|
+
# stray-capital gate. `Bill` in a document that also writes "bill"
|
|
1248
|
+
# survives on the tier; `Summer`, `Space`, `Love` and `Fight` do not.
|
|
1249
|
+
#
|
|
1250
|
+
# Measured on the 56-paper NWP corpus as a post-hoc arm over the recorded
|
|
1251
|
+
# spans: +7 false positives recovered, 0 public entities, 0 PII lost, and
|
|
1252
|
+
# it is the only free arm that moves papers-damaged-for-nothing on its own
|
|
1253
|
+
# (23 -> 20).
|
|
1254
|
+
def suppressed_as_a_word_the_writer_also_writes_lower_case?(tokens, lower_cased, is_given)
|
|
1255
|
+
return false unless tokens.length == 1
|
|
1256
|
+
|
|
1257
|
+
stripped = strip(tokens[0].downcase, ".,'’")
|
|
1258
|
+
return false unless lower_cased.include?(stripped)
|
|
1259
|
+
return false if is_given.call(stripped)
|
|
1260
|
+
|
|
1261
|
+
folded = without_clitic(stripped)
|
|
1262
|
+
!(folded != stripped && is_given.call(folded))
|
|
1263
|
+
end
|
|
1264
|
+
|
|
1124
1265
|
# The sentence-initial guard: drop a span whose only evidence is a capital
|
|
1125
1266
|
# that orthography required, unless a second channel vouches for it.
|
|
1126
1267
|
#
|
|
@@ -1143,9 +1284,11 @@ module Vicary
|
|
|
1143
1284
|
# **Do not "fix" it**; the tier feeding it was the defect, and that was
|
|
1144
1285
|
# addressed in 0.1.0 by adding SSA births to the given-name tier.
|
|
1145
1286
|
def suppressed_as_an_unevidenced_capital?(tokens, start, starts, emphasis, headings,
|
|
1146
|
-
written_as_a_capital, is_given
|
|
1287
|
+
written_as_a_capital, is_given,
|
|
1288
|
+
is_given_for_corroboration = nil)
|
|
1147
1289
|
capital_is_the_only_evidence?(tokens, start, starts, emphasis, headings) &&
|
|
1148
|
-
!corroborated?(tokens, written_as_a_capital, is_given
|
|
1290
|
+
!corroborated?(tokens, written_as_a_capital, is_given,
|
|
1291
|
+
is_given_for_corroboration)
|
|
1149
1292
|
end
|
|
1150
1293
|
|
|
1151
1294
|
# Whether this document capitalises words that cannot be names.
|
|
@@ -1166,6 +1309,17 @@ module Vicary
|
|
|
1166
1309
|
# therefore counts the names too, this cannot be satisfied by a document
|
|
1167
1310
|
# that simply names a lot of people. Headings are excluded because title
|
|
1168
1311
|
# case capitalises every word in one.
|
|
1312
|
+
#
|
|
1313
|
+
# Two channels. The stray-capital channel needs a
|
|
1314
|
+
# mid-sentence capital to land on a hand-curated list, so it can only ever
|
|
1315
|
+
# speak about words someone thought to curate. The second channel —
|
|
1316
|
+
# {.capitalises_inside_a_word?} — needs no list, because a capital in the
|
|
1317
|
+
# middle of a word is not a shape English produces under any rule.
|
|
1318
|
+
#
|
|
1319
|
+
# Measured on the 56-paper NWP corpus: the curated channel fires on 10
|
|
1320
|
+
# papers, the interior channel on 12 (14 counting headings, which it does
|
|
1321
|
+
# count), they overlap on 7, and 5 papers are reached by the interior
|
|
1322
|
+
# channel alone.
|
|
1169
1323
|
def capitalises_ordinary_words?(text, headings = [])
|
|
1170
1324
|
count = 0
|
|
1171
1325
|
each_match(text, MID_SENTENCE_CAP) do |m|
|
|
@@ -1178,7 +1332,90 @@ module Vicary
|
|
|
1178
1332
|
|
|
1179
1333
|
count += 1
|
|
1180
1334
|
end
|
|
1181
|
-
count >= STRAY_CAPITALS_MIN
|
|
1335
|
+
return true if count >= STRAY_CAPITALS_MIN
|
|
1336
|
+
|
|
1337
|
+
capitalises_inside_a_word?(text)
|
|
1338
|
+
end
|
|
1339
|
+
|
|
1340
|
+
# A capital in a non-initial position that no name form explains.
|
|
1341
|
+
#
|
|
1342
|
+
# Four exemptions, and each one is a real name shape rather than a hedge:
|
|
1343
|
+
# all-caps pieces (`BILL` is a writer who stopped using case, a different
|
|
1344
|
+
# defect with a different rule), apostrophes and hyphens split (`O'Brien`
|
|
1345
|
+
# and `Jean-Luc` are two initial capitals — see {PIECE}), particles (see
|
|
1346
|
+
# {INTERIOR_CAPITAL_PREFIXES}), and single characters (`T.V` splits to `T`
|
|
1347
|
+
# and `V`; an initial has no interior).
|
|
1348
|
+
#
|
|
1349
|
+
# What survives is `ChoaCh`, `PoSitive`, `grandParints`, `surPise` —
|
|
1350
|
+
# orthographic noise, and unlike a mid-sentence capital it cannot be
|
|
1351
|
+
# confused with correct English, because correct English has no such form.
|
|
1352
|
+
def interior_capital?(word)
|
|
1353
|
+
word.scan(PIECE).each do |piece|
|
|
1354
|
+
next if piece.length < 2 || !piece.match?(/\A[A-Za-z]+\z/) || upper?(piece)
|
|
1355
|
+
|
|
1356
|
+
lowered = piece.downcase
|
|
1357
|
+
start = 1
|
|
1358
|
+
INTERIOR_CAPITAL_PREFIXES.each do |prefix|
|
|
1359
|
+
next unless lowered.start_with?(prefix) && piece.length > prefix.length &&
|
|
1360
|
+
piece[prefix.length].match?(/[A-Z]/)
|
|
1361
|
+
|
|
1362
|
+
start = [start, prefix.length + 1].max
|
|
1363
|
+
end
|
|
1364
|
+
return true if piece[start..].to_s.match?(/[A-Z]/)
|
|
1365
|
+
end
|
|
1366
|
+
false
|
|
1367
|
+
end
|
|
1368
|
+
|
|
1369
|
+
# Whether this document puts a capital in the middle of a word.
|
|
1370
|
+
#
|
|
1371
|
+
# Headings are NOT excluded here, and that is a departure with a reason.
|
|
1372
|
+
# Everywhere else a heading's capitals are discounted because title case
|
|
1373
|
+
# put them there. Title case does not put a capital in the *middle* of a
|
|
1374
|
+
# word, so the argument does not transfer — and excluding headings anyway
|
|
1375
|
+
# costs signal that is measured rather than hypothetical: on the 56-paper
|
|
1376
|
+
# NWP corpus 14 papers carry an interior capital and 2 of them (`SPecial`,
|
|
1377
|
+
# `AFter`) carry it only inside a heading.
|
|
1378
|
+
#
|
|
1379
|
+
# The scan is driven off {INTERIOR_CAP_HINT} rather than run over every
|
|
1380
|
+
# word token, which is a filter and not a change of answer — see that
|
|
1381
|
+
# pattern. Each hit names a capital and the walk reaches the word carrying
|
|
1382
|
+
# it; the hit need not be interior itself (`'ChoaCh'` hints at its own
|
|
1383
|
+
# first letter), so the WHOLE token is handed over rather than the
|
|
1384
|
+
# position being judged here.
|
|
1385
|
+
def capitalises_inside_a_word?(text)
|
|
1386
|
+
pos = 0
|
|
1387
|
+
while (hint = INTERIOR_CAP_HINT.match(text, pos))
|
|
1388
|
+
capital = hint.begin(0) + 1
|
|
1389
|
+
boundary = text.rindex(NOT_WORD_CHARACTER, capital - 1)
|
|
1390
|
+
# A word begins on a letter, so a run of leading apostrophes or
|
|
1391
|
+
# hyphens the walk crossed belongs to no word — `'Abc` tokenises to
|
|
1392
|
+
# `Abc`. The capital is itself a letter, so this never runs past it.
|
|
1393
|
+
start = text.index(WORD_FIRST_CHARACTER, boundary.nil? ? 0 : boundary + 1)
|
|
1394
|
+
stop = text.index(NOT_WORD_CHARACTER, capital + 1) || text.length
|
|
1395
|
+
return true if interior_capital?(text[start...stop])
|
|
1396
|
+
|
|
1397
|
+
# Past the whole word: it has just been read in full, so a second
|
|
1398
|
+
# capital inside it would ask the same question again.
|
|
1399
|
+
pos = stop
|
|
1400
|
+
end
|
|
1401
|
+
false
|
|
1402
|
+
end
|
|
1403
|
+
|
|
1404
|
+
# Lower-cased forms of every word this document writes with a lower-case
|
|
1405
|
+
# initial.
|
|
1406
|
+
#
|
|
1407
|
+
# The mirror of {.mid_sentence_capitals}: the same scan read for the
|
|
1408
|
+
# opposite testimony. That method records the words a document capitalises
|
|
1409
|
+
# where orthography would not have and reads them as evidence those words
|
|
1410
|
+
# are names; this records the words the writer themself also wrote as
|
|
1411
|
+
# words.
|
|
1412
|
+
#
|
|
1413
|
+
# Case-insensitively *equal*, not merely similar — no plural fold, no edit
|
|
1414
|
+
# distance, no stem. The evidence is the writer's own hand on the same
|
|
1415
|
+
# letters, which is what keeps the rule free of any imported collision: it
|
|
1416
|
+
# consults no list, so it cannot inherit one's mistakes.
|
|
1417
|
+
def written_in_lower_case(text)
|
|
1418
|
+
document_casing(text, capitals: false).written_in_lower_case
|
|
1182
1419
|
end
|
|
1183
1420
|
|
|
1184
1421
|
# The mid-sentence guard: drop a lone capital a sloppy capitaliser chose.
|
|
@@ -1685,6 +1922,9 @@ module Vicary
|
|
|
1685
1922
|
# Options, all optional:
|
|
1686
1923
|
# * `:given_name` — turns on the lowercase route. Absent, this keys on
|
|
1687
1924
|
# capitalisation alone and misses lowercase writing by construction.
|
|
1925
|
+
# * `:given_name_corroboration` — the same tier at its permissive floor,
|
|
1926
|
+
# read by the sentence-initial rule ONLY. Absent, that rule reads
|
|
1927
|
+
# `:given_name` and nothing changes. Inert without `:given_name`.
|
|
1688
1928
|
# * `:title`, `:title_prefix` — protect work titles and fictional-character
|
|
1689
1929
|
# names from generation entirely. Absent, a student writing about a book
|
|
1690
1930
|
# has the book redacted.
|
|
@@ -1700,12 +1940,14 @@ module Vicary
|
|
|
1700
1940
|
# second half of the same rule and neither half works alone.
|
|
1701
1941
|
def find_candidates(text, options = {})
|
|
1702
1942
|
given_name = options[:given_name]
|
|
1943
|
+
given_name_corroboration = options[:given_name_corroboration]
|
|
1703
1944
|
title = options[:title]
|
|
1704
1945
|
title_prefix = options[:title_prefix]
|
|
1705
1946
|
settlement = options[:settlement]
|
|
1706
1947
|
headings_are_orthographic = options.fetch(:headings_are_orthographic, true)
|
|
1707
1948
|
title_relation_refusal = options.fetch(:title_relation_refusal, true)
|
|
1708
1949
|
mid_sentence_corroboration = options.fetch(:mid_sentence_corroboration, true)
|
|
1950
|
+
case_variance = options.fetch(:case_variance, true)
|
|
1709
1951
|
|
|
1710
1952
|
blocked = each_match(text, PROTECTED).map { |m| [m.begin(0), m.begin(0) + m[0].length] }
|
|
1711
1953
|
starts = sentence_starts(text)
|
|
@@ -1739,9 +1981,21 @@ module Vicary
|
|
|
1739
1981
|
blocked.any? { |block_start, block_end| start < block_end && finish > block_start }
|
|
1740
1982
|
end
|
|
1741
1983
|
|
|
1742
|
-
|
|
1743
|
-
#
|
|
1744
|
-
#
|
|
1984
|
+
# Both casing questions, read once from one pass over the words, for the
|
|
1985
|
+
# same reason `habit` is read once: two call sites computing it
|
|
1986
|
+
# separately could disagree.
|
|
1987
|
+
#
|
|
1988
|
+
# The lower-case half is asked for only where the writer marks proper
|
|
1989
|
+
# nouns at all: that rule reads the ABSENCE of a capital as testimony,
|
|
1990
|
+
# and the habit states in as many words that an absence means nothing in
|
|
1991
|
+
# a LOWERCASE or SILENT document. Running it there would suppress every
|
|
1992
|
+
# capital the writer did manage.
|
|
1993
|
+
casing = document_casing(
|
|
1994
|
+
text, starts, headings,
|
|
1995
|
+
lower_case: case_variance && !given_name.nil? && marks_proper_nouns?(habit),
|
|
1996
|
+
)
|
|
1997
|
+
written_as_a_capital = casing.written_as_a_capital
|
|
1998
|
+
lower_cased = casing.written_in_lower_case
|
|
1745
1999
|
stray_capitals = mid_sentence_corroboration && !given_name.nil? &&
|
|
1746
2000
|
capitalises_ordinary_words?(text, headings)
|
|
1747
2001
|
|
|
@@ -1763,12 +2017,22 @@ module Vicary
|
|
|
1763
2017
|
|
|
1764
2018
|
start = m.begin(0) + offset
|
|
1765
2019
|
next if is_protected.call(start, start + joined.length)
|
|
2020
|
+
|
|
2021
|
+
# Cheapest of the three capital-discounting rules and the only one
|
|
2022
|
+
# that reads neither position nor list, so it goes first.
|
|
2023
|
+
if !lower_cased.empty? && !given_name.nil? &&
|
|
2024
|
+
suppressed_as_a_word_the_writer_also_writes_lower_case?(run, lower_cased,
|
|
2025
|
+
given_name)
|
|
2026
|
+
next
|
|
2027
|
+
end
|
|
2028
|
+
|
|
1766
2029
|
# Requiring a second signal is only sound when there is a second
|
|
1767
2030
|
# signal to require, which is why this is reached only where an
|
|
1768
2031
|
# oracle exists.
|
|
1769
2032
|
if !given_name.nil? &&
|
|
1770
2033
|
suppressed_as_an_unevidenced_capital?(run, start, starts, emphasis, headings,
|
|
1771
|
-
written_as_a_capital, given_name
|
|
2034
|
+
written_as_a_capital, given_name,
|
|
2035
|
+
given_name_corroboration)
|
|
1772
2036
|
next
|
|
1773
2037
|
end
|
|
1774
2038
|
|
data/lib/vicary/gazetteer.rb
CHANGED
|
@@ -52,7 +52,8 @@ module Vicary
|
|
|
52
52
|
# forgotten here would read back as an empty set, and an empty KEEP tier
|
|
53
53
|
# redacts everything it was built to protect while presenting as
|
|
54
54
|
# over-aggressive tuning.
|
|
55
|
-
TIER_NAMES = %w[full short place given title
|
|
55
|
+
TIER_NAMES = %w[full short place given given_corroboration title
|
|
56
|
+
demonym settlement].freeze
|
|
56
57
|
|
|
57
58
|
# Name particles that may lead a two- or three-token *partial* surname.
|
|
58
59
|
#
|
|
@@ -179,7 +180,8 @@ module Vicary
|
|
|
179
180
|
# use rather than taken as constructor arguments, because they are functions
|
|
180
181
|
# of +title+ and must never be able to disagree with it.
|
|
181
182
|
class Index
|
|
182
|
-
attr_reader :full, :short, :place, :given, :
|
|
183
|
+
attr_reader :full, :short, :place, :given, :given_corroboration, :title,
|
|
184
|
+
:demonym, :settlement, :meta
|
|
183
185
|
|
|
184
186
|
def initialize(asset)
|
|
185
187
|
asset.tiers.each_key do |name|
|
|
@@ -198,6 +200,10 @@ module Vicary
|
|
|
198
200
|
@place = asset.tiers.fetch("place", EMPTY)
|
|
199
201
|
# Common given names. The INVERSE signal — see #common_given_name?.
|
|
200
202
|
@given = asset.tiers.fetch("given", EMPTY)
|
|
203
|
+
# Given names between the corroboration floor and the generation floor —
|
|
204
|
+
# the INCREMENT over +given+, never the whole set. Readers union the two;
|
|
205
|
+
# see #vouches_for_a_given_name_in_corroboration?.
|
|
206
|
+
@given_corroboration = asset.tiers.fetch("given_corroboration", EMPTY)
|
|
201
207
|
# Works and fictional characters — multi-token only. See #title?.
|
|
202
208
|
@title = asset.tiers.fetch("title", EMPTY)
|
|
203
209
|
# English demonyms — `cuban`, `nigerian`. A KEEP, see DEMONYM.
|
|
@@ -297,6 +303,28 @@ module Vicary
|
|
|
297
303
|
!key.empty? && !key.include?(" ") && given.include?(key)
|
|
298
304
|
end
|
|
299
305
|
|
|
306
|
+
# #common_given_name?, at the permissive corroboration floor.
|
|
307
|
+
#
|
|
308
|
+
# **Strictly wider than #common_given_name? and never narrower**, because
|
|
309
|
+
# +given_corroboration+ holds the increment and this unions it with
|
|
310
|
+
# +given+ rather than replacing it. A caller reaching for this one cannot
|
|
311
|
+
# accidentally get a smaller answer than the generation tier would give,
|
|
312
|
+
# which is the failure a second independently-built list would allow.
|
|
313
|
+
#
|
|
314
|
+
# The two floors exist because the two roles cost differently. A
|
|
315
|
+
# generation hit creates a span out of lower-case prose; a corroboration
|
|
316
|
+
# hit can only un-suppress a span the writer's own capital already
|
|
317
|
+
# proposed. The shipped 1,800-birth knee was measured against the first
|
|
318
|
+
# and does not transfer to the second.
|
|
319
|
+
#
|
|
320
|
+
# Consulted by Candidates.corroborated? and by nothing else.
|
|
321
|
+
def vouches_for_a_given_name_in_corroboration?(token)
|
|
322
|
+
key = Gazetteer.normalize(token)
|
|
323
|
+
return false if key.empty? || key.include?(" ")
|
|
324
|
+
|
|
325
|
+
given.include?(key) || given_corroboration.include?(key)
|
|
326
|
+
end
|
|
327
|
+
|
|
300
328
|
# True when +name+ is a town, city or village.
|
|
301
329
|
#
|
|
302
330
|
# **Not part of the notability decision, and deliberately not consulted by
|
|
@@ -391,6 +419,11 @@ module Vicary
|
|
|
391
419
|
load.common_given_name?(token)
|
|
392
420
|
end
|
|
393
421
|
|
|
422
|
+
# The given-name tier at its permissive, corroboration-only floor.
|
|
423
|
+
def vouches_for_a_given_name_in_corroboration?(token)
|
|
424
|
+
load.vouches_for_a_given_name_in_corroboration?(token)
|
|
425
|
+
end
|
|
426
|
+
|
|
394
427
|
# True when +name+ is a town or city — a TYPING signal, not a keep.
|
|
395
428
|
def settlement?(name)
|
|
396
429
|
load.settlement?(name)
|
data/lib/vicary/redact.rb
CHANGED
|
@@ -120,7 +120,16 @@ module Vicary
|
|
|
120
120
|
# The one difference between the two gazetteer levels. Absent rather than
|
|
121
121
|
# nil, so `gazetteer` and `gazetteer-lowercase` differ by the presence of a
|
|
122
122
|
# key rather than by a value the merge would have to strip.
|
|
123
|
-
|
|
123
|
+
if level == NAMES_LOWERCASE
|
|
124
|
+
oracles[:given_name] = ->(token) { Gazetteer.common_given_name?(token) }
|
|
125
|
+
# Paired deliberately: the corroboration floor is inert without
|
|
126
|
+
# `given_name` — every rule that could read it is gated on an oracle
|
|
127
|
+
# being supplied — so wiring it at the bare level too would be a dial
|
|
128
|
+
# that looks set and does nothing, which is worse than an absent one.
|
|
129
|
+
oracles[:given_name_corroboration] = lambda { |token|
|
|
130
|
+
Gazetteer.vouches_for_a_given_name_in_corroboration?(token)
|
|
131
|
+
}
|
|
132
|
+
end
|
|
124
133
|
oracles
|
|
125
134
|
end
|
|
126
135
|
|
data/lib/vicary/version.rb
CHANGED
|
@@ -6,5 +6,5 @@ module Vicary
|
|
|
6
6
|
# Shared across all three front doors on purpose: one detector, one number. A
|
|
7
7
|
# gem 0.3.0 that corresponds to nothing on PyPI cannot be reasoned about, and
|
|
8
8
|
# the parity claim is between *versions*, not between package names.
|
|
9
|
-
VERSION = "0.2.
|
|
9
|
+
VERSION = "0.2.15"
|
|
10
10
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: vicary
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.15
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Blake Thomas
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-09-
|
|
11
|
+
date: 2026-09-13 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: |
|
|
14
14
|
Finds the names a student writes about — classmates, teachers, relatives — and
|