@polycode-projects/the-mechanical-code-talker 6.0.18 → 6.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -23
- package/bin/tmct.mjs +16 -33
- package/corpus/LICENSES.json +0 -21
- package/corpus/README.md +10 -13
- package/corpus/reference/manifest.json +19 -19
- package/corpus/reference/shards/ref-01.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-04.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-08.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-10.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-11.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-17.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-20.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-25.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-2c.jsonl.gz +0 -0
- package/corpus/tier2/generate.mjs +6 -142
- package/corpus/tier2/manifest.json +0 -42
- package/package.json +6 -4
- package/src/adapters/corpus/child-seed.mjs +74 -0
- package/src/adapters/corpus/conceptnet.mjs +45 -26
- package/src/adapters/corpus/research-source.mjs +6 -2
- package/src/adapters/corpus/wikidata-live.mjs +92 -51
- package/src/adapters/memory/blocks.mjs +7 -1
- package/src/adapters/memory/core.mjs +505 -107
- package/src/adapters/memory/corpus-bands.mjs +27 -10
- package/src/adapters/memory/inspect.mjs +24 -5
- package/src/adapters/memory/rows.mjs +359 -30
- package/src/adapters/memory/shacl.mjs +10 -3
- package/src/domain/ask.mjs +27 -10
- package/src/domain/cli-verbs.mjs +3 -4
- package/src/domain/completions/group.mjs +8 -3
- package/src/domain/completions/infer.mjs +7 -2
- package/src/domain/completions/prune.mjs +5 -1
- package/src/domain/completions/rank.mjs +7 -2
- package/src/domain/digest/compose.mjs +5 -1
- package/src/domain/digest/select.mjs +12 -6
- package/src/domain/domain.mjs +15 -8
- package/src/domain/el-classify.mjs +11 -2
- package/src/domain/fact-phrase.mjs +86 -4
- package/src/domain/hash.mjs +9 -0
- package/src/domain/memory/bias.mjs +8 -4
- package/src/domain/memory/capability.mjs +12 -6
- package/src/domain/memory/fact-order.mjs +29 -0
- package/src/domain/memory/resolution.mjs +3 -0
- package/src/domain/news-feed.mjs +862 -92
- package/src/domain/reference-pack.mjs +5 -0
- package/src/domain/sense-gate.mjs +220 -0
- package/src/domain/sense-scope.mjs +116 -0
- package/src/domain/sense-split.mjs +1 -1
- package/src/domain/syllogise.mjs +60 -21
- package/src/domain/tableau.mjs +23 -14
- package/src/domain/term-ledger.mjs +16 -1
- package/src/domain/worlds-pack.mjs +5 -1
- package/src/services/adventure-autoplay.mjs +6 -1
- package/src/services/adventure-editor.mjs +43 -21
- package/src/services/adventure-viz.mjs +26 -9
- package/src/services/adventure.mjs +40 -10
- package/src/services/chat.mjs +270 -125
- package/src/services/extensions.mjs +51 -58
- package/src/services/extract-facts.mjs +906 -66
- package/src/services/init.mjs +4 -4
- package/src/services/ledger-viz.mjs +9 -4
- package/src/services/memory-panel-viz.mjs +4 -5
- package/src/services/mud-editor.mjs +40 -16
- package/src/services/mud-viz.mjs +8 -2
- package/src/services/mudiii-turn.mjs +5 -3
- package/src/services/mudiii-viz.mjs +8 -2
- package/src/services/news.mjs +306 -21
- package/src/services/research-viz.mjs +1 -1
- package/src/services/sprite-catalog-viz.mjs +10 -5
- package/src/surfaces/web/adventure-browser-entry.mjs +6 -12
- package/src/surfaces/web/memory-ask-browser.bundle.js +152 -151
- package/src/surfaces/web/mud-browser-entry.mjs +7 -11
- package/src/surfaces/web/research-browser-entry.mjs +5 -2
- package/corpus/tier2/aws.jsonl +0 -39
- package/corpus/tier2/java.jsonl +0 -31
- package/corpus/tier2/python.jsonl +0 -30
|
@@ -18,10 +18,11 @@
|
|
|
18
18
|
// full NLU: nothing here ever paraphrases or invents a fact the recognizer
|
|
19
19
|
// itself didn't produce.
|
|
20
20
|
//
|
|
21
|
-
// --optimistic ALSO run a bounded
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
-
//
|
|
21
|
+
// --optimistic ALSO run a bounded fuzzy tier over the sentences the strict
|
|
22
|
+
// recognizer skipped (optimisticTriples below): a copula, a
|
|
23
|
+
// lexicon relation verb, or one of the closed newswire event
|
|
24
|
+
// verbs read in a tighter frame, flanked by two resolvable
|
|
25
|
+
// entities, becomes a candidate triple, stored under its OWN
|
|
25
26
|
// low-trust source kind (optimistic-extract:<source>, prior 0.35 —
|
|
26
27
|
// below every curated pack, memory/trust.mjs) with NO operator or
|
|
27
28
|
// teach tag riding alongside, so a fuzzy candidate can never
|
|
@@ -51,8 +52,8 @@
|
|
|
51
52
|
//
|
|
52
53
|
// The extractor also says HOW it read a sentence, as named structural findings
|
|
53
54
|
// with their own detectors — never a score. A candidate the detectors show was
|
|
54
|
-
// mis-read is declined by name (`relative-clause-verb`, `fragment-term
|
|
55
|
-
// definitional frame ("X is the name for Y") declines the false isa and mints
|
|
55
|
+
// mis-read is declined by name (`relative-clause-verb`, `fragment-term`,
|
|
56
|
+
// `phrasal-particle`), and a definitional frame ("X is the name for Y") declines the false isa and mints
|
|
56
57
|
// the edge the sentence actually states (`mgx:nameFor`, `definitional-frame`).
|
|
57
58
|
// The ingest result reports both as `declined` and `minted`.
|
|
58
59
|
|
|
@@ -67,8 +68,9 @@ import { loadMemory, readFactRows, appendFacts, removeFacts } from "../adapters/
|
|
|
67
68
|
import { loadConfig } from "../adapters/config.mjs";
|
|
68
69
|
import { touchedFactRows } from "../domain/memory/touched-facts.mjs";
|
|
69
70
|
import { INGEST_SESSION_MARKER } from "../domain/memory/trust.mjs";
|
|
70
|
-
import { normFactTerm } from "../domain/hash.mjs";
|
|
71
|
+
import { normFactTerm, factIdForTriple } from "../domain/hash.mjs";
|
|
71
72
|
import { splitIdentifierWords } from "../domain/prose.mjs";
|
|
73
|
+
import { baseVerbSurface } from "../domain/fact-phrase.mjs";
|
|
72
74
|
import { winkInstance } from "../adapters/wink-model.mjs";
|
|
73
75
|
import {
|
|
74
76
|
loadLexicon, lookupNoun, lookupVerb, lookupAdjective, lookupProperName, predicateOf,
|
|
@@ -156,6 +158,14 @@ const COPULA_NAMING_PARTICIPLES = new Set(["termed", "known", "defined", "descri
|
|
|
156
158
|
// subject: "a mountain that has lava" is a fact about the volcano, so the
|
|
157
159
|
// relative clause's verb binds to the copula's own subject, not to its object.
|
|
158
160
|
const RELATIVE_PRONOUNS = new Set(["that", "which", "who", "whom", "whose"]);
|
|
161
|
+
// A pronoun standing between a copula and the nearest noun on its left IS that
|
|
162
|
+
// copula's subject: "many say it is just a matter of time" predicates about
|
|
163
|
+
// "it", and a scan that walks through the pronoun lands on whatever noun sits
|
|
164
|
+
// further left and stores a claim the sentence never made. A pronoun names
|
|
165
|
+
// nothing a fact can hold, so the subject side stops there and the frame
|
|
166
|
+
// abstains — the same answer the strict recognizer already gives a
|
|
167
|
+
// pronoun-subject claim.
|
|
168
|
+
const COPULA_SUBJECT_BLOCKERS = new Set([...COPULA_FRAME_BLOCKERS, "PRON"]);
|
|
159
169
|
// The copula-object heads that define a subject rather than classify it. "X is
|
|
160
170
|
// the name for Y" says what X names; it does not put X under the class "name".
|
|
161
171
|
// Followed by "for" or "of", one of these declines the isa and mints the edge
|
|
@@ -174,12 +184,178 @@ const MAX_TRIPLES_PER_SENTENCE = 4;
|
|
|
174
184
|
// abstains rather than guess, so a long noun pile never mints a stray class.
|
|
175
185
|
const ATTRIBUTIVE_CHAIN_MAX_HOPS = 8;
|
|
176
186
|
|
|
187
|
+
// The verbs a news report states an event with. The relation arm above reads a
|
|
188
|
+
// verb only when the lexicon declares one, and the lexicon's verb list is a
|
|
189
|
+
// software vocabulary — so a whole newswire paragraph ("the moon will
|
|
190
|
+
// completely block the sun") carries no relation under it at all. This closed
|
|
191
|
+
// band sits beside it: transitive event verbs, each of which takes a direct
|
|
192
|
+
// object naming the thing the event happened to, so the frame below can demand
|
|
193
|
+
// an adjacent noun on each side and get the actor and the affected thing.
|
|
194
|
+
//
|
|
195
|
+
// Verbs of speech and attribution are deliberately absent — "say", "tell",
|
|
196
|
+
// "add", "report", "accuse", "claim" each open a reported clause, and the noun
|
|
197
|
+
// after one is the subject of what was said, never the object of the saying.
|
|
198
|
+
// So are verbs whose everyday reading swamps their news one ("hold", "face",
|
|
199
|
+
// "follow", "lead", "reach", "back", "pass", "cut"): a band that admits those
|
|
200
|
+
// buys a handful of events and pays for it in nonsense.
|
|
201
|
+
const NEWSWIRE_RELATION_VERBS = new Set([
|
|
202
|
+
"hit", "strike", "kill", "injure", "wound", "damage", "destroy", "devastate",
|
|
203
|
+
"ban", "halt", "block", "bar", "suspend", "impose",
|
|
204
|
+
"arrest", "detain", "jail", "charge", "convict", "sentence", "deport", "release", "free",
|
|
205
|
+
"elect", "appoint", "oust", "overthrow",
|
|
206
|
+
"sign", "adopt", "approve", "reject", "veto",
|
|
207
|
+
"launch", "unveil", "seize", "capture", "invade", "attack", "bomb", "target",
|
|
208
|
+
"discover", "uncover", "rescue", "evacuate",
|
|
209
|
+
"spark", "trigger", "cause", "force", "deploy", "restore", "expand",
|
|
210
|
+
]);
|
|
211
|
+
// Two verbs the band already holds can report ONE event: a report wounds the
|
|
212
|
+
// people its next sentence says were injured, and the card then states the
|
|
213
|
+
// same act twice. Each pair below is one act under two words, so the left
|
|
214
|
+
// lemma reads as the right one and both sentences land on a single edge —
|
|
215
|
+
// the vocabulary-level form of the rule that already puts "releases" and
|
|
216
|
+
// "released" on one edge.
|
|
217
|
+
//
|
|
218
|
+
// The bar for a pair is that the two words mean one thing wherever the band
|
|
219
|
+
// reads them. A pair that merely overlaps keeps its own edge, and it fails the
|
|
220
|
+
// bar in either of two ways. It can name two different acts on the same people
|
|
221
|
+
// ("detain"/"jail", "oust"/"overthrow"). Or one word can carry a second sense
|
|
222
|
+
// the other does not: "free" is "release from custody" for a prisoner and
|
|
223
|
+
// "pull out of the rubble" for a rescue, so folding it onto "release" turns
|
|
224
|
+
// "rescuers free quake victim" into a jail delivery. Nothing in the sentence
|
|
225
|
+
// tells those two senses apart, so the pair stays out.
|
|
226
|
+
const NEWSWIRE_VERB_SYNONYMS = new Map([
|
|
227
|
+
["wound", "injure"],
|
|
228
|
+
["uncover", "discover"],
|
|
229
|
+
["bar", "ban"],
|
|
230
|
+
]);
|
|
231
|
+
|
|
232
|
+
// The particles a phrasal verb carries. A frame that drops one states a
|
|
233
|
+
// different event ("Frenzy … Takes Over London" → "takes London"), and one
|
|
234
|
+
// that reads it as the thing the event touched states nonsense ("stocks sell
|
|
235
|
+
// out" → sell "out"), so the pair below is read whole or not at all.
|
|
236
|
+
const PHRASAL_PARTICLES = new Set([
|
|
237
|
+
"over", "out", "up", "down", "off", "in", "on", "away", "back", "aside", "through",
|
|
238
|
+
]);
|
|
239
|
+
// The closed verb+particle pairs read as one predicate. Each is a phrasal verb
|
|
240
|
+
// whose meaning is not its bare verb's — "take over" is not "take", "sell out"
|
|
241
|
+
// is not "sell" — so the pair mints `mgx:<lemma>-<particle>`, which
|
|
242
|
+
// fact-phrase.mjs already reads back as "takes over". A verb+particle the list
|
|
243
|
+
// doesn't hold is left to the tiers it already went through.
|
|
244
|
+
const PHRASAL_VERB_PAIRS = new Set([
|
|
245
|
+
"take over", "take out", "take up", "take on", "take down", "take back",
|
|
246
|
+
"sell out", "sell off",
|
|
247
|
+
"carry out",
|
|
248
|
+
"step down", "step up", "step aside", "step in",
|
|
249
|
+
"call off", "call up", "call out",
|
|
250
|
+
"hand over", "hand out", "hand down",
|
|
251
|
+
"rule out",
|
|
252
|
+
"pull out", "pull off", "pull down", "pull back",
|
|
253
|
+
"set out", "set up", "set off", "set aside",
|
|
254
|
+
"break out", "break up", "break down", "break off",
|
|
255
|
+
"lay off", "lay out",
|
|
256
|
+
"shut down", "shut off",
|
|
257
|
+
"turn out", "turn down", "turn over", "turn away", "turn back",
|
|
258
|
+
"give up", "give back", "give in",
|
|
259
|
+
"bring down", "bring back", "bring in",
|
|
260
|
+
"roll out", "roll back",
|
|
261
|
+
"hold back", "hold up", "hold off",
|
|
262
|
+
"back down", "scale back", "push back", "push through",
|
|
263
|
+
"knock out", "knock down", "wipe out", "shoot down", "strike down",
|
|
264
|
+
"head off", "fend off",
|
|
265
|
+
"point out", "speak out",
|
|
266
|
+
"close down", "open up",
|
|
267
|
+
"drop out", "walk out", "walk back",
|
|
268
|
+
"wind down", "phase out", "sign off",
|
|
269
|
+
"fall back", "move on",
|
|
270
|
+
]);
|
|
271
|
+
// The prepositions a subject run climbs OUT of to the head that governs it.
|
|
272
|
+
// "Frenzy for solar eclipse glasses takes over London" is about the frenzy;
|
|
273
|
+
// the run beside the verb is the preposition's object, never the subject.
|
|
274
|
+
// Deliberately without the temporal ones (after/before/during/since): a
|
|
275
|
+
// fronted temporal phrase is adverbial, so the noun the climb would reach for
|
|
276
|
+
// heads nothing.
|
|
277
|
+
const SUBJECT_CHAIN_PREPOSITIONS = new Set([
|
|
278
|
+
"of", "for", "in", "on", "at", "from", "with", "about", "over",
|
|
279
|
+
"against", "between", "among", "across", "near", "behind", "around",
|
|
280
|
+
"under", "inside", "outside", "amid",
|
|
281
|
+
]);
|
|
282
|
+
// The counting words that head an of-frame the reader has to look through to
|
|
283
|
+
// find what the sentence is about. "triggers hundreds of evacuations" is a fact
|
|
284
|
+
// about the evacuations, and "discovers hundreds of ancient amphorae" about the
|
|
285
|
+
// amphorae. Every relation frame here reads through these; the newswire event
|
|
286
|
+
// frame widens the same climb to the container heads (OF_PARTITIVE_HEADS) and
|
|
287
|
+
// the classifier ones, so "charged a group of Cuban men" is about the men.
|
|
288
|
+
// Anything outside those sets names its own head, so "restore the sacred glow
|
|
289
|
+
// of fireflies" restores the glow, not the fireflies, and "a piece of cake" is
|
|
290
|
+
// about the piece.
|
|
291
|
+
const OF_COUNT_HEADS = new Set(["hundred", "hundreds", "thousand", "thousands", "million", "millions", "dozen", "dozens", "score", "scores", "handful"]);
|
|
292
|
+
// The count phrases newswire writes in front of what an event touched. Closed
|
|
293
|
+
// by list, and each has to close on the number itself, so a bare preposition
|
|
294
|
+
// ("attacked at dawn") is never mistaken for one.
|
|
295
|
+
const COUNT_PHRASE_RE = /^(?:more than|at least|at most|as many as|up to|fewer than|less than|nearly|almost|about|around|over|roughly)\s+\d[\d,.]*$/i;
|
|
296
|
+
const COUNT_PHRASE_MAX_TOKENS = 4;
|
|
297
|
+
// A verb whose own auxiliary is a be-form heads a passive or a progressive
|
|
298
|
+
// ("was arrested by ICE", "are disappearing"), and there the noun on the
|
|
299
|
+
// subject side is what the event happened TO, not who did it — an active read
|
|
300
|
+
// of one states the reverse of the sentence. The scan crosses adverbs only, so
|
|
301
|
+
// a modal chain that is still active ("will completely block") reads on.
|
|
302
|
+
const BE_AUXILIARIES = new Set(["is", "are", "was", "were", "be", "been", "being", "am"]);
|
|
303
|
+
// The prepositions an agentless passive states its subject's own condition
|
|
304
|
+
// with: "banned FROM parliament elections", "deported TO Mexico", "detained AT
|
|
305
|
+
// the border". Each says where the subject ended up, so the pair reads back as
|
|
306
|
+
// one predicate about the subject. Deliberately narrow — "charged WITH
|
|
307
|
+
// smuggling people" and "convicted OF fraud" take a whole clause or an
|
|
308
|
+
// abstraction after them, not a place a fact can hold, and "of" would collide
|
|
309
|
+
// with the of-chain climbs above.
|
|
310
|
+
const PASSIVE_STATE_PREPOSITIONS = new Set([
|
|
311
|
+
"from", "to", "in", "at", "into", "on", "near", "under", "over", "across", "off",
|
|
312
|
+
]);
|
|
313
|
+
|
|
314
|
+
// wink's tokenizer keeps a sentence-final full stop glued to the word before
|
|
315
|
+
// it when that word ends the text ("… block the sun." tokenizes as one PROPN
|
|
316
|
+
// "sun."), so a term read off the last token would otherwise be stored with
|
|
317
|
+
// the sentence's own punctuation in its key. Only a LONE trailing stop comes
|
|
318
|
+
// off: an abbreviation carries interior stops too ("U.S.", "P.K.K.") and keeps
|
|
319
|
+
// every one of them.
|
|
320
|
+
const stripSentenceFinalStop = (word) => {
|
|
321
|
+
const text = String(word ?? "");
|
|
322
|
+
return text.endsWith(".") && !text.slice(0, -1).includes(".") ? text.slice(0, -1) : text;
|
|
323
|
+
};
|
|
324
|
+
|
|
325
|
+
/** The closed-band lemma a word spells, or null. A Title Case headline carries
|
|
326
|
+
* no tag a reader can trust, so the band's own vocabulary is what identifies
|
|
327
|
+
* its verb: the word itself, or its -s fold ("Halts" → halt). */
|
|
328
|
+
function newswireVerbLemma(word) {
|
|
329
|
+
const surface = stripSentenceFinalStop(String(word ?? "")).toLowerCase();
|
|
330
|
+
if (NEWSWIRE_RELATION_VERBS.has(surface)) return surface;
|
|
331
|
+
const bare = surface.endsWith("s") ? surface.slice(0, -1) : "";
|
|
332
|
+
return bare && NEWSWIRE_RELATION_VERBS.has(bare) ? bare : null;
|
|
333
|
+
}
|
|
334
|
+
|
|
177
335
|
/** Fold an entity surface to its stored key: a lexicon noun's lemma, else the
|
|
178
336
|
* word's own normFactTerm (the optimistic tier mints unlisted content nouns
|
|
179
337
|
* the way the strict teach lane already mints "redis"). */
|
|
180
|
-
function foldEntity(word, lexicon) {
|
|
181
|
-
const
|
|
182
|
-
|
|
338
|
+
function foldEntity(word, lexicon, taggedLemma = "") {
|
|
339
|
+
const surface = stripSentenceFinalStop(word);
|
|
340
|
+
// A multi-word name is stored exactly as it reads. "United States" is the
|
|
341
|
+
// name; "united state" is a lemma fold of a word that was never on its own.
|
|
342
|
+
if (surface.includes(" ")) return normFactTerm(surface);
|
|
343
|
+
const noun = lookupNoun(lexicon, surface.toLowerCase());
|
|
344
|
+
return normFactTerm(noun ? noun.lemma : singularHead(surface, taggedLemma));
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/** The singular a head noun folds to when the lexicon carries no entry for it:
|
|
348
|
+
* the tagger's own lemma, and only where that lemma is the surface with an -s
|
|
349
|
+
* or -es taken off. A lemma that respells the word further ("analyses" →
|
|
350
|
+
* "analyzes") is not a singular, and a proper noun keeps its own spelling
|
|
351
|
+
* ("Wales", "Netherlands"), which is what a tagger returns for one anyway. */
|
|
352
|
+
function singularHead(surface, taggedLemma) {
|
|
353
|
+
const word = String(surface ?? "");
|
|
354
|
+
const lemma = String(taggedLemma ?? "");
|
|
355
|
+
if (!lemma || lemma === word) return word;
|
|
356
|
+
const lower = word.toLowerCase();
|
|
357
|
+
const base = lemma.toLowerCase();
|
|
358
|
+
return lower === `${base}s` || lower === `${base}es` ? lemma : word;
|
|
183
359
|
}
|
|
184
360
|
|
|
185
361
|
// The shortest word ingestText's fact-degree scan treats as a content-noun
|
|
@@ -187,36 +363,254 @@ function foldEntity(word, lexicon) {
|
|
|
187
363
|
// lexical fallback's stopword set doesn't already carry.
|
|
188
364
|
const CANDIDATE_TERM_MIN_LENGTH = 3;
|
|
189
365
|
|
|
366
|
+
// An abbreviated personal title carries a trailing stop, which is why wink
|
|
367
|
+
// tags it PROPN and glues it to the surname ("Mr./PROPN Gilman/PROPN"). A
|
|
368
|
+
// title addresses a person; it never names one, so it comes off the front of a
|
|
369
|
+
// name run however short the run is.
|
|
370
|
+
const HONORIFIC_NAME_PREFIXES = new Set([
|
|
371
|
+
"mr", "mrs", "ms", "mx", "dr", "prof", "rev",
|
|
372
|
+
"sen", "rep", "gov", "gen", "capt", "col", "lt", "sgt", "maj",
|
|
373
|
+
]);
|
|
374
|
+
|
|
375
|
+
/** The name a run of capitalized tokens states, front-trimmed. A run of three
|
|
376
|
+
* or more sheds a leading role noun the lexicon knows ("Prime Minister Keir
|
|
377
|
+
* Starmer" → "Keir Starmer") or a hyphenated compound that only Title Case
|
|
378
|
+
* lifted to a proper noun ("Ex-Marine Robert Gilman" → "Robert Gilman"). The
|
|
379
|
+
* trim stops at two tokens, so "Count Binface" and "Lake Kariba" keep the word
|
|
380
|
+
* that belongs to the name. An honorific comes off at any length. */
|
|
381
|
+
function trimNameRun(words, lexicon) {
|
|
382
|
+
const bare = (word) => stripSentenceFinalStop(word).toLowerCase();
|
|
383
|
+
let start = 0;
|
|
384
|
+
while (start < words.length - 1 && HONORIFIC_NAME_PREFIXES.has(bare(words[start]))) start += 1;
|
|
385
|
+
while (words.length - start >= 3) {
|
|
386
|
+
const first = bare(words[start]);
|
|
387
|
+
if (!first.includes("-") && !lookupNoun(lexicon, first)) break;
|
|
388
|
+
start += 1;
|
|
389
|
+
}
|
|
390
|
+
return words.slice(start);
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
// A headline is set in Title Case AND carries no sentence-final stop. Both
|
|
394
|
+
// halves matter, and they buy different things — see headlineReadPos.
|
|
395
|
+
const SENTENCE_FINAL_STOPS = new Set([".", "!", "?"]);
|
|
396
|
+
const readsAsHeadline = (values) => readsAsTitleCase(values)
|
|
397
|
+
&& !SENTENCE_FINAL_STOPS.has(String(values[values.length - 1] ?? ""));
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* How a sentence's parts of speech read once Title Case is allowed for. A
|
|
401
|
+
* tagger given a headline has no lowercase to work from and comes back wrong in
|
|
402
|
+
* both directions: the line's verb reads PROPN ("Thailand/PROPN Halts/PROPN
|
|
403
|
+
* New/PROPN Gun/PROPN"), while its plainest noun reads VERB ("Permits/VERB",
|
|
404
|
+
* "Shooting/VERB"). Two corrections, on two different conditions.
|
|
405
|
+
*
|
|
406
|
+
* In ANY Title Case sentence, a word the event band or the lexicon spells as a
|
|
407
|
+
* verb IS the verb (and a `lemmas` array passed in is corrected to match): no
|
|
408
|
+
* name run holds one of those, so promoting it can only split a run the
|
|
409
|
+
* capitals glued together.
|
|
410
|
+
*
|
|
411
|
+
* Demoting an unlisted VERB to a noun takes the stricter test — a real
|
|
412
|
+
* sentence can pass the capitalization test on its own ("The delegation met
|
|
413
|
+
* Prime Minister Keir Starmer."), and demoting its verb would cost every tag
|
|
414
|
+
* it already had right. The missing sentence-final stop is what tells a
|
|
415
|
+
* headline from that.
|
|
416
|
+
*
|
|
417
|
+
* Never the first token either way: a headline opens on its subject, and "Bar
|
|
418
|
+
* Refaeli" opens on a name the band also spells. Never a band word straight
|
|
419
|
+
* after a determiner ("The Free Press"). Any other sentence gets its tags back
|
|
420
|
+
* untouched.
|
|
421
|
+
*/
|
|
422
|
+
function headlineReadPos(values, pos, lemmas, lexicon) {
|
|
423
|
+
if (!readsAsTitleCase(values)) return pos;
|
|
424
|
+
const demoteUnlistedVerbs = readsAsHeadline(values);
|
|
425
|
+
const read = [...pos];
|
|
426
|
+
for (let i = 1; i < values.length; i += 1) {
|
|
427
|
+
const banded = read[i - 1] === "DET" ? null : newswireVerbLemma(values[i]);
|
|
428
|
+
// The tagger's own lemma still counts where it reaches a band verb the
|
|
429
|
+
// surface fold cannot ("Freed" → free), so a headline's past participle
|
|
430
|
+
// keeps the tag it had.
|
|
431
|
+
const taggedBandVerb = read[i] === "VERB" && NEWSWIRE_RELATION_VERBS.has(String(lemmas?.[i] ?? "").toLowerCase());
|
|
432
|
+
if (banded) {
|
|
433
|
+
read[i] = "VERB";
|
|
434
|
+
if (lemmas) lemmas[i] = banded;
|
|
435
|
+
} else if (taggedBandVerb || lookupVerb(lexicon, stripSentenceFinalStop(String(values[i])).toLowerCase())) {
|
|
436
|
+
read[i] = "VERB";
|
|
437
|
+
} else if (read[i] === "VERB" && demoteUnlistedVerbs) {
|
|
438
|
+
read[i] = "NOUN";
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
return read;
|
|
442
|
+
}
|
|
443
|
+
|
|
190
444
|
/** Every NOUN/PROPN token `sentences` names, surface-form occurrence-counted —
|
|
191
445
|
* a POS tagger reads unknown words by context, so an unlisted noun ("wombat")
|
|
192
|
-
* counts exactly like a lexicon-known one.
|
|
193
|
-
|
|
446
|
+
* counts exactly like a lexicon-known one. A contiguous run of two or more
|
|
447
|
+
* PROPN tokens counts ONCE, as the whole name it spells ("Robert Gilman",
|
|
448
|
+
* "United States"); the run's own words never count beside it, because half a
|
|
449
|
+
* name is half a lookup. */
|
|
450
|
+
function candidateTermOccurrencesPos(sentences, nlp, lexicon) {
|
|
194
451
|
const counts = new Map();
|
|
195
452
|
for (const sentence of sentences) {
|
|
196
453
|
let values;
|
|
197
454
|
let pos;
|
|
455
|
+
let lemmas;
|
|
198
456
|
try {
|
|
199
457
|
const doc = nlp.readDoc(String(sentence || ""));
|
|
200
458
|
values = doc.tokens().out(nlp.its.value);
|
|
201
459
|
pos = doc.tokens().out(nlp.its.pos);
|
|
460
|
+
lemmas = doc.tokens().out(nlp.its.lemma);
|
|
202
461
|
} catch { continue; }
|
|
462
|
+
const headline = readsAsHeadline(values);
|
|
463
|
+
const taggedPos = pos;
|
|
464
|
+
pos = headlineReadPos(values, pos, lemmas, lexicon);
|
|
465
|
+
// A headline read demotes an unlisted verb to a noun so a run can span it
|
|
466
|
+
// ("Mass Shooting"). That much is right for a run, and wrong for a term of
|
|
467
|
+
// its own: "Arrives" belongs inside no name and names nothing on its own.
|
|
468
|
+
const namesOnlyInsideARun = (k) => taggedPos[k] === "VERB" && pos[k] !== "VERB";
|
|
469
|
+
// A headline's capitals say nothing about which words spell a name, so its
|
|
470
|
+
// runs read over every noun; ordinary prose keeps the capital as the tell
|
|
471
|
+
// and runs over proper nouns alone.
|
|
472
|
+
const runsWith = (k) => (headline ? pos[k] === "NOUN" || pos[k] === "PROPN" : pos[k] === "PROPN");
|
|
473
|
+
// A role noun standing in front of a name is a title on that name, not a
|
|
474
|
+
// term of its own — "President Trump" is one lookup, while "president" and
|
|
475
|
+
// "trump" are two half ones. Only a common noun the lexicon knows opens a
|
|
476
|
+
// run this way, and a headline's own capitals carry no such distinction.
|
|
477
|
+
const opensRoleTitledName = (k) => !headline && pos[k] === "NOUN" && pos[k + 1] === "PROPN"
|
|
478
|
+
&& Boolean(lookupNoun(lexicon, stripSentenceFinalStop(String(values[k])).toLowerCase()));
|
|
203
479
|
for (let i = 0; i < values.length; i += 1) {
|
|
204
480
|
if (pos[i] !== "NOUN" && pos[i] !== "PROPN") continue;
|
|
481
|
+
let hi = i;
|
|
482
|
+
if (runsWith(i) || opensRoleTitledName(i)) {
|
|
483
|
+
while (hi + 1 < values.length && runsWith(hi + 1)) hi += 1;
|
|
484
|
+
}
|
|
485
|
+
if (hi > i) {
|
|
486
|
+
const name = trimNameRun(values.slice(i, hi + 1), lexicon).join(" ");
|
|
487
|
+
counts.set(name, (counts.get(name) || 0) + 1);
|
|
488
|
+
i = hi;
|
|
489
|
+
continue;
|
|
490
|
+
}
|
|
491
|
+
if (namesOnlyInsideARun(i)) continue;
|
|
205
492
|
counts.set(values[i], (counts.get(values[i]) || 0) + 1);
|
|
206
493
|
}
|
|
207
494
|
}
|
|
208
495
|
return counts;
|
|
209
496
|
}
|
|
210
497
|
|
|
498
|
+
// The tags a noun phrase opens with. One of these in front of a token settles
|
|
499
|
+
// that the token heads a noun phrase of its own, whatever follows it — a
|
|
500
|
+
// determiner ("the day it happened"), a possessive, a numeral, or the
|
|
501
|
+
// preposition that governs the phrase ("immigrants to countries they have no
|
|
502
|
+
// connection to").
|
|
503
|
+
const NOUN_PHRASE_OPENERS = new Set(["DET", "PRON", "PART", "NUM", "ADP"]);
|
|
504
|
+
|
|
505
|
+
/** Does the token at `i` stand where only a clause's own verb can, despite its
|
|
506
|
+
* NOUN tag? A tagger reads "many say it is just a matter of time" as a noun on
|
|
507
|
+
* the very word the clause turns on, and a term scan then offers "say" as
|
|
508
|
+
* something to look up. One closed frame says otherwise: a subject pronoun and
|
|
509
|
+
* its own verb right behind the token. The pronoun has to be a subject one, so
|
|
510
|
+
* a relative clause hanging off a real noun ("a camera that would cost …")
|
|
511
|
+
* keeps its reading. */
|
|
512
|
+
function readsAsClauseVerb(values, pos, i) {
|
|
513
|
+
if (i > 0 && NOUN_PHRASE_OPENERS.has(pos[i - 1])) return false;
|
|
514
|
+
if (pos[i + 1] !== "PRON" || RELATIVE_PRONOUNS.has(String(values[i + 1] ?? "").toLowerCase())) return false;
|
|
515
|
+
return pos[i + 2] === "VERB" || pos[i + 2] === "AUX";
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/** The same question for the other frame a newswire sentence hides a verb in:
|
|
519
|
+
* an infinitive right behind the token, with a finished noun phrase in front
|
|
520
|
+
* of it — "the central government moves to assert control". The noun phrase is
|
|
521
|
+
* what makes the reading safe, because an infinitive alone follows nouns
|
|
522
|
+
* constantly ("rapid development to cater to tourists", "the only person to
|
|
523
|
+
* break the sound barrier"): those all carry an adjective, a determiner or a
|
|
524
|
+
* preposition on the token itself, and only a token whose own left neighbour
|
|
525
|
+
* is a common noun reads as the verb that noun phrase governs. */
|
|
526
|
+
function readsAsInfinitiveClauseVerb(values, pos, i) {
|
|
527
|
+
if (i === 0 || pos[i - 1] !== "NOUN") return false;
|
|
528
|
+
if (String(values[i + 1] ?? "").toLowerCase() !== "to") return false;
|
|
529
|
+
return pos[i + 2] === "VERB";
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
/** The stored term keys `sentences` only ever uses as a clause's verb. Folded
|
|
533
|
+
* the same way `ungroundedTermOccurrences` folds its own counts, so a caller
|
|
534
|
+
* can subtract this set from those keys directly. A term the text also uses as
|
|
535
|
+
* a plain noun somewhere ("the peace talks" beside "he talks to them") stays
|
|
536
|
+
* out of the set — one verb reading never disqualifies a word that names
|
|
537
|
+
* something elsewhere in the same article.
|
|
538
|
+
*
|
|
539
|
+
* `nlp` follows this module's own convention: absent means the shared wink
|
|
540
|
+
* instance, and an explicit null (no model) returns an empty set, since the
|
|
541
|
+
* frames below are read off part-of-speech tags. */
|
|
542
|
+
export function termsUsedOnlyAsVerbs(sentences, { lexicon = loadLexicon(), nlp } = {}) {
|
|
543
|
+
const engine = nlp === undefined ? winkInstance() : nlp;
|
|
544
|
+
const asVerb = new Map();
|
|
545
|
+
const total = new Map();
|
|
546
|
+
if (!engine) return new Set();
|
|
547
|
+
for (const sentence of sentences) {
|
|
548
|
+
let values;
|
|
549
|
+
let pos;
|
|
550
|
+
let lemmas;
|
|
551
|
+
try {
|
|
552
|
+
const doc = engine.readDoc(String(sentence || ""));
|
|
553
|
+
values = doc.tokens().out(engine.its.value);
|
|
554
|
+
pos = doc.tokens().out(engine.its.pos);
|
|
555
|
+
lemmas = doc.tokens().out(engine.its.lemma);
|
|
556
|
+
} catch { continue; }
|
|
557
|
+
pos = headlineReadPos(values, pos, lemmas, lexicon);
|
|
558
|
+
for (let i = 0; i < values.length; i += 1) {
|
|
559
|
+
if (pos[i] !== "NOUN" && pos[i] !== "PROPN") continue;
|
|
560
|
+
const term = foldEntity(values[i], lexicon);
|
|
561
|
+
if (!term) continue;
|
|
562
|
+
total.set(term, (total.get(term) || 0) + 1);
|
|
563
|
+
if (readsAsClauseVerb(values, pos, i) || readsAsInfinitiveClauseVerb(values, pos, i)) {
|
|
564
|
+
asVerb.set(term, (asVerb.get(term) || 0) + 1);
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
const verbs = new Set();
|
|
569
|
+
for (const [term, count] of asVerb) if (count === total.get(term)) verbs.add(term);
|
|
570
|
+
return verbs;
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/** A sentence whose substantial words are nearly all capitalized is a headline
|
|
574
|
+
* set in Title Case, where a capital says nothing about which words spell a
|
|
575
|
+
* name. The POS tier reads such a sentence by tag and is unaffected; the
|
|
576
|
+
* lexical fallback has only the capitals, so it reads no name runs there. */
|
|
577
|
+
function readsAsTitleCase(words) {
|
|
578
|
+
const substantial = words.filter((word) => word.length >= 4);
|
|
579
|
+
if (substantial.length < 4) return false;
|
|
580
|
+
const capitalized = substantial.filter((word) => /^[A-Z]/.test(word)).length;
|
|
581
|
+
return capitalized / substantial.length >= 0.8;
|
|
582
|
+
}
|
|
583
|
+
|
|
211
584
|
/** The no-wink-model fallback: every word that is neither a closed-class
|
|
212
585
|
* scaffolding token nor a lexicon-known verb/adjective counts as a candidate
|
|
213
586
|
* noun — narrower than the POS tier (no context to lean on), but the same
|
|
214
|
-
* "an unlisted word can still be a content noun" posture.
|
|
587
|
+
* "an unlisted word can still be a content noun" posture. Capitalization
|
|
588
|
+
* stands in for the missing tags when it carries information: two or more
|
|
589
|
+
* capitalized words separated by nothing but spaces count once, as one name. */
|
|
215
590
|
function candidateTermOccurrencesLexical(sentences, lexicon) {
|
|
216
591
|
const counts = new Map();
|
|
217
592
|
for (const sentence of sentences) {
|
|
218
|
-
const
|
|
219
|
-
|
|
593
|
+
const text = String(sentence || "");
|
|
594
|
+
const matches = [...text.matchAll(/[A-Za-z][A-Za-z'-]*/g)];
|
|
595
|
+
const words = matches.map((match) => match[0]);
|
|
596
|
+
const titleCase = readsAsTitleCase(words);
|
|
597
|
+
const spacedRunEnd = (start) => {
|
|
598
|
+
let hi = start;
|
|
599
|
+
while (hi + 1 < matches.length && /^[A-Z]/.test(words[hi + 1])
|
|
600
|
+
&& !text.slice(matches[hi].index + words[hi].length, matches[hi + 1].index).trim()) hi += 1;
|
|
601
|
+
return hi;
|
|
602
|
+
};
|
|
603
|
+
for (let i = 0; i < words.length; i += 1) {
|
|
604
|
+
const word = words[i];
|
|
605
|
+
if (!titleCase && /^[A-Z]/.test(word)) {
|
|
606
|
+
const hi = spacedRunEnd(i);
|
|
607
|
+
if (hi > i) {
|
|
608
|
+
const name = trimNameRun(words.slice(i, hi + 1), lexicon).join(" ");
|
|
609
|
+
counts.set(name, (counts.get(name) || 0) + 1);
|
|
610
|
+
i = hi;
|
|
611
|
+
continue;
|
|
612
|
+
}
|
|
613
|
+
}
|
|
220
614
|
const lower = word.toLowerCase();
|
|
221
615
|
if (lower.length < CANDIDATE_TERM_MIN_LENGTH) continue;
|
|
222
616
|
if (OPTIMISTIC_SKIP.has(lower)) continue;
|
|
@@ -227,14 +621,47 @@ function candidateTermOccurrencesLexical(sentences, lexicon) {
|
|
|
227
621
|
return counts;
|
|
228
622
|
}
|
|
229
623
|
|
|
624
|
+
/** A single-word term that is one word of exactly one multi-word name the same
|
|
625
|
+
* text captured is that name's fragment, not a term of its own: "Gilman"
|
|
626
|
+
* beside "Robert Gilman" is the same person, and only the whole name is a
|
|
627
|
+
* question a reference lookup can answer. Its occurrences move onto the name.
|
|
628
|
+
* A word no captured name holds ("Russia", standing alone) is left where it
|
|
629
|
+
* is, and a word two names share is dropped rather than guessed onto one. */
|
|
630
|
+
function foldNameFragments(counts) {
|
|
631
|
+
const namesByWord = new Map();
|
|
632
|
+
for (const term of counts.keys()) {
|
|
633
|
+
if (!term.includes(" ")) continue;
|
|
634
|
+
for (const word of term.split(" ")) {
|
|
635
|
+
if (!namesByWord.has(word)) namesByWord.set(word, new Set());
|
|
636
|
+
namesByWord.get(word).add(term);
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
for (const [term, occurrences] of [...counts]) {
|
|
640
|
+
if (term.includes(" ")) continue;
|
|
641
|
+
const names = namesByWord.get(term);
|
|
642
|
+
if (!names) continue;
|
|
643
|
+
counts.delete(term);
|
|
644
|
+
if (names.size !== 1) continue;
|
|
645
|
+
const [name] = names;
|
|
646
|
+
counts.set(name, (counts.get(name) || 0) + occurrences);
|
|
647
|
+
}
|
|
648
|
+
return counts;
|
|
649
|
+
}
|
|
650
|
+
|
|
230
651
|
/** Every fact-ungrounded term `sentences` names: a candidate noun folded to
|
|
231
652
|
* its stored term key (`foldEntity`, so a ledger entry and a stored fact key
|
|
232
653
|
* the same term identically) that `rows` holds zero fact rows for — the
|
|
233
654
|
* fact-degree rule an ungrounded-term ledger admits by, independent of
|
|
234
655
|
* whether the lexicon happens to know the word. Occurrence-counted, so a
|
|
235
|
-
* term named three times outranks one named once.
|
|
236
|
-
|
|
237
|
-
|
|
656
|
+
* term named three times outranks one named once.
|
|
657
|
+
*
|
|
658
|
+
* `nlp` follows this module's own convention: absent means the shared wink
|
|
659
|
+
* instance, and an explicit null forces the lexical fallback. */
|
|
660
|
+
export function ungroundedTermOccurrences(sentences, rows, { lexicon = loadLexicon(), nlp } = {}) {
|
|
661
|
+
const engine = nlp === undefined ? winkInstance() : nlp;
|
|
662
|
+
const raw = engine
|
|
663
|
+
? candidateTermOccurrencesPos(sentences, engine, lexicon)
|
|
664
|
+
: candidateTermOccurrencesLexical(sentences, lexicon);
|
|
238
665
|
const grounded = new Set();
|
|
239
666
|
for (const row of rows) {
|
|
240
667
|
grounded.add(normFactTerm(row.subject));
|
|
@@ -243,9 +670,13 @@ function ungroundedTermOccurrences(sentences, rows, { lexicon, nlp }) {
|
|
|
243
670
|
const counts = new Map();
|
|
244
671
|
for (const [word, n] of raw) {
|
|
245
672
|
const term = foldEntity(word, lexicon);
|
|
246
|
-
if (!term
|
|
673
|
+
if (!term) continue;
|
|
247
674
|
counts.set(term, (counts.get(term) || 0) + n);
|
|
248
675
|
}
|
|
676
|
+
// Fragments fold onto their whole name BEFORE the fact-degree filter, so a
|
|
677
|
+
// name a fact already grounds takes its own fragments out with it.
|
|
678
|
+
foldNameFragments(counts);
|
|
679
|
+
for (const term of [...counts.keys()]) if (grounded.has(term)) counts.delete(term);
|
|
249
680
|
return counts;
|
|
250
681
|
}
|
|
251
682
|
|
|
@@ -257,11 +688,14 @@ function ungroundedTermOccurrences(sentences, rows, { lexicon, nlp }) {
|
|
|
257
688
|
function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false } = {}) {
|
|
258
689
|
let values;
|
|
259
690
|
let pos;
|
|
691
|
+
let lemmas;
|
|
260
692
|
try {
|
|
261
693
|
const doc = nlp.readDoc(String(sentence || ""));
|
|
262
694
|
values = doc.tokens().out(nlp.its.value);
|
|
263
695
|
pos = doc.tokens().out(nlp.its.pos);
|
|
696
|
+
lemmas = doc.tokens().out(nlp.its.lemma);
|
|
264
697
|
} catch { return { triples: [], declined: [], minted: [] }; }
|
|
698
|
+
pos = headlineReadPos(values, pos, lemmas, lexicon);
|
|
265
699
|
// A found noun is read as its whole contiguous NOUN/PROPN run, head-lemma
|
|
266
700
|
// folded — "a string instrument" is the class "string instrument", never
|
|
267
701
|
// its modifier "string"; a single-word run keeps the plain lemma fold.
|
|
@@ -272,32 +706,61 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
272
706
|
let hi = i;
|
|
273
707
|
while (lo - 1 >= 0 && isNounish(lo - 1)) lo -= 1;
|
|
274
708
|
while (hi + 1 < values.length && isNounish(hi + 1)) hi += 1;
|
|
275
|
-
if (lo === hi) return foldEntity(values[i], lexicon);
|
|
276
|
-
|
|
277
|
-
|
|
709
|
+
if (lo === hi) return foldEntity(values[i], lexicon, lemmas?.[i]);
|
|
710
|
+
// A run of proper nouns spells a name, and a name sheds the honorific or
|
|
711
|
+
// role word stacked in front of it — "Ex-Marine Robert Gilman" is Robert
|
|
712
|
+
// Gilman. A run holding any common noun is a compound, not a name, and
|
|
713
|
+
// keeps every word ("disk operating system").
|
|
714
|
+
let words = values.slice(lo, hi + 1);
|
|
715
|
+
if (words.every((_, k) => pos[lo + k] === "PROPN")) words = trimNameRun(words, lexicon);
|
|
716
|
+
const last = stripSentenceFinalStop(words[words.length - 1]);
|
|
717
|
+
const head = lookupNoun(lexicon, last.toLowerCase());
|
|
718
|
+
if (words.length === 1) return normFactTerm(head ? head.lemma : singularHead(last, lemmas?.[hi]));
|
|
719
|
+
return normFactTerm([...words.slice(0, -1), head ? head.lemma : singularHead(last, lemmas?.[hi])].join(" "));
|
|
720
|
+
};
|
|
721
|
+
// Where a leftward scan resumes when it meets a comma, or -1 when the comma
|
|
722
|
+
// stands between it and another clause. A news sentence names its subject
|
|
723
|
+
// first and then interrupts itself — "Yabloko, the Russian antiwar party, is
|
|
724
|
+
// banned …", "Robert Gilman, Freed by Russia, Arrives …" — so a scan that
|
|
725
|
+
// stops dead on a comma never reaches the subject at all. It may cross when
|
|
726
|
+
// nothing between the comma and either an earlier comma or the sentence start
|
|
727
|
+
// predicates: a relative clause ("the quake, which killed 100 people,
|
|
728
|
+
// damaged …") holds a verb, and that keeps the scan out.
|
|
729
|
+
const commaCrossingFrom = (close) => {
|
|
730
|
+
for (let k = close - 1; k >= 0; k -= 1) {
|
|
731
|
+
if (values[k] === ",") return k;
|
|
732
|
+
if (pos[k] === "VERB" || pos[k] === "AUX" || pos[k] === "PUNCT") return -1;
|
|
733
|
+
}
|
|
734
|
+
return close;
|
|
278
735
|
};
|
|
279
736
|
const nearestEntityIndex = (idx, step, blocked = null) => {
|
|
737
|
+
let mayCrossComma = step < 0;
|
|
280
738
|
for (let i = idx + step; i >= 0 && i < values.length; i += step) {
|
|
281
|
-
if (pos[i] === "PUNCT")
|
|
739
|
+
if (pos[i] === "PUNCT") {
|
|
740
|
+
if (!mayCrossComma || values[i] !== ",") break;
|
|
741
|
+
const resume = commaCrossingFrom(i);
|
|
742
|
+
if (resume < 0) break;
|
|
743
|
+
mayCrossComma = false;
|
|
744
|
+
i = resume;
|
|
745
|
+
continue;
|
|
746
|
+
}
|
|
282
747
|
if (blocked && blocked.has(pos[i])) break;
|
|
283
748
|
if (isNounish(i)) return i;
|
|
284
749
|
}
|
|
285
750
|
return null;
|
|
286
751
|
};
|
|
287
|
-
const nearestEntity = (idx, step, blocked = null) => {
|
|
288
|
-
const i = nearestEntityIndex(idx, step, blocked);
|
|
289
|
-
return i === null ? null : entityRunAt(i);
|
|
290
|
-
};
|
|
291
752
|
// The subject-side mirror of the copula-object of-chain rule: when a found
|
|
292
|
-
// subject run is the
|
|
293
|
-
// snow …"
|
|
294
|
-
//
|
|
295
|
-
//
|
|
296
|
-
//
|
|
297
|
-
//
|
|
298
|
-
//
|
|
299
|
-
//
|
|
300
|
-
|
|
753
|
+
// subject run is the object of a preposition ("the weight of all of the
|
|
754
|
+
// snow …", "frenzy for eclipse glasses …"), climb to the head that governs
|
|
755
|
+
// the phrase ("weight", "frenzy"), bounded to two hops. A classifier head
|
|
756
|
+
// (type/kind/sort/…) or a counting one (hundreds/dozens/…) reads THROUGH —
|
|
757
|
+
// "a kind of X" and "hundreds of X" are both about X — so the inner noun is
|
|
758
|
+
// kept. When the run is governed by a preposition but no readable noun heads
|
|
759
|
+
// the phrase (a mis-tagged head, e.g. "the top of the mountain …"), return
|
|
760
|
+
// null: an honest abstain, never the inner-noun confusion ("mountain",
|
|
761
|
+
// "snow"). An ungoverned run is returned unchanged. Returns a run-lo index
|
|
762
|
+
// to fold, or null to abstain.
|
|
763
|
+
const prepositionChainSkip = (k) => {
|
|
301
764
|
const p = pos[k];
|
|
302
765
|
return p === "DET" || p === "ADJ" || p === "ADV" || p === "NUM";
|
|
303
766
|
};
|
|
@@ -305,12 +768,14 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
305
768
|
let lo = runLoOf(found);
|
|
306
769
|
for (let hop = 0; hop < 2; hop += 1) {
|
|
307
770
|
let g = lo - 1;
|
|
308
|
-
while (g >= 0 &&
|
|
309
|
-
if (g < 0 || values[g]?.toLowerCase()
|
|
771
|
+
while (g >= 0 && prepositionChainSkip(g)) g -= 1;
|
|
772
|
+
if (g < 0 || !SUBJECT_CHAIN_PREPOSITIONS.has(values[g]?.toLowerCase())) return lo; // ungoverned run
|
|
310
773
|
let k = g - 1;
|
|
311
|
-
while (k >= 0 && !isNounish(k)
|
|
774
|
+
while (k >= 0 && !isNounish(k)
|
|
775
|
+
&& (prepositionChainSkip(k) || SUBJECT_CHAIN_PREPOSITIONS.has(values[k]?.toLowerCase()))) k -= 1;
|
|
312
776
|
if (k < 0 || !isNounish(k)) return null; // no readable head — abstain
|
|
313
|
-
|
|
777
|
+
const head = String(values[k]).toLowerCase();
|
|
778
|
+
if (OF_CLASSIFIER_HEADS.has(head) || OF_COUNT_HEADS.has(head)) return lo; // reads through to the inner noun
|
|
314
779
|
lo = runLoOf(k);
|
|
315
780
|
}
|
|
316
781
|
return lo;
|
|
@@ -343,10 +808,13 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
343
808
|
// is IN the oceans", "land is grouped INTO continents") — none of them
|
|
344
809
|
// class membership.
|
|
345
810
|
// An of-chain on the object reads through a classifier head to the real
|
|
346
|
-
// class ("a type of mammal" → mammal)
|
|
347
|
-
//
|
|
811
|
+
// class ("a type of mammal" → mammal). A container or counting head states
|
|
812
|
+
// quantity instead ("a large body of ice", "dozens of wolves"), and quantity
|
|
813
|
+
// is no class at all, so both decline the isa.
|
|
348
814
|
const copulaObjectAt = (i) => {
|
|
815
|
+
let sawDeterminer = false;
|
|
349
816
|
for (let j = i + 1; j < values.length; j += 1) {
|
|
817
|
+
if (pos[j] === "DET" || pos[j] === "NUM") sawDeterminer = true;
|
|
350
818
|
// A naming periphrasis ("… termed as …", "… known as …") keeps the
|
|
351
819
|
// frame copular: skip the participle and its "as" and read on.
|
|
352
820
|
if ((pos[j] === "VERB" || pos[j] === "AUX") && COPULA_NAMING_PARTICIPLES.has(values[j]?.toLowerCase())
|
|
@@ -378,6 +846,12 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
378
846
|
while (hi + 1 < values.length && isNounish(hi + 1)) hi += 1;
|
|
379
847
|
}
|
|
380
848
|
const headWord = String(values[hi]).toLowerCase();
|
|
849
|
+
// A bare "-ed" complement straight after the copula is a predicative
|
|
850
|
+
// participle the tagger mis-read as a noun ("dozens have been rescued"),
|
|
851
|
+
// never a class: a real class complement carries a determiner or number
|
|
852
|
+
// ("has been a doctor"). Short true nouns in -ed (bed, seed, need) stay
|
|
853
|
+
// under the length bound.
|
|
854
|
+
if (!sawDeterminer && headWord.length >= 5 && headWord.endsWith("ed")) return null;
|
|
381
855
|
const nextWord = values[hi + 1]?.toLowerCase();
|
|
382
856
|
// "latency is the name for the time period …" defines latency; it does not
|
|
383
857
|
// put latency under the class "name". The isa is declined and the object
|
|
@@ -392,7 +866,7 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
392
866
|
const nextIsOf = nextWord === "of";
|
|
393
867
|
if (!nextIsOf) return { label: entityRunAt(j), hi };
|
|
394
868
|
if (OF_CLASSIFIER_HEADS.has(headWord)) { i = hi + 1; j = hi + 1; continue; }
|
|
395
|
-
if (OF_PARTITIVE_HEADS.has(headWord)) return null;
|
|
869
|
+
if (OF_PARTITIVE_HEADS.has(headWord) || OF_COUNT_HEADS.has(headWord)) return null;
|
|
396
870
|
return { label: entityRunAt(j), hi };
|
|
397
871
|
}
|
|
398
872
|
return null;
|
|
@@ -409,7 +883,7 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
409
883
|
const copulaSubjectAt = (i) => {
|
|
410
884
|
let k = i - 1;
|
|
411
885
|
while (k >= 0 && pos[k] === "AUX") k -= 1;
|
|
412
|
-
const found = nearestEntityIndex(k + 1, -1,
|
|
886
|
+
const found = nearestEntityIndex(k + 1, -1, COPULA_SUBJECT_BLOCKERS);
|
|
413
887
|
if (found === null) return null;
|
|
414
888
|
const climbed = climbSubjectRun(found);
|
|
415
889
|
if (climbed === null) return null;
|
|
@@ -418,6 +892,17 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
418
892
|
return { label: entityRunAt(climbed), hi };
|
|
419
893
|
};
|
|
420
894
|
|
|
895
|
+
// Does the verb complex headed at `i` open with a be-form auxiliary? Adverbs
|
|
896
|
+
// in between are crossed; anything else ends the complex.
|
|
897
|
+
const beAuxiliaryBefore = (i) => {
|
|
898
|
+
for (let k = i - 1; k >= 0; k -= 1) {
|
|
899
|
+
if (pos[k] === "ADV" || pos[k] === "PART") continue;
|
|
900
|
+
if (pos[k] !== "AUX") return false;
|
|
901
|
+
if (BE_AUXILIARIES.has(String(values[k]).toLowerCase())) return true;
|
|
902
|
+
}
|
|
903
|
+
return false;
|
|
904
|
+
};
|
|
905
|
+
|
|
421
906
|
const triples = [];
|
|
422
907
|
const declined = [];
|
|
423
908
|
const minted = [];
|
|
@@ -433,6 +918,182 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
433
918
|
};
|
|
434
919
|
const decline = (finding, candidate) => { declined.push({ finding, candidate }); };
|
|
435
920
|
|
|
921
|
+
// The newswire event frame — a closed band of transitive event verbs
|
|
922
|
+
// (NEWSWIRE_RELATION_VERBS) read in a much tighter frame than the lexicon
|
|
923
|
+
// arm's. The lexicon's own verbs keep their loose scans; a verb only this
|
|
924
|
+
// band knows has to earn its triple:
|
|
925
|
+
//
|
|
926
|
+
// - wink must have tagged the token VERB, and its LEMMA must be in the
|
|
927
|
+
// band, so a past tense ("released", "adopted") reads where the
|
|
928
|
+
// lexicon's -s-only fold cannot. A verb the LEXICON already declares is
|
|
929
|
+
// skipped by surface, because the lexicon arm read that sentence first
|
|
930
|
+
// and this frame would mint the same act a second time;
|
|
931
|
+
// - a verb followed by one of the closed phrasal pairs is admitted on the
|
|
932
|
+
// pair instead, whichever vocabulary the bare verb belongs to: "take
|
|
933
|
+
// over" is a different relation from "take", so the lexicon's entry for
|
|
934
|
+
// the verb has nothing to say about it;
|
|
935
|
+
// - it must not sit in a relative clause, which has no subject of its own
|
|
936
|
+
// here;
|
|
937
|
+
// - a be-form auxiliary heads a passive or a progressive rather than
|
|
938
|
+
// declining the sentence: the -ing form marks the progressive and is
|
|
939
|
+
// skipped, a "by" complement mints the ACTIVE edge with the actor as
|
|
940
|
+
// subject, and an agentless passive states the patient's own condition;
|
|
941
|
+
// - subject and object are each the NEAREST noun run on their side with
|
|
942
|
+
// the copula frame's blockers applied, so neither scan crosses a verb, a
|
|
943
|
+
// preposition or a conjunction into another clause. "resigned from
|
|
944
|
+
// Cambridge" yields no object at all rather than "resign Cambridge".
|
|
945
|
+
// The subject scan starts left of the verb's OWN modal chain ("will
|
|
946
|
+
// completely block"), which is one verb complex rather than a crossing,
|
|
947
|
+
// and climbs out of any preposition that governs it ("a fire AT the
|
|
948
|
+
// hospital killed …" is about the fire). A counting of-chain on either
|
|
949
|
+
// side reads through to what the event really touched ("hundreds of
|
|
950
|
+
// ancient amphorae" → amphorae).
|
|
951
|
+
//
|
|
952
|
+
// A lemma the lexicon itself declares keeps the lexicon's predicate, so
|
|
953
|
+
// "releases" (the lexicon arm) and "released" (this one) land on one edge —
|
|
954
|
+
// and NEWSWIRE_VERB_SYNONYMS folds the band's own interchangeable pairs onto
|
|
955
|
+
// that same edge before the lookup, so "freed" lands there too.
|
|
956
|
+
const verbComplexStart = (i) => {
|
|
957
|
+
let k = i - 1;
|
|
958
|
+
while (k >= 0 && (pos[k] === "ADV" || pos[k] === "AUX" || pos[k] === "PART")) k -= 1;
|
|
959
|
+
return k + 1;
|
|
960
|
+
};
|
|
961
|
+
// A counting word alone. The lexicon arm reads through these and no others:
|
|
962
|
+
// a container head states what the thing is made of or held in, and a
|
|
963
|
+
// classifier rewrites a class, so neither one moves what a relation verb
|
|
964
|
+
// touched.
|
|
965
|
+
const readsThroughCountOf = (word) => OF_COUNT_HEADS.has(String(word ?? "").toLowerCase());
|
|
966
|
+
// The newswire event frame's wider read-through: the counting words, plus the
|
|
967
|
+
// container and classifier heads a headline writes its patient behind.
|
|
968
|
+
const readsThroughOf = (word) => {
|
|
969
|
+
const w = String(word ?? "").toLowerCase();
|
|
970
|
+
return readsThroughCountOf(w) || OF_PARTITIVE_HEADS.has(w) || OF_CLASSIFIER_HEADS.has(w);
|
|
971
|
+
};
|
|
972
|
+
// Walk a found run's of-chain to the noun the phrase is really about, at most
|
|
973
|
+
// two hops so a chain of prepositions can never wander into another clause.
|
|
974
|
+
// `readsThrough` says which heads qualify; a head outside that set stops the
|
|
975
|
+
// climb and keeps its own run.
|
|
976
|
+
const ofChainEntityIndex = (at, readsThrough) => {
|
|
977
|
+
for (let hop = 0; at !== null && hop < 2; hop += 1) {
|
|
978
|
+
let hi = at;
|
|
979
|
+
while (hi + 1 < values.length && isNounish(hi + 1)) hi += 1;
|
|
980
|
+
if (values[hi + 1]?.toLowerCase() !== "of" || !readsThrough(values[hi])) break;
|
|
981
|
+
const inner = nearestEntityIndex(hi + 1, +1);
|
|
982
|
+
if (inner === null) break;
|
|
983
|
+
at = inner;
|
|
984
|
+
}
|
|
985
|
+
return at;
|
|
986
|
+
};
|
|
987
|
+
const countChainEntityIndex = (idx, step) =>
|
|
988
|
+
ofChainEntityIndex(nearestEntityIndex(idx, step, COPULA_FRAME_BLOCKERS), readsThroughOf);
|
|
989
|
+
const countChainEntity = (idx, step) => {
|
|
990
|
+
const at = countChainEntityIndex(idx, step);
|
|
991
|
+
return at === null ? null : entityRunAt(at);
|
|
992
|
+
};
|
|
993
|
+
// The object a lexicon relation verb takes: the nearest run rightward, on the
|
|
994
|
+
// arm's own blocker-free scan ("relies ON redis" has to read past the
|
|
995
|
+
// preposition), then climbed through a counting of-chain so the fact lands on
|
|
996
|
+
// what was counted rather than on the count word.
|
|
997
|
+
const relationVerbObject = (i) => {
|
|
998
|
+
const at = ofChainEntityIndex(nearestEntityIndex(i, +1), readsThroughCountOf);
|
|
999
|
+
return at === null ? null : entityRunAt(at);
|
|
1000
|
+
};
|
|
1001
|
+
// The noun an event predicates about: the nearest run leftward, climbed out
|
|
1002
|
+
// of any preposition it is the object of, so a phrase-headed subject reads as
|
|
1003
|
+
// its own head rather than as the noun that happens to sit beside the verb.
|
|
1004
|
+
const newswireSubject = (idx) => {
|
|
1005
|
+
const at = countChainEntityIndex(idx, -1);
|
|
1006
|
+
if (at === null) return null;
|
|
1007
|
+
const climbed = climbSubjectRun(at);
|
|
1008
|
+
return climbed === null ? null : entityRunAt(climbed);
|
|
1009
|
+
};
|
|
1010
|
+
// The phrasal verb headed at `i`, when the closed table holds the pair: its
|
|
1011
|
+
// own minted predicate, and the particle's index, so an object scan reads
|
|
1012
|
+
// past the particle to what the event touched.
|
|
1013
|
+
const phrasalVerbAt = (i) => {
|
|
1014
|
+
const particle = stripSentenceFinalStop(String(values[i + 1] ?? "")).toLowerCase();
|
|
1015
|
+
if (!PHRASAL_PARTICLES.has(particle)) return null;
|
|
1016
|
+
const lemma = String(lemmas?.[i] ?? values[i]).toLowerCase();
|
|
1017
|
+
if (!PHRASAL_VERB_PAIRS.has(`${lemma} ${particle}`)) return null;
|
|
1018
|
+
return { predicate: `mgx:${lemma}-${particle}`, particleAt: i + 1 };
|
|
1019
|
+
};
|
|
1020
|
+
// The first token of the complement a verb takes, adverbs crossed. A passive
|
|
1021
|
+
// reads its own frame off this one word: "by" names the actor, a locative or
|
|
1022
|
+
// directional preposition names where the subject ended up.
|
|
1023
|
+
const complementHead = (i) => {
|
|
1024
|
+
for (let k = i + 1; k < values.length; k += 1) {
|
|
1025
|
+
if (pos[k] === "ADV") continue;
|
|
1026
|
+
return k;
|
|
1027
|
+
}
|
|
1028
|
+
return -1;
|
|
1029
|
+
};
|
|
1030
|
+
// Newswire writes the count before the thing counted — "killed more than 100
|
|
1031
|
+
// people", "injured at least 30 workers" — and the preposition inside the
|
|
1032
|
+
// count phrase stops an object scan dead. Each lead below is skipped whole,
|
|
1033
|
+
// and only where the number itself closes the phrase, so "attacked at dawn"
|
|
1034
|
+
// (no number, no count) is untouched.
|
|
1035
|
+
const skipCountPhrase = (idx) => {
|
|
1036
|
+
for (let n = COUNT_PHRASE_MAX_TOKENS; n >= 2; n -= 1) {
|
|
1037
|
+
const span = values.slice(idx + 1, idx + 1 + n);
|
|
1038
|
+
if (span.length === n && COUNT_PHRASE_RE.test(span.join(" "))) return idx + n;
|
|
1039
|
+
}
|
|
1040
|
+
return idx;
|
|
1041
|
+
};
|
|
1042
|
+
const readNewswireFrame = () => {
|
|
1043
|
+
for (let i = 1; i < values.length - 1; i += 1) {
|
|
1044
|
+
if (pos[i] !== "VERB") continue;
|
|
1045
|
+
const lemma = String(lemmas?.[i] ?? values[i]).toLowerCase();
|
|
1046
|
+
const phrasal = phrasalVerbAt(i);
|
|
1047
|
+
if (!phrasal) {
|
|
1048
|
+
if (lookupVerb(lexicon, String(values[i]).toLowerCase())) continue;
|
|
1049
|
+
if (!NEWSWIRE_RELATION_VERBS.has(lemma)) continue;
|
|
1050
|
+
}
|
|
1051
|
+
if (relativePronounBefore(i) >= 0) continue;
|
|
1052
|
+
const canonical = NEWSWIRE_VERB_SYNONYMS.get(lemma) || lemma;
|
|
1053
|
+
const declared = lookupVerb(lexicon, canonical);
|
|
1054
|
+
const predicate = phrasal
|
|
1055
|
+
? phrasal.predicate
|
|
1056
|
+
: (declared ? predicateOf(declared) : `mgx:${canonical}`);
|
|
1057
|
+
const surface = stripSentenceFinalStop(String(values[i])).toLowerCase();
|
|
1058
|
+
// A phrasal verb's own complement starts past the particle, so the frame
|
|
1059
|
+
// below reads "was taken over BY Google" the same way it reads a bare
|
|
1060
|
+
// participle's "by".
|
|
1061
|
+
const head = complementHead(phrasal ? phrasal.particleAt : i);
|
|
1062
|
+
const headWord = head === -1 ? "" : String(values[head]).toLowerCase();
|
|
1063
|
+
const beAuxiliary = beAuxiliaryBefore(i);
|
|
1064
|
+
// A progressive is not an event that happened, and its -ing form is the
|
|
1065
|
+
// one thing that tells it apart from the past participle a passive takes.
|
|
1066
|
+
if (beAuxiliary && surface.endsWith("ing")) continue;
|
|
1067
|
+
// "<patient> (was) <participle> by <actor>" — the sentence names both
|
|
1068
|
+
// roles, so it mints the ACTIVE edge with the actor on the subject side.
|
|
1069
|
+
// A reduced passive carries no auxiliary at all ("Boats Hit by Mystery
|
|
1070
|
+
// Attackers"), so the "by" is what identifies the frame.
|
|
1071
|
+
if (headWord === "by") {
|
|
1072
|
+
const actor = countChainEntity(head, +1);
|
|
1073
|
+
const patient = newswireSubject(verbComplexStart(i));
|
|
1074
|
+
if (actor && patient) push(actor, predicate, patient);
|
|
1075
|
+
continue;
|
|
1076
|
+
}
|
|
1077
|
+
if (beAuxiliary) {
|
|
1078
|
+
// "<patient> is <participle> <prep> <complement>" — no actor is named,
|
|
1079
|
+
// so nothing can take the subject side of an active edge. The claim the
|
|
1080
|
+
// sentence DOES make is about the patient's own condition, and that is
|
|
1081
|
+
// what the participle predicate states. A phrasal pair already carries
|
|
1082
|
+
// its particle as that condition, and no second preposition can restate
|
|
1083
|
+
// it, so an agentless phrasal passive abstains.
|
|
1084
|
+
if (phrasal || !PASSIVE_STATE_PREPOSITIONS.has(headWord)) continue;
|
|
1085
|
+
const patient = newswireSubject(verbComplexStart(i));
|
|
1086
|
+
const complement = countChainEntity(head, +1);
|
|
1087
|
+
if (patient && complement) push(patient, `mgx:${surface}-${headWord}`, complement);
|
|
1088
|
+
continue;
|
|
1089
|
+
}
|
|
1090
|
+
const subject = newswireSubject(verbComplexStart(i));
|
|
1091
|
+
const object = countChainEntity(phrasal ? phrasal.particleAt : skipCountPhrase(i), +1);
|
|
1092
|
+
if (!subject || !object) continue;
|
|
1093
|
+
push(subject, predicate, object);
|
|
1094
|
+
}
|
|
1095
|
+
};
|
|
1096
|
+
|
|
436
1097
|
// Pass 1 — the first clean copula frame yields the isa (all guards unchanged);
|
|
437
1098
|
// its subject and object-run end anchor the relative-clause continuation.
|
|
438
1099
|
let copulaSubject = null;
|
|
@@ -485,15 +1146,17 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
485
1146
|
if (OPTIMISTIC_COPULAS.has(word)) continue;
|
|
486
1147
|
const verb = lookupVerb(lexicon, word);
|
|
487
1148
|
if (!verb) continue;
|
|
1149
|
+
if (phrasalVerbAt(i)) continue; // the phrasal frame reads the pair whole
|
|
488
1150
|
const relative = relativePronounBefore(i);
|
|
489
1151
|
if (relative >= 0 && relative - 1 !== copulaObjHi) {
|
|
490
|
-
decline("relative-clause-verb", { subject: copulaSubject, predicate: predicateOf(verb), object:
|
|
1152
|
+
decline("relative-clause-verb", { subject: copulaSubject, predicate: predicateOf(verb), object: relationVerbObject(i) });
|
|
491
1153
|
continue;
|
|
492
1154
|
}
|
|
493
1155
|
const subject = relative >= 0 ? copulaSubject : climbedSubjectAt(i);
|
|
494
1156
|
if (subject === null) continue;
|
|
495
|
-
push(subject, predicateOf(verb),
|
|
1157
|
+
push(subject, predicateOf(verb), relationVerbObject(i));
|
|
496
1158
|
}
|
|
1159
|
+
readNewswireFrame();
|
|
497
1160
|
return { triples, declined, minted };
|
|
498
1161
|
}
|
|
499
1162
|
|
|
@@ -508,14 +1171,16 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
508
1171
|
if (pos[i] !== "VERB") continue;
|
|
509
1172
|
const verb = lookupVerb(lexicon, values[i].toLowerCase());
|
|
510
1173
|
if (!verb) continue;
|
|
1174
|
+
if (phrasalVerbAt(i)) continue; // the phrasal frame reads the pair whole
|
|
511
1175
|
const subject = climbedSubjectAt(i);
|
|
512
1176
|
if (relativePronounBefore(i) >= 0) {
|
|
513
|
-
decline("relative-clause-verb", { subject, predicate: predicateOf(verb), object:
|
|
1177
|
+
decline("relative-clause-verb", { subject, predicate: predicateOf(verb), object: relationVerbObject(i) });
|
|
514
1178
|
continue;
|
|
515
1179
|
}
|
|
516
1180
|
if (subject === null) continue;
|
|
517
|
-
push(subject, predicateOf(verb),
|
|
1181
|
+
push(subject, predicateOf(verb), relationVerbObject(i));
|
|
518
1182
|
}
|
|
1183
|
+
readNewswireFrame();
|
|
519
1184
|
return { triples, declined, minted };
|
|
520
1185
|
}
|
|
521
1186
|
|
|
@@ -559,8 +1224,9 @@ function optimisticTriplesLexical(sentence, lexicon) {
|
|
|
559
1224
|
|
|
560
1225
|
/**
|
|
561
1226
|
* The bounded triple candidates from a sentence the strict recognizer skipped:
|
|
562
|
-
* a copula (→ rdfs:subClassOf)
|
|
563
|
-
*
|
|
1227
|
+
* a copula (→ rdfs:subClassOf), past its object the relation verbs it grounds
|
|
1228
|
+
* (→ their predicates), and the closed newswire event band in its own tighter
|
|
1229
|
+
* frame, so one sentence contributes every fact it holds
|
|
564
1230
|
* ("a volcano is a mountain that has lava" → volcano ⊑ mountain AND volcano has
|
|
565
1231
|
* lava). Every triple passes the same entity/guard checks on its own, deduped,
|
|
566
1232
|
* capped at MAX_TRIPLES_PER_SENTENCE so a run-on never shatters into noise; []
|
|
@@ -643,6 +1309,53 @@ export function clauseCandidates(sentence, { nlp } = {}) {
|
|
|
643
1309
|
return out;
|
|
644
1310
|
}
|
|
645
1311
|
|
|
1312
|
+
// The verbs a report attributes a claim with. Closed and deliberately short:
|
|
1313
|
+
// each of these carries the claim through unchanged, so the clause beside one
|
|
1314
|
+
// is what the article states. Hedging and reversing verbs stay out — "denied",
|
|
1315
|
+
// "alleged", "claimed" and "suggested" each change what the sentence says
|
|
1316
|
+
// about the clause, and a frame that unwrapped them would store the opposite of
|
|
1317
|
+
// the report.
|
|
1318
|
+
const REPORTED_SPEECH_VERB_SRC = "said|says|told|reported|reports|announced|announces|stated|states|confirmed|confirms|added|wrote|writes";
|
|
1319
|
+
// "<claim>, President Trump said." — the attribution rides the tail after a
|
|
1320
|
+
// comma and closes the sentence. A closing quote mark may sit between them.
|
|
1321
|
+
// Group 1 is the speaker.
|
|
1322
|
+
const TRAILING_ATTRIBUTION_RE = new RegExp(
|
|
1323
|
+
`,\\s*["'“”‘’]?\\s*([\\w.'’-]+(?:\\s+[\\w.'’-]+){0,3})\\s+(?:${REPORTED_SPEECH_VERB_SRC})\\s*[.!?]?\\s*$`,
|
|
1324
|
+
"i",
|
|
1325
|
+
);
|
|
1326
|
+
// "President Trump said (that) <claim>", "Mr. Gilman's family had said <claim>"
|
|
1327
|
+
// — the attribution leads and the claim is everything past it. Group 1 is the
|
|
1328
|
+
// speaker, group 2 the claim.
|
|
1329
|
+
const LEADING_ATTRIBUTION_RE = new RegExp(
|
|
1330
|
+
`^\\s*([\\w.'’-]+(?:\\s+[\\w.'’-]+){0,4})\\s+(?:has\\s+|had\\s+|have\\s+)?(?:${REPORTED_SPEECH_VERB_SRC})\\s+(?:that\\s+)?(\\S.*)$`,
|
|
1331
|
+
"i",
|
|
1332
|
+
);
|
|
1333
|
+
|
|
1334
|
+
/**
|
|
1335
|
+
* The claim a sentence attributes to a speaker, and the speaker it names. A
|
|
1336
|
+
* report states most of what it knows this way, and the recognizer reading the
|
|
1337
|
+
* WHOLE sentence reads the attribution as the claim: "Officials said the quake
|
|
1338
|
+
* killed more than 100 people." came back as `officials mgx:say quake killed
|
|
1339
|
+
* more than 100 people`, a whole clause stored as a term. Stripping the
|
|
1340
|
+
* attribution leaves the claim the article is making, which is the thing worth
|
|
1341
|
+
* grounding.
|
|
1342
|
+
*
|
|
1343
|
+
* Returns { claim, speaker }. `speaker` is "" when no attribution frame fired,
|
|
1344
|
+
* or when the claim left behind is too short to be one — the sentence comes
|
|
1345
|
+
* back whole in that case, so there is nothing the speaker would hang off.
|
|
1346
|
+
* Where both frames fire ("Officials said X, police reported."), the LEADING
|
|
1347
|
+
* speaker wins: it is the one attached to the clause that survived.
|
|
1348
|
+
*/
|
|
1349
|
+
export function reportedClauseOf(sentence) {
|
|
1350
|
+
const text = String(sentence ?? "").trim();
|
|
1351
|
+
const trailing = TRAILING_ATTRIBUTION_RE.exec(text);
|
|
1352
|
+
const trailingStripped = trailing ? text.replace(TRAILING_ATTRIBUTION_RE, ".") : text;
|
|
1353
|
+
const leading = LEADING_ATTRIBUTION_RE.exec(trailingStripped);
|
|
1354
|
+
const claim = leading ? leading[2].trim() : trailingStripped;
|
|
1355
|
+
if (claim.split(/\s+/).length < 3) return { claim: text, speaker: "" };
|
|
1356
|
+
return { claim, speaker: (leading?.[1] ?? trailing?.[1] ?? "").trim() };
|
|
1357
|
+
}
|
|
1358
|
+
|
|
646
1359
|
// A stored term names a thing. These words open a predicate remainder or a new
|
|
647
1360
|
// clause, so a term that STARTS with one is the tail of a sentence a recognizer
|
|
648
1361
|
// frame over-read, never an entity — "has a population of 1,683,115" and "and
|
|
@@ -699,6 +1412,50 @@ const COMPASS_LEAD_WORDS = new Set([
|
|
|
699
1412
|
"northeast", "northwest", "southeast", "southwest",
|
|
700
1413
|
"northern", "southern", "eastern", "western",
|
|
701
1414
|
]);
|
|
1415
|
+
// A name is one noun phrase. These words open a new phrase or a new clause, so
|
|
1416
|
+
// a term carrying one BETWEEN its first and last word is a headline a frame
|
|
1417
|
+
// tore into subject + predicate + remainder: "colombia as rescuers free quake
|
|
1418
|
+
// victim", "keir starmer faces a vote", "new gun licenses after mass shooting",
|
|
1419
|
+
// "boats hit by mystery attackers" — all four reach the graph as a card's own
|
|
1420
|
+
// hub or object otherwise.
|
|
1421
|
+
//
|
|
1422
|
+
// "of" is deliberately absent: it is the one preposition real names are built
|
|
1423
|
+
// with ("house of representatives", "united states of america", "isle of man").
|
|
1424
|
+
// The rule reads STRICTLY interior positions for the same reason the lead and
|
|
1425
|
+
// trailing rules read the edges separately — "may" and "will" end real surnames
|
|
1426
|
+
// ("theresa may", "brian may") and open nothing there.
|
|
1427
|
+
const INTERIOR_CLAUSE_WORDS = new Set([
|
|
1428
|
+
"a", "an", "the",
|
|
1429
|
+
"and", "or", "but", "because", "since", "although", "though", "whereas", "while", "so",
|
|
1430
|
+
"if", "when", "then", "however", "as", "that", "which", "who", "whom", "whose",
|
|
1431
|
+
"is", "are", "was", "were", "be", "been", "being", "am", "has", "have", "had",
|
|
1432
|
+
"do", "does", "did", "can", "could", "will", "would", "should", "may", "might", "must",
|
|
1433
|
+
"in", "on", "at", "for", "to", "with", "from", "by", "into", "onto",
|
|
1434
|
+
"over", "under", "after", "before", "between", "during", "about", "near", "through",
|
|
1435
|
+
"against", "among", "within", "without", "per",
|
|
1436
|
+
]);
|
|
1437
|
+
|
|
1438
|
+
// Past participles that open a real name. Read on the lowercased stored key,
|
|
1439
|
+
// wink tags each of these VERB, so the POS rule below turns down "united
|
|
1440
|
+
// states" — the name world news reports most often — and every sibling built
|
|
1441
|
+
// the same way.
|
|
1442
|
+
const PARTICIPIAL_NAME_LEAD_WORDS = new Set(["united", "allied", "combined", "armed", "organized", "associated"]);
|
|
1443
|
+
|
|
1444
|
+
// A term the source itself wrapped in quotation marks is a title it quoted, and
|
|
1445
|
+
// a title is free to read as a clause — "Hackernews discusses \"Tim King,
|
|
1446
|
+
// AmigaDOS developer, has died\"" states a true fact about a whole headline.
|
|
1447
|
+
// The quotes are the tell, so they exempt the interior rule and nothing else.
|
|
1448
|
+
const QUOTED_TERM_RE = /^["“'‘].*["”'’]$/;
|
|
1449
|
+
|
|
1450
|
+
/** Does `term` carry a clause- or phrase-opening word strictly between its
|
|
1451
|
+
* first and last word? */
|
|
1452
|
+
export function carriesInteriorClauseWord(words) {
|
|
1453
|
+
if (words.length >= 2 && QUOTED_TERM_RE.test(`${words[0]} ${words[words.length - 1]}`)) return false;
|
|
1454
|
+
for (let i = 1; i < words.length - 1; i += 1) {
|
|
1455
|
+
if (INTERIOR_CLAUSE_WORDS.has(String(words[i]).toLowerCase())) return true;
|
|
1456
|
+
}
|
|
1457
|
+
return false;
|
|
1458
|
+
}
|
|
702
1459
|
|
|
703
1460
|
/** Does `term` read as a thing's name rather than a clause fragment? Bounds
|
|
704
1461
|
* the word count and rejects a leading conjunction, auxiliary, preposition,
|
|
@@ -717,7 +1474,9 @@ export function readsAsEntityTerm(term, nlp) {
|
|
|
717
1474
|
if (PARTICLE_LEAD_WORDS.has(first)) return false;
|
|
718
1475
|
if (PRONOUN_LEAD_WORDS.has(first.replace(CLITIC_SUFFIX_RE, ""))) return false;
|
|
719
1476
|
if (TRAILING_AUXILIARY_WORDS.has(words[words.length - 1].toLowerCase())) return false;
|
|
1477
|
+
if (carriesInteriorClauseWord(words)) return false;
|
|
720
1478
|
if (COMPASS_LEAD_WORDS.has(first) && words[1].toLowerCase() !== "of") return true;
|
|
1479
|
+
if (PARTICIPIAL_NAME_LEAD_WORDS.has(first)) return true;
|
|
721
1480
|
const engine = nlp === undefined ? winkInstance() : nlp;
|
|
722
1481
|
if (!engine) return true;
|
|
723
1482
|
try {
|
|
@@ -729,6 +1488,27 @@ export function readsAsEntityTerm(term, nlp) {
|
|
|
729
1488
|
|
|
730
1489
|
const readsAsEntityFact = (fact, nlp) => readsAsEntityTerm(fact.subject, nlp) && readsAsEntityTerm(fact.object, nlp);
|
|
731
1490
|
|
|
1491
|
+
/** Did the frame that read this row cut a phrasal verb in half? "as prices
|
|
1492
|
+
* surge and stocks sell out" names no object at all, so a row pairing the
|
|
1493
|
+
* bare verb with its own particle ("stocks mgx:sell out") states nothing —
|
|
1494
|
+
* the particle belongs to the verb. Read off the same closed pair table the
|
|
1495
|
+
* optimistic tier reads a phrasal verb whole by, so only a particle that verb
|
|
1496
|
+
* really takes declines the row. */
|
|
1497
|
+
export function splitsPhrasalVerb(fact) {
|
|
1498
|
+
const particle = String(fact?.object ?? "").trim().toLowerCase();
|
|
1499
|
+
if (!PHRASAL_PARTICLES.has(particle)) return false;
|
|
1500
|
+
const surface = String(fact?.predicate ?? "").split(":").pop().toLowerCase();
|
|
1501
|
+
return PHRASAL_VERB_PAIRS.has(`${surface} ${particle}`)
|
|
1502
|
+
|| PHRASAL_VERB_PAIRS.has(`${baseVerbSurface(surface)} ${particle}`);
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
/** Why a candidate row is turned down, or null when it stands: the phrasal
|
|
1506
|
+
* split first (it names the row's real problem), then the entity-term rule. */
|
|
1507
|
+
const declineFindingFor = (fact, nlp) => {
|
|
1508
|
+
if (splitsPhrasalVerb(fact)) return "phrasal-particle";
|
|
1509
|
+
return readsAsEntityFact(fact, nlp) ? null : "fragment-term";
|
|
1510
|
+
};
|
|
1511
|
+
|
|
732
1512
|
/** Does one raw token read as a code identifier rather than a word? camelCase
|
|
733
1513
|
* and snake_case split into several words, and a dot between letters or a path
|
|
734
1514
|
* separator names a module or a file. Read on the sentence's own surface,
|
|
@@ -772,6 +1552,24 @@ function findingsForRow(fact, readingFindings, identifierTerms) {
|
|
|
772
1552
|
return named;
|
|
773
1553
|
}
|
|
774
1554
|
|
|
1555
|
+
const REPORTED_SPEECH_FINDING = "reported-speech";
|
|
1556
|
+
const ATTRIBUTED_TO_PREDICATE = "mgx:attributedTo";
|
|
1557
|
+
|
|
1558
|
+
/** The reified attribution beside a claim row: `fact:<claimHash> |
|
|
1559
|
+
* mgx:attributedTo | <speaker>`. A fact is content-addressed by its own
|
|
1560
|
+
* triple, so naming the claim's group id costs no read back. Null when the
|
|
1561
|
+
* sentence attributed nothing. */
|
|
1562
|
+
function attributionRowFor(fact, { speaker, tag, observedAt }) {
|
|
1563
|
+
if (!speaker) return null;
|
|
1564
|
+
return {
|
|
1565
|
+
subject: factIdForTriple(fact.subject, fact.predicate, fact.object),
|
|
1566
|
+
predicate: ATTRIBUTED_TO_PREDICATE,
|
|
1567
|
+
object: speaker,
|
|
1568
|
+
provenance: tag,
|
|
1569
|
+
observedAt,
|
|
1570
|
+
};
|
|
1571
|
+
}
|
|
1572
|
+
|
|
775
1573
|
// A host that shares one thread with a UI hands the thread back through this;
|
|
776
1574
|
// a Node run leaves it unset and pays nothing.
|
|
777
1575
|
let ingestYield = null;
|
|
@@ -826,6 +1624,15 @@ function canonicalLines(facts, storeRows) {
|
|
|
826
1624
|
* ingestText — the single ingest seam. Grounds `text` into facts and returns a
|
|
827
1625
|
* structured result; the CLI, the browser and the tool layer all drive this.
|
|
828
1626
|
*
|
|
1627
|
+
* A sentence that attributes its claim to a speaker is handed to the recognizer
|
|
1628
|
+
* as the claim alone (`reportedClauseOf`); the ungrounded-term scan still reads
|
|
1629
|
+
* the sentence as written, so the speaker keeps reaching the enrichment queue.
|
|
1630
|
+
* Under `findings` the speaker is also stored, twice over: the claim carries the
|
|
1631
|
+
* `reported-speech` finding, and a reified `fact:<claimId> | mgx:attributedTo |
|
|
1632
|
+
* <speaker>` row goes into the same batch. The finding is the half that must
|
|
1633
|
+
* survive — a reader that loses the attribution still sees the claim say which
|
|
1634
|
+
* reading it came from, so the two are written together or not at all.
|
|
1635
|
+
*
|
|
829
1636
|
* text the raw string to ground.
|
|
830
1637
|
* options:
|
|
831
1638
|
* memoryDir write grounded facts here (a real .tmct memory dir). Omit for
|
|
@@ -848,13 +1655,15 @@ function canonicalLines(facts, storeRows) {
|
|
|
848
1655
|
* lexicon a loaded lexicon; the core vocabulary when absent.
|
|
849
1656
|
* findings record how each sentence read: mint the edge a definitional
|
|
850
1657
|
* frame states (`mgx:nameFor`) where the false isa was
|
|
851
|
-
* declined,
|
|
852
|
-
* `clause-fallback`, `pronoun-carry`, `definitional-frame
|
|
853
|
-
* the assertions this call writes
|
|
854
|
-
*
|
|
1658
|
+
* declined, attach the kept findings (`identifier-token`,
|
|
1659
|
+
* `clause-fallback`, `pronoun-carry`, `definitional-frame`,
|
|
1660
|
+
* `reported-speech`) to the assertions this call writes, and
|
|
1661
|
+
* write the attribution row beside each reported-speech claim.
|
|
1662
|
+
* Off by default; the declines themselves are reported either
|
|
1663
|
+
* way.
|
|
855
1664
|
*
|
|
856
|
-
* Returns { sentences, recognized, extracted, optimistic,
|
|
857
|
-
* minted, canonical? }.
|
|
1665
|
+
* Returns { sentences, recognized, extracted, optimistic, attributions, skipped,
|
|
1666
|
+
* declined, minted, canonical? }.
|
|
858
1667
|
* recognized — strict-recognized sentence count.
|
|
859
1668
|
* extracted — strict fact rows ({subject, predicate, object, provenance,
|
|
860
1669
|
* quantifier, sentence}), plus `extraction` when the row was
|
|
@@ -862,6 +1671,11 @@ function canonicalLines(facts, storeRows) {
|
|
|
862
1671
|
* optimistic — fuzzy candidate rows ({subject, predicate, object, provenance,
|
|
863
1672
|
* sentence}, same optional `extraction`); always [] unless
|
|
864
1673
|
* options.optimistic.
|
|
1674
|
+
* attributions
|
|
1675
|
+
* — the attribution rows written beside the two arrays above, in
|
|
1676
|
+
* the same shape. Reported apart from them because they are not
|
|
1677
|
+
* facts the article stated, so nothing counting what an article
|
|
1678
|
+
* taught should count them.
|
|
865
1679
|
* skipped — sentences neither tier grounded.
|
|
866
1680
|
* declined — every candidate a detector turned down, as
|
|
867
1681
|
* { sentence, finding, candidate }.
|
|
@@ -907,6 +1721,12 @@ export async function ingestText(text, {
|
|
|
907
1721
|
|
|
908
1722
|
const extracted = [];
|
|
909
1723
|
const optimisticFacts = [];
|
|
1724
|
+
// The reified attributions written beside the claim rows above. Kept out of
|
|
1725
|
+
// `extracted`/`optimistic` on purpose: those two arrays are what the article
|
|
1726
|
+
// stated, and they feed a snapshot's fact ids and the bench's own score. An
|
|
1727
|
+
// attribution says who said one of them, which is not another fact the
|
|
1728
|
+
// article stated.
|
|
1729
|
+
const attributions = [];
|
|
910
1730
|
// How each sentence read: the candidates a detector turned down, and the
|
|
911
1731
|
// edges minted where one was declined.
|
|
912
1732
|
const declined = [];
|
|
@@ -951,13 +1771,13 @@ export async function ingestText(text, {
|
|
|
951
1771
|
currentRows = afterRows;
|
|
952
1772
|
if (!recognized) { lastDecline = decline || lastDecline; return null; }
|
|
953
1773
|
const fresh = rows.filter((row) => !taggedIds.has(row.id));
|
|
954
|
-
const kept = fresh.filter((row) =>
|
|
1774
|
+
const kept = fresh.filter((row) => !declineFindingFor(row, termNlp));
|
|
955
1775
|
if (kept.length !== fresh.length) {
|
|
956
1776
|
for (const row of fresh) {
|
|
957
1777
|
if (kept.includes(row)) continue;
|
|
958
1778
|
declined.push({
|
|
959
1779
|
sentence: currentSentence,
|
|
960
|
-
finding:
|
|
1780
|
+
finding: declineFindingFor(row, termNlp),
|
|
961
1781
|
candidate: { subject: row.subject, predicate: row.predicate, object: row.object },
|
|
962
1782
|
});
|
|
963
1783
|
}
|
|
@@ -981,23 +1801,33 @@ export async function ingestText(text, {
|
|
|
981
1801
|
sentenceCount += 1;
|
|
982
1802
|
lastDecline = "";
|
|
983
1803
|
currentSentence = sentence;
|
|
984
|
-
const
|
|
985
|
-
|
|
1804
|
+
const asWritten = stripCitationResidue(sentence);
|
|
1805
|
+
// The ungrounded-term scan reads the sentence AS WRITTEN, so a speaker
|
|
1806
|
+
// the article named still reaches the enrichment queue; only the
|
|
1807
|
+
// recognizer reads the claim on its own.
|
|
1808
|
+
cleanedSentences.push(asWritten);
|
|
1809
|
+
const { claim: cleaned, speaker } = reportedClauseOf(asWritten);
|
|
986
1810
|
// The stored term keys this sentence names with an identifier-shaped
|
|
987
1811
|
// token, read off the surface before normFactTerm folds the shape away.
|
|
988
1812
|
// Null when this run records no findings.
|
|
989
1813
|
const identifierTerms = findings ? identifierTermsIn(cleaned) : null;
|
|
1814
|
+
// The finding every row off this sentence carries, and the speaker each
|
|
1815
|
+
// one is attributed to. Both halves ride the same switch: the finding is
|
|
1816
|
+
// what a reader falls back on when the attribution is gone, so writing
|
|
1817
|
+
// one without the other would invert the fallback.
|
|
1818
|
+
const attributedTo = identifierTerms && speaker ? speaker : "";
|
|
1819
|
+
const speechFindings = attributedTo ? [REPORTED_SPEECH_FINDING] : [];
|
|
990
1820
|
// How the strict tier reached its rows, when it did: the whole sentence
|
|
991
1821
|
// carries nothing, a later candidate is a clause fragment, and the
|
|
992
1822
|
// pronoun retry carried its subject in from an earlier sentence.
|
|
993
|
-
let readingFindings = [];
|
|
1823
|
+
let readingFindings = [...speechFindings];
|
|
994
1824
|
// Whole sentence first, then each closed-marker clause as a fallback.
|
|
995
1825
|
let rows = null;
|
|
996
1826
|
const candidates = clauseCandidates(cleaned, { nlp });
|
|
997
1827
|
for (let i = 0; i < candidates.length; i += 1) {
|
|
998
1828
|
rows = await strictRows(candidates[i]);
|
|
999
1829
|
if (!rows) continue;
|
|
1000
|
-
if (i > 0) readingFindings = ["clause-fallback"];
|
|
1830
|
+
if (i > 0) readingFindings = [...speechFindings, "clause-fallback"];
|
|
1001
1831
|
break;
|
|
1002
1832
|
}
|
|
1003
1833
|
// Bounded pronoun carry: a "they/it/these/those/this …" sentence the
|
|
@@ -1010,7 +1840,7 @@ export async function ingestText(text, {
|
|
|
1010
1840
|
const threaded = stripLeadingDiscourseAdverb(cleaned);
|
|
1011
1841
|
if (!rows && carrySubject && PRONOUN_LEAD_RE.test(threaded)) {
|
|
1012
1842
|
rows = await strictRows(threaded.replace(PRONOUN_LEAD_RE, `${articledSubject(carrySubject)} `));
|
|
1013
|
-
if (rows) readingFindings = ["pronoun-carry"];
|
|
1843
|
+
if (rows) readingFindings = [...speechFindings, "pronoun-carry"];
|
|
1014
1844
|
}
|
|
1015
1845
|
if (rows) {
|
|
1016
1846
|
recognizedSentences += 1;
|
|
@@ -1035,6 +1865,8 @@ export async function ingestText(text, {
|
|
|
1035
1865
|
...(extraction.length ? { extraction } : {}),
|
|
1036
1866
|
});
|
|
1037
1867
|
taggedIds.add(row.id);
|
|
1868
|
+
const attribution = attributionRowFor(row, { speaker: attributedTo, tag, observedAt });
|
|
1869
|
+
if (attribution) { writes.push(attribution); attributions.push({ ...attribution, sentence }); }
|
|
1038
1870
|
}
|
|
1039
1871
|
await appendFacts(dir, writes);
|
|
1040
1872
|
continue;
|
|
@@ -1046,8 +1878,9 @@ export async function ingestText(text, {
|
|
|
1046
1878
|
for (const { finding, candidate } of reading.declined) declined.push({ sentence, finding, candidate });
|
|
1047
1879
|
const keptCandidates = [];
|
|
1048
1880
|
for (const t of reading.triples) {
|
|
1049
|
-
|
|
1050
|
-
|
|
1881
|
+
const finding = declineFindingFor(t, termNlp);
|
|
1882
|
+
if (finding) declined.push({ sentence, finding, candidate: t });
|
|
1883
|
+
else keptCandidates.push(t);
|
|
1051
1884
|
}
|
|
1052
1885
|
// A minted edge rides its own finding: minted[].fact is the very object
|
|
1053
1886
|
// the candidate list holds, so the mint and its row match by identity
|
|
@@ -1063,12 +1896,15 @@ export async function ingestText(text, {
|
|
|
1063
1896
|
const tag = `optimistic-extract:${sourceTag}`;
|
|
1064
1897
|
const candidateWrites = [];
|
|
1065
1898
|
for (const t of keptCandidates) {
|
|
1066
|
-
const
|
|
1899
|
+
const mintFinding = mintFindings.has(t) ? [mintFindings.get(t)] : [];
|
|
1900
|
+
const extraction = findingsForRow(t, [...speechFindings, ...mintFinding], identifierTerms);
|
|
1067
1901
|
candidateWrites.push({
|
|
1068
1902
|
subject: t.subject, predicate: t.predicate, object: t.object, provenance: tag, observedAt,
|
|
1069
1903
|
...(extraction.length ? { extraction } : {}),
|
|
1070
1904
|
});
|
|
1071
1905
|
optimisticFacts.push({ ...t, provenance: tag, sentence, ...(extraction.length ? { extraction } : {}) });
|
|
1906
|
+
const attribution = attributionRowFor(t, { speaker: attributedTo, tag, observedAt });
|
|
1907
|
+
if (attribution) { candidateWrites.push(attribution); attributions.push({ ...attribution, sentence }); }
|
|
1072
1908
|
}
|
|
1073
1909
|
const { ids } = await appendFacts(dir, candidateWrites);
|
|
1074
1910
|
for (const id of ids) taggedIds.add(id);
|
|
@@ -1093,12 +1929,16 @@ export async function ingestText(text, {
|
|
|
1093
1929
|
const key = normFactTerm(term);
|
|
1094
1930
|
if (key && !ungroundedCounts.has(key)) ungroundedCounts.set(key, 1);
|
|
1095
1931
|
}
|
|
1932
|
+
// A decline names the word it tripped over, so the legacy set carries name
|
|
1933
|
+
// fragments too ("Gilman" out of "Robert Gilman"). Same fold, same reason.
|
|
1934
|
+
foldNameFragments(ungroundedCounts);
|
|
1096
1935
|
|
|
1097
1936
|
const result = {
|
|
1098
1937
|
sentences: sentenceCount,
|
|
1099
1938
|
recognized: recognizedSentences,
|
|
1100
1939
|
extracted,
|
|
1101
1940
|
optimistic: optimisticFacts,
|
|
1941
|
+
attributions,
|
|
1102
1942
|
skipped: sentenceCount - recognizedSentences - optimisticSentences,
|
|
1103
1943
|
ungroundedTerms: [...ungroundedTerms],
|
|
1104
1944
|
ungroundedCounts,
|