@polycode-projects/the-mechanical-code-talker 6.0.19 → 6.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -23
- package/bin/tmct.mjs +16 -33
- package/corpus/LICENSES.json +0 -21
- package/corpus/README.md +10 -13
- package/corpus/reference/manifest.json +19 -19
- package/corpus/reference/shards/ref-01.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-04.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-08.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-10.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-11.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-17.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-20.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-25.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-2c.jsonl.gz +0 -0
- package/corpus/tier2/generate.mjs +6 -142
- package/corpus/tier2/manifest.json +0 -42
- package/package.json +4 -4
- package/src/adapters/corpus/child-seed.mjs +74 -0
- package/src/adapters/corpus/conceptnet.mjs +45 -26
- package/src/adapters/memory/blocks.mjs +7 -1
- package/src/adapters/memory/core.mjs +453 -103
- package/src/adapters/memory/corpus-bands.mjs +33 -10
- package/src/adapters/memory/inspect.mjs +24 -5
- package/src/adapters/memory/rows.mjs +106 -9
- package/src/adapters/memory/shacl.mjs +10 -3
- package/src/domain/ask.mjs +27 -10
- package/src/domain/cli-verbs.mjs +3 -4
- package/src/domain/completions/group.mjs +8 -3
- package/src/domain/completions/infer.mjs +7 -2
- package/src/domain/completions/prune.mjs +5 -1
- package/src/domain/completions/rank.mjs +7 -2
- package/src/domain/digest/compose.mjs +5 -1
- package/src/domain/digest/select.mjs +12 -6
- package/src/domain/domain.mjs +15 -8
- package/src/domain/el-classify.mjs +11 -2
- package/src/domain/fact-phrase.mjs +86 -4
- package/src/domain/hash.mjs +9 -0
- package/src/domain/memory/bias.mjs +8 -4
- package/src/domain/memory/capability.mjs +12 -6
- package/src/domain/memory/fact-order.mjs +29 -0
- package/src/domain/memory/resolution.mjs +3 -0
- package/src/domain/news-feed.mjs +422 -56
- package/src/domain/reference-pack.mjs +5 -0
- package/src/domain/sense-scope.mjs +116 -0
- package/src/domain/sense-split.mjs +1 -1
- package/src/domain/syllogise.mjs +21 -13
- package/src/domain/tableau.mjs +23 -14
- package/src/domain/worlds-pack.mjs +5 -1
- package/src/services/adventure-autoplay.mjs +6 -1
- package/src/services/adventure-editor.mjs +43 -21
- package/src/services/adventure-viz.mjs +26 -9
- package/src/services/adventure.mjs +40 -10
- package/src/services/chat.mjs +253 -113
- package/src/services/extensions.mjs +51 -58
- package/src/services/extract-facts.mjs +670 -95
- package/src/services/init.mjs +4 -4
- package/src/services/ledger-viz.mjs +9 -4
- package/src/services/memory-panel-viz.mjs +4 -5
- package/src/services/mud-editor.mjs +40 -16
- package/src/services/mud-viz.mjs +8 -2
- package/src/services/mudiii-turn.mjs +5 -3
- package/src/services/mudiii-viz.mjs +8 -2
- package/src/services/news.mjs +277 -11
- package/src/services/research-viz.mjs +1 -1
- package/src/services/sprite-catalog-viz.mjs +10 -5
- package/src/surfaces/web/adventure-browser-entry.mjs +6 -12
- package/src/surfaces/web/memory-ask-browser.bundle.js +152 -151
- package/src/surfaces/web/mud-browser-entry.mjs +7 -11
- package/src/surfaces/web/research-browser-entry.mjs +5 -2
- package/corpus/tier2/aws.jsonl +0 -39
- package/corpus/tier2/java.jsonl +0 -31
- package/corpus/tier2/python.jsonl +0 -30
|
@@ -52,8 +52,8 @@
|
|
|
52
52
|
//
|
|
53
53
|
// The extractor also says HOW it read a sentence, as named structural findings
|
|
54
54
|
// with their own detectors — never a score. A candidate the detectors show was
|
|
55
|
-
// mis-read is declined by name (`relative-clause-verb`, `fragment-term
|
|
56
|
-
// definitional frame ("X is the name for Y") declines the false isa and mints
|
|
55
|
+
// mis-read is declined by name (`relative-clause-verb`, `fragment-term`,
|
|
56
|
+
// `phrasal-particle`), and a definitional frame ("X is the name for Y") declines the false isa and mints
|
|
57
57
|
// the edge the sentence actually states (`mgx:nameFor`, `definitional-frame`).
|
|
58
58
|
// The ingest result reports both as `declined` and `minted`.
|
|
59
59
|
|
|
@@ -68,8 +68,9 @@ import { loadMemory, readFactRows, appendFacts, removeFacts } from "../adapters/
|
|
|
68
68
|
import { loadConfig } from "../adapters/config.mjs";
|
|
69
69
|
import { touchedFactRows } from "../domain/memory/touched-facts.mjs";
|
|
70
70
|
import { INGEST_SESSION_MARKER } from "../domain/memory/trust.mjs";
|
|
71
|
-
import { normFactTerm } from "../domain/hash.mjs";
|
|
71
|
+
import { normFactTerm, factIdForTriple } from "../domain/hash.mjs";
|
|
72
72
|
import { splitIdentifierWords } from "../domain/prose.mjs";
|
|
73
|
+
import { baseVerbSurface } from "../domain/fact-phrase.mjs";
|
|
73
74
|
import { winkInstance } from "../adapters/wink-model.mjs";
|
|
74
75
|
import {
|
|
75
76
|
loadLexicon, lookupNoun, lookupVerb, lookupAdjective, lookupProperName, predicateOf,
|
|
@@ -157,6 +158,14 @@ const COPULA_NAMING_PARTICIPLES = new Set(["termed", "known", "defined", "descri
|
|
|
157
158
|
// subject: "a mountain that has lava" is a fact about the volcano, so the
|
|
158
159
|
// relative clause's verb binds to the copula's own subject, not to its object.
|
|
159
160
|
const RELATIVE_PRONOUNS = new Set(["that", "which", "who", "whom", "whose"]);
|
|
161
|
+
// A pronoun standing between a copula and the nearest noun on its left IS that
|
|
162
|
+
// copula's subject: "many say it is just a matter of time" predicates about
|
|
163
|
+
// "it", and a scan that walks through the pronoun lands on whatever noun sits
|
|
164
|
+
// further left and stores a claim the sentence never made. A pronoun names
|
|
165
|
+
// nothing a fact can hold, so the subject side stops there and the frame
|
|
166
|
+
// abstains — the same answer the strict recognizer already gives a
|
|
167
|
+
// pronoun-subject claim.
|
|
168
|
+
const COPULA_SUBJECT_BLOCKERS = new Set([...COPULA_FRAME_BLOCKERS, "PRON"]);
|
|
160
169
|
// The copula-object heads that define a subject rather than classify it. "X is
|
|
161
170
|
// the name for Y" says what X names; it does not put X under the class "name".
|
|
162
171
|
// Followed by "for" or "of", one of these declines the isa and mints the edge
|
|
@@ -199,20 +208,108 @@ const NEWSWIRE_RELATION_VERBS = new Set([
|
|
|
199
208
|
"discover", "uncover", "rescue", "evacuate",
|
|
200
209
|
"spark", "trigger", "cause", "force", "deploy", "restore", "expand",
|
|
201
210
|
]);
|
|
202
|
-
//
|
|
203
|
-
//
|
|
204
|
-
//
|
|
205
|
-
//
|
|
206
|
-
//
|
|
207
|
-
//
|
|
208
|
-
//
|
|
211
|
+
// Two verbs the band already holds can report ONE event: a report wounds the
|
|
212
|
+
// people its next sentence says were injured, and the card then states the
|
|
213
|
+
// same act twice. Each pair below is one act under two words, so the left
|
|
214
|
+
// lemma reads as the right one and both sentences land on a single edge —
|
|
215
|
+
// the vocabulary-level form of the rule that already puts "releases" and
|
|
216
|
+
// "released" on one edge.
|
|
217
|
+
//
|
|
218
|
+
// The bar for a pair is that the two words mean one thing wherever the band
|
|
219
|
+
// reads them. A pair that merely overlaps keeps its own edge, and it fails the
|
|
220
|
+
// bar in either of two ways. It can name two different acts on the same people
|
|
221
|
+
// ("detain"/"jail", "oust"/"overthrow"). Or one word can carry a second sense
|
|
222
|
+
// the other does not: "free" is "release from custody" for a prisoner and
|
|
223
|
+
// "pull out of the rubble" for a rescue, so folding it onto "release" turns
|
|
224
|
+
// "rescuers free quake victim" into a jail delivery. Nothing in the sentence
|
|
225
|
+
// tells those two senses apart, so the pair stays out.
|
|
226
|
+
const NEWSWIRE_VERB_SYNONYMS = new Map([
|
|
227
|
+
["wound", "injure"],
|
|
228
|
+
["uncover", "discover"],
|
|
229
|
+
["bar", "ban"],
|
|
230
|
+
]);
|
|
231
|
+
|
|
232
|
+
// The particles a phrasal verb carries. A frame that drops one states a
|
|
233
|
+
// different event ("Frenzy … Takes Over London" → "takes London"), and one
|
|
234
|
+
// that reads it as the thing the event touched states nonsense ("stocks sell
|
|
235
|
+
// out" → sell "out"), so the pair below is read whole or not at all.
|
|
236
|
+
const PHRASAL_PARTICLES = new Set([
|
|
237
|
+
"over", "out", "up", "down", "off", "in", "on", "away", "back", "aside", "through",
|
|
238
|
+
]);
|
|
239
|
+
// The closed verb+particle pairs read as one predicate. Each is a phrasal verb
|
|
240
|
+
// whose meaning is not its bare verb's — "take over" is not "take", "sell out"
|
|
241
|
+
// is not "sell" — so the pair mints `mgx:<lemma>-<particle>`, which
|
|
242
|
+
// fact-phrase.mjs already reads back as "takes over". A verb+particle the list
|
|
243
|
+
// doesn't hold is left to the tiers it already went through.
|
|
244
|
+
const PHRASAL_VERB_PAIRS = new Set([
|
|
245
|
+
"take over", "take out", "take up", "take on", "take down", "take back",
|
|
246
|
+
"sell out", "sell off",
|
|
247
|
+
"carry out",
|
|
248
|
+
"step down", "step up", "step aside", "step in",
|
|
249
|
+
"call off", "call up", "call out",
|
|
250
|
+
"hand over", "hand out", "hand down",
|
|
251
|
+
"rule out",
|
|
252
|
+
"pull out", "pull off", "pull down", "pull back",
|
|
253
|
+
"set out", "set up", "set off", "set aside",
|
|
254
|
+
"break out", "break up", "break down", "break off",
|
|
255
|
+
"lay off", "lay out",
|
|
256
|
+
"shut down", "shut off",
|
|
257
|
+
"turn out", "turn down", "turn over", "turn away", "turn back",
|
|
258
|
+
"give up", "give back", "give in",
|
|
259
|
+
"bring down", "bring back", "bring in",
|
|
260
|
+
"roll out", "roll back",
|
|
261
|
+
"hold back", "hold up", "hold off",
|
|
262
|
+
"back down", "scale back", "push back", "push through",
|
|
263
|
+
"knock out", "knock down", "wipe out", "shoot down", "strike down",
|
|
264
|
+
"head off", "fend off",
|
|
265
|
+
"point out", "speak out",
|
|
266
|
+
"close down", "open up",
|
|
267
|
+
"drop out", "walk out", "walk back",
|
|
268
|
+
"wind down", "phase out", "sign off",
|
|
269
|
+
"fall back", "move on",
|
|
270
|
+
]);
|
|
271
|
+
// The prepositions a subject run climbs OUT of to the head that governs it.
|
|
272
|
+
// "Frenzy for solar eclipse glasses takes over London" is about the frenzy;
|
|
273
|
+
// the run beside the verb is the preposition's object, never the subject.
|
|
274
|
+
// Deliberately without the temporal ones (after/before/during/since): a
|
|
275
|
+
// fronted temporal phrase is adverbial, so the noun the climb would reach for
|
|
276
|
+
// heads nothing.
|
|
277
|
+
const SUBJECT_CHAIN_PREPOSITIONS = new Set([
|
|
278
|
+
"of", "for", "in", "on", "at", "from", "with", "about", "over",
|
|
279
|
+
"against", "between", "among", "across", "near", "behind", "around",
|
|
280
|
+
"under", "inside", "outside", "amid",
|
|
281
|
+
]);
|
|
282
|
+
// The counting words that head an of-frame the reader has to look through to
|
|
283
|
+
// find what the sentence is about. "triggers hundreds of evacuations" is a fact
|
|
284
|
+
// about the evacuations, and "discovers hundreds of ancient amphorae" about the
|
|
285
|
+
// amphorae. Every relation frame here reads through these; the newswire event
|
|
286
|
+
// frame widens the same climb to the container heads (OF_PARTITIVE_HEADS) and
|
|
287
|
+
// the classifier ones, so "charged a group of Cuban men" is about the men.
|
|
288
|
+
// Anything outside those sets names its own head, so "restore the sacred glow
|
|
289
|
+
// of fireflies" restores the glow, not the fireflies, and "a piece of cake" is
|
|
290
|
+
// about the piece.
|
|
209
291
|
const OF_COUNT_HEADS = new Set(["hundred", "hundreds", "thousand", "thousands", "million", "millions", "dozen", "dozens", "score", "scores", "handful"]);
|
|
292
|
+
// The count phrases newswire writes in front of what an event touched. Closed
|
|
293
|
+
// by list, and each has to close on the number itself, so a bare preposition
|
|
294
|
+
// ("attacked at dawn") is never mistaken for one.
|
|
295
|
+
const COUNT_PHRASE_RE = /^(?:more than|at least|at most|as many as|up to|fewer than|less than|nearly|almost|about|around|over|roughly)\s+\d[\d,.]*$/i;
|
|
296
|
+
const COUNT_PHRASE_MAX_TOKENS = 4;
|
|
210
297
|
// A verb whose own auxiliary is a be-form heads a passive or a progressive
|
|
211
298
|
// ("was arrested by ICE", "are disappearing"), and there the noun on the
|
|
212
299
|
// subject side is what the event happened TO, not who did it — an active read
|
|
213
300
|
// of one states the reverse of the sentence. The scan crosses adverbs only, so
|
|
214
301
|
// a modal chain that is still active ("will completely block") reads on.
|
|
215
302
|
const BE_AUXILIARIES = new Set(["is", "are", "was", "were", "be", "been", "being", "am"]);
|
|
303
|
+
// The prepositions an agentless passive states its subject's own condition
|
|
304
|
+
// with: "banned FROM parliament elections", "deported TO Mexico", "detained AT
|
|
305
|
+
// the border". Each says where the subject ended up, so the pair reads back as
|
|
306
|
+
// one predicate about the subject. Deliberately narrow — "charged WITH
|
|
307
|
+
// smuggling people" and "convicted OF fraud" take a whole clause or an
|
|
308
|
+
// abstraction after them, not a place a fact can hold, and "of" would collide
|
|
309
|
+
// with the of-chain climbs above.
|
|
310
|
+
const PASSIVE_STATE_PREPOSITIONS = new Set([
|
|
311
|
+
"from", "to", "in", "at", "into", "on", "near", "under", "over", "across", "off",
|
|
312
|
+
]);
|
|
216
313
|
|
|
217
314
|
// wink's tokenizer keeps a sentence-final full stop glued to the word before
|
|
218
315
|
// it when that word ends the text ("… block the sun." tokenizes as one PROPN
|
|
@@ -225,16 +322,40 @@ const stripSentenceFinalStop = (word) => {
|
|
|
225
322
|
return text.endsWith(".") && !text.slice(0, -1).includes(".") ? text.slice(0, -1) : text;
|
|
226
323
|
};
|
|
227
324
|
|
|
325
|
+
/** The closed-band lemma a word spells, or null. A Title Case headline carries
|
|
326
|
+
* no tag a reader can trust, so the band's own vocabulary is what identifies
|
|
327
|
+
* its verb: the word itself, or its -s fold ("Halts" → halt). */
|
|
328
|
+
function newswireVerbLemma(word) {
|
|
329
|
+
const surface = stripSentenceFinalStop(String(word ?? "")).toLowerCase();
|
|
330
|
+
if (NEWSWIRE_RELATION_VERBS.has(surface)) return surface;
|
|
331
|
+
const bare = surface.endsWith("s") ? surface.slice(0, -1) : "";
|
|
332
|
+
return bare && NEWSWIRE_RELATION_VERBS.has(bare) ? bare : null;
|
|
333
|
+
}
|
|
334
|
+
|
|
228
335
|
/** Fold an entity surface to its stored key: a lexicon noun's lemma, else the
|
|
229
336
|
* word's own normFactTerm (the optimistic tier mints unlisted content nouns
|
|
230
337
|
* the way the strict teach lane already mints "redis"). */
|
|
231
|
-
function foldEntity(word, lexicon) {
|
|
338
|
+
function foldEntity(word, lexicon, taggedLemma = "") {
|
|
232
339
|
const surface = stripSentenceFinalStop(word);
|
|
233
340
|
// A multi-word name is stored exactly as it reads. "United States" is the
|
|
234
341
|
// name; "united state" is a lemma fold of a word that was never on its own.
|
|
235
342
|
if (surface.includes(" ")) return normFactTerm(surface);
|
|
236
343
|
const noun = lookupNoun(lexicon, surface.toLowerCase());
|
|
237
|
-
return normFactTerm(noun ? noun.lemma : surface);
|
|
344
|
+
return normFactTerm(noun ? noun.lemma : singularHead(surface, taggedLemma));
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/** The singular a head noun folds to when the lexicon carries no entry for it:
|
|
348
|
+
* the tagger's own lemma, and only where that lemma is the surface with an -s
|
|
349
|
+
* or -es taken off. A lemma that respells the word further ("analyses" →
|
|
350
|
+
* "analyzes") is not a singular, and a proper noun keeps its own spelling
|
|
351
|
+
* ("Wales", "Netherlands"), which is what a tagger returns for one anyway. */
|
|
352
|
+
function singularHead(surface, taggedLemma) {
|
|
353
|
+
const word = String(surface ?? "");
|
|
354
|
+
const lemma = String(taggedLemma ?? "");
|
|
355
|
+
if (!lemma || lemma === word) return word;
|
|
356
|
+
const lower = word.toLowerCase();
|
|
357
|
+
const base = lemma.toLowerCase();
|
|
358
|
+
return lower === `${base}s` || lower === `${base}es` ? lemma : word;
|
|
238
359
|
}
|
|
239
360
|
|
|
240
361
|
// The shortest word ingestText's fact-degree scan treats as a content-noun
|
|
@@ -269,6 +390,57 @@ function trimNameRun(words, lexicon) {
|
|
|
269
390
|
return words.slice(start);
|
|
270
391
|
}
|
|
271
392
|
|
|
393
|
+
// A headline is set in Title Case AND carries no sentence-final stop. Both
|
|
394
|
+
// halves matter, and they buy different things — see headlineReadPos.
|
|
395
|
+
const SENTENCE_FINAL_STOPS = new Set([".", "!", "?"]);
|
|
396
|
+
const readsAsHeadline = (values) => readsAsTitleCase(values)
|
|
397
|
+
&& !SENTENCE_FINAL_STOPS.has(String(values[values.length - 1] ?? ""));
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* How a sentence's parts of speech read once Title Case is allowed for. A
|
|
401
|
+
* tagger given a headline has no lowercase to work from and comes back wrong in
|
|
402
|
+
* both directions: the line's verb reads PROPN ("Thailand/PROPN Halts/PROPN
|
|
403
|
+
* New/PROPN Gun/PROPN"), while its plainest noun reads VERB ("Permits/VERB",
|
|
404
|
+
* "Shooting/VERB"). Two corrections, on two different conditions.
|
|
405
|
+
*
|
|
406
|
+
* In ANY Title Case sentence, a word the event band or the lexicon spells as a
|
|
407
|
+
* verb IS the verb (and a `lemmas` array passed in is corrected to match): no
|
|
408
|
+
* name run holds one of those, so promoting it can only split a run the
|
|
409
|
+
* capitals glued together.
|
|
410
|
+
*
|
|
411
|
+
* Demoting an unlisted VERB to a noun takes the stricter test — a real
|
|
412
|
+
* sentence can pass the capitalization test on its own ("The delegation met
|
|
413
|
+
* Prime Minister Keir Starmer."), and demoting its verb would cost every tag
|
|
414
|
+
* it already had right. The missing sentence-final stop is what tells a
|
|
415
|
+
* headline from that.
|
|
416
|
+
*
|
|
417
|
+
* Never the first token either way: a headline opens on its subject, and "Bar
|
|
418
|
+
* Refaeli" opens on a name the band also spells. Never a band word straight
|
|
419
|
+
* after a determiner ("The Free Press"). Any other sentence gets its tags back
|
|
420
|
+
* untouched.
|
|
421
|
+
*/
|
|
422
|
+
function headlineReadPos(values, pos, lemmas, lexicon) {
|
|
423
|
+
if (!readsAsTitleCase(values)) return pos;
|
|
424
|
+
const demoteUnlistedVerbs = readsAsHeadline(values);
|
|
425
|
+
const read = [...pos];
|
|
426
|
+
for (let i = 1; i < values.length; i += 1) {
|
|
427
|
+
const banded = read[i - 1] === "DET" ? null : newswireVerbLemma(values[i]);
|
|
428
|
+
// The tagger's own lemma still counts where it reaches a band verb the
|
|
429
|
+
// surface fold cannot ("Freed" → free), so a headline's past participle
|
|
430
|
+
// keeps the tag it had.
|
|
431
|
+
const taggedBandVerb = read[i] === "VERB" && NEWSWIRE_RELATION_VERBS.has(String(lemmas?.[i] ?? "").toLowerCase());
|
|
432
|
+
if (banded) {
|
|
433
|
+
read[i] = "VERB";
|
|
434
|
+
if (lemmas) lemmas[i] = banded;
|
|
435
|
+
} else if (taggedBandVerb || lookupVerb(lexicon, stripSentenceFinalStop(String(values[i])).toLowerCase())) {
|
|
436
|
+
read[i] = "VERB";
|
|
437
|
+
} else if (read[i] === "VERB" && demoteUnlistedVerbs) {
|
|
438
|
+
read[i] = "NOUN";
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
return read;
|
|
442
|
+
}
|
|
443
|
+
|
|
272
444
|
/** Every NOUN/PROPN token `sentences` names, surface-form occurrence-counted —
|
|
273
445
|
* a POS tagger reads unknown words by context, so an unlisted noun ("wombat")
|
|
274
446
|
* counts exactly like a lexicon-known one. A contiguous run of two or more
|
|
@@ -280,43 +452,124 @@ function candidateTermOccurrencesPos(sentences, nlp, lexicon) {
|
|
|
280
452
|
for (const sentence of sentences) {
|
|
281
453
|
let values;
|
|
282
454
|
let pos;
|
|
455
|
+
let lemmas;
|
|
283
456
|
try {
|
|
284
457
|
const doc = nlp.readDoc(String(sentence || ""));
|
|
285
458
|
values = doc.tokens().out(nlp.its.value);
|
|
286
459
|
pos = doc.tokens().out(nlp.its.pos);
|
|
460
|
+
lemmas = doc.tokens().out(nlp.its.lemma);
|
|
287
461
|
} catch { continue; }
|
|
462
|
+
const headline = readsAsHeadline(values);
|
|
463
|
+
const taggedPos = pos;
|
|
464
|
+
pos = headlineReadPos(values, pos, lemmas, lexicon);
|
|
465
|
+
// A headline read demotes an unlisted verb to a noun so a run can span it
|
|
466
|
+
// ("Mass Shooting"). That much is right for a run, and wrong for a term of
|
|
467
|
+
// its own: "Arrives" belongs inside no name and names nothing on its own.
|
|
468
|
+
const namesOnlyInsideARun = (k) => taggedPos[k] === "VERB" && pos[k] !== "VERB";
|
|
469
|
+
// A headline's capitals say nothing about which words spell a name, so its
|
|
470
|
+
// runs read over every noun; ordinary prose keeps the capital as the tell
|
|
471
|
+
// and runs over proper nouns alone.
|
|
472
|
+
const runsWith = (k) => (headline ? pos[k] === "NOUN" || pos[k] === "PROPN" : pos[k] === "PROPN");
|
|
473
|
+
// A role noun standing in front of a name is a title on that name, not a
|
|
474
|
+
// term of its own — "President Trump" is one lookup, while "president" and
|
|
475
|
+
// "trump" are two half ones. Only a common noun the lexicon knows opens a
|
|
476
|
+
// run this way, and a headline's own capitals carry no such distinction.
|
|
477
|
+
const opensRoleTitledName = (k) => !headline && pos[k] === "NOUN" && pos[k + 1] === "PROPN"
|
|
478
|
+
&& Boolean(lookupNoun(lexicon, stripSentenceFinalStop(String(values[k])).toLowerCase()));
|
|
288
479
|
for (let i = 0; i < values.length; i += 1) {
|
|
289
480
|
if (pos[i] !== "NOUN" && pos[i] !== "PROPN") continue;
|
|
290
481
|
let hi = i;
|
|
291
|
-
if (
|
|
482
|
+
if (runsWith(i) || opensRoleTitledName(i)) {
|
|
483
|
+
while (hi + 1 < values.length && runsWith(hi + 1)) hi += 1;
|
|
484
|
+
}
|
|
292
485
|
if (hi > i) {
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
const verbShaped = (w) => {
|
|
297
|
-
const word = stripSentenceFinalStop(String(w)).toLowerCase();
|
|
298
|
-
const lemma = word.endsWith("s") ? word.slice(0, -1) : word;
|
|
299
|
-
return NEWSWIRE_RELATION_VERBS.has(word) || NEWSWIRE_RELATION_VERBS.has(lemma)
|
|
300
|
-
|| Boolean(lookupVerb(lexicon, word));
|
|
301
|
-
};
|
|
302
|
-
let cut = -1;
|
|
303
|
-
for (let k = i + 1; k <= hi; k += 1) if (verbShaped(values[k])) { cut = k; break; }
|
|
304
|
-
const runEnd = cut === -1 ? hi : cut - 1;
|
|
305
|
-
if (runEnd > i) {
|
|
306
|
-
const name = trimNameRun(values.slice(i, runEnd + 1), lexicon).join(" ");
|
|
307
|
-
counts.set(name, (counts.get(name) || 0) + 1);
|
|
308
|
-
} else {
|
|
309
|
-
counts.set(values[i], (counts.get(values[i]) || 0) + 1);
|
|
310
|
-
}
|
|
311
|
-
i = cut === -1 ? hi : cut;
|
|
486
|
+
const name = trimNameRun(values.slice(i, hi + 1), lexicon).join(" ");
|
|
487
|
+
counts.set(name, (counts.get(name) || 0) + 1);
|
|
488
|
+
i = hi;
|
|
312
489
|
continue;
|
|
313
490
|
}
|
|
491
|
+
if (namesOnlyInsideARun(i)) continue;
|
|
314
492
|
counts.set(values[i], (counts.get(values[i]) || 0) + 1);
|
|
315
493
|
}
|
|
316
494
|
}
|
|
317
495
|
return counts;
|
|
318
496
|
}
|
|
319
497
|
|
|
498
|
+
// The tags a noun phrase opens with. One of these in front of a token settles
|
|
499
|
+
// that the token heads a noun phrase of its own, whatever follows it — a
|
|
500
|
+
// determiner ("the day it happened"), a possessive, a numeral, or the
|
|
501
|
+
// preposition that governs the phrase ("immigrants to countries they have no
|
|
502
|
+
// connection to").
|
|
503
|
+
const NOUN_PHRASE_OPENERS = new Set(["DET", "PRON", "PART", "NUM", "ADP"]);
|
|
504
|
+
|
|
505
|
+
/** Does the token at `i` stand where only a clause's own verb can, despite its
|
|
506
|
+
* NOUN tag? A tagger reads "many say it is just a matter of time" as a noun on
|
|
507
|
+
* the very word the clause turns on, and a term scan then offers "say" as
|
|
508
|
+
* something to look up. One closed frame says otherwise: a subject pronoun and
|
|
509
|
+
* its own verb right behind the token. The pronoun has to be a subject one, so
|
|
510
|
+
* a relative clause hanging off a real noun ("a camera that would cost …")
|
|
511
|
+
* keeps its reading. */
|
|
512
|
+
function readsAsClauseVerb(values, pos, i) {
|
|
513
|
+
if (i > 0 && NOUN_PHRASE_OPENERS.has(pos[i - 1])) return false;
|
|
514
|
+
if (pos[i + 1] !== "PRON" || RELATIVE_PRONOUNS.has(String(values[i + 1] ?? "").toLowerCase())) return false;
|
|
515
|
+
return pos[i + 2] === "VERB" || pos[i + 2] === "AUX";
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/** The same question for the other frame a newswire sentence hides a verb in:
|
|
519
|
+
* an infinitive right behind the token, with a finished noun phrase in front
|
|
520
|
+
* of it — "the central government moves to assert control". The noun phrase is
|
|
521
|
+
* what makes the reading safe, because an infinitive alone follows nouns
|
|
522
|
+
* constantly ("rapid development to cater to tourists", "the only person to
|
|
523
|
+
* break the sound barrier"): those all carry an adjective, a determiner or a
|
|
524
|
+
* preposition on the token itself, and only a token whose own left neighbour
|
|
525
|
+
* is a common noun reads as the verb that noun phrase governs. */
|
|
526
|
+
function readsAsInfinitiveClauseVerb(values, pos, i) {
|
|
527
|
+
if (i === 0 || pos[i - 1] !== "NOUN") return false;
|
|
528
|
+
if (String(values[i + 1] ?? "").toLowerCase() !== "to") return false;
|
|
529
|
+
return pos[i + 2] === "VERB";
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
/** The stored term keys `sentences` only ever uses as a clause's verb. Folded
|
|
533
|
+
* the same way `ungroundedTermOccurrences` folds its own counts, so a caller
|
|
534
|
+
* can subtract this set from those keys directly. A term the text also uses as
|
|
535
|
+
* a plain noun somewhere ("the peace talks" beside "he talks to them") stays
|
|
536
|
+
* out of the set — one verb reading never disqualifies a word that names
|
|
537
|
+
* something elsewhere in the same article.
|
|
538
|
+
*
|
|
539
|
+
* `nlp` follows this module's own convention: absent means the shared wink
|
|
540
|
+
* instance, and an explicit null (no model) returns an empty set, since the
|
|
541
|
+
* frames below are read off part-of-speech tags. */
|
|
542
|
+
export function termsUsedOnlyAsVerbs(sentences, { lexicon = loadLexicon(), nlp } = {}) {
|
|
543
|
+
const engine = nlp === undefined ? winkInstance() : nlp;
|
|
544
|
+
const asVerb = new Map();
|
|
545
|
+
const total = new Map();
|
|
546
|
+
if (!engine) return new Set();
|
|
547
|
+
for (const sentence of sentences) {
|
|
548
|
+
let values;
|
|
549
|
+
let pos;
|
|
550
|
+
let lemmas;
|
|
551
|
+
try {
|
|
552
|
+
const doc = engine.readDoc(String(sentence || ""));
|
|
553
|
+
values = doc.tokens().out(engine.its.value);
|
|
554
|
+
pos = doc.tokens().out(engine.its.pos);
|
|
555
|
+
lemmas = doc.tokens().out(engine.its.lemma);
|
|
556
|
+
} catch { continue; }
|
|
557
|
+
pos = headlineReadPos(values, pos, lemmas, lexicon);
|
|
558
|
+
for (let i = 0; i < values.length; i += 1) {
|
|
559
|
+
if (pos[i] !== "NOUN" && pos[i] !== "PROPN") continue;
|
|
560
|
+
const term = foldEntity(values[i], lexicon);
|
|
561
|
+
if (!term) continue;
|
|
562
|
+
total.set(term, (total.get(term) || 0) + 1);
|
|
563
|
+
if (readsAsClauseVerb(values, pos, i) || readsAsInfinitiveClauseVerb(values, pos, i)) {
|
|
564
|
+
asVerb.set(term, (asVerb.get(term) || 0) + 1);
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
const verbs = new Set();
|
|
569
|
+
for (const [term, count] of asVerb) if (count === total.get(term)) verbs.add(term);
|
|
570
|
+
return verbs;
|
|
571
|
+
}
|
|
572
|
+
|
|
320
573
|
/** A sentence whose substantial words are nearly all capitalized is a headline
|
|
321
574
|
* set in Title Case, where a capital says nothing about which words spell a
|
|
322
575
|
* name. The POS tier reads such a sentence by tag and is unaffected; the
|
|
@@ -442,6 +695,7 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
442
695
|
pos = doc.tokens().out(nlp.its.pos);
|
|
443
696
|
lemmas = doc.tokens().out(nlp.its.lemma);
|
|
444
697
|
} catch { return { triples: [], declined: [], minted: [] }; }
|
|
698
|
+
pos = headlineReadPos(values, pos, lemmas, lexicon);
|
|
445
699
|
// A found noun is read as its whole contiguous NOUN/PROPN run, head-lemma
|
|
446
700
|
// folded — "a string instrument" is the class "string instrument", never
|
|
447
701
|
// its modifier "string"; a single-word run keeps the plain lemma fold.
|
|
@@ -452,33 +706,61 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
452
706
|
let hi = i;
|
|
453
707
|
while (lo - 1 >= 0 && isNounish(lo - 1)) lo -= 1;
|
|
454
708
|
while (hi + 1 < values.length && isNounish(hi + 1)) hi += 1;
|
|
455
|
-
if (lo === hi) return foldEntity(values[i], lexicon);
|
|
456
|
-
|
|
709
|
+
if (lo === hi) return foldEntity(values[i], lexicon, lemmas?.[i]);
|
|
710
|
+
// A run of proper nouns spells a name, and a name sheds the honorific or
|
|
711
|
+
// role word stacked in front of it — "Ex-Marine Robert Gilman" is Robert
|
|
712
|
+
// Gilman. A run holding any common noun is a compound, not a name, and
|
|
713
|
+
// keeps every word ("disk operating system").
|
|
714
|
+
let words = values.slice(lo, hi + 1);
|
|
715
|
+
if (words.every((_, k) => pos[lo + k] === "PROPN")) words = trimNameRun(words, lexicon);
|
|
716
|
+
const last = stripSentenceFinalStop(words[words.length - 1]);
|
|
457
717
|
const head = lookupNoun(lexicon, last.toLowerCase());
|
|
458
|
-
|
|
718
|
+
if (words.length === 1) return normFactTerm(head ? head.lemma : singularHead(last, lemmas?.[hi]));
|
|
719
|
+
return normFactTerm([...words.slice(0, -1), head ? head.lemma : singularHead(last, lemmas?.[hi])].join(" "));
|
|
720
|
+
};
|
|
721
|
+
// Where a leftward scan resumes when it meets a comma, or -1 when the comma
|
|
722
|
+
// stands between it and another clause. A news sentence names its subject
|
|
723
|
+
// first and then interrupts itself — "Yabloko, the Russian antiwar party, is
|
|
724
|
+
// banned …", "Robert Gilman, Freed by Russia, Arrives …" — so a scan that
|
|
725
|
+
// stops dead on a comma never reaches the subject at all. It may cross when
|
|
726
|
+
// nothing between the comma and either an earlier comma or the sentence start
|
|
727
|
+
// predicates: a relative clause ("the quake, which killed 100 people,
|
|
728
|
+
// damaged …") holds a verb, and that keeps the scan out.
|
|
729
|
+
const commaCrossingFrom = (close) => {
|
|
730
|
+
for (let k = close - 1; k >= 0; k -= 1) {
|
|
731
|
+
if (values[k] === ",") return k;
|
|
732
|
+
if (pos[k] === "VERB" || pos[k] === "AUX" || pos[k] === "PUNCT") return -1;
|
|
733
|
+
}
|
|
734
|
+
return close;
|
|
459
735
|
};
|
|
460
736
|
const nearestEntityIndex = (idx, step, blocked = null) => {
|
|
737
|
+
let mayCrossComma = step < 0;
|
|
461
738
|
for (let i = idx + step; i >= 0 && i < values.length; i += step) {
|
|
462
|
-
if (pos[i] === "PUNCT")
|
|
739
|
+
if (pos[i] === "PUNCT") {
|
|
740
|
+
if (!mayCrossComma || values[i] !== ",") break;
|
|
741
|
+
const resume = commaCrossingFrom(i);
|
|
742
|
+
if (resume < 0) break;
|
|
743
|
+
mayCrossComma = false;
|
|
744
|
+
i = resume;
|
|
745
|
+
continue;
|
|
746
|
+
}
|
|
463
747
|
if (blocked && blocked.has(pos[i])) break;
|
|
464
748
|
if (isNounish(i)) return i;
|
|
465
749
|
}
|
|
466
750
|
return null;
|
|
467
751
|
};
|
|
468
|
-
const nearestEntity = (idx, step, blocked = null) => {
|
|
469
|
-
const i = nearestEntityIndex(idx, step, blocked);
|
|
470
|
-
return i === null ? null : entityRunAt(i);
|
|
471
|
-
};
|
|
472
752
|
// The subject-side mirror of the copula-object of-chain rule: when a found
|
|
473
|
-
// subject run is the
|
|
474
|
-
// snow …"
|
|
475
|
-
//
|
|
476
|
-
//
|
|
477
|
-
//
|
|
478
|
-
//
|
|
479
|
-
//
|
|
480
|
-
//
|
|
481
|
-
|
|
753
|
+
// subject run is the object of a preposition ("the weight of all of the
|
|
754
|
+
// snow …", "frenzy for eclipse glasses …"), climb to the head that governs
|
|
755
|
+
// the phrase ("weight", "frenzy"), bounded to two hops. A classifier head
|
|
756
|
+
// (type/kind/sort/…) or a counting one (hundreds/dozens/…) reads THROUGH —
|
|
757
|
+
// "a kind of X" and "hundreds of X" are both about X — so the inner noun is
|
|
758
|
+
// kept. When the run is governed by a preposition but no readable noun heads
|
|
759
|
+
// the phrase (a mis-tagged head, e.g. "the top of the mountain …"), return
|
|
760
|
+
// null: an honest abstain, never the inner-noun confusion ("mountain",
|
|
761
|
+
// "snow"). An ungoverned run is returned unchanged. Returns a run-lo index
|
|
762
|
+
// to fold, or null to abstain.
|
|
763
|
+
const prepositionChainSkip = (k) => {
|
|
482
764
|
const p = pos[k];
|
|
483
765
|
return p === "DET" || p === "ADJ" || p === "ADV" || p === "NUM";
|
|
484
766
|
};
|
|
@@ -486,12 +768,14 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
486
768
|
let lo = runLoOf(found);
|
|
487
769
|
for (let hop = 0; hop < 2; hop += 1) {
|
|
488
770
|
let g = lo - 1;
|
|
489
|
-
while (g >= 0 &&
|
|
490
|
-
if (g < 0 || values[g]?.toLowerCase()
|
|
771
|
+
while (g >= 0 && prepositionChainSkip(g)) g -= 1;
|
|
772
|
+
if (g < 0 || !SUBJECT_CHAIN_PREPOSITIONS.has(values[g]?.toLowerCase())) return lo; // ungoverned run
|
|
491
773
|
let k = g - 1;
|
|
492
|
-
while (k >= 0 && !isNounish(k)
|
|
774
|
+
while (k >= 0 && !isNounish(k)
|
|
775
|
+
&& (prepositionChainSkip(k) || SUBJECT_CHAIN_PREPOSITIONS.has(values[k]?.toLowerCase()))) k -= 1;
|
|
493
776
|
if (k < 0 || !isNounish(k)) return null; // no readable head — abstain
|
|
494
|
-
|
|
777
|
+
const head = String(values[k]).toLowerCase();
|
|
778
|
+
if (OF_CLASSIFIER_HEADS.has(head) || OF_COUNT_HEADS.has(head)) return lo; // reads through to the inner noun
|
|
495
779
|
lo = runLoOf(k);
|
|
496
780
|
}
|
|
497
781
|
return lo;
|
|
@@ -524,8 +808,9 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
524
808
|
// is IN the oceans", "land is grouped INTO continents") — none of them
|
|
525
809
|
// class membership.
|
|
526
810
|
// An of-chain on the object reads through a classifier head to the real
|
|
527
|
-
// class ("a type of mammal" → mammal)
|
|
528
|
-
//
|
|
811
|
+
// class ("a type of mammal" → mammal). A container or counting head states
|
|
812
|
+
// quantity instead ("a large body of ice", "dozens of wolves"), and quantity
|
|
813
|
+
// is no class at all, so both decline the isa.
|
|
529
814
|
const copulaObjectAt = (i) => {
|
|
530
815
|
let sawDeterminer = false;
|
|
531
816
|
for (let j = i + 1; j < values.length; j += 1) {
|
|
@@ -581,7 +866,7 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
581
866
|
const nextIsOf = nextWord === "of";
|
|
582
867
|
if (!nextIsOf) return { label: entityRunAt(j), hi };
|
|
583
868
|
if (OF_CLASSIFIER_HEADS.has(headWord)) { i = hi + 1; j = hi + 1; continue; }
|
|
584
|
-
if (OF_PARTITIVE_HEADS.has(headWord)) return null;
|
|
869
|
+
if (OF_PARTITIVE_HEADS.has(headWord) || OF_COUNT_HEADS.has(headWord)) return null;
|
|
585
870
|
return { label: entityRunAt(j), hi };
|
|
586
871
|
}
|
|
587
872
|
return null;
|
|
@@ -598,7 +883,7 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
598
883
|
const copulaSubjectAt = (i) => {
|
|
599
884
|
let k = i - 1;
|
|
600
885
|
while (k >= 0 && pos[k] === "AUX") k -= 1;
|
|
601
|
-
const found = nearestEntityIndex(k + 1, -1,
|
|
886
|
+
const found = nearestEntityIndex(k + 1, -1, COPULA_SUBJECT_BLOCKERS);
|
|
602
887
|
if (found === null) return null;
|
|
603
888
|
const climbed = climbSubjectRun(found);
|
|
604
889
|
if (climbed === null) return null;
|
|
@@ -640,55 +925,172 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
640
925
|
//
|
|
641
926
|
// - wink must have tagged the token VERB, and its LEMMA must be in the
|
|
642
927
|
// band, so a past tense ("released", "adopted") reads where the
|
|
643
|
-
// lexicon's -s-only fold cannot
|
|
644
|
-
//
|
|
645
|
-
//
|
|
928
|
+
// lexicon's -s-only fold cannot. A verb the LEXICON already declares is
|
|
929
|
+
// skipped by surface, because the lexicon arm read that sentence first
|
|
930
|
+
// and this frame would mint the same act a second time;
|
|
931
|
+
// - a verb followed by one of the closed phrasal pairs is admitted on the
|
|
932
|
+
// pair instead, whichever vocabulary the bare verb belongs to: "take
|
|
933
|
+
// over" is a different relation from "take", so the lexicon's entry for
|
|
934
|
+
// the verb has nothing to say about it;
|
|
646
935
|
// - it must not sit in a relative clause, which has no subject of its own
|
|
647
936
|
// here;
|
|
937
|
+
// - a be-form auxiliary heads a passive or a progressive rather than
|
|
938
|
+
// declining the sentence: the -ing form marks the progressive and is
|
|
939
|
+
// skipped, a "by" complement mints the ACTIVE edge with the actor as
|
|
940
|
+
// subject, and an agentless passive states the patient's own condition;
|
|
648
941
|
// - subject and object are each the NEAREST noun run on their side with
|
|
649
942
|
// the copula frame's blockers applied, so neither scan crosses a verb, a
|
|
650
943
|
// preposition or a conjunction into another clause. "resigned from
|
|
651
944
|
// Cambridge" yields no object at all rather than "resign Cambridge".
|
|
652
945
|
// The subject scan starts left of the verb's OWN modal chain ("will
|
|
653
|
-
// completely block"), which is one verb complex rather than a crossing
|
|
654
|
-
//
|
|
655
|
-
//
|
|
946
|
+
// completely block"), which is one verb complex rather than a crossing,
|
|
947
|
+
// and climbs out of any preposition that governs it ("a fire AT the
|
|
948
|
+
// hospital killed …" is about the fire). A counting of-chain on either
|
|
949
|
+
// side reads through to what the event really touched ("hundreds of
|
|
950
|
+
// ancient amphorae" → amphorae).
|
|
656
951
|
//
|
|
657
952
|
// A lemma the lexicon itself declares keeps the lexicon's predicate, so
|
|
658
|
-
// "releases" (the lexicon arm) and "released" (this one) land on one edge
|
|
953
|
+
// "releases" (the lexicon arm) and "released" (this one) land on one edge —
|
|
954
|
+
// and NEWSWIRE_VERB_SYNONYMS folds the band's own interchangeable pairs onto
|
|
955
|
+
// that same edge before the lookup, so "freed" lands there too.
|
|
659
956
|
const verbComplexStart = (i) => {
|
|
660
957
|
let k = i - 1;
|
|
661
958
|
while (k >= 0 && (pos[k] === "ADV" || pos[k] === "AUX" || pos[k] === "PART")) k -= 1;
|
|
662
959
|
return k + 1;
|
|
663
960
|
};
|
|
961
|
+
// A counting word alone. The lexicon arm reads through these and no others:
|
|
962
|
+
// a container head states what the thing is made of or held in, and a
|
|
963
|
+
// classifier rewrites a class, so neither one moves what a relation verb
|
|
964
|
+
// touched.
|
|
965
|
+
const readsThroughCountOf = (word) => OF_COUNT_HEADS.has(String(word ?? "").toLowerCase());
|
|
966
|
+
// The newswire event frame's wider read-through: the counting words, plus the
|
|
967
|
+
// container and classifier heads a headline writes its patient behind.
|
|
664
968
|
const readsThroughOf = (word) => {
|
|
665
969
|
const w = String(word ?? "").toLowerCase();
|
|
666
|
-
return
|
|
970
|
+
return readsThroughCountOf(w) || OF_PARTITIVE_HEADS.has(w) || OF_CLASSIFIER_HEADS.has(w);
|
|
667
971
|
};
|
|
668
|
-
|
|
669
|
-
|
|
972
|
+
// Walk a found run's of-chain to the noun the phrase is really about, at most
|
|
973
|
+
// two hops so a chain of prepositions can never wander into another clause.
|
|
974
|
+
// `readsThrough` says which heads qualify; a head outside that set stops the
|
|
975
|
+
// climb and keeps its own run.
|
|
976
|
+
const ofChainEntityIndex = (at, readsThrough) => {
|
|
670
977
|
for (let hop = 0; at !== null && hop < 2; hop += 1) {
|
|
671
978
|
let hi = at;
|
|
672
979
|
while (hi + 1 < values.length && isNounish(hi + 1)) hi += 1;
|
|
673
|
-
if (values[hi + 1]?.toLowerCase() !== "of" || !
|
|
980
|
+
if (values[hi + 1]?.toLowerCase() !== "of" || !readsThrough(values[hi])) break;
|
|
674
981
|
const inner = nearestEntityIndex(hi + 1, +1);
|
|
675
982
|
if (inner === null) break;
|
|
676
983
|
at = inner;
|
|
677
984
|
}
|
|
985
|
+
return at;
|
|
986
|
+
};
|
|
987
|
+
const countChainEntityIndex = (idx, step) =>
|
|
988
|
+
ofChainEntityIndex(nearestEntityIndex(idx, step, COPULA_FRAME_BLOCKERS), readsThroughOf);
|
|
989
|
+
const countChainEntity = (idx, step) => {
|
|
990
|
+
const at = countChainEntityIndex(idx, step);
|
|
991
|
+
return at === null ? null : entityRunAt(at);
|
|
992
|
+
};
|
|
993
|
+
// The object a lexicon relation verb takes: the nearest run rightward, on the
|
|
994
|
+
// arm's own blocker-free scan ("relies ON redis" has to read past the
|
|
995
|
+
// preposition), then climbed through a counting of-chain so the fact lands on
|
|
996
|
+
// what was counted rather than on the count word.
|
|
997
|
+
const relationVerbObject = (i) => {
|
|
998
|
+
const at = ofChainEntityIndex(nearestEntityIndex(i, +1), readsThroughCountOf);
|
|
678
999
|
return at === null ? null : entityRunAt(at);
|
|
679
1000
|
};
|
|
1001
|
+
// The noun an event predicates about: the nearest run leftward, climbed out
|
|
1002
|
+
// of any preposition it is the object of, so a phrase-headed subject reads as
|
|
1003
|
+
// its own head rather than as the noun that happens to sit beside the verb.
|
|
1004
|
+
const newswireSubject = (idx) => {
|
|
1005
|
+
const at = countChainEntityIndex(idx, -1);
|
|
1006
|
+
if (at === null) return null;
|
|
1007
|
+
const climbed = climbSubjectRun(at);
|
|
1008
|
+
return climbed === null ? null : entityRunAt(climbed);
|
|
1009
|
+
};
|
|
1010
|
+
// The phrasal verb headed at `i`, when the closed table holds the pair: its
|
|
1011
|
+
// own minted predicate, and the particle's index, so an object scan reads
|
|
1012
|
+
// past the particle to what the event touched.
|
|
1013
|
+
const phrasalVerbAt = (i) => {
|
|
1014
|
+
const particle = stripSentenceFinalStop(String(values[i + 1] ?? "")).toLowerCase();
|
|
1015
|
+
if (!PHRASAL_PARTICLES.has(particle)) return null;
|
|
1016
|
+
const lemma = String(lemmas?.[i] ?? values[i]).toLowerCase();
|
|
1017
|
+
if (!PHRASAL_VERB_PAIRS.has(`${lemma} ${particle}`)) return null;
|
|
1018
|
+
return { predicate: `mgx:${lemma}-${particle}`, particleAt: i + 1 };
|
|
1019
|
+
};
|
|
1020
|
+
// The first token of the complement a verb takes, adverbs crossed. A passive
|
|
1021
|
+
// reads its own frame off this one word: "by" names the actor, a locative or
|
|
1022
|
+
// directional preposition names where the subject ended up.
|
|
1023
|
+
const complementHead = (i) => {
|
|
1024
|
+
for (let k = i + 1; k < values.length; k += 1) {
|
|
1025
|
+
if (pos[k] === "ADV") continue;
|
|
1026
|
+
return k;
|
|
1027
|
+
}
|
|
1028
|
+
return -1;
|
|
1029
|
+
};
|
|
1030
|
+
// Newswire writes the count before the thing counted — "killed more than 100
|
|
1031
|
+
// people", "injured at least 30 workers" — and the preposition inside the
|
|
1032
|
+
// count phrase stops an object scan dead. Each lead below is skipped whole,
|
|
1033
|
+
// and only where the number itself closes the phrase, so "attacked at dawn"
|
|
1034
|
+
// (no number, no count) is untouched.
|
|
1035
|
+
const skipCountPhrase = (idx) => {
|
|
1036
|
+
for (let n = COUNT_PHRASE_MAX_TOKENS; n >= 2; n -= 1) {
|
|
1037
|
+
const span = values.slice(idx + 1, idx + 1 + n);
|
|
1038
|
+
if (span.length === n && COUNT_PHRASE_RE.test(span.join(" "))) return idx + n;
|
|
1039
|
+
}
|
|
1040
|
+
return idx;
|
|
1041
|
+
};
|
|
680
1042
|
const readNewswireFrame = () => {
|
|
681
1043
|
for (let i = 1; i < values.length - 1; i += 1) {
|
|
682
1044
|
if (pos[i] !== "VERB") continue;
|
|
683
|
-
if (lookupVerb(lexicon, String(values[i]).toLowerCase())) continue;
|
|
684
1045
|
const lemma = String(lemmas?.[i] ?? values[i]).toLowerCase();
|
|
685
|
-
|
|
686
|
-
if (
|
|
687
|
-
|
|
688
|
-
|
|
1046
|
+
const phrasal = phrasalVerbAt(i);
|
|
1047
|
+
if (!phrasal) {
|
|
1048
|
+
if (lookupVerb(lexicon, String(values[i]).toLowerCase())) continue;
|
|
1049
|
+
if (!NEWSWIRE_RELATION_VERBS.has(lemma)) continue;
|
|
1050
|
+
}
|
|
1051
|
+
if (relativePronounBefore(i) >= 0) continue;
|
|
1052
|
+
const canonical = NEWSWIRE_VERB_SYNONYMS.get(lemma) || lemma;
|
|
1053
|
+
const declared = lookupVerb(lexicon, canonical);
|
|
1054
|
+
const predicate = phrasal
|
|
1055
|
+
? phrasal.predicate
|
|
1056
|
+
: (declared ? predicateOf(declared) : `mgx:${canonical}`);
|
|
1057
|
+
const surface = stripSentenceFinalStop(String(values[i])).toLowerCase();
|
|
1058
|
+
// A phrasal verb's own complement starts past the particle, so the frame
|
|
1059
|
+
// below reads "was taken over BY Google" the same way it reads a bare
|
|
1060
|
+
// participle's "by".
|
|
1061
|
+
const head = complementHead(phrasal ? phrasal.particleAt : i);
|
|
1062
|
+
const headWord = head === -1 ? "" : String(values[head]).toLowerCase();
|
|
1063
|
+
const beAuxiliary = beAuxiliaryBefore(i);
|
|
1064
|
+
// A progressive is not an event that happened, and its -ing form is the
|
|
1065
|
+
// one thing that tells it apart from the past participle a passive takes.
|
|
1066
|
+
if (beAuxiliary && surface.endsWith("ing")) continue;
|
|
1067
|
+
// "<patient> (was) <participle> by <actor>" — the sentence names both
|
|
1068
|
+
// roles, so it mints the ACTIVE edge with the actor on the subject side.
|
|
1069
|
+
// A reduced passive carries no auxiliary at all ("Boats Hit by Mystery
|
|
1070
|
+
// Attackers"), so the "by" is what identifies the frame.
|
|
1071
|
+
if (headWord === "by") {
|
|
1072
|
+
const actor = countChainEntity(head, +1);
|
|
1073
|
+
const patient = newswireSubject(verbComplexStart(i));
|
|
1074
|
+
if (actor && patient) push(actor, predicate, patient);
|
|
1075
|
+
continue;
|
|
1076
|
+
}
|
|
1077
|
+
if (beAuxiliary) {
|
|
1078
|
+
// "<patient> is <participle> <prep> <complement>" — no actor is named,
|
|
1079
|
+
// so nothing can take the subject side of an active edge. The claim the
|
|
1080
|
+
// sentence DOES make is about the patient's own condition, and that is
|
|
1081
|
+
// what the participle predicate states. A phrasal pair already carries
|
|
1082
|
+
// its particle as that condition, and no second preposition can restate
|
|
1083
|
+
// it, so an agentless phrasal passive abstains.
|
|
1084
|
+
if (phrasal || !PASSIVE_STATE_PREPOSITIONS.has(headWord)) continue;
|
|
1085
|
+
const patient = newswireSubject(verbComplexStart(i));
|
|
1086
|
+
const complement = countChainEntity(head, +1);
|
|
1087
|
+
if (patient && complement) push(patient, `mgx:${surface}-${headWord}`, complement);
|
|
1088
|
+
continue;
|
|
1089
|
+
}
|
|
1090
|
+
const subject = newswireSubject(verbComplexStart(i));
|
|
1091
|
+
const object = countChainEntity(phrasal ? phrasal.particleAt : skipCountPhrase(i), +1);
|
|
689
1092
|
if (!subject || !object) continue;
|
|
690
|
-
|
|
691
|
-
push(subject, declared ? predicateOf(declared) : `mgx:${lemma}`, object);
|
|
1093
|
+
push(subject, predicate, object);
|
|
692
1094
|
}
|
|
693
1095
|
};
|
|
694
1096
|
|
|
@@ -744,14 +1146,15 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
744
1146
|
if (OPTIMISTIC_COPULAS.has(word)) continue;
|
|
745
1147
|
const verb = lookupVerb(lexicon, word);
|
|
746
1148
|
if (!verb) continue;
|
|
1149
|
+
if (phrasalVerbAt(i)) continue; // the phrasal frame reads the pair whole
|
|
747
1150
|
const relative = relativePronounBefore(i);
|
|
748
1151
|
if (relative >= 0 && relative - 1 !== copulaObjHi) {
|
|
749
|
-
decline("relative-clause-verb", { subject: copulaSubject, predicate: predicateOf(verb), object:
|
|
1152
|
+
decline("relative-clause-verb", { subject: copulaSubject, predicate: predicateOf(verb), object: relationVerbObject(i) });
|
|
750
1153
|
continue;
|
|
751
1154
|
}
|
|
752
1155
|
const subject = relative >= 0 ? copulaSubject : climbedSubjectAt(i);
|
|
753
1156
|
if (subject === null) continue;
|
|
754
|
-
push(subject, predicateOf(verb),
|
|
1157
|
+
push(subject, predicateOf(verb), relationVerbObject(i));
|
|
755
1158
|
}
|
|
756
1159
|
readNewswireFrame();
|
|
757
1160
|
return { triples, declined, minted };
|
|
@@ -768,13 +1171,14 @@ function optimisticTriplesPos(sentence, lexicon, nlp, { mintDefinitional = false
|
|
|
768
1171
|
if (pos[i] !== "VERB") continue;
|
|
769
1172
|
const verb = lookupVerb(lexicon, values[i].toLowerCase());
|
|
770
1173
|
if (!verb) continue;
|
|
1174
|
+
if (phrasalVerbAt(i)) continue; // the phrasal frame reads the pair whole
|
|
771
1175
|
const subject = climbedSubjectAt(i);
|
|
772
1176
|
if (relativePronounBefore(i) >= 0) {
|
|
773
|
-
decline("relative-clause-verb", { subject, predicate: predicateOf(verb), object:
|
|
1177
|
+
decline("relative-clause-verb", { subject, predicate: predicateOf(verb), object: relationVerbObject(i) });
|
|
774
1178
|
continue;
|
|
775
1179
|
}
|
|
776
1180
|
if (subject === null) continue;
|
|
777
|
-
push(subject, predicateOf(verb),
|
|
1181
|
+
push(subject, predicateOf(verb), relationVerbObject(i));
|
|
778
1182
|
}
|
|
779
1183
|
readNewswireFrame();
|
|
780
1184
|
return { triples, declined, minted };
|
|
@@ -905,6 +1309,53 @@ export function clauseCandidates(sentence, { nlp } = {}) {
|
|
|
905
1309
|
return out;
|
|
906
1310
|
}
|
|
907
1311
|
|
|
1312
|
+
// The verbs a report attributes a claim with. Closed and deliberately short:
|
|
1313
|
+
// each of these carries the claim through unchanged, so the clause beside one
|
|
1314
|
+
// is what the article states. Hedging and reversing verbs stay out — "denied",
|
|
1315
|
+
// "alleged", "claimed" and "suggested" each change what the sentence says
|
|
1316
|
+
// about the clause, and a frame that unwrapped them would store the opposite of
|
|
1317
|
+
// the report.
|
|
1318
|
+
const REPORTED_SPEECH_VERB_SRC = "said|says|told|reported|reports|announced|announces|stated|states|confirmed|confirms|added|wrote|writes";
|
|
1319
|
+
// "<claim>, President Trump said." — the attribution rides the tail after a
|
|
1320
|
+
// comma and closes the sentence. A closing quote mark may sit between them.
|
|
1321
|
+
// Group 1 is the speaker.
|
|
1322
|
+
const TRAILING_ATTRIBUTION_RE = new RegExp(
|
|
1323
|
+
`,\\s*["'“”‘’]?\\s*([\\w.'’-]+(?:\\s+[\\w.'’-]+){0,3})\\s+(?:${REPORTED_SPEECH_VERB_SRC})\\s*[.!?]?\\s*$`,
|
|
1324
|
+
"i",
|
|
1325
|
+
);
|
|
1326
|
+
// "President Trump said (that) <claim>", "Mr. Gilman's family had said <claim>"
|
|
1327
|
+
// — the attribution leads and the claim is everything past it. Group 1 is the
|
|
1328
|
+
// speaker, group 2 the claim.
|
|
1329
|
+
const LEADING_ATTRIBUTION_RE = new RegExp(
|
|
1330
|
+
`^\\s*([\\w.'’-]+(?:\\s+[\\w.'’-]+){0,4})\\s+(?:has\\s+|had\\s+|have\\s+)?(?:${REPORTED_SPEECH_VERB_SRC})\\s+(?:that\\s+)?(\\S.*)$`,
|
|
1331
|
+
"i",
|
|
1332
|
+
);
|
|
1333
|
+
|
|
1334
|
+
/**
|
|
1335
|
+
* The claim a sentence attributes to a speaker, and the speaker it names. A
|
|
1336
|
+
* report states most of what it knows this way, and the recognizer reading the
|
|
1337
|
+
* WHOLE sentence reads the attribution as the claim: "Officials said the quake
|
|
1338
|
+
* killed more than 100 people." came back as `officials mgx:say quake killed
|
|
1339
|
+
* more than 100 people`, a whole clause stored as a term. Stripping the
|
|
1340
|
+
* attribution leaves the claim the article is making, which is the thing worth
|
|
1341
|
+
* grounding.
|
|
1342
|
+
*
|
|
1343
|
+
* Returns { claim, speaker }. `speaker` is "" when no attribution frame fired,
|
|
1344
|
+
* or when the claim left behind is too short to be one — the sentence comes
|
|
1345
|
+
* back whole in that case, so there is nothing the speaker would hang off.
|
|
1346
|
+
* Where both frames fire ("Officials said X, police reported."), the LEADING
|
|
1347
|
+
* speaker wins: it is the one attached to the clause that survived.
|
|
1348
|
+
*/
|
|
1349
|
+
export function reportedClauseOf(sentence) {
|
|
1350
|
+
const text = String(sentence ?? "").trim();
|
|
1351
|
+
const trailing = TRAILING_ATTRIBUTION_RE.exec(text);
|
|
1352
|
+
const trailingStripped = trailing ? text.replace(TRAILING_ATTRIBUTION_RE, ".") : text;
|
|
1353
|
+
const leading = LEADING_ATTRIBUTION_RE.exec(trailingStripped);
|
|
1354
|
+
const claim = leading ? leading[2].trim() : trailingStripped;
|
|
1355
|
+
if (claim.split(/\s+/).length < 3) return { claim: text, speaker: "" };
|
|
1356
|
+
return { claim, speaker: (leading?.[1] ?? trailing?.[1] ?? "").trim() };
|
|
1357
|
+
}
|
|
1358
|
+
|
|
908
1359
|
// A stored term names a thing. These words open a predicate remainder or a new
|
|
909
1360
|
// clause, so a term that STARTS with one is the tail of a sentence a recognizer
|
|
910
1361
|
// frame over-read, never an entity — "has a population of 1,683,115" and "and
|
|
@@ -961,6 +1412,50 @@ const COMPASS_LEAD_WORDS = new Set([
|
|
|
961
1412
|
"northeast", "northwest", "southeast", "southwest",
|
|
962
1413
|
"northern", "southern", "eastern", "western",
|
|
963
1414
|
]);
|
|
1415
|
+
// A name is one noun phrase. These words open a new phrase or a new clause, so
|
|
1416
|
+
// a term carrying one BETWEEN its first and last word is a headline a frame
|
|
1417
|
+
// tore into subject + predicate + remainder: "colombia as rescuers free quake
|
|
1418
|
+
// victim", "keir starmer faces a vote", "new gun licenses after mass shooting",
|
|
1419
|
+
// "boats hit by mystery attackers" — all four reach the graph as a card's own
|
|
1420
|
+
// hub or object otherwise.
|
|
1421
|
+
//
|
|
1422
|
+
// "of" is deliberately absent: it is the one preposition real names are built
|
|
1423
|
+
// with ("house of representatives", "united states of america", "isle of man").
|
|
1424
|
+
// The rule reads STRICTLY interior positions for the same reason the lead and
|
|
1425
|
+
// trailing rules read the edges separately — "may" and "will" end real surnames
|
|
1426
|
+
// ("theresa may", "brian may") and open nothing there.
|
|
1427
|
+
const INTERIOR_CLAUSE_WORDS = new Set([
|
|
1428
|
+
"a", "an", "the",
|
|
1429
|
+
"and", "or", "but", "because", "since", "although", "though", "whereas", "while", "so",
|
|
1430
|
+
"if", "when", "then", "however", "as", "that", "which", "who", "whom", "whose",
|
|
1431
|
+
"is", "are", "was", "were", "be", "been", "being", "am", "has", "have", "had",
|
|
1432
|
+
"do", "does", "did", "can", "could", "will", "would", "should", "may", "might", "must",
|
|
1433
|
+
"in", "on", "at", "for", "to", "with", "from", "by", "into", "onto",
|
|
1434
|
+
"over", "under", "after", "before", "between", "during", "about", "near", "through",
|
|
1435
|
+
"against", "among", "within", "without", "per",
|
|
1436
|
+
]);
|
|
1437
|
+
|
|
1438
|
+
// Past participles that open a real name. Read on the lowercased stored key,
|
|
1439
|
+
// wink tags each of these VERB, so the POS rule below turns down "united
|
|
1440
|
+
// states" — the name world news reports most often — and every sibling built
|
|
1441
|
+
// the same way.
|
|
1442
|
+
const PARTICIPIAL_NAME_LEAD_WORDS = new Set(["united", "allied", "combined", "armed", "organized", "associated"]);
|
|
1443
|
+
|
|
1444
|
+
// A term the source itself wrapped in quotation marks is a title it quoted, and
|
|
1445
|
+
// a title is free to read as a clause — "Hackernews discusses \"Tim King,
|
|
1446
|
+
// AmigaDOS developer, has died\"" states a true fact about a whole headline.
|
|
1447
|
+
// The quotes are the tell, so they exempt the interior rule and nothing else.
|
|
1448
|
+
const QUOTED_TERM_RE = /^["“'‘].*["”'’]$/;
|
|
1449
|
+
|
|
1450
|
+
/** Does `term` carry a clause- or phrase-opening word strictly between its
|
|
1451
|
+
* first and last word? */
|
|
1452
|
+
export function carriesInteriorClauseWord(words) {
|
|
1453
|
+
if (words.length >= 2 && QUOTED_TERM_RE.test(`${words[0]} ${words[words.length - 1]}`)) return false;
|
|
1454
|
+
for (let i = 1; i < words.length - 1; i += 1) {
|
|
1455
|
+
if (INTERIOR_CLAUSE_WORDS.has(String(words[i]).toLowerCase())) return true;
|
|
1456
|
+
}
|
|
1457
|
+
return false;
|
|
1458
|
+
}
|
|
964
1459
|
|
|
965
1460
|
/** Does `term` read as a thing's name rather than a clause fragment? Bounds
|
|
966
1461
|
* the word count and rejects a leading conjunction, auxiliary, preposition,
|
|
@@ -979,7 +1474,9 @@ export function readsAsEntityTerm(term, nlp) {
|
|
|
979
1474
|
if (PARTICLE_LEAD_WORDS.has(first)) return false;
|
|
980
1475
|
if (PRONOUN_LEAD_WORDS.has(first.replace(CLITIC_SUFFIX_RE, ""))) return false;
|
|
981
1476
|
if (TRAILING_AUXILIARY_WORDS.has(words[words.length - 1].toLowerCase())) return false;
|
|
1477
|
+
if (carriesInteriorClauseWord(words)) return false;
|
|
982
1478
|
if (COMPASS_LEAD_WORDS.has(first) && words[1].toLowerCase() !== "of") return true;
|
|
1479
|
+
if (PARTICIPIAL_NAME_LEAD_WORDS.has(first)) return true;
|
|
983
1480
|
const engine = nlp === undefined ? winkInstance() : nlp;
|
|
984
1481
|
if (!engine) return true;
|
|
985
1482
|
try {
|
|
@@ -991,6 +1488,27 @@ export function readsAsEntityTerm(term, nlp) {
|
|
|
991
1488
|
|
|
992
1489
|
const readsAsEntityFact = (fact, nlp) => readsAsEntityTerm(fact.subject, nlp) && readsAsEntityTerm(fact.object, nlp);
|
|
993
1490
|
|
|
1491
|
+
/** Did the frame that read this row cut a phrasal verb in half? "as prices
|
|
1492
|
+
* surge and stocks sell out" names no object at all, so a row pairing the
|
|
1493
|
+
* bare verb with its own particle ("stocks mgx:sell out") states nothing —
|
|
1494
|
+
* the particle belongs to the verb. Read off the same closed pair table the
|
|
1495
|
+
* optimistic tier reads a phrasal verb whole by, so only a particle that verb
|
|
1496
|
+
* really takes declines the row. */
|
|
1497
|
+
export function splitsPhrasalVerb(fact) {
|
|
1498
|
+
const particle = String(fact?.object ?? "").trim().toLowerCase();
|
|
1499
|
+
if (!PHRASAL_PARTICLES.has(particle)) return false;
|
|
1500
|
+
const surface = String(fact?.predicate ?? "").split(":").pop().toLowerCase();
|
|
1501
|
+
return PHRASAL_VERB_PAIRS.has(`${surface} ${particle}`)
|
|
1502
|
+
|| PHRASAL_VERB_PAIRS.has(`${baseVerbSurface(surface)} ${particle}`);
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
/** Why a candidate row is turned down, or null when it stands: the phrasal
|
|
1506
|
+
* split first (it names the row's real problem), then the entity-term rule. */
|
|
1507
|
+
const declineFindingFor = (fact, nlp) => {
|
|
1508
|
+
if (splitsPhrasalVerb(fact)) return "phrasal-particle";
|
|
1509
|
+
return readsAsEntityFact(fact, nlp) ? null : "fragment-term";
|
|
1510
|
+
};
|
|
1511
|
+
|
|
994
1512
|
/** Does one raw token read as a code identifier rather than a word? camelCase
|
|
995
1513
|
* and snake_case split into several words, and a dot between letters or a path
|
|
996
1514
|
* separator names a module or a file. Read on the sentence's own surface,
|
|
@@ -1034,6 +1552,24 @@ function findingsForRow(fact, readingFindings, identifierTerms) {
|
|
|
1034
1552
|
return named;
|
|
1035
1553
|
}
|
|
1036
1554
|
|
|
1555
|
+
const REPORTED_SPEECH_FINDING = "reported-speech";
|
|
1556
|
+
const ATTRIBUTED_TO_PREDICATE = "mgx:attributedTo";
|
|
1557
|
+
|
|
1558
|
+
/** The reified attribution beside a claim row: `fact:<claimHash> |
|
|
1559
|
+
* mgx:attributedTo | <speaker>`. A fact is content-addressed by its own
|
|
1560
|
+
* triple, so naming the claim's group id costs no read back. Null when the
|
|
1561
|
+
* sentence attributed nothing. */
|
|
1562
|
+
function attributionRowFor(fact, { speaker, tag, observedAt }) {
|
|
1563
|
+
if (!speaker) return null;
|
|
1564
|
+
return {
|
|
1565
|
+
subject: factIdForTriple(fact.subject, fact.predicate, fact.object),
|
|
1566
|
+
predicate: ATTRIBUTED_TO_PREDICATE,
|
|
1567
|
+
object: speaker,
|
|
1568
|
+
provenance: tag,
|
|
1569
|
+
observedAt,
|
|
1570
|
+
};
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1037
1573
|
// A host that shares one thread with a UI hands the thread back through this;
|
|
1038
1574
|
// a Node run leaves it unset and pays nothing.
|
|
1039
1575
|
let ingestYield = null;
|
|
@@ -1088,6 +1624,15 @@ function canonicalLines(facts, storeRows) {
|
|
|
1088
1624
|
* ingestText — the single ingest seam. Grounds `text` into facts and returns a
|
|
1089
1625
|
* structured result; the CLI, the browser and the tool layer all drive this.
|
|
1090
1626
|
*
|
|
1627
|
+
* A sentence that attributes its claim to a speaker is handed to the recognizer
|
|
1628
|
+
* as the claim alone (`reportedClauseOf`); the ungrounded-term scan still reads
|
|
1629
|
+
* the sentence as written, so the speaker keeps reaching the enrichment queue.
|
|
1630
|
+
* Under `findings` the speaker is also stored, twice over: the claim carries the
|
|
1631
|
+
* `reported-speech` finding, and a reified `fact:<claimId> | mgx:attributedTo |
|
|
1632
|
+
* <speaker>` row goes into the same batch. The finding is the half that must
|
|
1633
|
+
* survive — a reader that loses the attribution still sees the claim say which
|
|
1634
|
+
* reading it came from, so the two are written together or not at all.
|
|
1635
|
+
*
|
|
1091
1636
|
* text the raw string to ground.
|
|
1092
1637
|
* options:
|
|
1093
1638
|
* memoryDir write grounded facts here (a real .tmct memory dir). Omit for
|
|
@@ -1110,13 +1655,15 @@ function canonicalLines(facts, storeRows) {
|
|
|
1110
1655
|
* lexicon a loaded lexicon; the core vocabulary when absent.
|
|
1111
1656
|
* findings record how each sentence read: mint the edge a definitional
|
|
1112
1657
|
* frame states (`mgx:nameFor`) where the false isa was
|
|
1113
|
-
* declined,
|
|
1114
|
-
* `clause-fallback`, `pronoun-carry`, `definitional-frame
|
|
1115
|
-
* the assertions this call writes
|
|
1116
|
-
*
|
|
1658
|
+
* declined, attach the kept findings (`identifier-token`,
|
|
1659
|
+
* `clause-fallback`, `pronoun-carry`, `definitional-frame`,
|
|
1660
|
+
* `reported-speech`) to the assertions this call writes, and
|
|
1661
|
+
* write the attribution row beside each reported-speech claim.
|
|
1662
|
+
* Off by default; the declines themselves are reported either
|
|
1663
|
+
* way.
|
|
1117
1664
|
*
|
|
1118
|
-
* Returns { sentences, recognized, extracted, optimistic,
|
|
1119
|
-
* minted, canonical? }.
|
|
1665
|
+
* Returns { sentences, recognized, extracted, optimistic, attributions, skipped,
|
|
1666
|
+
* declined, minted, canonical? }.
|
|
1120
1667
|
* recognized — strict-recognized sentence count.
|
|
1121
1668
|
* extracted — strict fact rows ({subject, predicate, object, provenance,
|
|
1122
1669
|
* quantifier, sentence}), plus `extraction` when the row was
|
|
@@ -1124,6 +1671,11 @@ function canonicalLines(facts, storeRows) {
|
|
|
1124
1671
|
* optimistic — fuzzy candidate rows ({subject, predicate, object, provenance,
|
|
1125
1672
|
* sentence}, same optional `extraction`); always [] unless
|
|
1126
1673
|
* options.optimistic.
|
|
1674
|
+
* attributions
|
|
1675
|
+
* — the attribution rows written beside the two arrays above, in
|
|
1676
|
+
* the same shape. Reported apart from them because they are not
|
|
1677
|
+
* facts the article stated, so nothing counting what an article
|
|
1678
|
+
* taught should count them.
|
|
1127
1679
|
* skipped — sentences neither tier grounded.
|
|
1128
1680
|
* declined — every candidate a detector turned down, as
|
|
1129
1681
|
* { sentence, finding, candidate }.
|
|
@@ -1169,6 +1721,12 @@ export async function ingestText(text, {
|
|
|
1169
1721
|
|
|
1170
1722
|
const extracted = [];
|
|
1171
1723
|
const optimisticFacts = [];
|
|
1724
|
+
// The reified attributions written beside the claim rows above. Kept out of
|
|
1725
|
+
// `extracted`/`optimistic` on purpose: those two arrays are what the article
|
|
1726
|
+
// stated, and they feed a snapshot's fact ids and the bench's own score. An
|
|
1727
|
+
// attribution says who said one of them, which is not another fact the
|
|
1728
|
+
// article stated.
|
|
1729
|
+
const attributions = [];
|
|
1172
1730
|
// How each sentence read: the candidates a detector turned down, and the
|
|
1173
1731
|
// edges minted where one was declined.
|
|
1174
1732
|
const declined = [];
|
|
@@ -1213,13 +1771,13 @@ export async function ingestText(text, {
|
|
|
1213
1771
|
currentRows = afterRows;
|
|
1214
1772
|
if (!recognized) { lastDecline = decline || lastDecline; return null; }
|
|
1215
1773
|
const fresh = rows.filter((row) => !taggedIds.has(row.id));
|
|
1216
|
-
const kept = fresh.filter((row) =>
|
|
1774
|
+
const kept = fresh.filter((row) => !declineFindingFor(row, termNlp));
|
|
1217
1775
|
if (kept.length !== fresh.length) {
|
|
1218
1776
|
for (const row of fresh) {
|
|
1219
1777
|
if (kept.includes(row)) continue;
|
|
1220
1778
|
declined.push({
|
|
1221
1779
|
sentence: currentSentence,
|
|
1222
|
-
finding:
|
|
1780
|
+
finding: declineFindingFor(row, termNlp),
|
|
1223
1781
|
candidate: { subject: row.subject, predicate: row.predicate, object: row.object },
|
|
1224
1782
|
});
|
|
1225
1783
|
}
|
|
@@ -1243,23 +1801,33 @@ export async function ingestText(text, {
|
|
|
1243
1801
|
sentenceCount += 1;
|
|
1244
1802
|
lastDecline = "";
|
|
1245
1803
|
currentSentence = sentence;
|
|
1246
|
-
const
|
|
1247
|
-
|
|
1804
|
+
const asWritten = stripCitationResidue(sentence);
|
|
1805
|
+
// The ungrounded-term scan reads the sentence AS WRITTEN, so a speaker
|
|
1806
|
+
// the article named still reaches the enrichment queue; only the
|
|
1807
|
+
// recognizer reads the claim on its own.
|
|
1808
|
+
cleanedSentences.push(asWritten);
|
|
1809
|
+
const { claim: cleaned, speaker } = reportedClauseOf(asWritten);
|
|
1248
1810
|
// The stored term keys this sentence names with an identifier-shaped
|
|
1249
1811
|
// token, read off the surface before normFactTerm folds the shape away.
|
|
1250
1812
|
// Null when this run records no findings.
|
|
1251
1813
|
const identifierTerms = findings ? identifierTermsIn(cleaned) : null;
|
|
1814
|
+
// The finding every row off this sentence carries, and the speaker each
|
|
1815
|
+
// one is attributed to. Both halves ride the same switch: the finding is
|
|
1816
|
+
// what a reader falls back on when the attribution is gone, so writing
|
|
1817
|
+
// one without the other would invert the fallback.
|
|
1818
|
+
const attributedTo = identifierTerms && speaker ? speaker : "";
|
|
1819
|
+
const speechFindings = attributedTo ? [REPORTED_SPEECH_FINDING] : [];
|
|
1252
1820
|
// How the strict tier reached its rows, when it did: the whole sentence
|
|
1253
1821
|
// carries nothing, a later candidate is a clause fragment, and the
|
|
1254
1822
|
// pronoun retry carried its subject in from an earlier sentence.
|
|
1255
|
-
let readingFindings = [];
|
|
1823
|
+
let readingFindings = [...speechFindings];
|
|
1256
1824
|
// Whole sentence first, then each closed-marker clause as a fallback.
|
|
1257
1825
|
let rows = null;
|
|
1258
1826
|
const candidates = clauseCandidates(cleaned, { nlp });
|
|
1259
1827
|
for (let i = 0; i < candidates.length; i += 1) {
|
|
1260
1828
|
rows = await strictRows(candidates[i]);
|
|
1261
1829
|
if (!rows) continue;
|
|
1262
|
-
if (i > 0) readingFindings = ["clause-fallback"];
|
|
1830
|
+
if (i > 0) readingFindings = [...speechFindings, "clause-fallback"];
|
|
1263
1831
|
break;
|
|
1264
1832
|
}
|
|
1265
1833
|
// Bounded pronoun carry: a "they/it/these/those/this …" sentence the
|
|
@@ -1272,7 +1840,7 @@ export async function ingestText(text, {
|
|
|
1272
1840
|
const threaded = stripLeadingDiscourseAdverb(cleaned);
|
|
1273
1841
|
if (!rows && carrySubject && PRONOUN_LEAD_RE.test(threaded)) {
|
|
1274
1842
|
rows = await strictRows(threaded.replace(PRONOUN_LEAD_RE, `${articledSubject(carrySubject)} `));
|
|
1275
|
-
if (rows) readingFindings = ["pronoun-carry"];
|
|
1843
|
+
if (rows) readingFindings = [...speechFindings, "pronoun-carry"];
|
|
1276
1844
|
}
|
|
1277
1845
|
if (rows) {
|
|
1278
1846
|
recognizedSentences += 1;
|
|
@@ -1297,6 +1865,8 @@ export async function ingestText(text, {
|
|
|
1297
1865
|
...(extraction.length ? { extraction } : {}),
|
|
1298
1866
|
});
|
|
1299
1867
|
taggedIds.add(row.id);
|
|
1868
|
+
const attribution = attributionRowFor(row, { speaker: attributedTo, tag, observedAt });
|
|
1869
|
+
if (attribution) { writes.push(attribution); attributions.push({ ...attribution, sentence }); }
|
|
1300
1870
|
}
|
|
1301
1871
|
await appendFacts(dir, writes);
|
|
1302
1872
|
continue;
|
|
@@ -1308,8 +1878,9 @@ export async function ingestText(text, {
|
|
|
1308
1878
|
for (const { finding, candidate } of reading.declined) declined.push({ sentence, finding, candidate });
|
|
1309
1879
|
const keptCandidates = [];
|
|
1310
1880
|
for (const t of reading.triples) {
|
|
1311
|
-
|
|
1312
|
-
|
|
1881
|
+
const finding = declineFindingFor(t, termNlp);
|
|
1882
|
+
if (finding) declined.push({ sentence, finding, candidate: t });
|
|
1883
|
+
else keptCandidates.push(t);
|
|
1313
1884
|
}
|
|
1314
1885
|
// A minted edge rides its own finding: minted[].fact is the very object
|
|
1315
1886
|
// the candidate list holds, so the mint and its row match by identity
|
|
@@ -1325,12 +1896,15 @@ export async function ingestText(text, {
|
|
|
1325
1896
|
const tag = `optimistic-extract:${sourceTag}`;
|
|
1326
1897
|
const candidateWrites = [];
|
|
1327
1898
|
for (const t of keptCandidates) {
|
|
1328
|
-
const
|
|
1899
|
+
const mintFinding = mintFindings.has(t) ? [mintFindings.get(t)] : [];
|
|
1900
|
+
const extraction = findingsForRow(t, [...speechFindings, ...mintFinding], identifierTerms);
|
|
1329
1901
|
candidateWrites.push({
|
|
1330
1902
|
subject: t.subject, predicate: t.predicate, object: t.object, provenance: tag, observedAt,
|
|
1331
1903
|
...(extraction.length ? { extraction } : {}),
|
|
1332
1904
|
});
|
|
1333
1905
|
optimisticFacts.push({ ...t, provenance: tag, sentence, ...(extraction.length ? { extraction } : {}) });
|
|
1906
|
+
const attribution = attributionRowFor(t, { speaker: attributedTo, tag, observedAt });
|
|
1907
|
+
if (attribution) { candidateWrites.push(attribution); attributions.push({ ...attribution, sentence }); }
|
|
1334
1908
|
}
|
|
1335
1909
|
const { ids } = await appendFacts(dir, candidateWrites);
|
|
1336
1910
|
for (const id of ids) taggedIds.add(id);
|
|
@@ -1364,6 +1938,7 @@ export async function ingestText(text, {
|
|
|
1364
1938
|
recognized: recognizedSentences,
|
|
1365
1939
|
extracted,
|
|
1366
1940
|
optimistic: optimisticFacts,
|
|
1941
|
+
attributions,
|
|
1367
1942
|
skipped: sentenceCount - recognizedSentences - optimisticSentences,
|
|
1368
1943
|
ungroundedTerms: [...ungroundedTerms],
|
|
1369
1944
|
ungroundedCounts,
|