@polycode-projects/the-mechanical-code-talker 6.0.18 → 6.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -23
- package/bin/tmct.mjs +16 -33
- package/corpus/LICENSES.json +0 -21
- package/corpus/README.md +10 -13
- package/corpus/reference/manifest.json +19 -19
- package/corpus/reference/shards/ref-01.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-04.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-08.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-10.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-11.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-17.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-20.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-25.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-2c.jsonl.gz +0 -0
- package/corpus/tier2/generate.mjs +6 -142
- package/corpus/tier2/manifest.json +0 -42
- package/package.json +6 -4
- package/src/adapters/corpus/child-seed.mjs +74 -0
- package/src/adapters/corpus/conceptnet.mjs +45 -26
- package/src/adapters/corpus/research-source.mjs +6 -2
- package/src/adapters/corpus/wikidata-live.mjs +92 -51
- package/src/adapters/memory/blocks.mjs +7 -1
- package/src/adapters/memory/core.mjs +505 -107
- package/src/adapters/memory/corpus-bands.mjs +27 -10
- package/src/adapters/memory/inspect.mjs +24 -5
- package/src/adapters/memory/rows.mjs +359 -30
- package/src/adapters/memory/shacl.mjs +10 -3
- package/src/domain/ask.mjs +27 -10
- package/src/domain/cli-verbs.mjs +3 -4
- package/src/domain/completions/group.mjs +8 -3
- package/src/domain/completions/infer.mjs +7 -2
- package/src/domain/completions/prune.mjs +5 -1
- package/src/domain/completions/rank.mjs +7 -2
- package/src/domain/digest/compose.mjs +5 -1
- package/src/domain/digest/select.mjs +12 -6
- package/src/domain/domain.mjs +15 -8
- package/src/domain/el-classify.mjs +11 -2
- package/src/domain/fact-phrase.mjs +86 -4
- package/src/domain/hash.mjs +9 -0
- package/src/domain/memory/bias.mjs +8 -4
- package/src/domain/memory/capability.mjs +12 -6
- package/src/domain/memory/fact-order.mjs +29 -0
- package/src/domain/memory/resolution.mjs +3 -0
- package/src/domain/news-feed.mjs +862 -92
- package/src/domain/reference-pack.mjs +5 -0
- package/src/domain/sense-gate.mjs +220 -0
- package/src/domain/sense-scope.mjs +116 -0
- package/src/domain/sense-split.mjs +1 -1
- package/src/domain/syllogise.mjs +60 -21
- package/src/domain/tableau.mjs +23 -14
- package/src/domain/term-ledger.mjs +16 -1
- package/src/domain/worlds-pack.mjs +5 -1
- package/src/services/adventure-autoplay.mjs +6 -1
- package/src/services/adventure-editor.mjs +43 -21
- package/src/services/adventure-viz.mjs +26 -9
- package/src/services/adventure.mjs +40 -10
- package/src/services/chat.mjs +270 -125
- package/src/services/extensions.mjs +51 -58
- package/src/services/extract-facts.mjs +906 -66
- package/src/services/init.mjs +4 -4
- package/src/services/ledger-viz.mjs +9 -4
- package/src/services/memory-panel-viz.mjs +4 -5
- package/src/services/mud-editor.mjs +40 -16
- package/src/services/mud-viz.mjs +8 -2
- package/src/services/mudiii-turn.mjs +5 -3
- package/src/services/mudiii-viz.mjs +8 -2
- package/src/services/news.mjs +306 -21
- package/src/services/research-viz.mjs +1 -1
- package/src/services/sprite-catalog-viz.mjs +10 -5
- package/src/surfaces/web/adventure-browser-entry.mjs +6 -12
- package/src/surfaces/web/memory-ask-browser.bundle.js +152 -151
- package/src/surfaces/web/mud-browser-entry.mjs +7 -11
- package/src/surfaces/web/research-browser-entry.mjs +5 -2
- package/corpus/tier2/aws.jsonl +0 -39
- package/corpus/tier2/java.jsonl +0 -31
- package/corpus/tier2/python.jsonl +0 -30
package/src/domain/news-feed.mjs
CHANGED
|
@@ -6,10 +6,11 @@
|
|
|
6
6
|
// sortFactIndividualsById holds for a CRDT-merged fact set.
|
|
7
7
|
|
|
8
8
|
import { sha256Bytes, normFactTerm } from "./hash.mjs";
|
|
9
|
-
import { FACT_PREDICATE_PHRASES, predicatePhrase, factSentence } from "./fact-phrase.mjs";
|
|
9
|
+
import { FACT_PREDICATE_PHRASES, predicatePhrase, predicateVerb, factSentence } from "./fact-phrase.mjs";
|
|
10
10
|
import { STOP_SET } from "./hub-terms.mjs";
|
|
11
11
|
import { articleFor } from "./digest/words.mjs";
|
|
12
12
|
import { provenanceTagToSource } from "./memory/trust.mjs";
|
|
13
|
+
import { buildSenseScope } from "./sense-scope.mjs";
|
|
13
14
|
|
|
14
15
|
export const NEWS_HUB_HOPS = 2; // fixed by design, not a knob
|
|
15
16
|
|
|
@@ -71,6 +72,72 @@ export function newsWindowRows(rows, { now, windowMs }) {
|
|
|
71
72
|
});
|
|
72
73
|
}
|
|
73
74
|
|
|
75
|
+
// ---------------------------------------------------------------------------
|
|
76
|
+
// Attributions: who a report said its claim came from.
|
|
77
|
+
// ---------------------------------------------------------------------------
|
|
78
|
+
|
|
79
|
+
// A reified attribution names its claim by that claim's own group id, so its
|
|
80
|
+
// subject is "fact:" and sixteen hex — normFactTerm's own carve-out shape, and
|
|
81
|
+
// nothing a source ever writes as a term.
|
|
82
|
+
const FACT_REFERENCE_TERM_RE = /^fact:[0-9a-f]{16}$/;
|
|
83
|
+
|
|
84
|
+
// A fact id is minted lowercase, and both the claim's own `id` and the
|
|
85
|
+
// attribution's subject go through this before either is used as a key — a
|
|
86
|
+
// speaker matched on one spelling and stored under another is a silent drop.
|
|
87
|
+
const factIdKey = (term) => String(term ?? "").trim().toLowerCase();
|
|
88
|
+
|
|
89
|
+
const namesAFactRow = (term) => FACT_REFERENCE_TERM_RE.test(factIdKey(term));
|
|
90
|
+
|
|
91
|
+
/** The fact `row` is ABOUT, when either of its sides names one rather than a
|
|
92
|
+
* thing, else "". */
|
|
93
|
+
function referencedFactId(row) {
|
|
94
|
+
const subject = factIdKey(row?.subject);
|
|
95
|
+
if (namesAFactRow(subject)) return subject;
|
|
96
|
+
const object = factIdKey(row?.object);
|
|
97
|
+
return namesAFactRow(object) ? object : "";
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** True when `row` is about another row rather than about the world. Every card
|
|
101
|
+
* lane scores, walks and prints terms, and `looksLikeEntityTerm` reads a bare
|
|
102
|
+
* `fact:285cf1618315591b` as a perfectly good one-word name, so a row like this
|
|
103
|
+
* loose in a lane can head a card with a hex id. */
|
|
104
|
+
export function isFactReferenceRow(row) {
|
|
105
|
+
return Boolean(referencedFactId(row));
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const ATTRIBUTED_TO_PREDICATE = "mgx:attributedTo";
|
|
109
|
+
|
|
110
|
+
/** Splits a fact set once, at the door: the claims a card may read, and the
|
|
111
|
+
* speakers each claim was attributed to (claim group id -> speaker[], sorted).
|
|
112
|
+
* Everything downstream — the hub gate, the adjacency index, the walk, the
|
|
113
|
+
* sentences, a card's own `factIds` — takes `claims`, so an attribution reaches
|
|
114
|
+
* no lane at all and the suppression cannot be missed one lane at a time.
|
|
115
|
+
*
|
|
116
|
+
* Pure and order-independent: one claim's speakers come back in the same sorted
|
|
117
|
+
* order whichever order the attributions arrived in, and a claim the fact set
|
|
118
|
+
* never names simply has no entry — an attribution can arrive before its claim,
|
|
119
|
+
* after it, or without it. */
|
|
120
|
+
export function partitionAttributions(rows) {
|
|
121
|
+
const claims = [];
|
|
122
|
+
const named = new Map();
|
|
123
|
+
for (const row of rows) {
|
|
124
|
+
if (!isFactReferenceRow(row)) {
|
|
125
|
+
claims.push(row);
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
if (row.predicate !== ATTRIBUTED_TO_PREDICATE) continue;
|
|
129
|
+
const claimId = factIdKey(row.subject);
|
|
130
|
+
const speaker = String(row.object ?? "").trim();
|
|
131
|
+
if (!claimId || !speaker) continue;
|
|
132
|
+
let speakers = named.get(claimId);
|
|
133
|
+
if (!speakers) named.set(claimId, (speakers = new Set()));
|
|
134
|
+
speakers.add(speaker);
|
|
135
|
+
}
|
|
136
|
+
const speakersByClaimId = new Map();
|
|
137
|
+
for (const [claimId, speakers] of named) speakersByClaimId.set(claimId, [...speakers].sort());
|
|
138
|
+
return { claims, speakersByClaimId };
|
|
139
|
+
}
|
|
140
|
+
|
|
74
141
|
// ---------------------------------------------------------------------------
|
|
75
142
|
// Item identity: what makes two fetched snapshots the same newsworthy item.
|
|
76
143
|
// ---------------------------------------------------------------------------
|
|
@@ -306,6 +373,27 @@ const ENTITY_TRAILING_AUXILIARY_WORDS = new Set([
|
|
|
306
373
|
"is", "are", "was", "were", "be", "been", "being", "am", "has", "have", "had",
|
|
307
374
|
]);
|
|
308
375
|
|
|
376
|
+
// A name is one noun phrase, so a word that opens a new phrase or clause
|
|
377
|
+
// BETWEEN a term's first and last word marks a headline a frame tore into
|
|
378
|
+
// subject + predicate + remainder ("colombia as rescuers free quake victim").
|
|
379
|
+
// Mirrors extract-facts.mjs's INTERIOR_CLAUSE_WORDS, for the same reason the
|
|
380
|
+
// sets above mirror their originals. "of" stays out: real names are built with
|
|
381
|
+
// it ("house of representatives").
|
|
382
|
+
// A term the source itself wrapped in quotation marks is a title it quoted, and
|
|
383
|
+
// a title is free to read as a clause. Mirrors extract-facts.mjs's own
|
|
384
|
+
// QUOTED_TERM_RE, and exempts the interior rule alone.
|
|
385
|
+
const ENTITY_QUOTED_TERM_RE = /^["“'‘].*["”'’]$/;
|
|
386
|
+
const ENTITY_INTERIOR_CLAUSE_WORDS = new Set([
|
|
387
|
+
"a", "an", "the",
|
|
388
|
+
"and", "or", "but", "because", "since", "although", "though", "whereas", "while", "so",
|
|
389
|
+
"if", "when", "then", "however", "as", "that", "which", "who", "whom", "whose",
|
|
390
|
+
"is", "are", "was", "were", "be", "been", "being", "am", "has", "have", "had",
|
|
391
|
+
"do", "does", "did", "can", "could", "will", "would", "should", "may", "might", "must",
|
|
392
|
+
"in", "on", "at", "for", "to", "with", "from", "by", "into", "onto",
|
|
393
|
+
"over", "under", "after", "before", "between", "during", "about", "near", "through",
|
|
394
|
+
"against", "among", "within", "without", "per",
|
|
395
|
+
]);
|
|
396
|
+
|
|
309
397
|
/** Does `term` read as a thing's name rather than a clause fragment? Bounds
|
|
310
398
|
* the word count and rejects a leading conjunction, auxiliary or
|
|
311
399
|
* preposition (test E's condition 3, PLAN_NEWSWORTHINESS.md section 2), plus
|
|
@@ -321,6 +409,11 @@ function looksLikeEntityTerm(term) {
|
|
|
321
409
|
if (ENTITY_PARTICLE_LEAD_WORDS.has(first)) return false;
|
|
322
410
|
if (ENTITY_PRONOUN_LEAD_WORDS.has(first.replace(ENTITY_CLITIC_SUFFIX_RE, ""))) return false;
|
|
323
411
|
if (ENTITY_TRAILING_AUXILIARY_WORDS.has(words[words.length - 1].toLowerCase())) return false;
|
|
412
|
+
if (!ENTITY_QUOTED_TERM_RE.test(`${words[0]} ${words[words.length - 1]}`)) {
|
|
413
|
+
for (let i = 1; i < words.length - 1; i += 1) {
|
|
414
|
+
if (ENTITY_INTERIOR_CLAUSE_WORDS.has(words[i].toLowerCase())) return false;
|
|
415
|
+
}
|
|
416
|
+
}
|
|
324
417
|
return true;
|
|
325
418
|
}
|
|
326
419
|
|
|
@@ -527,33 +620,52 @@ export function hubSeedTerms(hub) {
|
|
|
527
620
|
* `seedTerms` starts the walk from more than the hub itself (hubSeedTerms).
|
|
528
621
|
* Everything downstream that asks "is this the hub" — the report sentences,
|
|
529
622
|
* the sources, the neighbourhood — still reads the hub term alone, so a seed
|
|
530
|
-
* widens only what the card can draw background from.
|
|
623
|
+
* widens only what the card can draw background from.
|
|
624
|
+
*
|
|
625
|
+
* `excludeIds` drops rows before the cap rather than after it, so a card that
|
|
626
|
+
* gives a report away to another card (storyCoverage) spends the freed budget
|
|
627
|
+
* on rows it will actually show.
|
|
628
|
+
*
|
|
629
|
+
* `inSense` is the same-sense discipline (sense-scope.mjs): a `(term) =>
|
|
630
|
+
* boolean` test the walk applies to every term that is not a seed. A term it
|
|
631
|
+
* refuses collects no row and joins no frontier, so the walk stays inside the
|
|
632
|
+
* seeds' own sense instead of climbing a shared class node and coming back
|
|
633
|
+
* down another meaning of it. What a source actually REPORTED about a seed is
|
|
634
|
+
* exempt: a card never drops its own news to keep a sense tidy. */
|
|
531
635
|
export function subgraphAround(rows, hub, {
|
|
532
636
|
hops = NEWS_HUB_HOPS, cap = 60, adjacency = null, priorityIds = null, seedTerms = null,
|
|
637
|
+
excludeIds = null, inSense = null,
|
|
533
638
|
} = {}) {
|
|
534
639
|
const adj = adjacency ?? buildTermAdjacency(rows);
|
|
535
640
|
const hubTerm = normFactTerm(hub);
|
|
536
641
|
const seeds = (seedTerms?.length ? seedTerms : [hubTerm]).map((t) => normFactTerm(t)).filter(Boolean);
|
|
537
642
|
const visited = new Set(seeds);
|
|
643
|
+
const seedSet = new Set(seeds);
|
|
538
644
|
let frontier = [...new Set(seeds)].sort();
|
|
539
645
|
const collected = new Map();
|
|
540
646
|
const hopOf = new Map();
|
|
647
|
+
const isExcluded = excludeIds instanceof Set ? (id) => excludeIds.has(id) : () => false;
|
|
648
|
+
const isPriority = (id) => (priorityIds instanceof Set ? priorityIds.has(id) : Boolean(priorityIds?.includes?.(id)));
|
|
649
|
+
const inScope = typeof inSense === "function" ? (term) => seedSet.has(term) || inSense(term) : () => true;
|
|
541
650
|
for (let hop = 0; hop < hops; hop += 1) {
|
|
542
651
|
const nextFrontier = new Set();
|
|
543
652
|
for (const term of [...frontier].sort()) {
|
|
544
653
|
for (const idx of adj.byTerm.get(term) ?? []) {
|
|
545
654
|
const row = rows[idx];
|
|
655
|
+
if (isExcluded(row.id)) continue;
|
|
656
|
+
const [s, o] = adj.terms[idx];
|
|
657
|
+
const staysInSense = inScope(s) && inScope(o);
|
|
658
|
+
const isOwnReport = isPriority(row.id) && (seedSet.has(s) || seedSet.has(o));
|
|
659
|
+
if (!staysInSense && !isOwnReport) continue;
|
|
546
660
|
collected.set(row.id, row);
|
|
547
661
|
if (!hopOf.has(row.id)) hopOf.set(row.id, hop);
|
|
548
|
-
|
|
549
|
-
if (!visited.has(
|
|
550
|
-
if (!visited.has(o)) nextFrontier.add(o);
|
|
662
|
+
if (!visited.has(s) && inScope(s)) nextFrontier.add(s);
|
|
663
|
+
if (!visited.has(o) && inScope(o)) nextFrontier.add(o);
|
|
551
664
|
}
|
|
552
665
|
}
|
|
553
666
|
for (const term of nextFrontier) visited.add(term);
|
|
554
667
|
frontier = [...nextFrontier].sort();
|
|
555
668
|
}
|
|
556
|
-
const isPriority = (id) => (priorityIds instanceof Set ? priorityIds.has(id) : Boolean(priorityIds?.includes?.(id)));
|
|
557
669
|
return [...collected.values()]
|
|
558
670
|
.sort((a, b) => (isPriority(b.id) - isPriority(a.id))
|
|
559
671
|
|| (hopOf.get(a.id) - hopOf.get(b.id))
|
|
@@ -561,6 +673,32 @@ export function subgraphAround(rows, hub, {
|
|
|
561
673
|
.slice(0, cap);
|
|
562
674
|
}
|
|
563
675
|
|
|
676
|
+
// How far the walk goes from an entity the ARTICLE names rather than a fact,
|
|
677
|
+
// and how many rows it may bring back. One hop: what the graph says about that
|
|
678
|
+
// entity itself, never what it says about everything that entity touches. The
|
|
679
|
+
// hub's own two-hop walk is unchanged; this one runs beside it.
|
|
680
|
+
const ARTICLE_ENTITY_HOPS = 1;
|
|
681
|
+
const ARTICLE_ENTITY_ROW_CAP = 24;
|
|
682
|
+
|
|
683
|
+
/** The rows sitting one hop from an entity the card's article names. Seeded
|
|
684
|
+
* from `terms` rather than the hub, so a definition the graph holds about a
|
|
685
|
+
* name inside the headline reaches the card even when no fact of the card's
|
|
686
|
+
* own touches that name — "amigados is a disk operating system" beside a
|
|
687
|
+
* report whose only fact is that a site discussed the headline.
|
|
688
|
+
*
|
|
689
|
+
* `excludeIds` carries the card's reported rows as well as the ones another
|
|
690
|
+
* card claimed, so this walk returns background and nothing else: what a
|
|
691
|
+
* source reported is the hub walk's business. */
|
|
692
|
+
export function articleEntityRows(rows, terms, {
|
|
693
|
+
adjacency = null, excludeIds = null, cap = ARTICLE_ENTITY_ROW_CAP, inSense = null,
|
|
694
|
+
} = {}) {
|
|
695
|
+
const seedTerms = (terms || []).map((term) => normFactTerm(term)).filter(Boolean);
|
|
696
|
+
if (!seedTerms.length) return [];
|
|
697
|
+
return subgraphAround(rows, seedTerms[0], {
|
|
698
|
+
hops: ARTICLE_ENTITY_HOPS, cap, adjacency, seedTerms, excludeIds, inSense,
|
|
699
|
+
});
|
|
700
|
+
}
|
|
701
|
+
|
|
564
702
|
/** The strongest prior kind among `rows`, for the item's trust chip — read
|
|
565
703
|
* off each row's own `trust` (a number) and `sourceTypes` (the kind array
|
|
566
704
|
* readFactRows already computes), never a re-derivation of SOURCE_PRIOR. */
|
|
@@ -616,6 +754,50 @@ function joinWithAnd(items) {
|
|
|
616
754
|
return `${items.slice(0, -1).join(", ")} and ${items[items.length - 1]}`;
|
|
617
755
|
}
|
|
618
756
|
|
|
757
|
+
/** The speakers `speakersByClaimId` attributes to `rows`, deduped and sorted.
|
|
758
|
+
* Empty for rows nothing attributed, which is every row until a report stores
|
|
759
|
+
* one. */
|
|
760
|
+
function speakersFor(rows, speakersByClaimId) {
|
|
761
|
+
if (!(speakersByClaimId instanceof Map) || !speakersByClaimId.size) return [];
|
|
762
|
+
const named = new Set();
|
|
763
|
+
for (const row of rows) {
|
|
764
|
+
for (const speaker of speakersByClaimId.get(factIdKey(row.id)) ?? []) named.add(speaker);
|
|
765
|
+
}
|
|
766
|
+
return [...named].sort();
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
/** ", president trump said" — the article's own construction, folded onto the
|
|
770
|
+
* end of the claim's own sentence rather than printed as apparatus beside it. */
|
|
771
|
+
function speakerClause(speakers) {
|
|
772
|
+
return speakers.length ? `, ${joinWithAnd(speakers)} said` : "";
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
/** One row's whole sentence with its own speaker folded in. */
|
|
776
|
+
function attributedFactSentence(row, speakersByClaimId) {
|
|
777
|
+
return `${factSentence(row)}${speakerClause(speakersFor([row], speakersByClaimId))}`;
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
/** The speaker clause a GROUPED sentence may carry. One sentence stands for
|
|
781
|
+
* several rows, so the clause is only true where the attributed rows among them
|
|
782
|
+
* already name every object the sentence prints. A group whose UNattributed row
|
|
783
|
+
* brings an object of its own would put that object in a speaker's mouth, so
|
|
784
|
+
* the clause drops whole rather than narrowing to a sentence it no longer
|
|
785
|
+
* describes. */
|
|
786
|
+
function groupSpeakerClause(rows, printedObjects, speakersByClaimId) {
|
|
787
|
+
if (!(speakersByClaimId instanceof Map) || !speakersByClaimId.size) return "";
|
|
788
|
+
const named = new Set();
|
|
789
|
+
const spokenObjects = new Set();
|
|
790
|
+
for (const row of rows) {
|
|
791
|
+
const speakers = speakersByClaimId.get(factIdKey(row.id));
|
|
792
|
+
if (!speakers?.length) continue;
|
|
793
|
+
for (const speaker of speakers) named.add(speaker);
|
|
794
|
+
spokenObjects.add(row.object);
|
|
795
|
+
}
|
|
796
|
+
if (!named.size) return "";
|
|
797
|
+
if (!printedObjects.every((object) => spokenObjects.has(object))) return "";
|
|
798
|
+
return speakerClause([...named].sort());
|
|
799
|
+
}
|
|
800
|
+
|
|
619
801
|
// ---------------------------------------------------------------------------
|
|
620
802
|
// What one card reports, and whose neighbourhood it sits in.
|
|
621
803
|
// ---------------------------------------------------------------------------
|
|
@@ -721,6 +903,32 @@ const KNOWN_FACT_SENTENCE_CAP = 2;
|
|
|
721
903
|
// wall of text.
|
|
722
904
|
const OBJECTS_PER_SENTENCE = 6;
|
|
723
905
|
|
|
906
|
+
// Two report rows on one card can state ONE act under two words: a headline
|
|
907
|
+
// says a prisoner was "freed", the description says "released", and the graph
|
|
908
|
+
// is right to hold both edges because two different sentences really said so.
|
|
909
|
+
// The card is the thing that should say it once.
|
|
910
|
+
//
|
|
911
|
+
// The fold is here, at assembly, and NOT at extraction, because only here is
|
|
912
|
+
// the whole row set in view. Extraction reads one verb at a time, and folding
|
|
913
|
+
// "free" onto "release" there turned "rescuers free quake victim" into a jail
|
|
914
|
+
// delivery: the two senses of "free" split on the subject's kind, an open set,
|
|
915
|
+
// and a headline carries nothing to tell them apart. A card folds only where
|
|
916
|
+
// BOTH verbs already stand over the SAME subject naming the SAME people, which
|
|
917
|
+
// is the evidence extraction never had. A rescue card carries "free" alone, so
|
|
918
|
+
// nothing folds and it still reads as a rescue.
|
|
919
|
+
//
|
|
920
|
+
// The bar for a pair is that the two words name one act wherever a card can
|
|
921
|
+
// hold both. Verbs that merely share a subject and an object stay apart,
|
|
922
|
+
// opposites first among them: "russia detains X" and "russia releases X" are
|
|
923
|
+
// two claims about one prisoner, they are in no group together, and each keeps
|
|
924
|
+
// its own sentence.
|
|
925
|
+
const ONE_ACT_VERB_GROUPS = [
|
|
926
|
+
["free", "release"],
|
|
927
|
+
];
|
|
928
|
+
const ONE_ACT_CANONICAL_VERB = new Map(
|
|
929
|
+
ONE_ACT_VERB_GROUPS.flatMap((group) => group.map((verb) => [verb, group[0]])),
|
|
930
|
+
);
|
|
931
|
+
|
|
724
932
|
// A term the graph says is more than this many things is read across senses:
|
|
725
933
|
// "earthquake" is a natural event, a cognition, a social station and nine more,
|
|
726
934
|
// so no single class line about it is trustworthy on a card about one quake.
|
|
@@ -768,40 +976,58 @@ function identityFans(subgraphRows) {
|
|
|
768
976
|
return { senseFan, sourcedSenseFan, categoryFan };
|
|
769
977
|
}
|
|
770
978
|
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
979
|
+
// Where a background row's anchor term came from, and so which rows the
|
|
980
|
+
// disclosure names first: the card's own hub, then a term its report names,
|
|
981
|
+
// then an entity its article names that no fact of the card's own touches.
|
|
982
|
+
const ANCHOR_RANK_HUB = 0;
|
|
983
|
+
const ANCHOR_RANK_REPORT = 1;
|
|
984
|
+
const ANCHOR_RANK_ARTICLE = 2;
|
|
985
|
+
|
|
986
|
+
/** What this card is ABOUT, each term mapped to its anchor rank: its hub, the
|
|
987
|
+
* region of a hub the source spelled "settlement, region", the other terms its
|
|
988
|
+
* own report sentences name, and the entities its article names (`articleTerms`)
|
|
989
|
+
* — minus any of those that reads across senses. A quake card's report
|
|
774
990
|
* names "earthquake" and a place; the class term is where the graph's
|
|
775
991
|
* knowledge is thinnest and its senses widest, so background drawn through it
|
|
776
992
|
* is background about earthquakes in general, never about this quake. The hub
|
|
777
|
-
* itself is always in, whatever its sense count — the card is about it.
|
|
778
|
-
|
|
993
|
+
* itself is always in, whatever its sense count — the card is about it.
|
|
994
|
+
*
|
|
995
|
+
* An article entity earns the same reading a report term gets, one rank
|
|
996
|
+
* behind it: the story's own words name it, but no fact the card reports
|
|
997
|
+
* does. */
|
|
998
|
+
function cardSubjectTerms(hub, subgraphRows, { reportedIds = null, senseFan, articleTerms = [] }) {
|
|
779
999
|
const hubTerm = normFactTerm(hub);
|
|
780
1000
|
const isReported = idMembership(reportedIds);
|
|
781
|
-
const terms = new
|
|
1001
|
+
const terms = new Map();
|
|
1002
|
+
for (const seed of hubSeedTerms(hubTerm)) terms.set(seed, ANCHOR_RANK_HUB);
|
|
1003
|
+
const admit = (raw, rank) => {
|
|
1004
|
+
const term = normFactTerm(raw);
|
|
1005
|
+
if (!term || terms.has(term) || STOP_SET.has(term)) return;
|
|
1006
|
+
if ((senseFan.get(term) || 0) > IDENTITY_MAX_CLASSES) return;
|
|
1007
|
+
terms.set(term, rank);
|
|
1008
|
+
};
|
|
782
1009
|
for (const row of hubReportRows(hubTerm, subgraphRows, { reportedIds })) {
|
|
783
1010
|
if (!isReported(row.id)) continue;
|
|
784
|
-
for (const raw of [row.subject, row.object])
|
|
785
|
-
const term = normFactTerm(raw);
|
|
786
|
-
if (!term || terms.has(term) || STOP_SET.has(term)) continue;
|
|
787
|
-
if ((senseFan.get(term) || 0) > IDENTITY_MAX_CLASSES) continue;
|
|
788
|
-
terms.add(term);
|
|
789
|
-
}
|
|
1011
|
+
for (const raw of [row.subject, row.object]) admit(raw, ANCHOR_RANK_REPORT);
|
|
790
1012
|
}
|
|
1013
|
+
for (const raw of articleTerms) admit(raw, ANCHOR_RANK_ARTICLE);
|
|
791
1014
|
return terms;
|
|
792
1015
|
}
|
|
793
1016
|
|
|
794
1017
|
/** The background rows worth telling a reader about, ranked: a row touching
|
|
795
1018
|
* one of this card's own subject terms, whose other side is not a category
|
|
796
1019
|
* node, and — for an identity row — whose subject is not read across senses.
|
|
797
|
-
* Ranks the hub's own rows first, then
|
|
798
|
-
*
|
|
799
|
-
*
|
|
800
|
-
|
|
1020
|
+
* Ranks the hub's own rows first, then the ones its report names, then the
|
|
1021
|
+
* ones an entity in its article names, then by how specific the other side
|
|
1022
|
+
* is, then by content-addressed id, so the same fact set always yields the
|
|
1023
|
+
* same lines in the same order. */
|
|
1024
|
+
export function knownFactRows(hub, subgraphRows, {
|
|
1025
|
+
reportedIds = null, limit = KNOWN_FACT_ROW_LIMIT, articleTerms = [],
|
|
1026
|
+
} = {}) {
|
|
801
1027
|
const hubTerm = normFactTerm(hub);
|
|
802
1028
|
const isReported = idMembership(reportedIds);
|
|
803
1029
|
const { senseFan, sourcedSenseFan, categoryFan } = identityFans(subgraphRows);
|
|
804
|
-
const subjects = cardSubjectTerms(hub, subgraphRows, { reportedIds, senseFan });
|
|
1030
|
+
const subjects = cardSubjectTerms(hub, subgraphRows, { reportedIds, senseFan, articleTerms });
|
|
805
1031
|
const neighbourIds = new Set(neighbourRows(hub, subgraphRows, { reportedIds }).map((r) => r.id));
|
|
806
1032
|
|
|
807
1033
|
const scored = [];
|
|
@@ -822,14 +1048,14 @@ export function knownFactRows(hub, subgraphRows, { reportedIds = null, limit = K
|
|
|
822
1048
|
if ((categoryFan.get(other) || 0) > CATEGORY_FAN_MAX) continue;
|
|
823
1049
|
scored.push({
|
|
824
1050
|
row,
|
|
825
|
-
|
|
1051
|
+
anchorRank: subjects.get(anchor),
|
|
826
1052
|
otherCategoryFan: categoryFan.get(other) || 0,
|
|
827
1053
|
otherSenseFan: senseFan.get(other) || 0,
|
|
828
1054
|
});
|
|
829
1055
|
}
|
|
830
1056
|
|
|
831
1057
|
return scored
|
|
832
|
-
.sort((a, b) => (
|
|
1058
|
+
.sort((a, b) => (a.anchorRank - b.anchorRank)
|
|
833
1059
|
|| (a.otherCategoryFan - b.otherCategoryFan)
|
|
834
1060
|
|| (a.otherSenseFan - b.otherSenseFan)
|
|
835
1061
|
|| byId(a.row, b.row))
|
|
@@ -854,58 +1080,113 @@ function predicatesInRenderOrder(rows) {
|
|
|
854
1080
|
return [...curated, ...rest];
|
|
855
1081
|
}
|
|
856
1082
|
|
|
1083
|
+
/** The act a report predicate states, as the key two predicates share when
|
|
1084
|
+
* they say it in one word: the predicate's verb lemma folded onto its group's
|
|
1085
|
+
* canonical verb, with any particle beside it, so `tmct:releases` and
|
|
1086
|
+
* `mgx:free` both read "release" while `mgx:strike-near` stays apart from
|
|
1087
|
+
* `mgx:strike`. Empty for a predicate that states no act at all (an identity
|
|
1088
|
+
* row, a comparative, a passive participle), which folds with nothing. */
|
|
1089
|
+
function oneActKey(predicate) {
|
|
1090
|
+
const verb = predicateVerb(predicate);
|
|
1091
|
+
if (!verb) return "";
|
|
1092
|
+
return `${ONE_ACT_CANONICAL_VERB.get(verb.lemma) ?? verb.lemma} ${verb.particle}`;
|
|
1093
|
+
}
|
|
1094
|
+
|
|
857
1095
|
/** One sentence per (subject, predicate) group over `rows`, in the order the
|
|
858
1096
|
* rows arrive — the same shape the hub's own relation sentences take, so a
|
|
859
|
-
* background line reads like the rest of the paragraph rather than a dump.
|
|
860
|
-
|
|
1097
|
+
* background line reads like the rest of the paragraph rather than a dump.
|
|
1098
|
+
* Each entry carries the group's own rows alongside its text, so a caller
|
|
1099
|
+
* that needs to know which facts a sentence came from (the bench's noisy-
|
|
1100
|
+
* line scoring) reads them off the same grouping the sentence itself used,
|
|
1101
|
+
* never a second derivation of it. */
|
|
1102
|
+
function groupedFactSentenceEntries(rows, speakersByClaimId = null) {
|
|
861
1103
|
const groups = new Map();
|
|
862
1104
|
for (const row of rows) {
|
|
863
1105
|
const key = `${row.subject}${row.predicate}`;
|
|
864
1106
|
let group = groups.get(key);
|
|
865
|
-
if (!group) groups.set(key, (group = { subject: row.subject, predicate: row.predicate, objects: [] }));
|
|
1107
|
+
if (!group) groups.set(key, (group = { subject: row.subject, predicate: row.predicate, objects: [], rows: [] }));
|
|
866
1108
|
group.objects.push(row.object);
|
|
1109
|
+
group.rows.push(row);
|
|
867
1110
|
}
|
|
868
|
-
return [...groups.values()].map(({ subject, predicate, objects }) => {
|
|
1111
|
+
return [...groups.values()].map(({ subject, predicate, objects, rows: groupRows }) => {
|
|
869
1112
|
const sorted = [...objects].sort();
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
1113
|
+
// "rdf:type" reads "is a" from the curated phrase table, which is the wrong
|
|
1114
|
+
// article before a vowel and a second one in front of an object articleFor
|
|
1115
|
+
// has already spelled. The identity clause the paragraph opens with says a
|
|
1116
|
+
// bare "is" and lets articleFor choose; a background line says it the same
|
|
1117
|
+
// way. Every other predicate, "is a kind of" included, carries whatever
|
|
1118
|
+
// article it needs inside its own phrase and takes the object bare.
|
|
1119
|
+
const text = predicate === "rdf:type"
|
|
1120
|
+
? `${subject} is ${joinObjects(sorted.map((object) => `${articleFor(object)} ${object}`))}`
|
|
1121
|
+
: `${subject} ${predicatePhrase(predicate, subject)} ${joinObjects(sorted)}`;
|
|
1122
|
+
return { text: `${text}${groupSpeakerClause(groupRows, sorted, speakersByClaimId)}`, rows: groupRows };
|
|
874
1123
|
});
|
|
875
1124
|
}
|
|
876
1125
|
|
|
1126
|
+
function groupedFactSentences(rows, speakersByClaimId = null) {
|
|
1127
|
+
return groupedFactSentenceEntries(rows, speakersByClaimId).map((entry) => entry.text);
|
|
1128
|
+
}
|
|
1129
|
+
|
|
877
1130
|
/** The sentences a card's paragraph is made of, as four ordered blocks: the
|
|
878
1131
|
* report (what a source said inside the window), the identity clause, the
|
|
879
|
-
* related facts the graph already held, and the neighbourhood.
|
|
880
|
-
*
|
|
881
|
-
*
|
|
882
|
-
|
|
1132
|
+
* related facts the graph already held, and the neighbourhood. Each entry is
|
|
1133
|
+
* `{ text, rows }` — the rendered sentence and the fact row(s) it came from
|
|
1134
|
+
* — so a caller that needs to know which facts actually reached the printed
|
|
1135
|
+
* text (the bench's noisy-line scoring) reads them off the same blocks
|
|
1136
|
+
* `renderNewsParagraph` itself slices, never a second derivation of it.
|
|
1137
|
+
* Callers that render only one block — the "what the graph already knew"
|
|
1138
|
+
* disclosure — read the block they want instead of re-deriving it. */
|
|
1139
|
+
function paragraphBlocks(hub, subgraphRows, { reportedIds = null, articleTerms = [], speakersByClaimId = null } = {}) {
|
|
883
1140
|
const hubTerm = normFactTerm(hub);
|
|
884
1141
|
const isReported = idMembership(reportedIds);
|
|
885
1142
|
const hubRows = subgraphRows.filter((r) => normFactTerm(r.subject) === hubTerm);
|
|
886
1143
|
const reportedHubRows = hubRows.filter((r) => isReported(r.id));
|
|
887
|
-
|
|
1144
|
+
// Every sentence here shares the one subject, so the act key and the objects
|
|
1145
|
+
// are all that separate two of them. A predicate whose act a sentence above
|
|
1146
|
+
// it has already stated, over people that sentence already names, adds no
|
|
1147
|
+
// word a reader has not read — its rows join the sentence that says their
|
|
1148
|
+
// act, so a fact still counts as printed and the fold can lose nothing. A
|
|
1149
|
+
// predicate that brings a new name to the act keeps its own sentence.
|
|
1150
|
+
const reportGroups = [];
|
|
1151
|
+
const statedActs = new Map();
|
|
888
1152
|
for (const predicate of predicatesInRenderOrder(reportedHubRows)) {
|
|
889
|
-
if (IDENTITY_PREDICATES.has(predicate) ||
|
|
890
|
-
const
|
|
891
|
-
|
|
892
|
-
.map((r) => r.object)
|
|
893
|
-
.sort();
|
|
1153
|
+
if (IDENTITY_PREDICATES.has(predicate) || reportGroups.length >= REPORT_SENTENCE_CAP) continue;
|
|
1154
|
+
const groupRows = reportedHubRows.filter((r) => r.predicate === predicate);
|
|
1155
|
+
const objects = groupRows.map((r) => r.object).sort();
|
|
894
1156
|
if (!objects.length) continue;
|
|
895
|
-
|
|
1157
|
+
const actKey = oneActKey(predicate);
|
|
1158
|
+
const stated = actKey ? statedActs.get(actKey) : null;
|
|
1159
|
+
if (stated && objects.every((object) => stated.objects.has(object))) {
|
|
1160
|
+
stated.group.rows.push(...groupRows);
|
|
1161
|
+
continue;
|
|
1162
|
+
}
|
|
1163
|
+
const group = { text: `${hub} ${predicatePhrase(predicate, hub)} ${joinObjects(objects)}`, rows: groupRows, objects };
|
|
1164
|
+
reportGroups.push(group);
|
|
1165
|
+
if (actKey && !stated) statedActs.set(actKey, { group, objects: new Set(objects) });
|
|
896
1166
|
}
|
|
1167
|
+
// The speaker is read off the whole group, once the fold above has finished
|
|
1168
|
+
// moving rows into it — the row an article attributed is often the FOLDED one,
|
|
1169
|
+
// not the row whose words the sentence ended up wearing.
|
|
1170
|
+
const report = reportGroups.map(({ text, rows: groupRows, objects }) => ({
|
|
1171
|
+
text: `${text}${groupSpeakerClause(groupRows, objects, speakersByClaimId)}`,
|
|
1172
|
+
rows: groupRows,
|
|
1173
|
+
}));
|
|
897
1174
|
|
|
898
1175
|
// A hub that only ever appears as an OBJECT — the place a quake struck, the
|
|
899
1176
|
// story a site discussed — has no subject-side row to build a sentence from,
|
|
900
1177
|
// and its card came out blank. What was reported about it still says
|
|
901
1178
|
// something, so those rows render whole, subject and all.
|
|
902
1179
|
if (!report.length) {
|
|
903
|
-
const
|
|
1180
|
+
const aboutHubRows = subgraphRows
|
|
904
1181
|
.filter((r) => normFactTerm(r.object) === hubTerm && normFactTerm(r.subject) !== hubTerm && isReported(r.id))
|
|
905
1182
|
.sort(byId)
|
|
906
|
-
.slice(0, OBJECTS_PER_SENTENCE)
|
|
907
|
-
|
|
908
|
-
|
|
1183
|
+
.slice(0, OBJECTS_PER_SENTENCE);
|
|
1184
|
+
if (aboutHubRows.length) {
|
|
1185
|
+
report.push({
|
|
1186
|
+
text: aboutHubRows.map((r) => attributedFactSentence(r, speakersByClaimId)).join("; "),
|
|
1187
|
+
rows: aboutHubRows,
|
|
1188
|
+
});
|
|
1189
|
+
}
|
|
909
1190
|
}
|
|
910
1191
|
|
|
911
1192
|
// The identity clause follows the news, never leads it, and only when
|
|
@@ -913,23 +1194,48 @@ function paragraphBlocks(hub, subgraphRows, { reportedIds = null } = {}) {
|
|
|
913
1194
|
// closure names thirteen classes for "france" — a list nobody asked for,
|
|
914
1195
|
// most of it the wrong sense — so it opens no card.
|
|
915
1196
|
const identity = [];
|
|
916
|
-
const
|
|
917
|
-
|
|
918
|
-
.map((r) => r.object)
|
|
919
|
-
.sort();
|
|
1197
|
+
const identityRows = hubRows.filter((r) => IDENTITY_PREDICATES.has(r.predicate) && !isDerivedRow(r));
|
|
1198
|
+
const identityObjects = identityRows.map((r) => r.object).sort();
|
|
920
1199
|
const identityIsSingleSense = identityObjects.length > 0 && identityObjects.length <= IDENTITY_MAX_CLASSES;
|
|
921
1200
|
if (identityIsSingleSense) {
|
|
922
|
-
|
|
1201
|
+
const said = groupSpeakerClause(identityRows, identityObjects, speakersByClaimId);
|
|
1202
|
+
identity.push({
|
|
1203
|
+
text: `${hub} is ${joinObjects(identityObjects.map((object) => `${articleFor(object)} ${object}`))}${said}`,
|
|
1204
|
+
rows: identityRows,
|
|
1205
|
+
});
|
|
923
1206
|
}
|
|
924
1207
|
|
|
925
|
-
const known =
|
|
1208
|
+
const known = groupedFactSentenceEntries(
|
|
1209
|
+
knownFactRows(hub, subgraphRows, { reportedIds, articleTerms }),
|
|
1210
|
+
speakersByClaimId,
|
|
1211
|
+
);
|
|
926
1212
|
|
|
927
1213
|
const neighbours = neighbourRows(hub, subgraphRows, { reportedIds });
|
|
928
|
-
const around = neighbours.length
|
|
1214
|
+
const around = neighbours.length
|
|
1215
|
+
? [{
|
|
1216
|
+
text: `Around it: ${neighbours.map((r) => attributedFactSentence(r, speakersByClaimId)).join("; ")}`,
|
|
1217
|
+
rows: neighbours,
|
|
1218
|
+
}]
|
|
1219
|
+
: [];
|
|
929
1220
|
|
|
930
1221
|
return { report, identity, known, around };
|
|
931
1222
|
}
|
|
932
1223
|
|
|
1224
|
+
/** The paragraph's own sentence entries (`{ text, rows }`), in print order and
|
|
1225
|
+
* sliced to exactly what `renderNewsParagraph` shows — the per-block caps
|
|
1226
|
+
* (identity, known) and the paragraph-wide `SENTENCE_CAP` both applied.
|
|
1227
|
+
* Shared by `renderNewsParagraph` and `printedParagraphRows` so the two can
|
|
1228
|
+
* never drift: one reads `.text`, the other reads `.rows`. */
|
|
1229
|
+
function paragraphSentenceEntries(hub, subgraphRows, { reportedIds = null, articleTerms = [], speakersByClaimId = null } = {}) {
|
|
1230
|
+
const { report, identity, known, around } = paragraphBlocks(hub, subgraphRows, { reportedIds, articleTerms, speakersByClaimId });
|
|
1231
|
+
return [
|
|
1232
|
+
...report,
|
|
1233
|
+
...identity.slice(0, IDENTITY_SENTENCE_CAP),
|
|
1234
|
+
...known.slice(0, KNOWN_FACT_SENTENCE_CAP),
|
|
1235
|
+
...around,
|
|
1236
|
+
].slice(0, SENTENCE_CAP);
|
|
1237
|
+
}
|
|
1238
|
+
|
|
933
1239
|
/** A card's paragraph: what a source reported, then what the thing is, then
|
|
934
1240
|
* the related facts the graph already held about it, then its own
|
|
935
1241
|
* neighbourhood (neighbourRows). Every sentence shown is a grounded fact,
|
|
@@ -940,29 +1246,403 @@ function paragraphBlocks(hub, subgraphRows, { reportedIds = null } = {}) {
|
|
|
940
1246
|
* into what was reported (the lead sentences) and what the graph already held
|
|
941
1247
|
* (the background ones). Defaults to null, meaning every row counts as
|
|
942
1248
|
* reported. */
|
|
943
|
-
export function renderNewsParagraph(hub, subgraphRows, { reportedIds = null } = {}) {
|
|
944
|
-
const
|
|
945
|
-
|
|
946
|
-
...report,
|
|
947
|
-
...identity.slice(0, IDENTITY_SENTENCE_CAP),
|
|
948
|
-
...known.slice(0, KNOWN_FACT_SENTENCE_CAP),
|
|
949
|
-
...around,
|
|
950
|
-
].slice(0, SENTENCE_CAP);
|
|
1249
|
+
export function renderNewsParagraph(hub, subgraphRows, { reportedIds = null, articleTerms = [], speakersByClaimId = null } = {}) {
|
|
1250
|
+
const sentences = paragraphSentenceEntries(hub, subgraphRows, { reportedIds, articleTerms, speakersByClaimId })
|
|
1251
|
+
.map((entry) => entry.text);
|
|
951
1252
|
return sentences.length ? `${sentences.join(". ")}.` : "";
|
|
952
1253
|
}
|
|
953
1254
|
|
|
1255
|
+
/** Every fact row that survives into the card's rendered main paragraph —
|
|
1256
|
+
* the same rows `renderNewsParagraph` drew its sentences from, after every
|
|
1257
|
+
* cap it applies (per-block and the paragraph-wide `SENTENCE_CAP`). A row
|
|
1258
|
+
* computed but sliced away before render (an "Around it" clause cut by the
|
|
1259
|
+
* overall cap, an identity class beyond `IDENTITY_MAX_CLASSES`) never
|
|
1260
|
+
* appears here, because it never appears on the card either. */
|
|
1261
|
+
export function printedParagraphRows(hub, subgraphRows, { reportedIds = null, articleTerms = [], speakersByClaimId = null } = {}) {
|
|
1262
|
+
return paragraphSentenceEntries(hub, subgraphRows, { reportedIds, articleTerms, speakersByClaimId })
|
|
1263
|
+
.flatMap((entry) => entry.rows);
|
|
1264
|
+
}
|
|
1265
|
+
|
|
954
1266
|
/** The "what the graph already knew" disclosure: the same related facts the
|
|
955
1267
|
* paragraph leads with, at the disclosure's own fuller depth, and nothing the
|
|
956
1268
|
* card already reported. Empty when the graph held nothing about this card's
|
|
957
1269
|
* own subjects — a card with no background says so rather than filling the
|
|
958
1270
|
* space with whatever the two-hop walk happened to reach. */
|
|
959
|
-
export function renderKnownFactsParagraph(hub, subgraphRows, { reportedIds = null } = {}) {
|
|
960
|
-
const sentences = groupedFactSentences(
|
|
1271
|
+
export function renderKnownFactsParagraph(hub, subgraphRows, { reportedIds = null, articleTerms = [], speakersByClaimId = null } = {}) {
|
|
1272
|
+
const sentences = groupedFactSentences(
|
|
1273
|
+
knownFactRows(hub, subgraphRows, { reportedIds, articleTerms }),
|
|
1274
|
+
speakersByClaimId,
|
|
1275
|
+
);
|
|
961
1276
|
return sentences.length ? `${sentences.join(". ")}.` : "";
|
|
962
1277
|
}
|
|
963
1278
|
|
|
1279
|
+
// ---------------------------------------------------------------------------
|
|
1280
|
+
// What a card says beyond its own headline.
|
|
1281
|
+
// ---------------------------------------------------------------------------
|
|
1282
|
+
|
|
1283
|
+
// Two spellings of one headline — the source's own capitals and punctuation,
|
|
1284
|
+
// and the quoted, lower-cased form a fact's object carries — fold to the same
|
|
1285
|
+
// key, so the comparison answers to the words alone.
|
|
1286
|
+
const HEADLINE_NOISE_RE = /[^a-z0-9]+/g;
|
|
1287
|
+
|
|
1288
|
+
const headlineKey = (text) => String(text ?? "").toLowerCase().replace(HEADLINE_NOISE_RE, " ").trim();
|
|
1289
|
+
|
|
1290
|
+
/** Every term this card is about: its hub, whatever its own reports name, and
|
|
1291
|
+
* the entities its article names. Deliberately looser than
|
|
1292
|
+
* `cardSubjectTerms` — that one picks what a SENTENCE may draw on, and drops
|
|
1293
|
+
* a term the graph reads across senses; this one asks the plainer question of
|
|
1294
|
+
* what the card is about at all, which is what a count of held background
|
|
1295
|
+
* answers to. */
|
|
1296
|
+
function cardAboutTerms(hub, reports, articleTerms) {
|
|
1297
|
+
const terms = new Set(hubSeedTerms(normFactTerm(hub)));
|
|
1298
|
+
for (const row of reports) {
|
|
1299
|
+
for (const raw of [row.subject, row.object]) {
|
|
1300
|
+
const term = normFactTerm(raw);
|
|
1301
|
+
if (term) terms.add(term);
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
for (const raw of articleTerms) {
|
|
1305
|
+
const term = normFactTerm(raw);
|
|
1306
|
+
if (term) terms.add(term);
|
|
1307
|
+
}
|
|
1308
|
+
return terms;
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
/** What one card is carrying, and the reason a feed puts one card above
|
|
1312
|
+
* another.
|
|
1313
|
+
*
|
|
1314
|
+
* `claims` are the card's own reports (`hubReportRows`) that say something
|
|
1315
|
+
* about the world. `headlineMentions` are its own reports whose object IS one
|
|
1316
|
+
* of its headlines — "hackernews discuss <that headline>", a true fact that
|
|
1317
|
+
* tells a reader nothing the card's own quoted report already prints.
|
|
1318
|
+
* `background` is what the graph holds about the terms the card is about:
|
|
1319
|
+
* the rows behind its "what the graph already knew" lines, counted before the
|
|
1320
|
+
* paragraph's own caps cut them, and never another card's report.
|
|
1321
|
+
*
|
|
1322
|
+
* A source whose wire format carries no body can only ever mint headline
|
|
1323
|
+
* mentions, so a card off one lands at zero on both counts unless a lookup
|
|
1324
|
+
* attached something to a name in its headline. Those cards read last. They
|
|
1325
|
+
* still build, still cite their source, and `thin` marks them so a reader or
|
|
1326
|
+
* a bench run can count how many there were.
|
|
1327
|
+
*
|
|
1328
|
+
* Pure over the rows and the headline strings. A card with no headlines to
|
|
1329
|
+
* compare against (a caller that wired no sources) counts every report as a
|
|
1330
|
+
* claim: nothing is demoted without the evidence to demote it. */
|
|
1331
|
+
export function cardSubstance(hub, subgraphRows, { reportedIds = null, articleTerms = [], headlines = [] } = {}) {
|
|
1332
|
+
const isReported = idMembership(reportedIds);
|
|
1333
|
+
const ownHeadlines = new Set(headlines.map(headlineKey).filter(Boolean));
|
|
1334
|
+
|
|
1335
|
+
const reports = hubReportRows(hub, subgraphRows, { reportedIds });
|
|
1336
|
+
let claims = 0;
|
|
1337
|
+
let headlineMentions = 0;
|
|
1338
|
+
for (const row of reports) {
|
|
1339
|
+
if (ownHeadlines.has(headlineKey(row.object))) headlineMentions += 1;
|
|
1340
|
+
else claims += 1;
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
const about = cardAboutTerms(hub, reports, articleTerms);
|
|
1344
|
+
let background = 0;
|
|
1345
|
+
for (const row of subgraphRows) {
|
|
1346
|
+
if (isReported(row.id) || isDerivedRow(row)) continue;
|
|
1347
|
+
if (about.has(normFactTerm(row.subject)) || about.has(normFactTerm(row.object))) background += 1;
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
return { claims, background, headlineMentions, thin: claims === 0 && background === 0 };
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
/** How a feed orders two cards once their build moment is equal: the one that
|
|
1354
|
+
* reports more about the world first, then the one the graph holds more
|
|
1355
|
+
* around, then id. Reports outrank background on purpose — a card leads with
|
|
1356
|
+
* what a source said, and a hub whose common-noun object the seed graph
|
|
1357
|
+
* happens to know hundreds of edges about does not outrank a card carrying
|
|
1358
|
+
* more news. */
|
|
1359
|
+
function bySubstance(a, b) {
|
|
1360
|
+
return (b.substance.claims - a.substance.claims) || (b.substance.background - a.substance.background);
|
|
1361
|
+
}
|
|
1362
|
+
|
|
1363
|
+
// ---------------------------------------------------------------------------
|
|
1364
|
+
// Which story a card tells, and what it is called.
|
|
1365
|
+
// ---------------------------------------------------------------------------
|
|
1366
|
+
|
|
1367
|
+
// The item tag a news row carries, at whatever depth the ingest wrapper
|
|
1368
|
+
// nested it ("news:<sourceId>@<itemId>", "optimistic-extract:news:…"), plus
|
|
1369
|
+
// the fixture replay's own twin. Every sentence a card prints comes from a
|
|
1370
|
+
// row, so which story a card is telling is readable from the fact set alone —
|
|
1371
|
+
// no source map, no arrival order, no clock.
|
|
1372
|
+
const NEWS_STORY_TAG_RE = /(?:^|:)news(?:-fixture)?:([^\s|]+)/;
|
|
1373
|
+
|
|
1374
|
+
/** The newsworthy item one row reported, or "" when its provenance names
|
|
1375
|
+
* none. Pure. */
|
|
1376
|
+
export function newsStoryKey(row) {
|
|
1377
|
+
return NEWS_STORY_TAG_RE.exec(String(row?.provenance || ""))?.[1] || "";
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
/** Every reported row touching each term, subject side or object side. Built
|
|
1381
|
+
* once per feed assembly and shared, for the same reason buildTermAdjacency
|
|
1382
|
+
* is. */
|
|
1383
|
+
function reportedRowsByTerm(reported) {
|
|
1384
|
+
const byTerm = new Map();
|
|
1385
|
+
for (const row of reported) {
|
|
1386
|
+
for (const term of new Set([normFactTerm(row.subject), normFactTerm(row.object)])) {
|
|
1387
|
+
if (!term) continue;
|
|
1388
|
+
let rowsFor = byTerm.get(term);
|
|
1389
|
+
if (!rowsFor) byTerm.set(term, (rowsFor = []));
|
|
1390
|
+
rowsFor.push(row);
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
return byTerm;
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
// How many stories a term's own reports must span before it stops being one
|
|
1397
|
+
// story's subject. Two is enough: a term the day's second story also names is
|
|
1398
|
+
// a publication, a wire desk or a class every item shares, and a card headed
|
|
1399
|
+
// by one repeats whatever the per-story cards already said.
|
|
1400
|
+
const PUBLICATION_STORY_MIN = 2;
|
|
1401
|
+
|
|
1402
|
+
/** How many distinct newsworthy items each term's own reports came from. */
|
|
1403
|
+
function storyCountsByTerm(rowsByTerm) {
|
|
1404
|
+
const counts = new Map();
|
|
1405
|
+
for (const [term, rowsFor] of rowsByTerm) {
|
|
1406
|
+
const keys = new Set();
|
|
1407
|
+
for (const row of rowsFor) {
|
|
1408
|
+
const key = newsStoryKey(row);
|
|
1409
|
+
if (key) keys.add(key);
|
|
1410
|
+
}
|
|
1411
|
+
counts.set(term, keys.size);
|
|
1412
|
+
}
|
|
1413
|
+
return counts;
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
// The classes the graph's own identity rows use to say a thing is a place or
|
|
1417
|
+
// a person. The same closed set the bench's entity-preservation metric reads
|
|
1418
|
+
// off the seed, duplicated here (not imported) because the domain layer never
|
|
1419
|
+
// imports a script.
|
|
1420
|
+
const PLACE_OR_PERSON_CLASS_TERMS = new Set([
|
|
1421
|
+
"place", "person", "city", "country", "location", "nation", "continent",
|
|
1422
|
+
"capital", "state", "province", "town", "region",
|
|
1423
|
+
]);
|
|
1424
|
+
|
|
1425
|
+
/** Every term the graph itself types as a place or a person, whoever stated
|
|
1426
|
+
* the identity. A card prefers one of these for its own title: it is the
|
|
1427
|
+
* thing the story is about, where a clause the same report threw off is only
|
|
1428
|
+
* something that happened to it. Pure over `rows`. */
|
|
1429
|
+
export function placeAndPersonTerms(rows) {
|
|
1430
|
+
const grounded = new Set();
|
|
1431
|
+
for (const row of rows) {
|
|
1432
|
+
if (row.predicate !== "rdf:type" && row.predicate !== "rdfs:subClassOf") continue;
|
|
1433
|
+
if (!PLACE_OR_PERSON_CLASS_TERMS.has(normFactTerm(row.object))) continue;
|
|
1434
|
+
const subject = normFactTerm(row.subject);
|
|
1435
|
+
if (subject) grounded.add(subject);
|
|
1436
|
+
}
|
|
1437
|
+
return grounded;
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
// A predicate minted from a source's own verb, lemma only: "mgx:hit",
|
|
1441
|
+
// "mgx:discuss". The curated table's entries carry a capital ("mgx:partOf")
|
|
1442
|
+
// and a folded preposition carries a hyphen ("mgx:strike-near"), and neither
|
|
1443
|
+
// can be a single word inside a term, so neither belongs in the verb set.
|
|
1444
|
+
const MINTED_VERB_PREDICATE_RE = /^mgx:([a-z]+)$/;
|
|
1445
|
+
|
|
1446
|
+
/** The verbs this window's own reports minted a predicate from. A term
|
|
1447
|
+
* carrying one of them is a clause the extraction cut in half, and the graph
|
|
1448
|
+
* says so itself rather than a hand-written verb list saying it. */
|
|
1449
|
+
export function reportedVerbWords(reported) {
|
|
1450
|
+
const verbs = new Set();
|
|
1451
|
+
for (const row of reported) {
|
|
1452
|
+
const lemma = MINTED_VERB_PREDICATE_RE.exec(String(row.predicate || ""))?.[1];
|
|
1453
|
+
if (lemma) verbs.add(lemma);
|
|
1454
|
+
}
|
|
1455
|
+
return verbs;
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
// How many words a name runs to before it reads as a sentence about a thing
|
|
1459
|
+
// rather than the thing's name. "south sandwich islands region" is a place;
|
|
1460
|
+
// "boats hit by mystery attackers" is what happened to some.
|
|
1461
|
+
const CLAUSE_TITLE_MAX_WORDS = 4;
|
|
1462
|
+
const CLAUSE_WORD_EDGE_RE = /^[^a-z0-9]+|[^a-z0-9]+$/g;
|
|
1463
|
+
|
|
1464
|
+
/** Does `term` read as a clause rather than a name — longer than a name runs,
|
|
1465
|
+
* or carrying one of the verbs the window's own reports minted? A clause
|
|
1466
|
+
* never takes a card's title from a grounded entity term. */
|
|
1467
|
+
export function readsAsClauseTerm(term, verbWords) {
|
|
1468
|
+
const words = String(term ?? "").trim().toLowerCase().split(/\s+/).filter(Boolean);
|
|
1469
|
+
if (!words.length) return false;
|
|
1470
|
+
if (words.length > CLAUSE_TITLE_MAX_WORDS) return true;
|
|
1471
|
+
const verbs = verbWords instanceof Set ? verbWords : new Set(verbWords || []);
|
|
1472
|
+
return words.some((word) => verbs.has(word.replace(CLAUSE_WORD_EDGE_RE, "")));
|
|
1473
|
+
}
|
|
1474
|
+
|
|
1475
|
+
/** The term a card is titled and keyed by, given the hub the gate chose and
|
|
1476
|
+
* the reported rows that hub sits in. In order:
|
|
1477
|
+
*
|
|
1478
|
+
* 1. the hub itself, when the graph already types it as a place or a person;
|
|
1479
|
+
* 2. a place/person term the hub's own reports name — "bali" over "sacred
|
|
1480
|
+
* glow", "clacton" over "election";
|
|
1481
|
+
* 3. the subject those reports share, but only when the hub reads as a clause
|
|
1482
|
+
* and only when that subject tells one story of its own — the publication
|
|
1483
|
+
* every headline hangs off is never a card's name;
|
|
1484
|
+
* 4. the hub, unchanged.
|
|
1485
|
+
*
|
|
1486
|
+
* Ties inside a step go to the term the reports name most, then alphabetical,
|
|
1487
|
+
* so the same fact set always titles a card the same way. Nothing here invents
|
|
1488
|
+
* a term: every candidate is already on one of the card's own rows. */
|
|
1489
|
+
export function hubTitleTerm(hubTerm, hubRows, { placeOrPerson, verbWords, storyCountByTerm }) {
|
|
1490
|
+
if (placeOrPerson.has(hubTerm)) return hubTerm;
|
|
1491
|
+
|
|
1492
|
+
const namesAThing = (term) => Boolean(term) && term !== hubTerm && !STOP_SET.has(term)
|
|
1493
|
+
&& !isQuantityTerm(term) && looksLikeEntityTerm(term) && !readsAsClauseTerm(term, verbWords);
|
|
1494
|
+
|
|
1495
|
+
const groundedCounts = new Map();
|
|
1496
|
+
const subjectCounts = new Map();
|
|
1497
|
+
for (const row of hubRows) {
|
|
1498
|
+
const subject = normFactTerm(row.subject);
|
|
1499
|
+
const object = normFactTerm(row.object);
|
|
1500
|
+
for (const term of new Set([subject, object])) {
|
|
1501
|
+
if (namesAThing(term) && placeOrPerson.has(term)) groundedCounts.set(term, (groundedCounts.get(term) || 0) + 1);
|
|
1502
|
+
}
|
|
1503
|
+
if (object !== hubTerm || !namesAThing(subject)) continue;
|
|
1504
|
+
if ((storyCountByTerm.get(subject) || 0) >= PUBLICATION_STORY_MIN) continue;
|
|
1505
|
+
subjectCounts.set(subject, (subjectCounts.get(subject) || 0) + 1);
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
const mostNamed = (counts) => [...counts.entries()]
|
|
1509
|
+
.sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))[0]?.[0] || "";
|
|
1510
|
+
|
|
1511
|
+
return mostNamed(groundedCounts)
|
|
1512
|
+
|| (readsAsClauseTerm(hubTerm, verbWords) ? mostNamed(subjectCounts) : "")
|
|
1513
|
+
|| hubTerm;
|
|
1514
|
+
}
|
|
1515
|
+
|
|
1516
|
+
/** True when a source spelled this term as a settlement and its region at once
|
|
1517
|
+
* — "wana, pakistan", "mina, nevada". The source has already said which of a
|
|
1518
|
+
* report's terms is the thing that happened somewhere, so the term reads as a
|
|
1519
|
+
* place even where no identity row types it as one. hubSeedTerms holds the
|
|
1520
|
+
* same reading for the walk. */
|
|
1521
|
+
const namesASettlementAndRegion = (term) => hubSeedTerms(term).length > 1;
|
|
1522
|
+
|
|
1523
|
+
/** The gate's hubs retitled by hubTitleTerm, with two hubs that answer to the
|
|
1524
|
+
* same name merged into one. Each keeps the two readings storyCoverage ranks
|
|
1525
|
+
* cards by, taken on the title the card will actually wear. Keeps the gate's
|
|
1526
|
+
* own sort — changed count desc, then term asc. */
|
|
1527
|
+
function titledHubs(hubs, rows, reported, rowsByTerm) {
|
|
1528
|
+
const placeOrPerson = placeAndPersonTerms(rows);
|
|
1529
|
+
const verbWords = reportedVerbWords(reported);
|
|
1530
|
+
const storyCountByTerm = storyCountsByTerm(rowsByTerm);
|
|
1531
|
+
const changedByTitle = new Map();
|
|
1532
|
+
for (const { term, changed } of hubs) {
|
|
1533
|
+
const title = hubTitleTerm(term, rowsByTerm.get(term) || [], { placeOrPerson, verbWords, storyCountByTerm });
|
|
1534
|
+
const held = changedByTitle.get(title);
|
|
1535
|
+
if (held === undefined || changed > held) changedByTitle.set(title, changed);
|
|
1536
|
+
}
|
|
1537
|
+
return [...changedByTitle.entries()]
|
|
1538
|
+
.map(([term, changed]) => ({
|
|
1539
|
+
term,
|
|
1540
|
+
changed,
|
|
1541
|
+
namesAnEntity: placeOrPerson.has(term) || namesASettlementAndRegion(term),
|
|
1542
|
+
clauseShaped: readsAsClauseTerm(term, verbWords),
|
|
1543
|
+
reportSubject: (rowsByTerm.get(term) || []).some((row) => normFactTerm(row.subject) === term),
|
|
1544
|
+
}))
|
|
1545
|
+
.sort((a, b) => b.changed - a.changed || (a.term < b.term ? -1 : a.term > b.term ? 1 : 0));
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
/** How a story picks between the cards that want to tell it. In order: the
|
|
1549
|
+
* hub whose own reports span fewest stories, since a card about one story
|
|
1550
|
+
* beats a publication's roundup of it; then the hub that names a place or a
|
|
1551
|
+
* person, since that is what the story is about; then the hub that
|
|
1552
|
+
* reads as a name rather than a clause; then the one the report puts on the
|
|
1553
|
+
* subject side, the story's actor where the object is what happened to it;
|
|
1554
|
+
* then the gate's own changed count; then the term itself, so the answer never
|
|
1555
|
+
* comes down to arrival order. */
|
|
1556
|
+
function byCardClaim(a, b) {
|
|
1557
|
+
return a.storyCount - b.storyCount
|
|
1558
|
+
|| (Number(b.namesAnEntity) - Number(a.namesAnEntity))
|
|
1559
|
+
|| (Number(a.clauseShaped) - Number(b.clauseShaped))
|
|
1560
|
+
|| (Number(b.reportSubject) - Number(a.reportSubject))
|
|
1561
|
+
|| (b.changed - a.changed)
|
|
1562
|
+
|| (a.term < b.term ? -1 : a.term > b.term ? 1 : 0);
|
|
1563
|
+
}
|
|
1564
|
+
|
|
1565
|
+
/** Which stories each hub gets to tell, and which of its reports belong to
|
|
1566
|
+
* another card. One story mints one card: the hubs bid for it in byCardClaim
|
|
1567
|
+
* order and the first takes it, so a report that threw off both a subject hub
|
|
1568
|
+
* and an object hub ("ukraine" beside "air war", "london" beside "glass")
|
|
1569
|
+
* stops minting a card each. A hub left with no story of its own does not
|
|
1570
|
+
* mint, and one that keeps some carries only those — the "hackernews" card
|
|
1571
|
+
* that repeated both of the day's Hacker News cards wholesale is gone, and one
|
|
1572
|
+
* that repeated three of four carries the fourth alone.
|
|
1573
|
+
*
|
|
1574
|
+
* A row whose provenance names no story is never claimed and never dropped, so
|
|
1575
|
+
* a hub built only from those still mints.
|
|
1576
|
+
*
|
|
1577
|
+
* Returns hub term -> `{ mints, coveredRowIds }`. */
|
|
1578
|
+
export function storyCoverage(hubs, rowsByTerm) {
|
|
1579
|
+
const rowIdsByStory = new Map();
|
|
1580
|
+
for (const { term } of hubs) {
|
|
1581
|
+
const byStory = new Map();
|
|
1582
|
+
for (const row of rowsByTerm.get(term) || []) {
|
|
1583
|
+
const key = newsStoryKey(row);
|
|
1584
|
+
if (!key) continue;
|
|
1585
|
+
let ids = byStory.get(key);
|
|
1586
|
+
if (!ids) byStory.set(key, (ids = []));
|
|
1587
|
+
ids.push(row.id);
|
|
1588
|
+
}
|
|
1589
|
+
rowIdsByStory.set(term, byStory);
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
const bidders = hubs
|
|
1593
|
+
.map((hub) => ({ ...hub, storyCount: rowIdsByStory.get(hub.term).size }))
|
|
1594
|
+
.sort(byCardClaim);
|
|
1595
|
+
|
|
1596
|
+
const claimed = new Set();
|
|
1597
|
+
const coverage = new Map();
|
|
1598
|
+
for (const { term, storyCount } of bidders) {
|
|
1599
|
+
const stories = rowIdsByStory.get(term);
|
|
1600
|
+
const coveredRowIds = new Set();
|
|
1601
|
+
let ownStories = 0;
|
|
1602
|
+
for (const key of [...stories.keys()].sort()) {
|
|
1603
|
+
if (claimed.has(key)) {
|
|
1604
|
+
for (const id of stories.get(key)) coveredRowIds.add(id);
|
|
1605
|
+
continue;
|
|
1606
|
+
}
|
|
1607
|
+
claimed.add(key);
|
|
1608
|
+
ownStories += 1;
|
|
1609
|
+
}
|
|
1610
|
+
coverage.set(term, { mints: storyCount === 0 || ownStories > 0, coveredRowIds });
|
|
1611
|
+
}
|
|
1612
|
+
return coverage;
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1615
|
+
/** The entities this card's own article names, as terms the graph could hold
|
|
1616
|
+
* facts about. `articleEntityNames` reads them out of the text the card
|
|
1617
|
+
* already shows — each source's headline and the description beneath it — and
|
|
1618
|
+
* the same discipline the hub gate applies then filters them: no stop word, no
|
|
1619
|
+
* quantity, no class the graph's own identity rows name, nothing that reads as
|
|
1620
|
+
* a clause rather than a name. Sorted, so a card's background never depends on
|
|
1621
|
+
* the order the names came back in. */
|
|
1622
|
+
function cardArticleTerms(sources, articleEntityNames, { concepts, readsAsEntityTerm }) {
|
|
1623
|
+
const texts = [];
|
|
1624
|
+
for (const source of sources) {
|
|
1625
|
+
if (source.title) texts.push(source.title);
|
|
1626
|
+
if (source.summary) texts.push(source.summary);
|
|
1627
|
+
}
|
|
1628
|
+
if (!texts.length) return [];
|
|
1629
|
+
const terms = new Set();
|
|
1630
|
+
for (const name of articleEntityNames(texts) || []) {
|
|
1631
|
+
const term = normFactTerm(name);
|
|
1632
|
+
if (!term || STOP_SET.has(term) || isQuantityTerm(term) || concepts.has(term)) continue;
|
|
1633
|
+
if (!readsAsEntityTerm(term)) continue;
|
|
1634
|
+
terms.add(term);
|
|
1635
|
+
}
|
|
1636
|
+
return [...terms].sort();
|
|
1637
|
+
}
|
|
1638
|
+
|
|
964
1639
|
/** newsworthyHubs -> one item per hub (PLAN_NEWS_FEED.md section 6.6),
|
|
965
|
-
* paragraph included, sorted builtAt desc then
|
|
1640
|
+
* paragraph included, sorted builtAt desc, then by what the card carries
|
|
1641
|
+
* (`cardSubstance` through `bySubstance`), then id asc — every card of one
|
|
1642
|
+
* build shares a `builtAt`, so substance is what actually orders a feed, and
|
|
1643
|
+
* a card whose only fact restates its own headline reads after every card
|
|
1644
|
+
* with something to say. Nothing is dropped for being thin: each item carries
|
|
1645
|
+
* its own `substance` count instead. `sourcesByFactId`
|
|
966
1646
|
* maps fact ids to snapshot source links ({ title, url, name, publishedAt?
|
|
967
1647
|
* }); publishedAt is present only when the source snapshot carried one.
|
|
968
1648
|
* The gate (PLAN_NEWS_FEED.md section 17): `reportedRows` replaces `newsWindowRows`
|
|
@@ -977,19 +1657,83 @@ export function renderKnownFactsParagraph(hub, subgraphRows, { reportedIds = nul
|
|
|
977
1657
|
* nothing else. The two-hop walk reaches every row a shared class node
|
|
978
1658
|
* touches, so attributing the whole sub-graph gave one quake's card all 44 of
|
|
979
1659
|
* the day's quake headlines. An "Around it" neighbour is context the card
|
|
980
|
-
* borrows, and it carries its own citation on its own card.
|
|
981
|
-
|
|
982
|
-
|
|
1660
|
+
* borrows, and it carries its own citation on its own card.
|
|
1661
|
+
*
|
|
1662
|
+
* Between the gate and the render sit two more reads over the same reported
|
|
1663
|
+
* rows: `titledHubs` names each card after the entity its own report grounds
|
|
1664
|
+
* rather than a clause the report threw off, and `storyCoverage` leaves a
|
|
1665
|
+
* publication with only the stories no other card tells. Both are pure over
|
|
1666
|
+
* the fact set, so a feed built from the same rows in two orders still comes
|
|
1667
|
+
* back byte for byte.
|
|
1668
|
+
*
|
|
1669
|
+
* `articleEntityNames`, when the caller supplies one, reads the entity names
|
|
1670
|
+
* out of the text of the sources a card shows (news.mjs wires the services
|
|
1671
|
+
* layer's own capture, the same one the enrichment ledger admits terms by).
|
|
1672
|
+
* Those entities widen the card's background: what the graph holds about a
|
|
1673
|
+
* name inside the headline now reaches the card, where before only the
|
|
1674
|
+
* endpoints of its own facts did. They never widen its report — the walk they
|
|
1675
|
+
* seed excludes every reported row — so what a card claims a source said is
|
|
1676
|
+
* untouched. They also carry the card's sense when its hub is a phrase the
|
|
1677
|
+
* bands never place: the hub walk then keeps to the senses of the names the
|
|
1678
|
+
* article uses, where before it kept to nothing. */
|
|
1679
|
+
export function buildNewsItems(rows, {
|
|
1680
|
+
now, windowMs, limit = 6, sourcesByFactId = new Map(), readsAsEntityTerm, articleEntityNames = null,
|
|
1681
|
+
} = {}) {
|
|
1682
|
+
// Every lane below reads `claimRows`, never `rows`: the hub gate, the
|
|
1683
|
+
// adjacency index both walks share, the prior/concept/sense reads, the
|
|
1684
|
+
// sentences, and each card's own `factIds`. An attribution says something
|
|
1685
|
+
// about a row rather than about the world, and a lane handed one can score it
|
|
1686
|
+
// as a hub and head a card with a hex id.
|
|
1687
|
+
const { claims: claimRows, speakersByClaimId } = partitionAttributions(rows);
|
|
1688
|
+
const reported = reportedRows(claimRows, { now, windowMs });
|
|
983
1689
|
const reportedIds = new Set(reported.map((r) => r.id));
|
|
984
|
-
const adjacency = buildTermAdjacency(
|
|
985
|
-
const prior = priorTerms(
|
|
1690
|
+
const adjacency = buildTermAdjacency(claimRows);
|
|
1691
|
+
const prior = priorTerms(claimRows);
|
|
986
1692
|
const hubOptions = { now, windowMs, limit, adjacency, prior };
|
|
987
1693
|
if (readsAsEntityTerm) hubOptions.readsAsEntityTerm = readsAsEntityTerm;
|
|
988
|
-
const
|
|
989
|
-
const
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
1694
|
+
const rowsByTerm = reportedRowsByTerm(reported);
|
|
1695
|
+
const hubs = titledHubs(newsworthyHubs(claimRows, reported, hubOptions), claimRows, reported, rowsByTerm);
|
|
1696
|
+
const coverage = storyCoverage(hubs, rowsByTerm);
|
|
1697
|
+
const concepts = conceptTerms(claimRows);
|
|
1698
|
+
const namesEntities = readsAsEntityTerm || looksLikeEntityTerm;
|
|
1699
|
+
const senseScope = buildSenseScope(claimRows);
|
|
1700
|
+
const items = hubs.filter(({ term }) => coverage.get(term).mints).map(({ term, changed }) => {
|
|
1701
|
+
const coveredRowIds = coverage.get(term).coveredRowIds;
|
|
1702
|
+
const seeds = hubSeedTerms(term);
|
|
1703
|
+
const hubWalk = {
|
|
1704
|
+
adjacency,
|
|
1705
|
+
priorityIds: reportedIds,
|
|
1706
|
+
seedTerms: seeds,
|
|
1707
|
+
excludeIds: coveredRowIds,
|
|
1708
|
+
};
|
|
1709
|
+
// The card's sources have to be read before its sense can be chosen, and a
|
|
1710
|
+
// walk that admits no background at all is enough to read them: a report on
|
|
1711
|
+
// the hub sits on a seed, and every scope admits those. So this pass and
|
|
1712
|
+
// the real one below hand `hubReportRows` the same rows.
|
|
1713
|
+
const reportedRowsOnly = subgraphAround(claimRows, term, { ...hubWalk, inSense: () => false });
|
|
1714
|
+
const sources = collectSources(hubReportRows(term, reportedRowsOnly, { reportedIds }), sourcesByFactId);
|
|
1715
|
+
const articleTerms = articleEntityNames
|
|
1716
|
+
? cardArticleTerms(sources, articleEntityNames, { concepts, readsAsEntityTerm: namesEntities })
|
|
1717
|
+
: [];
|
|
1718
|
+
// A hub the bands never place has no sense of its own, so a scope anchored
|
|
1719
|
+
// on it refuses nothing and the walk fills the card with whatever it meets.
|
|
1720
|
+
// The entities the article names carry the card's sense instead, so its
|
|
1721
|
+
// background stays tied to its own text. When the bands place none of those
|
|
1722
|
+
// names either, the card keeps to the unplaced and reads sparse, which is
|
|
1723
|
+
// the price of not filling a card about a coined phrase with strays.
|
|
1724
|
+
const hubSense = senseScope.hasPlacedSense(seeds)
|
|
1725
|
+
? senseScope.sameSenseAs(seeds)
|
|
1726
|
+
: senseScope.sameSenseAs([...seeds, ...articleTerms], { admitAllWhenUnplaced: false });
|
|
1727
|
+
const hubRows = subgraphAround(claimRows, term, { ...hubWalk, inSense: hubSense });
|
|
1728
|
+
const heldIds = new Set(hubRows.map((r) => r.id));
|
|
1729
|
+
const articleRows = articleTerms.length
|
|
1730
|
+
? articleEntityRows(claimRows, articleTerms, {
|
|
1731
|
+
adjacency,
|
|
1732
|
+
excludeIds: new Set([...coveredRowIds, ...reportedIds]),
|
|
1733
|
+
inSense: senseScope.sameSenseAs(articleTerms),
|
|
1734
|
+
}).filter((r) => !heldIds.has(r.id))
|
|
1735
|
+
: [];
|
|
1736
|
+
const subgraphRows = articleRows.length ? [...hubRows, ...articleRows] : hubRows;
|
|
993
1737
|
const factIds = subgraphRows.map((r) => r.id).sort();
|
|
994
1738
|
const { background } = splitCardRows(subgraphRows, reportedIds);
|
|
995
1739
|
return {
|
|
@@ -997,15 +1741,18 @@ export function buildNewsItems(rows, { now, windowMs, limit = 6, sourcesByFactId
|
|
|
997
1741
|
hub: term,
|
|
998
1742
|
factIds,
|
|
999
1743
|
changedCount: changed,
|
|
1744
|
+
substance: cardSubstance(term, subgraphRows, {
|
|
1745
|
+
reportedIds, articleTerms, headlines: sources.map((s) => s.title),
|
|
1746
|
+
}),
|
|
1000
1747
|
builtAt: now,
|
|
1001
|
-
paragraph: renderNewsParagraph(term, subgraphRows, { reportedIds }),
|
|
1748
|
+
paragraph: renderNewsParagraph(term, subgraphRows, { reportedIds, articleTerms, speakersByClaimId }),
|
|
1002
1749
|
tier: tierOf(subgraphRows),
|
|
1003
|
-
sources
|
|
1750
|
+
sources,
|
|
1004
1751
|
background: background.map((r) => r.id).sort(),
|
|
1005
|
-
backgroundParagraph: renderKnownFactsParagraph(term, subgraphRows, { reportedIds }),
|
|
1752
|
+
backgroundParagraph: renderKnownFactsParagraph(term, subgraphRows, { reportedIds, articleTerms, speakersByClaimId }),
|
|
1006
1753
|
};
|
|
1007
1754
|
});
|
|
1008
|
-
return items.sort((a, b) => (toMs(b.builtAt) - toMs(a.builtAt)) || byId(a, b));
|
|
1755
|
+
return items.sort((a, b) => (toMs(b.builtAt) - toMs(a.builtAt)) || bySubstance(a, b) || byId(a, b));
|
|
1009
1756
|
}
|
|
1010
1757
|
|
|
1011
1758
|
// A `news:` tag, matched wherever it sits in a fact's provenance — bare
|
|
@@ -1018,19 +1765,42 @@ export function buildNewsItems(rows, { now, windowMs, limit = 6, sourcesByFactId
|
|
|
1018
1765
|
// is the deliberate exclusion the fixture-replay rows need.
|
|
1019
1766
|
const NEWS_PROVENANCE_RE = /(?:^|[:|]\s*)news:/;
|
|
1020
1767
|
|
|
1021
|
-
/** News-tagged fact ids to retract, oldest
|
|
1022
|
-
*
|
|
1023
|
-
*
|
|
1024
|
-
*
|
|
1768
|
+
/** News-tagged fact ids to retract, oldest observation first, ties by id — the
|
|
1769
|
+
* eviction the service applies at ingest time so the graph cannot grow past
|
|
1770
|
+
* `cap` unattended. Never selects a seed/taught/research/fixture-replay row.
|
|
1771
|
+
*
|
|
1772
|
+
* A claim and the attributions naming it evict as ONE unit. They carry the same
|
|
1773
|
+
* news tag and the same stamp but not the same id, so choosing row by row
|
|
1774
|
+
* routinely kept one half and dropped the other, leaving a speaker with no
|
|
1775
|
+
* claim or a claim whose surface can no longer say who said it. A unit that
|
|
1776
|
+
* straddles the cap goes whole: the graph lands under `cap`, never on half a
|
|
1777
|
+
* pair.
|
|
1778
|
+
*
|
|
1779
|
+
* The stamp comes from `rowObservedMs`, which is where a read row actually
|
|
1780
|
+
* carries it — `readFactRows` keeps observedAt on the assertion records, so
|
|
1781
|
+
* reading `row.observedAt` scored every real news row 0 and left the cap
|
|
1782
|
+
* evicting by id order. */
|
|
1025
1783
|
export function evictNewsFacts(rows, { cap }) {
|
|
1026
1784
|
const newsRows = rows.filter((r) => NEWS_PROVENANCE_RE.test(String(r.provenance || "")));
|
|
1027
1785
|
if (newsRows.length <= cap) return [];
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
const
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1786
|
+
|
|
1787
|
+
const units = new Map();
|
|
1788
|
+
for (const row of newsRows) {
|
|
1789
|
+
const key = referencedFactId(row) || row.id;
|
|
1790
|
+
let unit = units.get(key);
|
|
1791
|
+
if (!unit) units.set(key, (unit = { key, ids: [], observedMs: Infinity }));
|
|
1792
|
+
unit.ids.push(row.id);
|
|
1793
|
+
const t = rowObservedMs(row);
|
|
1794
|
+
unit.observedMs = Math.min(unit.observedMs, Number.isFinite(t) ? t : 0);
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
const target = newsRows.length - cap;
|
|
1798
|
+
const evicted = [];
|
|
1799
|
+
const oldestFirst = [...units.values()]
|
|
1800
|
+
.sort((a, b) => a.observedMs - b.observedMs || (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
|
|
1801
|
+
for (const unit of oldestFirst) {
|
|
1802
|
+
if (evicted.length >= target) break;
|
|
1803
|
+
evicted.push(...unit.ids.slice().sort());
|
|
1804
|
+
}
|
|
1805
|
+
return evicted;
|
|
1036
1806
|
}
|