@polycode-projects/the-mechanical-code-talker 6.0.17 → 6.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -1
- package/src/adapters/corpus/research-source.mjs +6 -2
- package/src/adapters/corpus/wikidata-live.mjs +92 -51
- package/src/adapters/memory/core.mjs +53 -5
- package/src/adapters/memory/rows.mjs +253 -21
- package/src/domain/news-feed.mjs +790 -75
- package/src/domain/sense-gate.mjs +220 -0
- package/src/domain/syllogise.mjs +39 -8
- package/src/domain/term-ledger.mjs +16 -1
- package/src/services/chat.mjs +17 -12
- package/src/services/extract-facts.mjs +284 -19
- package/src/services/news-viz.mjs +31 -0
- package/src/services/news.mjs +67 -14
- package/src/surfaces/web/memory-ask-browser.bundle.js +142 -142
package/src/domain/news-feed.mjs
CHANGED
|
@@ -150,16 +150,24 @@ const hasNonEmptyArray = (value) => Array.isArray(value) && value.length > 0;
|
|
|
150
150
|
// after it.
|
|
151
151
|
const provenanceHead = (provenance) => String(provenance || "").trim().split(/\s+/)[0] || "";
|
|
152
152
|
|
|
153
|
+
/** True when the graph reasoned this row out for itself rather than reading it
|
|
154
|
+
* somewhere — an entailment head, an environment, or a justification chain.
|
|
155
|
+
* A card never speaks from one: the subClassOf closure gives a common noun
|
|
156
|
+
* every parent class of every sense it has ("earthquake is a kind of
|
|
157
|
+
* cognition, a tentacle, a christians"), which is sound as inference and
|
|
158
|
+
* useless as a sentence about today's quake. */
|
|
159
|
+
export function isDerivedRow(row) {
|
|
160
|
+
return provenanceHead(row?.provenance).startsWith("entailed:")
|
|
161
|
+
|| hasNonEmptyArray(row?.environments) || hasNonEmptyArray(row?.justification);
|
|
162
|
+
}
|
|
163
|
+
|
|
153
164
|
/** "derived" | "background" | "reported" for one row, pure over the row plus
|
|
154
165
|
* `now` (PLAN_NEWS_FEED.md section 17.3, step one). Rules apply in order,
|
|
155
166
|
* first hit wins: a syllogised row is derived; an identity, universal or
|
|
156
167
|
* non-news-provenance row is background; a news/news-fixture row with no
|
|
157
168
|
* readable or in-window stamp is background; everything else is reported. */
|
|
158
169
|
export function classifyNewsRow(row, { now, windowMs }) {
|
|
159
|
-
if (
|
|
160
|
-
|| hasNonEmptyArray(row.environments) || hasNonEmptyArray(row.justification)) {
|
|
161
|
-
return "derived";
|
|
162
|
-
}
|
|
170
|
+
if (isDerivedRow(row)) return "derived";
|
|
163
171
|
if (GATE_IDENTITY_PREDICATES.has(row.predicate)) return "background";
|
|
164
172
|
if (UNIVERSAL_QUANTIFIERS.has(String(row.quantifier || "").toLowerCase())) return "background";
|
|
165
173
|
if (!REPORT_PROVENANCE_RE.test(String(row.provenance || ""))) return "background";
|
|
@@ -485,6 +493,26 @@ export function buildTermAdjacency(rows) {
|
|
|
485
493
|
return { byTerm, terms };
|
|
486
494
|
}
|
|
487
495
|
|
|
496
|
+
// A source names a place by settlement and region at once — "mina, nevada",
|
|
497
|
+
// "pedro bay, alaska", "san juan, puerto rico". The region is the trailing
|
|
498
|
+
// part, and it is the half the graph plausibly already holds facts about,
|
|
499
|
+
// while the joined term is new. So the walk seeds from the region as well as
|
|
500
|
+
// the whole name. The leading part stays out: "mina" the town and "mina" the
|
|
501
|
+
// myna bird are one string to a graph keyed on words, and the settlement half
|
|
502
|
+
// is exactly where that collision lands. Comma-separated only — splitting on
|
|
503
|
+
// spaces would turn "public investments fund" into three terms that name
|
|
504
|
+
// nothing.
|
|
505
|
+
export function hubSeedTerms(hub) {
|
|
506
|
+
const whole = normFactTerm(hub);
|
|
507
|
+
const seeds = [whole];
|
|
508
|
+
if (!whole.includes(",")) return seeds;
|
|
509
|
+
const region = normFactTerm(whole.split(",").pop());
|
|
510
|
+
if (!region || region === whole) return seeds;
|
|
511
|
+
if (STOP_SET.has(region) || isQuantityTerm(region) || !looksLikeEntityTerm(region)) return seeds;
|
|
512
|
+
seeds.push(region);
|
|
513
|
+
return seeds;
|
|
514
|
+
}
|
|
515
|
+
|
|
488
516
|
/** Breadth-first over subject/object adjacency from `hub`, exactly `hops`
|
|
489
517
|
* levels deep, then capped: a `priorityIds` row first, then the nearer hop,
|
|
490
518
|
* then content-addressed id. The cap never depends on `rows`' own order, only
|
|
@@ -494,19 +522,33 @@ export function buildTermAdjacency(rows) {
|
|
|
494
522
|
* `priorityIds` is what keeps a card about a term the graph already knows
|
|
495
523
|
* thousands of things about from being built out of an arbitrary slice of
|
|
496
524
|
* them: a hub like "france" reaches far more rows than the cap, and the one
|
|
497
|
-
* report that made it news would otherwise be the row that fell out.
|
|
498
|
-
|
|
525
|
+
* report that made it news would otherwise be the row that fell out.
|
|
526
|
+
*
|
|
527
|
+
* `seedTerms` starts the walk from more than the hub itself (hubSeedTerms).
|
|
528
|
+
* Everything downstream that asks "is this the hub" — the report sentences,
|
|
529
|
+
* the sources, the neighbourhood — still reads the hub term alone, so a seed
|
|
530
|
+
* widens only what the card can draw background from.
|
|
531
|
+
*
|
|
532
|
+
* `excludeIds` drops rows before the cap rather than after it, so a card that
|
|
533
|
+
* gives a report away to another card (storyCoverage) spends the freed budget
|
|
534
|
+
* on rows it will actually show. */
|
|
535
|
+
export function subgraphAround(rows, hub, {
|
|
536
|
+
hops = NEWS_HUB_HOPS, cap = 60, adjacency = null, priorityIds = null, seedTerms = null, excludeIds = null,
|
|
537
|
+
} = {}) {
|
|
499
538
|
const adj = adjacency ?? buildTermAdjacency(rows);
|
|
500
539
|
const hubTerm = normFactTerm(hub);
|
|
501
|
-
const
|
|
502
|
-
|
|
540
|
+
const seeds = (seedTerms?.length ? seedTerms : [hubTerm]).map((t) => normFactTerm(t)).filter(Boolean);
|
|
541
|
+
const visited = new Set(seeds);
|
|
542
|
+
let frontier = [...new Set(seeds)].sort();
|
|
503
543
|
const collected = new Map();
|
|
504
544
|
const hopOf = new Map();
|
|
545
|
+
const isExcluded = excludeIds instanceof Set ? (id) => excludeIds.has(id) : () => false;
|
|
505
546
|
for (let hop = 0; hop < hops; hop += 1) {
|
|
506
547
|
const nextFrontier = new Set();
|
|
507
548
|
for (const term of [...frontier].sort()) {
|
|
508
549
|
for (const idx of adj.byTerm.get(term) ?? []) {
|
|
509
550
|
const row = rows[idx];
|
|
551
|
+
if (isExcluded(row.id)) continue;
|
|
510
552
|
collected.set(row.id, row);
|
|
511
553
|
if (!hopOf.has(row.id)) hopOf.set(row.id, hop);
|
|
512
554
|
const [s, o] = adj.terms[idx];
|
|
@@ -525,6 +567,32 @@ export function subgraphAround(rows, hub, { hops = NEWS_HUB_HOPS, cap = 60, adja
|
|
|
525
567
|
.slice(0, cap);
|
|
526
568
|
}
|
|
527
569
|
|
|
570
|
+
// How far the walk goes from an entity the ARTICLE names rather than a fact,
|
|
571
|
+
// and how many rows it may bring back. One hop: what the graph says about that
|
|
572
|
+
// entity itself, never what it says about everything that entity touches. The
|
|
573
|
+
// hub's own two-hop walk is unchanged; this one runs beside it.
|
|
574
|
+
const ARTICLE_ENTITY_HOPS = 1;
|
|
575
|
+
const ARTICLE_ENTITY_ROW_CAP = 24;
|
|
576
|
+
|
|
577
|
+
/** The rows sitting one hop from an entity the card's article names. Seeded
|
|
578
|
+
* from `terms` rather than the hub, so a definition the graph holds about a
|
|
579
|
+
* name inside the headline reaches the card even when no fact of the card's
|
|
580
|
+
* own touches that name — "amigados is a disk operating system" beside a
|
|
581
|
+
* report whose only fact is that a site discussed the headline.
|
|
582
|
+
*
|
|
583
|
+
* `excludeIds` carries the card's reported rows as well as the ones another
|
|
584
|
+
* card claimed, so this walk returns background and nothing else: what a
|
|
585
|
+
* source reported is the hub walk's business. */
|
|
586
|
+
export function articleEntityRows(rows, terms, {
|
|
587
|
+
adjacency = null, excludeIds = null, cap = ARTICLE_ENTITY_ROW_CAP,
|
|
588
|
+
} = {}) {
|
|
589
|
+
const seedTerms = (terms || []).map((term) => normFactTerm(term)).filter(Boolean);
|
|
590
|
+
if (!seedTerms.length) return [];
|
|
591
|
+
return subgraphAround(rows, seedTerms[0], {
|
|
592
|
+
hops: ARTICLE_ENTITY_HOPS, cap, adjacency, seedTerms, excludeIds,
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
|
|
528
596
|
/** The strongest prior kind among `rows`, for the item's trust chip — read
|
|
529
597
|
* off each row's own `trust` (a number) and `sourceTypes` (the kind array
|
|
530
598
|
* readFactRows already computes), never a re-derivation of SOURCE_PRIOR. */
|
|
@@ -538,20 +606,37 @@ function tierOf(rows) {
|
|
|
538
606
|
return kinds[0] || "";
|
|
539
607
|
}
|
|
540
608
|
|
|
541
|
-
|
|
609
|
+
// How much of an item's own summary a card carries. A wire description runs
|
|
610
|
+
// about a line; an encyclopaedia extract runs several paragraphs, and a feed
|
|
611
|
+
// of fifty cards has a byte budget to keep (MAX_FEED_DOCUMENT_BYTES). Cut at
|
|
612
|
+
// the last word boundary inside the bound so the quote ends on a word.
|
|
613
|
+
const SOURCE_SUMMARY_MAX_CHARS = 400;
|
|
614
|
+
|
|
615
|
+
function clampSummary(summary) {
|
|
616
|
+
const text = String(summary || "").trim();
|
|
617
|
+
if (text.length <= SOURCE_SUMMARY_MAX_CHARS) return text;
|
|
618
|
+
const cut = text.slice(0, SOURCE_SUMMARY_MAX_CHARS);
|
|
619
|
+
const lastSpace = cut.lastIndexOf(" ");
|
|
620
|
+
return `${(lastSpace > 0 ? cut.slice(0, lastSpace) : cut).trimEnd()}…`;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
function collectSources(citedRows, sourcesByFactId) {
|
|
542
624
|
const get = (id) => (sourcesByFactId instanceof Map ? sourcesByFactId.get(id) : sourcesByFactId?.[id]);
|
|
543
625
|
const seen = new Set();
|
|
544
626
|
const sources = [];
|
|
545
|
-
for (const row of
|
|
627
|
+
for (const row of citedRows) {
|
|
546
628
|
const src = get(row.id);
|
|
547
629
|
if (!src) continue;
|
|
548
630
|
const key = src.url || src.title || "";
|
|
549
631
|
if (!key || seen.has(key)) continue;
|
|
550
632
|
seen.add(key);
|
|
551
633
|
const entry = { title: src.title || "", url: src.url || "", name: src.name || "" };
|
|
552
|
-
//
|
|
553
|
-
// card for an undated snapshot shows
|
|
634
|
+
// Both of these ride along only when the snapshot actually has one — a
|
|
635
|
+
// card for an undated or summary-less snapshot shows nothing there rather
|
|
636
|
+
// than a blank field.
|
|
554
637
|
if (src.publishedAt) entry.publishedAt = src.publishedAt;
|
|
638
|
+
const summary = clampSummary(src.summary);
|
|
639
|
+
if (summary) entry.summary = summary;
|
|
555
640
|
sources.push(entry);
|
|
556
641
|
}
|
|
557
642
|
return sources;
|
|
@@ -563,14 +648,245 @@ function joinWithAnd(items) {
|
|
|
563
648
|
return `${items.slice(0, -1).join(", ")} and ${items[items.length - 1]}`;
|
|
564
649
|
}
|
|
565
650
|
|
|
651
|
+
// ---------------------------------------------------------------------------
|
|
652
|
+
// What one card reports, and whose neighbourhood it sits in.
|
|
653
|
+
// ---------------------------------------------------------------------------
|
|
654
|
+
|
|
655
|
+
/** An id membership test over a Set, an array, or null — null meaning "every
|
|
656
|
+
* row counts", which is what a caller with no reported-row set of its own
|
|
657
|
+
* (the background paragraph, a direct render call) needs. */
|
|
658
|
+
function idMembership(ids) {
|
|
659
|
+
if (ids === null || ids === undefined) return () => true;
|
|
660
|
+
if (ids instanceof Set) return (id) => ids.has(id);
|
|
661
|
+
return (id) => ids.includes(id);
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
const rowObservedRank = (row) => {
|
|
665
|
+
const t = rowObservedMs(row);
|
|
666
|
+
return Number.isFinite(t) ? t : 0;
|
|
667
|
+
};
|
|
668
|
+
|
|
669
|
+
/** The reported rows of `subgraphRows` that touch `hub` directly — what this
|
|
670
|
+
* card actually reports, and so what it may attribute a source to. A row
|
|
671
|
+
* further out in the two-hop walk was reached through a term the hub merely
|
|
672
|
+
* shares; it belongs to some other card's report. */
|
|
673
|
+
export function hubReportRows(hub, subgraphRows, { reportedIds = null } = {}) {
|
|
674
|
+
const hubTerm = normFactTerm(hub);
|
|
675
|
+
const isReported = idMembership(reportedIds);
|
|
676
|
+
return subgraphRows.filter((row) => isReported(row.id)
|
|
677
|
+
&& (normFactTerm(row.subject) === hubTerm || normFactTerm(row.object) === hubTerm));
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
/** Every term sitting one edge from `hub` inside this card's own sub-graph. */
|
|
681
|
+
function hubLinkTerms(subgraphRows, hubTerm) {
|
|
682
|
+
const terms = new Set();
|
|
683
|
+
for (const row of subgraphRows) {
|
|
684
|
+
const s = normFactTerm(row.subject);
|
|
685
|
+
const o = normFactTerm(row.object);
|
|
686
|
+
if (s === hubTerm && o) terms.add(o);
|
|
687
|
+
else if (o === hubTerm && s) terms.add(s);
|
|
688
|
+
}
|
|
689
|
+
terms.delete(hubTerm);
|
|
690
|
+
return terms;
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
const NEIGHBOUR_ROW_LIMIT = 3;
|
|
694
|
+
|
|
695
|
+
/** This hub's own neighbourhood: the reported rows one further edge out,
|
|
696
|
+
* reached only through a link term specific enough to name a neighbourhood.
|
|
697
|
+
* A link term the clause cannot name in full — one reaching more rows than
|
|
698
|
+
* the sentence prints — is a category node, not a neighbour: "earthquake"
|
|
699
|
+
* sits between all 44 quakes of a day, so every quake card walked through it
|
|
700
|
+
* and printed the same arbitrary three. Naming a slice of a category is what
|
|
701
|
+
* made sibling cards identical, so a term over the clause's own capacity
|
|
702
|
+
* contributes nothing and a card with no specific link prints no "Around it"
|
|
703
|
+
* at all.
|
|
704
|
+
*
|
|
705
|
+
* Survivors rank by a predicate the hub's own report also used, then by
|
|
706
|
+
* observation time, then by id — this hub's choice, and a pure function of
|
|
707
|
+
* the fact set either way. */
|
|
708
|
+
export function neighbourRows(hub, subgraphRows, { reportedIds = null, limit = NEIGHBOUR_ROW_LIMIT } = {}) {
|
|
709
|
+
const hubTerm = normFactTerm(hub);
|
|
710
|
+
const isReported = idMembership(reportedIds);
|
|
711
|
+
const linkTerms = hubLinkTerms(subgraphRows, hubTerm);
|
|
712
|
+
|
|
713
|
+
const reachedByLinkTerm = new Map();
|
|
714
|
+
for (const row of subgraphRows) {
|
|
715
|
+
const s = normFactTerm(row.subject);
|
|
716
|
+
const o = normFactTerm(row.object);
|
|
717
|
+
if (s === hubTerm || o === hubTerm || !isReported(row.id)) continue;
|
|
718
|
+
for (const term of new Set([s, o])) {
|
|
719
|
+
if (!term || !linkTerms.has(term)) continue;
|
|
720
|
+
let reached = reachedByLinkTerm.get(term);
|
|
721
|
+
if (!reached) reachedByLinkTerm.set(term, (reached = []));
|
|
722
|
+
reached.push(row);
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
const candidates = new Map();
|
|
727
|
+
for (const [, reached] of reachedByLinkTerm) {
|
|
728
|
+
if (reached.length > limit) continue;
|
|
729
|
+
for (const row of reached) candidates.set(row.id, row);
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
const hubPredicates = new Set(hubReportRows(hub, subgraphRows, { reportedIds }).map((r) => r.predicate));
|
|
733
|
+
return [...candidates.values()]
|
|
734
|
+
.sort((a, b) => (Number(hubPredicates.has(b.predicate)) - Number(hubPredicates.has(a.predicate)))
|
|
735
|
+
|| (rowObservedRank(b) - rowObservedRank(a))
|
|
736
|
+
|| byId(a, b))
|
|
737
|
+
.slice(0, limit);
|
|
738
|
+
}
|
|
739
|
+
|
|
566
740
|
const IDENTITY_PREDICATES = new Set(["rdf:type", "rdfs:subClassOf"]);
|
|
567
|
-
const SENTENCE_CAP =
|
|
741
|
+
const SENTENCE_CAP = 6;
|
|
742
|
+
// The four blocks a card's paragraph is built from, in the order they read:
|
|
743
|
+
// what was reported, then what the thing is, then what the graph already knew
|
|
744
|
+
// about it, then its neighbourhood. Their caps add up past SENTENCE_CAP on
|
|
745
|
+
// purpose — the paragraph fills from the front, so a news-rich card spends its
|
|
746
|
+
// budget on the news and a quiet one spends it on background.
|
|
747
|
+
const REPORT_SENTENCE_CAP = 4;
|
|
748
|
+
const IDENTITY_SENTENCE_CAP = 1;
|
|
749
|
+
const KNOWN_FACT_SENTENCE_CAP = 2;
|
|
568
750
|
// How many objects one sentence names before it counts the rest. A live source
|
|
569
751
|
// reports the same relation over and over inside one window — every quake of
|
|
570
752
|
// the day strikes near somewhere — and an unbounded list turns a card into a
|
|
571
753
|
// wall of text.
|
|
572
754
|
const OBJECTS_PER_SENTENCE = 6;
|
|
573
755
|
|
|
756
|
+
// A term the graph says is more than this many things is read across senses:
|
|
757
|
+
// "earthquake" is a natural event, a cognition, a social station and nine more,
|
|
758
|
+
// so no single class line about it is trustworthy on a card about one quake.
|
|
759
|
+
// Same discipline as the neighbourhood's own category-node test — a node too
|
|
760
|
+
// wide for one clause to name in full says nothing — applied to senses instead
|
|
761
|
+
// of edges.
|
|
762
|
+
const IDENTITY_MAX_CLASSES = 2;
|
|
763
|
+
// A background line's far side, when this many terms in the sub-graph already
|
|
764
|
+
// fall under it, names a category rather than anything about this card:
|
|
765
|
+
// "france is related to place" is true of most of the graph. The hub's own
|
|
766
|
+
// identity clause is exempt — a crowded class is still this thing's own kind,
|
|
767
|
+
// and "france is a country" is the most useful line a card can carry.
|
|
768
|
+
const CATEGORY_FAN_MAX = 3;
|
|
769
|
+
// How many background rows the "what the graph already knew" disclosure names.
|
|
770
|
+
// The paragraph itself shows the first KNOWN_FACT_SENTENCE_CAP groups of these.
|
|
771
|
+
const KNOWN_FACT_ROW_LIMIT = 6;
|
|
772
|
+
|
|
773
|
+
/** Three fan-out readings of the identity rows inside one card's sub-graph.
|
|
774
|
+
* `senseFan` counts every class a term is said to BE, the entailment closure
|
|
775
|
+
* included — high means the graph reads the term across senses, which is what
|
|
776
|
+
* makes a common noun a poor subject for a card about one event.
|
|
777
|
+
* `sourcedSenseFan` counts only the classes something actually stated, which
|
|
778
|
+
* is what a printed "X is a Y" clause may draw on. `categoryFan` counts the
|
|
779
|
+
* distinct terms said to fall UNDER a term — high means a category node, the
|
|
780
|
+
* same reading the neighbourhood's own link-term test makes. All three are
|
|
781
|
+
* pure counts over the rows handed in, so a card's own sub-graph decides and
|
|
782
|
+
* no global blocklist is involved. */
|
|
783
|
+
function identityFans(subgraphRows) {
|
|
784
|
+
const senseFan = new Map();
|
|
785
|
+
const sourcedSenseFan = new Map();
|
|
786
|
+
const membersByClass = new Map();
|
|
787
|
+
for (const row of subgraphRows) {
|
|
788
|
+
if (!IDENTITY_PREDICATES.has(row.predicate)) continue;
|
|
789
|
+
const s = normFactTerm(row.subject);
|
|
790
|
+
const o = normFactTerm(row.object);
|
|
791
|
+
if (!s || !o) continue;
|
|
792
|
+
senseFan.set(s, (senseFan.get(s) || 0) + 1);
|
|
793
|
+
if (!isDerivedRow(row)) sourcedSenseFan.set(s, (sourcedSenseFan.get(s) || 0) + 1);
|
|
794
|
+
let members = membersByClass.get(o);
|
|
795
|
+
if (!members) membersByClass.set(o, (members = new Set()));
|
|
796
|
+
members.add(s);
|
|
797
|
+
}
|
|
798
|
+
const categoryFan = new Map();
|
|
799
|
+
for (const [term, members] of membersByClass) categoryFan.set(term, members.size);
|
|
800
|
+
return { senseFan, sourcedSenseFan, categoryFan };
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
// Where a background row's anchor term came from, and so which rows the
|
|
804
|
+
// disclosure names first: the card's own hub, then a term its report names,
|
|
805
|
+
// then an entity its article names that no fact of the card's own touches.
|
|
806
|
+
const ANCHOR_RANK_HUB = 0;
|
|
807
|
+
const ANCHOR_RANK_REPORT = 1;
|
|
808
|
+
const ANCHOR_RANK_ARTICLE = 2;
|
|
809
|
+
|
|
810
|
+
/** What this card is ABOUT, each term mapped to its anchor rank: its hub, the
|
|
811
|
+
* region of a hub the source spelled "settlement, region", the other terms its
|
|
812
|
+
* own report sentences name, and the entities its article names (`articleTerms`)
|
|
813
|
+
* — minus any of those that reads across senses. A quake card's report
|
|
814
|
+
* names "earthquake" and a place; the class term is where the graph's
|
|
815
|
+
* knowledge is thinnest and its senses widest, so background drawn through it
|
|
816
|
+
* is background about earthquakes in general, never about this quake. The hub
|
|
817
|
+
* itself is always in, whatever its sense count — the card is about it.
|
|
818
|
+
*
|
|
819
|
+
* An article entity earns the same reading a report term gets, one rank
|
|
820
|
+
* behind it: the story's own words name it, but no fact the card reports
|
|
821
|
+
* does. */
|
|
822
|
+
function cardSubjectTerms(hub, subgraphRows, { reportedIds = null, senseFan, articleTerms = [] }) {
|
|
823
|
+
const hubTerm = normFactTerm(hub);
|
|
824
|
+
const isReported = idMembership(reportedIds);
|
|
825
|
+
const terms = new Map();
|
|
826
|
+
for (const seed of hubSeedTerms(hubTerm)) terms.set(seed, ANCHOR_RANK_HUB);
|
|
827
|
+
const admit = (raw, rank) => {
|
|
828
|
+
const term = normFactTerm(raw);
|
|
829
|
+
if (!term || terms.has(term) || STOP_SET.has(term)) return;
|
|
830
|
+
if ((senseFan.get(term) || 0) > IDENTITY_MAX_CLASSES) return;
|
|
831
|
+
terms.set(term, rank);
|
|
832
|
+
};
|
|
833
|
+
for (const row of hubReportRows(hubTerm, subgraphRows, { reportedIds })) {
|
|
834
|
+
if (!isReported(row.id)) continue;
|
|
835
|
+
for (const raw of [row.subject, row.object]) admit(raw, ANCHOR_RANK_REPORT);
|
|
836
|
+
}
|
|
837
|
+
for (const raw of articleTerms) admit(raw, ANCHOR_RANK_ARTICLE);
|
|
838
|
+
return terms;
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
/** The background rows worth telling a reader about, ranked: a row touching
|
|
842
|
+
* one of this card's own subject terms, whose other side is not a category
|
|
843
|
+
* node, and — for an identity row — whose subject is not read across senses.
|
|
844
|
+
* Ranks the hub's own rows first, then the ones its report names, then the
|
|
845
|
+
* ones an entity in its article names, then by how specific the other side
|
|
846
|
+
* is, then by content-addressed id, so the same fact set always yields the
|
|
847
|
+
* same lines in the same order. */
|
|
848
|
+
export function knownFactRows(hub, subgraphRows, {
|
|
849
|
+
reportedIds = null, limit = KNOWN_FACT_ROW_LIMIT, articleTerms = [],
|
|
850
|
+
} = {}) {
|
|
851
|
+
const hubTerm = normFactTerm(hub);
|
|
852
|
+
const isReported = idMembership(reportedIds);
|
|
853
|
+
const { senseFan, sourcedSenseFan, categoryFan } = identityFans(subgraphRows);
|
|
854
|
+
const subjects = cardSubjectTerms(hub, subgraphRows, { reportedIds, senseFan, articleTerms });
|
|
855
|
+
const neighbourIds = new Set(neighbourRows(hub, subgraphRows, { reportedIds }).map((r) => r.id));
|
|
856
|
+
|
|
857
|
+
const scored = [];
|
|
858
|
+
for (const row of subgraphRows) {
|
|
859
|
+
if (isReported(row.id) || neighbourIds.has(row.id) || isDerivedRow(row)) continue;
|
|
860
|
+
const s = normFactTerm(row.subject);
|
|
861
|
+
const o = normFactTerm(row.object);
|
|
862
|
+
if (!s || !o || s === o) continue;
|
|
863
|
+
const anchorIsSubject = subjects.has(s);
|
|
864
|
+
if (!anchorIsSubject && !subjects.has(o)) continue;
|
|
865
|
+
const anchor = anchorIsSubject ? s : o;
|
|
866
|
+
const other = anchorIsSubject ? o : s;
|
|
867
|
+
if (IDENTITY_PREDICATES.has(row.predicate)) {
|
|
868
|
+
// What the hub itself IS belongs to the identity clause and is said once.
|
|
869
|
+
if (s === hubTerm) continue;
|
|
870
|
+
if ((sourcedSenseFan.get(s) || 0) > IDENTITY_MAX_CLASSES) continue;
|
|
871
|
+
}
|
|
872
|
+
if ((categoryFan.get(other) || 0) > CATEGORY_FAN_MAX) continue;
|
|
873
|
+
scored.push({
|
|
874
|
+
row,
|
|
875
|
+
anchorRank: subjects.get(anchor),
|
|
876
|
+
otherCategoryFan: categoryFan.get(other) || 0,
|
|
877
|
+
otherSenseFan: senseFan.get(other) || 0,
|
|
878
|
+
});
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
return scored
|
|
882
|
+
.sort((a, b) => (a.anchorRank - b.anchorRank)
|
|
883
|
+
|| (a.otherCategoryFan - b.otherCategoryFan)
|
|
884
|
+
|| (a.otherSenseFan - b.otherSenseFan)
|
|
885
|
+
|| byId(a.row, b.row))
|
|
886
|
+
.slice(0, limit)
|
|
887
|
+
.map((entry) => entry.row);
|
|
888
|
+
}
|
|
889
|
+
|
|
574
890
|
function joinObjects(objects) {
|
|
575
891
|
if (objects.length <= OBJECTS_PER_SENTENCE) return joinWithAnd(objects);
|
|
576
892
|
const shown = objects.slice(0, OBJECTS_PER_SENTENCE);
|
|
@@ -588,75 +904,427 @@ function predicatesInRenderOrder(rows) {
|
|
|
588
904
|
return [...curated, ...rest];
|
|
589
905
|
}
|
|
590
906
|
|
|
591
|
-
/**
|
|
592
|
-
*
|
|
593
|
-
*
|
|
594
|
-
*
|
|
595
|
-
*
|
|
596
|
-
*
|
|
597
|
-
*
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
907
|
+
/** One sentence per (subject, predicate) group over `rows`, in the order the
|
|
908
|
+
* rows arrive — the same shape the hub's own relation sentences take, so a
|
|
909
|
+
* background line reads like the rest of the paragraph rather than a dump.
|
|
910
|
+
* Each entry carries the group's own rows alongside its text, so a caller
|
|
911
|
+
* that needs to know which facts a sentence came from (the bench's noisy-
|
|
912
|
+
* line scoring) reads them off the same grouping the sentence itself used,
|
|
913
|
+
* never a second derivation of it. */
|
|
914
|
+
function groupedFactSentenceEntries(rows) {
|
|
915
|
+
const groups = new Map();
|
|
916
|
+
for (const row of rows) {
|
|
917
|
+
const key = `${row.subject}${row.predicate}`;
|
|
918
|
+
let group = groups.get(key);
|
|
919
|
+
if (!group) groups.set(key, (group = { subject: row.subject, predicate: row.predicate, objects: [], rows: [] }));
|
|
920
|
+
group.objects.push(row.object);
|
|
921
|
+
group.rows.push(row);
|
|
922
|
+
}
|
|
923
|
+
return [...groups.values()].map(({ subject, predicate, objects, rows: groupRows }) => {
|
|
924
|
+
const sorted = [...objects].sort();
|
|
925
|
+
// "rdf:type" reads "is a" from the curated phrase table, which is the wrong
|
|
926
|
+
// article before a vowel and a second one in front of an object articleFor
|
|
927
|
+
// has already spelled. The identity clause the paragraph opens with says a
|
|
928
|
+
// bare "is" and lets articleFor choose; a background line says it the same
|
|
929
|
+
// way. Every other predicate, "is a kind of" included, carries whatever
|
|
930
|
+
// article it needs inside its own phrase and takes the object bare.
|
|
931
|
+
const text = predicate === "rdf:type"
|
|
932
|
+
? `${subject} is ${joinObjects(sorted.map((object) => `${articleFor(object)} ${object}`))}`
|
|
933
|
+
: `${subject} ${predicatePhrase(predicate, subject)} ${joinObjects(sorted)}`;
|
|
934
|
+
return { text, rows: groupRows };
|
|
935
|
+
});
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
function groupedFactSentences(rows) {
|
|
939
|
+
return groupedFactSentenceEntries(rows).map((entry) => entry.text);
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
/** The sentences a card's paragraph is made of, as four ordered blocks: the
|
|
943
|
+
* report (what a source said inside the window), the identity clause, the
|
|
944
|
+
* related facts the graph already held, and the neighbourhood. Each entry is
|
|
945
|
+
* `{ text, rows }` — the rendered sentence and the fact row(s) it came from
|
|
946
|
+
* — so a caller that needs to know which facts actually reached the printed
|
|
947
|
+
* text (the bench's noisy-line scoring) reads them off the same blocks
|
|
948
|
+
* `renderNewsParagraph` itself slices, never a second derivation of it.
|
|
949
|
+
* Callers that render only one block — the "what the graph already knew"
|
|
950
|
+
* disclosure — read the block they want instead of re-deriving it. */
|
|
951
|
+
function paragraphBlocks(hub, subgraphRows, { reportedIds = null, articleTerms = [] } = {}) {
|
|
604
952
|
const hubTerm = normFactTerm(hub);
|
|
605
|
-
const isReported = reportedIds
|
|
606
|
-
? () => true
|
|
607
|
-
: (id) => (reportedIds instanceof Set ? reportedIds.has(id) : reportedIds.includes(id));
|
|
953
|
+
const isReported = idMembership(reportedIds);
|
|
608
954
|
const hubRows = subgraphRows.filter((r) => normFactTerm(r.subject) === hubTerm);
|
|
609
955
|
const reportedHubRows = hubRows.filter((r) => isReported(r.id));
|
|
610
|
-
const
|
|
611
|
-
(r) => normFactTerm(r.subject) !== hubTerm && normFactTerm(r.object) !== hubTerm && isReported(r.id),
|
|
612
|
-
);
|
|
613
|
-
|
|
614
|
-
const sentences = [];
|
|
615
|
-
|
|
616
|
-
const identityObjects = hubRows
|
|
617
|
-
.filter((r) => IDENTITY_PREDICATES.has(r.predicate))
|
|
618
|
-
.map((r) => r.object)
|
|
619
|
-
.sort();
|
|
620
|
-
if (identityObjects.length) {
|
|
621
|
-
const withArticles = identityObjects.map((object) => `${articleFor(object)} ${object}`);
|
|
622
|
-
sentences.push(`${hub} is ${joinObjects(withArticles)}`);
|
|
623
|
-
}
|
|
624
|
-
|
|
956
|
+
const report = [];
|
|
625
957
|
for (const predicate of predicatesInRenderOrder(reportedHubRows)) {
|
|
626
|
-
if (IDENTITY_PREDICATES.has(predicate) ||
|
|
627
|
-
const
|
|
628
|
-
|
|
629
|
-
.map((r) => r.object)
|
|
630
|
-
.sort();
|
|
958
|
+
if (IDENTITY_PREDICATES.has(predicate) || report.length >= REPORT_SENTENCE_CAP) continue;
|
|
959
|
+
const groupRows = reportedHubRows.filter((r) => r.predicate === predicate);
|
|
960
|
+
const objects = groupRows.map((r) => r.object).sort();
|
|
631
961
|
if (!objects.length) continue;
|
|
632
|
-
|
|
962
|
+
report.push({ text: `${hub} ${predicatePhrase(predicate, hub)} ${joinObjects(objects)}`, rows: groupRows });
|
|
633
963
|
}
|
|
634
964
|
|
|
635
965
|
// A hub that only ever appears as an OBJECT — the place a quake struck, the
|
|
636
966
|
// story a site discussed — has no subject-side row to build a sentence from,
|
|
637
967
|
// and its card came out blank. What was reported about it still says
|
|
638
968
|
// something, so those rows render whole, subject and all.
|
|
639
|
-
if (!
|
|
640
|
-
const
|
|
969
|
+
if (!report.length) {
|
|
970
|
+
const aboutHubRows = subgraphRows
|
|
641
971
|
.filter((r) => normFactTerm(r.object) === hubTerm && normFactTerm(r.subject) !== hubTerm && isReported(r.id))
|
|
642
972
|
.sort(byId)
|
|
643
|
-
.slice(0, OBJECTS_PER_SENTENCE)
|
|
644
|
-
|
|
645
|
-
|
|
973
|
+
.slice(0, OBJECTS_PER_SENTENCE);
|
|
974
|
+
if (aboutHubRows.length) {
|
|
975
|
+
report.push({ text: aboutHubRows.map((r) => factSentence(r)).join("; "), rows: aboutHubRows });
|
|
976
|
+
}
|
|
646
977
|
}
|
|
647
978
|
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
979
|
+
// The identity clause follows the news, never leads it, and only when
|
|
980
|
+
// something actually stated the hub's kind in one sense. The entailment
|
|
981
|
+
// closure names thirteen classes for "france" — a list nobody asked for,
|
|
982
|
+
// most of it the wrong sense — so it opens no card.
|
|
983
|
+
const identity = [];
|
|
984
|
+
const identityRows = hubRows.filter((r) => IDENTITY_PREDICATES.has(r.predicate) && !isDerivedRow(r));
|
|
985
|
+
const identityObjects = identityRows.map((r) => r.object).sort();
|
|
986
|
+
const identityIsSingleSense = identityObjects.length > 0 && identityObjects.length <= IDENTITY_MAX_CLASSES;
|
|
987
|
+
if (identityIsSingleSense) {
|
|
988
|
+
identity.push({
|
|
989
|
+
text: `${hub} is ${joinObjects(identityObjects.map((object) => `${articleFor(object)} ${object}`))}`,
|
|
990
|
+
rows: identityRows,
|
|
991
|
+
});
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
const known = groupedFactSentenceEntries(knownFactRows(hub, subgraphRows, { reportedIds, articleTerms }));
|
|
995
|
+
|
|
996
|
+
const neighbours = neighbourRows(hub, subgraphRows, { reportedIds });
|
|
997
|
+
const around = neighbours.length
|
|
998
|
+
? [{ text: `Around it: ${neighbours.map((r) => factSentence(r)).join("; ")}`, rows: neighbours }]
|
|
999
|
+
: [];
|
|
1000
|
+
|
|
1001
|
+
return { report, identity, known, around };
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
/** The paragraph's own sentence entries (`{ text, rows }`), in print order and
|
|
1005
|
+
* sliced to exactly what `renderNewsParagraph` shows — the per-block caps
|
|
1006
|
+
* (identity, known) and the paragraph-wide `SENTENCE_CAP` both applied.
|
|
1007
|
+
* Shared by `renderNewsParagraph` and `printedParagraphRows` so the two can
|
|
1008
|
+
* never drift: one reads `.text`, the other reads `.rows`. */
|
|
1009
|
+
function paragraphSentenceEntries(hub, subgraphRows, { reportedIds = null, articleTerms = [] } = {}) {
|
|
1010
|
+
const { report, identity, known, around } = paragraphBlocks(hub, subgraphRows, { reportedIds, articleTerms });
|
|
1011
|
+
return [
|
|
1012
|
+
...report,
|
|
1013
|
+
...identity.slice(0, IDENTITY_SENTENCE_CAP),
|
|
1014
|
+
...known.slice(0, KNOWN_FACT_SENTENCE_CAP),
|
|
1015
|
+
...around,
|
|
1016
|
+
].slice(0, SENTENCE_CAP);
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
/** A card's paragraph: what a source reported, then what the thing is, then
|
|
1020
|
+
* the related facts the graph already held about it, then its own
|
|
1021
|
+
* neighbourhood (neighbourRows). Every sentence shown is a grounded fact,
|
|
1022
|
+
* never a paraphrase of prose the grammar could not read, and the news always
|
|
1023
|
+
* leads.
|
|
1024
|
+
*
|
|
1025
|
+
* `reportedIds` (PLAN_NEWS_FEED.md section 17.4), when given, splits the rows
|
|
1026
|
+
* into what was reported (the lead sentences) and what the graph already held
|
|
1027
|
+
* (the background ones). Defaults to null, meaning every row counts as
|
|
1028
|
+
* reported. */
|
|
1029
|
+
export function renderNewsParagraph(hub, subgraphRows, { reportedIds = null, articleTerms = [] } = {}) {
|
|
1030
|
+
const sentences = paragraphSentenceEntries(hub, subgraphRows, { reportedIds, articleTerms }).map((entry) => entry.text);
|
|
1031
|
+
return sentences.length ? `${sentences.join(". ")}.` : "";
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
/** Every fact row that survives into the card's rendered main paragraph —
|
|
1035
|
+
* the same rows `renderNewsParagraph` drew its sentences from, after every
|
|
1036
|
+
* cap it applies (per-block and the paragraph-wide `SENTENCE_CAP`). A row
|
|
1037
|
+
* computed but sliced away before render (an "Around it" clause cut by the
|
|
1038
|
+
* overall cap, an identity class beyond `IDENTITY_MAX_CLASSES`) never
|
|
1039
|
+
* appears here, because it never appears on the card either. */
|
|
1040
|
+
export function printedParagraphRows(hub, subgraphRows, { reportedIds = null, articleTerms = [] } = {}) {
|
|
1041
|
+
return paragraphSentenceEntries(hub, subgraphRows, { reportedIds, articleTerms }).flatMap((entry) => entry.rows);
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
/** The "what the graph already knew" disclosure: the same related facts the
|
|
1045
|
+
* paragraph leads with, at the disclosure's own fuller depth, and nothing the
|
|
1046
|
+
* card already reported. Empty when the graph held nothing about this card's
|
|
1047
|
+
* own subjects — a card with no background says so rather than filling the
|
|
1048
|
+
* space with whatever the two-hop walk happened to reach. */
|
|
1049
|
+
export function renderKnownFactsParagraph(hub, subgraphRows, { reportedIds = null, articleTerms = [] } = {}) {
|
|
1050
|
+
const sentences = groupedFactSentences(knownFactRows(hub, subgraphRows, { reportedIds, articleTerms }));
|
|
1051
|
+
return sentences.length ? `${sentences.join(". ")}.` : "";
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
// ---------------------------------------------------------------------------
|
|
1055
|
+
// Which story a card tells, and what it is called.
|
|
1056
|
+
// ---------------------------------------------------------------------------
|
|
1057
|
+
|
|
1058
|
+
// The item tag a news row carries, at whatever depth the ingest wrapper
|
|
1059
|
+
// nested it ("news:<sourceId>@<itemId>", "optimistic-extract:news:…"), plus
|
|
1060
|
+
// the fixture replay's own twin. Every sentence a card prints comes from a
|
|
1061
|
+
// row, so which story a card is telling is readable from the fact set alone —
|
|
1062
|
+
// no source map, no arrival order, no clock.
|
|
1063
|
+
const NEWS_STORY_TAG_RE = /(?:^|:)news(?:-fixture)?:([^\s|]+)/;
|
|
1064
|
+
|
|
1065
|
+
/** The newsworthy item one row reported, or "" when its provenance names
|
|
1066
|
+
* none. Pure. */
|
|
1067
|
+
export function newsStoryKey(row) {
|
|
1068
|
+
return NEWS_STORY_TAG_RE.exec(String(row?.provenance || ""))?.[1] || "";
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
/** Every reported row touching each term, subject side or object side. Built
|
|
1072
|
+
* once per feed assembly and shared, for the same reason buildTermAdjacency
|
|
1073
|
+
* is. */
|
|
1074
|
+
function reportedRowsByTerm(reported) {
|
|
1075
|
+
const byTerm = new Map();
|
|
1076
|
+
for (const row of reported) {
|
|
1077
|
+
for (const term of new Set([normFactTerm(row.subject), normFactTerm(row.object)])) {
|
|
1078
|
+
if (!term) continue;
|
|
1079
|
+
let rowsFor = byTerm.get(term);
|
|
1080
|
+
if (!rowsFor) byTerm.set(term, (rowsFor = []));
|
|
1081
|
+
rowsFor.push(row);
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
return byTerm;
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
// How many stories a term's own reports must span before it stops being one
|
|
1088
|
+
// story's subject. Two is enough: a term the day's second story also names is
|
|
1089
|
+
// a publication, a wire desk or a class every item shares, and a card headed
|
|
1090
|
+
// by one repeats whatever the per-story cards already said.
|
|
1091
|
+
const PUBLICATION_STORY_MIN = 2;
|
|
1092
|
+
|
|
1093
|
+
/** How many distinct newsworthy items each term's own reports came from. */
|
|
1094
|
+
function storyCountsByTerm(rowsByTerm) {
|
|
1095
|
+
const counts = new Map();
|
|
1096
|
+
for (const [term, rowsFor] of rowsByTerm) {
|
|
1097
|
+
const keys = new Set();
|
|
1098
|
+
for (const row of rowsFor) {
|
|
1099
|
+
const key = newsStoryKey(row);
|
|
1100
|
+
if (key) keys.add(key);
|
|
1101
|
+
}
|
|
1102
|
+
counts.set(term, keys.size);
|
|
656
1103
|
}
|
|
1104
|
+
return counts;
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
// The classes the graph's own identity rows use to say a thing is a place or
|
|
1108
|
+
// a person. The same closed set the bench's entity-preservation metric reads
|
|
1109
|
+
// off the seed, duplicated here (not imported) because the domain layer never
|
|
1110
|
+
// imports a script.
|
|
1111
|
+
const PLACE_OR_PERSON_CLASS_TERMS = new Set([
|
|
1112
|
+
"place", "person", "city", "country", "location", "nation", "continent",
|
|
1113
|
+
"capital", "state", "province", "town", "region",
|
|
1114
|
+
]);
|
|
657
1115
|
|
|
658
|
-
|
|
659
|
-
|
|
1116
|
+
/** Every term the graph itself types as a place or a person, whoever stated
|
|
1117
|
+
* the identity. A card prefers one of these for its own title: it is the
|
|
1118
|
+
* thing the story is about, where a clause the same report threw off is only
|
|
1119
|
+
* something that happened to it. Pure over `rows`. */
|
|
1120
|
+
export function placeAndPersonTerms(rows) {
|
|
1121
|
+
const grounded = new Set();
|
|
1122
|
+
for (const row of rows) {
|
|
1123
|
+
if (row.predicate !== "rdf:type" && row.predicate !== "rdfs:subClassOf") continue;
|
|
1124
|
+
if (!PLACE_OR_PERSON_CLASS_TERMS.has(normFactTerm(row.object))) continue;
|
|
1125
|
+
const subject = normFactTerm(row.subject);
|
|
1126
|
+
if (subject) grounded.add(subject);
|
|
1127
|
+
}
|
|
1128
|
+
return grounded;
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
// A predicate minted from a source's own verb, lemma only: "mgx:hit",
|
|
1132
|
+
// "mgx:discuss". The curated table's entries carry a capital ("mgx:partOf")
|
|
1133
|
+
// and a folded preposition carries a hyphen ("mgx:strike-near"), and neither
|
|
1134
|
+
// can be a single word inside a term, so neither belongs in the verb set.
|
|
1135
|
+
const MINTED_VERB_PREDICATE_RE = /^mgx:([a-z]+)$/;
|
|
1136
|
+
|
|
1137
|
+
/** The verbs this window's own reports minted a predicate from. A term
|
|
1138
|
+
* carrying one of them is a clause the extraction cut in half, and the graph
|
|
1139
|
+
* says so itself rather than a hand-written verb list saying it. */
|
|
1140
|
+
export function reportedVerbWords(reported) {
|
|
1141
|
+
const verbs = new Set();
|
|
1142
|
+
for (const row of reported) {
|
|
1143
|
+
const lemma = MINTED_VERB_PREDICATE_RE.exec(String(row.predicate || ""))?.[1];
|
|
1144
|
+
if (lemma) verbs.add(lemma);
|
|
1145
|
+
}
|
|
1146
|
+
return verbs;
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
// How many words a name runs to before it reads as a sentence about a thing
|
|
1150
|
+
// rather than the thing's name. "south sandwich islands region" is a place;
|
|
1151
|
+
// "boats hit by mystery attackers" is what happened to some.
|
|
1152
|
+
const CLAUSE_TITLE_MAX_WORDS = 4;
|
|
1153
|
+
const CLAUSE_WORD_EDGE_RE = /^[^a-z0-9]+|[^a-z0-9]+$/g;
|
|
1154
|
+
|
|
1155
|
+
/** Does `term` read as a clause rather than a name — longer than a name runs,
|
|
1156
|
+
* or carrying one of the verbs the window's own reports minted? A clause
|
|
1157
|
+
* never takes a card's title from a grounded entity term. */
|
|
1158
|
+
export function readsAsClauseTerm(term, verbWords) {
|
|
1159
|
+
const words = String(term ?? "").trim().toLowerCase().split(/\s+/).filter(Boolean);
|
|
1160
|
+
if (!words.length) return false;
|
|
1161
|
+
if (words.length > CLAUSE_TITLE_MAX_WORDS) return true;
|
|
1162
|
+
const verbs = verbWords instanceof Set ? verbWords : new Set(verbWords || []);
|
|
1163
|
+
return words.some((word) => verbs.has(word.replace(CLAUSE_WORD_EDGE_RE, "")));
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
/** The term a card is titled and keyed by, given the hub the gate chose and
|
|
1167
|
+
* the reported rows that hub sits in. In order:
|
|
1168
|
+
*
|
|
1169
|
+
* 1. the hub itself, when the graph already types it as a place or a person;
|
|
1170
|
+
* 2. a place/person term the hub's own reports name — "bali" over "sacred
|
|
1171
|
+
* glow", "clacton" over "election";
|
|
1172
|
+
* 3. the subject those reports share, but only when the hub reads as a clause
|
|
1173
|
+
* and only when that subject tells one story of its own — the publication
|
|
1174
|
+
* every headline hangs off is never a card's name;
|
|
1175
|
+
* 4. the hub, unchanged.
|
|
1176
|
+
*
|
|
1177
|
+
* Ties inside a step go to the term the reports name most, then alphabetical,
|
|
1178
|
+
* so the same fact set always titles a card the same way. Nothing here invents
|
|
1179
|
+
* a term: every candidate is already on one of the card's own rows. */
|
|
1180
|
+
export function hubTitleTerm(hubTerm, hubRows, { placeOrPerson, verbWords, storyCountByTerm }) {
|
|
1181
|
+
if (placeOrPerson.has(hubTerm)) return hubTerm;
|
|
1182
|
+
|
|
1183
|
+
const namesAThing = (term) => Boolean(term) && term !== hubTerm && !STOP_SET.has(term)
|
|
1184
|
+
&& !isQuantityTerm(term) && looksLikeEntityTerm(term) && !readsAsClauseTerm(term, verbWords);
|
|
1185
|
+
|
|
1186
|
+
const groundedCounts = new Map();
|
|
1187
|
+
const subjectCounts = new Map();
|
|
1188
|
+
for (const row of hubRows) {
|
|
1189
|
+
const subject = normFactTerm(row.subject);
|
|
1190
|
+
const object = normFactTerm(row.object);
|
|
1191
|
+
for (const term of new Set([subject, object])) {
|
|
1192
|
+
if (namesAThing(term) && placeOrPerson.has(term)) groundedCounts.set(term, (groundedCounts.get(term) || 0) + 1);
|
|
1193
|
+
}
|
|
1194
|
+
if (object !== hubTerm || !namesAThing(subject)) continue;
|
|
1195
|
+
if ((storyCountByTerm.get(subject) || 0) >= PUBLICATION_STORY_MIN) continue;
|
|
1196
|
+
subjectCounts.set(subject, (subjectCounts.get(subject) || 0) + 1);
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
const mostNamed = (counts) => [...counts.entries()]
|
|
1200
|
+
.sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))[0]?.[0] || "";
|
|
1201
|
+
|
|
1202
|
+
return mostNamed(groundedCounts)
|
|
1203
|
+
|| (readsAsClauseTerm(hubTerm, verbWords) ? mostNamed(subjectCounts) : "")
|
|
1204
|
+
|| hubTerm;
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
/** True when a source spelled this term as a settlement and its region at once
|
|
1208
|
+
* — "wana, pakistan", "mina, nevada". The source has already said which of a
|
|
1209
|
+
* report's terms is the thing that happened somewhere, so the term reads as a
|
|
1210
|
+
* place even where no identity row types it as one. hubSeedTerms holds the
|
|
1211
|
+
* same reading for the walk. */
|
|
1212
|
+
const namesASettlementAndRegion = (term) => hubSeedTerms(term).length > 1;
|
|
1213
|
+
|
|
1214
|
+
/** The gate's hubs retitled by hubTitleTerm, with two hubs that answer to the
|
|
1215
|
+
* same name merged into one. Each keeps the two readings storyCoverage ranks
|
|
1216
|
+
* cards by, taken on the title the card will actually wear. Keeps the gate's
|
|
1217
|
+
* own sort — changed count desc, then term asc. */
|
|
1218
|
+
function titledHubs(hubs, rows, reported, rowsByTerm) {
|
|
1219
|
+
const placeOrPerson = placeAndPersonTerms(rows);
|
|
1220
|
+
const verbWords = reportedVerbWords(reported);
|
|
1221
|
+
const storyCountByTerm = storyCountsByTerm(rowsByTerm);
|
|
1222
|
+
const changedByTitle = new Map();
|
|
1223
|
+
for (const { term, changed } of hubs) {
|
|
1224
|
+
const title = hubTitleTerm(term, rowsByTerm.get(term) || [], { placeOrPerson, verbWords, storyCountByTerm });
|
|
1225
|
+
const held = changedByTitle.get(title);
|
|
1226
|
+
if (held === undefined || changed > held) changedByTitle.set(title, changed);
|
|
1227
|
+
}
|
|
1228
|
+
return [...changedByTitle.entries()]
|
|
1229
|
+
.map(([term, changed]) => ({
|
|
1230
|
+
term,
|
|
1231
|
+
changed,
|
|
1232
|
+
namesAnEntity: placeOrPerson.has(term) || namesASettlementAndRegion(term),
|
|
1233
|
+
clauseShaped: readsAsClauseTerm(term, verbWords),
|
|
1234
|
+
reportSubject: (rowsByTerm.get(term) || []).some((row) => normFactTerm(row.subject) === term),
|
|
1235
|
+
}))
|
|
1236
|
+
.sort((a, b) => b.changed - a.changed || (a.term < b.term ? -1 : a.term > b.term ? 1 : 0));
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
/** How a story picks between the cards that want to tell it. In order: the
|
|
1240
|
+
* hub whose own reports span fewest stories, since a card about one story
|
|
1241
|
+
* beats a publication's roundup of it; then the hub that names a place or a
|
|
1242
|
+
* person, since that is what the story is about; then the hub that
|
|
1243
|
+
* reads as a name rather than a clause; then the one the report puts on the
|
|
1244
|
+
* subject side, the story's actor where the object is what happened to it;
|
|
1245
|
+
* then the gate's own changed count; then the term itself, so the answer never
|
|
1246
|
+
* comes down to arrival order. */
|
|
1247
|
+
function byCardClaim(a, b) {
|
|
1248
|
+
return a.storyCount - b.storyCount
|
|
1249
|
+
|| (Number(b.namesAnEntity) - Number(a.namesAnEntity))
|
|
1250
|
+
|| (Number(a.clauseShaped) - Number(b.clauseShaped))
|
|
1251
|
+
|| (Number(b.reportSubject) - Number(a.reportSubject))
|
|
1252
|
+
|| (b.changed - a.changed)
|
|
1253
|
+
|| (a.term < b.term ? -1 : a.term > b.term ? 1 : 0);
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
/** Which stories each hub gets to tell, and which of its reports belong to
|
|
1257
|
+
* another card. One story mints one card: the hubs bid for it in byCardClaim
|
|
1258
|
+
* order and the first takes it, so a report that threw off both a subject hub
|
|
1259
|
+
* and an object hub ("ukraine" beside "air war", "london" beside "glass")
|
|
1260
|
+
* stops minting a card each. A hub left with no story of its own does not
|
|
1261
|
+
* mint, and one that keeps some carries only those — the "hackernews" card
|
|
1262
|
+
* that repeated both of the day's Hacker News cards wholesale is gone, and one
|
|
1263
|
+
* that repeated three of four carries the fourth alone.
|
|
1264
|
+
*
|
|
1265
|
+
* A row whose provenance names no story is never claimed and never dropped, so
|
|
1266
|
+
* a hub built only from those still mints.
|
|
1267
|
+
*
|
|
1268
|
+
* Returns hub term -> `{ mints, coveredRowIds }`. */
|
|
1269
|
+
export function storyCoverage(hubs, rowsByTerm) {
|
|
1270
|
+
const rowIdsByStory = new Map();
|
|
1271
|
+
for (const { term } of hubs) {
|
|
1272
|
+
const byStory = new Map();
|
|
1273
|
+
for (const row of rowsByTerm.get(term) || []) {
|
|
1274
|
+
const key = newsStoryKey(row);
|
|
1275
|
+
if (!key) continue;
|
|
1276
|
+
let ids = byStory.get(key);
|
|
1277
|
+
if (!ids) byStory.set(key, (ids = []));
|
|
1278
|
+
ids.push(row.id);
|
|
1279
|
+
}
|
|
1280
|
+
rowIdsByStory.set(term, byStory);
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1283
|
+
const bidders = hubs
|
|
1284
|
+
.map((hub) => ({ ...hub, storyCount: rowIdsByStory.get(hub.term).size }))
|
|
1285
|
+
.sort(byCardClaim);
|
|
1286
|
+
|
|
1287
|
+
const claimed = new Set();
|
|
1288
|
+
const coverage = new Map();
|
|
1289
|
+
for (const { term, storyCount } of bidders) {
|
|
1290
|
+
const stories = rowIdsByStory.get(term);
|
|
1291
|
+
const coveredRowIds = new Set();
|
|
1292
|
+
let ownStories = 0;
|
|
1293
|
+
for (const key of [...stories.keys()].sort()) {
|
|
1294
|
+
if (claimed.has(key)) {
|
|
1295
|
+
for (const id of stories.get(key)) coveredRowIds.add(id);
|
|
1296
|
+
continue;
|
|
1297
|
+
}
|
|
1298
|
+
claimed.add(key);
|
|
1299
|
+
ownStories += 1;
|
|
1300
|
+
}
|
|
1301
|
+
coverage.set(term, { mints: storyCount === 0 || ownStories > 0, coveredRowIds });
|
|
1302
|
+
}
|
|
1303
|
+
return coverage;
|
|
1304
|
+
}
|
|
1305
|
+
|
|
1306
|
+
/** The entities this card's own article names, as terms the graph could hold
|
|
1307
|
+
* facts about. `articleEntityNames` reads them out of the text the card
|
|
1308
|
+
* already shows — each source's headline and the description beneath it — and
|
|
1309
|
+
* the same discipline the hub gate applies then filters them: no stop word, no
|
|
1310
|
+
* quantity, no class the graph's own identity rows name, nothing that reads as
|
|
1311
|
+
* a clause rather than a name. Sorted, so a card's background never depends on
|
|
1312
|
+
* the order the names came back in. */
|
|
1313
|
+
function cardArticleTerms(sources, articleEntityNames, { concepts, readsAsEntityTerm }) {
|
|
1314
|
+
const texts = [];
|
|
1315
|
+
for (const source of sources) {
|
|
1316
|
+
if (source.title) texts.push(source.title);
|
|
1317
|
+
if (source.summary) texts.push(source.summary);
|
|
1318
|
+
}
|
|
1319
|
+
if (!texts.length) return [];
|
|
1320
|
+
const terms = new Set();
|
|
1321
|
+
for (const name of articleEntityNames(texts) || []) {
|
|
1322
|
+
const term = normFactTerm(name);
|
|
1323
|
+
if (!term || STOP_SET.has(term) || isQuantityTerm(term) || concepts.has(term)) continue;
|
|
1324
|
+
if (!readsAsEntityTerm(term)) continue;
|
|
1325
|
+
terms.add(term);
|
|
1326
|
+
}
|
|
1327
|
+
return [...terms].sort();
|
|
660
1328
|
}
|
|
661
1329
|
|
|
662
1330
|
/** newsworthyHubs -> one item per hub (PLAN_NEWS_FEED.md section 6.6),
|
|
@@ -666,19 +1334,66 @@ export function renderNewsParagraph(hub, subgraphRows, { reportedIds = null } =
|
|
|
666
1334
|
* The gate (PLAN_NEWS_FEED.md section 17): `reportedRows` replaces `newsWindowRows`
|
|
667
1335
|
* and `newsworthyHubs` replaces `scoreHubs` as this function's own inputs —
|
|
668
1336
|
* both keep their prior behaviour for every other caller. Each item's
|
|
669
|
-
* two-hop sub-graph then splits into its own `reported`/`background` rows
|
|
670
|
-
*
|
|
671
|
-
* `backgroundParagraph`
|
|
672
|
-
|
|
1337
|
+
* two-hop sub-graph then splits into its own `reported`/`background` rows:
|
|
1338
|
+
* the report leads the paragraph, the related background follows it, and
|
|
1339
|
+
* `backgroundParagraph` names that background at its own fuller depth for the
|
|
1340
|
+
* card's disclosure.
|
|
1341
|
+
*
|
|
1342
|
+
* A card attributes a source to the rows it reports (hubReportRows) and to
|
|
1343
|
+
* nothing else. The two-hop walk reaches every row a shared class node
|
|
1344
|
+
* touches, so attributing the whole sub-graph gave one quake's card all 44 of
|
|
1345
|
+
* the day's quake headlines. An "Around it" neighbour is context the card
|
|
1346
|
+
* borrows, and it carries its own citation on its own card.
|
|
1347
|
+
*
|
|
1348
|
+
* Between the gate and the render sit two more reads over the same reported
|
|
1349
|
+
* rows: `titledHubs` names each card after the entity its own report grounds
|
|
1350
|
+
* rather than a clause the report threw off, and `storyCoverage` leaves a
|
|
1351
|
+
* publication with only the stories no other card tells. Both are pure over
|
|
1352
|
+
* the fact set, so a feed built from the same rows in two orders still comes
|
|
1353
|
+
* back byte for byte.
|
|
1354
|
+
*
|
|
1355
|
+
* `articleEntityNames`, when the caller supplies one, reads the entity names
|
|
1356
|
+
* out of the text of the sources a card shows (news.mjs wires the services
|
|
1357
|
+
* layer's own capture, the same one the enrichment ledger admits terms by).
|
|
1358
|
+
* Those entities widen the card's background: what the graph holds about a
|
|
1359
|
+
* name inside the headline now reaches the card, where before only the
|
|
1360
|
+
* endpoints of its own facts did. They never widen its report — the walk they
|
|
1361
|
+
* seed excludes every reported row — so what a card claims a source said is
|
|
1362
|
+
* untouched. */
|
|
1363
|
+
export function buildNewsItems(rows, {
|
|
1364
|
+
now, windowMs, limit = 6, sourcesByFactId = new Map(), readsAsEntityTerm, articleEntityNames = null,
|
|
1365
|
+
} = {}) {
|
|
673
1366
|
const reported = reportedRows(rows, { now, windowMs });
|
|
674
1367
|
const reportedIds = new Set(reported.map((r) => r.id));
|
|
675
1368
|
const adjacency = buildTermAdjacency(rows);
|
|
676
1369
|
const prior = priorTerms(rows);
|
|
677
1370
|
const hubOptions = { now, windowMs, limit, adjacency, prior };
|
|
678
1371
|
if (readsAsEntityTerm) hubOptions.readsAsEntityTerm = readsAsEntityTerm;
|
|
679
|
-
const
|
|
680
|
-
const
|
|
681
|
-
|
|
1372
|
+
const rowsByTerm = reportedRowsByTerm(reported);
|
|
1373
|
+
const hubs = titledHubs(newsworthyHubs(rows, reported, hubOptions), rows, reported, rowsByTerm);
|
|
1374
|
+
const coverage = storyCoverage(hubs, rowsByTerm);
|
|
1375
|
+
const concepts = conceptTerms(rows);
|
|
1376
|
+
const namesEntities = readsAsEntityTerm || looksLikeEntityTerm;
|
|
1377
|
+
const items = hubs.filter(({ term }) => coverage.get(term).mints).map(({ term, changed }) => {
|
|
1378
|
+
const coveredRowIds = coverage.get(term).coveredRowIds;
|
|
1379
|
+
const hubRows = subgraphAround(rows, term, {
|
|
1380
|
+
adjacency,
|
|
1381
|
+
priorityIds: reportedIds,
|
|
1382
|
+
seedTerms: hubSeedTerms(term),
|
|
1383
|
+
excludeIds: coveredRowIds,
|
|
1384
|
+
});
|
|
1385
|
+
const sources = collectSources(hubReportRows(term, hubRows, { reportedIds }), sourcesByFactId);
|
|
1386
|
+
const articleTerms = articleEntityNames
|
|
1387
|
+
? cardArticleTerms(sources, articleEntityNames, { concepts, readsAsEntityTerm: namesEntities })
|
|
1388
|
+
: [];
|
|
1389
|
+
const heldIds = new Set(hubRows.map((r) => r.id));
|
|
1390
|
+
const articleRows = articleTerms.length
|
|
1391
|
+
? articleEntityRows(rows, articleTerms, {
|
|
1392
|
+
adjacency,
|
|
1393
|
+
excludeIds: new Set([...coveredRowIds, ...reportedIds]),
|
|
1394
|
+
}).filter((r) => !heldIds.has(r.id))
|
|
1395
|
+
: [];
|
|
1396
|
+
const subgraphRows = articleRows.length ? [...hubRows, ...articleRows] : hubRows;
|
|
682
1397
|
const factIds = subgraphRows.map((r) => r.id).sort();
|
|
683
1398
|
const { background } = splitCardRows(subgraphRows, reportedIds);
|
|
684
1399
|
return {
|
|
@@ -687,11 +1402,11 @@ export function buildNewsItems(rows, { now, windowMs, limit = 6, sourcesByFactId
|
|
|
687
1402
|
factIds,
|
|
688
1403
|
changedCount: changed,
|
|
689
1404
|
builtAt: now,
|
|
690
|
-
paragraph: renderNewsParagraph(term, subgraphRows, { reportedIds }),
|
|
1405
|
+
paragraph: renderNewsParagraph(term, subgraphRows, { reportedIds, articleTerms }),
|
|
691
1406
|
tier: tierOf(subgraphRows),
|
|
692
|
-
sources
|
|
1407
|
+
sources,
|
|
693
1408
|
background: background.map((r) => r.id).sort(),
|
|
694
|
-
backgroundParagraph:
|
|
1409
|
+
backgroundParagraph: renderKnownFactsParagraph(term, subgraphRows, { reportedIds, articleTerms }),
|
|
695
1410
|
};
|
|
696
1411
|
});
|
|
697
1412
|
return items.sort((a, b) => (toMs(b.builtAt) - toMs(a.builtAt)) || byId(a, b));
|