@polycode-projects/the-mechanical-code-talker 6.0.19 → 6.0.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +20 -23
  2. package/bin/tmct.mjs +16 -33
  3. package/corpus/LICENSES.json +0 -21
  4. package/corpus/README.md +10 -13
  5. package/corpus/reference/manifest.json +19 -19
  6. package/corpus/reference/shards/ref-01.jsonl.gz +0 -0
  7. package/corpus/reference/shards/ref-04.jsonl.gz +0 -0
  8. package/corpus/reference/shards/ref-08.jsonl.gz +0 -0
  9. package/corpus/reference/shards/ref-10.jsonl.gz +0 -0
  10. package/corpus/reference/shards/ref-11.jsonl.gz +0 -0
  11. package/corpus/reference/shards/ref-17.jsonl.gz +0 -0
  12. package/corpus/reference/shards/ref-20.jsonl.gz +0 -0
  13. package/corpus/reference/shards/ref-25.jsonl.gz +0 -0
  14. package/corpus/reference/shards/ref-2c.jsonl.gz +0 -0
  15. package/corpus/tier2/generate.mjs +6 -142
  16. package/corpus/tier2/manifest.json +0 -42
  17. package/package.json +4 -4
  18. package/src/adapters/corpus/child-seed.mjs +74 -0
  19. package/src/adapters/corpus/conceptnet.mjs +45 -26
  20. package/src/adapters/memory/blocks.mjs +7 -1
  21. package/src/adapters/memory/core.mjs +453 -103
  22. package/src/adapters/memory/corpus-bands.mjs +33 -10
  23. package/src/adapters/memory/inspect.mjs +24 -5
  24. package/src/adapters/memory/rows.mjs +106 -9
  25. package/src/adapters/memory/shacl.mjs +10 -3
  26. package/src/domain/ask.mjs +27 -10
  27. package/src/domain/cli-verbs.mjs +3 -4
  28. package/src/domain/completions/group.mjs +8 -3
  29. package/src/domain/completions/infer.mjs +7 -2
  30. package/src/domain/completions/prune.mjs +5 -1
  31. package/src/domain/completions/rank.mjs +7 -2
  32. package/src/domain/digest/compose.mjs +5 -1
  33. package/src/domain/digest/select.mjs +12 -6
  34. package/src/domain/domain.mjs +15 -8
  35. package/src/domain/el-classify.mjs +11 -2
  36. package/src/domain/fact-phrase.mjs +86 -4
  37. package/src/domain/hash.mjs +9 -0
  38. package/src/domain/memory/bias.mjs +8 -4
  39. package/src/domain/memory/capability.mjs +12 -6
  40. package/src/domain/memory/fact-order.mjs +29 -0
  41. package/src/domain/memory/resolution.mjs +3 -0
  42. package/src/domain/news-feed.mjs +422 -56
  43. package/src/domain/reference-pack.mjs +5 -0
  44. package/src/domain/sense-scope.mjs +116 -0
  45. package/src/domain/sense-split.mjs +1 -1
  46. package/src/domain/syllogise.mjs +21 -13
  47. package/src/domain/tableau.mjs +23 -14
  48. package/src/domain/worlds-pack.mjs +5 -1
  49. package/src/services/adventure-autoplay.mjs +6 -1
  50. package/src/services/adventure-editor.mjs +43 -21
  51. package/src/services/adventure-viz.mjs +26 -9
  52. package/src/services/adventure.mjs +40 -10
  53. package/src/services/chat.mjs +253 -113
  54. package/src/services/extensions.mjs +51 -58
  55. package/src/services/extract-facts.mjs +670 -95
  56. package/src/services/init.mjs +4 -4
  57. package/src/services/ledger-viz.mjs +9 -4
  58. package/src/services/memory-panel-viz.mjs +4 -5
  59. package/src/services/mud-editor.mjs +40 -16
  60. package/src/services/mud-viz.mjs +8 -2
  61. package/src/services/mudiii-turn.mjs +5 -3
  62. package/src/services/mudiii-viz.mjs +8 -2
  63. package/src/services/news.mjs +277 -11
  64. package/src/services/research-viz.mjs +1 -1
  65. package/src/services/sprite-catalog-viz.mjs +10 -5
  66. package/src/surfaces/web/adventure-browser-entry.mjs +6 -12
  67. package/src/surfaces/web/memory-ask-browser.bundle.js +152 -151
  68. package/src/surfaces/web/mud-browser-entry.mjs +7 -11
  69. package/src/surfaces/web/research-browser-entry.mjs +5 -2
  70. package/corpus/tier2/aws.jsonl +0 -39
  71. package/corpus/tier2/java.jsonl +0 -31
  72. package/corpus/tier2/python.jsonl +0 -30
@@ -199,8 +199,8 @@ source = ${JSON.stringify(config.research.source)}
199
199
  if (!Object.keys(extras).length) return out;
200
200
  return `${out}
201
201
  # Extension packs + bias (src/services/extensions.mjs) — written by \`tmct init --with-persona\`
202
- # or a manual edit. Recognized names (human, seon, code, conceptnet, tier2-aws,
203
- # tier2-python, tier2-java, tier2-general) override the shipped defaults; any
202
+ # or a manual edit. Recognized names (human, seon, code, conceptnet, child,
203
+ # namenet, tier2-general) override the shipped defaults; any
204
204
  # other name declares a new host-supplied bundle (needs its own "kind").
205
205
  # [bias] is a flat bundle-name -> weight table consumed by
206
206
  # src/domain/memory/bias.mjs's ranking.
@@ -298,8 +298,8 @@ export async function initRepo(dir, { force = false, seed, env = process.env, pe
298
298
  "",
299
299
  "Corpus bundles (activate by id — no local copy needed):",
300
300
  " tmct import --corpus human | human-medium | human-large",
301
- " tmct import --corpus seon | conceptnet | aws | python | java | general",
302
- " tmct import --corpus wordnet-xl | wordnet-full | namenet (large: tens of thousands of facts)",
301
+ " tmct import --corpus code | seon | conceptnet | general",
302
+ " tmct import --corpus wordnet-xl | wordnet-full | namenet | child (large: tens of thousands of facts)",
303
303
  "",
304
304
  "Ontology / lexicon resources are file paths declared as extension entries:",
305
305
  " tmct import --ontology <path> | --lexicon <path>",
@@ -25,6 +25,11 @@ import { dirname, join } from "node:path";
25
25
 
26
26
  const LEDGER_ROW_LIMIT_DEFAULT = 20000;
27
27
 
28
+ /** Codepoint order, never localeCompare — every string sorted below (a fact
29
+ * row's id or createdAt, a term label, a source bundle key, a predicate) is
30
+ * read off the stored fact set, so two locales have to render the same page. */
31
+ const byCodepoint = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
32
+
28
33
  // The ledger reads as an observability dashboard (dense stat tiles, bar
29
34
  // panels, a log-style fact stream) rather than an editorial page, so its
30
35
  // chrome runs the system sans everywhere instead of THEME_TOKENS_CSS's
@@ -168,7 +173,7 @@ export function computeLedgerDataFromPayload(payload, { focus, term, rowLimit =
168
173
  src: r.provenance || (r.sourceTypes || []).join(" ") || "unrecorded",
169
174
  };
170
175
  });
171
- rows.sort((a, b) => (b.createdAt || "").localeCompare(a.createdAt || "") || a.id.localeCompare(b.id));
176
+ rows.sort((a, b) => byCodepoint(b.createdAt || "", a.createdAt || "") || byCodepoint(a.id, b.id));
172
177
 
173
178
  // Term index + adjacency.
174
179
  const termMap = new Map(); // term -> { degree, best: {trust, prov}, newest }
@@ -187,7 +192,7 @@ export function computeLedgerDataFromPayload(payload, { focus, term, rowLimit =
187
192
  }
188
193
  const terms = [...termMap.values()]
189
194
  .map(({ term: t, degree, prov, newest }) => ({ term: t, degree, prov, newest }))
190
- .sort((a, b) => b.degree - a.degree || a.term.localeCompare(b.term));
195
+ .sort((a, b) => b.degree - a.degree || byCodepoint(a.term, b.term));
191
196
  const edges = rows.map((r) => ({ s: r.s, o: r.o, id: r.id }));
192
197
 
193
198
  // Focus: the asked term when it resolves; otherwise the newest taught
@@ -382,11 +387,11 @@ export function computeLedgerStats(rows, terms, contradictions) {
382
387
  }
383
388
  const bundles = [...bundleCounts.entries()]
384
389
  .map(([key, count]) => ({ key, label: bundleLabelFor(key), count }))
385
- .sort((a, b) => b.count - a.count || a.key.localeCompare(b.key))
390
+ .sort((a, b) => b.count - a.count || byCodepoint(a.key, b.key))
386
391
  .slice(0, BUNDLE_TOP_N);
387
392
  const predicates = [...predicateCounts.entries()]
388
393
  .map(([predicate, v]) => ({ predicate, phrase: v.phrase, count: v.count }))
389
- .sort((a, b) => b.count - a.count || a.predicate.localeCompare(b.predicate))
394
+ .sort((a, b) => b.count - a.count || byCodepoint(a.predicate, b.predicate))
390
395
  .slice(0, PREDICATE_TOP_N);
391
396
 
392
397
  const totalDegree = (terms || []).reduce((sum, t) => sum + (t.degree || 0), 0);
@@ -23,10 +23,9 @@ export function bandLabelFor(key) {
23
23
  "human-large": "human persona (large)",
24
24
  seon: "seon ontology",
25
25
  conceptnet: "ConceptNet",
26
- "tier2-aws": "AWS",
27
- "tier2-python": "Python",
28
- "tier2-java": "Java",
29
26
  "wordnet-xl": "WordNet",
27
+ namenet: "NameNet",
28
+ child: "child vocabulary",
30
29
  };
31
30
  return BAND_LABELS[key] || key;
32
31
  }
@@ -39,7 +38,7 @@ export function bandLabelFor(key) {
39
38
  export function statsSummaryLine(stats, bandLabel) {
40
39
  const BAND_ORDER = [
41
40
  "human", "human-medium", "human-large", "seon", "conceptnet",
42
- "tier2-aws", "tier2-python", "tier2-java", "wordnet-xl",
41
+ "wordnet-xl", "namenet", "child",
43
42
  "taught this session", "other",
44
43
  ];
45
44
  if (!stats || !stats.total) return "no starter memory; starting empty";
@@ -172,7 +171,7 @@ export function renderStatsPanelInto(panelEl, stats, opts) {
172
171
  const { bandLabel, taughtHint, onForget = null, persistNote = null } = opts;
173
172
  const BAND_ORDER = [
174
173
  "human", "human-medium", "human-large", "seon", "conceptnet",
175
- "tier2-aws", "tier2-python", "tier2-java", "wordnet-xl",
174
+ "wordnet-xl", "namenet", "child",
176
175
  "taught this session", "other",
177
176
  ];
178
177
  function bandRow(label, count) {
@@ -24,10 +24,15 @@
24
24
  // Two predicate families sync differently, for the reason adventure-editor.mjs's
25
25
  // own header sets out:
26
26
  // - PLACEMENT (currently-in/located-in/fixed-in/hidden-in), OPENNESS and MASS
27
- // are fold-versioned — foldWorldState already reads the newest write as the
28
- // current truth, which is how every turn's own move and mass drain lands — so
29
- // an edit to one is a plain new write superseding the old, never a
30
- // retraction. Mass belongs here and not below precisely because it MOVES: a
27
+ // are fold-versioned — foldWorldState ranks these rows by the (epoch, turn)
28
+ // pair stamped on the subject, which is how every turn's own move and mass
29
+ // drain lands — so an edit to one is a plain new write superseding the old,
30
+ // never a retraction. The superseding write is stamped one turn past the
31
+ // world's own turn count, exactly the way every in-game action's commit()
32
+ // writes, so it OUTRANKS what it replaces instead of merely arriving after
33
+ // it — at equal turn the fold takes whichever row comes later in the array,
34
+ // and array order is nobody's to promise once a peer's facts have merged
35
+ // in. Mass belongs here and not below precisely because it MOVES: a
31
36
  // playing animal's weight is a fold over turn snapshots, so an editor
32
37
  // diffing raw mass rows would show the seed and fight the drain.
33
38
  // - Everything else (typing, the class hierarchy, exits, the predator flag,
@@ -37,6 +42,8 @@
37
42
  // that fails to parse this keystroke must never be read as "this fact is
38
43
  // gone".
39
44
 
45
+ import { snapshotSubject } from "../domain/world-snapshot.mjs";
46
+
40
47
  export { wordBeforeCursor } from "./viz-theme.mjs";
41
48
 
42
49
  const PLACEMENT_KIND = "placement";
@@ -296,15 +303,26 @@ export function editableMudOtherRows(rows) {
296
303
 
297
304
  /** Plan the fact-store writes one parsed edit implies, from already-parsed
298
305
  * `triples` against the world's current `rows`/`state`. Returns
299
- * `{ toAppend, toRemoveIds }` — pure, no I/O.
306
+ * `{ toAppend, toRemoveIds, editTurn }` — pure, no I/O. `toAppend` rows are
307
+ * store-ready: a fold-versioned one already carries its `subject@turnN`
308
+ * stamp, and `editTurn` is the turn every one of them was stamped at, for
309
+ * the caller's own provenance tag.
310
+ *
311
+ * Placement, openness and mass triples are never retracted: one joins
312
+ * `toAppend` only when it actually differs from the subject's current folded
313
+ * value, so re-asserting an unchanged line doesn't append a no-op duplicate
314
+ * per keystroke. Each one is stamped one turn past the world's own turn
315
+ * count, which is what makes it outrank the row it supersedes rather than
316
+ * tie with it and win on array position.
300
317
  *
301
- * Placement and openness triples are never retracted: one joins `toAppend` only
302
- * when it actually differs from the subject's current folded value, so
303
- * re-asserting an unchanged line doesn't append a no-op duplicate per keystroke.
304
- * Everything else gets a real add/remove diff but the CALLER decides whether
305
- * `toRemoveIds` is safe to apply, and must skip it whenever the parse reported
306
- * any unrecognized line. */
318
+ * Everything else keeps its bare subject every reader takes those raw, and
319
+ * a stamped one names a subject no verb resolves. They get a real add/remove
320
+ * diff but the CALLER decides whether `toRemoveIds` is safe to apply, and
321
+ * must skip it whenever the parse reported any unrecognized line. */
307
322
  export function planMudEditorSync(rows, state, triples) {
323
+ const editTurn = (state?.turnCount ?? 0) + 1;
324
+ const editEpoch = state?.epoch ?? 0;
325
+ const stampedForFold = (t) => ({ ...t, subject: snapshotSubject(t.subject, editTurn, editEpoch) });
308
326
  const toAppend = [];
309
327
  // One "already said this" set per fold-versioned family, never one shared set:
310
328
  // a subject can carry a placement, an openness and a mass at once, and a
@@ -320,13 +338,13 @@ export function planMudEditorSync(rows, state, triples) {
320
338
  seen.add(t.subject);
321
339
  if (t.kind === PLACEMENT_KIND) {
322
340
  const current = state.placements?.get(t.subject);
323
- if (!current || current.predicate !== t.predicate || current.object !== t.object) toAppend.push(t);
341
+ if (!current || current.predicate !== t.predicate || current.object !== t.object) toAppend.push(stampedForFold(t));
324
342
  } else if (t.kind === OPENNESS_KIND) {
325
343
  const current = state.openness?.get(t.subject);
326
- if (!current || current.open !== (t.object === "true")) toAppend.push(t);
344
+ if (!current || current.open !== (t.object === "true")) toAppend.push(stampedForFold(t));
327
345
  } else {
328
346
  const current = state.masses?.get(t.subject);
329
- if (!current || Number(current.value) !== Number(t.object)) toAppend.push(t);
347
+ if (!current || Number(current.value) !== Number(t.object)) toAppend.push(stampedForFold(t));
330
348
  }
331
349
  }
332
350
 
@@ -342,7 +360,7 @@ export function planMudEditorSync(rows, state, triples) {
342
360
  for (const [key, id] of currentKeys) {
343
361
  if (!newOtherKeys.has(key)) toRemoveIds.push(id);
344
362
  }
345
- return { toAppend, toRemoveIds };
363
+ return { toAppend, toRemoveIds, editTurn };
346
364
  }
347
365
 
348
366
  /** The additive half of planMudEditorSync, for ONE already-parsed triple: the
@@ -350,7 +368,13 @@ export function planMudEditorSync(rows, state, triples) {
350
368
  * document says what the world contains, so a fact missing from it has gone;
351
369
  * one sentence only ever says what it says, so nothing it leaves out is
352
370
  * evidence of anything. Re-asserting a fact the world already holds appends
353
- * nothing, and `reason` says which of the two happened. Pure. */
371
+ * nothing, and `reason` says which of the two happened. Pure.
372
+ *
373
+ * The triple comes back with its bare subject, unlike planMudEditorSync's
374
+ * rows: a taught sentence can MINT the thing it talks about, and the caller
375
+ * picks the fresh id, so only the caller knows which subject the fold stamp
376
+ * belongs on. world-teach.mjs stamps a fold-versioned one at `turnCount + 1`
377
+ * there, for the same reason planMudEditorSync does. */
354
378
  export function planTaughtMudTriple(rows, state, triple) {
355
379
  if (!triple?.subject || !triple?.object) return { toAppend: [], reason: "nothing parsed" };
356
380
  if (triple.kind === PLACEMENT_KIND) {
@@ -170,7 +170,10 @@ export function carriedItemsFor(rows, state, character) {
170
170
  return [...state.placements]
171
171
  .filter(([, p]) => p.predicate === "mgx:located-in" && p.object === character)
172
172
  .map(([subject]) => ({ subject, spriteClass: spriteClassForObject(rows, subject) }))
173
- .sort((a, b) => a.subject.localeCompare(b.subject));
173
+ // Codepoint order, never localeCompare — inlined, not a module-level
174
+ // helper, because this function is spliced verbatim into the page's own
175
+ // inline script and can carry no outer-scope reference with it.
176
+ .sort((a, b) => (a.subject < b.subject ? -1 : a.subject > b.subject ? 1 : 0));
174
177
  }
175
178
 
176
179
  /** Every room's depth level, `Map<roomId, number>`, BFS from `root` at level
@@ -1994,7 +1997,10 @@ function pageScript() {
1994
1997
  svg: creature ? spriteSvgFor(speciesOfCharacter(subject), editRows, subject) : objectSvgFor(subject, editRows),
1995
1998
  });
1996
1999
  }
1997
- here.sort(function (a, b) { return a.subject.localeCompare(b.subject); });
2000
+ // Codepoint order, never localeCompare a placed subject traces back to
2001
+ // a world fact, and two readers must render the same room-detail cast
2002
+ // order regardless of the browser's own locale.
2003
+ here.sort(function (a, b) { return a.subject < b.subject ? -1 : a.subject > b.subject ? 1 : 0; });
1998
2004
  castEl.innerHTML = here.map(function (t) { return editCardHtml(t.subject, t.label, t.svg); }).join("");
1999
2005
  const kind = roomKindForRoom(editRows, selectedRoomId);
2000
2006
  const ways = [...(editState.exits.get(selectedRoomId) || new Map()).keys()].sort();
@@ -805,10 +805,12 @@ const PLACED_BY_PREDICATE = "mgx:placed-by";
805
805
 
806
806
  // The families foldTownSquareState ranks by (epoch, turn). A row in one of
807
807
  // them needs a snapshot subject or it ranks as turn 0 and loses to anything
808
- // already played about the same thing. mgx:placed-by is read raw and keeps
809
- // its bare subject, exactly as placeFood writes it.
808
+ // already played about the same thing. mgx:placed-by is one of them: placeFood
809
+ // writes the item's first placer under a bare subject when it mints the item,
810
+ // so a taught placer that stayed bare too would tie with it at turn 0 and win
811
+ // only by sitting later in the array.
810
812
  const TAUGHT_SNAPSHOT_PREDICATES = new Set([
811
- PLACEMENT_PREDICATE, MASS_PREDICATE, MOOD_PREDICATE, FACING_PREDICATE,
813
+ PLACEMENT_PREDICATE, MASS_PREDICATE, MOOD_PREDICATE, FACING_PREDICATE, PLACED_BY_PREDICATE,
812
814
  ]);
813
815
 
814
816
  // The closed cast vocabulary a taught sentence may name, matching the lane's
@@ -230,7 +230,10 @@ export function propPlacementsFrom(rows, assetManifest) {
230
230
  asset: byKey.get(entry.model) || null,
231
231
  });
232
232
  }
233
- placements.sort((a, b) => a.id.localeCompare(b.id));
233
+ // Codepoint order, never localeCompare — inlined, not a module-level
234
+ // helper, because this function is spliced verbatim into the page's own
235
+ // inline script and can carry no outer-scope reference with it.
236
+ placements.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
234
237
  return placements;
235
238
  }
236
239
 
@@ -286,7 +289,10 @@ export function currentPlacementsFrom(rows) {
286
289
  for (const [subject, entry] of latest) {
287
290
  if (!gone.has(subject)) placements.push({ subject, cell: entry.cell });
288
291
  }
289
- placements.sort((a, b) => a.subject.localeCompare(b.subject));
292
+ // Codepoint order, never localeCompare — inlined, not a module-level
293
+ // helper, because this function is spliced verbatim into the page's own
294
+ // inline script and can carry no outer-scope reference with it.
295
+ placements.sort((a, b) => (a.subject < b.subject ? -1 : a.subject > b.subject ? 1 : 0));
290
296
  return placements;
291
297
  }
292
298
 
@@ -23,10 +23,14 @@
23
23
  // ledger is built from state.ledger via ledgerFromPayload only
24
24
  // for the span of one call, then folded back with ledgerPayload.
25
25
  // providers { newsFetchers: Map<sourceId, { id, fetchItems }>,
26
- // getResearchProvider({ source }), preflightNewsUrl?(url) } —
26
+ // getResearchProvider({ source }), preflightNewsUrl?(url),
27
+ // queryBandTerm?({band, term, exclusiveStartKey, limit}) } —
27
28
  // every fetcher and provider this session may call, already
28
29
  // constructed (adapters own I/O construction, never this
29
- // module).
30
+ // module). `queryBandTerm` is subgraph-retrieval.mjs's
31
+ // `termQueryOverDocumentClient`, built once over the caller's own
32
+ // DynamoDB client; absent (the browser surface, tests with no
33
+ // band fixture) simply skips the band-grounding fallback below.
30
34
  // now a function returning the caller's clock reading (ISO string
31
35
  // or ms), or a fixed reading directly. Never read from here —
32
36
  // the wall clock enters only through this parameter.
@@ -41,7 +45,7 @@
41
45
  // section 3). Network lives in the adapters this module is handed, never
42
46
  // inside it.
43
47
 
44
- import { normFactTerm, normFactPredicate, factIdFor } from "../domain/hash.mjs";
48
+ import { normFactTerm, normFactPredicate, normText, factIdFor } from "../domain/hash.mjs";
45
49
  import {
46
50
  newsWindowRows, renderNewsParagraph, buildNewsItems, evictNewsFacts,
47
51
  conceptTerms, isQuantityTerm, newsItemKeys,
@@ -59,8 +63,11 @@ import {
59
63
  } from "../adapters/corpus/news-sources.mjs";
60
64
  import { DEFAULT_MIN_INTERVAL_MS } from "../adapters/corpus/courtesy.mjs";
61
65
  import { researchFacts } from "../adapters/corpus/research-source.mjs";
66
+ import { factFromBandRow } from "../adapters/memory/corpus-bands.mjs";
62
67
  import { throughSourceBreaker, sourceSkipStatusLine } from "../domain/source-breaker.mjs";
63
- import { ingestText, readsAsEntityTerm, ungroundedTermOccurrences } from "./extract-facts.mjs";
68
+ import {
69
+ ingestText, readsAsEntityTerm, ungroundedTermOccurrences, termsUsedOnlyAsVerbs,
70
+ } from "./extract-facts.mjs";
64
71
 
65
72
  export { NEWS_SOURCE_RECORDS, DEFAULT_NEWS_SOURCE_IDS, DEFAULT_NEWS_KB_IDS };
66
73
 
@@ -316,11 +323,19 @@ function buildSourcesByFactId(items) {
316
323
  * drops a term the graph already holds facts about, and a card wants precisely
317
324
  * those. A single word the lexicon reads as an everyday noun drops out here —
318
325
  * "developer" names nothing a lookup could define — while a name run keeps its
319
- * whole spelling, "tim king" and "amigados" alike. */
326
+ * whole spelling, "tim king" and "amigados" alike.
327
+ *
328
+ * A word the article only ever uses as a clause's verb names no entity either,
329
+ * however the tagger read it: "many say it is …" is the article saying
330
+ * something, and "the government moves to assert control" is the government
331
+ * moving. Either one reaching this list gives a card's background walk an
332
+ * anchor the article never named. */
320
333
  export function articleEntityNames(texts, { lexicon } = {}) {
321
334
  const lex = lexicon || loadLexicon();
335
+ const verbs = termsUsedOnlyAsVerbs(texts, { lexicon: lex });
322
336
  const names = [];
323
337
  for (const term of ungroundedTermOccurrences(texts, [], { lexicon: lex }).keys()) {
338
+ if (verbs.has(term)) continue;
324
339
  if (term.includes(" ") || !isVocabGroundedTerm(lex, term)) names.push(term);
325
340
  }
326
341
  return names.sort();
@@ -741,6 +756,139 @@ const KB_SOURCE_TO_RESEARCH_CHOICE = Object.freeze({
741
756
  "dbpedia-lookup": "dbpedia",
742
757
  });
743
758
 
759
+ const ISA_PREDICATES = new Set(["rdfs:subClassOf", "rdf:type"]);
760
+
761
+ const WORD_CHARACTER_RE = /[a-z0-9]/;
762
+
763
+ /** True when `prose` names `term` as a whole word — the test for whether a
764
+ * looked-up body is a STATEMENT about the term or merely a LABEL for it.
765
+ *
766
+ * A reference article's summary opens on the word it defines ("AmigaDOS is
767
+ * the disk operating system of the AmigaOS"). An entity description field
768
+ * does not: it is a bare noun phrase naming the class instead ("disk
769
+ * operating system of the AmigaOS"), and a sentence recognizer reading one
770
+ * finds a subject, a verb and an object inside the phrase itself — "disk
771
+ * operates a system of the amigaos", a claim about a word the lookup was
772
+ * never about. The class such a phrase names is what the structured tier
773
+ * already carries, so there is nothing to lose by leaving it unread. */
774
+ function proseNamesTerm(prose, term) {
775
+ const haystack = normText(prose).toLowerCase();
776
+ const needle = normFactTerm(term);
777
+ if (!haystack || !needle) return false;
778
+ for (let from = 0; ; from += 1) {
779
+ const at = haystack.indexOf(needle, from);
780
+ if (at < 0) return false;
781
+ const before = at === 0 ? "" : haystack[at - 1];
782
+ const after = haystack[at + needle.length] ?? "";
783
+ if (!WORD_CHARACTER_RE.test(before) && !WORD_CHARACTER_RE.test(after)) return true;
784
+ from = at;
785
+ }
786
+ }
787
+
788
+ /** Every name the looked-up definition is ABOUT: the term that was asked and
789
+ * the title the source answered with ("solar eclipse" and "Solar eclipse",
790
+ * "u.s. virgin islands" and "United States Virgin Islands"), each with its
791
+ * plural, since a body says "Solar eclipses can only happen during a new
792
+ * moon" as readily as it says "a solar eclipse". */
793
+ function definedTermKeys(term, article) {
794
+ const keys = new Set();
795
+ for (const name of [term, article?.term, article?.title]) {
796
+ const key = normFactTerm(name ?? "");
797
+ if (!key) continue;
798
+ keys.add(key);
799
+ keys.add(normFactTerm(pluralOf(key)));
800
+ }
801
+ return keys;
802
+ }
803
+
804
+ const SENTENCE_WORD_RE = /[a-z0-9][a-z0-9'’-]*/g;
805
+
806
+ const COPULAS = new Set(["is", "are", "was", "were", "be", "been", "being"]);
807
+
808
+ // Words that mean the copula opened something other than the term's class:
809
+ // a clause ("an earthquake is WHEN tectonic plates shake"), a denial ("Puerto
810
+ // Rico is NOT an independent country"), a second phrase ("is one OF the...")
811
+ // or a list ("is a state AND a...").
812
+ const CLASS_PHRASE_BREAKERS = new Set([
813
+ "not", "no", "never", "n't", "nor",
814
+ "when", "where", "while", "why", "how", "who", "whom", "whose", "which", "that",
815
+ "what", "because", "if", "whether", "unless", "until", "since", "though", "although",
816
+ "of", "in", "on", "at", "from", "with", "by", "to", "for", "about", "into", "onto",
817
+ "over", "under", "between", "among", "during", "near", "across", "through", "against",
818
+ "and", "or", "but",
819
+ ]);
820
+
821
+ /** True when `sentence` states `object` as the class the copula introduces —
822
+ * the test for whether a class claim read out of a definition body is what
823
+ * the body actually says the term is.
824
+ *
825
+ * The optimistic tier answers with a class it found somewhere in the
826
+ * sentence, which is right when the sentence is a definition ("A tsunami is
827
+ * a natural disaster" -> disaster) and wrong the moment the copula opens
828
+ * anything else. "An earthquake is when Earth's tectonic plates shake"
829
+ * yields "an earthquake is an earth"; "Puerto Rico is not an independent
830
+ * country" yields the exact claim the sentence denies. So the class has to
831
+ * sit inside the noun phrase the copula introduces: only determiners,
832
+ * numbers and adjectives may stand between them, never a clause, a
833
+ * preposition, a conjunction or a negation. A body that states its class
834
+ * some other way keeps it to itself, which costs a fact and never a wrong
835
+ * one. */
836
+ function bodyStatesClass(sentence, object) {
837
+ const words = String(sentence ?? "").toLowerCase().match(SENTENCE_WORD_RE) || [];
838
+ const head = (normFactTerm(object).match(SENTENCE_WORD_RE) || [])[0];
839
+ if (!head || !words.length) return false;
840
+ const plural = pluralOf(head);
841
+ const namesHead = (word) => {
842
+ const bare = word.replace(/['’]s$/, "");
843
+ return bare === head || bare === plural || pluralOf(bare) === head;
844
+ };
845
+ for (let at = 0; at < words.length; at += 1) {
846
+ if (!namesHead(words[at])) continue;
847
+ for (let back = at - 1; back >= 0; back -= 1) {
848
+ if (CLASS_PHRASE_BREAKERS.has(words[back])) break;
849
+ if (COPULAS.has(words[back])) return true;
850
+ }
851
+ }
852
+ return false;
853
+ }
854
+
855
+ /** The rows a definition body is allowed to give up: those about the term it
856
+ * defines, and — for a class claim — those the body states as that term's
857
+ * class. A body is evidence about its own subject, so a row whose subject is
858
+ * some other term the prose merely mentions ("Every year there are about two
859
+ * solar eclipses" -> a year is a kind of eclipse; "The Calpine Corporation
860
+ * operates and owns 19 of the 22 facilities" -> a corporation owns a
861
+ * facility) is not a definition of anything and is dropped. */
862
+ function factsDefinitionBodyStates(rows, termKeys) {
863
+ return rows.filter((row) => {
864
+ if (!termKeys.has(normFactTerm(row.subject))) return false;
865
+ if (!ISA_PREDICATES.has(row.predicate)) return true;
866
+ return bodyStatesClass(row.sentence, row.object);
867
+ });
868
+ }
869
+
870
+ const provenanceHead = (tag) => String(tag ?? "").split(/[\s@]/)[0];
871
+
872
+ const tripleKeyOf = (row) => `${normFactTerm(row.subject)}\0${normFactPredicate(row.predicate)}\0${normFactTerm(row.object)}`;
873
+
874
+ /** Retracts rows this call's ingest wrote and the definition-body rule then
875
+ * turned down. ingestText has already stored them by the time their rows
876
+ * come back, so they are matched by triple under this ingest's own
877
+ * provenance and removed — the same retract-after-write path the recognizer
878
+ * takes for a fragment term. */
879
+ async function retractRejectedBodyFacts(ctx, rejected, provenance) {
880
+ const { memoryDir, store, now } = ctx;
881
+ if (!rejected.length) return;
882
+ const keys = new Set(rejected.map(tripleKeyOf));
883
+ const rows = store.readFactRows(await store.loadMemory(memoryDir));
884
+ const ids = rows
885
+ .filter((row) => keys.has(tripleKeyOf(row)) && provenanceHead(row.provenance).endsWith(provenanceHead(provenance)))
886
+ .map((row) => row.id);
887
+ if (!ids.length) return;
888
+ await store.removeFacts(memoryDir, ids, { retractedAt: resolveNow(now) });
889
+ invalidateCache(ctx.cache);
890
+ }
891
+
744
892
  /** Grounds `article` under `provider`'s own provenance tag, through the SAME
745
893
  * ingest seam a polled article takes: the structured or isa facts the
746
894
  * research-source seam licenses (researchFacts), then the article's own
@@ -748,7 +896,23 @@ const KB_SOURCE_TO_RESEARCH_CHOICE = Object.freeze({
748
896
  * looked-up article therefore reaches the graph with the density of
749
897
  * relations a polled one does, and the feed's paraphrase templates have the
750
898
  * same kind of material to write a card from — a bare isa edge left the
751
- * reader with one bald sentence where a polled item got a paragraph. */
899
+ * reader with one bald sentence where a polled item got a paragraph.
900
+ *
901
+ * Two things a definition body is not allowed to do, both of them ways of
902
+ * restating the lookup's own class claim worse than the lookup already
903
+ * stated it. It is not read at all unless it names the term (proseNamesTerm,
904
+ * above). And where the lookup already licensed a class for the term, the
905
+ * body is read by the strict recognizer only: the optimistic tier's guess at
906
+ * a class can then only agree with what the structured rows say or truncate
907
+ * it — "AmigaDOS is the disk operating system" mints "amigados is a disk"
908
+ * beside the structured "amigados is a disk operating system" — while the
909
+ * relations the body states ("Rottnest has a lighthouse") are the strict
910
+ * tier's own work and survive either way.
911
+ *
912
+ * What the body does give up is then held to what it says about the term it
913
+ * defines (factsDefinitionBodyStates): a row about some other term the prose
914
+ * mentions, or a class the sentence never states, is retracted before it can
915
+ * reach the feed or a syllogism. */
752
916
  async function ingestResearchArticle(ctx, term, provider, article) {
753
917
  const { memoryDir, store, config, lexicon, now } = ctx;
754
918
  const provenance = provider.provenanceTag(term);
@@ -757,19 +921,25 @@ async function ingestResearchArticle(ctx, term, provider, article) {
757
921
  await store.appendFacts(memoryDir, structured);
758
922
  invalidateCache(ctx.cache);
759
923
  }
924
+ const lookupStatedClass = structured.some(
925
+ (f) => ISA_PREDICATES.has(f.predicate) && normFactTerm(f.subject) === normFactTerm(term),
926
+ );
760
927
 
761
928
  const prose = String(article.summary || article.text || "").trim();
762
- let ingested = { extracted: [], optimistic: [] };
763
- if (prose) {
764
- ingested = await ingestText(prose, {
765
- memoryDir, sourceTag: provenance, optimistic: true,
929
+ let fromBody = [];
930
+ if (prose && proseNamesTerm(prose, term)) {
931
+ const ingested = await ingestText(prose, {
932
+ memoryDir, sourceTag: provenance, optimistic: !lookupStatedClass,
766
933
  lexicon: lexicon || loadLexicon(), observedAt: resolveNow(now), findings: true,
767
934
  attributeToSource: true,
768
935
  });
769
936
  invalidateCache(ctx.cache);
937
+ const read = [...ingested.extracted, ...ingested.optimistic];
938
+ fromBody = factsDefinitionBodyStates(read, definedTermKeys(term, article));
939
+ await retractRejectedBodyFacts(ctx, read.filter((row) => !fromBody.includes(row)), provenance);
770
940
  }
771
941
 
772
- const facts = [...structured, ...ingested.extracted, ...ingested.optimistic];
942
+ const facts = [...structured, ...fromBody];
773
943
  if (!facts.length) return { facts: 0, derived: 0 };
774
944
  const distinct = new Set(facts.map(
775
945
  (f) => `${normFactTerm(f.subject)}\0${normFactPredicate(f.predicate)}\0${normFactTerm(f.object)}`,
@@ -783,6 +953,88 @@ async function ingestResearchArticle(ctx, term, provider, article) {
783
953
  return { facts: distinct.size, derived };
784
954
  }
785
955
 
956
+ // The bands this fallback reads. All three are reference content — dictionary
957
+ // senses and everyday commonsense edges, not news entities — so they ground
958
+ // the ordinary nouns a headline mentions in passing ("harbor", "senator",
959
+ // "penguin") without ever costing a KB round trip on those.
960
+ const BAND_TERM_LOOKUP_BANDS = Object.freeze(["child", "conceptnet", "wordnet-complete"]);
961
+ const BAND_TERM_LOOKUP_LIMIT = 50;
962
+ // A local DynamoDB Query has no courtesy throttle and nothing external to
963
+ // protect, but it still has to return: the enrich cycle shares the same wall
964
+ // budget every other phase does, and this runs once per candidate term. A
965
+ // slow or hung Query loses the race and reads as a miss — a timeout is a
966
+ // miss, never a guess — rather than stalling the whole cycle behind it.
967
+ // The bands are read in parallel against one deadline, so the budget buys
968
+ // every band at once: each is a single-partition begins_with read of its own,
969
+ // and the worst case stays one timeout however many bands there are.
970
+ const BAND_TERM_LOOKUP_TIMEOUT_MS = 750;
971
+
972
+ /** `term`'s rows from one band, as `{subject, predicate, object, provenance}`
973
+ * triples — or `[]` when the band carries nothing for it or the Query
974
+ * throws. A band that fails is a band with nothing to say, so the others'
975
+ * rows survive it. */
976
+ async function bandTermFacts(queryBandTerm, band, term) {
977
+ let response;
978
+ try {
979
+ response = await queryBandTerm({ band, term, limit: BAND_TERM_LOOKUP_LIMIT });
980
+ } catch {
981
+ return [];
982
+ }
983
+ const facts = [];
984
+ for (const row of response?.rows || []) {
985
+ const fact = factFromBandRow(row);
986
+ if (fact) facts.push(fact);
987
+ }
988
+ return facts;
989
+ }
990
+
991
+ /** `term`'s rows from every band, as `{subject, predicate, object,
992
+ * provenance}` triples ready for `appendFacts` — or `[]` when no band
993
+ * carries it. Each band's Query races the SAME deadline, so a band that
994
+ * hangs costs the cycle one timeout rather than one per band, and the bands
995
+ * that answered inside it keep their rows. A fact both a band and the seed
996
+ * already hold arrives with the band's own provenance, so it corroborates
997
+ * rather than duplicating. `ctx.providers.queryBandTerm` is absent on the
998
+ * browser surface and in most tests, so this is a no-op there by
999
+ * construction, not a special case here. */
1000
+ async function groundTermFromBand(ctx, term) {
1001
+ const queryBandTerm = ctx.providers?.queryBandTerm;
1002
+ if (typeof queryBandTerm !== "function") return [];
1003
+ let deadlineTimer;
1004
+ const deadline = new Promise((resolve) => {
1005
+ deadlineTimer = setTimeout(() => resolve(null), BAND_TERM_LOOKUP_TIMEOUT_MS);
1006
+ });
1007
+ try {
1008
+ const perBand = await Promise.all(BAND_TERM_LOOKUP_BANDS.map(
1009
+ (band) => Promise.race([bandTermFacts(queryBandTerm, band, term), deadline]),
1010
+ ));
1011
+ return perBand.flatMap((facts) => facts || []);
1012
+ } finally {
1013
+ clearTimeout(deadlineTimer);
1014
+ }
1015
+ }
1016
+
1017
+ /** Grounds `term` from `facts` — a band's own rows, already resolved
1018
+ * `{subject, predicate, object, provenance}` triples with no article prose
1019
+ * to walk, so this skips straight to append + syllogise rather than
1020
+ * repeating ingestResearchArticle's structured/prose split. Same shape of
1021
+ * return as ingestResearchArticle, so the caller folds it in identically. */
1022
+ async function ingestBandFacts(ctx, facts) {
1023
+ const { memoryDir, store, config } = ctx;
1024
+ await store.appendFacts(memoryDir, facts);
1025
+ invalidateCache(ctx.cache);
1026
+ const distinct = new Set(facts.map(
1027
+ (f) => `${normFactTerm(f.subject)}\0${normFactPredicate(f.predicate)}\0${normFactTerm(f.object)}`,
1028
+ ));
1029
+ let derived = 0;
1030
+ if (config.syllogismsPerIngest > 0) {
1031
+ const focus = [...termFocusOf(facts)];
1032
+ const res = await syllogise(memoryDir, { focus, expandFocus: true, budget: config.syllogismsPerIngest, store });
1033
+ derived = res?.count || 0;
1034
+ }
1035
+ return { facts: distinct.size, derived };
1036
+ }
1037
+
786
1038
  /** One KB lookup behind its source's circuit breaker. A source that has been
787
1039
  * timing out or throttling is skipped without a round trip, and the skip is
788
1040
  * reported so the cycle can tell "this source had nothing" from "this source
@@ -851,6 +1103,20 @@ export async function enrichTopTerms(ctx, { limit } = {}) {
851
1103
  // pending so the next cycle picks it up, never into the negative cache.
852
1104
  if (aborted) { markTerm(ledger, entry.term, "pending", nowVal); break; }
853
1105
  if (!hit) {
1106
+ // Every configured KB source came back empty (or none is configured) —
1107
+ // try the corpus bands before giving up. They're another source the
1108
+ // resolver can reach, not a replacement for the KB walk above: a
1109
+ // headline's proper nouns still need a KB lookup, but its everyday
1110
+ // vocabulary is often already sitting in a band.
1111
+ const bandFacts = await groundTermFromBand(ctx, entry.term);
1112
+ if (bandFacts.length) {
1113
+ const res = await ingestBandFacts(ctx, bandFacts);
1114
+ factsTotal += res.facts;
1115
+ derivedTotal += res.derived;
1116
+ markTerm(ledger, entry.term, "grounded", nowVal);
1117
+ enriched.push(entry.term);
1118
+ continue;
1119
+ }
854
1120
  // No source was asked, so nothing learned this term is missing —
855
1121
  // the same posture a stop takes, for the same reason.
856
1122
  markTerm(ledger, entry.term, askedAnySource ? "missed" : "pending", nowVal);
@@ -54,7 +54,7 @@ export function sourceLabelFor(source) {
54
54
  const BANDS = {
55
55
  human: "human persona", "human-medium": "human persona", "human-large": "human persona",
56
56
  seon: "SEON ontology", conceptnet: "ConceptNet",
57
- "tier2-aws": "AWS", "tier2-python": "Python", "tier2-java": "Java", "wordnet-xl": "WordNet",
57
+ "wordnet-xl": "WordNet", namenet: "NameNet", child: "child vocabulary",
58
58
  };
59
59
  const key = (source && source.key) || "";
60
60
  if (key === "taught") return { label: "taught by telling", tone: "taught" };