@polycode-projects/the-mechanical-code-talker 6.0.13 → 6.0.15

package/src/domain/news-feed.mjs

@@ -240,6 +240,19 @@ const ENTITY_FRAGMENT_LEAD_WORDS = new Set([
 // term; "back" alone is a fine noun.
 const ENTITY_PARTICLE_LEAD_WORDS = new Set(["back", "up", "down", "out", "off", "away", "along", "around"]);
 
+// A pronoun points back at whatever the last clause named, so a multi-word
+// term opening with one is a clause the split lost the subject of. A term
+// ending in a bare auxiliary is the front half of one. Both mirror
+// extract-facts.mjs's own lexical rules, for the same reason the sets above do.
+const ENTITY_PRONOUN_LEAD_WORDS = new Set([
+  "i", "he", "she", "it", "we", "they", "you", "me", "him", "them", "us",
+  "his", "her", "its", "their", "our", "your", "my",
+]);
+const ENTITY_CLITIC_SUFFIX_RE = /['’](?:s|re|ve|ll|d|m)$/;
+const ENTITY_TRAILING_AUXILIARY_WORDS = new Set([
+  "is", "are", "was", "were", "be", "been", "being", "am", "has", "have", "had",
+]);
+
 /** Does `term` read as a thing's name rather than a clause fragment? Bounds
  *  the word count and rejects a leading conjunction, auxiliary or
  *  preposition (test E's condition 3, PLAN_NEWSWORTHINESS.md section 2), plus
@@ -251,7 +264,10 @@ function looksLikeEntityTerm(term) {
   if (words.length > ENTITY_TERM_MAX_WORDS) return false;
   const first = words[0].toLowerCase().replace(/^[^a-z0-9]+/, "");
   if (!first || ENTITY_FRAGMENT_LEAD_WORDS.has(first)) return false;
-  if (words.length > 1 && ENTITY_PARTICLE_LEAD_WORDS.has(first)) return false;
+  if (words.length === 1) return true;
+  if (ENTITY_PARTICLE_LEAD_WORDS.has(first)) return false;
+  if (ENTITY_PRONOUN_LEAD_WORDS.has(first.replace(ENTITY_CLITIC_SUFFIX_RE, ""))) return false;
+  if (ENTITY_TRAILING_AUXILIARY_WORDS.has(words[words.length - 1].toLowerCase())) return false;
   return true;
 }
 
@@ -425,21 +441,29 @@ export function buildTermAdjacency(rows) {
 }
 
 /** Breadth-first over subject/object adjacency from `hub`, exactly `hops`
- *  levels deep, then capped by content-addressed id — so the cap never
- *  depends on `rows`' own order, only on which rows the hop-bounded walk
- *  actually reaches. */
-export function subgraphAround(rows, hub, { hops = NEWS_HUB_HOPS, cap = 60, adjacency = null } = {}) {
+ *  levels deep, then capped: a `priorityIds` row first, then the nearer hop,
+ *  then content-addressed id. The cap never depends on `rows`' own order, only
+ *  on which rows the hop-bounded walk actually reaches and how far out each
+ *  one sits.
+ *
+ *  `priorityIds` is what keeps a card about a term the graph already knows
+ *  thousands of things about from being built out of an arbitrary slice of
+ *  them: a hub like "france" reaches far more rows than the cap, and the one
+ *  report that made it news would otherwise be the row that fell out. */
+export function subgraphAround(rows, hub, { hops = NEWS_HUB_HOPS, cap = 60, adjacency = null, priorityIds = null } = {}) {
   const adj = adjacency ?? buildTermAdjacency(rows);
   const hubTerm = normFactTerm(hub);
   const visited = new Set([hubTerm]);
   let frontier = [hubTerm];
   const collected = new Map();
+  const hopOf = new Map();
   for (let hop = 0; hop < hops; hop += 1) {
     const nextFrontier = new Set();
     for (const term of [...frontier].sort()) {
       for (const idx of adj.byTerm.get(term) ?? []) {
         const row = rows[idx];
         collected.set(row.id, row);
+        if (!hopOf.has(row.id)) hopOf.set(row.id, hop);
         const [s, o] = adj.terms[idx];
         if (!visited.has(s)) nextFrontier.add(s);
         if (!visited.has(o)) nextFrontier.add(o);
@@ -448,7 +472,12 @@ export function subgraphAround(rows, hub, { hops = NEWS_HUB_HOPS, cap = 60, adja
     for (const term of nextFrontier) visited.add(term);
     frontier = [...nextFrontier].sort();
   }
-  return [...collected.values()].sort(byId).slice(0, cap);
+  const isPriority = (id) => (priorityIds instanceof Set ? priorityIds.has(id) : Boolean(priorityIds?.includes?.(id)));
+  return [...collected.values()]
+    .sort((a, b) => (isPriority(b.id) - isPriority(a.id))
+      || (hopOf.get(a.id) - hopOf.get(b.id))
+      || byId(a, b))
+    .slice(0, cap);
 }
 
 /** The strongest prior kind among `rows`, for the item's trust chip — read
@@ -487,6 +516,28 @@ function joinWithAnd(items) {
 
 const IDENTITY_PREDICATES = new Set(["rdf:type", "rdfs:subClassOf"]);
 const SENTENCE_CAP = 5;
+// How many objects one sentence names before it counts the rest. A live source
+// reports the same relation over and over inside one window — every quake of
+// the day strikes near somewhere — and an unbounded list turns a card into a
+// wall of text.
+const OBJECTS_PER_SENTENCE = 6;
+
+function joinObjects(objects) {
+  if (objects.length <= OBJECTS_PER_SENTENCE) return joinWithAnd(objects);
+  const shown = objects.slice(0, OBJECTS_PER_SENTENCE);
+  return `${shown.join(", ")} and ${objects.length - OBJECTS_PER_SENTENCE} more`;
+}
+
+/** The predicates `rows` carry, curated-table order first and then whatever
+ *  is left, sorted. A relation minted from a source's own verb ("mgx:hit",
+ *  "mgx:strike-near") has no curated entry, and reading the table alone left
+ *  every card built from live headlines with an empty paragraph. */
+function predicatesInRenderOrder(rows) {
+  const present = new Set(rows.map((r) => r.predicate));
+  const curated = Object.keys(FACT_PREDICATE_PHRASES).filter((predicate) => present.has(predicate));
+  const rest = [...present].filter((predicate) => !Object.hasOwn(FACT_PREDICATE_PHRASES, predicate)).sort();
+  return [...curated, ...rest];
+}
 
 /** The fixed five-sentence paraphrase template (PLAN_NEWS_FEED.md section
  *  8.3): identity first, then the hub's own relations grouped by predicate in
@@ -519,17 +570,30 @@ export function renderNewsParagraph(hub, subgraphRows, { reportedIds = null } =
     .sort();
   if (identityObjects.length) {
     const withArticles = identityObjects.map((object) => `${articleFor(object)} ${object}`);
-    sentences.push(`${hub} is ${joinWithAnd(withArticles)}`);
+    sentences.push(`${hub} is ${joinObjects(withArticles)}`);
   }
 
-  for (const predicate of Object.keys(FACT_PREDICATE_PHRASES)) {
+  for (const predicate of predicatesInRenderOrder(reportedHubRows)) {
     if (IDENTITY_PREDICATES.has(predicate) || sentences.length >= SENTENCE_CAP) continue;
     const objects = reportedHubRows
       .filter((r) => r.predicate === predicate)
       .map((r) => r.object)
       .sort();
     if (!objects.length) continue;
-    sentences.push(`${hub} ${predicatePhrase(predicate)} ${joinWithAnd(objects)}`);
+    sentences.push(`${hub} ${predicatePhrase(predicate)} ${joinObjects(objects)}`);
+  }
+
+  // A hub that only ever appears as an OBJECT — the place a quake struck, the
+  // story a site discussed — has no subject-side row to build a sentence from,
+  // and its card came out blank. What was reported about it still says
+  // something, so those rows render whole, subject and all.
+  if (!sentences.length) {
+    const aboutHub = subgraphRows
+      .filter((r) => normFactTerm(r.object) === hubTerm && normFactTerm(r.subject) !== hubTerm && isReported(r.id))
+      .sort(byId)
+      .slice(0, OBJECTS_PER_SENTENCE)
+      .map((r) => factSentence(r));
+    if (aboutHub.length) sentences.push(aboutHub.join("; "));
   }
 
   if (sentences.length < SENTENCE_CAP && secondHopRows.length) {
@@ -564,7 +628,7 @@ export function buildNewsItems(rows, { now, windowMs, limit = 6, sourcesByFactId
   if (readsAsEntityTerm) hubOptions.readsAsEntityTerm = readsAsEntityTerm;
   const hubs = newsworthyHubs(rows, reported, hubOptions);
   const items = hubs.map(({ term, changed }) => {
-    const subgraphRows = subgraphAround(rows, term, { adjacency });
+    const subgraphRows = subgraphAround(rows, term, { adjacency, priorityIds: reportedIds });
     const factIds = subgraphRows.map((r) => r.id).sort();
     const { background } = splitCardRows(subgraphRows, reportedIds);
     return {

package/src/domain/term-ledger.mjs

@@ -8,6 +8,33 @@ import { normFactTerm } from "./hash.mjs";
 
 const ITEM_IDS_CAP = 12;
 
+// A preposition, conjunction or degree/frequency adverb scaffolds a
+// sentence; it never names the thing the sentence is about, so no
+// occurrence count makes one a useful ledger term. Closed set, checked
+// against the already-normalized (lowercased) term. bumpTerms uses this to
+// keep a function word from ever being admitted; ledgerFromPayload uses it
+// to drop one that reached a persisted payload before this filter existed.
+const FUNCTION_WORD_TERMS = new Set([
+  // prepositions
+  "about", "above", "across", "after", "against", "along", "among", "around",
+  "at", "before", "behind", "below", "beneath", "beside", "between", "beyond",
+  "by", "despite", "down", "during", "except", "for", "from", "in", "into",
+  "near", "of", "off", "on", "onto", "out", "over", "since", "through",
+  "throughout", "to", "toward", "towards", "under", "underneath", "until",
+  "up", "upon", "with", "within", "without",
+  // conjunctions
+  "and", "or", "nor", "but", "so", "yet", "although", "because", "if",
+  "though", "unless", "when", "whenever", "whereas", "while", "than",
+  // degree and frequency adverbs
+  "very", "quite", "rather", "too", "just", "only", "even", "also", "still",
+  "already", "almost", "always", "never", "ever", "often", "sometimes",
+  "usually", "indeed", "however", "therefore", "thus", "hence", "meanwhile",
+]);
+
+function isFunctionWordTerm(term) {
+  return FUNCTION_WORD_TERMS.has(term);
+}
+
 /** Ledger entry field order fixed once here so `ledgerPayload` serializes
  *  byte-identically across peers regardless of insertion order elsewhere. */
 function newEntry(term, vocabGrounded, now) {
@@ -35,7 +62,7 @@ export function createTermLedger() {
 export function bumpTerms(ledger, termCounts, itemId, now, vocabGroundedByTerm = new Map()) {
   for (const [rawTerm, occurrences] of termCounts) {
     const term = normFactTerm(rawTerm);
-    if (!term) continue;
+    if (!term || isFunctionWordTerm(term)) continue;
     let entry = ledger.terms.get(term);
     if (!entry) {
       const vocabGrounded = vocabGroundedByTerm.has(rawTerm)
@@ -112,6 +139,7 @@ export function ledgerPayload(ledger) {
 export function ledgerFromPayload(payload) {
   const ledger = createTermLedger();
   for (const entry of payload?.terms ?? []) {
+    if (isFunctionWordTerm(entry.term)) continue;
     ledger.terms.set(entry.term, { ...entry, itemIds: [...(entry.itemIds ?? [])] });
   }
   return ledger;

package/src/services/chat.mjs

@@ -18487,14 +18487,15 @@ async function factRowSnapshot(memoryDir) {
   try { return readStoredFactRows(await loadMemoryStore(memoryDir)); } catch { return null; }
 }
 
-/** The Fact rows this turn wrote, diffed against the snapshot taken before it.
- *  Empty when the turn had no store to write to, or wrote nothing. */
+/** The Fact rows this turn wrote, diffed against the snapshot taken before it,
+ *  with the after-snapshot handed back beside them. Empty when the turn had no
+ *  store to write to, or wrote nothing. */
 async function factsTouchedSince(memoryDir, before) {
-  if (!before) return [];
+  if (!before) return { factsTouched: [], factRowsAfter: null };
   const after = await factRowSnapshot(memoryDir);
-  if (!after) return [];
+  if (!after) return { factsTouched: [], factRowsAfter: null };
   const { touchedFactRows } = await import("../domain/memory/touched-facts.mjs");
-  return touchedFactRows(before, after);
+  return { factsTouched: touchedFactRows(before, after), factRowsAfter: after };
 }
 
 /** A whole-line miss whose only problem is a closed filler clause in front of a
@@ -18519,12 +18520,19 @@ async function answerWithoutFillerPrefix(input, options, missed) {
 }
 
 /** Run one turn and report which Fact rows it wrote, as `factsTouched` beside
- *  the answer/record/logLines every caller already reads. The dispatch itself
- *  is dispatchTurn, below; this wrapper exists so the field lands on EVERY
- *  return path (dispatched, conversational, multi-sentence) from one place. */
+ *  the answer/record/logLines every caller already reads, with the after-fold
+ *  the diff was taken against as `factRowsAfter`. The dispatch itself is
+ *  dispatchTurn, below; this wrapper exists so the field lands on EVERY return
+ *  path (dispatched, conversational, multi-sentence) from one place.
+ *
+ *  `options.factRowsBefore` is the caller's own already-folded view of the
+ *  store, standing in for the before-snapshot. A caller running many turns over
+ *  one document folds once and threads it; folding a seed-sized graph again per
+ *  turn, to read back the rows the caller just handed over, is the most
+ *  expensive thing an ingest does. */
 export async function runTurn(input, options = {}) {
   const memoryDir = options?.memoryDir ?? null;
-  const before = await factRowSnapshot(memoryDir);
+  const before = options?.factRowsBefore || await factRowSnapshot(memoryDir);
   let result;
   try {
     result = await dispatchTurn(input, options);
@@ -18541,7 +18549,7 @@ export async function runTurn(input, options = {}) {
     });
   }
   if (!result || typeof result !== "object") return result;
-  return { ...result, factsTouched: await factsTouchedSince(memoryDir, before) };
+  return { ...result, ...(await factsTouchedSince(memoryDir, before)) };
 }
 
 async function dispatchTurn(input, { config, source = defaultSource, graph = null, focus = null, last = null, memoryDir = null, sessionId = "", env = process.env, lexicon = null, narrate = false, liveReference = false, researchSource = null, onLiveLookup = null, vocabHint = null, tel = null, biasByBundle = {}, factRowsCache: injectedFactRowsCache = null, planState = null, gameConfig = null, uiContext = "cli", synthesisBudget = AUTO_SYNTHESIS_BUDGET, researchState = null, researchConfig = null, newsState = null, newsConfig = null, newsProviders = null, discourse = null, _noSplit = false, actingSubject = "player", codeDomainActive = null, laneVocab = null, domainPacks = null, retrieval = null } = {}) {

package/src/services/extract-facts.mjs

@@ -66,6 +66,7 @@ import { splitSentencesPreservingPaths, stripCitationResidue } from "./sentences
 import { loadMemory, readFactRows, appendFacts, removeFacts } from "../adapters/memory/core.mjs";
 import { loadConfig } from "../adapters/config.mjs";
 import { touchedFactRows } from "../domain/memory/touched-facts.mjs";
+import { INGEST_SESSION_MARKER } from "../domain/memory/trust.mjs";
 import { normFactTerm } from "../domain/hash.mjs";
 import { splitIdentifierWords } from "../domain/prose.mjs";
 import { winkInstance } from "../adapters/wink-model.mjs";
@@ -104,18 +105,22 @@ export function parseArgs(argv) {
  * on a browser-sized graph — so the caller threads one fold from sentence to
  * sentence instead of paying a fresh one per candidate.
  */
-async function runSentence(sentence, { config, memoryDir, env, beforeRows }) {
+async function runSentence(sentence, { config, memoryDir, env, beforeRows, sessionId = "" }) {
   const before = beforeRows || readFactRows(await loadMemory(memoryDir));
   if (ingestYield) await ingestYield();
-  const { record, answer } = await runTurn(sentence, { config, memoryDir, sessionId: uuidv7(), env });
+  // The turn takes the caller's fold as its own before-view and hands back the
+  // after-fold it already had to take, so one sentence costs one fold rather
+  // than three of the same graph.
+  const { record, answer, factsTouched, factRowsAfter } = await runTurn(sentence, {
+    config, memoryDir, sessionId: sessionId || uuidv7(), env, factRowsBefore: before,
+  });
+  const after = factRowsAfter || before;
   // Only an assert turn can have written a Fact, so only an assert turn earns
-  // the post-turn fold; every other turn hands the caller's own view straight
-  // back untouched.
+  // a fresh view; every other turn hands the caller's own straight back.
   if (record?.via !== "assert") return { recognized: false, rows: [], afterRows: before, decline: String(answer || "") };
   if (ingestYield) await ingestYield();
-  const after = readFactRows(await loadMemory(memoryDir));
   if (record?.miss) return { recognized: false, rows: [], afterRows: after, decline: String(answer || "") };
-  return { recognized: true, rows: touchedFactRows(before, after), afterRows: after };
+  return { recognized: true, rows: factsTouched || touchedFactRows(before, after), afterRows: after };
 }
 
 /** The recognizer's own words for why it turned a sentence down, when it named
@@ -668,6 +673,32 @@ const FRAGMENT_LEAD_TAGS = new Set(["VERB", "AUX", "ADP", "CCONJ", "SCONJ", "PAR
 // which names nothing. Read lexically so a checkout with no wink model catches
 // it too, and only for a multi-word term — "back" alone is a fine noun.
 const PARTICLE_LEAD_WORDS = new Set(["back", "up", "down", "out", "off", "away", "along", "around"]);
+// A pronoun names nothing on its own — it points back at whatever the last
+// clause named — so a multi-word term opening with one is a clause the split
+// lost the subject of ("he's also destroyed the city's soul"), never a name. A
+// one-word term is exempt for the same reason the particle rule exempts one:
+// "us" is also how a headline writes the United States.
+const PRONOUN_LEAD_WORDS = new Set([
+  "i", "he", "she", "it", "we", "they", "you", "me", "him", "them", "us",
+  "his", "her", "its", "their", "our", "your", "my",
+]);
+const CLITIC_SUFFIX_RE = /['’](?:s|re|ve|ll|d|m)$/;
+// A term ending in a bare auxiliary is the front half of a clause the split cut
+// ("rooms were"), never the whole of a name.
+const TRAILING_AUXILIARY_WORDS = new Set([
+  "is", "are", "was", "were", "be", "been", "being", "am", "has", "have", "had",
+]);
+// A compass word opening a place name is a modifier, not a clause lead —
+// "north korea", "south sandwich islands". A tagger reading the LOWERCASED
+// term has no capital left to tell the place from the direction and tags
+// "north"/"south" as an adverb, so the POS rule below would turn every one of
+// them down. Followed by "of" the word really is heading a prepositional
+// phrase ("north of the border"), and that stays declined.
+const COMPASS_LEAD_WORDS = new Set([
+  "north", "south", "east", "west",
+  "northeast", "northwest", "southeast", "southwest",
+  "northern", "southern", "eastern", "western",
+]);
 
 /** Does `term` read as a thing's name rather than a clause fragment? Bounds
  *  the word count and rejects a leading conjunction, auxiliary, preposition,
@@ -684,6 +715,9 @@ export function readsAsEntityTerm(term, nlp) {
   if (FRAGMENT_LEAD_WORDS.has(first)) return false;
   if (words.length === 1) return true;
   if (PARTICLE_LEAD_WORDS.has(first)) return false;
+  if (PRONOUN_LEAD_WORDS.has(first.replace(CLITIC_SUFFIX_RE, ""))) return false;
+  if (TRAILING_AUXILIARY_WORDS.has(words[words.length - 1].toLowerCase())) return false;
+  if (COMPASS_LEAD_WORDS.has(first) && words[1].toLowerCase() !== "of") return true;
   const engine = nlp === undefined ? winkInstance() : nlp;
   if (!engine) return true;
   try {
@@ -800,6 +834,13 @@ function canonicalLines(facts, storeRows) {
  *                 are the only output.
  *     sourceTag   the label the audit provenance carries (extracted:<tag> /
  *                 optimistic-extract:<tag>). Default "text".
+ *     attributeToSource
+ *                 file the recognizer's own assertion under `sourceTag`'s
+ *                 publication instead of a fresh chat session per sentence.
+ *                 Off by default: an operator running `tmct extract` over their
+ *                 own notes IS the asserting party, so that lane keeps minting
+ *                 a session. A feed or a reference work is not, and one
+ *                 publication's sentences must never corroborate each other.
  *     optimistic  also run the fuzzy tier over strict-skipped sentences.
  *     canonical   include a `canonical` array: one enriched triple line per
  *                 ingested fact.
@@ -830,7 +871,12 @@ function canonicalLines(facts, storeRows) {
 export async function ingestText(text, {
   memoryDir = null, sourceTag = "text", optimistic = false,
   canonical = false, config = null, lexicon = null, observedAt = "", findings = false,
+  attributeToSource = false,
 } = {}) {
+  // The session id every sentence's recognizer turn runs under. Stable and
+  // derived from the publication when the caller attributes to it, so the whole
+  // run lands on one Source; a fresh uuid per sentence otherwise (runSentence).
+  const recognizerSessionId = attributeToSource ? `${INGEST_SESSION_MARKER}${sourceTag.split("@")[0]}` : "";
   // Paragraphs first (blank-line separated), so the pronoun carry never bridges
   // a topic break: a fresh paragraph clears the last-subject it would resolve
   // "they"/"it" against. Each paragraph then splits into sentences the shared
@@ -900,7 +946,7 @@ export async function ingestText(text, {
     if (ingestYield) await ingestYield();
     const knownIds = new Set(currentRows.map((r) => r.id));
     const { recognized, rows, afterRows, decline } = await runSentence(form, {
-      config: cfg, memoryDir: dir, env: runEnv, beforeRows: currentRows,
+      config: cfg, memoryDir: dir, env: runEnv, beforeRows: currentRows, sessionId: recognizerSessionId,
     });
     currentRows = afterRows;
     if (!recognized) { lastDecline = decline || lastDecline; return null; }

package/src/services/news.mjs

@@ -338,6 +338,7 @@ async function ingestSnapshotFacts(ctx, snapshot) {
   const sourceTag = `news:${snapshot.sourceId}@${snapshot.id}`;
   const result = await ingestText(text, {
     memoryDir, sourceTag, optimistic: true, lexicon: lex, observedAt: nowVal, findings: true,
+    attributeToSource: true,
   });
   invalidateCache(cache);
 
@@ -662,6 +663,7 @@ async function ingestResearchArticle(ctx, term, provider, article) {
     ingested = await ingestText(prose, {
       memoryDir, sourceTag: provenance, optimistic: true,
       lexicon: lexicon || loadLexicon(), observedAt: resolveNow(now), findings: true,
+      attributeToSource: true,
     });
     invalidateCache(ctx.cache);
   }