@polycode-projects/the-mechanical-code-talker 6.0.12 → 6.0.14

package/src/domain/memory/trust.mjs

@@ -107,6 +107,60 @@ function parsePeerNodeTagRest(rest) {
 const LIVE_REFERENCE_PACK = "wikipedia-live";
 const referenceKindFor = (pack) => (pack === LIVE_REFERENCE_PACK ? "referenceLive" : "reference");
 
+/** The part of a tag that names WHICH publication it came from, with the
+ *  per-item tail cut off: `news:nyt-world@item-91` names the NYT world feed,
+ *  and the item id beside it says which article, not which publisher. One feed
+ *  is one asserting party however many articles it runs, so the tail stays on
+ *  the tag for audit and out of the Source identity — the same split
+ *  `world:<name>:turnN` and `mud:<character>:turnN` already make. */
+const publicationKeyOf = (rest) => String(rest || "").split("@")[0];
+
+/** The same cut for a tag the fuzzy tier labels its own Source with: the feed
+ *  or the reference work, never the article. `news:nyt-world@item-91` folds to
+ *  `news:nyt-world`, `research:wikidata:otter` to `research:wikidata`, and the
+ *  research lane's `research:otter@2` to `research`. */
+function publicationSourceLabel(rest) {
+  if (rest.startsWith("news:")) return publicationKeyOf(rest);
+  const { pack } = researchTagToSource(rest.slice("research:".length));
+  return `research:${pack}`;
+}
+
+/** `teach:chat:ingest#<tag>@<ts>` — the ingest seam driving the chat teach lane
+ *  as a RECOGNIZER over a document. The party asserting is the document's own
+ *  publisher, which the embedded `<tag>` names, so the record lands on that
+ *  publisher's Source rather than on a chat session. Without it an ingest mints
+ *  a throwaway teach Source per SENTENCE, and one publication's sentences
+ *  corroborate each other for free. */
+export const INGEST_SESSION_MARKER = "ingest#";
+
+/** The two `research:` tag shapes, both live-fetched at query time and both
+ *  scored at the referenceLive prior, below every curated pack:
+ *    research:<source>:<term>   one KB adapter's own lookup (researchSourceTag)
+ *    research:<topic>@<depth>   the research lane's fan-out, which records how
+ *                               far it reached rather than which adapter answered
+ *  The `@` is what tells them apart — a folded term never carries one. Either
+ *  way the PACK names the reference work and the article segment names the page
+ *  inside it, so `sourceIdFor` can key one Source per work. */
+function researchTagToSource(rest) {
+  const at = rest.lastIndexOf("@");
+  if (at >= 0) return { kind: "referenceLive", pack: "research", article: rest.slice(0, at).trim() || "unknown" };
+  const colon = rest.indexOf(":");
+  if (colon < 0) return { kind: "referenceLive", pack: "research", article: rest.trim() || "unknown" };
+  return { kind: "referenceLive", pack: rest.slice(0, colon) || "research", article: rest.slice(colon + 1).trim() };
+}
+
+/** The publisher a chat-shaped tag stands in for, when its session slot carries
+ *  the ingest marker; null for an ordinary session, which is every tag a person
+ *  actually typed. One level only — an embedded chat tag is refused rather than
+ *  followed, so a hand-written tag cannot nest its way anywhere. */
+function ingestPublisherOf(chat) {
+  if (!chat.sessionId?.startsWith(INGEST_SESSION_MARKER)) return null;
+  const embedded = chat.sessionId.slice(INGEST_SESSION_MARKER.length);
+  if (embedded.startsWith("teach:") || embedded.startsWith("ace:")) return null;
+  const publisher = provenanceTagToSource(embedded);
+  return publisher ? { ...publisher, ...(chat.createdAt ? { createdAt: chat.createdAt } : {}) } : null;
+}
+
 export function provenanceTagToSource(tag) {
   const t = String(tag || "").trim();
   if (!t) return null;
@@ -119,17 +173,7 @@ export function provenanceTagToSource(tag) {
     const pack = rest.slice(0, colon) || "unknown";
     return { kind: referenceKindFor(pack), pack, article: rest.slice(colon + 1) };
   }
-  // research:<topic>@<depth> — the research lane's Simple English Wikipedia
-  // loads. Live-fetched at query time like the wikipedia-live pack, so it
-  // scores at the same referenceLive prior, below every curated pack. Parsed
-  // from the FULL tag (a topic may contain spaces); the depth segment records
-  // how far the fan-out reached and is not part of the Source identity.
-  if (t.startsWith("research:")) {
-    const rest = t.slice("research:".length);
-    const at = rest.lastIndexOf("@");
-    const topic = (at >= 0 ? rest.slice(0, at) : rest).trim();
-    return { kind: "referenceLive", pack: "research", article: topic || "unknown" };
-  }
+  if (t.startsWith("research:")) return researchTagToSource(t.slice("research:".length));
   const head = t.split(/\s+/)[0]; // drop trailing " /r/IsA" etc.
   if (head.startsWith("corpus-weak:")) return { kind: "corpusWeak", name: head.slice("corpus-weak:".length) || "unknown" };
   if (head.startsWith("corpus:")) return { kind: "corpus", name: head.slice("corpus:".length) || "unknown" };
@@ -148,7 +192,10 @@ export function provenanceTagToSource(tag) {
   // `mud:` prefix so `src:corpus:mud:<character>` can never collide with a
   // `world:<name>` Source that happens to share the literal name.
   if (head.startsWith("mud:")) return { kind: "corpus", name: `mud:${head.slice("mud:".length).split(":")[0] || "unknown"}` };
-  if (head.startsWith("ace:")) return { kind: "operator", ...parseChatTagRest(head.slice("ace:".length)) };
+  if (head.startsWith("ace:")) {
+    const chat = parseChatTagRest(head.slice("ace:".length));
+    return ingestPublisherOf(chat) || { kind: "operator", ...chat };
+  }
   if (head.startsWith("teach:")) {
     const rest = head.slice("teach:".length);
     // teach:peer:<name>#node:<id>@<ts> — a peer's own relabeled tag off the
@@ -156,7 +203,8 @@ export function provenanceTagToSource(tag) {
     const peerNode = parsePeerNodeTagRest(rest);
     if (peerNode) return peerNode;
     // the chat teach lane's natural frames — chat.mjs's teachProvenanceTag
-    return { kind: "teach", ...parseChatTagRest(rest) };
+    const chat = parseChatTagRest(rest);
+    return ingestPublisherOf(chat) || { kind: "teach", ...chat };
   }
   if (head.startsWith("web:")) return { kind: "web", url: head.slice("web:".length) };
   if (head.startsWith("url:")) return { kind: "web", url: head.slice("url:".length) };
@@ -172,10 +220,23 @@ export function provenanceTagToSource(tag) {
   // the bare extracted: fallback below, which still catches every other
   // extracted: caller unchanged. A bare news:<sourceId>@<itemId> tag (a
   // future caller that writes it directly, with no extracted: wrapper)
-  // scores the same way.
-  if (head.startsWith("extracted:news:")) return { kind: "web", url: head.slice("extracted:".length) };
-  if (head.startsWith("news:")) return { kind: "web", url: head };
-  if (head.startsWith("optimistic-extract:")) return { kind: "optimisticExtract", name: head.slice("optimistic-extract:".length) || "unknown" };
+  // scores the same way. One Source per FEED: the item id rides the tag for
+  // audit and stays out of the identity.
+  if (head.startsWith("extracted:news:")) return { kind: "web", url: publicationKeyOf(head.slice("extracted:".length)) };
+  if (head.startsWith("news:")) return { kind: "web", url: publicationKeyOf(head) };
+  // The same wrapper over a KB lookup's own tag: the reference work that
+  // answered is the asserting party, not the ingest run that read it.
+  if (head.startsWith("extracted:research:")) return researchTagToSource(head.slice("extracted:research:".length));
+  // The fuzzy tier keeps a Source of its OWN — a candidate the strict
+  // recognizer skipped must never corroborate a curated fact — but one per
+  // publication, not one per article.
+  if (head.startsWith("optimistic-extract:")) {
+    const rest = head.slice("optimistic-extract:".length);
+    const publication = rest.startsWith("news:") || rest.startsWith("research:")
+      ? publicationSourceLabel(rest)
+      : rest;
+    return { kind: "optimisticExtract", name: publication || "unknown" };
+  }
   if (head.startsWith("extracted:")) return { kind: "extracted", name: head.slice("extracted:".length) || "unknown" };
   if (head.startsWith("entailed:")) return { kind: "entailed", rule: head.slice("entailed:".length) };
   if (head.startsWith("chat:") || head.startsWith("session:") || head.startsWith("operator")) return { kind: "operator" };

package/src/services/chat.mjs

@@ -18487,14 +18487,15 @@ async function factRowSnapshot(memoryDir) {
   try { return readStoredFactRows(await loadMemoryStore(memoryDir)); } catch { return null; }
 }
 
-/** The Fact rows this turn wrote, diffed against the snapshot taken before it.
- *  Empty when the turn had no store to write to, or wrote nothing. */
+/** The Fact rows this turn wrote, diffed against the snapshot taken before it,
+ *  with the after-snapshot handed back beside them. Empty when the turn had no
+ *  store to write to, or wrote nothing. */
 async function factsTouchedSince(memoryDir, before) {
-  if (!before) return [];
+  if (!before) return { factsTouched: [], factRowsAfter: null };
   const after = await factRowSnapshot(memoryDir);
-  if (!after) return [];
+  if (!after) return { factsTouched: [], factRowsAfter: null };
   const { touchedFactRows } = await import("../domain/memory/touched-facts.mjs");
-  return touchedFactRows(before, after);
+  return { factsTouched: touchedFactRows(before, after), factRowsAfter: after };
 }
 
 /** A whole-line miss whose only problem is a closed filler clause in front of a
@@ -18519,12 +18520,19 @@ async function answerWithoutFillerPrefix(input, options, missed) {
 }
 
 /** Run one turn and report which Fact rows it wrote, as `factsTouched` beside
- *  the answer/record/logLines every caller already reads. The dispatch itself
- *  is dispatchTurn, below; this wrapper exists so the field lands on EVERY
- *  return path (dispatched, conversational, multi-sentence) from one place. */
+ *  the answer/record/logLines every caller already reads, with the after-fold
+ *  the diff was taken against as `factRowsAfter`. The dispatch itself is
+ *  dispatchTurn, below; this wrapper exists so the field lands on EVERY return
+ *  path (dispatched, conversational, multi-sentence) from one place.
+ *
+ *  `options.factRowsBefore` is the caller's own already-folded view of the
+ *  store, standing in for the before-snapshot. A caller running many turns over
+ *  one document folds once and threads it; folding a seed-sized graph again per
+ *  turn, to read back the rows the caller just handed over, is the most
+ *  expensive thing an ingest does. */
 export async function runTurn(input, options = {}) {
   const memoryDir = options?.memoryDir ?? null;
-  const before = await factRowSnapshot(memoryDir);
+  const before = options?.factRowsBefore || await factRowSnapshot(memoryDir);
   let result;
   try {
     result = await dispatchTurn(input, options);
@@ -18541,7 +18549,7 @@ export async function runTurn(input, options = {}) {
     });
   }
   if (!result || typeof result !== "object") return result;
-  return { ...result, factsTouched: await factsTouchedSince(memoryDir, before) };
+  return { ...result, ...(await factsTouchedSince(memoryDir, before)) };
 }
 
 async function dispatchTurn(input, { config, source = defaultSource, graph = null, focus = null, last = null, memoryDir = null, sessionId = "", env = process.env, lexicon = null, narrate = false, liveReference = false, researchSource = null, onLiveLookup = null, vocabHint = null, tel = null, biasByBundle = {}, factRowsCache: injectedFactRowsCache = null, planState = null, gameConfig = null, uiContext = "cli", synthesisBudget = AUTO_SYNTHESIS_BUDGET, researchState = null, researchConfig = null, newsState = null, newsConfig = null, newsProviders = null, discourse = null, _noSplit = false, actingSubject = "player", codeDomainActive = null, laneVocab = null, domainPacks = null, retrieval = null } = {}) {

package/src/services/extract-facts.mjs

@@ -63,9 +63,10 @@ import { basename, join, resolve } from "node:path";
 import { runTurn, uuidv7, stripLeadingDiscourseAdverb } from "./chat.mjs";
 import { beginsWithVowelSound, grammarRules } from "./finish.mjs";
 import { splitSentencesPreservingPaths, stripCitationResidue } from "./sentences.mjs";
-import { loadMemory, readFactRows, appendFact, removeFacts } from "../adapters/memory/core.mjs";
+import { loadMemory, readFactRows, appendFacts, removeFacts } from "../adapters/memory/core.mjs";
 import { loadConfig } from "../adapters/config.mjs";
 import { touchedFactRows } from "../domain/memory/touched-facts.mjs";
+import { INGEST_SESSION_MARKER } from "../domain/memory/trust.mjs";
 import { normFactTerm } from "../domain/hash.mjs";
 import { splitIdentifierWords } from "../domain/prose.mjs";
 import { winkInstance } from "../adapters/wink-model.mjs";
@@ -104,18 +105,22 @@ export function parseArgs(argv) {
  * on a browser-sized graph — so the caller threads one fold from sentence to
  * sentence instead of paying a fresh one per candidate.
  */
-async function runSentence(sentence, { config, memoryDir, env, beforeRows }) {
+async function runSentence(sentence, { config, memoryDir, env, beforeRows, sessionId = "" }) {
   const before = beforeRows || readFactRows(await loadMemory(memoryDir));
   if (ingestYield) await ingestYield();
-  const { record, answer } = await runTurn(sentence, { config, memoryDir, sessionId: uuidv7(), env });
+  // The turn takes the caller's fold as its own before-view and hands back the
+  // after-fold it already had to take, so one sentence costs one fold rather
+  // than three of the same graph.
+  const { record, answer, factsTouched, factRowsAfter } = await runTurn(sentence, {
+    config, memoryDir, sessionId: sessionId || uuidv7(), env, factRowsBefore: before,
+  });
+  const after = factRowsAfter || before;
   // Only an assert turn can have written a Fact, so only an assert turn earns
-  // the post-turn fold; every other turn hands the caller's own view straight
-  // back untouched.
+  // a fresh view; every other turn hands the caller's own straight back.
   if (record?.via !== "assert") return { recognized: false, rows: [], afterRows: before, decline: String(answer || "") };
   if (ingestYield) await ingestYield();
-  const after = readFactRows(await loadMemory(memoryDir));
   if (record?.miss) return { recognized: false, rows: [], afterRows: after, decline: String(answer || "") };
-  return { recognized: true, rows: touchedFactRows(before, after), afterRows: after };
+  return { recognized: true, rows: factsTouched || touchedFactRows(before, after), afterRows: after };
 }
 
 /** The recognizer's own words for why it turned a sentence down, when it named
@@ -800,6 +805,13 @@ function canonicalLines(facts, storeRows) {
  *                 are the only output.
  *     sourceTag   the label the audit provenance carries (extracted:<tag> /
  *                 optimistic-extract:<tag>). Default "text".
+ *     attributeToSource
+ *                 file the recognizer's own assertion under `sourceTag`'s
+ *                 publication instead of a fresh chat session per sentence.
+ *                 Off by default: an operator running `tmct extract` over their
+ *                 own notes IS the asserting party, so that lane keeps minting
+ *                 a session. A feed or a reference work is not, and one
+ *                 publication's sentences must never corroborate each other.
  *     optimistic  also run the fuzzy tier over strict-skipped sentences.
  *     canonical   include a `canonical` array: one enriched triple line per
  *                 ingested fact.
@@ -830,7 +842,12 @@ function canonicalLines(facts, storeRows) {
 export async function ingestText(text, {
   memoryDir = null, sourceTag = "text", optimistic = false,
   canonical = false, config = null, lexicon = null, observedAt = "", findings = false,
+  attributeToSource = false,
 } = {}) {
+  // The session id every sentence's recognizer turn runs under. Stable and
+  // derived from the publication when the caller attributes to it, so the whole
+  // run lands on one Source; a fresh uuid per sentence otherwise (runSentence).
+  const recognizerSessionId = attributeToSource ? `${INGEST_SESSION_MARKER}${sourceTag.split("@")[0]}` : "";
   // Paragraphs first (blank-line separated), so the pronoun carry never bridges
   // a topic break: a fresh paragraph clears the last-subject it would resolve
   // "they"/"it" against. Each paragraph then splits into sentences the shared
@@ -900,7 +917,7 @@ export async function ingestText(text, {
     if (ingestYield) await ingestYield();
     const knownIds = new Set(currentRows.map((r) => r.id));
     const { recognized, rows, afterRows, decline } = await runSentence(form, {
-      config: cfg, memoryDir: dir, env: runEnv, beforeRows: currentRows,
+      config: cfg, memoryDir: dir, env: runEnv, beforeRows: currentRows, sessionId: recognizerSessionId,
     });
     currentRows = afterRows;
     if (!recognized) { lastDecline = decline || lastDecline; return null; }
@@ -971,9 +988,14 @@ export async function ingestText(text, {
           const subjects = new Set(rows.map((r) => r.subject));
           if (subjects.size === 1) carrySubject = [...subjects][0];
           const tag = `extracted:${sourceTag}`;
+          // One write for the whole sentence, not one per row: a write reads
+          // and re-derives the whole graph, so N of them cost N times what one
+          // carrying the same N rows does. The batch stays inside the sentence
+          // — a later sentence still reads everything the earlier ones wrote.
+          const writes = [];
           for (const row of rows) {
             const extraction = findingsForRow(row, readingFindings, identifierTerms);
-            await appendFact(dir, {
+            writes.push({
               subject: row.subject, predicate: row.predicate, object: row.object,
               provenance: tag, quantifier: row.quantifier || "", observedAt,
               ...(extraction.length ? { extraction } : {}),
@@ -985,6 +1007,7 @@ export async function ingestText(text, {
             });
             taggedIds.add(row.id);
           }
+          await appendFacts(dir, writes);
           continue;
         }
         const ungrounded = ungroundedTermsIn(lastDecline);
@@ -1009,15 +1032,17 @@ export async function ingestText(text, {
         if (!keptCandidates.length) continue;
         optimisticSentences += 1;
         const tag = `optimistic-extract:${sourceTag}`;
+        const candidateWrites = [];
         for (const t of keptCandidates) {
           const extraction = findingsForRow(t, mintFindings.has(t) ? [mintFindings.get(t)] : [], identifierTerms);
-          const written = await appendFact(dir, {
+          candidateWrites.push({
             subject: t.subject, predicate: t.predicate, object: t.object, provenance: tag, observedAt,
             ...(extraction.length ? { extraction } : {}),
           });
           optimisticFacts.push({ ...t, provenance: tag, sentence, ...(extraction.length ? { extraction } : {}) });
-          taggedIds.add(written.id);
         }
+        const { ids } = await appendFacts(dir, candidateWrites);
+        for (const id of ids) taggedIds.add(id);
       }
     }
 

package/src/services/news.mjs

@@ -338,6 +338,7 @@ async function ingestSnapshotFacts(ctx, snapshot) {
   const sourceTag = `news:${snapshot.sourceId}@${snapshot.id}`;
   const result = await ingestText(text, {
     memoryDir, sourceTag, optimistic: true, lexicon: lex, observedAt: nowVal, findings: true,
+    attributeToSource: true,
   });
   invalidateCache(cache);
 
@@ -494,6 +495,27 @@ function mergeSnapshotsById(existing, incoming, cap) {
   return { items, added };
 }
 
+const fetchedAtMs = (snapshot) => {
+  const ms = toMs(snapshot?.fetchedAt);
+  return Number.isFinite(ms) ? ms : 0;
+};
+
+/** True once a snapshot has been through a grounding round. `mergeSnapshotsById`
+ *  files a snapshot the moment it arrives, so "known" and "grounded" are two
+ *  different states and only this one means the facts landed. */
+const isGroundedSnapshot = (snapshot) => (snapshot?.processedRounds || 0) > 0;
+
+/** One source's fetched-but-not-yet-grounded snapshots, oldest first. A cycle
+ *  that ran out of time leaves its backlog here, so the next cycle works
+ *  through that before anything newer and the same article is never ingested
+ *  twice. Ordered off the snapshots' own fields, so two cycles reading the same
+ *  state take the same work in the same order. */
+function pendingSnapshotsFor(items, sourceId) {
+  return (items || [])
+    .filter((snap) => snap?.sourceId === sourceId && !isGroundedSnapshot(snap))
+    .sort((a, b) => (fetchedAtMs(a) - fetchedAtMs(b)) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+}
+
 function emptyCycleAccumulator(at) {
   return { at, sentences: 0, recognized: 0, optimisticCount: 0, factsAdded: 0, termsResolved: 0, derived: 0 };
 }
@@ -548,21 +570,28 @@ export async function pollNewsSources(ctx) {
       perSource.push({ sourceId, status: "failed" });
       continue;
     }
+    let added = [];
     if (result.notModified) {
       recordSuccess(health, nowVal, "not-modified");
-      perSource.push({ sourceId, status: "not-modified" });
-      continue;
+    } else {
+      recordSuccess(health, nowVal, "ok");
+      const merged = mergeSnapshotsById(state.items, result.items, config.itemCap);
+      state.items = merged.items;
+      added = merged.added;
+      newItemsTotal += added.length;
     }
-    recordSuccess(health, nowVal, "ok");
-    const { items: mergedItems, added } = mergeSnapshotsById(state.items, result.items, config.itemCap);
-    state.items = mergedItems;
-    newItemsTotal += added.length;
 
+    // Everything this source has fetched and not yet grounded, not just what
+    // arrived on this fetch: a source that answers 304 still has a backlog to
+    // finish, and an aborted cycle's leftovers would otherwise sit in the state
+    // marked known and never be read again.
     const before = emptyCycleAccumulator(nowVal);
     const after = emptyCycleAccumulator(nowVal);
-    for (const snapshot of added) {
+    let grounded = 0;
+    for (const snapshot of pendingSnapshotsFor(state.items, sourceId)) {
       if (shouldAbort()) { aborted = true; break; }
       const r = await ingestNewsSnapshot(ctx, snapshot);
+      grounded += 1;
       after.sentences += r.sentences;
       after.recognized += r.recognized;
       after.optimisticCount += r.optimisticCount;
@@ -571,10 +600,17 @@ export async function pollNewsSources(ctx) {
       factsTotal += r.facts;
       derivedTotal += r.derived;
     }
-    if (added.length) {
+    if (grounded) {
       state.metrics = [...(state.metrics || []), cycleMetrics(before, after, { source: sourceId })];
     }
-    perSource.push({ sourceId, status: "ok", newItems: added.length });
+    const pendingLeft = pendingSnapshotsFor(state.items, sourceId).length;
+    perSource.push({
+      sourceId,
+      status: result.notModified ? "not-modified" : "ok",
+      newItems: added.length,
+      grounded,
+      pending: pendingLeft,
+    });
     if (aborted) break;
   }
 
@@ -627,6 +663,7 @@ async function ingestResearchArticle(ctx, term, provider, article) {
     ingested = await ingestText(prose, {
       memoryDir, sourceTag: provenance, optimistic: true,
       lexicon: lexicon || loadLexicon(), observedAt: resolveNow(now), findings: true,
+      attributeToSource: true,
     });
     invalidateCache(ctx.cache);
   }