@polycode-projects/the-mechanical-code-talker 6.0.12 → 6.0.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polycode-projects/the-mechanical-code-talker",
-  "version": "6.0.12",
+  "version": "6.0.13",
   "private": false,
   "type": "module",
   "description": "The Mechanical Code Talker (tmct) — a tolerant, offline, $0 chat surface that guides you toward precision queries about a software repository. ELIZA/PARRY-style but domain-obsessed with code. No model calls; indexes a repo on request (tmct index) or reads any producer's graph.",

package/src/adapters/memory/core.mjs

@@ -766,12 +766,22 @@ const ROW_NODE_ID_KEY = "nodeId";
  *  `wrapRowBackendOverSqliteSeed`, the entry point that takes one.
  *  `sqliteSeedOverlayRows` are rows that layer over that seed and under the
  *  session's own (a turn's retrieved corpus subgraph), read-only exactly as
- *  the seed is. */
+ *  the seed is.
+ *
+ *  `copyOnRead: false` hands every reader the assembled payload itself instead
+ *  of a copy of it. Copying a seed-sized payload is the most expensive thing
+ *  this backend does — over a second per read at 60k facts — and a caller that
+ *  drives the whole session through this one handle (a worker cycle, a request
+ *  handler) has nobody to protect the payload from. Anyone sharing a handle
+ *  across independent readers leaves the default alone: without the copy, a
+ *  reader that mutates what it read has changed the store's own view of
+ *  itself. */
 export function wrapRowBackend(impl, {
   basePayload = null,
   sqliteSeedStore = null,
   sqliteSeedOverlayRows = null,
   onOversizedRow = "throw",
+  copyOnRead = true,
   log = undefined,
 } = {}) {
   const problems = rowBackendProblems(impl);
@@ -795,6 +805,7 @@ export function wrapRowBackend(impl, {
     baseRows: null,
     storedRows: null,
     onOversizedRow,
+    copyOnRead,
     log,
   };
 }
@@ -805,8 +816,8 @@ export function wrapRowBackend(impl, {
  *  never materialized as a row array: the assembled payload is the only copy
  *  of it this process holds. `overlayRows` layer over the seed and under the
  *  session's own rows. */
-export function wrapRowBackendOverSqliteSeed(impl, sqliteSeedStore, { overlayRows = null, onOversizedRow = "throw", log = undefined } = {}) {
-  return wrapRowBackend(impl, { sqliteSeedStore, sqliteSeedOverlayRows: overlayRows, onOversizedRow, log });
+export function wrapRowBackendOverSqliteSeed(impl, sqliteSeedStore, { overlayRows = null, onOversizedRow = "throw", copyOnRead = true, log = undefined } = {}) {
+  return wrapRowBackend(impl, { sqliteSeedStore, sqliteSeedOverlayRows: overlayRows, onOversizedRow, copyOnRead, log });
 }
 
 /** Drain whatever `readRows()` returned: the contract allows an array or an
@@ -863,7 +874,7 @@ async function ensureRowPayload(handle) {
       prefixes: await readRowMeta(handle, ROW_META_PREFIXES_KEY, seedScalar(handle, "prefixes", empty.prefixes)),
     };
     if (handle.sqliteSeedStore) {
-      handle.cachedPayload = assembleSqliteSeededPayload(handle, meta);
+      handle.cachedPayload = migrateStoredMemory(assembleSqliteSeededPayload(handle, meta));
     } else {
       // "keep": the base overlay's own rows never reach the wire (persistRowPayload
       // excludes every seed key from every write via seedOnlyKeys below), so the
@@ -872,16 +883,49 @@ async function ensureRowPayload(handle) {
       // band's own high-fan-out property (one edge per fact is normal, not a
       // pathology) and break every read that depends on it.
       handle.baseRows = payloadToRows(handle.basePayload || empty, { onOversizedRow: "keep" });
-      handle.cachedPayload = rowsToPayload(overlayRows(handle.baseRows, handle.storedRows), { meta });
+      handle.cachedPayload = migrateStoredMemory(rowsToPayload(overlayRows(handle.baseRows, handle.storedRows), { meta }));
     }
   }
   return handle.cachedPayload;
 }
 
 /** loadMemory's read for Backend D: one `readRows()` per cold open, then a
- *  clone of the assembled payload per call. */
+ *  clone of the assembled payload per call — unless the handle was opened
+ *  `copyOnRead: false`, where the reader gets the assembled payload itself. */
 async function readRowPayload(handle) {
-  return cloneJson(await ensureRowPayload(handle));
+  const payload = await ensureRowPayload(handle);
+  return handle.copyOnRead === false ? payload : cloneJson(payload);
+}
+
+/** A payload a mutation can work on without any of it reaching the one it was
+ *  copied from. Every container a write reaches into is its own: the individuals
+ *  array and each individual, the edge groups and each group's example list.
+ *  What stays shared is what a write never changes in place — an individual's
+ *  `attributes` array is REPLACED by `setAttr`, never pushed onto, and an edge
+ *  is replaced rather than edited, so both sides keep reading their own.
+ *
+ *  This is `structuredClone`'s job done at the granularity writes actually use.
+ *  At seed scale the deep copy runs well over a second and a cycle pays it on
+ *  every fact it grounds; this is a few milliseconds of pointer copying. */
+function mutablePayloadCopy(payload) {
+  return {
+    ...payload,
+    individuals: (payload.individuals || []).map((ind) => ({ ...ind })),
+    objectProperties: (payload.objectProperties || []).map((group) => ({
+      ...group, examples: [...(group.examples || [])],
+    })),
+  };
+}
+
+/** Forget a row handle's assembled payload, so the next read rebuilds it from
+ *  the store. The one thing that must happen after a mutation dies part-way
+ *  through: the payload it was changing is neither what the store holds nor a
+ *  coherent graph. */
+function dropAssembledRowPayload(handle) {
+  if (!isRowHandle(handle)) return;
+  handle.cachedPayload = null;
+  handle.storedRows = null;
+  handle.baseRows = null;
 }
 
 /** A record with its audit stamp removed. `mgx:updatedAt` moves on every
@@ -935,9 +979,7 @@ async function persistRowPayload(handle, payload) {
     await handle.impl.putMeta(ROW_META_MEMORY_KEY, JSON.stringify(payload.memory ?? emptyMemory().memory));
     await handle.impl.putMeta(ROW_META_PREFIXES_KEY, JSON.stringify(payload.prefixes ?? emptyMemory().prefixes));
   } catch (e) {
-    handle.cachedPayload = null;
-    handle.storedRows = null;
-    handle.baseRows = null;
+    dropAssembledRowPayload(handle);
     throw e;
   }
   const removed = new Set(removals);
@@ -949,13 +991,13 @@ async function persistRowPayload(handle, payload) {
     patchAssembledPayload(handle.cachedPayload, meta, writes);
   } else if (handle.sqliteSeedStore) {
     handle.cachedPayload = null;
-    handle.cachedPayload = assembleSqliteSeededPayload(handle, meta);
+    handle.cachedPayload = migrateStoredMemory(assembleSqliteSeededPayload(handle, meta));
   } else {
     // Dropped before the rebuild, not after it: the payload this replaces is the
     // largest object the handle holds, and keeping it reachable while the next
     // one assembles doubles the peak for no reason.
     handle.cachedPayload = null;
-    handle.cachedPayload = rowsToPayload(overlayRows(handle.baseRows, handle.storedRows), { meta });
+    handle.cachedPayload = migrateStoredMemory(rowsToPayload(overlayRows(handle.baseRows, handle.storedRows), { meta }));
   }
 }
 
@@ -1951,7 +1993,9 @@ export async function snapshotMemory(dir, { retentionVersions } = {}) {
 export async function loadMemory(dir) {
   if (isMemoryHandle(dir)) return migrateStoredMemory(dir.payload);
   if (isSqliteHandle(dir)) return migrateStoredMemory(readSqlitePayload(dir));
-  if (isRowHandle(dir)) return migrateStoredMemory(await readRowPayload(dir));
+  // Not migrated here: a row handle migrates the payload once, as it assembles
+  // it, so every read after the first is spared two walks of the whole graph.
+  if (isRowHandle(dir)) return readRowPayload(dir);
   let text;
   try {
     text = await readFile(memoryGraphFile(dir), "utf8");
@@ -2315,15 +2359,31 @@ function retractionsFor(payload, groupId) {
  *  linear scan in that case. */
 const memoryIndexOf = (payload) => payload?.[MEMORY_INDEX] || null;
 
+/** Load, change, persist. A row handle always works on a copy here, even one
+ *  opened `copyOnRead: false`: a write drops every change that lands on a
+ *  seed-owned row, so a mutation applied straight to the assembled payload
+ *  would leave the handle holding changes the store refused. The copy is what
+ *  keeps "what this handle reads" and "what a fresh handle would assemble" the
+ *  same thing.
+ *
+ *  A row handle does skip the prose index built here: `persistRowPayload`
+ *  re-derives every derived structure from the rows it just wrote, so building
+ *  one now builds it twice over the whole graph. */
 async function mutateMemory(dir, fn) {
-  const payload = await loadMemory(dir);
-  buildMemoryIndex(payload);
-  const out = (await fn(payload)) ?? payload;
-  migrateLegacyProvenance(out);
-  recomputeSourceReliability(out);
-  out.proseIndex = buildProseIndex(out.individuals);
-  await persistMemory(dir, out);
-  return out;
+  const overRowHandle = isRowHandle(dir);
+  const payload = overRowHandle ? mutablePayloadCopy(await ensureRowPayload(dir)) : await loadMemory(dir);
+  try {
+    buildMemoryIndex(payload);
+    const out = (await fn(payload)) ?? payload;
+    migrateLegacyProvenance(out);
+    recomputeSourceReliability(out);
+    if (!overRowHandle) out.proseIndex = buildProseIndex(out.individuals);
+    await persistMemory(dir, out);
+    return overRowHandle ? dir.cachedPayload : out;
+  } catch (e) {
+    dropAssembledRowPayload(dir);
+    throw e;
+  }
 }
 
 const labelOf = (text) => (text.length > LABEL_CAP ? text.slice(0, LABEL_CAP - 1) + "…" : text);
@@ -2552,7 +2612,10 @@ function recomputeSourceReliability(payload) {
   if (!Array.isArray(payload?.individuals) || !Array.isArray(payload?.objectProperties)) return;
   const rows = readFactRows(payload); // Fact-only — contradiction accounting is inherently Fact-shaped
   const contradictedFactIds = new Set();
-  for (const group of findContradictions(payload)) for (const r of group) contradictedFactIds.add(r.id);
+  // The fold above is the same one findContradictions would take for itself,
+  // and over a seed-sized graph it is half a second. Nothing changes the
+  // payload between the two, so it goes across.
+  for (const group of findContradictions(payload, { factRows: rows })) for (const r of group) contradictedFactIds.add(r.id);
 
   const bySource = new Map(); // sessionSourceId -> { factsAsserted, factsContradicted }
   for (const row of rows) {
@@ -4195,8 +4258,8 @@ export const MULTI_VALUED_PREDICATES = MERGE_PREDICATES;
  *  only what its own clock could not order (the resolver's trust and codepoint
  *  tie-breaks — see resolveSiblingGroups), so ordinary succession stops reading
  *  as disagreement; every other predicate keeps the full keep-both contract. */
-export function findContradictions(memory, { floor = CONTRADICTION_TRUST_FLOOR } = {}) {
-  const rows = readFactRows(memory).filter((r) => r.trust >= floor);
+export function findContradictions(memory, { floor = CONTRADICTION_TRUST_FLOOR, factRows = null } = {}) {
+  const rows = (factRows || readFactRows(memory)).filter((r) => r.trust >= floor);
   const byKey = new Map();
   for (const r of rows) {
     if (resolutionStrategyFor(r.predicate) === RESOLUTION_MERGE) continue;

package/src/services/extract-facts.mjs

@@ -63,7 +63,7 @@ import { basename, join, resolve } from "node:path";
 import { runTurn, uuidv7, stripLeadingDiscourseAdverb } from "./chat.mjs";
 import { beginsWithVowelSound, grammarRules } from "./finish.mjs";
 import { splitSentencesPreservingPaths, stripCitationResidue } from "./sentences.mjs";
-import { loadMemory, readFactRows, appendFact, removeFacts } from "../adapters/memory/core.mjs";
+import { loadMemory, readFactRows, appendFacts, removeFacts } from "../adapters/memory/core.mjs";
 import { loadConfig } from "../adapters/config.mjs";
 import { touchedFactRows } from "../domain/memory/touched-facts.mjs";
 import { normFactTerm } from "../domain/hash.mjs";
@@ -971,9 +971,14 @@ export async function ingestText(text, {
           const subjects = new Set(rows.map((r) => r.subject));
           if (subjects.size === 1) carrySubject = [...subjects][0];
           const tag = `extracted:${sourceTag}`;
+          // One write for the whole sentence, not one per row: a write reads
+          // and re-derives the whole graph, so N of them cost N times what one
+          // carrying the same N rows does. The batch stays inside the sentence
+          // — a later sentence still reads everything the earlier ones wrote.
+          const writes = [];
           for (const row of rows) {
             const extraction = findingsForRow(row, readingFindings, identifierTerms);
-            await appendFact(dir, {
+            writes.push({
               subject: row.subject, predicate: row.predicate, object: row.object,
               provenance: tag, quantifier: row.quantifier || "", observedAt,
               ...(extraction.length ? { extraction } : {}),
@@ -985,6 +990,7 @@ export async function ingestText(text, {
             });
             taggedIds.add(row.id);
           }
+          await appendFacts(dir, writes);
           continue;
         }
         const ungrounded = ungroundedTermsIn(lastDecline);
@@ -1009,15 +1015,17 @@ export async function ingestText(text, {
         if (!keptCandidates.length) continue;
         optimisticSentences += 1;
         const tag = `optimistic-extract:${sourceTag}`;
+        const candidateWrites = [];
         for (const t of keptCandidates) {
           const extraction = findingsForRow(t, mintFindings.has(t) ? [mintFindings.get(t)] : [], identifierTerms);
-          const written = await appendFact(dir, {
+          candidateWrites.push({
             subject: t.subject, predicate: t.predicate, object: t.object, provenance: tag, observedAt,
             ...(extraction.length ? { extraction } : {}),
           });
           optimisticFacts.push({ ...t, provenance: tag, sentence, ...(extraction.length ? { extraction } : {}) });
-          taggedIds.add(written.id);
         }
+        const { ids } = await appendFacts(dir, candidateWrites);
+        for (const id of ids) taggedIds.add(id);
       }
     }
 

package/src/services/news.mjs

@@ -494,6 +494,27 @@ function mergeSnapshotsById(existing, incoming, cap) {
   return { items, added };
 }
 
+const fetchedAtMs = (snapshot) => {
+  const ms = toMs(snapshot?.fetchedAt);
+  return Number.isFinite(ms) ? ms : 0;
+};
+
+/** True once a snapshot has been through a grounding round. `mergeSnapshotsById`
+ *  files a snapshot the moment it arrives, so "known" and "grounded" are two
+ *  different states and only this one means the facts landed. */
+const isGroundedSnapshot = (snapshot) => (snapshot?.processedRounds || 0) > 0;
+
+/** One source's fetched-but-not-yet-grounded snapshots, oldest first. A cycle
+ *  that ran out of time leaves its backlog here, so the next cycle works
+ *  through that before anything newer and the same article is never ingested
+ *  twice. Ordered off the snapshots' own fields, so two cycles reading the same
+ *  state take the same work in the same order. */
+function pendingSnapshotsFor(items, sourceId) {
+  return (items || [])
+    .filter((snap) => snap?.sourceId === sourceId && !isGroundedSnapshot(snap))
+    .sort((a, b) => (fetchedAtMs(a) - fetchedAtMs(b)) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+}
+
 function emptyCycleAccumulator(at) {
   return { at, sentences: 0, recognized: 0, optimisticCount: 0, factsAdded: 0, termsResolved: 0, derived: 0 };
 }
@@ -548,21 +569,28 @@ export async function pollNewsSources(ctx) {
       perSource.push({ sourceId, status: "failed" });
       continue;
     }
+    let added = [];
     if (result.notModified) {
       recordSuccess(health, nowVal, "not-modified");
-      perSource.push({ sourceId, status: "not-modified" });
-      continue;
+    } else {
+      recordSuccess(health, nowVal, "ok");
+      const merged = mergeSnapshotsById(state.items, result.items, config.itemCap);
+      state.items = merged.items;
+      added = merged.added;
+      newItemsTotal += added.length;
     }
-    recordSuccess(health, nowVal, "ok");
-    const { items: mergedItems, added } = mergeSnapshotsById(state.items, result.items, config.itemCap);
-    state.items = mergedItems;
-    newItemsTotal += added.length;
 
+    // Everything this source has fetched and not yet grounded, not just what
+    // arrived on this fetch: a source that answers 304 still has a backlog to
+    // finish, and an aborted cycle's leftovers would otherwise sit in the state
+    // marked known and never be read again.
     const before = emptyCycleAccumulator(nowVal);
     const after = emptyCycleAccumulator(nowVal);
-    for (const snapshot of added) {
+    let grounded = 0;
+    for (const snapshot of pendingSnapshotsFor(state.items, sourceId)) {
       if (shouldAbort()) { aborted = true; break; }
       const r = await ingestNewsSnapshot(ctx, snapshot);
+      grounded += 1;
       after.sentences += r.sentences;
       after.recognized += r.recognized;
       after.optimisticCount += r.optimisticCount;
@@ -571,10 +599,17 @@ export async function pollNewsSources(ctx) {
       factsTotal += r.facts;
       derivedTotal += r.derived;
     }
-    if (added.length) {
+    if (grounded) {
       state.metrics = [...(state.metrics || []), cycleMetrics(before, after, { source: sourceId })];
     }
-    perSource.push({ sourceId, status: "ok", newItems: added.length });
+    const pendingLeft = pendingSnapshotsFor(state.items, sourceId).length;
+    perSource.push({
+      sourceId,
+      status: result.notModified ? "not-modified" : "ok",
+      newItems: added.length,
+      grounded,
+      pending: pendingLeft,
+    });
     if (aborted) break;
   }