@polycode-projects/the-mechanical-code-talker 6.0.17 → 6.0.19

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polycode-projects/the-mechanical-code-talker",
-  "version": "6.0.17",
+  "version": "6.0.19",
   "private": false,
   "type": "module",
   "description": "The Mechanical Code Talker (tmct) — a tolerant, offline, $0 chat surface that guides you toward precision queries about a software repository. ELIZA/PARRY-style but domain-obsessed with code. No model calls; indexes a repo on request (tmct index) or reads any producer's graph.",
@@ -120,6 +120,8 @@
     "probe:news-sources": "node scripts/probe-news-sources.mjs",
     "bench:news": "node scripts/news-bench/run.mjs --seed=xl",
     "bench:news:fast": "node scripts/news-bench/run.mjs --seed=fixture",
+    "bench:news:iterate": "node scripts/news-bench/iterate.mjs",
+    "bench:inputs": "node scripts/news-bench/ensure-bench-inputs.mjs",
     "check:links": "node scripts/check-links.mjs",
     "check:pii": "node scripts/pii-lint.mjs",
     "check:pack": "node scripts/check-pack-manifest.mjs",

package/src/adapters/corpus/research-source.mjs

@@ -77,9 +77,13 @@ export function isResearchSource(source) {
  *  `research:<source>:<folded term>`. memory/trust.mjs reads the `research:`
  *  prefix back as a referenceLive Source, so a live-fetched fact scores below
  *  every curated pack, and the source segment keeps which adapter fetched it
- *  readable off the fact itself. */
+ *  readable off the fact itself. A multi-word term's internal spaces fold to
+ *  underscores (ConceptNet's own /c/en/foo_bar convention, which normFactTerm
+ *  already unwinds on the way in) so the tag stays one whitespace-free token —
+ *  the same shape every other single-segment provenance tag carries, and safe
+ *  under a caller that reads only the tag's first whitespace-split word. */
 export function researchSourceTag(sourceName, term) {
-  return `research:${sourceName}:${normFactTerm(term)}`;
+  return `research:${sourceName}:${normFactTerm(term).replace(/ /g, "_")}`;
 }
 
 /** The facts a looked-up row licenses, each stamped with the source's own tag.

package/src/adapters/corpus/wikidata-live.mjs

@@ -1,5 +1,5 @@
-// corpus/wikidata-live.mjs — the live Wikidata research source. Three small
-// GET round trips against www.wikidata.org's Action API, mapped onto tmct's
+// corpus/wikidata-live.mjs — the live Wikidata research source. Small GET
+// round trips against www.wikidata.org's Action API, mapped onto tmct's
 // seed-ontology relations at THIS boundary and nowhere else: everything
 // downstream reads ordinary tmct facts and never learns the word "Wikidata".
 //
@@ -11,12 +11,17 @@
 // own rate policy and a SPARQL string built per term.
 //
 // The round trips:
-//   1. wbsearchentities — the item whose English label matches the term.
-//   2. wbgetentities    — that item's label, description, revision, claims.
-//   3. wbgetentities    — the English labels of the mapped claims' object
-//                         items, batched into ONE request. A claim's value is
-//                         a Q-id, and a stored fact's object has to be a human
-//                         term. Skipped when nothing mapped.
+//   1. wbsearchentities — every candidate item whose English label matches
+//                         the term, best fold first.
+//   2. wbgetentities    — a candidate's label, description, revision, claims.
+//   3. wbgetentities    — the English labels of that candidate's mapped
+//                         claims' object items, batched into ONE request. A
+//                         claim's value is a Q-id, and a stored fact's object
+//                         has to be a human term. Skipped when nothing
+//                         mapped.
+// Steps 2 and 3 repeat, candidate by candidate, only while the current one
+// turns out to be a media or document work sharing the term's name rather
+// than the term itself — the common case still costs three round trips.
 //
 // Courtesy is structural, mirroring wikipedia-live.mjs: one in-flight lookup
 // at a time, a minimum interval between round trips, a 429/maxlag cool-off
@@ -72,6 +77,29 @@ export const WIKIDATA_PROPERTY_RELATIONS = Object.freeze({
 // it would store from a prose lead sentence.
 const ISA_PROPERTIES = ["P279", "P31"];
 
+// A closed list of Wikidata item classes that name a media or document work,
+// not the everyday concept a term search asked for. A search on "canadian
+// companies" or "continents" can land on a paper or an album that merely
+// SHARES the term's name — Wikidata's own title match, not a definition.
+// Folded through normFactTerm the same way every isa term is, so the check
+// compares like with like. Each class is a Wikidata English label read
+// straight off the item, not a guess at one — refine this list from what
+// Wikidata actually returns, keep it named and small.
+const MEDIA_WORK_CLASSES = new Set([
+  "scholarly article",
+  "album",
+  "song",
+  "single",
+  "film",
+  "television series",
+  "television series episode",
+  "band",
+  "musical group",
+  "video game",
+  "book",
+  "novel",
+]);
+
 // How much of one item a single lookup reads: at most this many object values
 // per property, and this many facts in total. A busy item like "human" carries
 // hundreds of statements, and a research lookup wants the shape of the thing,
@@ -81,23 +109,27 @@ const MAX_FACTS_PER_ITEM = 12;
 
 const ITEM_ID_RE = /^Q[1-9][0-9]*$/;
 
-/** The searched item whose English label folds onto the key, or null — the
+/** Every candidate item whose English label folds onto the key, exact folds
+ *  first then prefix folds, each group in the search result's own order — the
  *  topic-drift guard, matching wikipedia-live.mjs's: "quasar" may resolve to
  *  "quasar" or "quasars", never to the first suggestion about something else.
  *  An exact fold beats a prefix fold wherever it appears in the result list,
  *  so a search that ranks "Quasars (album)" above "quasar" still lands on the
- *  term the caller asked for. */
-function matchingItemId(key, body) {
+ *  term the caller asked for first. Returning every candidate, not just the
+ *  best one, lets the caller step to the next title match when the best one
+ *  turns out to be a media work sharing the name. */
+function candidateItemIds(key, body) {
   const results = Array.isArray(body?.search) ? body.search : [];
-  let prefixMatch = null;
+  const exact = [];
+  const prefix = [];
   for (const hit of results) {
     const id = String(hit?.id ?? "");
     if (!ITEM_ID_RE.test(id)) continue;
     const folded = normFactTerm(hit?.label ?? "");
-    if (folded === key) return id;
-    if (prefixMatch === null && folded.startsWith(key)) prefixMatch = id;
+    if (folded === key) exact.push(id);
+    else if (folded.startsWith(key)) prefix.push(id);
   }
-  return prefixMatch;
+  return [...exact, ...prefix];
 }
 
 /** Every mapped claim on an entity as {predicate, id} pairs, capped per
@@ -178,6 +210,11 @@ export function createWikidataLiveProvider({
     return termById;
   }
 
+  /** The looked-up item, or null when every candidate that matched the
+   *  search either has no readable entity or turns out to be a media/document
+   *  work sharing the term's name — a media-class isa is a wrong identity, not
+   *  a definition, so the caller steps to the next title match instead of
+   *  accepting it. Exhausting every candidate this way is the term missing. */
   async function roundTrips(key) {
     const search = await gate.fetchJson(actionUrl({
       action: "wbsearchentities",
@@ -187,45 +224,49 @@ export function createWikidataLiveProvider({
       limit: "5",
       search: key,
     }));
-    const id = search ? matchingItemId(key, search) : null;
-    if (!id) return null;
+    const candidateIds = search ? candidateItemIds(key, search) : [];
 
-    const read = await gate.fetchJson(actionUrl({
-      action: "wbgetentities",
-      languages: "en",
-      props: "labels|descriptions|claims|info",
-      ids: id,
-    }));
-    const entity = read?.entities?.[id];
-    if (!entity) return null;
+    for (const id of candidateIds) {
+      const read = await gate.fetchJson(actionUrl({
+        action: "wbgetentities",
+        languages: "en",
+        props: "labels|descriptions|claims|info",
+        ids: id,
+      }));
+      const entity = read?.entities?.[id];
+      if (!entity) continue;
 
-    const claims = mappedClaims(entity);
-    const termById = await termsForIds(claims.map((c) => c.id));
-    const provenance = researchSourceTag(sourceName, key);
-    const facts = [];
-    const seen = new Set();
-    for (const claim of claims) {
-      const object = termById.get(claim.id);
-      if (!object || object === key || seen.has(`${claim.predicate}\0${object}`)) continue;
-      seen.add(`${claim.predicate}\0${object}`);
-      facts.push({ subject: key, predicate: claim.predicate, object, provenance });
-    }
+      const claims = mappedClaims(entity);
+      const termById = await termsForIds(claims.map((c) => c.id));
+      const isa = isaFrom(claims, termById, key);
+      if (isa && MEDIA_WORK_CLASSES.has(isa)) continue;
 
-    const description = String(entity.descriptions?.en?.value ?? "");
-    const row = {
-      term: key,
-      title: String(entity.labels?.en?.value ?? ""),
-      text: description,
-      summary: sentencesUpTo(description, SUMMARY_CHAR_CAP),
-      url: `${origin}/wiki/${id}`,
-      revid: Number(entity.lastrevid),
-      source: WIKIDATA_SOURCE_LABEL,
-      licence: WIKIDATA_LICENCE,
-    };
-    const isa = isaFrom(claims, termById, key);
-    if (isa) row.isa = isa;
-    if (facts.length) row.facts = facts;
-    return isResearchSourceRow(row) ? row : null;
+      const provenance = researchSourceTag(sourceName, key);
+      const facts = [];
+      const seen = new Set();
+      for (const claim of claims) {
+        const object = termById.get(claim.id);
+        if (!object || object === key || seen.has(`${claim.predicate}\0${object}`)) continue;
+        seen.add(`${claim.predicate}\0${object}`);
+        facts.push({ subject: key, predicate: claim.predicate, object, provenance });
+      }
+
+      const description = String(entity.descriptions?.en?.value ?? "");
+      const row = {
+        term: key,
+        title: String(entity.labels?.en?.value ?? ""),
+        text: description,
+        summary: sentencesUpTo(description, SUMMARY_CHAR_CAP),
+        url: `${origin}/wiki/${id}`,
+        revid: Number(entity.lastrevid),
+        source: WIKIDATA_SOURCE_LABEL,
+        licence: WIKIDATA_LICENCE,
+      };
+      if (isa) row.isa = isa;
+      if (facts.length) row.facts = facts;
+      if (isResearchSourceRow(row)) return row;
+    }
+    return null;
   }
 
   return {

package/src/adapters/memory/core.mjs

@@ -16,7 +16,7 @@
 
 import { access, mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
-import { proseTokensFor, buildProseIndex } from "../../domain/prose.mjs";
+import { proseTokensFor } from "../../domain/prose.mjs";
 import { fnv1aHex, normText, normFactTerm, normFactPredicate, factIdFor, factIdForTriple } from "../../domain/hash.mjs";
 
 // Fact identity (normalization + id derivation) lives in hash.mjs — the one
@@ -68,7 +68,7 @@ export {
 // fine as long as neither side READS an imported binding while the other's
 // body is still running — rows.mjs builds its class map on first use, and
 // every use here is inside a function.
-import { payloadToRows, rowsToPayload, diffRows, renormalizeAssembledPayload } from "./rows.mjs";
+import { payloadToRows, rowsToPayload, diffRows, renormalizeAssembledPayload, renormalizeProseIndex } from "./rows.mjs";
 
 // The rollup vocabulary and its tuning constants live with the compaction
 // layer; re-exported here so store consumers keep one import site.
@@ -235,6 +235,14 @@ export function isMemoryOrSqliteHandle(dir) {
   return isMemoryHandle(dir) || isSqliteHandle(dir) || isRowHandle(dir);
 }
 
+/** Move a store handle's write stamp on. `foldedFactRows` keys the fold it
+ *  holds to this number, so anything that changes what the store would fold to
+ *  — a landed write, a payload about to be mutated in place, a dropped
+ *  assembly, a seed assigned over the top — stamps here first. */
+function stampStoreWrite(dir) {
+  if (isMemoryOrSqliteHandle(dir)) dir.storeWrites = (dir.storeWrites || 0) + 1;
+}
+
 /** Backend B — pure in-memory store: `{ backend: "memory", payload }` held by
  *  the caller (never module-global). Zero file I/O; distinct from
  *  `--ephemeral`, which still round-trips a throwaway temp dir. */
@@ -250,7 +258,9 @@ export function createInMemoryStore() {
  *  `seedPayload` is null/undefined — a browser session with nothing to seed
  *  keeps its own fresh empty payload untouched. */
 export function applySeedPayload(memoryDir, seedPayload) {
-  if (seedPayload) memoryDir.payload = { ...memoryDir.payload, ...seedPayload };
+  if (!seedPayload) return;
+  memoryDir.payload = { ...memoryDir.payload, ...seedPayload };
+  stampStoreWrite(memoryDir);
 }
 
 /** A structurally independent copy of a memory payload — `structuredClone`
@@ -923,6 +933,7 @@ function mutablePayloadCopy(payload) {
  *  coherent graph. */
 function dropAssembledRowPayload(handle) {
   if (!isRowHandle(handle)) return;
+  stampStoreWrite(handle);
   handle.cachedPayload = null;
   handle.storedRows = null;
   handle.baseRows = null;
@@ -1271,7 +1282,12 @@ const cloneJson = (v) => (v === undefined ? v : structuredClone(v));
  *  persist would delete them as absent-from-payload. */
 function readSqlitePayload(handle) {
   const dataVersion = handle.db.prepare("PRAGMA data_version").get()?.data_version;
-  if (handle.cachedPayload && handle.cachedDataVersion !== dataVersion) handle.cachedPayload = null;
+  if (handle.cachedPayload && handle.cachedDataVersion !== dataVersion) {
+    // Another connection committed, so the fold this handle holds describes a
+    // store that no longer exists — it goes with the payload it was taken of.
+    stampStoreWrite(handle);
+    handle.cachedPayload = null;
+  }
   if (!handle.cachedPayload) {
     handle.cachedPayload = buildSqlitePayloadFromRows(handle);
     // The head index rides the same cache lifecycle as the payload it indexes,
@@ -2198,6 +2214,7 @@ function migrateFactAssertionKeys(payload) {
  *  SQL write (Backend C, persistSqlitePayload), or a diffed row write into an
  *  injected store (Backend D, persistRowPayload). */
 async function persistMemory(dir, payload) {
+  stampStoreWrite(dir);
   if (isMemoryHandle(dir)) { dir.payload = payload; return; }
   if (isSqliteHandle(dir)) { persistSqlitePayload(dir, payload); return; }
   if (isRowHandle(dir)) { await persistRowPayload(dir, payload); return; }
@@ -2372,12 +2389,16 @@ const memoryIndexOf = (payload) => payload?.[MEMORY_INDEX] || null;
 async function mutateMemory(dir, fn) {
   const overRowHandle = isRowHandle(dir);
   const payload = overRowHandle ? mutablePayloadCopy(await ensureRowPayload(dir)) : await loadMemory(dir);
+  // Stamped before `fn` runs as well as after it lands: Backend B's own
+  // `loadMemory` hands back the live payload, so the store stops matching any
+  // fold taken of it the moment `fn` starts changing it.
+  stampStoreWrite(dir);
   try {
     buildMemoryIndex(payload);
     const out = (await fn(payload)) ?? payload;
     migrateLegacyProvenance(out);
     recomputeSourceReliability(out);
-    if (!overRowHandle) out.proseIndex = buildProseIndex(out.individuals);
+    if (!overRowHandle) renormalizeProseIndex(out);
     await persistMemory(dir, out);
     return overRowHandle ? dir.cachedPayload : out;
   } catch (e) {
@@ -3867,6 +3888,33 @@ export function readFactRows(memory, opts = {}) {
   return foldFactRows(memory, factFoldContext(memory), opts);
 }
 
+/** `readFactRows` over a whole store, held between writes.
+ *
+ *  Folding the graph is the most expensive read tmct does, and the fold is a
+ *  pure function of the payload — so between two writes every caller asking for
+ *  it is asking the same question. This answers it once. `stampStoreWrite`
+ *  moves the stamp the held fold is keyed to, at both ends of `mutateMemory`
+ *  (the single seam every backend's writes pass through) and wherever else a
+ *  handle's payload is replaced or dropped, so a fold taken before a write can
+ *  never be served after one.
+ *
+ *  A repo-path dir has no handle to hold anything on, and another process can
+ *  write its file between two reads, so it folds fresh every call exactly as
+ *  before. */
+export async function foldedFactRows(dir) {
+  if (!isMemoryOrSqliteHandle(dir)) return readFactRows(await loadMemory(dir));
+  const stamp = dir.storeWrites || 0;
+  if (dir.heldFactRows && dir.heldFactRowsStamp === stamp) return dir.heldFactRows;
+  const rows = readFactRows(await loadMemory(dir));
+  // A write that landed while this fold was running has already moved the
+  // stamp; holding these rows would serve that write's own reader stale ones.
+  if ((dir.storeWrites || 0) === stamp) {
+    dir.heldFactRows = rows;
+    dir.heldFactRowsStamp = stamp;
+  }
+  return rows;
+}
+
 /** The fold itself, over whatever slice of the graph a context was built for.
  *  `readFactRows` hands it the whole graph; a caller that only needs certain
  *  (subject, predicate) pairs hands it a scoped context and gets exactly the