@polycode-projects/the-mechanical-code-talker 3.0.0 → 3.0.2

package/README.md

@@ -802,6 +802,17 @@ self-contained HTML file you can open in a browser:
                                rows; --term resolves via the same normalization chat uses.
 ```
 
+`tmct digest <term>` turns what the graph knows about one term into a short, readable
+paragraph — the vocabulary-side sibling of `tmct cli digest`'s code map. The narrative
+leads, its sources follow, and the full fact count points at the ledger for the rest:
+
+```output:help:digest
+  tmct digest <term>           a readable digest of what the graph knows about one term:
+       [--repo <abs>]          a bounded narrative first (selected, sense-filtered,
+       [--graph <path>]        deduped), then its sources and the stored-fact count.
+       [--config <path>]       The vocabulary-side sibling of `cli digest`'s code map.
+```
+
 `tmct serve` runs an Anthropic Messages API-compatible HTTP endpoint over the graph,
 so a tool-loop client can call tmct like a model, at $0:
 

package/bin/tmct.mjs

@@ -920,6 +920,43 @@ async function main() {
     // on disk: one copy-paste Bash invocation per tool, rewritten on every init.
     process.stdout.write(`cold-tool catalog: ${await writeToolsCatalog(repoRoot)}\n`);
 
+    // `--with-persona code`: on top of the corpus-vocabulary bias `initRepo` just wrote,
+    // also run the repository INDEXER (`tmct index`'s own machinery) against this repo's
+    // real source, so one command produces a `.tmct/graph.json` backed by the repo itself —
+    // not just a bias preset. `chat --repo` already reads whatever graph is on disk; this
+    // is the onboarding path that puts one there. Failure-tolerant like the corpus seed
+    // above: a repo that can't be indexed (no supported source, or a parse error) degrades
+    // to an initialized-but-graphless repo, never a crashed init.
+    if (personaName === "code") {
+      try {
+        const { indexRepository } = await import("../src/index/index-repo.mjs");
+        const stats = await indexRepository(repoRoot);
+        for (const { pass, message } of stats.gitErrors || []) {
+          process.stderr.write(`tmct init: WARNING git history pass '${pass}' — ${message} (graph built without those edges)\n`);
+        }
+        const perLang = Object.entries(stats.perLang)
+          .map(([lang, s]) => `${lang}: ${s.modules} modules, ${s.symbols} symbols`).join("; ");
+        const kib = (stats.bytes / 1024).toFixed(1);
+        process.stdout.write(
+          `code persona: indexed the repo — wrote ${stats.graphFile} (${stats.modules} modules, ${stats.symbols} symbols; ${kib} KiB)\n`
+          + (perLang ? `${perLang}\n` : "no supported source found under the repo\n"),
+        );
+        // The capability the graph just unlocked, said out loud — a graphless
+        // repo (no supported source) gets no such promise.
+        if (stats.modules > 0) {
+          process.stdout.write(
+            `indexed ${stats.modules} modules (${stats.symbols} symbols) — code questions now work in \`tmct chat\`: `
+            + `try "which modules import <path>" or "what does <module> do".\n`,
+          );
+        }
+        if (stats.failures?.length) {
+          process.stderr.write(`tmct init: ${stats.failures.length} file(s) failed to parse (skipped): ${stats.failures.slice(0, 5).join(", ")}${stats.failures.length > 5 ? ", …" : ""}\n`);
+        }
+      } catch (e) {
+        process.stderr.write(`tmct init: code persona indexing skipped (${e?.message || e})\n`);
+      }
+    }
+
     // `--corpus`/`--ontology`/`--lexicon` now mean "activate this bundle and
     // PERSIST that into tmct.toml" — so a second `tmct init` (or the next chat
     // bootstrap) remembers the choice, unlike the old ad hoc path, which had
@@ -1011,6 +1048,14 @@ async function main() {
       `tmct index — wrote ${stats.graphFile} (${stats.modules} modules, ${stats.symbols} symbols; ${kib} KiB)\n`
       + (perLang ? `${perLang}\n` : "no supported source found under the repo\n"),
     );
+    // Same discoverability line the code-persona init prints: the graph just
+    // unlocked code questions, so say so — but never over an empty graph.
+    if (stats.modules > 0) {
+      process.stdout.write(
+        `indexed ${stats.modules} modules (${stats.symbols} symbols) — code questions now work in \`tmct chat\`: `
+        + `try "which modules import <path>" or "what does <module> do".\n`,
+      );
+    }
     if (stats.failures?.length) {
       process.stderr.write(`tmct index: ${stats.failures.length} file(s) failed to parse (skipped): ${stats.failures.slice(0, 5).join(", ")}${stats.failures.length > 5 ? ", …" : ""}\n`);
     }
@@ -1283,6 +1328,59 @@ async function main() {
     return;
   }
 
+  if (mode === "digest") {
+    // `tmct digest <term>` — the vocabulary-side digest: turn what the graph
+    // knows about one term into a bounded, readable paragraph, deterministically
+    // (src/domain/digest, wired through corpus/digest-bank.mjs), beside the
+    // code-side `tmct cli digest`. Same repo/backend resolution as `viz`.
+    const rest = process.argv.slice(3);
+    const term = rest.find((a) => !a.startsWith("-"));
+    if (!term) {
+      process.stderr.write("tmct digest — name a term: `tmct digest <term>`\n");
+      process.exitCode = 1;
+      return;
+    }
+    const { resolveRuntimeConfig } = await import("../src/services/cli-args.mjs");
+    const { openMemoryBackend, loadMemory, readFactRows, normFactTerm } = await import("../src/adapters/memory/core.mjs");
+    const { digestTermFromRows } = await import("../src/adapters/corpus/digest-bank.mjs");
+    const { repo, toml } = await resolveRuntimeConfig({ argv: rest });
+    const backendChoice = String(process.env.TMCT_MEMORY_BACKEND || toml?.memory?.backend || "").trim().toLowerCase();
+    const { dir: memoryDir, close: closeMemoryStore } = await openMemoryBackend(repo, backendChoice);
+    let rows;
+    try { rows = readFactRows(await loadMemory(memoryDir)); } finally { await closeMemoryStore(); }
+    // Fold a naive plural to its base on BOTH sides so "aardvark" reaches a
+    // stored "aardvarks" and vice versa — the store keeps whichever number the
+    // teaching sentence used.
+    const baseTerm = (s) => {
+      const x = normFactTerm(s);
+      if (x.endsWith("es")) return x.slice(0, -2);
+      if (x.endsWith("s")) return x.slice(0, -1);
+      return x;
+    };
+    const wantedBase = baseTerm(term);
+    const termRows = rows.filter((r) => baseTerm(r.subject) === wantedBase);
+    if (!termRows.length) {
+      process.stdout.write(`I don't have anything stored about "${term}".\n`);
+      return;
+    }
+    // Render under the term's own stored spelling, not the query's — a plural
+    // query ("doctors") reads back as the singular the store holds ("doctor").
+    const subject = termRows[0].subject;
+    const article = digestTermFromRows(subject, termRows, rows, { budget: 12 });
+    if (!article || !article.paragraphs.length) {
+      // No structure bank, or nothing the selector kept — surface the plain
+      // fact count rather than an empty digest.
+      process.stdout.write(`${subject} — ${termRows.length} fact(s) stored. Run \`tmct viz --focus ${subject}\` for the full ledger.\n`);
+      return;
+    }
+    const sources = article.sources.map((s) => s.provenance).filter(Boolean);
+    const out = [`${subject}`, ...article.paragraphs];
+    if (sources.length) out.push("", `Sources: ${[...new Set(sources)].join("; ")}`);
+    out.push(`${article.detail.factCount} fact(s) stored — \`tmct viz --focus ${subject}\` shows them all.`);
+    process.stdout.write(out.join("\n") + "\n");
+    return;
+  }
+
   if (mode === "serve") {
     // `tmct serve` — the Phase-A capability-router interface: an Anthropic
     // Messages API-compatible HTTP endpoint (POST /v1/messages) over the graph.

package/data/templates/constructions/digest-sentence-structures.toml

@@ -0,0 +1,97 @@
+# data/templates/constructions/digest-sentence-structures.toml — the digest
+# layer's sentence-structure bank (PLAN_DIGEST.md stage 2). A closed table of
+# sentence skeletons keyed by relation FAMILY and FORM, hand-authored and
+# reviewed as DATA, in the same committed-TOML idiom the construction banks in
+# this directory already use. A new family or a reworded frame is an edit to
+# this file, never a code change — src/domain/digest/structures.mjs validates
+# and renders these skeletons but never invents one.
+#
+# The existing construction-bank loader (adapters/corpus/construction-banks.mjs)
+# reads only [[relation]] and [[construction]] tables, so it ignores this file's
+# [[structure]] rows; the digest layer reads them through its own path.
+#
+# Each row is one (family, form) pairing:
+#   family  one of the digest families (isa | location | partOf | capableOf |
+#           usedFor) — the closed set stage 1 groups predicates under. Each is
+#           verb-coherent: every predicate in it shares this frame's verb.
+#   form    single   — one fact of the family
+#           several  — two or more facts of the family, merged into one clause
+#           chained  — one isa fact plus its rendered ancestry chain
+#   template  the skeleton. Placeholders the renderer fills (words.mjs supplies
+#             the article, plural and casing):
+#               {TERM} {TERM_CAP}          the term, lower / sentence-leading
+#               {A_TERM} {A_TERM_CAP}      "an aardvark" / "An aardvark"
+#               {TERMS} {TERMS_CAP}        "aardvarks" / "Aardvarks"
+#               {PRONOUN} {PRONOUN_CAP}    "it" / "It" (after first mention)
+#               {OBJECT}                   the first object, raw
+#               {A_OBJECT}                 "a mammal" (article chosen per word)
+#               {OBJECTS_A}                "a mammal, a burrowing animal, and …"
+#               {OBJECTS_PLURAL}           "mammals, burrowing animals, and …"
+#               {OBJECTS_RAW}              "dig, and swim" (no article)
+#               {CHAIN}                    "a mammal, and so an animal"
+#           Any placeholder with no value for this call renders empty.
+
+# ---- isa: the definition frame (copula) -----------------------------------
+
+[[structure]]
+family = "isa"
+form = "single"
+template = "{A_TERM_CAP} is {A_OBJECT}."
+
+[[structure]]
+family = "isa"
+form = "several"
+template = "{TERMS_CAP} are {OBJECTS_A}."
+
+[[structure]]
+family = "isa"
+form = "chained"
+template = "{A_TERM_CAP} is {CHAIN}."
+
+# ---- location: where it is found (atLocation / locatedNear) ----------------
+
+[[structure]]
+family = "location"
+form = "single"
+template = "{PRONOUN_CAP} is found in {OBJECT}."
+
+[[structure]]
+family = "location"
+form = "several"
+template = "{PRONOUN_CAP} is found in {OBJECTS_RAW}."
+
+# ---- partOf: what it belongs to -------------------------------------------
+
+[[structure]]
+family = "partOf"
+form = "single"
+template = "{PRONOUN_CAP} is part of {A_OBJECT}."
+
+[[structure]]
+family = "partOf"
+form = "several"
+template = "{PRONOUN_CAP} is part of {OBJECTS_A}."
+
+# ---- capableOf: what it can do --------------------------------------------
+
+[[structure]]
+family = "capableOf"
+form = "single"
+template = "{PRONOUN_CAP} can {OBJECT}."
+
+[[structure]]
+family = "capableOf"
+form = "several"
+template = "{PRONOUN_CAP} can {OBJECTS_RAW}."
+
+# ---- usedFor: what it is used for -----------------------------------------
+
+[[structure]]
+family = "usedFor"
+form = "single"
+template = "{PRONOUN_CAP} is used for {OBJECT}."
+
+[[structure]]
+family = "usedFor"
+form = "several"
+template = "{PRONOUN_CAP} is used for {OBJECTS_RAW}."

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polycode-projects/the-mechanical-code-talker",
-  "version": "3.0.0",
+  "version": "3.0.2",
   "private": false,
   "type": "module",
   "description": "The Mechanical Code Talker (tmct) — a tolerant, offline, $0 chat surface that guides you toward precision queries about a software repository. ELIZA/PARRY-style but domain-obsessed with code. No model calls; indexes a repo on request (tmct index) or reads any producer's graph.",
@@ -130,8 +130,14 @@
     "example:polyglot": "node --disable-warning=ExperimentalWarning bin/tmct.mjs chat --repo examples/polyglot --ephemeral",
     "chatbench:run": "node chatbench/run.mjs",
     "chatbench:judge": "node chatbench/judge.mjs",
+    "chatbench:judge:cached": "node chatbench/judge.mjs --cache chatbench/verdict-cache.json",
     "agentbench:run": "node agentbench/run.mjs",
     "infbench": "node infbench/generate-cases.mjs && node infbench/run.mjs",
+    "idxbench:run": "node idxbench/run.mjs",
+    "researchbench:run": "node researchbench/run.mjs",
+    "synthbench:code": "node synthbench/code/run.mjs",
+    "ingestbench:run": "node ingestbench/run.mjs",
+    "ingestbench:judge": "node ingestbench/judge.mjs",
     "corpus:matrix": "node scripts/corpus-matrix.mjs",
     "corpus:matrix:gaps": "node scripts/corpus-matrix.mjs --gaps",
     "template:coverage": "node scripts/template-coverage.mjs",

package/src/adapters/corpus/digest-bank.mjs

@@ -0,0 +1,62 @@
+// corpus/digest-bank.mjs — the filesystem side of the digest layer's stage-5
+// wiring: read the committed sentence-structure bank and build the pre-parsed
+// table the pure layer (src/domain/digest/) expects, then run one term end to
+// end from a set of fact rows. The pure store scan and the pure pipeline live
+// in src/domain/digest; only the TOML read lives here, so every node surface
+// (chat, `tmct digest`, the page generators) shares one seam.
+//
+// A browser bundle stubs this module out (the ask/chat dock never digests
+// in-page — the pages digest client-side from a table embedded at build time),
+// so every consumer treats a null article as "fall back to the flat fact list".
+
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { join, dirname } from "node:path";
+import { parse as parseToml } from "smol-toml";
+import { buildStructureTable, digestTerm } from "../../domain/digest/index.mjs";
+import { digestStoreStats, chainsForObjects, isaObjectsOf } from "../../domain/digest/store-stats.mjs";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const BANK_FILE = join(HERE, "..", "..", "..", "data", "templates", "constructions", "digest-sentence-structures.toml");
+
+/** The raw [[structure]] rows parsed from the committed bank, once. A missing or
+ *  unparseable bank yields an empty list — the digest degrades to the caller's
+ *  flat fallback rather than throwing, the same posture the construction-bank
+ *  loader takes. The raw rows (not the built table) are cached so a surface that
+ *  needs to embed them for a browser render gets them without re-reading. */
+let cachedStructures = null;
+export function readDigestStructures() {
+  if (cachedStructures) return cachedStructures;
+  try {
+    const parsed = parseToml(readFileSync(BANK_FILE, "utf8"));
+    cachedStructures = Array.isArray(parsed.structure) ? parsed.structure : [];
+  } catch {
+    cachedStructures = [];
+  }
+  return cachedStructures;
+}
+
+let cachedTable = null;
+/** The built structure table, once. Empty when the bank is unavailable. */
+export function loadDigestStructureTable() {
+  if (cachedTable) return cachedTable;
+  cachedTable = buildStructureTable(readDigestStructures());
+  return cachedTable;
+}
+
+/**
+ * Digest one term end to end from fact rows. `termRows` are the rows whose
+ * subject is the term; `allRows` is the whole store the statistics scan over
+ * (pass the same array for both when the caller has already narrowed to one
+ * term). Returns the term-article shape, or null when the structure bank is
+ * unavailable (the browser-stub case) so the surface falls back to its flat
+ * rendering. `opts.budget` picks the per-surface fact budget.
+ */
+export function digestTermFromRows(term, termRows, allRows, opts = {}) {
+  const table = loadDigestStructureTable();
+  if (!table || table.size === 0) return null;
+  const rows = termRows || [];
+  const store = digestStoreStats(allRows || rows);
+  const chains = chainsForObjects(store.subClassEdges, isaObjectsOf(rows));
+  return digestTerm(term, rows, store, table, { ...opts, chains });
+}

package/src/adapters/toml-config.mjs

@@ -102,6 +102,14 @@ export async function normalizeConfig(raw, { configDir } = {}) {
     const arr = Array.isArray(src.graph_files) ? src.graph_files : [src.graph_files];
     cfg.graphFiles = arr.map((p) => resolve(dir, String(p)));
   }
+  // [graph] read_only — a chat session against this repo READS the graph but
+  // writes nothing back into its .tmct/: no per-turn session upsert into
+  // graph.json, no transcript/sidecar logs, no memory droppings. A committed
+  // example fixture sets it so a plain `tmct chat --repo examples/<x>` (no
+  // --ephemeral) can never rewrite the hand-stamped graph. Sparse like the
+  // rest: absent when unset, so "unset" stays distinguishable from "false".
+  const graph = src.graph || {};
+  if (graph.read_only !== undefined) cfg.graph = { readOnly: graph.read_only };
   const corpus = src.corpus || {};
   if (corpus.tier !== undefined) cfg.corpus = { tier: corpus.tier };
   const seed = src.seed || {};
@@ -134,6 +142,11 @@ export async function normalizeConfig(raw, { configDir } = {}) {
   // resolveResearchConfig's job.
   if (src.research !== undefined) cfg.research = src.research;
 
+  // Discourse-record knob (src/domain/discourse.mjs): sparse PASS-THROUGH,
+  // same discipline — the raw `[discourse]` table (max_referents) rides
+  // through; default-filling is resolveDiscourseConfig's job.
+  if (src.discourse !== undefined) cfg.discourse = src.discourse;
+
   const idx = src.index || {};
   const index = {};
   if (idx.languages !== undefined) index.languages = idx.languages;

package/src/domain/ask.mjs

@@ -1841,18 +1841,26 @@ function evalRecentCommits(graph) {
 const COMMIT_FILTER_DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
 /** Resolves the pivot (a literal ISO date, or a named commit whose own date
  *  becomes the pivot), then filters every Commit's date against it. An
- *  unresolvable pivot declines honestly (pivotResolved:false). */
+ *  unresolvable pivot declines honestly (pivotResolved:false).
+ *
+ *  A resolved answer also carries `referents` — the typed discourse referents
+ *  this answer establishes (the result set, and the pivot commit the question
+ *  was ABOUT, which the focus rules deliberately refuse to hold). Emitted
+ *  here, beside the fully typed result, before render() flattens it all to a
+ *  sentence; the session layer is what registers them into its record. */
 function evalCommitFilter(graph, ast) {
   const { op, pivotRaw } = ast;
   const dateOf = (c) => String((c.attributes || []).find((a) => a.key === "date")?.value || "").slice(0, 10);
   let pivotDate = null;
   let pivotId = null;
+  let pivotLabel = null;
   if (COMMIT_FILTER_DATE_RE.test(pivotRaw)) {
     pivotDate = pivotRaw;
   } else {
     const { match, ambiguous } = resolveObject(graph, pivotRaw, { expectedClass: "Commit" });
     if (match && !ambiguous && match.class === "Commit" && dateOf(match)) {
       pivotId = match.id;
+      pivotLabel = match.label;
       pivotDate = dateOf(match);
     }
   }
@@ -1867,7 +1875,24 @@ function evalCommitFilter(graph, ast) {
       return d === pivotDate; // "on"
     })
     .sort((a, b) => dateOf(b).localeCompare(dateOf(a)));
-  return { compositeKind: "commitFilter", op, pivotRaw, pivotDate, pivotResolved: true, matches };
+  const referents = [];
+  if (matches.length) {
+    referents.push({
+      kind: "set", class: "Commit",
+      label: `${matches.length} commit${matches.length === 1 ? "" : "s"} ${op} ${pivotRaw}`,
+      ids: matches.map((c) => c.id),
+      attrs: { count: matches.length, op, ...(pivotId ? { pivot: pivotId } : {}) },
+      lane: "commitFilter",
+    });
+  }
+  if (pivotId) {
+    referents.push({
+      kind: "event", class: "Commit", label: pivotLabel,
+      ids: [pivotId], attrs: { date: pivotDate },
+      lane: "commitFilter",
+    });
+  }
+  return { compositeKind: "commitFilter", op, pivotRaw, pivotDate, pivotResolved: true, matches, referents };
 }
 
 /** Temporal over a nested set: the commits that touched any member of the
@@ -2371,6 +2396,14 @@ function joinedQueryForm(term) {
     .replace(/[\s\-_]+/g, "");
 }
 
+/** Ceiling of tier 3's weakest scoring band (the term-component-overlap
+ *  fraction): a candidate scores exactly this when EVERY component of the
+ *  term appears in its label, strictly less on a partial overlap. Every
+ *  NAME-evidence band (stem/containment/joined/derivational) scores
+ *  hundreds and up, so score < this ceiling means the candidate shares only
+ *  some generic path components with the term. */
+const COMPONENT_OVERLAP_MAX = 10;
+
 /** Resolve a free-text object/subject term against the graph's individuals, in
  *  priority order: a sha-shaped term first resolves against Commit
  *  individuals by unique id/label prefix, then (1) exact label/id match, (2)
@@ -2549,7 +2582,7 @@ function resolveObjectCore(graph, term, { expectedClass = null } = {}) {
         // stem hit from the tier above, nor a fuller-fraction overlap on another
         // candidate. termComps.length > 0 is guaranteed here (overlap > 0 requires
         // at least one termComps entry to have matched).
-        scored.push({ ind: m, score: (overlap / termComps.length) * 10 });
+        scored.push({ ind: m, score: (overlap / termComps.length) * COMPONENT_OVERLAP_MAX });
       }
     }
   }
@@ -2557,9 +2590,20 @@ function resolveObjectCore(graph, term, { expectedClass = null } = {}) {
   if (scored.length) {
     const [best, ...rest] = scored;
     const tied = rest.filter((x) => x.score === best.score);
+    // A PARTIAL component-overlap runner-up (score < COMPONENT_OVERLAP_MAX:
+    // some but not all of the term's own tokens, e.g. graph-merge.mjs
+    // sharing only "graph"/"mjs" with "graph-build.mjs") is not another
+    // reading of the term, and disclosing it in the "(answering for X — N
+    // other matches)" note reads as leakage. A candidate that matched EVERY
+    // term token (score === COMPONENT_OVERLAP_MAX) or matched by name
+    // evidence (score above the band) stays disclosed. A partial-overlap
+    // winner keeps its partial peers — they are all the evidence there is.
+    const disclosable = best.score >= COMPONENT_OVERLAP_MAX
+      ? rest.filter((x) => x.score >= COMPONENT_OVERLAP_MAX)
+      : rest;
     return {
       match: best.ind,
-      candidates: rest.slice(0, 4).map((x) => x.ind),
+      candidates: disclosable.slice(0, 4).map((x) => x.ind),
       tier: 3,
       ambiguous: tied.length > 0,
     };
@@ -4384,6 +4428,11 @@ export function ask(graph, query, { contextId = null, nlp = undefined, prev = nu
       // edit-distance, announced in the content as "assuming you meant …");
       // null for every literal-identifier tier.
       matchedVia: result.matchedVia || null,
+      // The typed discourse referents this answer established (see
+      // evalCommitFilter) — additive, present only when a lane emitted them,
+      // so a session layer can register them without re-deriving the answer's
+      // typed content from its rendered sentence.
+      ...(Array.isArray(result.referents) && result.referents.length ? { discourse: result.referents } : {}),
       ...(rendered.ambiguous ? { candidates: rendered.candidates, candidateParses: rendered.candidateParses } : {}),
     },
   };

package/src/domain/cli-verbs.mjs

@@ -137,6 +137,17 @@ export const CLI_VERBS = [
       { flag: "[--config <path>]", prose: ["--term <word> override it); --output defaults to", "ledger.html in the cwd; --limit caps the embedded fact", "rows; --term resolves via the same normalization chat uses."] },
     ],
   },
+  {
+    mode: "digest",
+    errorLabel: "digest",
+    usage: "tmct digest <term>",
+    prose: ["a readable digest of what the graph knows about one term:"],
+    flags: [
+      { flag: "[--repo <abs>]", prose: ["a bounded narrative first (selected, sense-filtered,"] },
+      { flag: "[--graph <path>]", prose: ["deduped), then its sources and the stored-fact count."] },
+      { flag: "[--config <path>]", prose: ["The vocabulary-side sibling of `cli digest`'s code map."] },
+    ],
+  },
   {
     mode: "serve",
     errorLabel: "serve",