@polycode-projects/the-mechanical-code-talker 3.0.5 → 3.0.7

package/src/services/extract-facts.mjs

@@ -53,7 +53,8 @@ import { readFile, writeFile, mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { basename, join, resolve } from "node:path";
 
-import { runTurn, uuidv7 } from "./chat.mjs";
+import { runTurn, uuidv7, stripLeadingDiscourseAdverb } from "./chat.mjs";
+import { beginsWithVowelSound, grammarRules } from "./finish.mjs";
 import { splitSentencesPreservingPaths, stripCitationResidue } from "./sentences.mjs";
 import { loadMemory, readFactRows, appendFact } from "../adapters/memory/core.mjs";
 import { loadConfig } from "../adapters/config.mjs";
@@ -125,6 +126,12 @@ const RELATIVE_PRONOUNS = new Set(["that", "which", "who", "whom", "whose"]);
 // At most this many triples from one sentence — a bound so a run-on can never
 // shatter into noise, not a first-wins cap.
 const MAX_TRIPLES_PER_SENTENCE = 4;
+// How far a copula object scan walks past an attributive-adjective compound
+// (wink tokenizes "medium-sized" as NOUN + "-" + VERB and never re-fuses it) to
+// reach the real head noun through a coordinate modifier list
+// (", burrowing, nocturnal mammal"). A small, explicit bound: past it the object
+// abstains rather than guess, so a long noun pile never mints a stray class.
+const ATTRIBUTIVE_CHAIN_MAX_HOPS = 8;
 
 /** Fold an entity surface to its stored key: a lexicon noun's lemma, else the
  *  word's own normFactTerm (the optimistic tier mints unlisted content nouns
@@ -242,6 +249,25 @@ function optimisticTriplesPos(sentence, lexicon, nlp) {
       if (!isNounish(j)) continue;
       let hi = j;
       while (hi + 1 < values.length && isNounish(hi + 1)) hi += 1;
+      // A NOUN immediately followed by "-" then a VERB or ADJ is the left half of
+      // an attributive-adjective compound wink never re-fused ("medium-sized"),
+      // not the class. Walk forward through the coordinate modifier list (hyphens,
+      // commas, "and", further ADJ/VERB tokens) to the real head noun and re-point
+      // there; abstain if none appears within the bound, never mint the modifier.
+      if (values[hi + 1] === "-" && (pos[hi + 2] === "VERB" || pos[hi + 2] === "ADJ")) {
+        let head = null;
+        let k = hi + 1;
+        for (let hop = 0; hop < ATTRIBUTIVE_CHAIN_MAX_HOPS && k < values.length; hop += 1, k += 1) {
+          if (isNounish(k)) { head = k; break; }
+          const w = values[k]?.toLowerCase();
+          if (w === "-" || w === "," || w === "and" || pos[k] === "ADJ" || pos[k] === "VERB" || pos[k] === "CCONJ") continue;
+          break;
+        }
+        if (head === null) return null;
+        j = head;
+        hi = head;
+        while (hi + 1 < values.length && isNounish(hi + 1)) hi += 1;
+      }
       const headWord = String(values[hi]).toLowerCase();
       const nextIsOf = values[hi + 1]?.toLowerCase() === "of";
       if (!nextIsOf) return { label: entityRunAt(j), hi };
@@ -434,6 +460,17 @@ export function clauseCandidates(sentence, { nlp } = {}) {
 // live focus, never a stale paragraph carry.
 const PRONOUN_LEAD_RE = /^(?:they|it|these|those|this)\b\s*/i;
 
+/** Re-article a bare carried subject so the retried sentence is a grammatical
+ *  habitual surface the recognizer accepts: "cell" → "a cell", "orbit" → "an
+ *  orbit". Uses the same vowel-sound-aware article rule (grammar-rules.toml)
+ *  the chat recognizer's own capability rewrite uses, rather than a hardcoded
+ *  "a". */
+function articledSubject(subject) {
+  const articleRule = grammarRules().find((r) => r.kind === "article");
+  const article = articleRule && beginsWithVowelSound(subject, articleRule) ? "an" : "a";
+  return `${article} ${subject}`;
+}
+
 /** A readable predicate for canonical output: the local part of an rdfs:/ace:
  *  CURIE, otherwise the predicate verbatim. */
 const readablePredicate = (predicate) => String(predicate).replace(/^[a-z]+:/i, "");
@@ -532,9 +569,14 @@ export async function ingestText(text, {
         }
         // Bounded pronoun carry: a "they/it/these/those/this …" sentence the
         // recognizer skipped is retried once with the paragraph's last grounded
-        // subject in the pronoun's place. Never a chat turn — ingest only.
-        if (!rows && carrySubject && PRONOUN_LEAD_RE.test(cleaned)) {
-          rows = await strictRows(cleaned.replace(PRONOUN_LEAD_RE, `${carrySubject} `));
+        // subject in the pronoun's place. A leading ordinal/temporal discourse
+        // adverb ("Then it splits.") is stripped first so the pronoun reaches
+        // the sentence front; the carried subject is re-articled ("a cell") so
+        // the retry is a grammatical habitual surface. Never a chat turn —
+        // ingest only.
+        const threaded = stripLeadingDiscourseAdverb(cleaned);
+        if (!rows && carrySubject && PRONOUN_LEAD_RE.test(threaded)) {
+          rows = await strictRows(threaded.replace(PRONOUN_LEAD_RE, `${articledSubject(carrySubject)} `));
         }
         if (rows) {
           recognizedSentences += 1;

package/src/services/index.mjs

@@ -6,7 +6,7 @@
 // ask.mjs keeps the core primitives (resolveObject, traverse, render) and
 // the ask() orchestration.
 
-// Chat surface (also reachable as the `./chat` subpath export).
+// Chat surface.
 // createSession is the library's session sink — the same focus/last/planState
 // threading and memory-backend seam every shell shares; runTurn is the pure
 // single-turn engine underneath it.
@@ -35,6 +35,9 @@ setConstructionBanks(readConstructionFiles);
 // Graph traversal primitives.
 export { relationKind, impactClosure } from "../domain/codegraph.mjs";
 
+// Graph-service construction for a provider-supplied code graph.
+export { createGraphService } from "../adapters/providers/graph-service.mjs";
+
 // Tool dispatch (slash-commands and CLI tool calls route through here).
 export { dispatchTool } from "../tools/server.mjs";
 
@@ -49,13 +52,13 @@ export { foldSessionLogs } from "./fold.mjs";
 // fetchEntities() is the one read path.
 export { fetchEntities, registerProvider } from "../adapters/source.mjs";
 
-// `tmct init` onboarding (also reachable as the `./init` subpath export).
+// `tmct init` onboarding.
 // init.mjs and toml-config.mjs each export a same-named `CONFIG_FILE`
 // constant ("tmct.toml") — aliased here so both can ride the one `.` entry
 // point without colliding.
 export { initRepo, defaultConfig, renderTomlConfig, PERSONA_PRESETS, CONFIG_FILE as INIT_CONFIG_FILE } from "./init.mjs";
 
-// tmct.toml loading (also reachable as the `./toml-config` subpath export).
+// tmct.toml loading.
 export { CONFIG_FILE as TOML_CONFIG_FILE } from "../adapters/toml-config.mjs";
 
 // The "detailed answer" completions pipeline (also reachable as the

package/src/services/ledger-viz.mjs

@@ -795,7 +795,7 @@ ${ledgerBundleAvailable ? `<script src="./ledger-browser.bundle.js"></script>` :
   // LEDGER (declared "let" in the embed script above this one — a classic,
   // non-module script's top-level bindings are reachable as bare identifiers
   // by every later script tag in the document, never as window.LEDGER; see
-  // e2e/pages-ledger.test.mjs's own note on this) and its indexes start as
+  // test-e2e/pages-ledger.test.mjs's own note on this) and its indexes start as
   // the server-rendered snapshot but are REASSIGNED wholesale after a live
   // teach (applyLedgerData, below) — a successful teach through the dock
   // changes the underlying graph, and the page has to show that, not keep

package/src/services/research.mjs

@@ -25,6 +25,7 @@
 
 import { normFactTerm } from "../domain/hash.mjs";
 import { loadLexicon, lookupNoun } from "../domain/grammar/lexicon.mjs";
+import { defaultNlp } from "../domain/interpret/nlp-registry.mjs";
 
 /** The search key a topic folds to: normFactTerm, then the lexicon lemma
  *  when the noun is known ("owls" → "owl") — the same fold the live
@@ -210,12 +211,87 @@ function queuedFolds(state) {
   return seen;
 }
 
+// The lexical-token split a fan-out candidate and the seed topic both fold
+// through before comparison — lowercased, split on anything that isn't a
+// letter or digit, empty pieces dropped.
+const LEXICAL_SPLIT_RE = /[^a-z0-9]+/i;
+function lexicalTokens(text) {
+  return String(text || "").toLowerCase().split(LEXICAL_SPLIT_RE).filter(Boolean);
+}
+function sharesLexicalToken(title, seedTokens) {
+  if (!seedTokens.size) return false;
+  for (const tok of lexicalTokens(title)) if (seedTokens.has(tok)) return true;
+  return false;
+}
+
+// A citation-shaped candidate: an ISBN/DOI prefix, a bare year, or a
+// "Nth century" phrase — the shape raw Wikipedia link lists carry for their
+// source apparatus rather than a kin article.
+const CITATION_PREFIX_RE = /^(isbn|doi)[\s:.-]/i;
+const CITATION_BARE_YEAR_RE = /^\d{3,4}$/;
+const CITATION_CENTURY_RE = /\b\d{1,2}(?:st|nd|rd|th)\s+century\b/i;
+function isCitationShaped(title) {
+  const t = String(title || "").trim();
+  if (!t) return false;
+  return CITATION_PREFIX_RE.test(t) || CITATION_BARE_YEAR_RE.test(t) || CITATION_CENTURY_RE.test(t);
+}
+
+const SINGLE_WORD_TITLE_RE = /^[A-Za-z][A-Za-z'-]*$/;
+// A short, neutral sentence to tag a bare candidate word inside — wink-nlp
+// reads an isolated capitalized word as PROPN with no sentence context to
+// tell it otherwise, so the word is lowercased and read at this fixed slot
+// inside real subject/verb/object context instead.
+const NOUN_CARRIER_PREFIX = ["this", "is", "about", "the"];
+const NOUN_CARRIER_SUFFIX = ["and", "its", "history"];
+const NOUN_CARRIER_WORD_INDEX = NOUN_CARRIER_PREFIX.length;
+
+/** True when `title` is a single word a general-English POS tagger reads as
+ *  a common noun (hub articles like "Earth"/"Geology") rather than a proper
+ *  noun (kin articles like "Hawaii") — false whenever `nlp` is unavailable,
+ *  never a throw. */
+function readsAsCommonNoun(title, nlp) {
+  if (!nlp || typeof nlp.posTags !== "function") return false;
+  const t = String(title || "").trim();
+  if (!SINGLE_WORD_TITLE_RE.test(t)) return false;
+  const carrier = [...NOUN_CARRIER_PREFIX, t.toLowerCase(), ...NOUN_CARRIER_SUFFIX];
+  let tags;
+  try { tags = nlp.posTags(carrier); } catch { return false; }
+  return Array.isArray(tags) && tags[NOUN_CARRIER_WORD_INDEX] === "NOUN";
+}
+
+/** Stable-sorts fan-out candidates into relevance tiers — never reorders
+ *  within a tier, only reprioritizes between them — from information the
+ *  runtime actually has (title strings, the seed topic, an optional POS
+ *  tagger), never bench-only ground truth:
+ *  0. shares a lexical token with `seedTopic` ("Active volcano" ~ "Volcano");
+ *  1. everything else, in original document order (the fallback tier);
+ *  2. a single-word title a POS tagger reads as a common noun rather than a
+ *     proper noun — the hub-article signal ("Earth", "Geology");
+ *  3. a citation-shaped title (ISBN/DOI prefix, bare year, "Nth century").
+ *  With no `nlp` adapter registered, tier 2 never fires (everything that
+ *  would have landed there stays in tier 1) — degrades gracefully, never
+ *  throws, never drops a candidate. */
+export function relevanceOrder(titles, seedTopic, nlp = null) {
+  const list = Array.isArray(titles) ? titles : [];
+  const seedTokens = new Set(lexicalTokens(seedTopic));
+  const tiers = [[], [], [], []];
+  for (const title of list) {
+    if (sharesLexicalToken(title, seedTokens)) { tiers[0].push(title); continue; }
+    if (isCitationShaped(title)) { tiers[3].push(title); continue; }
+    if (readsAsCommonNoun(title, nlp)) { tiers[2].push(title); continue; }
+    tiers[1].push(title);
+  }
+  return [...tiers[0], ...tiers[1], ...tiers[2], ...tiers[3]];
+}
+
 /** Queue `article`'s lead-section links at `fromDepth + 1`, subject to the run's
  *  depth ceiling, its per-fan-out cap and — crucially — its TOTAL node budget:
  *  the number added never pushes grounded+pending past `maxTopics`. Sets
  *  `state.nodeCapReached` when the budget (not the depth, not a lack of links)
- *  is what stopped the fan-out, so the progress line can say so. Returns the
- *  titles it enqueued. */
+ *  is what stopped the fan-out, so the progress line can say so. Candidates are
+ *  relevance-ordered (relevanceOrder) before the fan-out cap truncates them, so
+ *  a capped fetch keeps kin articles over generic hubs. Returns the titles it
+ *  enqueued. */
 async function enqueueFrom(state, article, fromDepth, provider) {
   const childDepth = fromDepth + 1;
   if (childDepth > runMaxDepth(state)) return [];
@@ -226,10 +302,11 @@ async function enqueueFrom(state, article, fromDepth, provider) {
   if (want <= 0) { state.nodeCapReached = true; return []; }
   let linked = null;
   try { linked = await provider.linkedTitles(article.title, { limit: want + 2 }); } catch { linked = null; }
+  const ordered = relevanceOrder(linked || [], article.title, defaultNlp());
   const seen = queuedFolds(state);
   if (!state.depths) state.depths = {};
   const added = [];
-  for (const title of linked || []) {
+  for (const title of ordered) {
     const folded = normFactTerm(title);
     if (!folded || seen.has(folded)) continue;
     seen.add(folded);

package/src/surfaces/web/chat-browser-entry.mjs

@@ -10,12 +10,21 @@
 // built seed payload (scripts/build-chat-seed.mjs), so teach turns, recall,
 // proof chains and the honest miss all run client-side with zero I/O.
 //
-// Two browser traps this file owns so no caller can fall into them:
+// Three browser traps this file owns so no caller can fall into them:
 //   - runTurn defaults `env` to process.env, and a browser has no `process`
 //     global — every turn here passes `env: {}` explicitly;
 //   - the uuid adapter needs node:crypto — the session id comes from the
 //     Web Crypto API instead, with a Date.now fallback for contexts
-//     without it.
+//     without it;
+//   - setDigestStructures is imported from adapters/corpus/digest-bank.mjs.
+//     Under the bundle build, scripts/build-chat-bundle.mjs swaps that import
+//     for a live in-memory twin, and this call actually feeds it the page's
+//     embedded structure rows; this file is ALSO imported directly by plain
+//     Node (test-e2e/web-chat-memory.test.mjs, exercising the same engine contract
+//     without a browser), where the import resolves to the real fs+TOML
+//     adapter — its own setDigestStructures is a documented no-op there
+//     (see that module's header), so the call is harmless either way, never
+//     a load error and never a behavior change on the Node side.
 import { runTurn, vocabExampleHint } from "../../services/chat.mjs";
 import { createInMemoryStore, normFactTerm, loadMemory, readFactRows } from "../../adapters/memory/core.mjs";
 import { serializeFactsJsonl } from "../../adapters/memory/export-jsonl.mjs";
@@ -24,6 +33,7 @@ import { provenanceTagToSource } from "../../domain/memory/trust.mjs";
 import { parseEntities } from "../../domain/codegraph.mjs";
 import { loadLexicon } from "../../domain/grammar/lexicon.mjs";
 import { registerWinkModel } from "../../adapters/wink-model.mjs";
+import { setDigestStructures } from "../../adapters/corpus/digest-bank.mjs";
 // The reference-pack provider seam: the page registers a fetch-backed
 // provider over public/reference-pack/ so the engine's pack lookups work
 // where the gzipped fs layout cannot (the module's own fs loader degrades to
@@ -50,11 +60,21 @@ import { openPersistedStore } from "./idb-persist.mjs";
  * payload carries the starter vocabulary, so an unseeded session offers the
  * teach pointer instead of an example it cannot answer.
  *
+ * `digestStructures` (optional) is the page's build-time-embedded
+ * [[structure]] rows from the digest sentence-structure bank (chat-page-viz.mjs's
+ * DIGEST_STRUCTURES) — fed once to the bundle's live digest-bank twin so a
+ * long "what is X" answer leads with a composed digest instead of always
+ * falling back to the flat fact list (the bundle's stub used to return null
+ * unconditionally). Harmless to call more than once per page load: the table
+ * is module-scope, last write wins, and every session created after this one
+ * shares it.
+ *
  * Returns { memoryDir, sessionId, turn }. `turn(line)` resolves to
  * { answer, end, record, plan } and threads focus/last/planState between
  * calls exactly as the CLI session does.
  */
-export function createChatSession({ seedPayload = null, vocabSeeded = false, liveReference = false, onLiveLookup = null, synthesisBudget = 12 } = {}) {
+export function createChatSession({ seedPayload = null, vocabSeeded = false, liveReference = false, onLiveLookup = null, synthesisBudget = 12, digestStructures = null } = {}) {
+  setDigestStructures(digestStructures || []);
   const memoryDir = createInMemoryStore();
   // Spread onto the store's own empty payload so a partial seed (individuals
   // and objectProperties only) still carries the classes/prefixes scaffolding

package/src/surfaces/web/digest-client.mjs

@@ -14,7 +14,8 @@
 
 import { buildStructureTable, digestTerm } from "../../domain/digest/index.mjs";
 import { digestStoreStats, chainsForObjects, isaObjectsOf } from "../../domain/digest/store-stats.mjs";
-import { readFactRows } from "../../adapters/memory/core.mjs";
+import { readFactRows, normFactTerm } from "../../adapters/memory/core.mjs";
+import { factTermVariants } from "../../services/chat.mjs";
 
 /**
  * Digest one term end to end from fact rows and a pre-parsed structure table.
@@ -40,14 +41,18 @@ export function digestTermFromRowsBrowser(term, termRows, allRows, structures, o
  * in-browser surface already holds (the ledger's embedded PAYLOAD, the ledger
  * dock's live memoryDir.payload, the research session's store). Scans the
  * payload once for the term's own rows and the whole-store statistics, so a
- * caller never has to run readFactRows itself.
+ * caller never has to run readFactRows itself. Matches `term` against a fact's
+ * subject through the same spelling-variant fold `factTermVariants` gives the
+ * "what is X" fact reader (plural/irregular-plural folding), so a term seeded
+ * as "dog" is still found when queried as "dogs".
  *
  * Returns the render-ready view (see digestViewFromArticle) or null when the
  * term has no rows, no structures were supplied, or the selector kept nothing.
  */
 export function digestTermFromPayloadBrowser(payload, term, structures, opts = {}) {
   const rows = readFactRows(payload || { individuals: [], objectProperties: [] });
-  const termRows = rows.filter((r) => r.subject === term);
+  const variants = factTermVariants(normFactTerm, term);
+  const termRows = rows.filter((r) => variants.has(r.subject));
   if (!termRows.length) return null;
   const article = digestTermFromRowsBrowser(term, termRows, rows, structures, opts);
   return digestViewFromArticle(article);

package/src/surfaces/web/graph-ask-browser-entry.mjs

@@ -0,0 +1,41 @@
+// graph-ask-browser-entry.mjs — the public, browser-safe entry point for
+// tmct's CODE-graph query engine (`./ask-browser` package export).
+//
+// A plain re-export, zero side effects, zero `globalThis` assignment: unlike
+// this directory's other *-browser-entry.mjs files (built into an IIFE and
+// inlined into a specific tmct page), this one is meant to be imported by a
+// DOWNSTREAM consumer's own bundler (e.g. seonix's website chat panel) as a
+// genuine ESM module, not inlined by tmct's own build scripts.
+//
+// It exists because the package root (`src/services/index.mjs`, and its
+// `.`/exports subpath) also runs Node-side composition wiring at module-load
+// time — `setDefaultNlpAdapter(nlpAdapter)` from `../adapters/ask-nlp.mjs`
+// (wink-nlp, a ~4MB model) and `setConstructionBanks(readConstructionFiles)`
+// from `../adapters/corpus/construction-banks.mjs` (fs reads) — neither of
+// which can run or bundle in a browser. This file imports only ask.mjs and
+// graph-service.mjs directly, so none of that wiring is reachable from here.
+//
+// Verified with `esbuild --bundle --platform=browser --format=esm`
+// (scripts/verify-ask-browser-entry.mjs). ask()'s optional strategies
+// (ace.mjs, constructions.mjs) and the wink lemma/POS adapter are all reached
+// through nlp-registry.mjs's `defaultNlp()` registry seam and each call
+// site's own `typeof X !== "undefined"` guard, not a static import, so a
+// consumer bundling this entry point alone never links wink-nlp or any
+// fs-reading corpus module — none of the ask-nlp.mjs/ace.mjs/
+// constructions.mjs/construction-banks.mjs/digest-bank.mjs family of stubs
+// scripts/build-ask-bundle.mjs needs for chat.mjs is required here.
+//
+// One real gap remains: `createGraphService` pulls in source-slice.mjs,
+// which imports `node:path` (join/resolve/sep) for span math, so a browser
+// bundler with no Node shims will fail on that one specifier. Apply the
+// same generic `stubNodeBuiltins` plugin scripts/lib/browser-bundle.mjs
+// already exports for tmct's own bundles (or an equivalent node:path shim —
+// e.g. esbuild's own `--define`/polyfill options, or a bundler that already
+// ships a node:path shim, as most do). Re-run
+// `node scripts/verify-ask-browser-entry.mjs` (clean) and
+// `node scripts/verify-ask-browser-entry.mjs --with-stubs` (with the shim)
+// after touching ask.mjs's or graph-service.mjs's import graph — if the
+// no-stub run ever passes clean, or a new specifier shows up, update this
+// comment to match what esbuild actually reports.
+export { ask, resolveObject } from "../../domain/ask.mjs";
+export { createGraphService } from "../../adapters/providers/graph-service.mjs";