@polycode-projects/the-mechanical-code-talker 0.3.0 → 0.4.0

package/src/init.mjs

@@ -0,0 +1,264 @@
+// init.mjs — `tmct init`: the interface's onboarding surface.
+//
+// One command takes a bare directory (a user's repo, or a host package such as
+// seonix) to a WORKING tmct install: it creates the `.tmct/` artifact tree,
+// writes an externalised `tmct.toml` (the seonix.toml documented-config pattern),
+// seeds the committed tier-1 corpus into memory, and records provenance of what
+// it did. See ROADMAP Phase 8 ("Distribution: tmct init") and the Phase-4
+// corpus-tiering policy.
+//
+//   initRepo(dir, { force?, seed?, env? }) → { created, config, seeded, ... }
+//
+// DESIGN RULES (load-bearing):
+//   - OFFLINE, DETERMINISTIC, $0. The seed is the tier-1 committed ConceptNet
+//     slice already in the tarball — no network, ever. The $0-offline default is
+//     inviolable (ROADMAP Phase 4); tiers 2-3 are additive config, never run here.
+//   - IDEMPOTENT and NON-DESTRUCTIVE. Safe to re-run. A benign re-init NEVER
+//     throws — it returns an honest result whose `message` says nothing changed.
+//     Existing `tmct.toml` and an existing seed are preserved unless `force`.
+//   - FAILURE-TOLERANT SEED. A missing/broken corpus degrades to an unseeded (but
+//     still initialised) repo — the directory scaffold and config always land.
+//
+// The seed marker + limit + prefer mirror src/chat.mjs's W3 bootstrap
+// (SEED_MARKER_REL / SEED_LIMIT / SEED_PREFER) ON PURPOSE: both write the same
+// `.tmct/memory/corpus-seed.json`, so whichever of `tmct init` and first-run
+// bootstrap happens first wins and the other short-circuits. They are re-declared
+// here (not imported) to keep init off chat.mjs's heavy module graph.
+
+import { mkdir, readFile, writeFile, stat } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+export const CONFIG_FILE = "tmct.toml";
+export const PROVENANCE_REL = join(".tmct", "init.json");
+export const MEMORY_DIR_REL = join(".tmct", "memory");
+export const SESSIONS_DIR_REL = join(".tmct", "sessions");
+export const SEED_MARKER_REL = join(".tmct", "memory", "corpus-seed.json");
+
+/** How many corpus facts the seed writes — matches chat.mjs SEED_LIMIT so an
+ *  init-seeded repo and a bootstrap-seeded repo carry the identical slice. */
+export const SEED_LIMIT = 500;
+
+/** Predicate preference for the capped seed (definitional band first) — matches
+ *  chat.mjs SEED_PREFER so "what is a cache?" answers land in the first 500. */
+export const SEED_PREFER = ["rdfs:subClassOf", "rdf:type", "mgx:usedFor", "mgx:partOf", "mgx:capableOf"];
+
+/** The shipped default config — the exact shape written into `tmct.toml` and
+ *  echoed back in the result's `config`. Absent file ⇒ these values apply. */
+export function defaultConfig() {
+  return {
+    graphFile: join(".tmct", "graph.json"),
+    corpus: { tier: "tier1" },
+    seed: { enabled: true, limit: SEED_LIMIT },
+  };
+}
+
+/** Read this package's version (best-effort, for provenance). */
+async function tmctVersion() {
+  try {
+    const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
+    const pkg = JSON.parse(await readFile(pkgPath, "utf8"));
+    return pkg.version || null;
+  } catch {
+    return null;
+  }
+}
+
+async function exists(p) {
+  try {
+    await stat(p);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Render the documented, commented `tmct.toml` for a config object. Hand-written
+ *  (not smol-toml stringify) so every key ships with the prose that makes the file
+ *  a self-explaining config surface — the seonix.toml pattern. The output parses
+ *  back cleanly through toml-config.mjs's loadTomlConfig. */
+export function renderTomlConfig(config = defaultConfig()) {
+  const c = { ...defaultConfig(), ...config };
+  const corpus = { ...defaultConfig().corpus, ...(config.corpus || {}) };
+  const seed = { ...defaultConfig().seed, ...(config.seed || {}) };
+  return `# tmct.toml — the mechanical code talker, project configuration.
+# Written by \`tmct init\`. An ABSENT file means shipped defaults (this file
+# just makes them explicit and editable). Documented in the repository-interface
+# onboarding surface (ROADMAP Phase 8, "Distribution: tmct init").
+
+# Where the code-graph JSON artifact lives, relative to this file. The
+# TMCT_GRAPH_FILE environment variable overrides it at runtime.
+graph_file = ${JSON.stringify(c.graphFile)}
+
+[corpus]
+# Corpus-tiering policy (ROADMAP Phase 4). The $0-offline default is inviolable;
+# higher tiers are ADDITIVE and never required to answer.
+#   "tier1" — committed slice only. Offline, $0. The default.
+#   "tier2" — also fetch growable corpora at seed time (network, once, cached).
+#   "tier3" — also consult live sources at question time (network, per-query, opt-in).
+tier = ${JSON.stringify(corpus.tier)}
+
+[seed]
+# Seed the committed tier-1 ConceptNet slice into .tmct/memory during init.
+# Offline and deterministic. Set false, or export TMCT_NO_SEED=1, to opt out —
+# the repo still initialises, just empty of corpus facts.
+enabled = ${seed.enabled ? "true" : "false"}
+# How many facts the seed writes (definitional band first).
+limit = ${Number(seed.limit)}
+`;
+}
+
+/** Should the seed run? Explicit `opts.seed` wins; otherwise the config's
+ *  `seed.enabled`; and TMCT_NO_SEED=<non-empty> is a hard veto over both (the
+ *  documented environment opt-out, honoured even when config says enabled). */
+function seedRequested({ optSeed, configEnabled, env }) {
+  const noSeed = env && String(env.TMCT_NO_SEED || "").trim();
+  if (noSeed) return false;
+  if (optSeed !== undefined) return Boolean(optSeed);
+  return Boolean(configEnabled);
+}
+
+/**
+ * Initialise `dir` for tmct. Idempotent, non-destructive, offline.
+ *
+ * @param {string} dir  target directory (a repo root, or a host package root).
+ * @param {object} [opts]
+ * @param {boolean} [opts.force]  re-write tmct.toml + re-record provenance even
+ *   when the repo is already initialised (never deletes memory/seed data).
+ * @param {boolean} [opts.seed]   force seeding on/off, overriding tmct.toml's
+ *   `seed.enabled` (TMCT_NO_SEED still vetoes).
+ * @param {object}  [opts.env]    environment (for TMCT_NO_SEED); defaults to
+ *   process.env.
+ * @returns {Promise<{
+ *   created: string[], config: object, seeded: boolean,
+ *   alreadyInitialized: boolean, seedResult: (object|null), message: string
+ * }>} `created` lists the ABSOLUTE paths this call brought into being (empty on a
+ *   benign no-op re-init). Never throws on a benign re-init or a corpus failure.
+ */
+export async function initRepo(dir, { force = false, seed, env = process.env } = {}) {
+  const root = resolve(dir);
+  const created = [];
+  const paths = {
+    tmct: join(root, ".tmct"),
+    memory: join(root, MEMORY_DIR_REL),
+    sessions: join(root, SESSIONS_DIR_REL),
+    toml: join(root, CONFIG_FILE),
+    provenance: join(root, PROVENANCE_REL),
+    marker: join(root, SEED_MARKER_REL),
+  };
+
+  const wasInitialized = await exists(paths.provenance);
+
+  // ---- 1. The artifact directory scaffold (always idempotent) ----
+  for (const d of [paths.tmct, paths.memory, paths.sessions]) {
+    if (!(await exists(d))) {
+      await mkdir(d, { recursive: true });
+      created.push(d);
+    }
+  }
+
+  // ---- 2. The externalised config (preserve an existing file unless force) ----
+  let config = defaultConfig();
+  const tomlPresent = await exists(paths.toml);
+  if (!tomlPresent || force) {
+    await writeFile(paths.toml, renderTomlConfig(config));
+    if (!tomlPresent) created.push(paths.toml);
+  } else {
+    // Honour the user's committed tmct.toml — read its knobs back so the returned
+    // config (and the seed decision) reflect what's actually on disk.
+    config = await readWrittenConfig(paths.toml, config);
+  }
+
+  // ---- 3. Seed the tier-1 committed corpus (offline, failure-tolerant) ----
+  let seeded = false;
+  let seedResult = null;
+  let seedNote = "";
+  const wantSeed = seedRequested({ optSeed: seed, configEnabled: config.seed?.enabled, env });
+  if (!wantSeed) {
+    seedNote = env && String(env.TMCT_NO_SEED || "").trim()
+      ? "seed skipped (TMCT_NO_SEED set)"
+      : "seed skipped (disabled)";
+  } else if ((await exists(paths.marker)) && !force) {
+    seedNote = "seed skipped (already seeded — marker present)";
+  } else {
+    try {
+      const { seedMemory } = await import("./corpus/conceptnet.mjs");
+      const limit = Number(config.seed?.limit) || SEED_LIMIT;
+      seedResult = await seedMemory(root, { limit, prefer: SEED_PREFER });
+      const markerNew = !(await exists(paths.marker));
+      await mkdir(dirname(paths.marker), { recursive: true });
+      await writeFile(
+        paths.marker,
+        JSON.stringify({
+          seededAt: new Date().toISOString(),
+          limit,
+          appended: seedResult.appended,
+          skipped: seedResult.skipped,
+        }) + "\n",
+      );
+      if (markerNew) created.push(paths.marker);
+      seeded = true;
+    } catch (err) {
+      // Corpus unavailable/broken → an initialised-but-unseeded repo, not a crash.
+      seedNote = `seed skipped (corpus unavailable: ${err && err.message ? err.message : err})`;
+    }
+  }
+
+  // ---- 4. Record provenance (what was created, when, by which version) ----
+  const provenanceNew = !(await exists(paths.provenance));
+  const provenance = {
+    tool: "tmct init",
+    tmctVersion: await tmctVersion(),
+    initializedAt: new Date().toISOString(),
+    dir: root,
+    config,
+    seeded,
+    seedResult,
+    created,
+  };
+  await writeFile(paths.provenance, JSON.stringify(provenance, null, 2) + "\n");
+  if (provenanceNew) created.push(paths.provenance);
+
+  const alreadyInitialized = wasInitialized && !force;
+  const message = buildMessage({ alreadyInitialized, force, created, seeded, seedNote, seedResult });
+
+  return { created, config, seeded, alreadyInitialized, seedResult, message };
+}
+
+/** Read a present tmct.toml back into the canonical config shape (so a re-init
+ *  respects the on-disk file). Falls back to `base` on any read/parse trouble —
+ *  init must never crash on a malformed user file; the runtime loader
+ *  (toml-config.mjs) is where a bad file surfaces its error. */
+async function readWrittenConfig(tomlPath, base) {
+  try {
+    const { loadTomlConfig } = await import("./toml-config.mjs");
+    const raw = await loadTomlConfig(dirname(tomlPath));
+    if (!raw) return base;
+    const cfg = { ...base };
+    if (raw.graph_file !== undefined) cfg.graphFile = String(raw.graph_file);
+    if (raw.corpus && raw.corpus.tier !== undefined) cfg.corpus = { ...cfg.corpus, tier: raw.corpus.tier };
+    if (raw.seed) {
+      cfg.seed = { ...cfg.seed };
+      if (raw.seed.enabled !== undefined) cfg.seed.enabled = Boolean(raw.seed.enabled);
+      if (raw.seed.limit !== undefined) cfg.seed.limit = Number(raw.seed.limit);
+    }
+    return cfg;
+  } catch {
+    return base;
+  }
+}
+
+function buildMessage({ alreadyInitialized, force, created, seeded, seedNote, seedResult }) {
+  if (alreadyInitialized && created.length === 0) {
+    return `Already initialized — nothing to do (re-run with force to rewrite tmct.toml). ${seedNote || ""}`.trim();
+  }
+  const parts = [];
+  parts.push(force && alreadyInitialized ? "Re-initialized" : "Initialized");
+  parts.push(`tmct here (${created.length} path${created.length === 1 ? "" : "s"} created).`);
+  if (seeded && seedResult) {
+    parts.push(`Seeded ${seedResult.appended} corpus fact${seedResult.appended === 1 ? "" : "s"} into memory.`);
+  } else if (seedNote) {
+    parts.push(seedNote + ".");
+  }
+  return parts.join(" ");
+}

package/src/interpret/normalize.mjs

@@ -90,6 +90,40 @@ export function applyNegationFrames(text) {
   return text;
 }
 
+// ---- §B1 negation — the SET-COMPLEMENT frame (Cycle 5, PLAN_CYCLE_4.md). Recognizes
+// a BARE set-negation query — "which X do not <verb> Y", "X that don't <verb> Y",
+// "modules not importing Y", "which X are not <qualifier>" — and returns a descriptor
+// {entWord, predicate} that ask.mjs's compositional grammar turns into a bounded
+// complement (allOfClass(kind) MINUS the positive result set). Deliberately SEPARATE
+// from applyNegationFrames/NEGATION_FRAMES above: that table is a rhetorical
+// double-negative rewriter that REMOVES negation ("there isn't anything calling it" ->
+// "what calls it") and its docblock forbids scope parsing; this detector PRESERVES the
+// negation as a set operation. Returns null when no set-negation marker is present, so
+// every affirmative query passes through untouched (the active-voice regression guard).
+// The entWord is validated against the entity vocabulary by the caller, which also
+// enforces the bounded-universe refusal for the non-enumerable "changes" pseudo-type. ----
+const NEGATION_SET_RE = new RegExp(
+  "^(?:which|what|who|list|show(?:\\s+me)?|find|give\\s+me)?\\s*(?:the\\s+|all\\s+)?" // optional frame + determiner
+  + "([a-z][a-z-]*)\\s+"                    // (1) the entity kind noun
+  + "(?:(?:that|which|who)\\s+)?"           // optional relative pronoun
+  + "(?:(?:do|does|did|are|is|was|were|have|has)\\s+)?" // optional auxiliary
+  + "not\\s+(.+)$",                         // the negation marker + (2) the predicate
+  "i",
+);
+
+/** Recognize a bare set-negation query and return {entWord, predicate}, or null when
+ *  no set-negation marker ("not") follows an entity noun. Pure text analysis — the
+ *  caller (ask.mjs's parseNegation) validates the entity kind, refuses the
+ *  non-enumerable "changes" universe, and builds the complement AST. */
+export function matchNegationSet(text) {
+  const m = String(text || "").match(NEGATION_SET_RE);
+  if (!m) return null;
+  const entWord = m[1].toLowerCase();
+  const predicate = m[2].trim();
+  if (!predicate) return null;
+  return { entWord, predicate };
+}
+
 // ---- shared text-prep helpers (used by the strategies, the compositional
 // grammar, and ask.mjs's relaxation cascade alike) ----
 

package/src/interpret/strategies/keywords.mjs

@@ -7,11 +7,20 @@
 
 import {
   VERB_TO_KIND, ENTITY_TO_TYPE, MODIFIER_TO_KIND,
-  WHERE_MARKERS, MENTION_MARKERS,
+  WHERE_MARKERS, MENTION_MARKERS, PLACEHOLDER_NOUNS, PASSIVE_PARTICIPLE_TO_KIND,
 } from "../../ask-vocab.mjs";
 import { STOPWORDS } from "../normalize.mjs";
 import { VOCAB_WORDS, eligibleForCanon, fuzzyVocabWord } from "../fuzzy.mjs";
 
+// Reversible-passive detection (Cycle 6, PLAN_CYCLE_4.md): the passive auxiliaries that,
+// together with an agent-marking "by", flip the active reading, and the wh-words that
+// mark a QUESTIONED agent ("by which classes" / stranded "who is X tested by"). Bare
+// "do/does/did" are deliberately EXCLUDED — "which X do not <verb> Y" is a NEGATION, not
+// a passive, and must be left for the compositional complement frame.
+const PASSIVE_AUX = new Set(["is", "are", "was", "were", "be", "been", "being", "get", "gets", "got"]);
+const WH_WORDS = new Set(["which", "what", "who", "whom", "whose"]);
+const PLACEHOLDER_SET = new Set(PLACEHOLDER_NOUNS.map((w) => w.toLowerCase()));
+
 /** Find the longest phrase from `table`'s keys that appears as a contiguous
  *  run of `words` (case already lowercased by the caller). Longest-match-first
  *  (multi-word phrases before single words) so "co-changes with" isn't
@@ -107,6 +116,17 @@ export function parseKeywordSpot(text, nlp = null) {
     verbHit = findPhrase(fuzzyWords, VERB_TO_KIND);
     if (verbHit) canonWords = fuzzyWords;
   }
+  if (!verbHit && lcWords.includes("by")) {
+    // passive-participle rescue (Cycle 6): a participle whose kind is NOT a standalone
+    // active verb here ("defined" belongs to "is defined in"; bare "inherited" has no
+    // active key) still marks a passive when a passive auxiliary and an agent "by" are
+    // present. Consulted ONLY on this exact gate (aux + by), so the active grammar and
+    // the where-marker routing ("where is X defined") are never disturbed.
+    for (let i = 0; i < lcWords.length; i += 1) {
+      const k = PASSIVE_PARTICIPLE_TO_KIND[lcWords[i]];
+      if (k && lcWords.slice(0, i).some((w) => PASSIVE_AUX.has(w))) { verbHit = { kind: k, start: i, end: i + 1 }; break; }
+    }
+  }
   if (!verbHit) return null;
   // POS consumer (wink adapter, Node-side only): rescue the ONE decomposition this
   // strategy provably mis-parses — a relation word used as a NOUN in a "the
@@ -156,6 +176,42 @@ export function parseKeywordSpot(text, nlp = null) {
     if (objText) return { shape: "when", entityType: null, modifier: "direct", kind: "touches", object: objText };
   }
 
+  // reversible passive (Cycle 6, PLAN_CYCLE_4.md): "PATIENT is VERBed BY AGENT" — an
+  // agent-marking "by" plus a passive auxiliary flips the active reading, so the AGENT
+  // (after "by") is the edge SUBJECT and the PATIENT the edge OBJECT. Object-first
+  // phrasing is otherwise read subject-first and the edge traversed backwards. Fires
+  // ONLY on a genuine agent "by": a passive auxiliary before the verb AND a standalone
+  // "by" NOT already swallowed into a multi-word verb phrase ("touched by"/"modified
+  // by" are single touches verbs, so their "by" is consumed and never triggers this) —
+  // an active query whose object merely contains a "by" token is untouched (the
+  // regression guard). The single NAMED role term becomes the object; whether the AGENT
+  // is named ("by b.test.mjs" → forward from the agent) or QUESTIONED ("by which
+  // classes" / a stranded "…tested by" → reverse over the patient) picks the direction.
+  const byIdx = lcWords.indexOf("by");
+  const hasPassiveAux = lcWords.slice(0, verbHit.start).some((w) => PASSIVE_AUX.has(w));
+  if (byIdx >= 0 && !consumed.has(byIdx) && hasPassiveAux) {
+    const roleWords = [];
+    for (let i = 0; i < words.length; i += 1) {
+      const w = lcWords[i];
+      if (consumed.has(i) || STOPWORDS.has(w) || w === "by" || PASSIVE_AUX.has(w)
+        || WH_WORDS.has(w) || PLACEHOLDER_SET.has(w)) continue;
+      roleWords.push(words[i]);
+    }
+    const object = roleWords.join(" ").trim();
+    if (object) {
+      // the first meaningful token after "by" (skipping only articles) decides direction:
+      // a wh-word or nothing → the agent is questioned (reverse over the named patient);
+      // a named token → the agent is given (forward from it).
+      let nextAfterBy = null;
+      for (let i = byIdx + 1; i < lcWords.length; i += 1) {
+        if (lcWords[i] === "the" || lcWords[i] === "a" || lcWords[i] === "an") continue;
+        nextAfterBy = lcWords[i]; break;
+      }
+      const agentNamed = nextAfterBy != null && !WH_WORDS.has(nextAfterBy) && !ENTITY_TO_TYPE[nextAfterBy];
+      return { shape: agentNamed ? "forward" : "reverse", entityType, modifier, kind, object };
+    }
+  }
+
   if (beforeText && afterText) return { shape: "ask", entityType: null, modifier: "direct", kind, subject: beforeText, object: afterText };
   if (afterText) return { shape: "reverse", entityType, modifier, kind, object: afterText };
   // forward keeps the spotted entityType ("which modules did commit <sha> touch" is a

package/src/memory/blocks.mjs

@@ -24,6 +24,15 @@
 import { mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { splitIdentifierWords, tokenizeProse } from "../prose.mjs";
+import { SOURCE_PRIOR } from "./trust.mjs";
+
+// A block inherits the trust of the Source it was folded from: a session block
+// is operator-chat (1.0), a corpus block its corpus Source (0.7). Retrieval
+// weights relevance × trust via a BOUNDED factor (≈ 0.5 + trust → ~[0.5, 1.5]),
+// a capped nudge so a weakly-trusted but perfectly-relevant block still surfaces.
+const DEFAULT_BLOCK_SOURCE_TYPE = "operator";
+const blockTrust = (sourceType) => SOURCE_PRIOR[sourceType] ?? SOURCE_PRIOR.operator;
+const trustFactorOf = (trust) => 0.5 + (typeof trust === "number" ? trust : SOURCE_PRIOR.operator);
 
 export const BLOCKS_DIR_REL = join(".tmct", "memory", "blocks");
 const INDEX_NAME = "index.json";
@@ -134,14 +143,20 @@ function rerank(index) {
  * duplicated (fold.mjs's re-fold idempotency rests on this).
  * Returns the block's index entry { file, tokens, rank }.
  */
-export async function saveBlock(dir, { id, text }) {
+export async function saveBlock(dir, { id, text, sourceType = DEFAULT_BLOCK_SOURCE_TYPE, createdAt = "" }) {
   if (!id) throw new Error("a block needs an id");
   const bdir = blocksDir(dir);
   await mkdir(bdir, { recursive: true });
   const file = `${safeName(id)}.txt`;
   await atomicWrite(join(bdir, file), String(text ?? ""));
   const index = await loadBlockIndex(dir);
-  index.blocks[id] = { file, tokens: tokenizeBlock(text) };
+  const prior = index.blocks[id];
+  index.blocks[id] = {
+    file, tokens: tokenizeBlock(text),
+    // first-write-wins createdAt (step (a)) + the block's inherited source trust (step (d)).
+    createdAt: prior?.createdAt || createdAt || new Date().toISOString(),
+    sourceType, trust: blockTrust(sourceType),
+  };
   rerank(index);
   await atomicWrite(join(bdir, INDEX_NAME), JSON.stringify(index));
   return index.blocks[id];
@@ -189,7 +204,12 @@ export async function retrieveBlocks(dir, query, k = 3) {
     for (const t of qTokens) if (sets[i].has(t)) idfSum += idf.get(t);
     if (idfSum <= 0) continue;
     const rank = b.rank ?? 0;
-    scored.push({ id, score: idfSum * (1 + rank), rank, file: b.file });
+    // relevance × connectivity × TRUST — a bounded trustFactor (~[0.5, 1.5]) so a
+    // corroborated/operator block outranks a lone low-trust one on a relevance tie,
+    // yet a weakly-trusted but perfectly-relevant block still surfaces.
+    const trust = typeof b.trust === "number" ? b.trust : blockTrust(b.sourceType);
+    const trustFactor = trustFactorOf(trust);
+    scored.push({ id, score: idfSum * (1 + rank) * trustFactor, rank, trust, file: b.file });
   }
   scored.sort((a, b) => b.score - a.score || b.rank - a.rank || a.id.localeCompare(b.id));
   const top = scored.slice(0, Math.max(1, k));