@polycode-projects/the-mechanical-code-talker 1.3.2 → 1.4.0

package/src/init.mjs

@@ -28,6 +28,7 @@
 import { mkdir, readFile, writeFile, stat } from "node:fs/promises";
 import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+import { stringify as stringifyToml } from "smol-toml";
 
 export const CONFIG_FILE = "tmct.toml";
 export const PROVENANCE_REL = join(".tmct", "init.json");
@@ -55,6 +56,19 @@ export function defaultConfig() {
   };
 }
 
+/** `tmct init --with-persona <name>` presets (Part 7 of the extension-pack
+ *  batch): a named bundle of `extensions`/`bias` overrides, written into
+ *  tmct.toml alongside the plain defaults. `code` is TODAY'S IMPLICIT
+ *  DEFAULT made explicit — an empty `extensions` override (seon + conceptnet
+ *  are already the shipped builtin defaults; nothing to add) plus an
+ *  EXPLICIT bias table naming them both at neutral weight, so a repo that
+ *  chose the `code` persona has a self-documenting tmct.toml rather than
+ *  relying on an unstated implicit default. Kept minimal on purpose — this
+ *  batch's job is the persona SEAM, not a curated library of presets. */
+export const PERSONA_PRESETS = Object.freeze({
+  code: { extensions: {}, bias: { seon: 1.0, conceptnet: 1.0 } },
+});
+
 /** Read this package's version (best-effort, for provenance). */
 async function tmctVersion() {
   try {
@@ -83,7 +97,7 @@ export function renderTomlConfig(config = defaultConfig()) {
   const c = { ...defaultConfig(), ...config };
   const corpus = { ...defaultConfig().corpus, ...(config.corpus || {}) };
   const seed = { ...defaultConfig().seed, ...(config.seed || {}) };
-  return `# tmct.toml — the mechanical code talker, project configuration.
+  const base = `# tmct.toml — the mechanical code talker, project configuration.
 # Written by \`tmct init\`. An ABSENT file means shipped defaults (this file
 # just makes them explicit and editable). Documented in the repository-interface
 # onboarding surface (ROADMAP Phase 8, "Distribution: tmct init").
@@ -109,6 +123,23 @@ enabled = ${seed.enabled ? "true" : "false"}
 # To cap it, uncomment and set a number (definitional band first):
 ${seed.limit != null ? `limit = ${Number(seed.limit)}` : "# limit = 500"}
 `;
+  // Extension-pack / bias sections (src/extensions.mjs) — ONLY emitted when a
+  // caller actually supplies them (an explicit `--with-persona`, or a manual
+  // override); the plain zero-flag `tmct init` output stays BYTE-IDENTICAL to
+  // before this feature existed. Rendered via smol-toml's own stringify (not
+  // hand-written prose like the base file above) — a plain, uncommented
+  // config fragment is honest about being machine-written/round-tripped.
+  const extras = {};
+  if (config.extensions !== undefined) extras.extensions = config.extensions;
+  if (config.bias !== undefined) extras.bias = config.bias;
+  if (!Object.keys(extras).length) return base;
+  return `${base}
+# Extension packs + bias (src/extensions.mjs) — written by \`tmct init --with-persona\`
+# or a manual edit. Recognized names (seon, conceptnet, tier2-aws, tier2-python,
+# tier2-java) override the shipped defaults; any other name declares a new
+# host-supplied bundle (needs its own "kind"). [bias] is a flat bundle-name ->
+# weight table consumed by src/memory/bias.mjs's ranking.
+${stringifyToml(extras)}`;
 }
 
 /** Should the seed run? Explicit `opts.seed` wins; otherwise the config's
@@ -132,13 +163,20 @@ function seedRequested({ optSeed, configEnabled, env }) {
  *   `seed.enabled` (TMCT_NO_SEED still vetoes).
  * @param {object}  [opts.env]    environment (for TMCT_NO_SEED); defaults to
  *   process.env.
+ * @param {object}  [opts.persona] a resolved PERSONA_PRESETS entry
+ *   ({extensions?, bias?}) to merge into the FRESH config before it's written
+ *   (Part 7, `tmct init --with-persona <name>`). Name -> preset resolution
+ *   and unknown-name validation are the CALLER'S job (bin/tmct.mjs) — this
+ *   only ever sees an already-resolved preset object (or nothing). Has no
+ *   effect when tmct.toml already exists and `force` isn't set (the existing
+ *   "preserve a user's tmct.toml" rule wins, same as `seed`/`corpus.tier`).
  * @returns {Promise<{
  *   created: string[], config: object, seeded: boolean,
  *   alreadyInitialized: boolean, seedResult: (object|null), message: string
  * }>} `created` lists the ABSOLUTE paths this call brought into being (empty on a
  *   benign no-op re-init). Never throws on a benign re-init or a corpus failure.
  */
-export async function initRepo(dir, { force = false, seed, env = process.env } = {}) {
+export async function initRepo(dir, { force = false, seed, env = process.env, persona = null } = {}) {
   const root = resolve(dir);
   const created = [];
   const paths = {
@@ -162,6 +200,15 @@ export async function initRepo(dir, { force = false, seed, env = process.env } =
 
   // ---- 2. The externalised config (preserve an existing file unless force) ----
   let config = defaultConfig();
+  // Persona overrides (Part 7) apply ONLY to a FRESH write — an empty preset
+  // field (e.g. `code`'s `extensions: {}`) is a genuine no-op, never an
+  // explicit-empty-section write (renderTomlConfig only emits [extensions]/
+  // [bias] when the merged config actually carries a non-empty one — the
+  // plain zero-flag `tmct init` output stays byte-identical either way).
+  if (persona) {
+    if (persona.extensions && Object.keys(persona.extensions).length) config.extensions = persona.extensions;
+    if (persona.bias && Object.keys(persona.bias).length) config.bias = persona.bias;
+  }
   const tomlPresent = await exists(paths.toml);
   if (!tomlPresent || force) {
     await writeFile(paths.toml, renderTomlConfig(config));
@@ -172,7 +219,16 @@ export async function initRepo(dir, { force = false, seed, env = process.env } =
     config = await readWrittenConfig(paths.toml, config);
   }
 
-  // ---- 3. Seed the tier-1 committed corpus (offline, failure-tolerant) ----
+  // ---- 3. Seed the committed corpus (offline, failure-tolerant) ----
+  // DELIBERATE BUG FIX (this batch): `tmct init`'s zero-flag seed step used to
+  // seed ONLY the ConceptNet band, never the curated SEON ontology — unlike
+  // chat.mjs's own first-run bootstrap (seedBootstrapMemory), which has always
+  // seeded BOTH. Both now go through the SAME unified loop
+  // (src/extensions.mjs's resolveExtensions + seedActiveCorpusEntries), so
+  // `tmct init`'s seed matches chat's bootstrap exactly — this changes
+  // `initRepo`'s seeded fact COUNT (test/init.test.mjs's seed-count assertions
+  // were updated for the larger post-fix totals, deliberately, in the same
+  // commit as this fix).
   let seeded = false;
   let seedResult = null;
   let seedNote = "";
@@ -185,18 +241,42 @@ export async function initRepo(dir, { force = false, seed, env = process.env } =
     seedNote = "seed skipped (already seeded — marker present)";
   } else {
     try {
-      const { seedMemory } = await import("./corpus/conceptnet.mjs");
-      const limit = config.seed?.limit != null ? Number(config.seed.limit) : SEED_LIMIT;
-      seedResult = await seedMemory(root, { limit, prefer: SEED_PREFER });
+      const { resolveExtensions, seedActiveCorpusEntries } = await import("./extensions.mjs");
+      const { entries } = await resolveExtensions(root);
+      // `tmct.toml`'s `[seed] limit` knob is documented as capping the tier-1
+      // ConceptNet band specifically (the curated SEON ontology is small and
+      // always seeds whole) — so it overrides ONLY the resolved "conceptnet"
+      // entry's limit, exactly like the pre-fix single-corpus seed did.
+      if (config.seed?.limit != null && entries.has("conceptnet")) {
+        entries.set("conceptnet", { ...entries.get("conceptnet"), limit: Number(config.seed.limit) });
+      }
+      const { appended, skipped, total, perBundle } = await seedActiveCorpusEntries(root, entries);
+      // seedActiveCorpusEntries is failure-tolerant PER BUNDLE (a bad third-party
+      // pack never aborts the others) — but initRepo's own "FAILURE-TOLERANT
+      // SEED" contract is about the SEED AS A WHOLE degrading honestly. If every
+      // active bundle failed (e.g. the memory graph file itself is unwritable —
+      // see test/init.test.mjs "seed failure degrades"), re-throw the first
+      // bundle's error so the SAME outer catch below reports the familiar "seed
+      // skipped (corpus unavailable: …)" note, rather than claiming success with
+      // zero facts actually written.
+      const bundleNames = Object.keys(perBundle);
+      const allFailed = bundleNames.length > 0 && bundleNames.every((n) => perBundle[n].error);
+      if (allFailed) throw new Error(perBundle[bundleNames[0]].error);
+      seedResult = {
+        appended, skipped, total, perBundle,
+        seon: perBundle.seon?.appended || 0,
+        conceptnet: perBundle.conceptnet?.appended || 0,
+      };
       const markerNew = !(await exists(paths.marker));
       await mkdir(dirname(paths.marker), { recursive: true });
       await writeFile(
         paths.marker,
         JSON.stringify({
           seededAt: new Date().toISOString(),
-          limit,
+          limit: config.seed?.limit != null ? Number(config.seed.limit) : SEED_LIMIT,
           appended: seedResult.appended,
           skipped: seedResult.skipped,
+          perBundle,
         }) + "\n",
       );
       if (markerNew) created.push(paths.marker);
@@ -245,6 +325,11 @@ async function readWrittenConfig(tomlPath, base) {
       if (raw.seed.enabled !== undefined) cfg.seed.enabled = Boolean(raw.seed.enabled);
       if (raw.seed.limit !== undefined) cfg.seed.limit = Number(raw.seed.limit);
     }
+    // Sparse pass-through (src/extensions.mjs validates; this layer just carries
+    // the raw tables through unmodified, same discipline as toml-config.mjs's
+    // normalizeConfig).
+    if (raw.extensions !== undefined) cfg.extensions = raw.extensions;
+    if (raw.bias !== undefined) cfg.bias = raw.bias;
     return cfg;
   } catch {
     return base;

package/src/memory/bias.mjs

@@ -0,0 +1,77 @@
+// memory/bias.mjs — bias-weighted fact ranking (extension-pack batch, Part 6).
+//
+// A small, PURE module, deliberately SEPARATE from trust.mjs's existing closed
+// 3-input computeTrust contract (many tests pin computeTrust's exact
+// signature; this module never touches it). Trust answers "how much do I
+// believe this fact"; bias answers a DIFFERENT question an operator asks
+// explicitly — "when two facts disagree, which BUNDLE do I prefer to hear
+// from first" (the worked example: is a class part of code, or part of a
+// school — a `[bias]` table lets an operator say "for MY repo, weight the
+// code-vocabulary bundle over the general-English one").
+//
+//   biasForSourceId(sourceId, biasByBundle)  "src:corpus:<name>" -> weight (default 1)
+//   biasForRow(row, biasByBundle)            max bias across row.sourceIds
+//   rankByBiasThenTrust(rows, biasByBundle)  stable: bias desc, trust desc, original order
+//
+// biasByBundle is the flat `{ bundleName: number }` table src/extensions.mjs's
+// resolveExtensions() reads out of tmct.toml's `[bias]` table — resolved ONCE
+// per chat session and threaded through, never re-read per turn.
+//
+// CRITICAL CONTRACT (the operator's own "disclosed, never dropped"
+// requirement): bias only REORDERS a hit list. It must NEVER drop, hide, or
+// silently prefer a lower-biased fact over a higher-trust one within the same
+// bias tier — every hit rankByBiasThenTrust is given still comes back out,
+// same length, same members, just reordered.
+
+/** A Fact's statedBy Source id is shaped "src:corpus:<bundleName>" for every
+ *  corpus-kind Source (memory/core.mjs's own sourceIdFor — see its `corpus`
+ *  case). Any other shape (operator/teach/provider/web/entailed Source ids,
+ *  or a malformed/absent id) is NOT a corpus bundle and always ranks at the
+ *  neutral bias of 1 — bias is a corpus-bundle-only concept, never a proxy
+ *  for a different trust dimension. */
+const CORPUS_SOURCE_RE = /^src:corpus:(.+)$/;
+
+/** The bias weight of one Source id, resolved against `biasByBundle`. Default
+ *  1 (neutral) for a non-corpus source id, an unconfigured bundle name, or a
+ *  non-numeric configured value (never throws — a malformed `[bias]` entry is
+ *  caught earlier, at resolveExtensions() load time). */
+export function biasForSourceId(sourceId, biasByBundle = {}) {
+  const m = CORPUS_SOURCE_RE.exec(String(sourceId || ""));
+  if (!m) return 1;
+  const v = biasByBundle?.[m[1]];
+  return typeof v === "number" && Number.isFinite(v) ? v : 1;
+}
+
+/** The bias weight of one fact ROW (readFactRows' shape: `{..., sourceIds}`)
+ *  — the MAX bias across its (possibly several, corroborating) Sources, so a
+ *  fact corroborated by both a neutral and a high-bias bundle ranks at the
+ *  higher of the two, never averaged down. A row with no sourceIds (or an
+ *  empty array) ranks at the neutral 1. */
+export function biasForRow(row, biasByBundle = {}) {
+  const ids = Array.isArray(row?.sourceIds) ? row.sourceIds : [];
+  if (!ids.length) return 1;
+  return Math.max(...ids.map((id) => biasForSourceId(id, biasByBundle)));
+}
+
+/**
+ * Rank fact rows by bias (desc), then trust (desc), then original relative
+ * order (STABLE — Array.prototype.sort is stable in Node, reinforced here
+ * with an explicit index tiebreak so the guarantee never depends on engine
+ * internals). Every row that goes in comes back out — same length, same
+ * members — this ONLY reorders, it never filters or drops a hit, honouring
+ * the "disclosed, never dropped" contract every caller in chat.mjs relies on.
+ *
+ * With an empty/absent `biasByBundle` (the default, unconfigured case) every
+ * row's bias is 1 — a true no-op tier, so the sort degrades to trust-desc,
+ * ties broken by original order: BYTE-IDENTICAL to today's behaviour for any
+ * caller that previously sorted by trust alone (or didn't sort at all and
+ * every row happened to share the same trust, the common single-session
+ * operator-taught case).
+ */
+export function rankByBiasThenTrust(rows, biasByBundle = {}) {
+  const list = Array.isArray(rows) ? rows : [];
+  return list
+    .map((row, index) => ({ row, index, bias: biasForRow(row, biasByBundle) }))
+    .sort((a, b) => (b.bias - a.bias) || ((b.row?.trust ?? 0) - (a.row?.trust ?? 0)) || (a.index - b.index))
+    .map((x) => x.row);
+}

package/src/memory/blocks.mjs

@@ -15,8 +15,12 @@
 //   2. QUERY-time IDF match — each query token is weighted by rarity across
 //      blocks (idf = log(1 + N/(1+df)), the codegraph.mjs locate discipline),
 //      so a whole-question query is not dominated by its ubiquitous words.
-// retrieveBlocks() scores idf-sum × (1 + rank): the IDF match decides topic,
-// the static rank breaks ties toward well-connected blocks.
+// retrieveBlocks() scores idf-sum × (1 + rank) ÷ sqrt(1 + degree): the IDF
+// match decides topic, the static rank breaks ties toward well-connected
+// blocks, and the degree divisor (hub dampening, ported from codegraph.mjs's
+// spiralExpand degree-quantile gate; on by default here) stops a block that
+// merely shares vocabulary with disproportionately many others — a generic
+// "boilerplate" hub — from winning on inflated rank alone.
 //
 // All writes are temp+rename atomic; saveBlock is an upsert (same id replaces —
 // what makes fold.mjs's re-fold idempotent).
@@ -79,20 +83,15 @@ export async function loadBlockIndex(dir) {
 }
 
 /**
- * Iterative PageRank over the block-similarity graph. `tokensById` is a plain
- * { id: tokens[] } map; an undirected edge joins two blocks sharing at least
- * `overlapMin` tokens. Standard damped iteration (d=0.85, 20 rounds), dangling
- * mass redistributed evenly, ranks summing to ~1. Pure — returns { id: rank }.
+ * Build the block-similarity adjacency shared by rankBlocks and degreeOf:
+ * `tokensById` is a plain { id: tokens[] } map; an undirected edge joins two
+ * blocks sharing at least `overlapMin` tokens. Returns { ids, neighbours }
+ * (neighbours[i] is an array of adjacent indices into ids).
  */
-export function rankBlocks(tokensById, {
-  damping = PAGERANK_DAMPING, iterations = PAGERANK_ITERATIONS, overlapMin = OVERLAP_MIN,
-} = {}) {
+function buildNeighbours(tokensById, overlapMin) {
   const ids = Object.keys(tokensById || {});
   const N = ids.length;
-  if (!N) return {};
   const sets = ids.map((id) => new Set(tokensById[id] || []));
-
-  // similarity edges: shared-token count ≥ overlapMin (undirected → both directions)
   const neighbours = ids.map(() => []);
   for (let i = 0; i < N; i += 1) {
     for (let j = i + 1; j < N; j += 1) {
@@ -107,6 +106,35 @@ export function rankBlocks(tokensById, {
       }
     }
   }
+  return { ids, neighbours };
+}
+
+/**
+ * Each block's degree (neighbour count) in the same block-similarity graph
+ * rankBlocks runs PageRank over. Pure — returns { id: degree }. Used to
+ * dampen retrieveBlocks' hub bias (a block linked to many others shouldn't
+ * win purely by being well-connected — codegraph.mjs's spiralExpand applies
+ * the same degree-quantile idea to module search).
+ */
+export function degreeOf(tokensById, { overlapMin = OVERLAP_MIN } = {}) {
+  const { ids, neighbours } = buildNeighbours(tokensById, overlapMin);
+  const out = {};
+  for (let i = 0; i < ids.length; i += 1) out[ids[i]] = neighbours[i].length;
+  return out;
+}
+
+/**
+ * Iterative PageRank over the block-similarity graph. `tokensById` is a plain
+ * { id: tokens[] } map; an undirected edge joins two blocks sharing at least
+ * `overlapMin` tokens. Standard damped iteration (d=0.85, 20 rounds), dangling
+ * mass redistributed evenly, ranks summing to ~1. Pure — returns { id: rank }.
+ */
+export function rankBlocks(tokensById, {
+  damping = PAGERANK_DAMPING, iterations = PAGERANK_ITERATIONS, overlapMin = OVERLAP_MIN,
+} = {}) {
+  const { ids, neighbours } = buildNeighbours(tokensById, overlapMin);
+  const N = ids.length;
+  if (!N) return {};
 
   let rank = new Array(N).fill(1 / N);
   for (let round = 0; round < iterations; round += 1) {
@@ -133,7 +161,11 @@ function rerank(index) {
   const tokensById = {};
   for (const [id, b] of Object.entries(index.blocks)) tokensById[id] = b.tokens || [];
   const ranks = rankBlocks(tokensById);
-  for (const [id, b] of Object.entries(index.blocks)) b.rank = ranks[id] ?? 0;
+  const degrees = degreeOf(tokensById);
+  for (const [id, b] of Object.entries(index.blocks)) {
+    b.rank = ranks[id] ?? 0;
+    b.degree = degrees[id] ?? 0;
+  }
   return index;
 }
 
@@ -177,9 +209,10 @@ export async function removeBlock(dir, id) {
 /**
  * The top-k blocks a question "touches": IDF-weighted token match (rarity-
  * weighted, so common words can't dominate) combined with the static PageRank
- * (score × (1 + rank) — on an IDF tie the better-connected block wins).
- * Returns [{ id, score, rank, file, text }], best first; [] when nothing
- * matches (never a guessed block).
+ * (score × (1 + rank), divided by sqrt(1 + degree) to dampen hubs — a block
+ * that shares vocabulary with disproportionately many others doesn't win on
+ * inflated rank alone). Returns [{ id, score, rank, file, text }], best
+ * first; [] when nothing matches (never a guessed block).
  */
 export async function retrieveBlocks(dir, query, k = 3) {
   const index = await loadBlockIndex(dir);
@@ -204,12 +237,18 @@ export async function retrieveBlocks(dir, query, k = 3) {
     for (const t of qTokens) if (sets[i].has(t)) idfSum += idf.get(t);
     if (idfSum <= 0) continue;
     const rank = b.rank ?? 0;
+    const degree = b.degree ?? 0;
     // relevance × connectivity × TRUST — a bounded trustFactor (~[0.5, 1.5]) so a
     // corroborated/operator block outranks a lone low-trust one on a relevance tie,
-    // yet a weakly-trusted but perfectly-relevant block still surfaces.
+    // yet a weakly-trusted but perfectly-relevant block still surfaces. Divided by
+    // sqrt(1 + degree) — hub dampening (the codegraph.mjs spiralExpand idea, ported
+    // here): a block that shares vocabulary with disproportionately many others
+    // shouldn't win purely by being well-connected.
     const trust = typeof b.trust === "number" ? b.trust : blockTrust(b.sourceType);
     const trustFactor = trustFactorOf(trust);
-    scored.push({ id, score: idfSum * (1 + rank) * trustFactor, rank, trust, file: b.file });
+    scored.push({
+      id, score: (idfSum * (1 + rank) * trustFactor) / Math.sqrt(1 + degree), rank, trust, file: b.file,
+    });
   }
   scored.sort((a, b) => b.score - a.score || b.rank - a.rank || a.id.localeCompare(b.id));
   const top = scored.slice(0, Math.max(1, k));