@polycode-projects/the-mechanical-code-talker 6.0.19 → 6.0.21

package/corpus/tier2/manifest.json

@@ -2,48 +2,6 @@
   "version": 1,
   "generated": "by corpus/tier2/generate.mjs",
   "corpuses": [
-    {
-      "id": "aws",
-      "kind": "domain",
-      "description": "Amazon Web Services core services and primitives (S3, Lambda, DynamoDB, EC2, IAM, SQS) mapped to general cloud/CS concepts.",
-      "source": {
-        "kind": "curated",
-        "tool": "corpus/tier2/generate.mjs"
-      },
-      "file": "aws.jsonl",
-      "facts": 39,
-      "bytes": 4802,
-      "sha256": "7ab3e656bc4bc717be3c50d97af030e8592d6597cde182b15390a1dfaca9755a",
-      "license": "MPL-2.0"
-    },
-    {
-      "id": "python",
-      "kind": "language",
-      "description": "Python language constructs and stdlib types mapped to the shared CS concept vocabulary (list->array, dict->hash table, …).",
-      "source": {
-        "kind": "curated",
-        "tool": "corpus/tier2/generate.mjs"
-      },
-      "file": "python.jsonl",
-      "facts": 30,
-      "bytes": 3794,
-      "sha256": "009245d024357bc4285e967fba486e32349ccec8fbe2796ba13011cb2be4c19c",
-      "license": "MPL-2.0"
-    },
-    {
-      "id": "java",
-      "kind": "language",
-      "description": "Java language and JVM constructs mapped to the shared CS concept vocabulary (ArrayList->list, HashMap->hash table, …).",
-      "source": {
-        "kind": "curated",
-        "tool": "corpus/tier2/generate.mjs"
-      },
-      "file": "java.jsonl",
-      "facts": 31,
-      "bytes": 3920,
-      "sha256": "d089426833c393f6f1574fe75c4ac94a1483ee80f72f79bd7fd32aaaa24a6e44",
-      "license": "MPL-2.0"
-    },
     {
       "id": "general",
       "kind": "domain",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polycode-projects/the-mechanical-code-talker",
-  "version": "6.0.19",
+  "version": "6.0.21",
   "private": false,
   "type": "module",
   "description": "The Mechanical Code Talker (tmct) — a tolerant, offline, $0 chat surface that guides you toward precision queries about a software repository. ELIZA/PARRY-style but domain-obsessed with code. No model calls; indexes a repo on request (tmct index) or reads any producer's graph.",
@@ -140,14 +140,14 @@
     "chat:repo": "node --disable-warning=ExperimentalWarning bin/tmct.mjs chat --repo",
     "chat:plain": "node --disable-warning=ExperimentalWarning bin/tmct.mjs chat --plain",
     "chat:narrate": "node --disable-warning=ExperimentalWarning bin/tmct.mjs chat --narrate",
-    "init": "node --disable-warning=ExperimentalWarning bin/tmct.mjs init && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus seon && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus conceptnet && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus aws && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus python && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus java",
+    "init": "node --disable-warning=ExperimentalWarning bin/tmct.mjs init && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus code && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus conceptnet",
     "init:small": "node --disable-warning=ExperimentalWarning bin/tmct.mjs init",
     "init:sqlite": "npm run init:small",
     "init:persona:human": "node --disable-warning=ExperimentalWarning bin/tmct.mjs init --with-persona human",
     "init:persona:empty": "node --disable-warning=ExperimentalWarning bin/tmct.mjs init --with-persona empty",
     "init:large": "npm run init",
-    "init:xl": "node --disable-warning=ExperimentalWarning bin/tmct.mjs init --persona-size large && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus seon && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus conceptnet && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus aws && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus python && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus java && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus wordnet-xl",
-    "init:xxl": "node --disable-warning=ExperimentalWarning bin/tmct.mjs init --persona-size large && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus seon && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus conceptnet && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus aws && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus python && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus java && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus wordnet-full && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus namenet",
+    "init:xl": "node --disable-warning=ExperimentalWarning bin/tmct.mjs init --persona-size large && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus code && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus conceptnet && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus wordnet-xl && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus namenet && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus child",
+    "init:xxl": "node --disable-warning=ExperimentalWarning bin/tmct.mjs init --persona-size large && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus code && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus conceptnet && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus wordnet-full && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus namenet && node --disable-warning=ExperimentalWarning bin/tmct.mjs import --corpus child",
     "memory": "node --disable-warning=ExperimentalWarning bin/tmct.mjs memory",
     "memory:verbose": "node --disable-warning=ExperimentalWarning bin/tmct.mjs memory --verbose",
     "syllogise": "node --disable-warning=ExperimentalWarning bin/tmct.mjs syllogise",

package/src/adapters/corpus/child-seed.mjs

@@ -0,0 +1,74 @@
+// corpus/child-seed.mjs — the BULK reader for the shipped CHILD triples pack
+// (corpus/child/), used when the pack seeds a store as an ordinary corpus band.
+// src/adapters/corpus/child-pack.mjs is the other half: one term at a time, on a
+// miss, for the clean-miss cascade. This one walks every shard once.
+//
+// The shards already carry tmct-vocabulary triples ({subject, predicate,
+// object}), so there is no conceptnet-map.toml step here — the rows come out of
+// the pack build already mapped.
+
+import { readFileSync } from "node:fs";
+import { gunzipSync } from "node:zlib";
+import { join } from "node:path";
+import { CHILD_SHARD_COUNT, isChildFactsRow } from "../../domain/child-pack.mjs";
+import { appendNewFacts, preferThenLimit } from "./conceptnet.mjs";
+
+/** Every shard basename the pack can hold, in order — the naming contract
+ *  shardNameFor writes, walked rather than listed, so this reader needs no
+ *  directory listing and runs in a browser bundle. */
+function shardBasenames() {
+  return Array.from(
+    { length: CHILD_SHARD_COUNT },
+    (_, i) => `child-${i.toString(16).padStart(2, "0")}`,
+  );
+}
+
+/** Every distinct triple in `packDir`'s shards, in shard then row order, as
+ *  appendFact-shaped facts. A term appears in one shard as a subject and in
+ *  another as an object, so the same edge arrives twice — the first spelling
+ *  wins and the duplicate is dropped here, before the store ever sees it.
+ *  Absent or malformed shards contribute nothing rather than throwing, the same
+ *  tolerance the per-term loader gives. */
+export function loadChildPackFacts(packDir, provenancePrefix = "corpus:child") {
+  const facts = [];
+  const seen = new Set();
+  for (const shard of shardBasenames()) {
+    let body;
+    try {
+      body = gunzipSync(readFileSync(join(packDir, "shards", `${shard}.jsonl.gz`))).toString("utf8");
+    } catch {
+      continue;
+    }
+    for (const line of body.split("\n")) {
+      if (!line.trim()) continue;
+      let row;
+      try {
+        row = JSON.parse(line);
+      } catch {
+        continue;
+      }
+      if (!isChildFactsRow(row)) continue;
+      for (const f of row.facts) {
+        const key = `${f.subject} ${f.predicate} ${f.object}`;
+        if (seen.has(key)) continue;
+        seen.add(key);
+        facts.push({
+          subject: f.subject,
+          predicate: f.predicate,
+          object: f.object,
+          provenance: `${provenancePrefix} ${row.term}`,
+        });
+      }
+    }
+  }
+  return facts;
+}
+
+/** Seed `packDir`'s triples into `dir`'s memory through the same idempotent
+ *  append every other corpus band uses. `limit`/`prefer` behave exactly as they
+ *  do for a slice-shaped band. Returns { appended, skipped, total }. */
+export async function seedChildPack(dir, { packDir, provenancePrefix, limit, prefer } = {}) {
+  const facts = preferThenLimit(loadChildPackFacts(packDir, provenancePrefix), prefer, limit);
+  const { appended, skipped } = await appendNewFacts(dir, facts);
+  return { appended, skipped, total: facts.length };
+}

package/src/adapters/corpus/conceptnet.mjs

@@ -25,7 +25,7 @@ export const MAP_FILE = join(PKG_ROOT, "src", "adapters", "corpus", "conceptnet-
 // The tier-1 curated Software-Engineering ontology (SEON): concepts.jsonl shares
 // ConceptNet's slice shape and loads through the same loadSlice/loadMap/toFacts path.
 // definitions.jsonl is a separate {term, definition, sense} list for lexicon lookups.
-// Tier-2 corpuses (aws/python/java) share the slice shape too. The data lives under
+// Tier-2 corpuses share the slice shape too. The data lives under
 // corpus/domains/code/ (the code domain pack's own directory); the bundle name
 // "seon" and its "corpus:seon" provenance prefix are unchanged by the move —
 // both are declared literally in src/services/extensions.mjs, not derived from
@@ -127,35 +127,30 @@ export function toFacts(assertions, map, provenancePrefix = "corpus:conceptnet")
   return facts;
 }
 
-/** Seed a repo's memory graph (<dir>/.tmct/memory/graph.json) from the committed slice.
- *  `limit` caps the facts written; `prefer` stable-partitions facts by predicate (so a
- *  capped seed favors the definitional band over whatever trivia the slice opens with).
- *  Idempotent: pre-reads the store to skip triples already there, then writes survivors
- *  in one batched appendFacts call. Returns { appended, skipped, total }.
- *  `provenancePrefix` tags facts (default "corpus:conceptnet").
- *
- *  `captureUnknownContext` (default false): also runs unknown-ingest.mjs's
- *  ingestUnknownFromAssertions so a term that only appears in an `ace="none"` row (never
- *  reified as a Fact) still lands in memory, tagged with the passage it was found in.
- *  `unknownContextLimit` bounds how many distinct terms one call captures (default 500).
- *  Loaded dynamically to avoid a load-time import cycle with unknown-ingest.mjs. */
-export async function seedMemory(dir, {
-  limit, slicePath = SLICE_FILE, mapPath = MAP_FILE, prefer, provenancePrefix,
-  captureUnknownContext = false, unknownContextLimit,
-} = {}) {
-  const [assertions, map] = await Promise.all([loadSlice(slicePath), loadMap(mapPath)]);
-  let facts = toFacts(assertions, map, provenancePrefix);
+/** Stable-partition `facts` so the `prefer` predicates come first, then take the
+ *  first `limit` of them. A capped band therefore buys its definitional backbone
+ *  before it buys trivia. Either argument may be absent. */
+export function preferThenLimit(facts, prefer, limit) {
+  let out = facts;
   if (Array.isArray(prefer) && prefer.length) {
     const rank = new Map(prefer.map((p, i) => [p, i]));
-    facts = facts.slice().sort((a, b) => (rank.get(a.predicate) ?? prefer.length) - (rank.get(b.predicate) ?? prefer.length));
+    out = out.slice().sort((a, b) => (rank.get(a.predicate) ?? prefer.length) - (rank.get(b.predicate) ?? prefer.length));
   }
-  if (limit !== undefined) facts = facts.slice(0, limit);
+  return limit === undefined ? out : out.slice(0, limit);
+}
+
+// Keyed with normFactTerm so it matches the store's own normalized read-back.
+const factKey = (s, p, o) => `${normFactTerm(s)} ${p} ${normFactTerm(o)}`;
 
-  // Keyed with normFactTerm so it matches the store's own normalized read-back.
-  const factKey = (s, p, o) => `${normFactTerm(s)} ${p} ${normFactTerm(o)}`;
+/** Write every fact `dir`'s store does not already hold, in one batched append,
+ *  so re-seeding a band is idempotent and a term shared by two bands converges
+ *  to one fact. `memory` is the store's already-loaded contents (callers that
+ *  need it for their own work pass theirs rather than paying a second read).
+ *  Returns { appended, skipped }. */
+export async function appendNewFacts(dir, facts, memory) {
+  const store = memory ?? await loadMemory(dir);
   const existing = new Set();
-  const memory = await loadMemory(dir);
-  for (const ind of memory.individuals || []) {
+  for (const ind of store.individuals || []) {
     if (ind?.class !== "Fact") continue;
     const get = (key) => (ind.attributes || []).find((x) => x.key === key)?.value;
     existing.add(factKey(get("subject"), get("predicate"), get("object")));
@@ -173,6 +168,30 @@ export async function seedMemory(dir, {
     toWrite.push(fact);
   }
   const res = await appendFacts(dir, toWrite);
+  return { appended: res.appended, skipped: skipped + res.skipped };
+}
+
+/** Seed a repo's memory graph (<dir>/.tmct/memory/graph.json) from the committed slice.
+ *  `limit` caps the facts written; `prefer` stable-partitions facts by predicate (so a
+ *  capped seed favors the definitional band over whatever trivia the slice opens with).
+ *  Idempotent: pre-reads the store to skip triples already there, then writes survivors
+ *  in one batched appendFacts call. Returns { appended, skipped, total }.
+ *  `provenancePrefix` tags facts (default "corpus:conceptnet").
+ *
+ *  `captureUnknownContext` (default false): also runs unknown-ingest.mjs's
+ *  ingestUnknownFromAssertions so a term that only appears in an `ace="none"` row (never
+ *  reified as a Fact) still lands in memory, tagged with the passage it was found in.
+ *  `unknownContextLimit` bounds how many distinct terms one call captures (default 500).
+ *  Loaded dynamically to avoid a load-time import cycle with unknown-ingest.mjs. */
+export async function seedMemory(dir, {
+  limit, slicePath = SLICE_FILE, mapPath = MAP_FILE, prefer, provenancePrefix,
+  captureUnknownContext = false, unknownContextLimit,
+} = {}) {
+  const [assertions, map] = await Promise.all([loadSlice(slicePath), loadMap(mapPath)]);
+  const facts = preferThenLimit(toFacts(assertions, map, provenancePrefix), prefer, limit);
+
+  const memory = await loadMemory(dir);
+  const { appended, skipped } = await appendNewFacts(dir, facts, memory);
 
   let unknown;
   if (captureUnknownContext) {
@@ -185,7 +204,7 @@ export async function seedMemory(dir, {
   }
 
   return {
-    appended: res.appended, skipped: skipped + res.skipped, total: facts.length,
+    appended, skipped, total: facts.length,
     ...(unknown ? { unknown } : {}),
   };
 }

package/src/adapters/memory/blocks.mjs

@@ -27,6 +27,12 @@ const MAX_TOKENS_PER_BLOCK = 800; // beyond tokenizeProse's per-doc cap: union o
 
 const blocksDir = (dir) => join(dir, BLOCKS_DIR_REL);
 
+// Codepoint order, never localeCompare — the block ids this settles ties on
+// decide WHICH blocks retrieveBlocks hands back, so a locale-dependent
+// comparison would let two readers retrieve different text from one store.
+// src/domain/memory/fact-order.mjs states the rule for the fact rows.
+const byCodepoint = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
+
 /** Only safe, filesystem-friendly block file names (ids are session uuids or
  *  corpus slugs; anything else is normalized, never trusted into a path). */
 const safeName = (id) => String(id).replace(/[^A-Za-z0-9._-]+/g, "_").slice(0, 120) || "_";
@@ -216,7 +222,7 @@ export async function retrieveBlocks(dir, query, k = 3) {
       id, score: (idfSum * (1 + rank) * trustFactor) / Math.sqrt(1 + degree), rank, trust, file: b.file,
     });
   }
-  scored.sort((a, b) => b.score - a.score || b.rank - a.rank || a.id.localeCompare(b.id));
+  scored.sort((a, b) => b.score - a.score || b.rank - a.rank || byCodepoint(a.id, b.id));
   const top = scored.slice(0, Math.max(1, k));
   for (const hit of top) {
     try { hit.text = await readFile(join(blocksDir(dir), hit.file), "utf8"); }