@polycode-projects/the-mechanical-code-talker 1.9.1 → 1.10.0

package/src/corpus/conceptnet.mjs

@@ -1,24 +1,14 @@
-// corpus/conceptnet.mjs — the ConceptNet slice loader + memory seeder
-// (ROADMAP Phase 2, "ConceptNet corpus slice").
+// corpus/conceptnet.mjs — the ConceptNet slice loader + memory seeder.
 //
 //   loadSlice(path?)        stream corpus/conceptnet/slice.jsonl → assertions
 //   loadMap(path?)          src/corpus/conceptnet-map.toml → Map(rel → row)
 //   toFacts(assertions,map) assertions → appendFact-shaped triples
-//   seedMemory(dir, opts)   write them into <dir>/.tmct/memory via appendFacts (one batched write)
+//   seedMemory(dir, opts)   write them into <dir>/.tmct/memory via appendFacts
 //
-// The slice is committed data (one JSON object per line: {start, rel, end,
-// surfaceText?, weight}; en→en only; CC-BY-SA 4.0 for ConceptNet-derived rows
-// — see corpus/conceptnet/LICENSE-NOTICE). The mapping table decides which
-// relations become memory facts and under which predicate URI; rows marked
-// ace = "none" are deliberate non-emissions. A slice relation MISSING from
-// the table is a drift error — loud, never guessed around.
-//
-// Seeding goes through src/memory/core.mjs appendFacts() (memory is import-only
-// here): fact ids are content-hashed from (s,p,o), so re-seeding is idempotent by
-// construction. seedMemory pre-loads the store once and skips triples already
-// present, then hands the survivors to appendFacts as ONE batched read-modify-
-// write — so seeding the whole slice is O(N), not the O(N²) a per-fact appendFact
-// loop would incur (the 6 k-fact slice: ~7 min → a couple of seconds).
+// The slice is committed data (en→en only; CC-BY-SA 4.0 — see
+// corpus/conceptnet/LICENSE-NOTICE). The mapping table decides which relations become
+// memory facts and under which predicate; a slice relation missing from the table is a
+// drift error, loud, never guessed around.
 
 import { createReadStream } from "node:fs";
 import { readFile } from "node:fs/promises";
@@ -32,29 +22,18 @@ const PKG_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
 export const SLICE_FILE = join(PKG_ROOT, "corpus", "conceptnet", "slice.jsonl");
 export const MAP_FILE = join(PKG_ROOT, "src", "corpus", "conceptnet-map.toml");
 
-// The tier-1 curated Software-Engineering ontology (SEON). concepts.jsonl is in
-// the SAME slice shape as ConceptNet ({start, rel, end, weight}), so it loads +
-// maps through the identical loadSlice/loadMap/toFacts path — just with a
-// "corpus:seon" provenance prefix. definitions.jsonl is a separate {term,
-// definition, sense} list the chat answer layer prefers for a lexicon term's
-// "what is a <term>". tier-2 corpuses (aws/python/java) share the slice shape too.
+// The tier-1 curated Software-Engineering ontology (SEON): concepts.jsonl shares
+// ConceptNet's slice shape and loads through the same loadSlice/loadMap/toFacts path.
+// definitions.jsonl is a separate {term, definition, sense} list for lexicon lookups.
+// Tier-2 corpuses (aws/python/java) share the slice shape too.
 export const SEON_CONCEPTS_FILE = join(PKG_ROOT, "corpus", "seon", "concepts.jsonl");
 export const SEON_DEFINITIONS_FILE = join(PKG_ROOT, "corpus", "seon", "definitions.jsonl");
 export const TIER2_DIR = join(PKG_ROOT, "corpus", "tier2");
 export const TIER2_MANIFEST_FILE = join(TIER2_DIR, "manifest.json");
 
-// corpus/wordnet/generate.mjs's output: the Open English WordNet ->
-// ConceptNet-shape conversion, same slice shape/loader path as tier-1/tier-2.
-// Lives alongside its own generator (corpus/conceptnet/'s own precedent:
-// fetch-slice.mjs + slice.jsonl share one directory) rather than under a new
-// "tier-3" name — corpus/README.md's tiering policy already uses "tier-3" for
-// something else entirely (runtime-learned facts, NEVER committed), and this
-// bundle is curated + committed, the same shape as a tier-2 corpus, just too
-// large/mechanically-derived to hand-curate. "wordnet-xl"/"wordnet-full" are
-// wired as BUILTIN_EXTENSIONS corpus entries in src/extensions.mjs, so
-// `tmct import --corpus wordnet-xl` resolves directly there rather than
-// through TIER2_MANIFEST_FILE's id lookup — see that module's own
-// BUILTIN_EXTENSIONS comment.
+// corpus/wordnet/generate.mjs's output: the Open English WordNet -> ConceptNet-shape
+// conversion, same slice shape/loader path as tier-1/tier-2. "wordnet-xl"/"wordnet-full"
+// are wired as BUILTIN_EXTENSIONS corpus entries in src/extensions.mjs.
 export const WORDNET_DIR = join(PKG_ROOT, "corpus", "wordnet");
 export const WORDNET_MANIFEST_FILE = join(WORDNET_DIR, "manifest.json");
 
@@ -114,17 +93,10 @@ export const termText = (uri) => {
   return m ? m[1].replace(/_/g, " ") : null;
 };
 
-/** Map slice assertions → appendFact-shaped triples:
- *    { subject, predicate, object, provenance }
- *  (provenance is a STRING — exactly what src/memory/core.mjs appendFact
- *  takes; it names the corpus and the originating ConceptNet relation).
- *  Rows whose relation maps ace="none" are skipped — deliberate non-emission.
- *  A relation with NO row in the map throws: that is table drift, not data.
- *
- *  `provenancePrefix` names the corpus half of the provenance string; it defaults
- *  to "corpus:conceptnet" so the ConceptNet seed stays BYTE-IDENTICAL to before.
- *  The seon / tier-2 corpuses reuse the same slice shape, tagged "corpus:seon" or
- *  "corpus:tier2:<id>" so a reader can tell a curated SE fact from ConceptNet noise. */
+/** Map slice assertions → appendFact-shaped triples { subject, predicate, object,
+ *  provenance }. Rows whose relation maps ace="none" are skipped; a relation with no row
+ *  in the map throws (table drift, not data). `provenancePrefix` tags the corpus half of
+ *  provenance, e.g. "corpus:seon" / "corpus:tier2:<id>" for non-ConceptNet callers. */
 export function toFacts(assertions, map, provenancePrefix = "corpus:conceptnet") {
   const facts = [];
   for (const a of assertions) {
@@ -136,10 +108,8 @@ export function toFacts(assertions, map, provenancePrefix = "corpus:conceptnet")
     const subject = termText(a.start);
     const object = termText(a.end);
     if (!subject || !object) continue; // non-en endpoint slipped in — filtered, not fatal
-    // mgx:relatedTo is real but low-precision (undirected, ambiguous) — routed
-    // through the corpus-weak: prefix so memory/trust.mjs's SOURCE_PRIOR.corpusWeak
-    // (below plain corpus, above web) applies, computed from the Source's kind
-    // like every other tier, never hand-set on the Fact.
+    // mgx:relatedTo is low-precision (undirected, ambiguous) — routed through the
+    // corpus-weak: prefix so memory/trust.mjs's SOURCE_PRIOR.corpusWeak applies.
     const prefix = row.predicate === "mgx:relatedTo"
       ? provenancePrefix.replace(/^corpus:/, "corpus-weak:")
       : provenancePrefix;
@@ -153,39 +123,18 @@ export function toFacts(assertions, map, provenancePrefix = "corpus:conceptnet")
   return facts;
 }
 
-/** Seed a repo's memory graph (<dir>/.tmct/memory/graph.json) from the
- *  committed slice. Options: limit (cap the facts written — handy for tests
- *  and fast bootstraps), slicePath/mapPath overrides, and `prefer` — an array
- *  of predicate URIs that STABLE-partitions the facts before the limit is
- *  applied (facts whose predicate appears earlier in `prefer` come first;
- *  everything else keeps slice order after them). A capped bootstrap seed
- *  wants the DEFINITIONAL band ("a cache is a kind of buffer") ahead of the
- *  location trivia the slice happens to open with; without `prefer` the
- *  behavior is byte-identical to before.
- *
- *  Idempotent twice over: appendFacts' content-hashed ids make a blind
- *  re-append an upsert, and we pre-read the store once to skip triples that
- *  are already there (so re-seeding costs one read, not N rewrites). The
- *  survivors are written in ONE batched appendFacts call, not a per-fact loop.
- *  Returns { appended, skipped, total }. `provenancePrefix` is threaded through to
- *  toFacts (default "corpus:conceptnet" → byte-identical seed) so a seon/tier-2
- *  corpus can tag its facts "corpus:seon" / "corpus:tier2:<id>".
+/** Seed a repo's memory graph (<dir>/.tmct/memory/graph.json) from the committed slice.
+ *  `limit` caps the facts written; `prefer` stable-partitions facts by predicate (so a
+ *  capped seed favors the definitional band over whatever trivia the slice opens with).
+ *  Idempotent: pre-reads the store to skip triples already there, then writes survivors
+ *  in one batched appendFacts call. Returns { appended, skipped, total }.
+ *  `provenancePrefix` tags facts (default "corpus:conceptnet").
  *
- *  `captureUnknownContext` (default false — every existing call stays
- *  byte-identical): when true, also runs corpus/unknown-ingest.mjs's
- *  `ingestUnknownFromAssertions` over this same assertions/map pair — the
- *  PLAN_AGENTS.md §4 "context-preserving ingestion for unknown words"
- *  mechanism — so a term that ONLY ever appears in a row `toFacts` silently
- *  drops (an `ace = "none"` relation like RelatedTo/HasContext) still lands
- *  in memory, tagged with the passage it was found in, instead of vanishing.
- *  `unknownContextLimit` bounds how many distinct unknown terms one call
- *  captures (default 500 — see that module's own doc comment for why an
- *  unbounded sweep over a wide slice would not be "bounded, not padding").
- *  The result's `unknown` key is present only when the flag is set. Loaded
- *  dynamically (not a static import) to avoid a load-time import cycle with
- *  unknown-ingest.mjs, which itself statically imports `termText` from here —
- *  the same "avoid the cycle" discipline extensions.mjs's
- *  seedActiveCorpusEntries already uses for this very module. */
+ *  `captureUnknownContext` (default false): also runs unknown-ingest.mjs's
+ *  ingestUnknownFromAssertions so a term that only appears in an `ace="none"` row (never
+ *  reified as a Fact) still lands in memory, tagged with the passage it was found in.
+ *  `unknownContextLimit` bounds how many distinct terms one call captures (default 500).
+ *  Loaded dynamically to avoid a load-time import cycle with unknown-ingest.mjs. */
 export async function seedMemory(dir, {
   limit, slicePath = SLICE_FILE, mapPath = MAP_FILE, prefer, provenancePrefix,
   captureUnknownContext = false, unknownContextLimit,
@@ -194,15 +143,11 @@ export async function seedMemory(dir, {
   let facts = toFacts(assertions, map, provenancePrefix);
   if (Array.isArray(prefer) && prefer.length) {
     const rank = new Map(prefer.map((p, i) => [p, i]));
-    // stable partition: Array.prototype.sort is stable in Node, so equal-rank
-    // facts keep their slice order — deterministic across runs by construction.
     facts = facts.slice().sort((a, b) => (rank.get(a.predicate) ?? prefer.length) - (rank.get(b.predicate) ?? prefer.length));
   }
   if (limit !== undefined) facts = facts.slice(0, limit);
 
-  // One read up front: what does the store already reify? Keys are built with
-  // memory's own normFactTerm so they match the normalized read-back exactly
-  // (appendFact converges /c/en/foo_bar, tmct:Foo and "Foo bar" to one term).
+  // Keyed with normFactTerm so it matches the store's own normalized read-back.
   const factKey = (s, p, o) => `${normFactTerm(s)} ${p} ${normFactTerm(o)}`;
   const existing = new Set();
   const memory = await loadMemory(dir);
@@ -223,9 +168,6 @@ export async function seedMemory(dir, {
     existing.add(key);
     toWrite.push(fact);
   }
-  // ONE read-modify-write for the whole seed (was one per fact — O(N²) I/O, ~7 min
-  // for the 6 k-fact slice). appendFacts also skips any malformed row rather than
-  // throwing, so its skipped count folds into the dedup skips here.
   const res = await appendFacts(dir, toWrite);
 
   let unknown;

package/src/corpus/templates.mjs

@@ -1,5 +1,5 @@
-// corpus/templates.mjs — the response-template library + SE phrase book loaders
-// (ROADMAP Phase 2, items 4+7). Plain diffable data in, strict renderers out:
+// corpus/templates.mjs — the response-template library + SE phrase book loaders.
+// Plain diffable data in, strict renderers out:
 //
 //   data/templates/responses.jsonl   {id, class, template, register} rows
 //   data/phrasebook/software-phrases.txt  one phrase pattern per line
@@ -8,8 +8,8 @@
 // loadTemplates() validates the whole file (parse, required fields, unique
 // ids) and caches; render(id, slots) is then synchronous and STRICT — an
 // unknown id or a missing slot throws, it never emits a half-filled sentence.
-// The response surface (Phase 1 pipeline) fills templates from grounded data
-// only, so a thrown slot is a programming error, not a user-facing miss.
+// The response surface fills templates from grounded data only, so a thrown
+// slot is a programming error, not a user-facing miss.
 
 import { readFile } from "node:fs/promises";
 import { fileURLToPath } from "node:url";
@@ -19,20 +19,14 @@ const PKG_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
 export const TEMPLATES_FILE = join(PKG_ROOT, "data", "templates", "responses.jsonl");
 export const PHRASEBOOK_FILE = join(PKG_ROOT, "data", "phrasebook", "software-phrases.txt");
 
-// Registers (Phase 6, archive/PLAN_FORMULAIC_COMPETENCE.md): `terse|friendly` are the
-// conversational bands; `technical` is the C1 / technical-paper band whose
-// templates render item-5 mechanical conclusions (count / comparison /
-// superlative + the provenance we already compute) as advanced prose. A
-// technical template is FORMULAIC COMPETENCE: it renders via:"template", so the
-// dual banding counts it in the PERFORMANCE band only, never the productive one.
+// Registers: `terse|friendly` are the conversational bands; `technical` is the C1
+// technical-paper band, rendering mechanical conclusions (count/comparison/superlative
+// + provenance) as advanced prose.
 const REGISTERS = new Set(["terse", "friendly", "technical"]);
 const SLOT_RE = /\{([A-Za-z][A-Za-z0-9]*)\}/g;
 
-// Slot-lint for the technical band: a technical template may ONLY fill from the
-// mechanical values tmct actually computes (counts, comparisons, superlatives,
-// scopes, provenance) — no free-text slot can smuggle unattributable prose into
-// the C1 register. Every technical row must also carry a {provenance} fill (the
-// item-5 "+ provenance" contract: an advanced claim always shows its source).
+// Slot-lint for the technical band: only mechanical values, no free-text slot — and
+// every technical row must carry a {provenance} fill.
 export const TECHNICAL_SLOTS = Object.freeze(new Set([
   "subject", "count", "noun", "scope", "comparison", "metric", "unit",
   "superlative", "provenance",
@@ -47,18 +41,12 @@ export function slotsOf(template) {
   return out;
 }
 
-// --- Segmentation IR (Phase 7, lever 1) -------------------------------------
+// --- Segmentation IR ---------------------------------------------------------
 // A rendered answer is ALSO a list of typed spans: [{ type, text }, …] with
-// type ∈ prose | entity | path | number | code | provenance | receipt. The
-// invariant law is byte-exact reconstruction: flatten(segments) === render().
-// Everything except `prose` is PROTECTED — finishing (a later wave) may only
-// transform prose spans, so a grammar rule can never touch a fact.
-//
-// Slot kinds map a template hole to its protected span type. A slot fill is
-// ALWAYS protected (never prose): it is grounded data, not our wording. The
-// specific type is derived from the slot name; unknown slots fall back to the
-// conservative `entity` (protect-when-unsure). Bytes never depend on the type,
-// only on the fill, so the type is metadata layered over an exact split.
+// type ∈ prose | entity | path | number | code | provenance | receipt, satisfying
+// flatten(segments) === render(). Everything except `prose` is PROTECTED — a grammar
+// rule may only transform prose spans, never touch a fact. A slot fill is always
+// protected; unknown slot names fall back to the conservative `entity`.
 const SLOT_KIND = {
   count: "number",
   when: "number",
@@ -152,22 +140,11 @@ export async function loadTemplates(path = TEMPLATES_FILE) {
   return byId;
 }
 
-/** Load + merge several EXTENSION-PACK template files (each independently
- *  validated by loadTemplates() — same loud-on-malformed-row guarantee) into
- *  one Map<id,row>. Every extension-pack template id MUST be namespaced
- *  "<packname>:<id>" — enforced here as a validation rule (also the check
- *  Part 4's validateExtensionPack runs on a single candidate file), rather
- *  than inventing a same-id collision-precedence policy: a bare, unnamespaced
- *  id, or the SAME id appearing under two different paths, both throw loudly
- *  naming the offending path/id.
- *
- *  loadTemplates(path) mutates the module's own render()-serving `cache` as a
- *  side effect; this function restores it to whatever it was before the merge
- *  ran, so calling loadTemplatesMerged() never clobbers the "current" default
- *  templates map for an unrelated caller (e.g. a concurrent render() call
- *  elsewhere in the same process). The returned map is NOT installed as the
- *  render() default — a caller that wants render() to serve the merged set
- *  passes it explicitly: render(id, slots, mergedMap). */
+/** Load + merge several EXTENSION-PACK template files into one Map<id,row>. Every
+ *  extension-pack template id must be namespaced "<packname>:<id>" (a bare or duplicate
+ *  id throws, naming the offending path/id). Restores the module's `cache` to its
+ *  pre-call state afterward — the returned map is not installed as the render() default;
+ *  a caller passes it explicitly: render(id, slots, mergedMap). */
 export async function loadTemplatesMerged(paths = []) {
   const savedCache = cache;
   const merged = new Map();
@@ -196,9 +173,6 @@ export async function loadTemplatesMerged(paths = []) {
  *  `templates` explicitly to bypass the module cache, e.g. in tests). */
 export function render(id, slots = {}, templates = cache) {
   if (!templates) throw new Error("render() before loadTemplates() — load the template library first");
-  // render() IS the flattened segmentation, by construction: the byte output is
-  // provably identical to the old `.replace(SLOT_RE, …)` (test/segments.test.mjs
-  // renders every responses.jsonl row both ways and asserts equality).
   return flatten(renderSegments(id, slots, templates));
 }
 

package/src/corpus/unknown-ingest.mjs

@@ -1,70 +1,29 @@
-// corpus/unknown-ingest.mjs — context-preserving ingestion for unknown terms
-// (PLAN_AGENTS.md §4 Phase 1, the "still not built at all" bullet).
+// corpus/unknown-ingest.mjs — context-preserving ingestion for unknown terms.
 //
-// The problem this closes: `toFacts()` (conceptnet.mjs) only emits a Fact for
-// a relation the ACE-OWL map marks axiom-worthy (`ace != "none"`) — a row
-// whose relation is RelatedTo/Synonym/FormOf/SimilarTo/HasContext/etc is
-// SILENTLY skipped (`if (row.ace === "none") continue;`), on purpose, because
-// the relation itself doesn't fit a clean OWL axiom. That is the right call
-// for the AXIOM graph, but it means a term that ONLY ever shows up in one of
-// those dropped rows — never as the endpoint of a relation tmct actually
-// reifies — has NO anchor in memory at all. A wider seed set (a broader
-// ConceptNet slice, a tier-2 bundle, Phase 4's scraped web content) makes
-// this common: real terms, genuinely mentioned, quietly vanishing.
+// `toFacts()` (conceptnet.mjs) skips a row whose relation isn't axiom-worthy
+// (`ace === "none"`, e.g. RelatedTo/Synonym/HasContext), which is right for the axiom
+// graph but leaves a term that only ever appears in such a row with no anchor in memory
+// at all. This module captures those terms as a separate, honestly-labelled kind of
+// individual: a term is "unknown" if it's never been the subject/object of any reified
+// Fact (in memory already, or in this same seeding batch). Each captured term gets a
+// `mgx:contextPassage` fact (the passage it was found in — ConceptNet's own surfaceText,
+// or the map's `surface` template) and `mgx:coOccursWith` links to its row's other
+// endpoint plus any already-known terms recognizable in the same passage (bounded) —
+// deliberately closed-vocabulary co-occurrence, not distributional/embedding inference.
 //
-// This module does NOT change what counts as an axiom. It adds a SEPARATE,
-// honestly-labelled kind of individual for exactly the terms that would
-// otherwise vanish: a term is "unknown" here if it has never been the
-// subject/object of any reified Fact — not in memory already, and not in the
-// mapped facts this same seeding batch is about to write. For each dropped
-// (ace="none") row that touches an unknown term, the term becomes a Fact
-// tagged with the PASSAGE it was found in (ConceptNet's own `surfaceText`
-// when present, else the map's own `surface` template filled with the row's
-// two endpoints — both are committed, closed-vocabulary text; nothing here
-// ever generates free text), via a dedicated `mgx:contextPassage` predicate.
-// The row's OTHER endpoint (always) plus any already-known term recognizable
-// by exact word/bigram match in the passage (bounded to a handful) are linked
-// to the unknown term via one plain, closed-vocabulary co-occurrence
-// predicate, `mgx:coOccursWith` — deliberately NOT distributional/embedding
-// meaning induction (PLAN_AGENTS.md's own scoping): this buys traceable
-// context, never automatic sense disambiguation.
-//
-// Reuses src/memory/core.mjs's existing appendFacts/loadMemory/normFactTerm
-// machinery unmodified — a captured term is a completely ordinary Fact
-// individual (same trust/provenance/Source pipeline every other fact gets),
-// just carrying two new-but-closed predicates instead of an ACE-OWL one.
-//
-// Wiring: `seedMemory` (conceptnet.mjs) accepts an opt-in
-// `captureUnknownContext: true` (default false — every existing seed call
-// stays byte-identical) that calls `ingestUnknownFromAssertions` after the
-// mapped facts are computed, dynamically imported (the same "avoid a static
-// import cycle" discipline extensions.mjs's seedActiveCorpusEntries already
-// uses for conceptnet.mjs itself).
-//
-// NOT covered here (out of scope for this module, see the caller's report):
-// the LIVE chat teach/miss path (src/chat.mjs) has its own, separate
-// unknown-word moment — a visitor's utterance mentioning a term the grammar
-// can't classify — which has no "assertion batch" or ConceptNet-shaped
-// surfaceText to draw a passage from at all. That needs its own hook (the
-// raw utterance text IS the passage there); this module only ever consumes
-// {start, rel, end, surfaceText?} shaped rows, so it cannot be reused as-is
-// for that path without a chat.mjs-side adapter. See the report for the
-// exact shape that hook would need — deliberately not built here.
+// Wired as an opt-in `captureUnknownContext: true` on seedMemory (conceptnet.mjs),
+// dynamically imported to avoid a load-time cycle with conceptnet.mjs.
 
 import { appendFacts, loadMemory, normFactTerm, FACT_CLASS } from "../memory/core.mjs";
 import { termText } from "./conceptnet.mjs";
 
-/** unknown term -> the passage it was captured from (object = the passage
- *  text itself, capped by normFactTerm's own TEXT_CAP like any fact term). */
+/** unknown term -> the passage it was captured from. */
 export const CONTEXT_PASSAGE_PREDICATE = "mgx:contextPassage";
-/** plain, undirected-in-spirit co-occurrence edge: term -> another term seen
- *  in the SAME passage. Deliberately the ONE relation this module ever
- *  emits for "these two showed up together" — no relation-type inference. */
+/** term -> another term seen in the SAME passage — no relation-type inference. */
 export const CO_OCCURS_PREDICATE = "mgx:coOccursWith";
 
-// Function words filtered out of passage word/bigram scanning — never
-// candidates for a co-occurrence link (a "the"<->term edge would be noise,
-// not context). Closed, small, hand-curated — not a general stopword list.
+// Function words filtered out of passage word/bigram scanning (a "the"<->term edge
+// would be noise, not context). Closed, small, hand-curated.
 const STOPWORDS = new Set([
   "a", "an", "the", "is", "are", "was", "were", "be", "being", "been",
   "to", "of", "for", "in", "on", "at", "with", "by", "as", "and", "or",
@@ -73,11 +32,8 @@ const STOPWORDS = new Set([
 
 const MAX_EXTRA_LINKS_PER_PASSAGE = 3; // bounded — not an open-ended sweep
 
-/** The set of terms tmct already "recognizes": every subject/object across
- *  every reified Fact currently in memory, union every term the mapped facts
- *  THIS batch is about to write introduce. A term with a real Fact anywhere
- *  already has structured knowledge — only a term that never gets one is a
- *  candidate for context-only capture. Pure; does not mutate `memory`. */
+/** The set of terms tmct already "recognizes": every subject/object across every
+ *  reified Fact in memory, union every term this batch's mapped facts introduce. Pure. */
 export function knownTermsFrom(memory, mappedFacts) {
   const known = new Set();
   for (const ind of memory?.individuals || []) {
@@ -95,11 +51,9 @@ export function knownTermsFrom(memory, mappedFacts) {
   return known;
 }
 
-/** The human-readable context passage for one DROPPED assertion — ConceptNet's
- *  own `surfaceText` (bracket-stripped) when present, else the mapping row's
- *  own `surface` template filled with the two endpoint terms. Both sources
- *  are committed, closed-vocabulary text — this never generates free text.
- *  Returns null when neither source is available (never fatal). */
+/** The human-readable context passage for one dropped assertion — ConceptNet's own
+ *  `surfaceText` (bracket-stripped) when present, else the map's `surface` template
+ *  filled with the two endpoint terms. Returns null when neither is available. */
 export function passageFor(assertion, row) {
   const raw = assertion?.surfaceText;
   if (typeof raw === "string" && raw.trim()) {
@@ -113,11 +67,8 @@ export function passageFor(assertion, row) {
   return null;
 }
 
-/** Single words + adjacent bigrams in `passage` that are already-known terms
- *  (present in `known`), excluding anything in `exclude` (the row's own two
- *  endpoints — already linked directly). Bounded to
- *  MAX_EXTRA_LINKS_PER_PASSAGE hits; independent of `known`'s size (a Set
- *  lookup per token/bigram, never a substring sweep over every known term). */
+/** Single words + adjacent bigrams in `passage` that are already-known terms, excluding
+ *  `exclude` (the row's own two endpoints). Bounded to MAX_EXTRA_LINKS_PER_PASSAGE hits. */
 function knownMentionsIn(passage, known, exclude) {
   const tokens = String(passage || "").toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter(Boolean);
   const hits = [];
@@ -138,27 +89,15 @@ function knownMentionsIn(passage, known, exclude) {
 }
 
 /**
- * Capture unknown terms out of the assertions a corpus-seeding batch is
- * about to (or just did) write. Scans only rows whose relation maps to
- * `ace = "none"` (the genuinely-dropped rows) — a mapped (ace != "none") row
- * always gets a real reified Fact via toFacts()/appendFacts() already, so it
- * is never a "silently dropped" case this module needs to rescue.
+ * Capture unknown terms out of the assertions a corpus-seeding batch is about to (or
+ * just did) write. Scans only rows whose relation maps to `ace = "none"`.
  *
- * { assertions, map, mappedFacts, memory?, provenancePrefix?, limit? }:
- *   - assertions/map: the SAME loadSlice()/loadMap() results seedMemory has.
- *   - mappedFacts: the toFacts() output for this same batch (feeds
- *     knownTermsFrom so a term this batch itself just defined isn't
- *     re-captured as "unknown").
- *   - memory: a pre-loaded payload (seedMemory already has one) — loaded
- *     fresh via loadMemory(dir) when omitted.
- *   - provenancePrefix: tags the captured facts, default "corpus:unknown".
- *   - limit: caps how many DISTINCT unknown terms get captured in one call
- *     (default 500) — a wide slice's RelatedTo rows alone number in the tens
- *     of thousands; this keeps one run bounded, not a flood.
+ * { assertions, map, mappedFacts, memory?, provenancePrefix?, limit? }: assertions/map are
+ * the same loadSlice()/loadMap() results seedMemory has; mappedFacts is this batch's
+ * toFacts() output; memory defaults to loadMemory(dir); limit (default 500) caps distinct
+ * unknown terms captured per call.
  *
- * Returns { captured, linked, appended, skipped } — `captured` = distinct
- * unknown terms newly given a contextPassage fact, `linked` = co-occurrence
- * edges written (the direct pair plus any bounded extra known-term hits).
+ * Returns { captured, linked, appended, skipped }.
  */
 export async function ingestUnknownFromAssertions(dir, {
   assertions, map, mappedFacts = [], memory, provenancePrefix = "corpus:unknown", limit = 500,

package/src/embed.mjs

@@ -1,27 +1,15 @@
-// embed.mjs — deterministic static-embedding lookup (model2vec potion-base-8M).
-// PLAN_SEON_TUNING.md §7.6(5b), 2026-07-02 library-leverage review.
+// embed.mjs — deterministic static-embedding lookup (model2vec potion-base-8M,
+// 29,528 WordPiece subwords × 256 fp32 dims). Pure table lookup + float
+// arithmetic, no ONNX runtime, no network after the one-time fetch — the same
+// text always embeds to the same vector.
 //
-// The honest "near-LLM" locate lever: a STATIC per-token embedding table (model2vec's
-// potion-base-8M, MIT — 29,528 WordPiece subwords × 256 fp32 dims, ~30 MB) read straight
-// from its safetensors export, mean-pooled and L2-normalised. No ONNX runtime, no model
-// calls, no network after the one-time fetch — pure table lookup + float arithmetic, so
-// the same text always embeds to the same vector ($0, deterministic, offline).
+// The safetensors reader and WordPiece tokenizer below are hand-rolled: both
+// formats are simple enough to parse directly, avoiding an ONNX/HF tokenizer
+// dependency for it.
 //
-// Dependency choice (documented per the review): the safetensors format is an 8-byte LE
-// header length + JSON header + raw little-endian tensor bytes, and the tokenizer is a
-// plain Bert-style WordPiece (tokenizer.json: BertNormalizer lowercase + BertPreTokenizer
-// + greedy longest-match with "##" continuations) — both are small enough to hand-roll
-// with node built-ins, so neither the @yarflam/potion-base-8m fallback package nor an HF
-// tokenizer dependency is taken. Numerical intent follows model2vec's own encode (subword
-// ids WITHOUT the [CLS]/[SEP] template, mean pool, normalize per config.json) — exact
-// float parity with the Python lib is not claimed; determinism and rank usefulness are.
-//
-// The weights are NEVER committed and NEVER in the npm package: they live in the
-// gitignored vendor/ tree (fetched by scripts/fetch-embeddings.mjs, `npm run
-// refs:embeddings`, SHA-256-pinned like the repo's other binary artefacts). Everything
-// here degrades gracefully — loadEmbedder() returns null when the weights dir is absent,
-// and callers (codegraph.mjs's opt-in embedRank, scripts/rank-gate.mjs) no-op with a
-// clear note instead of failing, so CI/tests never require the 30 MB download.
+// Weights are gitignored (vendor/, fetched by scripts/fetch-embeddings.mjs)
+// and never in the npm package; loadEmbedder() returns null when absent so
+// CI/tests never require the download.
 
 import { readFileSync, existsSync } from "node:fs";
 import { join, dirname } from "node:path";