@polycode-projects/the-mechanical-code-talker 1.8.20 → 1.9.1

package/src/init.mjs

@@ -148,6 +148,27 @@ enabled = ${seed.enabled ? "true" : "false"}
 # To cap it, uncomment and set a number (definitional band first):
 ${seed.limit != null ? `limit = ${Number(seed.limit)}` : "# limit = 500"}
 `;
+  // [memory] backend — ONLY emitted when a caller actually supplies it (an
+  // explicit `tmct init --memory-backend <...>`, or a manual override); the
+  // plain zero-flag `tmct init` output stays BYTE-IDENTICAL to before this
+  // knob existed, same discipline as the extras block below. "default" is
+  // written out explicitly rather than omitted, so `--memory-backend default`
+  // leaves a self-documenting trace of the choice (mirrors --with-persona's
+  // own "make the default explicit" behaviour).
+  let out = base;
+  if (config.memory && config.memory.backend !== undefined) {
+    out += `
+[memory]
+# Storage backend for taught facts + the memory graph (PLAN_SEED.md §6).
+# Precedence: --memory-backend flag > TMCT_MEMORY_BACKEND env > this file >
+# "default" (the built-in fallback).
+#   "default" — the flat OWL-labelled JSON file under .tmct/memory/. The default.
+#   "memory"  — in-process only; nothing written to disk (a library caller's option).
+#   "sqlite"  — a local SQLite file at .tmct/memory/graph.sqlite.
+backend = ${JSON.stringify(config.memory.backend)}
+`;
+  }
+
   // Extension-pack / bias sections (src/extensions.mjs) — ONLY emitted when a
   // caller actually supplies them (an explicit `--with-persona`, or a manual
   // override); the plain zero-flag `tmct init` output stays BYTE-IDENTICAL to
@@ -157,8 +178,8 @@ ${seed.limit != null ? `limit = ${Number(seed.limit)}` : "# limit = 500"}
   const extras = {};
   if (config.extensions !== undefined) extras.extensions = config.extensions;
   if (config.bias !== undefined) extras.bias = config.bias;
-  if (!Object.keys(extras).length) return base;
-  return `${base}
+  if (!Object.keys(extras).length) return out;
+  return `${out}
 # Extension packs + bias (src/extensions.mjs) — written by \`tmct init --with-persona\`
 # or a manual edit. Recognized names (human, seon, conceptnet, tier2-aws,
 # tier2-python, tier2-java, tier2-general) override the shipped defaults; any
@@ -196,13 +217,21 @@ function seedRequested({ optSeed, configEnabled, env }) {
  *   only ever sees an already-resolved preset object (or nothing). Has no
  *   effect when tmct.toml already exists and `force` isn't set (the existing
  *   "preserve a user's tmct.toml" rule wins, same as `seed`/`corpus.tier`).
+ * @param {string}  [opts.memoryBackend]  "default" | "memory" | "sqlite" — merged
+ *   into the FRESH config's `[memory] backend` before it's written (same
+ *   "fresh write only" rule as `persona`, above; `tmct init --memory-backend
+ *   <...>` on an ALREADY-initialized repo is bin/tmct.mjs's own job, mirroring
+ *   how `--graph` amends an existing tmct.toml post-hoc). Also selects which
+ *   backend the corpus SEED below (step 3) writes into — src/memory/core.mjs's
+ *   `openMemoryBackend`, the same resolver chat.mjs's createSession uses, so a
+ *   seeded fact and a later chat-taught fact always land in the same store.
  * @returns {Promise<{
  *   created: string[], config: object, seeded: boolean,
  *   alreadyInitialized: boolean, seedResult: (object|null), message: string
  * }>} `created` lists the ABSOLUTE paths this call brought into being (empty on a
  *   benign no-op re-init). Never throws on a benign re-init or a corpus failure.
  */
-export async function initRepo(dir, { force = false, seed, env = process.env, persona = null } = {}) {
+export async function initRepo(dir, { force = false, seed, env = process.env, persona = null, memoryBackend = null } = {}) {
   const root = resolve(dir);
   const created = [];
   const paths = {
@@ -235,6 +264,7 @@ export async function initRepo(dir, { force = false, seed, env = process.env, pe
     if (persona.extensions && Object.keys(persona.extensions).length) config.extensions = persona.extensions;
     if (persona.bias && Object.keys(persona.bias).length) config.bias = persona.bias;
   }
+  if (memoryBackend) config.memory = { ...(config.memory || {}), backend: memoryBackend };
   const tomlPresent = await exists(paths.toml);
   if (!tomlPresent || force) {
     await writeFile(paths.toml, renderTomlConfig(config));
@@ -266,50 +296,70 @@ export async function initRepo(dir, { force = false, seed, env = process.env, pe
   } else if ((await exists(paths.marker)) && !force) {
     seedNote = "seed skipped (already seeded — marker present)";
   } else {
-    try {
-      const { resolveExtensions, seedActiveCorpusEntries } = await import("./extensions.mjs");
-      const { entries } = await resolveExtensions(root);
-      // `tmct.toml`'s `[seed] limit` knob is documented as capping the tier-1
-      // ConceptNet band specifically (the curated SEON ontology is small and
-      // always seeds whole) — so it overrides ONLY the resolved "conceptnet"
-      // entry's limit, exactly like the pre-fix single-corpus seed did.
-      if (config.seed?.limit != null && entries.has("conceptnet")) {
-        entries.set("conceptnet", { ...entries.get("conceptnet"), limit: Number(config.seed.limit) });
+    // BUG FIX (found in review): this step used to call seedActiveCorpusEntries
+    // with the plain `root` string ALWAYS — Backend A only — regardless of
+    // `config.memory.backend`. A `tmct init --memory-backend sqlite` repo ended
+    // up with its corpus facts trapped in an inert .tmct/memory/graph.json that
+    // a sqlite-backend chat session (createSession, which IS backend-aware)
+    // createSession resolves it — src/memory/core.mjs's openMemoryBackend — so
+    // the seed lands in whichever backend `config.memory.backend` actually
+    // names. "memory" is skipped outright: it's an in-process-only store that
+    // vanishes the moment this one-shot init process exits, so seeding it is
+    // pure wasted work — a later `tmct chat --memory-backend memory` opens a
+    // brand new, unrelated in-memory store anyway.
+    const backendChoice = String(config.memory?.backend || "").trim().toLowerCase();
+    if (backendChoice === "memory") {
+      seedNote = "seed skipped (memory backend is in-process only — nothing would persist past this command)";
+    } else {
+      const { openMemoryBackend } = await import("./memory/core.mjs");
+      const { dir: memoryDir, close: closeMemoryStore } = await openMemoryBackend(root, backendChoice);
+      try {
+        const { resolveExtensions, seedActiveCorpusEntries } = await import("./extensions.mjs");
+        const { entries } = await resolveExtensions(root);
+        // `tmct.toml`'s `[seed] limit` knob is documented as capping the tier-1
+        // ConceptNet band specifically (the curated SEON ontology is small and
+        // always seeds whole) — so it overrides ONLY the resolved "conceptnet"
+        // entry's limit, exactly like the pre-fix single-corpus seed did.
+        if (config.seed?.limit != null && entries.has("conceptnet")) {
+          entries.set("conceptnet", { ...entries.get("conceptnet"), limit: Number(config.seed.limit) });
+        }
+        const { appended, skipped, total, perBundle } = await seedActiveCorpusEntries(memoryDir, entries);
+        // seedActiveCorpusEntries is failure-tolerant PER BUNDLE (a bad third-party
+        // pack never aborts the others) — but initRepo's own "FAILURE-TOLERANT
+        // SEED" contract is about the SEED AS A WHOLE degrading honestly. If every
+        // active bundle failed (e.g. the memory graph file itself is unwritable —
+        // see test/init.test.mjs "seed failure degrades"), re-throw the first
+        // bundle's error so the SAME outer catch below reports the familiar "seed
+        // skipped (corpus unavailable: …)" note, rather than claiming success with
+        // zero facts actually written.
+        const bundleNames = Object.keys(perBundle);
+        const allFailed = bundleNames.length > 0 && bundleNames.every((n) => perBundle[n].error);
+        if (allFailed) throw new Error(perBundle[bundleNames[0]].error);
+        seedResult = {
+          appended, skipped, total, perBundle,
+          seon: perBundle.seon?.appended || 0,
+          conceptnet: perBundle.conceptnet?.appended || 0,
+        };
+        const markerNew = !(await exists(paths.marker));
+        await mkdir(dirname(paths.marker), { recursive: true });
+        await writeFile(
+          paths.marker,
+          JSON.stringify({
+            seededAt: new Date().toISOString(),
+            limit: config.seed?.limit != null ? Number(config.seed.limit) : SEED_LIMIT,
+            appended: seedResult.appended,
+            skipped: seedResult.skipped,
+            perBundle,
+          }) + "\n",
+        );
+        if (markerNew) created.push(paths.marker);
+        seeded = true;
+      } catch (err) {
+        // Corpus unavailable/broken → an initialised-but-unseeded repo, not a crash.
+        seedNote = `seed skipped (corpus unavailable: ${err && err.message ? err.message : err})`;
+      } finally {
+        await closeMemoryStore();
       }
-      const { appended, skipped, total, perBundle } = await seedActiveCorpusEntries(root, entries);
-      // seedActiveCorpusEntries is failure-tolerant PER BUNDLE (a bad third-party
-      // pack never aborts the others) — but initRepo's own "FAILURE-TOLERANT
-      // SEED" contract is about the SEED AS A WHOLE degrading honestly. If every
-      // active bundle failed (e.g. the memory graph file itself is unwritable —
-      // see test/init.test.mjs "seed failure degrades"), re-throw the first
-      // bundle's error so the SAME outer catch below reports the familiar "seed
-      // skipped (corpus unavailable: …)" note, rather than claiming success with
-      // zero facts actually written.
-      const bundleNames = Object.keys(perBundle);
-      const allFailed = bundleNames.length > 0 && bundleNames.every((n) => perBundle[n].error);
-      if (allFailed) throw new Error(perBundle[bundleNames[0]].error);
-      seedResult = {
-        appended, skipped, total, perBundle,
-        seon: perBundle.seon?.appended || 0,
-        conceptnet: perBundle.conceptnet?.appended || 0,
-      };
-      const markerNew = !(await exists(paths.marker));
-      await mkdir(dirname(paths.marker), { recursive: true });
-      await writeFile(
-        paths.marker,
-        JSON.stringify({
-          seededAt: new Date().toISOString(),
-          limit: config.seed?.limit != null ? Number(config.seed.limit) : SEED_LIMIT,
-          appended: seedResult.appended,
-          skipped: seedResult.skipped,
-          perBundle,
-        }) + "\n",
-      );
-      if (markerNew) created.push(paths.marker);
-      seeded = true;
-    } catch (err) {
-      // Corpus unavailable/broken → an initialised-but-unseeded repo, not a crash.
-      seedNote = `seed skipped (corpus unavailable: ${err && err.message ? err.message : err})`;
     }
   }
 
@@ -356,6 +406,9 @@ async function readWrittenConfig(tomlPath, base) {
     // normalizeConfig).
     if (raw.extensions !== undefined) cfg.extensions = raw.extensions;
     if (raw.bias !== undefined) cfg.bias = raw.bias;
+    if (raw.memory && raw.memory.backend !== undefined) {
+      cfg.memory = { ...cfg.memory, backend: raw.memory.backend };
+    }
     return cfg;
   } catch {
     return base;

package/src/interpret/normalize.mjs

@@ -119,9 +119,7 @@ const KIND_NOUN_ANAPHORA_RE = /\b(this|that)\s+(class|module|function|method|att
  *  "file"->"Module" convention every other lane in this grammar already
  *  uses)? Returns that class, or null when no such anaphora is present.
  *  Deliberately SEPARATE from normalizeQuery's own KIND_NOUN_ANAPHORA_RE
- *  replace just above (which permanently collapses "this file" to bare
- *  "this", discarding the kind-noun signal for good, by design — see that
- *  replace's own docblock): this never mutates its input and has no effect
+ *  replace just above (: this never mutates its input and has no effect
  *  on normalizeQuery's behavior, signature, or any of its many call sites.
  *  A caller that needs BOTH the collapsed pronoun AND the kind it stood for
  *  (chat.mjs's runAsk, at its pronoun-reuse site) calls this side-channel on

package/src/memory/core.mjs

@@ -108,7 +108,7 @@ const MEMORY_VOCABULARY = [
   { prop: DERIVED_FROM_PROP, predicate: "derivedFrom", note: "umbrella: a Fact derived from a Source (or another Fact). ext ref prov:wasDerivedFrom (UNVERIFIED-pending-web-check)" },
   { prop: STATED_BY_PROP, predicate: "statedBy", note: "subPropertyOf derivedFrom: a Source directly asserts this Fact (one edge per independent source — replaces the factProvenance union)" },
   { prop: CANONICALISED_FROM_PROP, predicate: "canonicalisedFrom", note: "subPropertyOf derivedFrom: a canonical Fact cleaned from a raw Block/Source, never replacing it" },
-  { prop: "mgx:sourceType", note: "a Source's kind: operator | teach | provider | corpus | web | entailed (the trust-prior key)" },
+  { prop: "mgx:sourceType", note: "a Source's kind: operator | teach | provider | corpus | corpusWeak | extracted | web | entailed (the trust-prior key)" },
   { prop: "mgx:sourceUrl", note: "a web Source's URL" },
   { prop: "mgx:sourceRule", note: "an entailed Source's rule id" },
   { prop: "mgx:sourceReliability", note: "actor-level (session-scoped) trust nudge in [0.5,1.5], neutral 1.0 when absent — materialised by recomputeSourceReliability from a session's asserted-vs-contradicted track record (memory/trust.mjs's sessionReliabilityFrom); folds into computeTrust's per-source prior" },
@@ -312,6 +312,43 @@ export function closeSqliteMemoryStore(handle) {
   if (isSqliteHandle(handle)) handle.db.close();
 }
 
+/**
+ * Resolve an already-lowercased/trimmed backend token ("memory" | "sqlite" |
+ * anything else, including "" or "default") into `{ dir, close }`: the exact
+ * `dir` value every dir-taking export in this module (loadMemory, appendFact,
+ * appendFacts, seedMemory via src/corpus/conceptnet.mjs, …) already accepts —
+ * a plain repo-path string for Backend A, or a live Backend B/C handle — plus
+ * an idempotent `close()` cleanup (a no-op for A/B; closes the sqlite
+ * connection for C).
+ *
+ * The ONE shared resolver for the storage-backend seam (PLAN_SEED.md §6):
+ * originally inlined only in chat.mjs's createSession, factored out here so
+ * `tmct init`'s corpus seed (src/init.mjs) and bin/tmct.mjs's
+ * `--corpus`/`--ontology`/`--lexicon` activation seed the SAME backend a
+ * later `tmct chat` in that repo will read — a caller resolving "sqlite"
+ * independently and picking a different db path (or a different in-memory
+ * store) would silently split a repo's memory across two places that can
+ * never see each other. `backendChoice` is a precedence result the CALLER
+ * already computed (CLI flag > env > tmct.toml > default, or whatever subset
+ * applies); this function does no precedence resolution of its own — it only
+ * maps the resolved token to a handle. `repoRoot` is used only for Backend C's
+ * db path (`<repoRoot>/.tmct/memory/graph.sqlite`); Backend A returns
+ * `repoRoot` itself unchanged, byte-identical to every existing plain-string
+ * caller.
+ */
+export async function openMemoryBackend(repoRoot, backendChoice) {
+  if (backendChoice === BACKEND_MEMORY) {
+    return { dir: createInMemoryStore(), close: async () => {} };
+  }
+  if (backendChoice === BACKEND_SQLITE) {
+    const dbPath = join(repoRoot, ".tmct", "memory", "graph.sqlite");
+    await mkdir(dirname(dbPath), { recursive: true });
+    const handle = await createSqliteMemoryStore(dbPath);
+    return { dir: handle, close: async () => closeSqliteMemoryStore(handle) };
+  }
+  return { dir: repoRoot, close: async () => {} };
+}
+
 /** Deep-clone a JSON-safe value. Used two ways here: (1) readSqlitePayload
  *  hands every CALLER a clone of the cache, never the live cached object
  *  itself, so this backend keeps the same "fresh object every call" contract
@@ -725,8 +762,56 @@ async function persistMemory(dir, payload) {
  *  reached). `await fn(payload)` is a documented no-op for every existing
  *  SYNC caller (appendUtterance(s), appendFacts) — awaiting a non-Promise
  *  value just resolves to it, byte-identical behaviour to calling it plain. */
+// ---- mutateMemory-scoped lookup index (PLAN_GRAPH_SCAN.md Phase 1) ----------
+// syncFactSources's per-fact bookkeeping (upsertSource, upsertIndividual,
+// upsertEdge's statedBy path, statedByObjectsFor, sourcesByIdMap) used to each
+// re-scan payload.individuals or the statedBy edge list from scratch, turning
+// one appendFacts batch of n facts into O(n^2) work. mutateMemory now builds
+// three lookup Maps once per call (one O(n) pass) and attaches them to payload
+// under a Symbol key — JSON.stringify skips Symbol-keyed properties
+// automatically, so persistMemory's graph.json write is byte-identical to
+// before. Every helper below checks for the Symbol slot: present → O(1) Map
+// lookup; absent (a bare payload object built outside mutateMemory, e.g. a
+// test fixture) → today's exact linear-scan fallback, so nothing outside
+// mutateMemory's own call chain can observe a behaviour change. The index is
+// discarded when mutateMemory returns — it never survives across calls, so
+// there is no invalidation logic to get wrong.
+const MEMORY_INDEX = Symbol("mutateMemory lookup index");
+
+/** Build the three lookup Maps from the just-loaded payload and attach them
+ *  under MEMORY_INDEX. Any code that pushes a new individual into
+ *  payload.individuals, or a new statedBy edge, must also write the matching
+ *  index entry in that same statement (see upsertIndividual/upsertSource/
+ *  upsertEdge/appendFacts below) — the same discipline appendFacts's own
+ *  local `byId` Map already used for the Fact upsert, generalised here. */
+function buildMemoryIndex(payload) {
+  const individualsById = new Map();
+  const sourcesById = new Map();
+  const statedByBySubject = new Map();
+  for (const ind of payload.individuals || []) {
+    if (!ind?.id) continue;
+    individualsById.set(ind.id, ind);
+    if (ind.class === SOURCE_CLASS) sourcesById.set(ind.id, ind);
+  }
+  const statedGroup = (payload.objectProperties || []).find((g) => g?.prop === STATED_BY_PROP);
+  for (const e of statedGroup?.examples || []) {
+    if (!e?.subject) continue;
+    const list = statedByBySubject.get(e.subject);
+    if (list) list.push(e.object);
+    else statedByBySubject.set(e.subject, [e.object]);
+  }
+  payload[MEMORY_INDEX] = { individualsById, sourcesById, statedByBySubject };
+  return payload[MEMORY_INDEX];
+}
+
+/** The active lookup index for this payload, or null when this payload wasn't
+ *  built by mutateMemory (a bare test fixture) — callers fall back to a
+ *  linear scan in that case. */
+const memoryIndexOf = (payload) => payload?.[MEMORY_INDEX] || null;
+
 async function mutateMemory(dir, fn) {
   const payload = await loadMemory(dir);
+  buildMemoryIndex(payload);
   const out = (await fn(payload)) ?? payload;
   migrateLegacyProvenance(out);
   recomputeSourceReliability(out);
@@ -771,6 +856,11 @@ function sourceIdFor(desc) {
     case "teach": return { id: desc.sessionId ? `${TEACH_SOURCE_ID}:${desc.sessionId}` : TEACH_SOURCE_ID, type: "teach" };
     case "provider": return { id: `src:provider:${desc.name}`, type: "provider" };
     case "corpus": return { id: `src:corpus:${desc.name}`, type: "corpus" };
+    // One Source per source-file basename (the corpus precedent above), not per
+    // extraction run — re-running scripts/extract-facts-from-text.mjs over the
+    // SAME file collapses onto the same Source instead of minting a new one
+    // every time, matching corpus's "one Source per named dataset" idiom.
+    case "extracted": return { id: `src:extracted:${desc.name}`, type: "extracted" };
     case "web": return { id: `src:learned:web:${fnv1aHex(String(desc.url || ""))}`, type: "web", url: String(desc.url || "") };
     case "entailed": return { id: `src:entailed:${desc.rule}`, type: "entailed", rule: String(desc.rule || "") };
     default: return null;
@@ -785,9 +875,10 @@ const sourceLabel = (id) => String(id).replace(/^src:/, "");
 function upsertSource(payload, desc, createdAtCandidate) {
   const info = sourceIdFor(desc);
   if (!info) return null;
-  const prior = payload.individuals.find((i) => i?.id === info.id);
+  const idx = memoryIndexOf(payload);
+  const prior = idx ? idx.individualsById.get(info.id) : payload.individuals.find((i) => i?.id === info.id);
   const created = firstWriteCreatedAt(prior, desc?.createdAt || createdAtCandidate);
-  upsertIndividual(payload, {
+  const ind = {
     id: info.id, label: sourceLabel(info.id), class: SOURCE_CLASS,
     derived_from: [], mentions: [],
     attributes: [
@@ -797,7 +888,9 @@ function upsertSource(payload, desc, createdAtCandidate) {
       ...(info.url ? [{ prop: "mgx:sourceUrl", key: "sourceUrl", value: info.url }] : []),
       ...(info.rule ? [{ prop: "mgx:sourceRule", key: "sourceRule", value: info.rule }] : []),
     ],
-  });
+  };
+  const stored = upsertIndividual(payload, ind);
+  if (idx) idx.sourcesById.set(info.id, stored);
   return info.id;
 }
 
@@ -821,19 +914,32 @@ function parseChatTagRest(rest) {
  * set — the inverse the migration and the live write path both name Sources
  * through. The tag formats are exactly what the writers produce:
  *   corpus:conceptnet /r/IsA   → { kind:"corpus",   name:"conceptnet" }
+ *   corpus-weak:conceptnet /r/RelatedTo → { kind:"corpusWeak", name:"conceptnet" }
  *   ace:chat:<session>@<ts>    → { kind:"operator",  createdAt:<ts>, sessionId:<session> }
  *   teach:chat:<session>@<ts>  → { kind:"teach",     createdAt:<ts>, sessionId:<session> }
  *   web:<url> | url:<url>      → { kind:"web",       url:<url> }
+ *   extracted:<file-basename>  → { kind:"extracted", name:<file-basename> }
  *   entailed:<rule>            → { kind:"entailed",  rule:<rule> }
  * chat:/session: refs map to the operator; an unknown tag → null (no Source).
+ * `corpus-weak:` is the SAME corpus provenance shape as `corpus:`, just naming
+ * a lower trust-prior kind (memory/trust.mjs SOURCE_PRIOR.corpusWeak) for
+ * facts whose underlying relation is real but low-precision (e.g. ConceptNet's
+ * undirected /r/RelatedTo) — trust stays computed from the Source's kind, never
+ * hand-set on the Fact.
  * The session-id segment (Part B: session-scoped actor-level trust) feeds
  * sourceIdFor, which mints a PER-SESSION Source id when present, instead of
  * collapsing every session onto one singleton operator/teach Source.
+ * `extracted:` is scripts/extract-facts-from-text.mjs's own audit tag, layered
+ * ADDITIVELY (via appendFact's provenance union) on top of whatever the
+ * runTurn recognizer already wrote (ace:/teach:) — see that script for why:
+ * it distinguishes "this document evidenced this fact" from ordinary chat
+ * speech, at its own trust-prior tier (memory/trust.mjs SOURCE_PRIOR.extracted).
  */
 export function provenanceTagToSource(tag) {
   const t = String(tag || "").trim();
   if (!t) return null;
   const head = t.split(/\s+/)[0]; // drop trailing " /r/IsA" etc.
+  if (head.startsWith("corpus-weak:")) return { kind: "corpusWeak", name: head.slice("corpus-weak:".length) || "unknown" };
   if (head.startsWith("corpus:")) return { kind: "corpus", name: head.slice("corpus:".length) || "unknown" };
   if (head.startsWith("ace:")) return { kind: "operator", ...parseChatTagRest(head.slice("ace:".length)) };
   if (head.startsWith("teach:")) {
@@ -842,6 +948,7 @@ export function provenanceTagToSource(tag) {
   }
   if (head.startsWith("web:")) return { kind: "web", url: head.slice("web:".length) };
   if (head.startsWith("url:")) return { kind: "web", url: head.slice("url:".length) };
+  if (head.startsWith("extracted:")) return { kind: "extracted", name: head.slice("extracted:".length) || "unknown" };
   if (head.startsWith("entailed:")) return { kind: "entailed", rule: head.slice("entailed:".length) };
   if (head.startsWith("chat:") || head.startsWith("session:") || head.startsWith("operator")) return { kind: "operator" };
   return null;
@@ -850,13 +957,23 @@ export function provenanceTagToSource(tag) {
 /** Map a payload's Source individuals into the { id: Source } shape computeTrust
  *  resolves against. */
 function sourcesByIdMap(payload) {
+  const idx = memoryIndexOf(payload);
   const m = {};
+  if (idx) {
+    // idx.sourcesById is kept incrementally correct by upsertSource, so this
+    // is O(distinct Sources) — a handful, roughly one per corpus/provider —
+    // never O(all individuals), unlike the fallback rebuild below.
+    for (const [id, ind] of idx.sourcesById) m[id] = ind;
+    return m;
+  }
   for (const i of payload.individuals) if (i?.class === SOURCE_CLASS) m[i.id] = i;
   return m;
 }
 
 /** The Source ids a Fact is statedBy, read off the edge group. */
 function statedByObjectsFor(payload, factId) {
+  const idx = memoryIndexOf(payload);
+  if (idx) return (idx.statedByBySubject.get(factId) || []).slice();
   const g = payload.objectProperties.find((x) => x?.prop === STATED_BY_PROP);
   return (g?.examples || []).filter((e) => e?.subject === factId).map((e) => e.object);
 }
@@ -983,8 +1100,9 @@ function recomputeSourceReliability(payload) {
   }
   if (!bySource.size) return;
 
+  const idx = memoryIndexOf(payload);
   for (const [sid, counts] of bySource) {
-    const source = payload.individuals.find((i) => i?.id === sid);
+    const source = idx ? idx.individualsById.get(sid) : payload.individuals.find((i) => i?.id === sid);
     if (!source) continue;
     setAttr(source, SOURCE_RELIABILITY_PROP, "sourceReliability", String(sessionReliabilityFrom(counts)));
     // Own-attribute mutation in place (PLAN_VIZ.md §2) — same reasoning as recomputeFactTrust.
@@ -997,16 +1115,38 @@ function recomputeSourceReliability(payload) {
   const affected = new Set();
   for (const e of statedGroup?.examples || []) if (bySource.has(e?.object)) affected.add(e.subject);
   for (const id of affected) {
-    const ind = payload.individuals.find((i) => i?.id === id);
+    const ind = idx ? idx.individualsById.get(id) : payload.individuals.find((i) => i?.id === id);
     if (ind) recomputeFactTrust(payload, ind);
   }
 }
 
-/** Upsert an individual by id (replace-in-place keeps ordering stable). */
+/** Upsert an individual by id (replace-in-place keeps ordering stable).
+ *  Returns the individual object actually stored in payload.individuals — the
+ *  caller (e.g. upsertSource) should index THAT reference, not `ind` itself,
+ *  since the indexed path below merges into the prior object in place rather
+ *  than replacing the array slot. When a lookup index is present (built by
+ *  mutateMemory), an existing individual is updated via Object.assign — same
+ *  array position AND same object identity as before, so it stays trivially
+ *  in sync with individualsById without a second Map write, and a brand-new
+ *  individual is pushed + indexed in the same statement. Absent an index
+ *  (a bare payload built outside mutateMemory), this is EXACTLY the original
+ *  findIndex + replace-or-push code. */
 function upsertIndividual(payload, ind) {
+  const idx = memoryIndexOf(payload);
+  if (idx) {
+    const prior = idx.individualsById.get(ind.id);
+    if (prior) {
+      Object.assign(prior, ind);
+      return prior;
+    }
+    payload.individuals.push(ind);
+    idx.individualsById.set(ind.id, ind);
+    return ind;
+  }
   const i = payload.individuals.findIndex((x) => x?.id === ind.id);
-  if (i >= 0) payload.individuals[i] = ind;
-  else payload.individuals.push(ind);
+  if (i >= 0) { payload.individuals[i] = ind; return ind; }
+  payload.individuals.push(ind);
+  return ind;
 }
 
 /** Upsert one edge into the named relation group (dedupe by subject>object). Stamps `createdAt`
@@ -1020,6 +1160,27 @@ function upsertEdge(payload, { predicate, prop }, edge) {
     group = { predicate, prop, count: 0, examples: [] };
     payload.objectProperties.push(group);
   }
+  // statedBy-only fast path (PLAN_GRAPH_SCAN.md Phase 1): statedByBySubject
+  // tracks, per fact, the small list of Source ids already stated it (almost
+  // always 0-1 during a seed), so the overwhelmingly common case — a brand
+  // new (subject,object) statedBy pair — can append directly without the
+  // find+filter scan of the WHOLE statedBy edge list below. Every other
+  // predicate (saidInSession, inReplyTo, ...) is untouched and always takes
+  // the original path.
+  const idx = prop === STATED_BY_PROP ? memoryIndexOf(payload) : null;
+  if (idx) {
+    const existing = idx.statedByBySubject.get(edge.subject);
+    if (!existing || !existing.includes(edge.object)) {
+      group.examples.push({ ...edge, createdAt: edge.createdAt || nowIso() });
+      group.count = group.examples.length;
+      if (existing) existing.push(edge.object);
+      else idx.statedByBySubject.set(edge.subject, [edge.object]);
+      return;
+    }
+    // Rare re-assert of the exact same (subject,object) pair — fall through
+    // to the exact original find+filter dance so first-write-wins createdAt
+    // is preserved; the index is kept accurate below too.
+  }
   // Edges are flat ({subject, object, ...}), not attribute-bearing individuals, so this can't
   // reuse firstWriteCreatedAt (which reads `.attributes`) directly — same discipline, edge shape:
   // the prior edge's OWN createdAt wins if it has one, else the incoming candidate, else now.
@@ -1030,6 +1191,11 @@ function upsertEdge(payload, { predicate, prop }, edge) {
   );
   group.examples.push({ ...edge, createdAt });
   group.count = group.examples.length;
+  if (idx) {
+    const list = idx.statedByBySubject.get(edge.subject) || [];
+    if (!list.includes(edge.object)) list.push(edge.object);
+    idx.statedByBySubject.set(edge.subject, list);
+  }
 }
 
 /** Recount `classes[]` from the individuals — every memory class stays counted
@@ -1285,7 +1451,13 @@ export async function appendFacts(dir, facts) {
   if (!prepared.length) return { ids, appended: 0, skipped };
   await mutateMemory(dir, (payload) => {
     // id → individual index for O(1) upsert (the array grows to thousands).
-    const byId = new Map(payload.individuals.map((i) => [i?.id, i]));
+    // When mutateMemory already built the Symbol-keyed lookup index, reuse
+    // THAT Map directly (same object) instead of rescanning payload.individuals
+    // a second time — every `byId.set` below then also keeps
+    // idx.individualsById correct for upsertSource/recomputeSourceReliability's
+    // later lookups in this same mutation, with no extra write.
+    const idx = memoryIndexOf(payload);
+    const byId = idx ? idx.individualsById : new Map(payload.individuals.map((i) => [i?.id, i]));
     const touched = [];
     const seen = new Set();
     const trustOptsById = new Map();
@@ -1314,10 +1486,15 @@ export async function appendFacts(dir, facts) {
           ...(f.justification && f.justification.length ? [{ prop: "mgx:factJustification", key: "justification", value: f.justification.join(" ") }] : []),
         ],
       };
-      // Upsert into BOTH the array (replace-in-place keeps order) and the index.
-      if (prior) payload.individuals[payload.individuals.indexOf(prior)] = ind;
-      else payload.individuals.push(ind);
-      byId.set(f.id, ind);
+      // Upsert via the shared helper — O(1) via the index (Object.assign in
+      // place when `prior` exists, push+index when it's new), same as every
+      // other upsert path now. Previously this did its own inline
+      // `payload.individuals.indexOf(prior)` array scan on a re-assert within
+      // the same batch — an O(n) fallback that could still blow up a batch
+      // heavy with within-file duplicate triples; upsertIndividual has no
+      // such case left.
+      const stored = upsertIndividual(payload, ind);
+      byId.set(f.id, stored);
       ids.push(f.id);
       if (!seen.has(f.id)) { seen.add(f.id); touched.push(f.id); }
       // Last-prepared-row-wins per id for the trust hook opts (mirrors the

package/src/memory/trust.mjs

@@ -4,7 +4,8 @@
 // Trust is a COMPUTED attribute of a Fact — never hand-set — a pure function of
 // its Source edges, those Sources' types, and its mgx:createdAt. Three inputs
 // combine:
-//   - a Source-TYPE PRIOR (operator > teach > provider > corpus > web > entailed);
+//   - a Source-TYPE PRIOR (operator > teach > provider > corpus > corpusWeak >
+//     extracted > web > entailed);
 //   - CORROBORATION over the fact's distinct Sources by noisy-OR
 //     (1 − Π(1 − wᵢ), capped at 1) — two independent web sources (0.4) reach
 //     0.64, a lone operator fact is already 1.0;
@@ -35,17 +36,37 @@ export const TRUST_SCORE_PROP = "mgx:trustScore";
 export const TRUST_INPUTS_PROP = "mgx:trustInputs";
 
 /** Source-type priors — the ordering operator > teach > provider-graph >
- *  curated-corpus > web > unverified-entailment. `teach` is the chat teach
- *  lane's natural-frame writes ("remember that …", "<Name> owns <X>") — still
- *  operator speech, but through a looser recognizer than the ACE-parsed
- *  operator assert, so it sits just below the operator prior. The entailed
- *  value is a FLOOR before premise adjustment (see the entailed hook below). */
+ *  curated-corpus > weak-corpus > extracted-document > web > unverified-
+ *  entailment. `teach` is the chat teach lane's natural-frame writes
+ *  ("remember that …", "<Name> owns <X>") — still operator speech, but
+ *  through a looser recognizer than the ACE-parsed operator assert, so it
+ *  sits just below the operator prior. The entailed value is a FLOOR before
+ *  premise adjustment (see the entailed hook below).
+ *
+ *  `corpusWeak` is for corpus-sourced facts whose underlying relation is real
+ *  but low-precision (ConceptNet's /r/RelatedTo — ambiguous, undirected
+ *  association, unlike the specific typed relations the plain `corpus` prior
+ *  covers) — still a curated, committed dataset (above `web`), just not
+ *  asserting the same strength of claim (below `corpus`).
+ *
+ *  `extracted` is scripts/extract-facts-from-text.mjs's batch reader: it runs
+ *  the SAME deterministic teach/assert recognizer as `teach`/`operator`, but
+ *  unattended over an arbitrary document nobody in-session vetted sentence by
+ *  sentence — the recognizer is exact/closed-set (no guessing), but the
+ *  SOURCE DOCUMENT is unreviewed, so it sits just above `web` (also an
+ *  unreviewed external source) and below `corpusWeak`/`corpus` (curated,
+ *  committed datasets) and `teach` (a human typing into the live chat).
+ *
+ *  Both are computed from the Source's type exactly like every other tier —
+ *  never hand-set on a Fact directly. */
 export const SOURCE_PRIOR = Object.freeze({
   operator: 1.0,
   teach: 0.95,
   provider: 0.9,
   corpus: 0.7,
+  corpusWeak: 0.55,
   web: 0.4,
+  extracted: 0.45,
   entailed: 0.3,
 });
 

package/src/memory-ask-browser-entry.mjs

@@ -0,0 +1,36 @@
+// memory-ask-browser-entry.mjs — the esbuild entry for `tmct viz`'s embedded
+// "Ask the graph" panel's MEMORY-graph engine (PLAN_VIZ_MEMORY.md Bug 1 fix).
+//
+// Bug 1: the panel bundled ONLY ask.mjs — tmct's code-graph query engine
+// ("which modules import X"), which has no concept of Facts/corpus data at
+// all, so a memory-graph question like "what is a dog" always missed even
+// though the page's own embedded payload had the answer. This is the SECOND,
+// narrow browser entry point src/ask-browser-entry.mjs's own doc comment
+// anticipates: re-exports just `factAnswer` (src/chat.mjs) — tmct's REAL
+// memory-graph answer engine, the same one `npm run chat` uses — plus
+// `createInMemoryStore` (src/memory/core.mjs), which is how the panel hands
+// `factAnswer` the page's already-embedded PAYLOAD with ZERO fs I/O: a
+// Backend-B handle's `loadMemory` branch returns `handle.payload` directly, no
+// bundle-time module shimming needed (see factAnswer's own doc comment,
+// src/chat.mjs, for the full reasoning — a simpler, more robust mechanism than
+// intercepting loadMemory at bundle time, since it reuses machinery the
+// codebase already ships and tests, rather than a new esbuild-only code path
+// that could drift from the real one).
+//
+// `factAnswer` is called with `envelope: null, miss: true` — the exact,
+// already-documented "no envelope available" bootstrap path (chat.mjs's own
+// comments: "the FIRST turn of a graph-less session... leaves `envelope` null
+// for the rest of THIS turn's processing" — a real, tested code path, not a
+// hack) — which arms factAnswer's own bare-question regex fallbacks
+// (BARE_WHATIS_RE and friends) to parse the query directly, with no
+// dependency on the much larger structural-graph parse pipeline
+// (dispatchTool/loadGraph, server.mjs) that pipeline needs a real --repo code
+// index for and this panel has no use for.
+import { factAnswer } from "./chat.mjs";
+import { createInMemoryStore, normFactTerm } from "./memory/core.mjs";
+
+// normFactTerm is re-exported too — viz.mjs's client-side "focus follows the
+// memory-engine's answer" heuristic (guessTermIdFromQuery) needs the SAME
+// term normalization the CLI's own `--term` seed flag and factAnswer's own
+// subject/object matching use, never a second hand-rolled copy.
+globalThis.tmctMemoryAsk = { factAnswer, createInMemoryStore, normFactTerm };