@polycode-projects/the-mechanical-code-talker 1.5.4 → 1.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +123 -14
  2. package/ROADMAP.md +233 -1392
  3. package/bin/tmct.mjs +479 -98
  4. package/corpus/README.md +3 -0
  5. package/corpus/generated/README.md +43 -0
  6. package/corpus/generated/ace-surface-variants.jsonl +17 -0
  7. package/corpus/generated/manifest.json +9 -0
  8. package/corpus/tier2/generate.mjs +14668 -0
  9. package/corpus/tier2/human-examples-large.jsonl +1928 -0
  10. package/corpus/tier2/human-examples-medium.jsonl +356 -0
  11. package/corpus/tier2/human-examples.jsonl +120 -0
  12. package/corpus/tier2/human-large.jsonl +12001 -0
  13. package/corpus/tier2/human-medium.jsonl +944 -0
  14. package/corpus/tier2/human.jsonl +664 -0
  15. package/corpus/tier2/manifest.json +42 -0
  16. package/package.json +14 -8
  17. package/src/answer-variants.json +47 -0
  18. package/src/answer-variants.mjs +67 -0
  19. package/src/ask-browser-entry.mjs +34 -0
  20. package/src/ask-browser.bundle.js +5095 -0
  21. package/src/ask-vocab.mjs +93 -8
  22. package/src/ask.mjs +451 -49
  23. package/src/chat.mjs +1391 -141
  24. package/src/cli-args.mjs +164 -0
  25. package/src/codegraph.mjs +170 -32
  26. package/src/completions/graph-adapter.mjs +118 -0
  27. package/src/extensions.mjs +100 -19
  28. package/src/grammar/ace.mjs +85 -3
  29. package/src/grammar/lexicon-core.json +9531 -63
  30. package/src/grammar/lexicon.mjs +58 -8
  31. package/src/graph-merge.mjs +114 -0
  32. package/src/index.mjs +14 -0
  33. package/src/init.mjs +40 -14
  34. package/src/interpret/normalize.mjs +88 -3
  35. package/src/interpret/strategies/grammar.mjs +10 -0
  36. package/src/interpret/strategies/keywords.mjs +20 -0
  37. package/src/interpret/strategies/noise-strip.mjs +73 -4
  38. package/src/memory/core.mjs +466 -8
  39. package/src/router/goal-reasoner.mjs +41 -7
  40. package/src/router/guardrail.mjs +37 -7
  41. package/src/router/resolver.mjs +50 -4
  42. package/src/sessions.mjs +5 -1
  43. package/src/source.mjs +54 -1
  44. package/src/syllogise.mjs +398 -27
  45. package/src/toml-config.mjs +13 -4
  46. package/src/viz.mjs +541 -0
@@ -57,6 +57,12 @@ export const DERIVED_FROM_PROP = "mgx:derivedFrom"; // umbrella: Fact →
57
57
  export const STATED_BY_PROP = "mgx:statedBy"; // a Source directly asserts a Fact
58
58
  export const CANONICALISED_FROM_PROP = "mgx:canonicalisedFrom"; // a canonical Fact ← its raw form
59
59
  export const CREATED_AT_PROP = "mgx:createdAt"; // first-write-wins ISO-8601 on every individual
60
+ // DERIVED at read/render time for most individuals (codegraph.mjs's derivedUpdatedAt: an
61
+ // individual's own createdAt, or the max createdAt over every edge touching it) — this constant
62
+ // exists for the handful of call sites that mutate an individual's OWN attributes in place
63
+ // without necessarily touching an edge (upsertSession, recomputeFactTrust,
64
+ // recomputeSourceReliability), where the derived rule alone can't see the change (PLAN_VIZ.md §2).
65
+ export const UPDATED_AT_PROP = "mgx:updatedAt";
60
66
  export const SOURCE_RELIABILITY_PROP = "mgx:sourceReliability"; // actor-level (session-scoped) trust nudge on a Source, [0.5,1.5]
61
67
 
62
68
  // The bare (session-less) singleton Source ids — the fallback for an
@@ -98,6 +104,7 @@ const MEMORY_VOCABULARY = [
98
104
  { prop: "mgx:ruleBaseCase", note: "recursive only: the base-case relation name (hop zero)" },
99
105
  { prop: "mgx:ruleRecStep", note: "recursive only: the self-referential recursive-step relation name" },
100
106
  { prop: CREATED_AT_PROP, note: "when an individual was FIRST written, ISO-8601 (first-write-wins on upsert); the audit 'when', the recency input to trust, the novelty signal" },
107
+ { prop: UPDATED_AT_PROP, note: "when an individual's OWN attributes were last mutated in place (upsertSession, recomputeFactTrust, recomputeSourceReliability) — most individuals never carry this and instead derive 'updated' from codegraph.mjs's derivedUpdatedAt (max createdAt over their edges)" },
101
108
  { prop: DERIVED_FROM_PROP, predicate: "derivedFrom", note: "umbrella: a Fact derived from a Source (or another Fact). ext ref prov:wasDerivedFrom (UNVERIFIED-pending-web-check)" },
102
109
  { prop: STATED_BY_PROP, predicate: "statedBy", note: "subPropertyOf derivedFrom: a Source directly asserts this Fact (one edge per independent source — replaces the factProvenance union)" },
103
110
  { prop: CANONICALISED_FROM_PROP, predicate: "canonicalisedFrom", note: "subPropertyOf derivedFrom: a canonical Fact cleaned from a raw Block/Source, never replacing it" },
@@ -141,12 +148,430 @@ export function emptyMemory() {
141
148
  * independent path-resolution copies (core.mjs's mutateMemory and fold.mjs's
142
149
  * writeMemoryGraph used to compute this path separately). */
143
150
  export function resolveMemoryGraphFile(dir, version = null) {
151
+ if (isMemoryHandle(dir) || isSqliteHandle(dir)) {
152
+ throw new Error("resolveMemoryGraphFile: dir is a memory/sqlite handle, not a file path (Backend A only)");
153
+ }
144
154
  if (version === null) return join(dir, MEMORY_GRAPH_REL);
145
155
  return join(dir, MEMORY_DIR_REL, `graph.v${version}.json`);
146
156
  }
147
157
 
148
158
  const memoryGraphFile = (dir) => resolveMemoryGraphFile(dir);
149
159
 
160
+ // ---- Storage-backend seam (PLAN_SEED.md §6) ---------------------------------
161
+ //
162
+ // Every dir-taking export in this file historically assumed `dir` was a plain
163
+ // string repo path that resolveMemoryGraphFile joins into an on-disk file
164
+ // (Backend A, unchanged below — still the exact byte-identical default for
165
+ // every existing caller that never opts into anything else).
166
+ //
167
+ // `dir` may now ALSO be a memory HANDLE: a small tagged object created by
168
+ // createInMemoryStore() (Backend B, pure in-memory, zero disk I/O) or
169
+ // createSqliteMemoryStore() (Backend C, a live node:sqlite connection kept
170
+ // open for the session's lifetime). loadMemory/mutateMemory below recognize
171
+ // both and dispatch the LOAD/PERSIST steps only; every other function in this
172
+ // file (appendFact, appendFacts, appendUtterance(s), appendRule,
173
+ // readFactRows, findRuleByName, resolveRelationChase(Reverse),
174
+ // findContradictions) takes `memory`/`dir` exactly as before and never
175
+ // branches on backend — they operate on the plain JS payload object
176
+ // mutateMemory hands them, regardless of where it came from or where it goes
177
+ // next. That is the whole point of the seam: id hashing, provenance/trust
178
+ // computation, migrateLegacyProvenance, recomputeSourceReliability and
179
+ // buildProseIndex are backend-agnostic logic, unchanged either way.
180
+ //
181
+ // snapshotMemory (manifest-versioned snapshots) and resolveMemoryGraphFile
182
+ // stay Backend-A-only (a handle has no on-disk file to snapshot) — both throw
183
+ // a clear error if given a handle rather than silently doing the wrong thing.
184
+
185
+ const BACKEND_MEMORY = "memory";
186
+ const BACKEND_SQLITE = "sqlite";
187
+
188
+ function isMemoryHandle(dir) {
189
+ return !!dir && typeof dir === "object" && dir.backend === BACKEND_MEMORY;
190
+ }
191
+ function isSqliteHandle(dir) {
192
+ return !!dir && typeof dir === "object" && dir.backend === BACKEND_SQLITE;
193
+ }
194
+ function isMemoryOrSqliteHandle(dir) {
195
+ return isMemoryHandle(dir) || isSqliteHandle(dir);
196
+ }
197
+
198
+ /**
199
+ * Backend B — pure in-memory store (new). A plain JS object held by the
200
+ * CALLER (never module-global state, which would break multiple concurrent
201
+ * sessions in one process): `{ backend: "memory", payload }`. loadMemory
202
+ * returns `payload` directly (the live reference, not a fresh parse — there
203
+ * is nothing to parse); mutateMemory's persist step is a no-op assignment
204
+ * (`handle.payload = out` — already the same object in every real caller,
205
+ * since none of appendFact/appendFacts/appendUtterance(s)/appendRule ever
206
+ * return a NEW object from their mutateMemory callback, they all mutate the
207
+ * payload in place). ZERO readFile/writeFile/JSON.parse/JSON.stringify calls
208
+ * ever happen for this backend — verified directly by this module's own
209
+ * dispatch (no fs import is even reachable from this path) and by
210
+ * test/memory-backend-memory.test.mjs's fs-spy assertions.
211
+ *
212
+ * Distinct from `--ephemeral` (createSession): ephemeral mode still does real
213
+ * readFile/JSON.parse/writeFile round-trips against a throwaway mkdtemp temp
214
+ * dir every turn — "disposable disk," not "no disk." Backend B is genuinely
215
+ * disk-free.
216
+ */
217
+ export function createInMemoryStore() {
218
+ return { backend: BACKEND_MEMORY, payload: emptyMemory() };
219
+ }
220
+
221
+ // ---- Backend C — SQLite (new; schema shape adapted from seonix's src/store.mjs,
222
+ // write model is NOT) ----------------------------------------------------------
223
+ //
224
+ // seonix (a sibling repo consuming tmct as a library) already has a working,
225
+ // opt-in node:sqlite store (SEONIX_STORE=sqlite, node:sqlite lazily imported,
226
+ // zero external dependency): an `ids`/`nodes`/`relations`/`edges`/`meta` table
227
+ // set. Its WRITE MODEL is a full rebuild-and-atomic-swap on every write — correct
228
+ // for seonix's problem (read-latency on a relatively static, rebuild-on-change
229
+ // code graph), wrong for tmct's (write-heavy, one-fact-at-a-time accumulation
230
+ // across a session's lifetime): lifting it as-is would just replace "rewrite the
231
+ // whole JSON file per turn" with "rebuild the whole SQLite file per turn."
232
+ //
233
+ // tmct's Backend C reuses the SHAPE, not the write model: real per-row
234
+ // INSERT/REPLACE/DELETE against a LIVE, OPEN connection kept for the session's
235
+ // lifetime (see createSqliteMemoryStore/closeSqliteMemoryStore below),
236
+ // diffed against whatever is already on that row so only touched
237
+ // individuals/edges are ever written — not seonix's rebuild-and-swap.
238
+ //
239
+ // Schema, adapted (not ported) to tmct's actual payload shape (emptyMemory(),
240
+ // above — { generated_at, memory, prefixes, vocabulary, classes,
241
+ // objectProperties, individuals, proseIndex }, distinct from seonix's code-graph
242
+ // `entities` shape): seonix's separate integer-interning `ids` table exists to
243
+ // cover edge endpoints that AREN'T always node ids (e.g. an `inherits` edge's
244
+ // `ext:<Base>` target). tmct's own edge groups (saidInSession, inReplyTo,
245
+ // statedBy, canonicalisedFrom) only ever link two individuals-table ids, so
246
+ // that interning table is dropped here — individuals/edges reference each
247
+ // other by their natural TEXT id directly, a deliberate simplification over a
248
+ // literal port. A Fact's reified rdf:subject/rdf:predicate/rdf:object live as
249
+ // ATTRIBUTES on the individual (tmct's own reification style, no seonix
250
+ // equivalent), so they ride inside that individual's own JSON blob column —
251
+ // no separate fact-triple columns needed.
252
+ //
253
+ // Cached, incrementally patched reads (closes the PLAN_SEED.md §6 gap the
254
+ // prior "honest shortcut" comment used to flag here): the READ side
255
+ // (readSqlitePayload) reconstructs the FULL in-memory payload shape from real
256
+ // SQL SELECTs only ONCE — the first call for a given handle, or the first
257
+ // call after a failed/rolled-back write — and stashes the result on
258
+ // `handle.cachedPayload`. Every later call returns a deep clone of that cache
259
+ // directly, with ZERO SQL queries: no re-SELECT of individuals, no
260
+ // per-relation edge SELECT. The WRITE side (persistSqlitePayload) was never a
261
+ // shortcut: it already diffs the incoming payload against what is already in
262
+ // each row/edge-group and only issues a real INSERT/REPLACE/DELETE for what
263
+ // actually changed (write cost proportional to what changed this turn, not to
264
+ // the total store size). It now ALSO applies that exact same diff to
265
+ // `handle.cachedPayload` in lockstep — patching only the individuals/edges it
266
+ // actually wrote to SQLite, in the same order SQLite itself would reorder
267
+ // them (a changed row gets a fresh rowid and sorts last) — so the cache never
268
+ // goes stale, and never needs a re-query to catch up either. If a write fails
269
+ // mid-transaction (ROLLBACK), the partially-patched cache is not trusted: it
270
+ // is invalidated so the NEXT read does an honest full rebuild instead of
271
+ // risking a state that was never actually committed.
272
+
273
+ const SQLITE_DDL = `
274
+ CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL);
275
+ CREATE TABLE IF NOT EXISTS individuals (id TEXT PRIMARY KEY, ord INTEGER NOT NULL, class TEXT, label TEXT, json TEXT NOT NULL);
276
+ CREATE TABLE IF NOT EXISTS relations (prop TEXT PRIMARY KEY, ord INTEGER NOT NULL, predicate TEXT, count INTEGER);
277
+ CREATE TABLE IF NOT EXISTS edges (prop TEXT NOT NULL, subject TEXT NOT NULL, object TEXT NOT NULL, subject_label TEXT, object_label TEXT, extra TEXT, PRIMARY KEY (prop, subject, object));
278
+ CREATE INDEX IF NOT EXISTS edges_by_prop ON edges(prop);
279
+ `;
280
+
281
+ // Edge keys with dedicated columns; any other key on an edge example object
282
+ // (none exist in core.mjs's own edge groups today, but a future/external
283
+ // writer might add one) round-trips via the `extra` JSON column, same
284
+ // discipline as seonix's own STD_EDGE_KEYS.
285
+ const STD_EDGE_KEYS = new Set(["subject", "object", "subjectLabel", "objectLabel"]);
286
+
287
+ /**
288
+ * Open (creating if absent) a resident node:sqlite connection for `dbPath` and
289
+ * return a Backend C handle: `{ backend: "sqlite", db, dbPath }`. `node:sqlite`
290
+ * is imported LAZILY here — calling this function is the ONLY way it is ever
291
+ * loaded, so a caller that never opts into this backend never even imports it
292
+ * (matching seonix's own SEONIX_STORE=sqlite gating discipline, and tmct's
293
+ * minimal-deps philosophy: zero external dependency either way).
294
+ *
295
+ * The connection is meant to be opened ONCE per session and kept open for the
296
+ * session's lifetime (not re-opened per call) — close it via
297
+ * closeSqliteMemoryStore when the session ends.
298
+ */
299
+ export async function createSqliteMemoryStore(dbPath) {
300
+ const { DatabaseSync } = await import("node:sqlite");
301
+ const db = new DatabaseSync(dbPath);
302
+ db.exec("PRAGMA journal_mode = WAL");
303
+ db.exec("PRAGMA synchronous = NORMAL");
304
+ db.exec(SQLITE_DDL);
305
+ return { backend: BACKEND_SQLITE, db, dbPath };
306
+ }
307
+
308
+ /** Close a Backend C handle's connection. A no-op for anything else (so a
309
+ * caller that doesn't know which backend it has can call this unconditionally
310
+ * at session end). */
311
+ export function closeSqliteMemoryStore(handle) {
312
+ if (isSqliteHandle(handle)) handle.db.close();
313
+ }
314
+
315
+ /** Deep-clone a JSON-safe value. Used two ways here: (1) readSqlitePayload
316
+ * hands every CALLER a clone of the cache, never the live cached object
317
+ * itself, so this backend keeps the same "fresh object every call" contract
318
+ * Backend A's JSON.parse(readFile()) always had — nothing outside this
319
+ * module can mutate handle.cachedPayload by mutating what loadMemory
320
+ * returned; and (2) persistSqlitePayload clones a value INTO the cache so
321
+ * the cache never ends up aliasing a piece of the caller's own payload
322
+ * object (which mutateMemory's caller may go on to mutate further). */
323
+ const cloneJson = (v) => (v === undefined ? v : structuredClone(v));
324
+
325
+ /** The loadMemory-equivalent read for Backend C. First call for a handle (or
326
+ * first call after a failed write invalidated the cache): a real, full
327
+ * reconstruction from SQL SELECTs, same as before — then it is stashed on
328
+ * `handle.cachedPayload`. Every later call, with no write in between, skips
329
+ * SQL entirely and returns a clone of that cache (see the module-comment
330
+ * above SQLITE_DDL for the full mechanism). A brand-new store (no meta rows
331
+ * written yet) reconstructs to the same shape emptyMemory() returns, so a
332
+ * fresh handle behaves like Backend A's ENOENT-bootstrap and Backend B's
333
+ * fresh createInMemoryStore(). */
334
+ function readSqlitePayload(handle) {
335
+ if (!handle.cachedPayload) handle.cachedPayload = buildSqlitePayloadFromRows(handle);
336
+ return cloneJson(handle.cachedPayload);
337
+ }
338
+
339
+ /** The actual SQL reconstruction — unchanged from the pre-cache implementation,
340
+ * just extracted so readSqlitePayload can call it only when the cache is
341
+ * cold. */
342
+ function buildSqlitePayloadFromRows(handle) {
343
+ const db = handle.db;
344
+ const empty = emptyMemory();
345
+ const getMeta = (k, fallback) => {
346
+ const row = db.prepare("SELECT v FROM meta WHERE k = ?").get(k);
347
+ return row ? JSON.parse(row.v) : fallback;
348
+ };
349
+
350
+ const individuals = db.prepare("SELECT json FROM individuals ORDER BY ord").all()
351
+ .map((r) => JSON.parse(r.json));
352
+
353
+ const edgesForProp = db.prepare(
354
+ "SELECT subject, object, subject_label, object_label, extra FROM edges WHERE prop = ? ORDER BY rowid",
355
+ );
356
+ const objectProperties = db.prepare("SELECT prop, predicate, count FROM relations ORDER BY ord").all()
357
+ .map((r) => ({
358
+ predicate: r.predicate,
359
+ prop: r.prop,
360
+ count: r.count,
361
+ examples: edgesForProp.all(r.prop).map((e) => {
362
+ const edge = { subject: e.subject, object: e.object, subjectLabel: e.subject_label, objectLabel: e.object_label };
363
+ if (e.extra) Object.assign(edge, JSON.parse(e.extra));
364
+ return edge;
365
+ }),
366
+ }));
367
+
368
+ return {
369
+ generated_at: getMeta("generated_at", empty.generated_at),
370
+ memory: getMeta("memory", empty.memory),
371
+ prefixes: getMeta("prefixes", empty.prefixes),
372
+ vocabulary: getMeta("vocabulary", empty.vocabulary),
373
+ classes: getMeta("classes", empty.classes),
374
+ objectProperties,
375
+ individuals,
376
+ proseIndex: getMeta("proseIndex", empty.proseIndex),
377
+ };
378
+ }
379
+
380
+ // ---- handle.cachedPayload mirrors --------------------------------------
381
+ // Applied by persistSqlitePayload in LOCKSTEP with the SQL statement sitting
382
+ // right beside each call — same condition (only when the SQL write actually
383
+ // runs), same effect, so the cache always ends up holding exactly what a
384
+ // fresh SQL reconstruction would now produce. `cache.individuals`/
385
+ // `cache.objectProperties` are mutated in place; persistSqlitePayload is
386
+ // responsible for invalidating the whole cache on a rolled-back write (these
387
+ // helpers assume the surrounding transaction succeeds).
388
+
389
+ /** Mirrors `INSERT OR REPLACE INTO individuals(...)`: an existing id is
390
+ * replaced IN PLACE (same array position, matching how SQL keeps that row's
391
+ * `ord` — and so its sort position — unchanged on an update); a new id is
392
+ * appended (matching a fresh row getting the next `ord`). */
393
+ function cacheUpsertIndividual(cache, ind) {
394
+ const clone = cloneJson(ind);
395
+ const i = cache.individuals.findIndex((x) => x?.id === ind.id);
396
+ if (i >= 0) cache.individuals[i] = clone;
397
+ else cache.individuals.push(clone);
398
+ }
399
+
400
+ /** Mirrors the individuals delete loop: drop any cached individual whose id
401
+ * isn't in the just-persisted payload's full id set. */
402
+ function cacheDropIndividualsExcept(cache, seenIds) {
403
+ cache.individuals = cache.individuals.filter((i) => seenIds.has(i?.id));
404
+ }
405
+
406
+ /** Find-or-create the cached edge group for `prop` — mirrors a relation row
407
+ * being implicitly created the first time a group is written. */
408
+ function cacheGroupFor(cache, prop) {
409
+ let g = cache.objectProperties.find((x) => x?.prop === prop);
410
+ if (!g) {
411
+ g = { predicate: null, prop, count: 0, examples: [] };
412
+ cache.objectProperties.push(g);
413
+ }
414
+ return g;
415
+ }
416
+
417
+ /** Mirrors `INSERT OR REPLACE INTO edges(...)`: SQLite deletes-then-reinserts
418
+ * a changed/new row on conflict, so it gets a fresh (highest) rowid and sorts
419
+ * LAST under readSqlitePayload's `ORDER BY rowid` — moving the entry to the
420
+ * end of `examples` here (rather than replacing it in place) reproduces that
421
+ * ordering exactly, without a re-SELECT. Rebuilds the cached edge shape the
422
+ * same way buildSqlitePayloadFromRows does (subject/object/labels + any
423
+ * extra keys), from the same `extraKeys` persistSqlitePayload already
424
+ * computed for the SQL `extra` column, so it never re-derives them. Same
425
+ * NUL-delimited (subject,object) key discipline as the SQL diff beside it
426
+ * (a space could be forged by a term that contains one; NUL never occurs in
427
+ * a normalized term). */
428
+ function cacheUpsertEdge(group, edge, extraKeys) {
429
+ const key = `${edge.subject}\u0000${edge.object}`;
430
+ group.examples = group.examples.filter((e) => `${e.subject}\u0000${e.object}` !== key);
431
+ const cached = {
432
+ subject: edge.subject, object: edge.object,
433
+ subjectLabel: edge.subjectLabel ?? null, objectLabel: edge.objectLabel ?? null,
434
+ };
435
+ if (extraKeys.length) Object.assign(cached, cloneJson(Object.fromEntries(extraKeys.map((k) => [k, edge[k]]))));
436
+ group.examples.push(cached);
437
+ }
438
+
439
+ /** Mirrors the per-group edge delete loop: drop any cached edge in this group
440
+ * whose (subject,object) key isn't in the just-persisted group's key set. */
441
+ function cacheDropEdgesExcept(group, newKeys) {
442
+ group.examples = group.examples.filter((e) => newKeys.has(`${e.subject}\u0000${e.object}`));
443
+ }
444
+
445
+ /** Mirrors the relations delete loop: drop any cached edge group whose prop
446
+ * wasn't in the just-persisted payload's `objectProperties`. */
447
+ function cacheDropGroupsExcept(cache, seenProps) {
448
+ cache.objectProperties = cache.objectProperties.filter((g) => seenProps.has(g?.prop));
449
+ }
450
+
451
+ /** Persist a mutated payload into a Backend C handle: real per-row
452
+ * INSERT/REPLACE/DELETE, diffed against whatever is ALREADY in the table for
453
+ * that individual id / (prop,subject,object) edge key — not seonix's
454
+ * rebuild-and-swap. Every statement runs inside one transaction so a session
455
+ * never observes (or leaves on disk) a half-applied mutation.
456
+ *
457
+ * Also patches `handle.cachedPayload` (if one exists yet — it always will in
458
+ * practice, since mutateMemory always calls loadMemory before this) in the
459
+ * same lockstep as the SQL diff below, so a subsequent loadMemory() never has
460
+ * to re-query SQLite to see this write (see the module comment above
461
+ * SQLITE_DDL). On a rolled-back write, the cache is invalidated rather than
462
+ * left holding a partially-applied patch — the next read does an honest
463
+ * full rebuild instead. */
464
+ function persistSqlitePayload(handle, payload) {
465
+ const db = handle.db;
466
+ const empty = emptyMemory();
467
+ const cache = handle.cachedPayload || null;
468
+ db.exec("BEGIN IMMEDIATE");
469
+ try {
470
+ const setMeta = db.prepare("INSERT OR REPLACE INTO meta(k, v) VALUES (?, ?)");
471
+ setMeta.run("generated_at", JSON.stringify(payload.generated_at ?? empty.generated_at));
472
+ setMeta.run("memory", JSON.stringify(payload.memory ?? empty.memory));
473
+ setMeta.run("prefixes", JSON.stringify(payload.prefixes ?? empty.prefixes));
474
+ setMeta.run("vocabulary", JSON.stringify(payload.vocabulary ?? empty.vocabulary));
475
+ setMeta.run("classes", JSON.stringify(payload.classes ?? empty.classes));
476
+ setMeta.run("proseIndex", JSON.stringify(payload.proseIndex ?? empty.proseIndex));
477
+ if (cache) {
478
+ cache.generated_at = cloneJson(payload.generated_at ?? empty.generated_at);
479
+ cache.memory = cloneJson(payload.memory ?? empty.memory);
480
+ cache.prefixes = cloneJson(payload.prefixes ?? empty.prefixes);
481
+ cache.vocabulary = cloneJson(payload.vocabulary ?? empty.vocabulary);
482
+ cache.classes = cloneJson(payload.classes ?? empty.classes);
483
+ cache.proseIndex = cloneJson(payload.proseIndex ?? empty.proseIndex);
484
+ }
485
+
486
+ // individuals: a real per-row upsert, only for an id that is new or whose
487
+ // JSON actually changed since the last persist — every other row is left
488
+ // untouched (no whole-table rewrite).
489
+ const getInd = db.prepare("SELECT ord, json FROM individuals WHERE id = ?");
490
+ const maxOrd = db.prepare("SELECT COALESCE(MAX(ord), -1) AS m FROM individuals").get().m;
491
+ let nextOrd = maxOrd + 1;
492
+ const upsertInd = db.prepare("INSERT OR REPLACE INTO individuals(id, ord, class, label, json) VALUES (?, ?, ?, ?, ?)");
493
+ const seenIds = new Set();
494
+ for (const ind of payload.individuals || []) {
495
+ seenIds.add(ind.id);
496
+ const json = JSON.stringify(ind);
497
+ const existing = getInd.get(ind.id);
498
+ if (existing && existing.json === json) continue; // unchanged — skip the write entirely (cache already matches)
499
+ const ord = existing ? existing.ord : nextOrd++;
500
+ upsertInd.run(ind.id, ord, ind.class ?? null, ind.label ?? null, json);
501
+ if (cache) cacheUpsertIndividual(cache, ind);
502
+ }
503
+ // Removal (no appendX function in this file ever removes an individual
504
+ // today — dead code path in practice, kept for correctness): a cheap
505
+ // index-only scan of the primary-key column only, never the JSON payload.
506
+ const deleteInd = db.prepare("DELETE FROM individuals WHERE id = ?");
507
+ for (const row of db.prepare("SELECT id FROM individuals").all()) {
508
+ if (!seenIds.has(row.id)) deleteInd.run(row.id);
509
+ }
510
+ if (cache) cacheDropIndividualsExcept(cache, seenIds);
511
+
512
+ // objectProperties/edges: per-edge diff WITHIN each group, scoped to that
513
+ // group's own rows (edges_by_prop) rather than the whole edges table — a
514
+ // group that gains one new edge (e.g. statedBy, touched by nearly every
515
+ // appendFact call) writes exactly that one new row, not the group's
516
+ // entire history.
517
+ const getRelOrd = db.prepare("SELECT ord FROM relations WHERE prop = ?");
518
+ const maxRelOrd = db.prepare("SELECT COALESCE(MAX(ord), -1) AS m FROM relations").get().m;
519
+ let nextRelOrd = maxRelOrd + 1;
520
+ const upsertRel = db.prepare("INSERT OR REPLACE INTO relations(prop, ord, predicate, count) VALUES (?, ?, ?, ?)");
521
+ const edgesForProp = db.prepare("SELECT subject, object, subject_label, object_label, extra FROM edges WHERE prop = ?");
522
+ const upsertEdge = db.prepare("INSERT OR REPLACE INTO edges(prop, subject, object, subject_label, object_label, extra) VALUES (?, ?, ?, ?, ?, ?)");
523
+ const deleteEdge = db.prepare("DELETE FROM edges WHERE prop = ? AND subject = ? AND object = ?");
524
+ const seenProps = new Set();
525
+ for (const group of payload.objectProperties || []) {
526
+ seenProps.add(group.prop);
527
+ const existingRows = edgesForProp.all(group.prop);
528
+ const existingByKey = new Map(existingRows.map((r) => [`${r.subject}\u0000${r.object}`, r]));
529
+ const newKeys = new Set();
530
+ const cacheGroup = cache ? cacheGroupFor(cache, group.prop) : null;
531
+ for (const e of group.examples || []) {
532
+ const key = `${e.subject}\u0000${e.object}`;
533
+ newKeys.add(key);
534
+ const extraKeys = Object.keys(e).filter((k) => !STD_EDGE_KEYS.has(k));
535
+ const extra = extraKeys.length ? JSON.stringify(Object.fromEntries(extraKeys.map((k) => [k, e[k]]))) : null;
536
+ const existing = existingByKey.get(key);
537
+ const unchanged = existing
538
+ && (existing.subject_label ?? null) === (e.subjectLabel ?? null)
539
+ && (existing.object_label ?? null) === (e.objectLabel ?? null)
540
+ && (existing.extra ?? null) === (extra ?? null);
541
+ if (unchanged) continue;
542
+ upsertEdge.run(group.prop, e.subject, e.object, e.subjectLabel ?? null, e.objectLabel ?? null, extra);
543
+ if (cacheGroup) cacheUpsertEdge(cacheGroup, e, extraKeys);
544
+ }
545
+ for (const key of existingByKey.keys()) {
546
+ if (newKeys.has(key)) continue;
547
+ const [s, o] = key.split("\u0000");
548
+ deleteEdge.run(group.prop, s, o);
549
+ }
550
+ if (cacheGroup) cacheDropEdgesExcept(cacheGroup, newKeys);
551
+ const relCount = Number.isFinite(group.count) ? group.count : (group.examples || []).length;
552
+ const relOrd = getRelOrd.get(group.prop)?.ord ?? nextRelOrd++;
553
+ upsertRel.run(group.prop, relOrd, group.predicate ?? null, relCount);
554
+ if (cacheGroup) { cacheGroup.predicate = group.predicate ?? null; cacheGroup.count = relCount; }
555
+ }
556
+ for (const row of db.prepare("SELECT prop FROM relations").all()) {
557
+ if (seenProps.has(row.prop)) continue;
558
+ db.prepare("DELETE FROM edges WHERE prop = ?").run(row.prop);
559
+ db.prepare("DELETE FROM relations WHERE prop = ?").run(row.prop);
560
+ }
561
+ if (cache) cacheDropGroupsExcept(cache, seenProps);
562
+
563
+ db.exec("COMMIT");
564
+ } catch (e) {
565
+ db.exec("ROLLBACK");
566
+ // The cache may hold a partially-applied patch at this point (some of the
567
+ // loop bodies above already mutated it before the failure) that was never
568
+ // actually committed to SQLite — never trust it silently. Drop it so the
569
+ // next loadMemory() call does an honest full rebuild instead.
570
+ handle.cachedPayload = undefined;
571
+ throw e;
572
+ }
573
+ }
574
+
150
575
  /** Atomic write of raw text (temp in the same dir + rename) — the discipline
151
576
  * every writer in this module (and fold.mjs/sessions.mjs's own copies) uses:
152
577
  * a crash never destroys the previous file, a concurrent reader never sees a
@@ -209,6 +634,9 @@ const resolveManifestFile = (dir) => join(dir, MEMORY_MANIFEST_REL);
209
634
  * the snapshot just written (or null if skipped); `prunedVersion` is the
210
635
  * number pruned, or null if nothing was in range to prune yet. */
211
636
  export async function snapshotMemory(dir, { retentionVersions } = {}) {
637
+ if (isMemoryOrSqliteHandle(dir)) {
638
+ throw new Error("snapshotMemory only supports the flat-JSON backend (Backend A) — a memory/sqlite handle has no on-disk graph.json to snapshot");
639
+ }
212
640
  const graphFile = resolveMemoryGraphFile(dir);
213
641
  let graphText;
214
642
  try {
@@ -251,10 +679,14 @@ export async function snapshotMemory(dir, { retentionVersions } = {}) {
251
679
  return { skipped: false, version: v, prunedVersion };
252
680
  }
253
681
 
254
- /** Load the memory graph for a repo dir. A missing store is the bootstrap:
255
- * return the empty payload (uncached the first append creates the file).
256
- * The result is a raw entities payload; parseEntities() loads it. */
682
+ /** Load the memory graph for a repo dir OR a Backend B/C handle (see the
683
+ * storage-backend seam above `createInMemoryStore`). A missing Backend-A
684
+ * store is the bootstrap: return the empty payload (uncached the first
685
+ * append creates the file). The result is a raw entities payload;
686
+ * parseEntities() loads it. */
257
687
  export async function loadMemory(dir) {
688
+ if (isMemoryHandle(dir)) return dir.payload;
689
+ if (isSqliteHandle(dir)) return readSqlitePayload(dir);
258
690
  let text;
259
691
  try {
260
692
  text = await readFile(memoryGraphFile(dir), "utf8");
@@ -265,6 +697,19 @@ export async function loadMemory(dir) {
265
697
  return JSON.parse(text);
266
698
  }
267
699
 
700
+ /** Persist a mutated payload back to `dir` — the seam's other half. Backend A
701
+ * (unchanged): atomic write of the whole file. Backend B: the payload IS the
702
+ * handle's live object already (every real caller mutates in place); this
703
+ * assignment is a documented no-op safety net, never I/O. Backend C: a real,
704
+ * diffed per-row INSERT/UPDATE/DELETE against the live connection — see
705
+ * persistSqlitePayload. */
706
+ async function persistMemory(dir, payload) {
707
+ if (isMemoryHandle(dir)) { dir.payload = payload; return; }
708
+ if (isSqliteHandle(dir)) { persistSqlitePayload(dir, payload); return; }
709
+ await mkdir(dirname(memoryGraphFile(dir)), { recursive: true });
710
+ await atomicWriteJson(memoryGraphFile(dir), payload);
711
+ }
712
+
268
713
  /** Fresh read → mutate → atomic write. Serialized per call; every public append
269
714
  * goes through here so a concurrent reader never sees a torn store. The lazy,
270
715
  * idempotent legacy-provenance migration rides this same cycle (step (b)): any
@@ -286,8 +731,7 @@ async function mutateMemory(dir, fn) {
286
731
  migrateLegacyProvenance(out);
287
732
  recomputeSourceReliability(out);
288
733
  out.proseIndex = buildProseIndex(out.individuals);
289
- await mkdir(dirname(memoryGraphFile(dir)), { recursive: true });
290
- await atomicWriteJson(memoryGraphFile(dir), out);
734
+ await persistMemory(dir, out);
291
735
  return out;
292
736
  }
293
737
 
@@ -425,7 +869,9 @@ function statedByObjectsFor(payload, factId) {
425
869
  * conclusion's trust premise-derived (`min(premiseTrusts) × ruleConfidence`)
426
870
  * instead of riding the bare entailed prior. Absent (the default, `{}`), this
427
871
  * is a no-op passthrough — every existing caller's score is byte-identical
428
- * (PLAN_INFERENCE_TESTING.md §4 stage 2's exit criterion). */
872
+ * (PLAN_INFERENCE_TESTING.md §4 stage 2's exit criterion). Also stamps
873
+ * `mgx:updatedAt` (PLAN_VIZ.md §2) — this mutates the Fact's own attributes in place without
874
+ * necessarily touching an edge, so the derived "max over edges" updatedAt rule can't see it. */
429
875
  function recomputeFactTrust(payload, fact, nowMs = Date.now(), trustOpts = {}) {
430
876
  const sourceIds = statedByObjectsFor(payload, fact.id);
431
877
  const createdAt = (fact.attributes || []).find((a) => a?.prop === CREATED_AT_PROP)?.value || "";
@@ -436,6 +882,7 @@ function recomputeFactTrust(payload, fact, nowMs = Date.now(), trustOpts = {}) {
436
882
  });
437
883
  setAttr(fact, TRUST_SCORE_PROP, "trustScore", String(score));
438
884
  setAttr(fact, TRUST_INPUTS_PROP, "trustInputs", JSON.stringify(inputs));
885
+ setAttr(fact, UPDATED_AT_PROP, "updatedAt", new Date(nowMs).toISOString());
439
886
  }
440
887
 
441
888
  /** Reconcile a Fact's Sources + statedBy edges with its (unchanged, compat)
@@ -540,6 +987,8 @@ function recomputeSourceReliability(payload) {
540
987
  const source = payload.individuals.find((i) => i?.id === sid);
541
988
  if (!source) continue;
542
989
  setAttr(source, SOURCE_RELIABILITY_PROP, "sourceReliability", String(sessionReliabilityFrom(counts)));
990
+ // Own-attribute mutation in place (PLAN_VIZ.md §2) — same reasoning as recomputeFactTrust.
991
+ setAttr(source, UPDATED_AT_PROP, "updatedAt", new Date().toISOString());
543
992
  }
544
993
 
545
994
  // Re-materialise trust for EVERY individual statedBy a recomputed session
@@ -560,17 +1009,26 @@ function upsertIndividual(payload, ind) {
560
1009
  else payload.individuals.push(ind);
561
1010
  }
562
1011
 
563
- /** Upsert one edge into the named relation group (dedupe by subject>object). */
1012
+ /** Upsert one edge into the named relation group (dedupe by subject>object). Stamps `createdAt`
1013
+ * on the edge, first-write-wins over the same (subject,object) pair — mirrors
1014
+ * `firstWriteCreatedAt`'s discipline: a re-upserted edge keeps its original creation time
1015
+ * rather than resetting to "now" on every write. This is the only place in the codebase edges
1016
+ * get a timestamp at all — `codegraph.mjs`'s `derivedUpdatedAt` reads it back. */
564
1017
  function upsertEdge(payload, { predicate, prop }, edge) {
565
1018
  let group = payload.objectProperties.find((g) => g?.prop === prop);
566
1019
  if (!group) {
567
1020
  group = { predicate, prop, count: 0, examples: [] };
568
1021
  payload.objectProperties.push(group);
569
1022
  }
1023
+ // Edges are flat ({subject, object, ...}), not attribute-bearing individuals, so this can't
1024
+ // reuse firstWriteCreatedAt (which reads `.attributes`) directly — same discipline, edge shape:
1025
+ // the prior edge's OWN createdAt wins if it has one, else the incoming candidate, else now.
1026
+ const prior = (group.examples || []).find((e) => e?.subject === edge.subject && e?.object === edge.object);
1027
+ const createdAt = prior?.createdAt || edge.createdAt || nowIso();
570
1028
  group.examples = (group.examples || []).filter(
571
1029
  (e) => !(e?.subject === edge.subject && e?.object === edge.object),
572
1030
  );
573
- group.examples.push(edge);
1031
+ group.examples.push({ ...edge, createdAt });
574
1032
  group.count = group.examples.length;
575
1033
  }
576
1034
 
@@ -197,13 +197,20 @@ export function threatsAmong(candidateName, _pending) {
197
197
  }
198
198
 
199
199
  /** Resolve the FOCUS the request scopes the goal model to — an entity binding
200
- * (extractEntity + the graph oracle), NOT an intent keyword. Returns the bound
201
- * individual or null (no bindable focus => a whole-graph / global goal). */
200
+ * (extractEntity + the graph oracle), NOT an intent keyword. Returns
201
+ * `{match, ambiguous, candidates}`: `match` is the bound individual or null (no
202
+ * bindable focus => a whole-graph / global goal); `ambiguous`+`candidates` let
203
+ * the caller distinguish "genuinely no focus" from "a focus term that TIED"
204
+ * (PLAN_BREADTH_FIRST_NLU.md §4 — the latter must never silently fall back to
205
+ * a global answer, since that would silently answer a DIFFERENT goal than the
206
+ * one the user actually named). */
202
207
  function focusOf(request, ctx) {
203
208
  const term = extractEntity(String(request || ""));
204
- if (!term || !ctx || !ctx.resolve) return null;
209
+ if (!term || !ctx || !ctx.resolve) return { match: null, ambiguous: false, candidates: [] };
205
210
  const r = ctx.resolve(term);
206
- return r && r.match && !r.ambiguous ? r.match : null;
211
+ if (r && r.match && !r.ambiguous) return { match: r.match, ambiguous: false, candidates: [] };
212
+ if (r && r.ambiguous) return { match: null, ambiguous: true, candidates: [r.match, ...(r.candidates || [])].filter(Boolean) };
213
+ return { match: null, ambiguous: false, candidates: [] };
207
214
  }
208
215
 
209
216
  /** The GLOBAL-MODE DOMAIN GATE's primitive: what entity CLASS (if any) did
@@ -268,14 +275,41 @@ export function dropCondition(intention, observed, mode, focus, focusClass) {
268
275
  * omits it). PLAN_CODE.md Track 1's synthesis oracle (synthbench/rules/
269
276
  * oracle.mjs) is the one caller that overrides it, with a CLONED array
270
277
  * holding a candidate rule — the candidate then runs through this exact same
271
- * meta-loop a hand-written rule does, never a parallel re-implementation. */
272
- export async function goalReason(request, tools, ctx, { driver = "goal-0.8.1", ruleSet = GOAL_RULES } = {}) {
278
+ * meta-loop a hand-written rule does, never a parallel re-implementation.
279
+ *
280
+ * `pinnedFocus` (internal — PLAN_BREADTH_FIRST_NLU.md §4's breadth-first
281
+ * ambiguity fix) skips `focusOf` entirely and scopes straight to the given
282
+ * individual: the mechanism the ambiguous-focus branch below uses to run this
283
+ * SAME meta-loop once per tied candidate, "pin, don't re-resolve" (the same
284
+ * idiom PLAN_BREADTH_FIRST_NLU.md §1 uses for ask.mjs's entity ties). No
285
+ * product call site ever passes it. */
286
+ export async function goalReason(request, tools, ctx, { driver = "goal-0.8.1", ruleSet = GOAL_RULES, pinnedFocus = null } = {}) {
273
287
  const declared = Array.isArray(tools) ? tools : [];
274
288
 
275
289
  // STEP 1 — deduce the goal scope from the DECLARED model + a bound focus,
276
290
  // then SELECT the goal-rule by pure applicability (no request keyword ever):
277
291
  // a bound focus reads scoped, no focus reads global (keystone arbitration).
278
- const focus = focusOf(request, ctx);
292
+ const focusRes = pinnedFocus ? { match: pinnedFocus, ambiguous: false, candidates: [] } : focusOf(request, ctx);
293
+ const focus = focusRes.match;
294
+
295
+ // An AMBIGUOUS focus term must never silently collapse to "global" — that
296
+ // would silently answer a DIFFERENT goal (whole-graph keystone arbitration)
297
+ // than the one the user actually named. Refuse honestly, the same "never a
298
+ // guess" discipline resolver.mjs/guardrail.mjs apply to an ambiguous resolved
299
+ // term — and, when the tied candidates share ONE class a declared goal-rule
300
+ // scopes and a dispatcher is wired, ADDITIONALLY run this SAME meta-loop once
301
+ // per (pinned) candidate, so a machine caller gets both the honest "still
302
+ // ambiguous" signal and every candidate's real composed answer.
303
+ if (!pinnedFocus && focusRes.ambiguous) {
304
+ const term = extractEntity(String(request || ""));
305
+ const pool = focusRes.candidates.slice(0, 4);
306
+ const why2 = `open-world: focus "${term}" is ambiguous (${pool.map((c) => c.label).join(", ")}) — narrow it`;
307
+ const scopable = pool.length > 0 && pool.every((c) => c.class === pool[0].class) && ruleSet.some((r) => r.focusClass === pool[0].class);
308
+ if (!scopable || !ctx.dispatch) return refuse(why2, driver);
309
+ const candidateResults = [];
310
+ for (const c of pool) candidateResults.push({ candidate: c.label, result: await goalReason(request, tools, ctx, { driver, ruleSet, pinnedFocus: c }) });
311
+ return { ...refuse(why2, driver), candidateResults };
312
+ }
279
313
  const mode = focus ? "scoped" : "global";
280
314
 
281
315
  // The open-world goal-generation seam, named honestly: a resolved focus whose