@hviana/sema 0.7.2 → 0.7.5

package/src/store.ts

@@ -285,6 +285,17 @@ export class BoundedMap<K, V> {
 export interface Store {
   readonly D: number;
 
+  /** The seed the artifact was TRAINED with, recovered from the store's own
+   *  `train.seed` metadata, or null for a store that was never trained.
+   *
+   *  This is not decoration: the seed feeds the alphabet and the seat keyring
+   *  (see the Mind constructor), so folding a query under any other seed lands
+   *  in a different vector space than the one the artifact's nodes were folded
+   *  into.  A Mind opening a trained store MUST adopt this seed unless the
+   *  caller explicitly overrides it — the same discipline that recovers
+   *  `train.D` and `geometry.maxGroup` from the metadata. */
+  readonly trainSeed: number | null;
+
   /** The work accumulator for the inference call in flight, or null.  The
    *  Mind attaches one per profiled response and detaches it after (see
    *  src/meter.ts).  A store MUST only ever write to it — no read may reach
@@ -917,6 +928,10 @@ export abstract class AbstractStore implements Store {
 
   protected _D: number;
   protected _maxGroup: number;
+  /** `train.seed` recovered by the backend at open, or null when the store was
+   *  never trained.  A backend that omits it simply reports null, which leaves
+   *  the caller's configured seed in force. */
+  protected _trainSeed: number | null = null;
   protected readonly minHaloMass: number;
   protected readonly efSearch: number;
   protected readonly overfetch: number;
@@ -1091,6 +1106,13 @@ export abstract class AbstractStore implements Store {
     return this._D;
   }
 
+  /** The seed the artifact was trained with, recovered from `train.seed` at
+   *  open.  Null for a store that was never trained.  See
+   *  {@link Store.trainSeed} for why this must govern inference. */
+  get trainSeed(): number | null {
+    return this._trainSeed;
+  }
+
   /** Await the async initialisation performed by the concrete constructor. */
   protected async _ensureReady(): Promise<void> {
     if (!this._ready) throw new Error("Store: not open");
@@ -1133,9 +1155,6 @@ export abstract class AbstractStore implements Store {
     return rec;
   }
 
-  /** Reconstruct the bytes a node spans by traversing the DAG bottom-up.
-   *  Iterative post-order on an explicit stack — the call stack never sees the
-   *  tree depth, so even an adversarial chain of nodes stays safe. */
   /** How many reads hit a MISSING node record this session (a dangling edge
    *  or kid id).  Zero in a healthy store; a growing count means references
    *  outlive their records — the read degrades safely to empty bytes, this
@@ -1147,6 +1166,25 @@ export abstract class AbstractStore implements Store {
    *  unprofiled store pays one null check per read and allocates nothing. */
   meter: Meter | null = null;
 
+  /** Reconstruct the bytes a node spans by traversing the DAG bottom-up.
+   *  Iterative post-order on an explicit stack — the call stack never sees the
+   *  tree depth, so even an adversarial chain of nodes stays safe.
+   *
+   *  TERMINATION.  The walk memoizes into a LOCAL map, and `_bytesCache` is
+   *  consulted only as a warm hint whose hit is immediately promoted into that
+   *  map.  It used to use `_bytesCache` itself as the memo, which is not a
+   *  memo at all: it EVICTS, and its `"smallest"` policy prefers precisely the
+   *  freshly-resolved small children that the pending parents on the stack are
+   *  waiting for.  A parent then finds them uncached again, re-pushes them,
+   *  they are re-resolved, re-inserted, re-evicted — the loop makes no
+   *  progress and never exits.  Latent until the cache saturates, then
+   *  unconditional: observed in the wild at 19.9M nodes with the 20 MB cache
+   *  pinned at 19,999,962/20,000,000 bytes, spinning 8h45m on a node whose
+   *  whole content was 124 bytes (5 kids, 2 of them perpetually re-evicted).
+   *  Because the loop is synchronous, no timer could fire — the trainer's stall
+   *  watchdog never got a turn either.  A local map resolves each node at most
+   *  once per call, so the walk terminates by construction and `_bytesCache`
+   *  goes back to being a pure speed hint. */
   bytes(id: NodeId): Uint8Array {
     if (this.meter) {
       this.meter.byteReads++;
@@ -1162,12 +1200,25 @@ export abstract class AbstractStore implements Store {
 
     const stack: NodeId[] = [id];
     const cache = this._bytesCache;
+    // The walk's own memo.  Entries are the same shared arrays `_bytesCache`
+    // holds (no extra copy), and it lives exactly as long as this call.
+    const done = new Map<NodeId, Uint8Array>();
 
     while (stack.length > 0) {
       const nid = stack[stack.length - 1]; // peek
 
-      // Already resolved by an earlier traversal.
-      if (cache.get(nid)) {
+      // Already resolved by this walk — the ONLY authority the readiness test
+      // below trusts, because it cannot be evicted underneath us.
+      if (done.has(nid)) {
+        stack.pop();
+        continue;
+      }
+
+      // Warm hint: a hit is promoted into `done` in the same step, so from
+      // here on the entry is pinned for the rest of the walk.
+      const warm = cache.get(nid);
+      if (warm !== undefined) {
+        done.set(nid, warm);
         stack.pop();
         continue;
       }
@@ -1180,22 +1231,27 @@ export abstract class AbstractStore implements Store {
         // The cache makes the empty read permanent for the session; the
         // counter survives as the visible trace.
         this.danglingReads++;
+        done.set(nid, _ZERO);
         cache.set(nid, _ZERO);
         stack.pop();
         continue;
       }
       if (rec.leaf) {
-        cache.set(nid, new Uint8Array(rec.leaf));
+        // COPY before caching: rec.leaf is the node record's own buffer, and
+        // handing it out would let one mutating caller corrupt the record.
+        const leaf = new Uint8Array(rec.leaf);
+        done.set(nid, leaf);
+        cache.set(nid, leaf);
         stack.pop();
         continue;
       }
 
-      // Branch — push any uncached children (reverse order so they resolve
-      // left-to-right).  If every child is already cached, concatenate now.
+      // Branch — push any unresolved children (reverse order so they resolve
+      // left-to-right).  If every child is resolved, concatenate now.
       const kids = rec.kids ?? [];
       let ready = true;
       for (let i = kids.length - 1; i >= 0; i--) {
-        if (!cache.get(kids[i])) {
+        if (!done.has(kids[i])) {
           stack.push(kids[i]);
           ready = false;
         }
@@ -1203,11 +1259,12 @@ export abstract class AbstractStore implements Store {
       if (!ready) continue;
 
       stack.pop();
-      const out = concat(kids.map((k) => cache.get(k)!));
+      const out = concat(kids.map((k) => done.get(k)!));
+      done.set(nid, out);
       cache.set(nid, out);
     }
 
-    const out = cache.get(id) ?? _ZERO;
+    const out = done.get(id) ?? _ZERO;
     if (this.meter) this.meter.bytesRead += out.length;
     return out;
   }
@@ -1730,16 +1787,35 @@ export abstract class AbstractStore implements Store {
    *  common-prefix / common-suffix trim: whatever remains after both trims is
    *  the single differing span (substitution, insertion or deletion), and both
    *  remainders must fit the budget.  Scattered differences leave a wide
-   *  middle and are rejected. */
+   *  middle and are rejected.
+   *
+   *  Every read here is CAPPED (§2.8).  It used to open with
+   *  `bytesPrefix(k, Number.MAX_SAFE_INTEGER)` — the ALL sentinel, i.e. the
+   *  full materialising `bytes()` read — on the deposit hot path, and only
+   *  then compare lengths.  So a candidate the length test was about to reject
+   *  had already been reconstructed byte for byte.  The LENGTHS decide first
+   *  instead, from the `contentLen` memo the interning order has already built
+   *  bottom-up, and the target's length is itself read under a cap: a target
+   *  longer than `la + W` is rejected without touching one of its bytes.
+   *  Same semantics — the old capped `b` read would have produced
+   *  `a.length + W + 1` here and failed the very same test — strictly fewer
+   *  byte reads.  The `+ 1` on each byte cap keeps `_prefix`'s
+   *  "complete reconstruction" test true, so the results still cache. */
   private differsByOneWindow(
     kids: NodeId[],
     targetId: NodeId,
     W: number,
   ): boolean {
-    const a = concat(
-      kids.map((k) => this.bytesPrefix(k, Number.MAX_SAFE_INTEGER)),
-    );
-    const b = this.bytesPrefix(targetId, a.length + W + 1);
+    const lens = kids.map((k) => this.contentLen(k));
+    let la = 0;
+    for (const n of lens) la += n;
+    const cap = la + W + 1;
+    // `contentLen` under a cap returns a clamped LOWER BOUND once the partial
+    // sum reaches it, so `>= cap` is exactly "longer than la + W".
+    const lb = this.contentLen(targetId, cap);
+    if (lb >= cap || Math.abs(la - lb) > W) return false;
+    const a = concat(kids.map((k, i) => this.bytesPrefix(k, lens[i] + 1)));
+    const b = this.bytesPrefix(targetId, lb + 1);
     if (Math.abs(a.length - b.length) > W) return false;
     const n = Math.min(a.length, b.length);
     let i = 0;

package/test/89-completion-recursion.test.mjs

@@ -77,15 +77,35 @@ const mix = (x) => {
   return (x ^ (x >>> 15)) >>> 0;
 };
 
-/** Four-word windows of the repo's own English prose.  Code fences, inline
- *  code and link targets are stripped so what is left is language, which is
- *  where the fragment overlap lives. */
+/** Four-word windows of the repo's own prose, taken from the CODE's comments
+ *  under src (TypeScript) and test (the suites) — never from documentation.  A
+ *  test that reads docs is coupled to every documentation edit: deleting one
+ *  root file once took its corpus below the non-vacuity guard and broke every
+ *  release.  Code comments are prose too, and the code is always here.
+ *
+ *  Comment markers, code fences, inline code and link targets are stripped so
+ *  what is left is language, which is where the fragment overlap lives. */
 function fragments() {
   const out = [];
-  for (const f of readdirSync(REPO).filter((f) => f.endsWith(".md")).sort()) {
-    let t = readFileSync(join(REPO, f), "utf8");
-    t = t.replace(/```[\s\S]*?```/g, " ").replace(/`[^`]*`/g, " ");
-    t = t.replace(/\[[^\]]*\]\([^)]*\)/g, " ");
+  const files = [];
+  const walk = (dir, exts) => {
+    for (const e of readdirSync(dir, { withFileTypes: true })) {
+      if (
+        e.name.startsWith(".") || e.name === "node_modules" || e.name === "dist"
+      ) continue;
+      const p = join(dir, e.name);
+      if (e.isDirectory()) walk(p, exts);
+      else if (exts.some((x) => e.name.endsWith(x))) files.push(p);
+    }
+  };
+  walk(join(REPO, "src"), [".ts"]);
+  walk(join(REPO, "test"), [".mjs"]);
+  for (const f of files.sort()) {
+    const text = readFileSync(f, "utf8");
+    let t = "";
+    for (const m of text.matchAll(/\/\*[\s\S]*?\*\/|\/\/[^\n]*/g)) {
+      t += " " + m[0];
+    }
     t = t.toLowerCase().replace(/[^a-z ]+/g, " ").replace(/\s+/g, " ");
     const w = t.split(" ").filter(Boolean);
     for (let i = 0; i + 4 < w.length; i += 2) {
@@ -140,9 +160,9 @@ const SIZES = [750, 1000, 1500];
 test("completion recursion: per-query work does not grow with the corpus", async () => {
   assert.ok(
     FRAG.length > 4000,
-    `only ${FRAG.length} prose fragments found in ${REPO}/*.md — this test ` +
-      `draws its corpus from the repo's own documentation; with the prose gone ` +
-      `it can no longer exercise the completion recursion at all`,
+    `only ${FRAG.length} prose fragments found in the source comments — this ` +
+      `test draws its corpus from src/**/*.ts and test/**/*.mjs; with the ` +
+      `comments gone it can no longer exercise the completion recursion at all`,
   );
 
   const searches = [], pops = [], answers = [], secs = [];

package/test/96-bytes-walk-termination.test.mjs

@@ -0,0 +1,115 @@
+// 96-bytes-walk-termination.test.mjs — `bytes()` must TERMINATE when its
+// traversal memo cannot hold the reconstruction's working set.
+//
+// `bytes()` is an iterative post-order walk: it peeks the top of an explicit
+// stack, pushes any child that is not yet resolved, and concatenates once every
+// child is.  Its termination argument needs a memo that only ever GROWS.  It
+// used `_bytesCache` — a byte-accounted BoundedMap that EVICTS, and whose
+// `"smallest"` policy deliberately prefers the cheapest-to-rebuild entries,
+// i.e. exactly the small, freshly-resolved children the parents still sitting
+// on the stack are waiting for.  The parent finds them unresolved again,
+// re-pushes them, they are re-resolved, re-inserted and re-evicted.  No
+// progress.  The loop never exits.
+//
+// It stayed hidden because it needs the cache to be SATURATED, which only a
+// long run reaches.  OBSERVED IN THE FIELD: a training run at 19.9M nodes with
+// the 20 MB cache pinned at 19,999,962 bytes spun for 8h45m of 100% CPU on one
+// node — whose entire content was 124 bytes, with 5 kids, 2 of them
+// perpetually re-evicted.  Sampled 45s apart through the V8 inspector, the
+// walk's root id, the node's length and the cache's byte count were all
+// identical; only the stack depth oscillated between 1 and 2.
+//
+// And the loop is SYNCHRONOUS, so nothing could observe it: the trainer's
+// 15-minute stall watchdog is a timer, and a timer cannot fire while the
+// microtask/JS stack is occupied.  The run looked alive for 9 hours.
+//
+// The fix makes the walk memoize into a LOCAL map (`_bytesCache` demoted to a
+// warm hint whose hit is promoted into that map immediately), so each node
+// resolves at most once per call and termination is structural.
+//
+// This test does NOT depend on the eviction cursor's position — the field case
+// reached the defect stochastically, via the cursor sweeping the map's
+// recently-inserted tail.  Here the subtree's children simply sum to more bytes
+// than the ceiling, so no cursor position can save it: pre-fix this file hangs
+// forever, and `node --test` reports it only as a timeout.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { SQliteStore } from "../dist/src/store-sqlite.js";
+
+const D = 64;
+const gist = () => {
+  const v = new Float32Array(D);
+  v[0] = 1;
+  return v;
+};
+
+test("bytes() terminates when its memo cannot hold the working set", async () => {
+  // A ceiling small enough to saturate in milliseconds.  The mechanism is
+  // scale-free: this is the only thing scaled down from the field case.
+  const CEILING = 4096;
+  const store = new SQliteStore({
+    D,
+    bytesCacheMax: CEILING,
+    path: ":memory:",
+  });
+
+  // One ordinary 9,600-byte form: 300 distinct 32-byte children under a root.
+  const KIDS = 300, WIDTH = 32;
+  const kids = [];
+  for (let b = 0; b < KIDS; b++) {
+    const buf = new Uint8Array(WIDTH);
+    for (let i = 0; i < WIDTH; i++) buf[i] = (b * 131 + i * 17) & 0xff;
+    kids.push(await store.putLeaf(buf, gist()));
+  }
+  const root = await store.putBranch(kids, gist());
+  const expected = store.contentLen(root);
+  assert.equal(expected, KIDS * WIDTH);
+
+  // Churn the subtree out of the memo and leave it AT its ceiling — the steady
+  // state every long run reaches.
+  for (let k = 0; k < 4000; k++) {
+    const buf = new Uint8Array(48);
+    for (let i = 0; i < 48; i++) buf[i] = (k * 7919 + i * 251) & 0xff;
+    await store.putLeaf(buf, gist());
+  }
+
+  // The premise the guard rests on: the memo genuinely cannot hold the working
+  // set.  If a future change grows the ceiling or shrinks the fixture, this
+  // assertion fails LOUDLY rather than letting the test pass vacuously.
+  assert.ok(
+    KIDS * WIDTH > CEILING,
+    `fixture must exceed the memo ceiling (${KIDS * WIDTH} vs ${CEILING})`,
+  );
+
+  // Pre-fix this call never returns.  Post-fix it is sub-millisecond.
+  const out = store.bytes(root);
+  assert.equal(out.length, expected);
+  for (let b = 0; b < KIDS; b++) {
+    for (let i = 0; i < WIDTH; i++) {
+      assert.equal(out[b * WIDTH + i], (b * 131 + i * 17) & 0xff);
+    }
+  }
+});
+
+test("differsByOneWindow's reads are capped — no ALL-sentinel read on deposit", async () => {
+  // §2.8: the near-dedup byte check used to open with
+  // `bytesPrefix(k, Number.MAX_SAFE_INTEGER)` — the ALL sentinel, which routes
+  // to the full materialising `bytes()` — and only THEN compare lengths.  A
+  // candidate the length test was about to reject had already been rebuilt byte
+  // for byte, and that read is what dragged the deposit path into the walk
+  // above.  Lengths now decide first, from the `contentLen` memo.
+  const src = await import("node:fs").then((fs) =>
+    fs.readFileSync(new URL("../src/store.ts", import.meta.url), "utf8")
+  );
+  const body = src.slice(src.indexOf("private differsByOneWindow"));
+  const end = body.indexOf("\n  }\n");
+  const fn = body.slice(0, end);
+  assert.ok(
+    !fn.includes("MAX_SAFE_INTEGER"),
+    "differsByOneWindow must not read with the ALL sentinel",
+  );
+  assert.ok(
+    fn.includes("this.contentLen("),
+    "differsByOneWindow must decide on lengths before reading bytes",
+  );
+});

package/test/97-store-seed.test.mjs

@@ -0,0 +1,105 @@
+// 97-store-seed.test.mjs — a trained store's OWN seed governs the Mind that
+// opens it.
+//
+// `train.seed` is persisted by the trainer (example/train_base/main.ts) and the
+// trainer refuses to resume a store under a different seed, so the value is
+// authoritative for the artifact.  The seed feeds `makeKeyring`, `Space.rand`
+// and the `Alphabet` in the Mind constructor: folding a query under any other
+// seed lands in a DIFFERENT vector space than the one the artifact's nodes were
+// folded into, so recognition and resonance read the wrong space and every
+// answer degrades silently.
+//
+// The store recovers `train.D` and `geometry.maxGroup` from its own metadata at
+// open; `train.seed` must be recovered the same way, and a Mind that did not
+// receive an explicit seed must adopt it.  An explicit caller seed still wins.
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { DEFAULT_CONFIG, Mind } from "../dist/src/index.js";
+import { SQliteStore } from "../dist/src/store-sqlite.js";
+
+/** A store that was never trained carries no seed and leaves the caller's
+ *  configured default in force. */
+test("an untrained store reports no trainSeed and keeps the default seed", async () => {
+  const store = new SQliteStore({ path: ":memory:", D: 256 });
+  const mind = new Mind({ store });
+  assert.equal(store.trainSeed, null);
+  assert.equal(mind.cfg.seed, DEFAULT_CONFIG.seed);
+  await store.close();
+});
+
+/** The artifact's seed is recovered at open and adopted by a Mind that was not
+ *  given one; an explicit seed still overrides it. */
+test("a trained store's seed is recovered and adopted unless overridden", async () => {
+  const dir = mkdtempSync(join(tmpdir(), "sema-seed-"));
+  const stem = join(dir, "trained");
+  const TRAIN_SEED = 7;
+
+  // Build the artifact: ingest under an explicit seed, then persist the seed
+  // exactly as the trainer does.
+  {
+    const store = new SQliteStore({ path: stem, D: 256 });
+    const mind = new Mind({ seed: TRAIN_SEED, store });
+    await mind.ingest([["the sky is blue", "blue"]]);
+    await store.setMeta("train.seed", String(TRAIN_SEED));
+    store.commit();
+    await store.close();
+  }
+
+  // Reopen WITHOUT a seed: the store's own seed must stand.
+  {
+    const store = new SQliteStore({ path: stem, D: 256 });
+    assert.equal(store.trainSeed, TRAIN_SEED);
+    const adopted = new Mind({ store });
+    assert.equal(adopted.cfg.seed, TRAIN_SEED);
+    await store.close();
+  }
+
+  // Reopen WITH an explicit seed: the caller wins over the artifact.
+  {
+    const store = new SQliteStore({ path: stem, D: 256 });
+    const explicit = new Mind({ seed: 3, store });
+    assert.equal(explicit.cfg.seed, 3);
+    await store.close();
+  }
+
+  rmSync(dir, { recursive: true, force: true });
+});
+
+/** The adopted seed is the one the answer is computed under: a store ingested
+ *  under seed 7 answers a query identically when reopened without a seed and
+ *  when reopened with seed 7 passed explicitly. */
+test("adopting the artifact seed reproduces the artifact's answer", async () => {
+  const dir = mkdtempSync(join(tmpdir(), "sema-seed-"));
+  const stem = join(dir, "trained");
+  const TRAIN_SEED = 7;
+  const QUESTION = "the sky is blue";
+
+  let artifactAnswer;
+  {
+    const store = new SQliteStore({ path: stem, D: 256 });
+    const mind = new Mind({ seed: TRAIN_SEED, store });
+    await mind.ingest([
+      ["the sky is blue", "blue"],
+      ["the grass is green", "green"],
+    ]);
+    artifactAnswer = (await mind.respondText(QUESTION)).trim();
+    await store.setMeta("train.seed", String(TRAIN_SEED));
+    store.commit();
+    await store.close();
+  }
+  assert.equal(artifactAnswer, "blue");
+
+  {
+    const store = new SQliteStore({ path: stem, D: 256 });
+    const adopted = new Mind({ store });
+    assert.equal(adopted.cfg.seed, TRAIN_SEED);
+    assert.equal((await adopted.respondText(QUESTION)).trim(), artifactAnswer);
+    await store.close();
+  }
+
+  rmSync(dir, { recursive: true, force: true });
+});