@hviana/sema 0.6.0 → 0.7.1

package/test/91-branch-bytes-cache.test.mjs

@@ -0,0 +1,152 @@
+// 91-branch-bytes-cache.test.mjs — reconstructing a node twice must not
+// re-walk it.
+//
+// `bytesPrefix` rebuilds a node's bytes by descending the DAG, one `store.get`
+// per node visited.  `_prefix` consults `_bytesCache` for EVERY id but used to
+// populate it only for leaves, so a BRANCH re-walked its entire subtree on every
+// request — and because the DAG is hash-consed, the same children recur under
+// many different parents.
+//
+// MEASURED on the trained store (18,938,834 nodes), ONE 1,314-byte query:
+//   _prefix calls (all levels) : 20,021,474
+//   distinct ids               :    469,083   → 42.7x reuse
+//   avoidable by a cache       : 19,552,391   → 97.7%
+//   top-level calls            :     87,789 over 76,849 distinct (1.1x)
+//   hottest id                 : a single-byte leaf, 2,599,984 reconstructions
+//
+// The 1.1x at the top level is why this hid for so long: measured there, reuse
+// looks absent and a cache looks worthless.  All of the reuse is one level down.
+//
+// `_bytesCache` was already the right home — a byte-accounted BoundedMap with
+// "smallest"/"clock" eviction, the configuration this codebase reserves for a
+// TRANSPARENT cache (evicting costs a re-read and nothing else).  Reconstruction
+// is a pure function of the store, so it qualifies; only the population was
+// missing.
+//
+// WHAT THIS FILE ACTUALLY GUARDS — read this before trusting it.
+//
+// The re-walk assertion routes through `_prefix` (a CAPPED read), because that
+// is where the fix lives.  `bytesPrefix(id, ALL)` short-circuits to `bytes()`
+// (store.ts `bytesPrefix`: `maxLen >= 0x7fffffff → this.bytes(id)`), and
+// `bytes()` has its OWN, pre-existing branch cache — so a re-walk assertion
+// built on the ALL sentinel stays green when the `_prefix` fix is reverted and
+// guards nothing (this file once did exactly that, and its header claimed the
+// cause was "the node carries `flat` bytes", which is false: `flat` is
+// STRUCTURAL — a branch stores its bytes flat iff every kid is an implicit
+// single-byte leaf (store.ts `flatKidsBytes`) — and the fixture's node is
+// non-flat, kids of real chunk nodes.  There is no size threshold that drops
+// `flat`).
+//
+// With the fix, a capped read that completes the node caches the BRANCH
+// (`got < maxLen` guard), so a second capped read is a full-cache hit and costs
+// 0 node reads; with the fix reverted it re-reads the root (1 node read) because
+// only the LEAVES are cached.  The signal is small at fixture scale precisely
+// because the fixture's tree is shallow (root → flat chunks → leaves); the real
+// 5.4–7.7× nodeRecords reduction is verified on the trained store instead.
+//
+// The TRUNCATION assertion at the end IS a real guard, verified red: with the
+// `got < maxLen` condition removed, it fails with "a capped read poisoned the
+// cache: the full read came back 29 bytes instead of 59".  That is the
+// dangerous half of this fix — a truncated prefix served as a node's whole
+// content would silently corrupt every later reader — so that is the half worth
+// having a test for.
+//
+// The fix's real effect is verified on the trained store instead: identical
+// answers and identical `bytesRead`/`byteReads`/`junctionPops`, with
+// `nodeRecords` falling 1,182,651 → 218,449 · 2,462,577 → 321,794 ·
+// 2,492,035 → 381,020 (5.4–7.7×), and the 1,314-byte query 14.0 s → 9.4 s.
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { Mind } from "../dist/src/index.js";
+import { SQliteStore } from "../dist/src/store-sqlite.js";
+import { Meter } from "../dist/src/meter.js";
+
+const ALL = 0x7fffffff;
+
+test("a branch's bytes are reconstructed once, not re-walked", async () => {
+  const store = new SQliteStore({ path: ":memory:", D: 256 });
+  const mind = new Mind({ seed: 7, store });
+  // Long deposits, so the nodes have real interior structure to re-walk.
+  await mind.ingest([
+    ["alpha", "the quick brown fox jumps over the lazy dog again and again"],
+    ["beta", "the quick brown fox jumps over the lazy cat again and again"],
+    ["gamma", "a quick brown fox once jumped over a lazy dog and then rested"],
+  ]);
+
+  // A branch node with interior structure — the deposit's own root.
+  const tree = mind.perceive(
+    "the quick brown fox jumps over the lazy dog again and again",
+  );
+  const id = mind.resolve(new TextEncoder().encode(
+    "the quick brown fox jumps over the lazy dog again and again",
+  ));
+  assert.ok(id !== null, "the deposited form resolves to a stored node");
+  assert.ok(tree.kids && tree.kids.length > 1, "and it has interior structure");
+
+  // Route through `_prefix`, not `bytes()`: the ALL sentinel short-circuits to
+  // `bytes()`, whose branch cache is pre-existing and would keep the assertion
+  // green even with the `_prefix` fix reverted.  A capped read past the node's
+  // full length completes the walk (so the branch gets cached, `got < maxLen`)
+  // while staying off the ALL fast path.  `contentLen` warms `_recCache`/`_lenCache`
+  // only, never `_bytesCache`, so the first read below starts cold.
+  const fullLen = store.contentLen(id);
+  store.meter = new Meter();
+  const capped = fullLen + 1;
+
+  // FIRST reconstruction — this one legitimately walks the subtree.
+  const first = store.bytesPrefix(id, capped);
+  const walked = store.meter.nodeRecords;
+  assert.ok(
+    first.length > 0,
+    "the node reconstructs to bytes",
+  );
+  // NON-VACUITY: if the first read did no node reads either, the store answered
+  // from some other cache and this test proves nothing about re-walking.
+  assert.ok(
+    walked > 0,
+    `the first reconstruction did no node reads at all (nodeRecords=${walked}), ` +
+      `so there is no re-walk for this test to detect`,
+  );
+
+  // SECOND reconstruction of the SAME node — must be free.
+  const before = store.meter.nodeRecords;
+  const second = store.bytesPrefix(id, capped);
+  const again = store.meter.nodeRecords - before;
+
+  console.log(
+    `    first reconstruction: ${walked} node reads; second: ${again}`,
+  );
+  assert.deepEqual(second, first, "the cached bytes are the same bytes");
+  assert.equal(
+    again,
+    0,
+    `re-reading the same node cost ${again} node reads (the first cost ` +
+      `${walked}) — reconstruction is a pure function of the store, so the ` +
+      `second request must be served from _bytesCache, not re-walked`,
+  );
+
+  // AND a truncated read must never be cached AS the whole node: serving a
+  // prefix as the full content would silently corrupt every later reader.
+  const cut = Math.max(1, first.length >> 1);
+  const store2 = new SQliteStore({ path: ":memory:", D: 256 });
+  const mind2 = new Mind({ seed: 7, store: store2 });
+  await mind2.ingest([[
+    "alpha",
+    "the quick brown fox jumps over the lazy dog again and again",
+  ]]);
+  const id2 = mind2.resolve(new TextEncoder().encode(
+    "the quick brown fox jumps over the lazy dog again and again",
+  ));
+  store2.bytesPrefix(id2, cut); // truncated FIRST, so a bad cache would poison
+  const whole = store2.bytesPrefix(id2, ALL);
+  assert.equal(
+    whole.length,
+    first.length,
+    `a capped read poisoned the cache: the full read came back ${whole.length} ` +
+      `bytes instead of ${first.length}`,
+  );
+
+  await store.close();
+  await store2.close();
+});

package/test/93-regime-prediction.test.mjs

@@ -0,0 +1,148 @@
+// 93-regime-prediction.test.mjs — the retrieval/composition regime (R8) is
+// exposed as a structured trace step, without changing inference.
+//
+// After the FIRST mechanism runs (cover, which §2.6 places first and floors at
+// 0), the market's whole outcome is already determined by the one cost ladder:
+// the consensus climb runs exactly when `worthRunning(2 * STEP)` is true —
+// CAST (floor 2·STEP) is the cheapest mechanism that first-touches it.  An
+// incumbent at or below that floor prunes CAST and, with it, the climb
+// (retrieval); anything above — or no incumbent — runs the full market and the
+// climb (composition).  The step is purely observational: it is built only
+// under a trace (optional-chaining short-circuits it otherwise), and it never
+// alters which candidate wins.  The assertions here check the payload's
+// STRUCTURE and its consistency with the actual market outcome, never that
+// inference itself changed.
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { Mind } from "../dist/src/index.js";
+import { SQliteStore } from "../dist/src/store-sqlite.js";
+
+const mk = (seed = 7) =>
+  new Mind({ seed, store: new SQliteStore({ path: ":memory:", D: 256 }) });
+
+/** Collect the full step stream for one traced query. */
+async function trace(mind, q) {
+  const steps = [];
+  const ans = await mind.respondText(q, (s) => steps.push(s));
+  return { steps, ans };
+}
+
+function regimeStep(steps) {
+  return steps.filter((s) => s.mechanism.at(-1) === "regimePrediction");
+}
+
+test("1. a fully-grounding query predicts retrieval, and the market prunes", async () => {
+  const m = mk();
+  await m.ingest([
+    [
+      "who wrote romeo and juliet",
+      "William Shakespeare wrote Romeo and Juliet.",
+    ],
+  ]);
+  const { steps } = await trace(m, "who wrote romeo and juliet");
+  await m.store.close();
+
+  const r = regimeStep(steps);
+  assert.equal(r.length, 1, "exactly one regimePrediction step per response");
+  const d = r[0].data;
+  assert.equal(d.version, 1);
+  assert.equal(d.regime, "retrieval");
+  assert.ok(
+    d.incumbentGrade <= d.climbFloorGrade,
+    "incumbent grade at or under the climb floor",
+  );
+  assert.equal(
+    d.climbFloorGrade,
+    2,
+    "2·STEP = 2 (CAST's floor), in grade units",
+  );
+});
+
+test("2. an ungroundable query predicts composition with no incumbent", async () => {
+  const m = mk();
+  await m.ingest([["alpha", "beta"]]);
+  const { steps } = await trace(m, "qzx zzjf vbnm plkj");
+  await m.store.close();
+
+  const r = regimeStep(steps);
+  assert.equal(r.length, 1);
+  assert.equal(r[0].data.regime, "composition");
+  assert.equal(
+    r[0].data.incumbentGrade,
+    null,
+    "nothing grounded — no incumbent",
+  );
+});
+
+test("3. a partially-grounding query predicts composition (one unexplained byte outbids every floor)", async () => {
+  const m = mk();
+  await m.ingest([
+    ["the capital of france", "The capital of France is Paris."],
+  ]);
+  const { steps } = await trace(m, "what is the capital of france");
+  await m.store.close();
+
+  const r = regimeStep(steps);
+  assert.equal(r[0].data.regime, "composition");
+  assert.ok(
+    r[0].data.incumbentGrade > r[0].data.climbFloorGrade,
+    `incumbent grade ${r[0].data.incumbentGrade} must exceed the climb floor ` +
+      `${
+        r[0].data.climbFloorGrade
+      } — PASS prices each unexplained byte at 1000`,
+  );
+});
+
+test("5. a fully-covered multi-move query (grade above the climb floor) still predicts composition", async () => {
+  // Three contiguous trained forms cover the whole query with three STEP moves
+  // and NO unexplained bytes — incumbent grade 3, which the old
+  // `worthRunning(CONCEPT + STEP)` boundary (≤ 11) mislabelled "retrieval"
+  // even though CAST's 2·STEP floor still runs the consensus climb.  The
+  // regime must follow the climb, not the market's maximum floor.
+  const m = mk();
+  await m.ingest([
+    ["abcdefgh", "ABCDEFGH"],
+    ["ijklmnop", "IJKLMNOP"],
+    ["qrstuvwx", "QRSTUVWX"],
+  ]);
+  const { steps } = await trace(m, "abcdefghijklmnopqrstuvwx");
+  await m.store.close();
+
+  const r = regimeStep(steps);
+  assert.equal(r.length, 1);
+  const d = r[0].data;
+  assert.equal(d.regime, "composition", "the climb runs, so it is composition");
+  assert.ok(
+    d.incumbentGrade > d.climbFloorGrade,
+    `incumbent grade ${d.incumbentGrade} must exceed the climb floor ${d.climbFloorGrade}`,
+  );
+  // The prediction's own premise: the climb DID run (CAST first-touched it).
+  const climbed = steps.some((s) => s.mechanism.at(-1) === "climbConsensus");
+  assert.ok(climbed, "the consensus climb must actually run in this regime");
+});
+
+test("4. the prediction is observational — an untraced response is byte-identical", async () => {
+  const mk2 = () =>
+    new Mind({
+      seed: 7,
+      store: new SQliteStore({ path: ":memory:", D: 256 }),
+    });
+  const q = "who wrote romeo and juliet";
+  const corpus = [[
+    "who wrote romeo and juliet",
+    "William Shakespeare wrote Romeo and Juliet.",
+  ]];
+
+  const a = mk2();
+  await a.ingest(corpus);
+  const plain = await a.respondText(q);
+  await a.store.close();
+
+  const b = mk2();
+  await b.ingest(corpus);
+  const traced = await b.respondText(q, () => {});
+  await b.store.close();
+
+  assert.equal(traced, plain, "attaching a trace must not change the answer");
+});

package/test/94-cross-region-budget.test.mjs

@@ -0,0 +1,67 @@
+// 94-cross-region-budget.test.mjs — the cross-region junction ladder shares ONE
+// k·W allowance per evidence tier once atoms are hubs, instead of letting each
+// candidate pair spend its own √N·W drift budget (attention.ts crossRegionVotes,
+// §2.17's derived gate = traverse.atomIsHub).
+//
+// This is a PERFORMANCE regression test, not a behaviour test: the shared
+// budget is byte-identical at every scale — a pair whose container is not
+// reached within the allowance falls through to the resonance tier exactly as a
+// per-pair walk that exhausted its own budget would — so the only observable is
+// the meter's junctionPops counter.  A ~4.3k-fact fixture is NOT optional:
+// atomIsHub is false in every small-store suite, so a conventional fixture
+// would pass while the per-pair drift (measured: 160k junction pops, 31% of
+// think) is fully present.  The atomIsHub assertion below fails loudly if the
+// crossover ever moves, so this suite can never silently stop covering the
+// branch it exists for.
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { Mind } from "../dist/src/index.js";
+import { SQliteStore } from "../dist/src/store-sqlite.js";
+import { atomIsHub, corpusN } from "../dist/src/mind/traverse.js";
+
+test("crossRegion shares one k·W allowance per tier once atoms are hubs", async () => {
+  const m = new Mind({
+    seed: 7,
+    store: new SQliteStore({ path: ":memory:" }),
+    profile: true,
+  });
+
+  // Common windows, each in ≤ √N contexts so the hub guard does NOT abstain —
+  // a non-hub cone is exactly the drift the shared budget must bound.  Three
+  // such windows ("aaaa", "bbbb", "cccc") that never co-occur give the query
+  // below three strong regions and three pairs, every one of whose junction
+  // walks drifts through both windows' cones and finds no container.
+  const per = 60;
+  const corpus = [];
+  for (const w of ["aaaa", "bbbb", "cccc"]) {
+    for (let i = 0; i < per; i++) corpus.push([`${w} ${i}`, `a ${w} ${i}`]);
+  }
+  // Filler edge sources push N past atomIsHub (N > ~4096 at maxGroup=4).
+  for (let i = 0; i < 4300; i++) corpus.push([`filler-${i}`, `f${i}`]);
+  await m.ingest(corpus);
+
+  const N = corpusN(m);
+  assert.ok(
+    atomIsHub(m, N),
+    `fixture must cross atomIsHub (N=${N}); the shared budget is scale-gated ` +
+      `and this suite would otherwise assert nothing`,
+  );
+
+  await m.respondText("aaaa bbbb cccc");
+
+  const c = m.lastCost;
+  const crossPops = c.phases["climb.crossRegion"]?.counters.junctionPops ?? 0;
+  const k = m.cfg.recallQueryK * 2; // pre.k — the ladder's pair budget breadth
+  const W = m.space.maxGroup;
+
+  assert.ok(crossPops > 0, "the query must actually run cross-region walks");
+  assert.ok(
+    crossPops <= 2 * k * W,
+    `crossRegion junction pops ${crossPops} must stay within the shared ` +
+      `2·k·W = ${2 * k * W} allowance (exact + synonym tiers), not the ` +
+      `per-pair √N·W drift`,
+  );
+
+  await m.store.close();
+});

package/test/95-wide-resonance-removed.test.mjs

@@ -0,0 +1,109 @@
+// 95-wide-resonance-removed.test.mjs — the substitution bridge and prefix
+// completion must propose from BOUNDED sources, never from the exhaustive-√N
+// ANN scan that `Precomputed.wideResonance()` used to run (removed; see
+// pipeline-mechanism.ts's REMOVED note).
+//
+// This is a PERFORMANCE regression test, not a behaviour test: the wide list
+// was a PROPOSAL source whose consumers byte-verify every candidate (§2.3), so
+// removing it is byte-identical wherever the bounded sources supply the same
+// candidate set — and the meter's phase map is the only observable that says
+// whether the exhaustive machinery still exists.  Red-on-revert: re-adding
+// wideResonance re-creates the `wideResonance` phase (and, when the query's
+// top hit clears conceptThreshold, a full-index ANN scan inside it), so the
+// phase-absence assertion below fails.
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { Mind } from "../dist/src/index.js";
+import { SQliteStore } from "../dist/src/store-sqlite.js";
+
+const dec = (b) => new TextDecoder().decode(b).replace(/\0+$/, "");
+
+// The test/49 resonance-proposed corpus: the query "what is the capital of
+// france" resonates STRAIGHT to the trained "What is the capital of France?"
+// (nearest whole-query hit, so its top hit clears conceptThreshold — the exact
+// condition that made the old wideResonance go exhaustive) yet falls below the
+// reach bar, so recall refuses and the bridge runs to recover the fact.
+const CORPUS = [
+  ["What is the capital of France?", "The capital of France is Paris."],
+  ["What is the capital of Spain?", "The capital of Spain is Madrid."],
+  ["What is the capital of Italy?", "The capital of Italy is Rome."],
+  // Lowercase mid-sentence occurrences attest the case-folded windows the
+  // substitution's corroboration gate requires.
+  ["He wrote of france and of spain.", "Then he flew home to italy."],
+  ["She spoke of france in her diary.", "Her diary told of france."],
+];
+
+test("the bridge's refusal path never descends the exhaustive-√N ANN scan", async () => {
+  const m = new Mind({
+    seed: 7,
+    store: new SQliteStore({ path: ":memory:" }),
+    profile: true,
+  });
+  await m.ingest(CORPUS);
+
+  const r = await m.respond("what is the capital of france");
+  assert.ok(
+    dec(r.bytes).includes("Paris"),
+    `the bridge must still ground the fact through its bounded proposals, got ${
+      JSON.stringify(dec(r.bytes))
+    }`,
+  );
+
+  const c = m.lastCost;
+  // The fixture actually ran the bridge — without this, a silently-skipped
+  // fixture would pass the phase-absence assertion while asserting nothing.
+  assert.ok(
+    c.phases["substitutionBridge"],
+    "the fixture must run the substitution bridge",
+  );
+  // The exhaustive-√N phase is gone.  Its removal is the whole point: the
+  // bridge's proposal source is the response's ONE top-k read (memoized), and
+  // every proposal is byte-verified downstream, so the full-index scan bought
+  // recall at O(index) cost for an O(k) need.
+  assert.equal(
+    c.phases["wideResonance"],
+    undefined,
+    "wideResonance (the exhaustive √N scan) must be removed from the response",
+  );
+  // The bridge phase itself accrues no content-index ANN reads: proposals come
+  // from the memoized `resonance()`, never a fresh exhaustive descend.
+  const bridgeAnn = c.phases["substitutionBridge"].counters.annVectorReads ?? 0;
+  assert.equal(
+    bridgeAnn,
+    0,
+    "the substitution bridge must not descend the content ANN on its own",
+  );
+
+  await m.store.close();
+});
+
+test("prefix completion proposes from the write-side window index, not the ANN", async () => {
+  const m = new Mind({
+    seed: 7,
+    store: new SQliteStore({ path: ":memory:" }),
+    profile: true,
+  });
+  await m.ingest(CORPUS);
+
+  // A strict byte-prefix of the trained answer form.  Its gist cannot rank its
+  // own continuation (cos falls below reachThreshold), so the content-addressed
+  // window walk is the correct proposal source — and the one prefix-completion
+  // now tries FIRST.
+  const r = await m.respond("The capital of France is");
+  assert.ok(
+    dec(r.bytes).includes("Paris"),
+    `expected the trained form through the content-addressed prefix supply, got ${
+      JSON.stringify(dec(r.bytes))
+    }`,
+  );
+
+  const c = m.lastCost;
+  assert.equal(
+    c.phases["wideResonance"],
+    undefined,
+    "wideResonance (the exhaustive √N scan) must be removed from the response",
+  );
+
+  await m.store.close();
+});