@hviana/sema 0.2.3 → 0.2.5

package/test/48-recognise-turn-connective.test.mjs

@@ -0,0 +1,125 @@
+// 48-recognise-turn-connective.test.mjs — recognise() must find a trained
+// form even when the asker prepends a real discourse connective ("And ")
+// to a follow-up turn — not boundary noise, a genuine extra word.
+//
+// Traced live (analyze_training.ts, dialogue D geography thread):
+// "What is the capital of France?" then "And what is the capital of
+// Spain?" answered with an unrelated CAST analog (Japan/Moon) instead of
+// "Madrid is the capital of Spain." — root-caused directly against the
+// live store: the turn "And what is the capital of Spain?" canon-misses as
+// ONE 33-byte node (turn/segment scale); canonResolve on the WHOLE turn
+// fails because "and " is real content, not an equivalence the injected
+// canonicalizer folds away.  The shipped ±1/k-trim fallbacks are bounded to
+// chunk-scale (<= W²) specifically to avoid rediscovering a smaller
+// subtree's own content as a duplicate, overlapping site at root scale
+// (see test/46's regression case) — but that bound also blocked recovering
+// this turn, which is naturally larger than one chunk.
+//
+// Fix: a canon-miss composite ALSO tries canonResolve from every position
+// UP TO W chunk-widths from its own left edge that the query's OWN
+// structural fold already treats as a chunk boundary (`starts` — the same
+// set the canonical pass privileges with full chain reach).  This is NOT a
+// blind offset guess (the size-bounded k-trim loop above it already covers
+// that): `starts.has(p)` is fold EVIDENCE the query itself produced —
+// "And " is a genuine W=4-byte leaf-parent chunk, so its own end (byte 65)
+// is already a `starts` boundary before this code ever runs (foldTree
+// visits children before parents).  Unbounded in the NODE's size (turn/
+// segment scale is fine), bounded in candidate COUNT (at most W probes,
+// each an O(1) starts.has() before paying for the real canonResolve fold).
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { Mind, SQliteStore } from "../dist/src/index.js";
+import { textCanon } from "../dist/src/canon.js";
+import { recognise } from "../dist/src/mind/recognition.js";
+import { latin1Key, perceive, resolve } from "../dist/src/mind/primitives.js";
+
+const enc = (s) => new TextEncoder().encode(s);
+const dec = (b) => new TextDecoder().decode(b).replace(/\0+$/, "");
+
+test("recognise(): a real leading connective word ('And ') still finds the trained follow-up form", async () => {
+  const m = new Mind({ seed: 7, store: new SQliteStore({ path: ":memory:" }) });
+  await m.ingest([
+    ["What is the capital of France?", "The capital of France is Paris."],
+    ["What is the capital of Spain?", "Madrid is the capital of Spain."],
+  ]);
+  await m.buildCanonIndex(textCanon);
+  m.canon = m._canonFor(textCanon);
+  m.canonMemo = new Map();
+
+  const query = enc(
+    "What is the capital of France?The capital of France is Paris." +
+      "And what is the capital of Spain?",
+  );
+  const expected = resolve(m, enc("What is the capital of Spain?"));
+  assert.ok(
+    expected !== null,
+    "sanity: the trained fact must resolve standalone",
+  );
+
+  // The live shape is STABLE-PREFIX folded (each turn folds independently
+  // from its own local offset 0 — see mind.ts's _growContext) so "And "
+  // lands on a genuine chunk boundary; a plain fold of the whole query (no
+  // boundaries) radix-aligns from byte 0 instead, which happens to reduce
+  // this particular query to the already-shipped ±1 case and would not
+  // exercise this fix at all.  Priming perceiveMemo with the boundary-aware
+  // tree (content-keyed, exactly what _growContext itself does) makes this
+  // recognise() call see what the real multi-turn pipeline sees, without
+  // going through respondTurn's own memo (a second recognise() call on the
+  // same bytes is deliberately non-idempotent — see recognise()'s own
+  // doc comment — so this must be the FIRST call on this content).
+  const boundaries = [30, 61];
+  m.perceiveMemo = new Map();
+  m.perceiveMemo.set(
+    latin1Key(query),
+    perceive(m, query, undefined, undefined, boundaries),
+  );
+
+  const rec = recognise(m, query);
+  const hit = rec.sites.find((s) => s.payload === expected);
+  assert.ok(
+    hit,
+    `expected a recognised site for the trained Spain fact despite the ` +
+      `leading "And " connective, got sites: ` +
+      JSON.stringify(rec.sites.map((s) => [s.start, s.end, s.payload])),
+  );
+  await m.store.close();
+});
+
+test("multi-turn: a follow-up turn prefixed with a real connective word grounds the NEW fact, not a CAST analog", async () => {
+  const m = new Mind({ seed: 7, store: new SQliteStore({ path: ":memory:" }) });
+  await m.ingest([
+    ["What is the capital of France?", "The capital of France is Paris."],
+    ["What is the capital of Spain?", "Madrid is the capital of Spain."],
+    [
+      "some other prompt",
+      "What is the capital of Japan? Tokyo is the capital of Japan.",
+    ],
+  ]);
+  // The live store builds this at training time; respondTurn's own
+  // automatic canon wiring (_canonFor) only wraps the canonicalizer
+  // function — the INDEX canonResolve searches is a separate, opt-in
+  // build step, same as test 44/46.
+  await m.buildCanonIndex(textCanon);
+
+  const conv = m.beginConversation();
+  await m.respondTurn(conv, "What is the capital of France?");
+  const r2 = await m.respondTurn(conv, "And what is the capital of Spain?");
+  const got = dec(r2.response.bytes);
+
+  assert.ok(
+    got.includes("Madrid") && got.includes("Spain"),
+    `expected the NEW Spain fact to ground the answer, got ${
+      JSON.stringify(got)
+    } (provenance: ${r2.response.provenance})`,
+  );
+  assert.ok(
+    !got.includes("Japan") && !got.includes("Tokyo"),
+    `must not fall back to an unrelated CAST analog, got ${
+      JSON.stringify(got)
+    }`,
+  );
+
+  m.endConversation(conv);
+  await m.store.close();
+});

package/test/49-natural-units-synonym-bridge.test.mjs

@@ -0,0 +1,175 @@
+// 49-natural-units-synonym-bridge.test.mjs — irrefutable demonstration of
+// the architectural gap investigated this session: a query using a
+// near-synonym ("biggest") of a word that was only ever trained in a
+// DIFFERENT phrasing ("largest") finds nothing, even though the fact
+// itself is trained and the synonym relationship is corroborated many
+// times over in the corpus.
+//
+// Root-caused directly (prior turn): "biggest"/"largest" NEVER resolve as
+// independent nodes at all — deposit() only interns whole sentences (as
+// one flat branch) plus tiny W-1/W leaf-id windows (canonical.ts); a
+// 7-byte word floating mid-sentence falls in the gap between those two
+// scales and is never independently addressable, so the EXISTING
+// halo/synonym machinery (conceptHop, already used by CAST) never gets an
+// entry point to it — the halo system is real and works, it is simply
+// never asked about a node that was never minted.
+//
+// Fix (units.ts): a derived, corpus-statistics-only notion of a "natural
+// unit" — a run of adjacent chunks whose pairing recurs at least as often
+// as either chunk alone (the same principle behind BPE/content-defined
+// chunking, computed from this store's own reuse/containment counts, no
+// injected modality-specific segmenter).  Interned at deposit time,
+// pairwise halo-poured as each other's company; read at query time via the
+// same derived merge (existingUnits) plus a halo-corroborated substitution
+// fallback in recognise().
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { Mind } from "../dist/src/index.js";
+import { SQliteStore } from "../dist/src/store-sqlite.js";
+import { resolve } from "../dist/src/mind/primitives.js";
+
+const enc = (s) => new TextEncoder().encode(s);
+const dec = (b) => new TextDecoder().decode(b).replace(/\0+$/, "");
+
+// Parallel "X is the biggest/largest Y" constructions across several
+// different nouns — real cross-sentence corroboration for the
+// biggest~largest collocation pattern, not a single coincidental pairing.
+const TRAIN = [
+  [
+    "What is the largest planet in our solar system?",
+    "The largest planet is Jupiter.",
+  ],
+  ["What is the biggest ocean on Earth?", "The biggest ocean is the Pacific."],
+  ["What is the largest country by area?", "The largest country is Russia."],
+  [
+    "What is the biggest mammal on Earth?",
+    "The biggest mammal is the blue whale.",
+  ],
+  [
+    "What is the largest desert in the world?",
+    "The largest desert is the Sahara.",
+  ],
+  ["What is the biggest city in Japan?", "The biggest city is Tokyo."],
+  ["What is the largest lake in Africa?", "The largest lake is Lake Victoria."],
+  ["What is the biggest island on Earth?", "The biggest island is Greenland."],
+];
+
+test("baseline: 'biggest'/'largest' never resolve as independent nodes without natural units", async () => {
+  const m = new Mind({ seed: 7, store: new SQliteStore({ path: ":memory:" }) });
+  await m.ingest(TRAIN);
+  // This assertion documents the ROOT CAUSE (still true even after the
+  // fix — units.ts does not change what resolve() itself returns for a
+  // bare word; it changes what recognise() can bridge to via halos).
+  assert.equal(resolve(m, enc("biggest")), null);
+  assert.equal(resolve(m, enc("largest")), null);
+  await m.store.close();
+});
+
+test("irrefutable failure: a trained fact is unreachable through an untrained near-synonym", async () => {
+  const m = new Mind({ seed: 7, store: new SQliteStore({ path: ":memory:" }) });
+  await m.ingest(TRAIN);
+
+  // Sanity: the TRAINED phrasing works.
+  const trained = await m.respond(
+    "What is the largest planet in our solar system?",
+  );
+  assert.ok(
+    dec(trained.bytes).includes("Jupiter"),
+    `sanity: trained phrasing must answer, got ${
+      JSON.stringify(dec(trained.bytes))
+    }`,
+  );
+
+  // The untrained near-synonym phrasing.  NOTE: a query that keeps the
+  // trained sentence's own long, near-identical scaffolding around the one
+  // swapped word ("What is the biggest planet in our solar system?")
+  // already succeeds WITHOUT this fix — recallByResonance's WHOLE-QUERY
+  // gist is similar enough (one word out of a long, highly-repetitive
+  // sentence) to clear the reach threshold on its own, an existing
+  // capability this test must not accidentally exercise instead.  Deliberately
+  // SHORT and restructured ("Name the biggest planet.") so the whole-query
+  // gist has no such shortcut — confirmed directly (see this session's
+  // investigation) to return silence at baseline: recognise() finds 0
+  // sites, and the whole-query resonance score is nowhere near the reach
+  // bar.  Only a WORD-level bridge from "biggest" to its corroborated
+  // synonym "largest" can recover the trained fact here.
+  const untrained = await m.respond("Name the biggest planet.");
+  assert.ok(
+    dec(untrained.bytes).includes("Jupiter"),
+    `expected the trained fact to be reachable through the corroborated ` +
+      `synonym "biggest" ~ "largest", got ${
+        JSON.stringify(dec(untrained.bytes))
+      }`,
+  );
+  await m.store.close();
+});
+
+test("resonance-proposed bridge: casing/punctuation paraphrase reaches the trained fact", async () => {
+  // Root-caused live on the trained store (2026-07-20): the query
+  // "what is the capital of france" resonates STRAIGHT to the trained
+  // "What is the capital of France?" (nearest whole-query hit) yet stayed
+  // silent — the case-changed bytes fall below the reach bar, the canon
+  // twin misses on the absent "?", and the bridge's own W-window climb
+  // cannot single the right context out of hundreds sharing common
+  // windows.  Recall now hands its resonance ranking to the bridge as
+  // PROPOSED candidates (rank-only, byte-verified there), which recovers
+  // the fact through two corroborated one-byte case substitutions.
+  const m = new Mind({ seed: 7, store: new SQliteStore({ path: ":memory:" }) });
+  await m.ingest([
+    ["What is the capital of France?", "The capital of France is Paris."],
+    ["What is the capital of Spain?", "The capital of Spain is Madrid."],
+    ["What is the capital of Italy?", "The capital of Italy is Rome."],
+    // Lowercase mid-sentence occurrences attest the case-folded windows
+    // the substitution's corroboration gate requires — including the
+    // frame-bearing " of france" the expansion absorbs into.
+    ["He wrote of france and of spain.", "Then he flew home to italy."],
+    ["She spoke of france in her diary.", "Her diary told of france."],
+  ]);
+  const r = await m.respond("what is the capital of france");
+  assert.ok(
+    dec(r.bytes).includes("Paris"),
+    `expected the trained fact through the resonance-proposed bridge, got ${
+      JSON.stringify(dec(r.bytes))
+    }`,
+  );
+  await m.store.close();
+});
+
+test(
+  "FIXED 2026-07-20: a proper-noun substitution must not voice a wrong " +
+    "fact when the true answer has no outgoing edge to compete as a " +
+    "bridge candidate (RAW BALANCE gate, see bridge.ts)",
+  async () => {
+    // This miniature corpus does NOT reproduce the live bug (recall's
+    // whole-query resonance already answers correctly here — verified: it
+    // takes the real 17.9M-node store's specific candidate ranking to
+    // surface the failure mode).  Kept anyway as a standing regression
+    // check on the shape of the bug (terminal fact vs. a same-shaped fact
+    // WITH a continuation); the authoritative fix verification was run
+    // directly against the trained store: "The capital of France is"
+    // used to bridge through a substitution reading "of Fra[nce]" as
+    // "of Spain si[nce]" (raw mismatch (3,8) bytes, badly imbalanced);
+    // the RAW BALANCE gate (dominates(min(uLen,cLen), max(uLen,cLen)))
+    // now refuses it, and recall correctly falls through to an honest
+    // echo of the true trained fact instead.
+    const m = new Mind({
+      seed: 7,
+      store: new SQliteStore({ path: ":memory:" }),
+    });
+    await m.ingest([
+      ["What is the capital of France?", "The capital of France is Paris."],
+      [
+        "Madrid has been the capital of Spain since 1561.",
+        "It was established as such by Philip III.",
+      ],
+    ]);
+    const r = await m.respond("The capital of France is");
+    assert.ok(
+      !dec(r.bytes).includes("Spain"),
+      `expected no wrong-entity substitution, got ${
+        JSON.stringify(dec(r.bytes))
+      }`,
+    );
+  },
+);

package/test/50-cast-analog-consensus-floor.test.mjs

@@ -0,0 +1,179 @@
+// 50-cast-analog-consensus-floor.test.mjs — CAST's analogical comparison
+// must not voice content the query never named through a climb root whose
+// consensus vote is below consensusFloor(N) = ln(N) + 1/2 — the SAME trust
+// bar recallByResonance applies before grounding through a climb root.
+//
+// The live bug (traced on the real trained store, 325k edge sources, floor
+// 13.2): "Tell me the name of the biggest planet orbiting our sun." shares
+// only stopword scraps ("the", "our", generic bigrams) with an unrelated
+// haiku exemplar; those scraps pooled a climb vote of just 1.92, yet the
+// climb still committed the exemplar as FIRST root (the first root is
+// deliberately floor-free in attention.ts — "the dominant one always
+// grounds").  CAST's comparison schema then cited an analog reached through
+// a continuation hop (never named by the query), and voiced the haiku's
+// continuation ("Winds of change blow free…") as the answer — a wrong
+// answer where recall, extraction, and the ALU all honestly refused.
+//
+// The gate (cast.ts): comparison may cite a hop-reached (unnamed) analog
+// only when some committed root's vote clears consensusFloor(N); a
+// DIRECTLY aligned analog (the query's own bytes evidence it, e.g. test/29
+// C1's "Steel is hard") needs no floor.  Derived, never tuned — the floor
+// is the one recall.ts already gates the same trust decision with.
+//
+// This miniature reproduces the exact shape: ~200 filler pairs whose LONG
+// contexts (longer than the probe query) can never be direct analogs, so
+// their short continuations arrive only as continuation-hop descendants;
+// the exemplar's distinctive words also occur across fillers, diluting its
+// root vote below the floor (measured: 4.24 and 5.51 against floor 5.81)
+// while it still ranks first and commits floor-free.  Confirmed red at
+// baseline: both probes echoed the haiku continuation with provenance
+// "cast"; with the gate both are honest silence.
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { Mind } from "../dist/src/index.js";
+import { SQliteStore } from "../dist/src/store-sqlite.js";
+
+const dec = (b) => new TextDecoder().decode(b).replace(/\0+$/, "");
+
+const subjects = [
+  "cat",
+  "dog",
+  "bird",
+  "fish",
+  "horse",
+  "sheep",
+  "goat",
+  "duck",
+  "frog",
+  "deer",
+  "wolf",
+  "bear",
+  "lion",
+  "tiger",
+  "mouse",
+  "rabbit",
+  "otter",
+  "eagle",
+  "shark",
+  "whale",
+];
+const verbs = [
+  "describes",
+  "praises",
+  "watches",
+  "follows",
+  "studies",
+  "admires",
+  "draws",
+  "paints",
+  "names",
+  "sings",
+];
+const objects = [
+  "freedom",
+  "feeling",
+  "titles",
+  "songs",
+  "times",
+  "stories",
+  "poems",
+  "rivers",
+  "shadows",
+  "seasons",
+];
+
+const TRAIN = [];
+for (const s of subjects) {
+  for (const v of verbs) {
+    const o =
+      objects[(subjects.indexOf(s) + verbs.indexOf(v)) % objects.length];
+    TRAIN.push([
+      `The ${s} quietly ${v} the many ${o} it meets today in the wide green garden.`,
+      `Indeed the ${s} ${v} ${o}.`,
+    ]);
+  }
+}
+TRAIN.push([
+  "Create a haiku that describes the feeling of freedom.",
+  "Winds of change blow free\nFeeling of lightness fills my soul",
+]);
+
+// SHORT filler contexts: continuations become directly ALIGNED points, so
+// the junk analog arrives with frame-tier similarity as an aligned point —
+// the configuration the consensusFloor/naming gates alone cannot separate
+// from test/29 C1's legitimate small-corpus comparison.  The separator is
+// the IGNORED-KNOWN principle (dismissedKnownContent): under an untrusted
+// root, a frame-tier comparison must account for every STORED window of
+// the query — here "songs"/"times"/"planet"-class trained content is left
+// in gaps, so comparison refuses; C1's gaps ("How ", " like ") are
+// untrained and stay tolerable.  Confirmed red at baseline: both probes
+// echoed the haiku + a filler continuation with provenance "cast".
+const SHORT_TRAIN = [];
+for (const s of subjects) {
+  for (const v of verbs) {
+    const o =
+      objects[(subjects.indexOf(s) + verbs.indexOf(v)) % objects.length];
+    SHORT_TRAIN.push([
+      `The ${s} ${v} the ${o} today.`,
+      `Indeed the ${s} ${v} ${o}.`,
+    ]);
+  }
+}
+SHORT_TRAIN.push([
+  "Create a haiku that describes the feeling of freedom.",
+  "Winds of change blow free\nFeeling of lightness fills my soul",
+]);
+
+test("comparison refuses an aligned frame-tier analog that dismisses stored query content", async () => {
+  const m = new Mind({ seed: 7, store: new SQliteStore({ path: ":memory:" }) });
+  await m.ingest(SHORT_TRAIN);
+  for (
+    const q of [
+      "Give me the title of the finest song describing our times.",
+      "Tell me the name of the biggest planet orbiting our sun.",
+    ]
+  ) {
+    const r = await m.respond(q);
+    const t = dec(r.bytes);
+    assert.equal(
+      t,
+      "",
+      `expected honest silence for ${JSON.stringify(q)}, got ${
+        JSON.stringify(t)
+      }`,
+    );
+  }
+  await m.store.close();
+});
+
+test("comparison refuses a hop-reached analog under a below-floor root", async () => {
+  const m = new Mind({ seed: 7, store: new SQliteStore({ path: ":memory:" }) });
+  await m.ingest(TRAIN);
+
+  // Neither query names any trained fact — the corpus knows nothing about
+  // songs, titles, or planets.  The only honest answer is silence; at
+  // baseline both echoed the haiku exemplar's continuation via CAST.
+  for (
+    const q of [
+      "Give me the title of the finest song describing our times.",
+      "Tell me the name of the biggest planet orbiting our sun.",
+    ]
+  ) {
+    const r = await m.respond(q);
+    const t = dec(r.bytes);
+    assert.ok(
+      !t.includes("Winds of change"),
+      `CAST voiced an unrelated exemplar's continuation off a below-floor ` +
+        `root for ${JSON.stringify(q)}, got ${JSON.stringify(t)}`,
+    );
+    assert.equal(
+      t,
+      "",
+      `expected honest silence for ${JSON.stringify(q)}, got ${
+        JSON.stringify(t)
+      }`,
+    );
+  }
+  await m.store.close();
+});