@hviana/sema 0.1.7 → 0.1.8

package/src/mind/primitives.ts

@@ -14,6 +14,8 @@ import {
   hilbertBytes,
   stackGrids,
 } from "../geometry.js";
+import { canonHash } from "../canon.js";
+import { bytesEqual } from "../bytes.js";
 import { ALL } from "./types.js";
 import type { Input, MindContext } from "./types.js";
 
@@ -199,7 +201,70 @@ export function foldTree(
  *  unknown. */
 export function resolve(ctx: MindContext, bytes: Uint8Array): number | null {
   if (bytes.length === 0) return null;
-  return foldTree(ctx, perceive(ctx, bytes), 0).node;
+  const exact = foldTree(ctx, perceive(ctx, bytes), 0).node;
+  if (exact !== null) return exact;
+  return canonResolve(ctx, bytes);
+}
+
+/** Equivalence-class resolution: when the exact content-addressed lookup
+ *  misses, find a stored node whose CANONICAL key equals the span's — the
+ *  store's canon index proposes candidates by key hash, and each is verified
+ *  by re-canonicalizing its bytes (hash-then-verify, like every content
+ *  lookup).  Among verified candidates, one that leads somewhere (has a
+ *  continuation edge) is preferred; ties break to the lowest id — a corpus
+ *  property, not a seed property.  Null when the response carries no
+ *  canonicalizer, the store has no canon index, or nothing verifies. */
+export function canonResolve(
+  ctx: MindContext,
+  bytes: Uint8Array,
+): number | null {
+  const canon = ctx.canon;
+  const store = ctx.store;
+  if (canon === null || !store.canonFind) return null;
+  if (bytes.length < 2) return null;
+  const memo = ctx.canonMemo;
+  const memoKey = memo ? latin1Key(bytes) : "";
+  if (memo) {
+    const hit = memo.get(memoKey);
+    if (hit !== undefined) return hit;
+  }
+  const set = (v: number | null): number | null => {
+    memo?.set(memoKey, v);
+    return v;
+  };
+  const key = canon(bytes);
+  if (key.length === 0) return set(null);
+  // A stored form that IS canonical is not in the index (buildCanonIndex
+  // skips identity rows) — the exact content-addressed lookup of the
+  // canonical bytes finds it directly.
+  if (key.length !== bytes.length || !bytesEqual(key, bytes)) {
+    const direct = foldTree(ctx, perceive(ctx, key), 0).node;
+    if (direct !== null) return set(direct);
+  }
+  const candidates = store.canonFind(canonHash(key));
+  if (candidates.length === 0) return set(null);
+  let best: number | null = null;
+  let bestLeads = false;
+  for (const id of candidates) {
+    const bytesOf = read(ctx, id);
+    const stored = canon(bytesOf);
+    if (stored.length !== key.length || !bytesEqual(stored, key)) continue;
+    // The index stores FLAT content twins; the id the exact path would have
+    // resolved for these bytes is their FOLD — the deposit-shaped node that
+    // carries the edges and halos.  Re-folding the candidate's bytes lands
+    // on exactly the node the canonical-case query would have found.
+    const folded = foldTree(ctx, perceive(ctx, bytesOf), 0).node;
+    const use = folded ?? id;
+    const leads = store.hasNext(use) || store.haloMass(use) > 0;
+    if (
+      best === null || (leads && !bestLeads) ||
+      (leads === bestLeads && use < best)
+    ) {
+      best = use;
+      bestLeads = leads;
+    }
+  }
+  return set(best);
 }
 
 /** Walk a perceived tree in POST-ORDER with byte offsets — children before

package/src/mind/recognition.ts

@@ -8,6 +8,7 @@ import { rItem } from "./trace.js";
 
 import type { MindContext, Recognition, Segment } from "./types.js";
 import {
+  canonResolve,
   foldTree,
   gistOf,
   latin1Key,
@@ -96,6 +97,15 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
       leaves.push({ start, end, bytes: n.leaf ?? new Uint8Array(0), node });
     }
     if (node !== null) emit(start, end, node);
+    // Canonical fallback: a subtree whose exact content-addressed lookup
+    // missed may still be a stored form under the response's equivalence
+    // (case, width, whitespace — whatever the injected canonicalizer says).
+    // O(subtree bytes) per miss, memoised per response; a no-op when no
+    // canonicalizer was injected or the store has no canon index.
+    else if (end - start >= 2) {
+      const cid = canonResolve(ctx, bytes.subarray(start, end));
+      if (cid !== null) emit(start, end, cid);
+    }
     if (isChunk(n)) {
       starts.add(start);
       // Try every sub-span within this leaf-parent.

package/src/mind/types.ts

@@ -144,6 +144,16 @@ export interface MindContext extends GraphSearchHost {
   cfg: MindConfig;
   search: GraphSearch;
   trace: Rationale | null;
+  /** The content canonicalizer for THIS response, or null — injected by the
+   *  modality entry point (respondText passes the text canonicalizer; a
+   *  binary respond passes none).  Resolution uses it as a fallback: when
+   *  the exact content-addressed lookup misses, the span's canonical key is
+   *  probed against the store's canon index (see src/canon.ts).  The core
+   *  never inspects what the equivalence IS. */
+  canon: ((bytes: Uint8Array) => Uint8Array) | null;
+  /** Per-response memo of canonical-fallback resolutions, keyed by the
+   *  span's latin1 content key.  Null outside respond(). */
+  canonMemo: Map<string, number | null> | null;
   /** Memo of the consensus climb — content-keyed (latin1) so results
    *  persist across conversation turns where the same byte spans recur.
    *  Null outside respond(); during respondTurn() the conversation's

package/src/store-sqlite.ts

@@ -134,6 +134,18 @@ CREATE TABLE IF NOT EXISTS contain (
   id      INTEGER PRIMARY KEY,
   parents BLOB NOT NULL
 );
+-- CANONICAL-FORM index (Store.canonAdd/canonFind): h is the 32-bit hash of a
+-- node's CANONICAL content key (the modality's canonicalizer output — case-
+-- folded, whitespace-collapsed text, etc.; the store never sees the key
+-- itself, only its hash).  Same hash-then-verify discipline as idx_node_h:
+-- the caller re-canonicalizes each candidate's bytes before trusting it, so
+-- a collision costs a read, never a wrong id.  WITHOUT ROWID on (h, id):
+-- the composite key IS the lookup index and gives free dedup.
+CREATE TABLE IF NOT EXISTS canon (
+  h  INTEGER NOT NULL,
+  id INTEGER NOT NULL,
+  PRIMARY KEY (h, id)
+) WITHOUT ROWID;
 CREATE TABLE IF NOT EXISTS snapshot (
   id INTEGER PRIMARY KEY CHECK (id = 1),
   data BLOB NOT NULL
@@ -241,6 +253,10 @@ export class SQliteStore extends AbstractStore implements Store {
   private _selPrev: any = null;
   private _setMeta: any = null;
   private _getMeta: any = null;
+  private _insCanon: any = null;
+  private _selCanon: any = null;
+  private _cntCanon: any = null;
+  private _selContentFrom: any = null;
   private _delMeta: any = null;
   private _insSnapshot: any = null;
   private _selSnapshot: any = null;
@@ -990,6 +1006,58 @@ export class SQliteStore extends AbstractStore implements Store {
     this._delMeta.run(key);
   }
 
+  // -- Canonical-form index (Store optional capability) --
+
+  canonAdd(h: number, id: number): void {
+    if (!this._insCanon) {
+      this._insCanon = this.sqlite!.prepare(
+        "INSERT OR IGNORE INTO canon (h, id) VALUES (?, ?)",
+      );
+    }
+    // Join the deferred write transaction (committed by flush/commit), so a
+    // bulk index build coalesces instead of paying autocommit per row.
+    this._dbBeginTx();
+    this._insCanon.run(h, id);
+  }
+
+  canonFind(h: number): number[] {
+    if (!this._selCanon) {
+      this._selCanon = this.sqlite!.prepare(
+        "SELECT id FROM canon WHERE h = ?",
+      );
+    }
+    return (this._selCanon.all(h) as Array<{ id: number }>).map((r) => r.id);
+  }
+
+  canonCount(): number {
+    if (!this._cntCanon) {
+      this._cntCanon = this.sqlite!.prepare(
+        "SELECT count(*) AS c FROM canon",
+      );
+    }
+    return (this._cntCanon.get() as { c: number }).c;
+  }
+
+  eachContent(
+    cb: (id: number, bytes: Uint8Array) => void,
+    fromId = 0,
+  ): void {
+    if (!this._selContentFrom) {
+      // Content-bearing nodes are FLAT branches: leaf present, kids an
+      // empty (zero-length) blob — the same population PART 2 of the
+      // diagnostics calls "distinct content spans".
+      this._selContentFrom = this.sqlite!.prepare(
+        "SELECT id, leaf FROM node " +
+          "WHERE id >= ? AND leaf IS NOT NULL " +
+          "AND (kids IS NULL OR length(kids) = 0)",
+      );
+    }
+    for (const row of this._selContentFrom.iterate(fromId)) {
+      const r = row as { id: number; leaf: Uint8Array };
+      cb(r.id, new Uint8Array(r.leaf));
+    }
+  }
+
   // -- Snapshot --
 
   protected _dbSaveSnapshot(bytes: Uint8Array): void {

package/src/store.ts

@@ -417,6 +417,33 @@ export interface Store {
    *  the node has no halo row. */
   haloMass(id: NodeId): number;
 
+  // ── canonical-form index (optional capability) ─────────────────────────
+  // A small hash index from CANONICAL content keys to node ids, enabling
+  // equivalence-class resolution ("WHAT" finds the stored "What") without
+  // the store knowing what the equivalence IS: the canonicalizer lives with
+  // the modality (see src/canon.ts) and is applied by the CALLER — the store
+  // only maps 32-bit key hashes to candidate ids, and the caller verifies
+  // canon(stored bytes) === key before trusting any candidate (the same
+  // hash-then-verify discipline as the node table's own `h` index).
+  // Backends that do not implement the capability leave all three absent;
+  // resolution then simply has no canonical fallback.
+
+  /** Record that node `id`'s canonical key hashes to `h`.  Idempotent. */
+  canonAdd?(h: number, id: NodeId): void;
+  /** All candidate node ids whose canonical key hashes to `h` (collisions
+   *  included — the caller verifies). */
+  canonFind?(h: number): NodeId[];
+  /** Number of (h, id) rows in the canon index — 0 means never built. */
+  canonCount?(): number;
+  /** Visit every content-bearing node (flat branch: `leaf` present, no
+   *  kids) — the population a canonical index is built over.  `fromId`
+   *  restricts the scan to ids ≥ fromId, so an index refresh after further
+   *  training only visits the new rows. */
+  eachContent?(
+    cb: (id: NodeId, bytes: Uint8Array) => void,
+    fromId?: NodeId,
+  ): void;
+
   // ── lifecycle ──────────────────────────────────────────────────────────
   size(): Promise<number>;
   saveSnapshot(bytes: Uint8Array): Promise<void>;

package/test/13-conversation.test.mjs

@@ -45,15 +45,17 @@ async function teachConversation(mind, turns) {
 // ═══════════════════════════════════════════════════════════════════════
 // predictNext — given the turns spoken so far, predict the next turn.
 //
-// Uses the Conversation API: beginConversation → respondTurn for each
-// prior turn → respondTurnText for the query → endConversation.
+// Uses the Conversation API: beginConversation → addTurn for each prior
+// turn (BOTH speakers' lines are history being replayed, not questions to
+// answer — respondTurn would answer each one AND append its own reply to
+// the context) → respondTurnText for the final turn → endConversation.
 // Turns are raw strings, concatenated by the Mind — no separator.
 // ═══════════════════════════════════════════════════════════════════════
 async function predictNext(mind, priorTurns) {
   if (priorTurns.length === 0) return "";
   const conv = mind.beginConversation();
   for (let i = 0; i < priorTurns.length - 1; i++) {
-    await mind.respondTurn(conv, priorTurns[i]);
+    mind.addTurn(conv, priorTurns[i]);
   }
   const { response } = await mind.respondTurnText(
     conv,
@@ -244,8 +246,9 @@ test("E1: begin → respondTurn → end completes a single-turn conversation", a
   const conv = mind.beginConversation();
   const { response, state } = await mind.respondTurnText(conv, "hello");
   assert.equal(response, "world");
-  // Single turn — no boundaries yet.
-  assert.equal(state.boundaries.length, 0);
+  // The reply is part of the exchange: one boundary, between the turn and
+  // the appended reply.
+  assert.equal(state.boundaries.length, 1);
   mind.endConversation(conv);
   await mind.store.close();
 });
@@ -255,13 +258,15 @@ test("E2: multi-turn conversation accumulates context and boundaries", async ()
   await teachConversation(mind, CAT);
 
   const conv = mind.beginConversation();
-  // First turn establishes the subject.
-  await mind.respondTurn(conv, "I adopted a cat");
+  // First turn establishes the subject — scripted history, so it is FED
+  // (addTurn), not asked.
+  mind.addTurn(conv, "I adopted a cat");
 
   // Second turn is the pivot query.
   const t2 = await mind.respondTurnText(conv, "what is its name?");
-  // Now we have one boundary: the byte position after the first turn.
-  assert.equal(t2.state.boundaries.length, 1);
+  // Two boundaries: after the first turn, and after the pivot (the reply
+  // is part of the exchange).
+  assert.equal(t2.state.boundaries.length, 2);
   assert.equal(t2.response, "her name is Whiskers");
 
   mind.endConversation(conv);
@@ -272,28 +277,26 @@ test("E3: save → end → restore → continue works", async () => {
   const mind = newMind();
   await teachConversation(mind, CAT);
 
-  // First two turns.
+  // First two turns (context fed, pivot asked).
   const conv = mind.beginConversation();
-  await mind.respondTurn(conv, "I adopted a cat");
+  mind.addTurn(conv, "I adopted a cat");
   const { state: saved } = await mind.respondTurnText(
     conv,
     "what is its name?",
   );
-  assert.equal(saved.boundaries.length, 1);
+  assert.equal(saved.boundaries.length, 2);
 
   // Save and end.
   mind.endConversation(conv);
 
   // Restore from the saved state into a new handle.
   const conv2 = mind.beginConversation(saved);
-  const { response, state: state2 } = await mind.respondTurnText(
-    conv2,
-    "her name is Whiskers",
-  );
-  // The restored conversation continues from where it left off.
-  assert.ok(response.length > 0 || response === "");
-  // Boundaries are intact from the restore.
-  assert.equal(state2.boundaries.length, 2);
+  // Boundaries are intact from the restore, and the conversation continues
+  // from where it left off.
+  const state2 = mind.addTurn(conv2, "and she likes to nap");
+  assert.equal(state2.boundaries.length, saved.boundaries.length + 1);
+  const { response } = await mind.respondTurnText(conv2, "what is its name?");
+  assert.ok(typeof response === "string");
 
   mind.endConversation(conv2);
   await mind.store.close();
@@ -336,8 +339,8 @@ test("E7: multiple concurrent conversations are independent", async () => {
   const dogConv = mind.beginConversation();
 
   // Feed the distinguishing context to each.
-  await mind.respondTurn(catConv, "I adopted a cat");
-  await mind.respondTurn(dogConv, "I adopted a dog");
+  mind.addTurn(catConv, "I adopted a cat");
+  mind.addTurn(dogConv, "I adopted a dog");
 
   // The same pivot turn — different answers.
   const catR = await mind.respondTurnText(catConv, "what is its name?");
@@ -347,9 +350,9 @@ test("E7: multiple concurrent conversations are independent", async () => {
   assert.equal(dogR.response, "his name is Rex");
   assert.notEqual(catR.response, dogR.response);
 
-  // Each conversation has its own boundaries.
-  assert.equal(catR.state.boundaries.length, 1);
-  assert.equal(dogR.state.boundaries.length, 1);
+  // Each conversation has its own boundaries (turn|pivot, pivot|reply).
+  assert.equal(catR.state.boundaries.length, 2);
+  assert.equal(dogR.state.boundaries.length, 2);
 
   // Ending one does not affect the other.
   mind.endConversation(catConv);
@@ -384,9 +387,11 @@ test("E9: respond and respondTurn give the same answer for the same cumulative b
     "I adopted a catwhat is its name?",
   );
 
-  // Via respondTurn — the new way.
+  // Via the Conversation API — the first turn is fed (addTurn keeps the
+  // cumulative bytes identical to the raw concatenation above; respondTurn
+  // would append its own reply between the turns).
   const conv = mind.beginConversation();
-  await mind.respondTurnText(conv, "I adopted a cat");
+  mind.addTurn(conv, "I adopted a cat");
   const { response: viaConv } = await mind.respondTurnText(
     conv,
     "what is its name?",
@@ -396,3 +401,48 @@ test("E9: respond and respondTurn give the same answer for the same cumulative b
   assert.equal(viaRespond, viaConv);
   await mind.store.close();
 });
+
+test("E10: an empty turn neither grows the context nor marks a boundary", async () => {
+  const mind = newMind();
+  await teachConversation(mind, CAT);
+
+  const conv = mind.beginConversation();
+  mind.addTurn(conv, "I adopted a cat");
+  const before = mind.conversationState(conv);
+
+  // Empty turns are no turns: addTurn and respondTurn alike must leave the
+  // context bytes and the (strictly increasing) boundaries untouched.
+  const viaAdd = mind.addTurn(conv, "");
+  assert.equal(viaAdd.context.length, before.context.length);
+  assert.deepEqual(viaAdd.boundaries, before.boundaries);
+
+  // respondTurn("") answers the ACCUMULATED context (an empty turn adds no
+  // bytes and no boundary of its own) — only its reply may grow the state,
+  // through the same append+boundary path as any turn.
+  const { response: r0, state: viaTurn } = await mind.respondTurnText(
+    conv,
+    "",
+  );
+  const replyLen = new TextEncoder().encode(r0).length;
+  assert.equal(viaTurn.context.length, before.context.length + replyLen);
+  assert.equal(
+    viaTurn.boundaries.length,
+    before.boundaries.length + (replyLen > 0 ? 1 : 0),
+  );
+  for (let i = 1; i < viaTurn.boundaries.length; i++) {
+    assert.ok(viaTurn.boundaries[i] > viaTurn.boundaries[i - 1]);
+  }
+
+  // The conversation still answers afterwards — the zero-growth refold must
+  // not have corrupted the pyramid's shared raw interiors.  (The empty
+  // turn's own reply may have advanced the exchange, so the answer is
+  // asserted by content, not byte-exactly.)
+  const { response } = await mind.respondTurnText(conv, "what is its name?");
+  assert.ok(
+    response.includes("her name is Whiskers"),
+    `expected the name to surface, got ${JSON.stringify(response)}`,
+  );
+
+  mind.endConversation(conv);
+  await mind.store.close();
+});

package/test/35-prefix-edge.test.mjs

@@ -0,0 +1,86 @@
+// test/35-prefix-edge.test.mjs — RC7: right-edge suffix inheritance
+//
+// When ingestPair learns C → D, every right-aligned byte suffix of C
+// that is already a known form inherits the edge.  Gate: ≥ 2 structural
+// parents, or (halo > 0 ∧ already an edge source).  Pure answers do
+// not qualify — they are destinations, not sources.
+//
+// All phrases verified via instrumentation first (see MISTAKES.md).
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { Mind } from "../dist/src/index.js";
+import { SQliteStore } from "../dist/src/store-sqlite.js";
+import { resolve } from "../dist/src/mind/primitives.js";
+
+const enc = new TextEncoder();
+const dec = new TextDecoder();
+const text = (b) => dec.decode(b.filter((x) => x !== 0));
+const mk = (s = 7) =>
+  new Mind({ seed: s, store: new SQliteStore({ path: ":memory:" }) });
+
+// Establish via pair (gives halo mass).  The second deposit where the
+// form appears as a fold-tree constituent gives ≥ 1 structural parent.
+async function establish(m, phrase) {
+  await m.ingest([phrase, "established"]);
+}
+
+test("A1 — right-aligned known suffix gains edge", async () => {
+  const m = mk();
+  await establish(m, "planet in our solar system");
+  const sid = resolve(m, enc.encode("planet in our solar system"));
+
+  await m.ingest([
+    "What is the largest planet in our solar system",
+    "Jupiter.",
+  ]);
+  const next = m.store.nextFirst(sid, 10).map((n) => text(m.store.bytes(n)));
+  assert.ok(next.some((s) => s.includes("Jupiter")));
+  await m.store.close();
+});
+
+test("A2 — suffix NOT previously known gets NO edge", async () => {
+  const m = mk();
+  const before = m.store.edgeSourceCount();
+  await m.ingest(["What is the largest planet", "Jupiter."]);
+  assert.equal(m.store.edgeSourceCount() - before, 1);
+  await m.store.close();
+});
+
+test("A3 — unestablished ≥ W suffix does NOT inherit", async () => {
+  const m = mk();
+  const before = m.store.edgeSourceCount();
+  // "prefix abcd" has "abcd" as a ≥W suffix, but it was never established.
+  await m.ingest(["prefix abcd", "answer"]);
+  // Only the full context should be a new edge source — no suffix edge.
+  assert.equal(m.store.edgeSourceCount() - before, 1);
+  await m.store.close();
+});
+
+test("B1 — inherited edge is present after propagation", async () => {
+  const m = mk();
+  await establish(m, "largest planet in our solar system");
+  await m.ingest([
+    "What is the largest planet in our solar system",
+    "Jupiter.",
+  ]);
+  const sid = resolve(m, enc.encode("largest planet in our solar system"));
+  const next = m.store.nextFirst(sid, 10).map((n) => text(m.store.bytes(n)));
+  assert.ok(next.some((s) => s.includes("Jupiter")));
+  await m.store.close();
+});
+
+test("B2 — bidirectional pair via suffix inheritance", async () => {
+  const m = mk();
+  await establish(m, "thank you");
+  await establish(m, "merci");
+  await m.ingest(["translates to merci", "thank you"]);
+  await m.ingest(["translates to thank you", "merci"]);
+
+  const sid = resolve(m, enc.encode("thank you"));
+  const next = m.store.nextFirst(sid, 10).map((n) => text(m.store.bytes(n)));
+  // The establish edge ("established") is still there, but the inherited
+  // edge ("merci") must ALSO be present.
+  assert.ok(next.some((s) => s.includes("merci")));
+  await m.store.close();
+});