@hviana/sema 0.1.7 → 0.1.9

package/src/store-sqlite.ts

@@ -134,6 +134,18 @@ CREATE TABLE IF NOT EXISTS contain (
   id      INTEGER PRIMARY KEY,
   parents BLOB NOT NULL
 );
+-- CANONICAL-FORM index (Store.canonAdd/canonFind): h is the 32-bit hash of a
+-- node's CANONICAL content key (the modality's canonicalizer output — case-
+-- folded, whitespace-collapsed text, etc.; the store never sees the key
+-- itself, only its hash).  Same hash-then-verify discipline as idx_node_h:
+-- the caller re-canonicalizes each candidate's bytes before trusting it, so
+-- a collision costs a read, never a wrong id.  WITHOUT ROWID on (h, id):
+-- the composite key IS the lookup index and gives free dedup.
+CREATE TABLE IF NOT EXISTS canon (
+  h  INTEGER NOT NULL,
+  id INTEGER NOT NULL,
+  PRIMARY KEY (h, id)
+) WITHOUT ROWID;
 CREATE TABLE IF NOT EXISTS snapshot (
   id INTEGER PRIMARY KEY CHECK (id = 1),
   data BLOB NOT NULL
@@ -241,6 +253,10 @@ export class SQliteStore extends AbstractStore implements Store {
   private _selPrev: any = null;
   private _setMeta: any = null;
   private _getMeta: any = null;
+  private _insCanon: any = null;
+  private _selCanon: any = null;
+  private _cntCanon: any = null;
+  private _selContentFrom: any = null;
   private _delMeta: any = null;
   private _insSnapshot: any = null;
   private _selSnapshot: any = null;
@@ -990,6 +1006,58 @@ export class SQliteStore extends AbstractStore implements Store {
     this._delMeta.run(key);
   }
 
+  // -- Canonical-form index (Store optional capability) --
+
+  canonAdd(h: number, id: number): void {
+    if (!this._insCanon) {
+      this._insCanon = this.sqlite!.prepare(
+        "INSERT OR IGNORE INTO canon (h, id) VALUES (?, ?)",
+      );
+    }
+    // Join the deferred write transaction (committed by flush/commit), so a
+    // bulk index build coalesces instead of paying autocommit per row.
+    this._dbBeginTx();
+    this._insCanon.run(h, id);
+  }
+
+  canonFind(h: number): number[] {
+    if (!this._selCanon) {
+      this._selCanon = this.sqlite!.prepare(
+        "SELECT id FROM canon WHERE h = ?",
+      );
+    }
+    return (this._selCanon.all(h) as Array<{ id: number }>).map((r) => r.id);
+  }
+
+  canonCount(): number {
+    if (!this._cntCanon) {
+      this._cntCanon = this.sqlite!.prepare(
+        "SELECT count(*) AS c FROM canon",
+      );
+    }
+    return (this._cntCanon.get() as { c: number }).c;
+  }
+
+  eachContent(
+    cb: (id: number, bytes: Uint8Array) => void,
+    fromId = 0,
+  ): void {
+    if (!this._selContentFrom) {
+      // Content-bearing nodes are FLAT branches: leaf present, kids an
+      // empty (zero-length) blob — the same population PART 2 of the
+      // diagnostics calls "distinct content spans".
+      this._selContentFrom = this.sqlite!.prepare(
+        "SELECT id, leaf FROM node " +
+          "WHERE id >= ? AND leaf IS NOT NULL " +
+          "AND (kids IS NULL OR length(kids) = 0)",
+      );
+    }
+    for (const row of this._selContentFrom.iterate(fromId)) {
+      const r = row as { id: number; leaf: Uint8Array };
+      cb(r.id, new Uint8Array(r.leaf));
+    }
+  }
+
   // -- Snapshot --
 
   protected _dbSaveSnapshot(bytes: Uint8Array): void {

package/src/store.ts

@@ -417,6 +417,33 @@ export interface Store {
    *  the node has no halo row. */
   haloMass(id: NodeId): number;
 
+  // ── canonical-form index (optional capability) ─────────────────────────
+  // A small hash index from CANONICAL content keys to node ids, enabling
+  // equivalence-class resolution ("WHAT" finds the stored "What") without
+  // the store knowing what the equivalence IS: the canonicalizer lives with
+  // the modality (see src/canon.ts) and is applied by the CALLER — the store
+  // only maps 32-bit key hashes to candidate ids, and the caller verifies
+  // canon(stored bytes) === key before trusting any candidate (the same
+  // hash-then-verify discipline as the node table's own `h` index).
+  // Backends that do not implement the capability leave all three absent;
+  // resolution then simply has no canonical fallback.
+
+  /** Record that node `id`'s canonical key hashes to `h`.  Idempotent. */
+  canonAdd?(h: number, id: NodeId): void;
+  /** All candidate node ids whose canonical key hashes to `h` (collisions
+   *  included — the caller verifies). */
+  canonFind?(h: number): NodeId[];
+  /** Number of (h, id) rows in the canon index — 0 means never built. */
+  canonCount?(): number;
+  /** Visit every content-bearing node (flat branch: `leaf` present, no
+   *  kids) — the population a canonical index is built over.  `fromId`
+   *  restricts the scan to ids ≥ fromId, so an index refresh after further
+   *  training only visits the new rows. */
+  eachContent?(
+    cb: (id: NodeId, bytes: Uint8Array) => void,
+    fromId?: NodeId,
+  ): void;
+
   // ── lifecycle ──────────────────────────────────────────────────────────
   size(): Promise<number>;
   saveSnapshot(bytes: Uint8Array): Promise<void>;

package/test/13-conversation.test.mjs

@@ -45,15 +45,17 @@ async function teachConversation(mind, turns) {
 // ═══════════════════════════════════════════════════════════════════════
 // predictNext — given the turns spoken so far, predict the next turn.
 //
-// Uses the Conversation API: beginConversation → respondTurn for each
-// prior turn → respondTurnText for the query → endConversation.
+// Uses the Conversation API: beginConversation → addTurn for each prior
+// turn (BOTH speakers' lines are history being replayed, not questions to
+// answer — respondTurn would answer each one AND append its own reply to
+// the context) → respondTurnText for the final turn → endConversation.
 // Turns are raw strings, concatenated by the Mind — no separator.
 // ═══════════════════════════════════════════════════════════════════════
 async function predictNext(mind, priorTurns) {
   if (priorTurns.length === 0) return "";
   const conv = mind.beginConversation();
   for (let i = 0; i < priorTurns.length - 1; i++) {
-    await mind.respondTurn(conv, priorTurns[i]);
+    mind.addTurn(conv, priorTurns[i]);
   }
   const { response } = await mind.respondTurnText(
     conv,
@@ -244,8 +246,9 @@ test("E1: begin → respondTurn → end completes a single-turn conversation", a
   const conv = mind.beginConversation();
   const { response, state } = await mind.respondTurnText(conv, "hello");
   assert.equal(response, "world");
-  // Single turn — no boundaries yet.
-  assert.equal(state.boundaries.length, 0);
+  // The reply is part of the exchange: one boundary, between the turn and
+  // the appended reply.
+  assert.equal(state.boundaries.length, 1);
   mind.endConversation(conv);
   await mind.store.close();
 });
@@ -255,13 +258,15 @@ test("E2: multi-turn conversation accumulates context and boundaries", async ()
   await teachConversation(mind, CAT);
 
   const conv = mind.beginConversation();
-  // First turn establishes the subject.
-  await mind.respondTurn(conv, "I adopted a cat");
+  // First turn establishes the subject — scripted history, so it is FED
+  // (addTurn), not asked.
+  mind.addTurn(conv, "I adopted a cat");
 
   // Second turn is the pivot query.
   const t2 = await mind.respondTurnText(conv, "what is its name?");
-  // Now we have one boundary: the byte position after the first turn.
-  assert.equal(t2.state.boundaries.length, 1);
+  // Two boundaries: after the first turn, and after the pivot (the reply
+  // is part of the exchange).
+  assert.equal(t2.state.boundaries.length, 2);
   assert.equal(t2.response, "her name is Whiskers");
 
   mind.endConversation(conv);
@@ -272,28 +277,26 @@ test("E3: save → end → restore → continue works", async () => {
   const mind = newMind();
   await teachConversation(mind, CAT);
 
-  // First two turns.
+  // First two turns (context fed, pivot asked).
   const conv = mind.beginConversation();
-  await mind.respondTurn(conv, "I adopted a cat");
+  mind.addTurn(conv, "I adopted a cat");
   const { state: saved } = await mind.respondTurnText(
     conv,
     "what is its name?",
   );
-  assert.equal(saved.boundaries.length, 1);
+  assert.equal(saved.boundaries.length, 2);
 
   // Save and end.
   mind.endConversation(conv);
 
   // Restore from the saved state into a new handle.
   const conv2 = mind.beginConversation(saved);
-  const { response, state: state2 } = await mind.respondTurnText(
-    conv2,
-    "her name is Whiskers",
-  );
-  // The restored conversation continues from where it left off.
-  assert.ok(response.length > 0 || response === "");
-  // Boundaries are intact from the restore.
-  assert.equal(state2.boundaries.length, 2);
+  // Boundaries are intact from the restore, and the conversation continues
+  // from where it left off.
+  const state2 = mind.addTurn(conv2, "and she likes to nap");
+  assert.equal(state2.boundaries.length, saved.boundaries.length + 1);
+  const { response } = await mind.respondTurnText(conv2, "what is its name?");
+  assert.ok(typeof response === "string");
 
   mind.endConversation(conv2);
   await mind.store.close();
@@ -336,8 +339,8 @@ test("E7: multiple concurrent conversations are independent", async () => {
   const dogConv = mind.beginConversation();
 
   // Feed the distinguishing context to each.
-  await mind.respondTurn(catConv, "I adopted a cat");
-  await mind.respondTurn(dogConv, "I adopted a dog");
+  mind.addTurn(catConv, "I adopted a cat");
+  mind.addTurn(dogConv, "I adopted a dog");
 
   // The same pivot turn — different answers.
   const catR = await mind.respondTurnText(catConv, "what is its name?");
@@ -347,9 +350,9 @@ test("E7: multiple concurrent conversations are independent", async () => {
   assert.equal(dogR.response, "his name is Rex");
   assert.notEqual(catR.response, dogR.response);
 
-  // Each conversation has its own boundaries.
-  assert.equal(catR.state.boundaries.length, 1);
-  assert.equal(dogR.state.boundaries.length, 1);
+  // Each conversation has its own boundaries (turn|pivot, pivot|reply).
+  assert.equal(catR.state.boundaries.length, 2);
+  assert.equal(dogR.state.boundaries.length, 2);
 
   // Ending one does not affect the other.
   mind.endConversation(catConv);
@@ -384,9 +387,11 @@ test("E9: respond and respondTurn give the same answer for the same cumulative b
     "I adopted a catwhat is its name?",
   );
 
-  // Via respondTurn — the new way.
+  // Via the Conversation API — the first turn is fed (addTurn keeps the
+  // cumulative bytes identical to the raw concatenation above; respondTurn
+  // would append its own reply between the turns).
   const conv = mind.beginConversation();
-  await mind.respondTurnText(conv, "I adopted a cat");
+  mind.addTurn(conv, "I adopted a cat");
   const { response: viaConv } = await mind.respondTurnText(
     conv,
     "what is its name?",
@@ -396,3 +401,48 @@ test("E9: respond and respondTurn give the same answer for the same cumulative b
   assert.equal(viaRespond, viaConv);
   await mind.store.close();
 });
+
+test("E10: an empty turn neither grows the context nor marks a boundary", async () => {
+  const mind = newMind();
+  await teachConversation(mind, CAT);
+
+  const conv = mind.beginConversation();
+  mind.addTurn(conv, "I adopted a cat");
+  const before = mind.conversationState(conv);
+
+  // Empty turns are no turns: addTurn and respondTurn alike must leave the
+  // context bytes and the (strictly increasing) boundaries untouched.
+  const viaAdd = mind.addTurn(conv, "");
+  assert.equal(viaAdd.context.length, before.context.length);
+  assert.deepEqual(viaAdd.boundaries, before.boundaries);
+
+  // respondTurn("") answers the ACCUMULATED context (an empty turn adds no
+  // bytes and no boundary of its own) — only its reply may grow the state,
+  // through the same append+boundary path as any turn.
+  const { response: r0, state: viaTurn } = await mind.respondTurnText(
+    conv,
+    "",
+  );
+  const replyLen = new TextEncoder().encode(r0).length;
+  assert.equal(viaTurn.context.length, before.context.length + replyLen);
+  assert.equal(
+    viaTurn.boundaries.length,
+    before.boundaries.length + (replyLen > 0 ? 1 : 0),
+  );
+  for (let i = 1; i < viaTurn.boundaries.length; i++) {
+    assert.ok(viaTurn.boundaries[i] > viaTurn.boundaries[i - 1]);
+  }
+
+  // The conversation still answers afterwards — the zero-growth refold must
+  // not have corrupted the pyramid's shared raw interiors.  (The empty
+  // turn's own reply may have advanced the exchange, so the answer is
+  // asserted by content, not byte-exactly.)
+  const { response } = await mind.respondTurnText(conv, "what is its name?");
+  assert.ok(
+    response.includes("her name is Whiskers"),
+    `expected the name to surface, got ${JSON.stringify(response)}`,
+  );
+
+  mind.endConversation(conv);
+  await mind.store.close();
+});

package/test/35-prefix-edge.test.mjs

@@ -0,0 +1,86 @@
+// test/35-prefix-edge.test.mjs — RC7: right-edge suffix inheritance
+//
+// When ingestPair learns C → D, every right-aligned byte suffix of C
+// that is already a known form inherits the edge.  Gate: ≥ 2 structural
+// parents, or (halo > 0 ∧ already an edge source).  Pure answers do
+// not qualify — they are destinations, not sources.
+//
+// All phrases verified via instrumentation first (see MISTAKES.md).
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { Mind } from "../dist/src/index.js";
+import { SQliteStore } from "../dist/src/store-sqlite.js";
+import { resolve } from "../dist/src/mind/primitives.js";
+
+const enc = new TextEncoder();
+const dec = new TextDecoder();
+const text = (b) => dec.decode(b.filter((x) => x !== 0));
+const mk = (s = 7) =>
+  new Mind({ seed: s, store: new SQliteStore({ path: ":memory:" }) });
+
+// Establish via pair (gives halo mass).  The second deposit where the
+// form appears as a fold-tree constituent gives ≥ 1 structural parent.
+async function establish(m, phrase) {
+  await m.ingest([phrase, "established"]);
+}
+
+test("A1 — right-aligned known suffix gains edge", async () => {
+  const m = mk();
+  await establish(m, "planet in our solar system");
+  const sid = resolve(m, enc.encode("planet in our solar system"));
+
+  await m.ingest([
+    "What is the largest planet in our solar system",
+    "Jupiter.",
+  ]);
+  const next = m.store.nextFirst(sid, 10).map((n) => text(m.store.bytes(n)));
+  assert.ok(next.some((s) => s.includes("Jupiter")));
+  await m.store.close();
+});
+
+test("A2 — suffix NOT previously known gets NO edge", async () => {
+  const m = mk();
+  const before = m.store.edgeSourceCount();
+  await m.ingest(["What is the largest planet", "Jupiter."]);
+  assert.equal(m.store.edgeSourceCount() - before, 1);
+  await m.store.close();
+});
+
+test("A3 — unestablished ≥ W suffix does NOT inherit", async () => {
+  const m = mk();
+  const before = m.store.edgeSourceCount();
+  // "prefix abcd" has "abcd" as a ≥W suffix, but it was never established.
+  await m.ingest(["prefix abcd", "answer"]);
+  // Only the full context should be a new edge source — no suffix edge.
+  assert.equal(m.store.edgeSourceCount() - before, 1);
+  await m.store.close();
+});
+
+test("B1 — inherited edge is present after propagation", async () => {
+  const m = mk();
+  await establish(m, "largest planet in our solar system");
+  await m.ingest([
+    "What is the largest planet in our solar system",
+    "Jupiter.",
+  ]);
+  const sid = resolve(m, enc.encode("largest planet in our solar system"));
+  const next = m.store.nextFirst(sid, 10).map((n) => text(m.store.bytes(n)));
+  assert.ok(next.some((s) => s.includes("Jupiter")));
+  await m.store.close();
+});
+
+test("B2 — bidirectional pair via suffix inheritance", async () => {
+  const m = mk();
+  await establish(m, "thank you");
+  await establish(m, "merci");
+  await m.ingest(["translates to merci", "thank you"]);
+  await m.ingest(["translates to thank you", "merci"]);
+
+  const sid = resolve(m, enc.encode("thank you"));
+  const next = m.store.nextFirst(sid, 10).map((n) => text(m.store.bytes(n)));
+  // The establish edge ("established") is still there, but the inherited
+  // edge ("merci") must ALSO be present.
+  assert.ok(next.some((s) => s.includes("merci")));
+  await m.store.close();
+});

package/dist/example/demo.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/example/demo.js

@@ -1,39 +0,0 @@
-// demo.ts — one short session that drives the WHOLE pipeline from one memory.
-//
-// We give Sema a handful of plain notes, then ask things that no single note
-// answers. The headline query is the third one: from three worked examples Sema
-// learns the shape of "X was painted by Y", lifts the painter out of a sentence
-// it has NEVER seen, and then — in the same pass — reasons forward to a separate
-// fact about that painter. The reply contains no word from the question. That is
-// retrieval, generalization, and reasoning composing as a single act, with every
-// step traceable back to the notes behind it.
-import { Mind } from "../src/index.js";
-import { SQliteStore } from "../src/store-sqlite.js";
-async function main() {
-    const mind = new Mind({ store: new SQliteStore({ path: ":memory:" }) });
-    const ask = async (q) => (await mind.respondText(q)).trim();
-    // ── Jot down what we know. Each line is just (context → what follows). ──
-    await mind.ingest([
-        // One relation, shown three times — a pattern taught purely by example:
-        ["The Mona Lisa was painted by Leonardo da Vinci.", "Leonardo da Vinci"],
-        ["The Starry Night was painted by Vincent van Gogh.", "Vincent van Gogh"],
-        [
-            "The Night Watch was painted by Rembrandt van Rijn.",
-            "Rembrandt van Rijn",
-        ],
-        // One stray fact, keyed on a name none of the examples mention:
-        ["Pablo Picasso", "Pablo Picasso co-founded the Cubist movement"],
-    ]);
-    // 1) GENERALIZE — apply the learned pattern to an unseen sentence and read out
-    //    the painter. "Pablo Picasso" was never given as an answer; Sema locates it
-    //    by analogy to the three examples.
-    console.log(await ask("The Weeping Woman was painted by Pablo Picasso."));
-    // → "Pablo Picasso co-founded the Cubist movement"
-    //   …and, having found the painter, it KEEPS GOING: the name bridges into the
-    //   one fact it holds about him. The answer appears in no word of the question.
-    // 2) COMPUTE — exact arithmetic, grounded right where the notes go silent.
-    console.log(await ask("a museum charges 12*4 for a family ticket"));
-    // → "48"
-    await mind.store.close();
-}
-main();

package/dist/example/train_base.d.ts

@@ -1,87 +0,0 @@
-export interface Episode {
-    context: string;
-    continuation: string;
-}
-export type TrainingItem = string | Episode;
-/** Dedup + trim a concept's items: drop empty/degenerate pairs and exact
- *  repeats so a concept never deposits the same form twice. */
-export declare function refineItems(items: TrainingItem[]): TrainingItem[];
-/** One normalized SmolSent row. */
-export interface SmolSentRow {
-    src: string;
-    trg: string;
-    sl: string;
-    tl: string;
-}
-/** Normalize a raw datasets-server row into a SmolSentRow, or null when it lacks
- *  both sides or a side is implausibly large (a dump, not a sentence). */
-export declare function toSmolSentRow(row: unknown): SmolSentRow | null;
-/** Translate ONE SmolSent pair into SEMA facts: the two sentences are one
- *  meaning in two languages, so bind them BOTH ways. refineItems drops the
- *  degenerate case where src === trg. */
-export declare function smolSentRowToItems(row: SmolSentRow): TrainingItem[];
-/** One normalized Aya row. */
-export interface AyaRow {
-    inputs: string;
-    targets: string;
-    language: string;
-}
-/** Normalize a raw datasets-server row object into an AyaRow, or null when it
- *  lacks a usable prompt/answer or a field is implausibly large (a dump, not a
- *  cognitive example). Trims surrounding whitespace; keeps inner text verbatim
- *  (human prose, possibly multi-paragraph). */
-export declare function toAyaRow(row: unknown): AyaRow | null;
-/** Translate ONE Aya row into SEMA training items. A row is a single human
- *  (question → answer) exchange — exactly one FACT, the (inputs → targets) edge.
- *  No standalone-answer experience and no one-exchange "cumulative" walk: a lone
- *  Q→A is not multi-turn, and both would only replicate the same edge. */
-export declare function ayaRowToItems(row: AyaRow): TrainingItem[];
-/** A single oasst2 message node (the fields we use; the tree nests via replies). */
-interface OasstNode {
-    role?: string;
-    text?: string;
-    rank?: number | null;
-    deleted?: boolean;
-    replies?: OasstNode[];
-}
-/** One conversational turn extracted from a tree. */
-export interface OasstTurn {
-    role: string;
-    text: string;
-}
-/** Collapse a conversation tree to ONE linear path: at each node, descend into
- *  its best-ranked, non-deleted reply (rank 0 preferred; unranked sorts last).
- *  Returns the ordered turns (already strictly alternating in this corpus). */
-export declare function bestOasstPath(root: OasstNode): OasstTurn[];
-/** Translate ONE multi-turn oasst2 conversation into SEMA training items.
- *
- *  This is the ONE stage where cumulative continuous context is truly necessary:
- *  the data is a real multi-turn dialogue, and what must be learned is how each
- *  turn follows from the WHOLE conversation so far — not from the previous turn
- *  alone. The conversation is emitted ONLY as the accumulated walk; standalone
- *  turn experiences and local adjacent-pair facts are NOT emitted (they are
- *  subsumed by it and would merely replicate the content).
- *
- *  The walk is byte-for-byte the pattern proven in test/13-conversation.test.mjs
- *  ("teachConversation"): each turn is the continuation of all prior turns joined
- *  by "\n", with BARE turn text — NO "User:/Assistant:" labels. Roles already
- *  alternate by position in an oasst2 best-path (the root is a prompter), so a
- *  label adds nothing the position does not, while a clean continuation matches
- *  the test's recall (predictNext queries bare prior turns) and lets a turn share
- *  its gist with the same text elsewhere (e.g. an Aya question stored bare).
- *
- *  Returns [] for a conversation below the multi-turn threshold, so callers can
- *  simply skip empties. */
-export declare function oasstConversationToItems(turns: OasstTurn[]): TrainingItem[];
-/** One normalized General-Knowledge row. */
-export interface GenKnowRow {
-    question: string;
-    answer: string;
-}
-/** Normalize a raw datasets-server row into a GenKnowRow, or null when it lacks
- *  a usable question/answer or a side is implausibly large (corruption). */
-export declare function toGenKnowRow(row: unknown): GenKnowRow | null;
-/** Translate ONE General-Knowledge row into SEMA items: exactly one
- *  (question → answer) FACT. refineItems drops a degenerate question === answer. */
-export declare function genKnowRowToItems(row: GenKnowRow): TrainingItem[];
-export {};