@hviana/sema 0.1.6 → 0.1.7

package/src/mind/recognition.ts

@@ -7,8 +7,14 @@
 import { rItem } from "./trace.js";
 
 import type { MindContext, Recognition, Segment } from "./types.js";
-import { foldTree, gistOf, perceive, resolve } from "./primitives.js";
-import { leadsSomewhere } from "./traverse.js";
+import {
+  foldTree,
+  gistOf,
+  latin1Key,
+  perceive,
+  resolve,
+} from "./primitives.js";
+import { atomIsHub, corpusN, leadsSomewhere } from "./traverse.js";
 import { chainReach, leafIdAt } from "./canonical.js";
 import { isChunk, type Sema } from "../sema.js";
 import type { Leaf, Site } from "./graph-search.js";
@@ -26,15 +32,15 @@ import type { Leaf, Site } from "./graph-search.js";
  *
  *  Both O(n · maxGroup) bounded O(1) probes — never a scan of the corpus. */
 export function recognise(ctx: MindContext, bytes: Uint8Array): Recognition {
-  // Per-response memo (see MindContext.recogniseMemo): think, articulate and
-  // the pre-consume pass all recognise the same byte objects; each repeat is
-  // O(n·maxGroup²) store probes.  Skipped while tracing so every call still
-  // emits its rationale step.
+  // Content-keyed memo — works for both single-turn respond() and multi-turn
+  // respondTurn() (where the map persists across calls).  Skipped while
+  // tracing so every call still emits its rationale step.
   if (ctx.recogniseMemo && !ctx.trace) {
-    const hit = ctx.recogniseMemo.get(bytes);
+    const key = latin1Key(bytes);
+    const hit = ctx.recogniseMemo.get(key);
     if (hit !== undefined) return hit;
     const fresh = recogniseImpl(ctx, bytes);
-    ctx.recogniseMemo.set(bytes, fresh);
+    ctx.recogniseMemo.set(key, fresh);
     return fresh;
   }
   return recogniseImpl(ctx, bytes);
@@ -64,7 +70,19 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
     return id;
   };
 
+  // Byte atoms (implicit negative-id single-byte leaves) are admitted as
+  // recognised sites only while atoms can still DISCRIMINATE at this corpus
+  // scale (see {@link atomIsHub}).  On a small store a single-letter fact
+  // ("a" → "A") is genuine learnt content and its site is essential; on a
+  // large one every letter of every query would otherwise become a
+  // "recognised form" — the bridge then finds junction connectors between
+  // bare letters, cover follows edges hanging off them, and pure noise
+  // ("qq8f3kz9…") grounds to an arbitrary learnt sentence instead of
+  // silence.  Atoms stay available as leaves (PASS-carried literals) and
+  // through exact tier-0 resolution regardless.
+  const atomsAreHubs = atomIsHub(ctx, corpusN(ctx));
   const emit = (start: number, end: number, id: number) => {
+    if (id < 0 && atomsAreHubs) return;
     if (leadsSomewhere(ctx, id)) {
       sites.push({ start, end, payload: id });
     }

package/src/mind/traverse.ts

@@ -115,6 +115,30 @@ export function edgeAncestors(
   const hit = memo?.get(id);
   if (hit !== undefined) return hit;
 
+  // BYTE-ATOM COMMONALITY.  A single-byte leaf (implicit negative id) has no
+  // structural parents BY CONSTRUCTION — atoms are never linked into the kid
+  // or contain tables — so this climb cannot observe its containment at all.
+  // The walk below would see only the atom's own direct edges and report
+  // contextsReached ≈ 1, turning the MOST common content in the store into
+  // the MOST discriminative voter (observed on a 325K-context store: every
+  // recognised single-letter site voted full ln N for the one fact whose
+  // continuation is that letter, and their pooled sum out-voted every
+  // genuine anchor).  An unmeasurable containment must not default to
+  // "maximally rare": it is bounded below by the uniform expectation over
+  // the byte alphabet — N contexts, each at least one chunk of up to W of
+  // the 256 possible atoms, reach ≥ N·W/256 contexts per atom on average
+  // (see {@link atomReach}).  When that floor itself exceeds the hub bound
+  // √N the atom is a hub at this corpus scale and the climb abstains
+  // (saturated) — the atom's own edges remain fully traversable (tier-0
+  // exact recall, chooseNext, project); only its say as a consensus voter
+  // is withdrawn.  On a small store the floor stays ≤ √N and the atom
+  // climbs exactly as before, so single-letter facts keep working.
+  if (id < 0 && atomIsHub(ctx, contextCount)) {
+    const reach = { roots: [], contextsReached: 0, saturated: true };
+    memo?.set(id, reach);
+    return reach;
+  }
+
   const bound = Math.ceil(Math.sqrt(contextCount));
   const roots: number[] = [];
   const seen = new Set<number>([id]);
@@ -257,6 +281,29 @@ export function prevOf(ctx: MindContext, id: number): number[] {
   return ctx.store.prev(id);
 }
 
+/** The uniform-expectation floor on a byte atom's corpus commonality: N
+ *  learnt contexts, each at least one perception chunk of up to W of the 256
+ *  possible byte values, contain a given atom in ≥ N·W/256 contexts on
+ *  average.  An atom's TRUE containment is unmeasurable (atoms carry no
+ *  kid/contain links by construction), so this floor is the honest stand-in:
+ *  derived entirely from the corpus scale N, the perception window W, and
+ *  the alphabet size — never tuned. */
+export function atomReach(ctx: MindContext, contextCount: number): number {
+  return Math.max(
+    1,
+    Math.ceil((contextCount * ctx.space.maxGroup) / 256),
+  );
+}
+
+/** Whether a byte atom is a hub at this corpus scale — its commonality floor
+ *  {@link atomReach} exceeds the hub bound √N.  Below it (small stores) an
+ *  atom votes and is recognised exactly as any stored form; above it the
+ *  alphabet is scaffolding everywhere and abstains. */
+export function atomIsHub(ctx: MindContext, contextCount: number): boolean {
+  return atomReach(ctx, contextCount) >
+    Math.ceil(Math.sqrt(Math.max(2, contextCount)));
+}
+
 /** Whether a node LEADS SOMEWHERE — it bears a continuation edge or a halo.
  *  The admission predicate recognition filters sites with (HOW_IT_WORKS
  *  §15.3): a form that leads nowhere contributes nothing to any derivation.

package/src/mind/types.ts

@@ -144,28 +144,27 @@ export interface MindContext extends GraphSearchHost {
   cfg: MindConfig;
   search: GraphSearch;
   trace: Rationale | null;
-  climbMemo: WeakMap<Uint8Array, Map<string, AttentionRead>> | null;
-  /** Per-response memo of {@link recognise} keyed by the byte-array OBJECT
-   *  (think, articulate, and the post-grounding pre-consume all recognise the
-   *  same query/answer objects).  Valid because the store is read-only while
-   *  a response is in flight; bypassed when a trace is attached so every
-   *  recognise still emits its rationale step.  Null outside respond(). */
-  recogniseMemo: WeakMap<Uint8Array, Recognition> | null;
-  /** Per-response memo of {@link perceive} keyed by the byte-array OBJECT —
-   *  the GENERAL memo the result-level ones (recogniseMemo, climbMemo,
-   *  _gistCache) each partially compensate for: resolve(), gistOf(), and
-   *  every mechanism's re-perception of the same query/answer object hit it
-   *  (a reason hop used to fold the same answer three times).  Valid because
-   *  the store is read-only while a response is in flight and perception is
-   *  a pure function of bytes; only inference-shaped calls (plain Uint8Array,
-   *  no leafAt/lookup capabilities) are memoised, so the deposit path never
-   *  sees it.  Keyed by CONTENT (latin1 of the bytes), not object identity —
-   *  mechanisms materialise the same span in fresh subarrays constantly
-   *  (measured on a trained store: 46% of one response's perceptions were
-   *  byte-identical repeats an identity key missed).  NOT bypassed under
-   *  trace — perception emits no rationale steps, so there is nothing a memo
-   *  hit could swallow.  Null outside respond(). */
+  /** Memo of the consensus climb — content-keyed (latin1) so results
+   *  persist across conversation turns where the same byte spans recur.
+   *  Null outside respond(); during respondTurn() the conversation's
+   *  persistent map is swapped in. */
+  climbMemo: Map<string, Map<string, AttentionRead>> | null;
+  /** Memo of {@link recognise} — content-keyed (latin1) so recognised
+   *  forms carry forward across conversation turns.  Bypassed while a
+   *  trace is attached.  Null outside respond(). */
+  recogniseMemo: Map<string, Recognition> | null;
+  /** Memo of {@link perceive} — content-keyed (latin1).  The general
+   *  cache the result-level memos each partially compensate for.  NOT
+   *  bypassed under trace — perception emits no rationale steps.
+   *  Null outside respond(). */
   perceiveMemo: Map<string, Sema> | null;
+  /** Subtree-resolution cache: Sema node → its store id and byte length.
+   *  Populated by {@link foldTree} during inference; checked before
+   *  walking children.  When a conversation's pyramid reuses prefix
+   *  subtrees, this cache lets {@link recognise} skip them entirely —
+   *  O(suffix) instead of O(context).  Mind-lifetime (WeakMap keys are
+   *  the Sema objects the pyramid keeps alive). */
+  _resolvedSubtrees: WeakMap<Sema, { id: number; len: number }> | null;
   _edgeGuide: Vec | null;
   _edgeChoice: Map<number, number>;
   _prevSeen: Set<number> | null;

package/src/store-sqlite.ts

@@ -1048,6 +1048,10 @@ export class SQliteStore extends AbstractStore implements Store {
     return this.content ? this.content.physicalSize : 0;
   }
 
+  protected _vecContentClusterCount(): number {
+    return this.content ? this.content.clusterCount : 0;
+  }
+
   protected _vecContentCompact(): void {
     this.content!.compact();
   }
@@ -1099,6 +1103,10 @@ export class SQliteStore extends AbstractStore implements Store {
     return this.halos ? this.halos.physicalSize : 0;
   }
 
+  protected _vecHaloClusterCount(): number {
+    return this.halos ? this.halos.clusterCount : 0;
+  }
+
   protected _vecHaloCompact(): void {
     this.halos!.compact();
   }

package/src/store.ts

@@ -724,6 +724,7 @@ export abstract class AbstractStore implements Store {
   protected abstract _vecContentSize(): number;
   protected abstract _vecContentLastReads(): number;
   protected abstract _vecContentPhysicalSize(): number;
+  protected abstract _vecContentClusterCount(): number;
   protected abstract _vecContentCompact(): void;
   /** Live content-index entries whose INTERNAL id is > `after`, as
    *  {ext, internal} pairs in internal-id order.  Internal ids are monotone
@@ -759,8 +760,24 @@ export abstract class AbstractStore implements Store {
    *  the tombstone-ratio compaction trigger compares physical size against. */
   protected abstract _vecHaloSize(): number;
   protected abstract _vecHaloPhysicalSize(): number;
+  protected abstract _vecHaloClusterCount(): number;
   protected abstract _vecHaloCompact(): void;
 
+  /** Derived query breadth for a partitioned index of C clusters: probe √C
+   *  of them (the same √-of-the-population convention as the hub bound √N).
+   *  The IVF maps ef → nprobe as ceil(ef/4), so ef = 4·⌈√C⌉ probes exactly
+   *  ⌈√C⌉ clusters.  A FIXED efSearch stops scaling the moment the
+   *  collection outgrows it: at 4,270 clusters the default 64 probed 16
+   *  clusters (0.4%), and an exact stored match of a query routinely sat in
+   *  an unprobed cluster — recall silently degraded as the store grew.  The
+   *  configured efSearch remains the floor for small collections. */
+  protected efFor(clusterCount: number): number {
+    return Math.max(
+      this.efSearch,
+      4 * Math.ceil(Math.sqrt(Math.max(1, clusterCount))),
+    );
+  }
+
   // ── Config ─────────────────────────────────────────────────────────────
 
   protected _D: number;
@@ -1615,7 +1632,7 @@ export abstract class AbstractStore implements Store {
     const results = this._vecContentQuery(
       normalize(copy(v)),
       k * this.overfetch,
-      this.efSearch,
+      this.efFor(this._vecContentClusterCount()),
     );
     const out: Hit[] = [];
     for (const r of results) {
@@ -1966,7 +1983,7 @@ export abstract class AbstractStore implements Store {
     const results = this._vecHaloQuery(
       normalize(copy(v)),
       k * this.overfetch,
-      this.efSearch,
+      this.efFor(this._vecHaloClusterCount()),
     );
     const out: Hit[] = [];
     for (const r of results) {

package/test/13-conversation.test.mjs

@@ -14,14 +14,10 @@
 // The assertions describe BEHAVIOUR only — they never mention how context is
 // represented — so they stay valid for any implementation.
 //
-// Note: Sema trains on the accumulated context string ("turn0\nturn1\nturn2")
-// and queries the same accumulated string at inference. This is not a
-// weakness — LLMs work the same way: a chat model receives the full
-// conversation history as its prompt, and that same history must be
-// provided at inference to produce the next turn. The difference is in
-// how the answer is produced: Sema composes it from learned graph forms;
-// an LLM samples it from a parametric distribution. Neither is "just a
-// lookup."
+// Sema trains on the raw concatenation of prior turns and queries the same
+// accumulated bytes at inference.  The Conversation API tracks turn-boundary
+// offsets explicitly so no separator character is needed — the geometry never
+// inspects content to find turn boundaries.
 // ─────────────────────────────────────────────────────────────────────────
 
 import { test } from "node:test";
@@ -34,16 +30,14 @@ const newMind = () => new Mind({ seed: 7 });
 // ═══════════════════════════════════════════════════════════════════════
 // teachConversation — store one conversation (an ordered list of turns).
 //
-// Accumulate every prior turn into the context so each episode carries the
-// full conversation history:  (t₀ → t₁), (t₀+t₁ → t₂), (t₀+t₁+t₂ → t₃) …
-// A pivot turn ("what is its name?") that appears in two conversations now
-// produces different episode vectors because the context side encodes which
-// conversation it belongs to.  All SEMA operations remain sublinear — the
-// river cuts any context into O(log N) chunks regardless of length.
+// Accumulate prior turns by raw concatenation — no separator character.
+// Each episode carries the full conversation history: (t₀ → t₁),
+// (t₀+t₁ → t₂), (t₀+t₁+t₂ → t₃) … where + is byte concatenation.
 // ═══════════════════════════════════════════════════════════════════════
 async function teachConversation(mind, turns) {
+  let context = "";
   for (let i = 0; i + 1 < turns.length; i++) {
-    const context = turns.slice(0, i + 1).join("\n");
+    context += turns[i];
     await mind.ingest(context, turns[i + 1]);
   }
 }
@@ -51,13 +45,22 @@ async function teachConversation(mind, turns) {
 // ═══════════════════════════════════════════════════════════════════════
 // predictNext — given the turns spoken so far, predict the next turn.
 //
-// Join every prior turn into one accumulated context, mirroring the storage
-// pattern in teachConversation.  The resulting query string is the exact
-// context side of the matching episode, so recall resolves unambiguously.
+// Uses the Conversation API: beginConversation → respondTurn for each
+// prior turn → respondTurnText for the query → endConversation.
+// Turns are raw strings, concatenated by the Mind — no separator.
 // ═══════════════════════════════════════════════════════════════════════
 async function predictNext(mind, priorTurns) {
-  const context = priorTurns.join("\n");
-  return await mind.respondText(context);
+  if (priorTurns.length === 0) return "";
+  const conv = mind.beginConversation();
+  for (let i = 0; i < priorTurns.length - 1; i++) {
+    await mind.respondTurn(conv, priorTurns[i]);
+  }
+  const { response } = await mind.respondTurnText(
+    conv,
+    priorTurns[priorTurns.length - 1],
+  );
+  mind.endConversation(conv);
+  return response;
 }
 
 // A small convenience: teach a conversation, then ask for the continuation
@@ -226,3 +229,170 @@ test("D2: prediction is deterministic across runs", async () => {
   };
   assert.equal(await run(), await run());
 });
+
+// ═══════════════════════════════════════════════════════════════════════
+// Section E — Conversation API lifecycle & cache management
+//
+// Tests for the beginConversation / respondTurn / endConversation API
+// itself, independent of the conversation-training pattern above.
+// ═══════════════════════════════════════════════════════════════════════
+
+test("E1: begin → respondTurn → end completes a single-turn conversation", async () => {
+  const mind = newMind();
+  await mind.ingest("hello", "world");
+
+  const conv = mind.beginConversation();
+  const { response, state } = await mind.respondTurnText(conv, "hello");
+  assert.equal(response, "world");
+  // Single turn — no boundaries yet.
+  assert.equal(state.boundaries.length, 0);
+  mind.endConversation(conv);
+  await mind.store.close();
+});
+
+test("E2: multi-turn conversation accumulates context and boundaries", async () => {
+  const mind = newMind();
+  await teachConversation(mind, CAT);
+
+  const conv = mind.beginConversation();
+  // First turn establishes the subject.
+  await mind.respondTurn(conv, "I adopted a cat");
+
+  // Second turn is the pivot query.
+  const t2 = await mind.respondTurnText(conv, "what is its name?");
+  // Now we have one boundary: the byte position after the first turn.
+  assert.equal(t2.state.boundaries.length, 1);
+  assert.equal(t2.response, "her name is Whiskers");
+
+  mind.endConversation(conv);
+  await mind.store.close();
+});
+
+test("E3: save → end → restore → continue works", async () => {
+  const mind = newMind();
+  await teachConversation(mind, CAT);
+
+  // First two turns.
+  const conv = mind.beginConversation();
+  await mind.respondTurn(conv, "I adopted a cat");
+  const { state: saved } = await mind.respondTurnText(
+    conv,
+    "what is its name?",
+  );
+  assert.equal(saved.boundaries.length, 1);
+
+  // Save and end.
+  mind.endConversation(conv);
+
+  // Restore from the saved state into a new handle.
+  const conv2 = mind.beginConversation(saved);
+  const { response, state: state2 } = await mind.respondTurnText(
+    conv2,
+    "her name is Whiskers",
+  );
+  // The restored conversation continues from where it left off.
+  assert.ok(response.length > 0 || response === "");
+  // Boundaries are intact from the restore.
+  assert.equal(state2.boundaries.length, 2);
+
+  mind.endConversation(conv2);
+  await mind.store.close();
+});
+
+test("E4: endConversation is idempotent", async () => {
+  const mind = newMind();
+  const conv = mind.beginConversation();
+  mind.endConversation(conv);
+  // Second end on the same handle — must not throw.
+  mind.endConversation(conv);
+  await mind.store.close();
+});
+
+test("E5: respondTurn on an ended conversation throws", async () => {
+  const mind = newMind();
+  const conv = mind.beginConversation();
+  mind.endConversation(conv);
+  await assert.rejects(
+    () => mind.respondTurn(conv, "hello"),
+    /not found/,
+  );
+  await mind.store.close();
+});
+
+test("E6: conversationState returns null after endConversation", async () => {
+  const mind = newMind();
+  const conv = mind.beginConversation();
+  mind.endConversation(conv);
+  assert.equal(mind.conversationState(conv), null);
+  await mind.store.close();
+});
+
+test("E7: multiple concurrent conversations are independent", async () => {
+  const mind = newMind();
+  await teachConversation(mind, CAT);
+  await teachConversation(mind, DOG);
+
+  const catConv = mind.beginConversation();
+  const dogConv = mind.beginConversation();
+
+  // Feed the distinguishing context to each.
+  await mind.respondTurn(catConv, "I adopted a cat");
+  await mind.respondTurn(dogConv, "I adopted a dog");
+
+  // The same pivot turn — different answers.
+  const catR = await mind.respondTurnText(catConv, "what is its name?");
+  const dogR = await mind.respondTurnText(dogConv, "what is its name?");
+
+  assert.equal(catR.response, "her name is Whiskers");
+  assert.equal(dogR.response, "his name is Rex");
+  assert.notEqual(catR.response, dogR.response);
+
+  // Each conversation has its own boundaries.
+  assert.equal(catR.state.boundaries.length, 1);
+  assert.equal(dogR.state.boundaries.length, 1);
+
+  // Ending one does not affect the other.
+  mind.endConversation(catConv);
+  assert.equal(mind.conversationState(catConv), null);
+  assert.notEqual(mind.conversationState(dogConv), null);
+
+  // End the second — both are now gone.
+  mind.endConversation(dogConv);
+  assert.equal(mind.conversationState(dogConv), null);
+
+  // Ending again is idempotent.
+  mind.endConversation(dogConv);
+  await mind.store.close();
+});
+
+test("E8: respondTurn with a forged handle throws", async () => {
+  const mind = newMind();
+  // { id: 999 } was never returned by beginConversation.
+  await assert.rejects(
+    () => mind.respondTurn({ id: 999 }, "hello"),
+    /not found/,
+  );
+  await mind.store.close();
+});
+
+test("E9: respond and respondTurn give the same answer for the same cumulative bytes", async () => {
+  const mind = newMind();
+  await teachConversation(mind, CAT);
+
+  // Via respond() — raw concatenation, same as respondTurn.
+  const viaRespond = await mind.respondText(
+    "I adopted a catwhat is its name?",
+  );
+
+  // Via respondTurn — the new way.
+  const conv = mind.beginConversation();
+  await mind.respondTurnText(conv, "I adopted a cat");
+  const { response: viaConv } = await mind.respondTurnText(
+    conv,
+    "what is its name?",
+  );
+  mind.endConversation(conv);
+
+  assert.equal(viaRespond, viaConv);
+  await mind.store.close();
+});