@hviana/sema 0.1.6 → 0.1.8

package/src/mind/mind.ts

@@ -14,6 +14,8 @@ import { bindSeat, fold, Sema, Space } from "../sema.js";
 import { Alphabet } from "../alphabet.js";
 import {
   bytesToTree,
+  bytesToTreePyramid,
+  type FoldPyramid,
   Grid,
   gridToTree,
   hilbertBytes,
@@ -23,6 +25,7 @@ import {
 import { BoundedMap, type Store } from "../store.js";
 import { SQliteStore } from "../store-sqlite.js";
 import { type MindConfig, resolveConfig } from "../config.js";
+import { type Canon, canonHash, textCanon } from "../canon.js";
 import {
   type CandidateSpan,
   coverSequence,
@@ -68,6 +71,53 @@ export interface Response {
   provenance?: import("./pipeline.js").Provenance;
 }
 
+/** Serializable state of a conversation — can be saved and restored across
+ *  sessions.  The Mind never interprets the bytes; it only tracks their
+ *  cumulative lengths so the caller can reconstruct turn boundaries later
+ *  without inspecting content. */
+export interface ConversationState {
+  /** The accumulated context bytes — raw concatenation of every turn's
+   *  bytes in order.  No separator is inserted; the boundary offsets
+   *  ({@link boundaries}) tell the caller where each turn ends. */
+  context: Uint8Array;
+  /** Cumulative byte length after each completed turn.  Sorted, strictly
+   *  increasing, each {@code < context.length}.  The first turn's length
+   *  is `boundaries[0]`; the second turn starts at that offset, and so
+   *  on.  Empty for a single-turn or new conversation. */
+  boundaries: number[];
+}
+
+/** An active conversation handle.  Opaque — interact through the Mind's
+ *  conversation methods ({@link Mind.beginConversation},
+ *  {@link Mind.respondTurn}, {@link Mind.endConversation}). */
+export interface Conversation {
+  readonly id: number;
+}
+
+/** Internal per-conversation state.
+ *
+ *  The {@link pyramid} IS the conversation — the accumulated internal
+ *  processing state.  {@link bytes} is the raw accumulated input, kept
+ *  in sync with the pyramid for O(1) concatenation.
+ *
+ *  Memos persist across turns so the inference pipeline does not
+ *  re-process the prefix.  Content-keyed (latin1) — each turn's fresh
+ *  {@code Uint8Array} would never hit an object-identity key.
+ *
+ *  {@link resolvedSubtrees} caches foldTree resolutions at the Sema-node
+ *  level.  When the pyramid reuses prefix subtrees (identical objects),
+ *  foldTree returns their ids immediately — O(suffix) instead of
+ *  O(context) for every tree walk. */
+interface ConversationData {
+  pyramid: FoldPyramid;
+  bytes: Uint8Array;
+  boundaries: number[];
+  perceiveMemo: Map<string, Sema>;
+  recogniseMemo: Map<string, Recognition>;
+  climbMemo: Map<string, Map<string, AttentionRead>>;
+  resolvedSubtrees: WeakMap<Sema, { id: number; len: number }>;
+}
+
 // Mind module imports
 import type { AttentionRead, MindContext, Recognition } from "./types.js";
 import { changedNodes, liftAnswer, spliceAll } from "./types.js";
@@ -75,6 +125,7 @@ import {
   foldTree,
   gistOf,
   inputBytes,
+  latin1Key,
   perceive as perceiveImpl,
   read,
   resolve as resolveImpl,
@@ -110,6 +161,12 @@ export interface MindOptions {
   mechanismFactories?: ((
     host: import("../extension.js").ExtensionHost,
   ) => import("./pipeline-mechanism.js").PipelineMechanism)[];
+  /** Content canonicalizer applied to EVERY response (any modality) for
+   *  equivalence-class resolution — see src/canon.ts.  Text entry points
+   *  ({@link Mind.respondText}, {@link Mind.respondTurnText}) inject the
+   *  Unicode text canonicalizer automatically when this is unset; pass
+   *  `false` to disable canonical resolution everywhere. */
+  canon?: Canon | false;
 }
 
 // ═══════════════════════════════════════════════════════════════════════════
@@ -132,19 +189,36 @@ export class Mind implements MindContext {
   /** The live rationale tracer for the inference currently in flight, or null. */
   trace: Rationale | null = null;
 
-  /** Per-response memo of the consensus climb.  NOTE: this memo and
-   *  {@link recogniseMemo} are BYPASSED while a rationale trace is attached
-   *  (every mechanism must emit its own steps), so a traced respond re-pays
-   *  up to four consensus climbs plus repeat recognitions — that is where the
-   *  traced-vs-untraced latency multiple comes from, by design. */
-  climbMemo: WeakMap<Uint8Array, Map<string, AttentionRead>> | null = null;
+  /** The content canonicalizer for the response in flight — see
+   *  {@link MindContext.canon}.  Injected per response by the modality entry
+   *  point; null when the response carries no equivalence. */
+  canon: Canon | null = null;
+
+  /** Per-response canonical-resolution memo — see {@link MindContext.canonMemo}. */
+  canonMemo: Map<string, number | null> | null = null;
+
+  /** The Mind-level canon option: a canonicalizer to use for EVERY response,
+   *  `false` to disable canonical resolution, or null to let each entry
+   *  point decide (text entry points inject {@link textCanon}). */
+  private _canonOpt: Canon | false | null = null;
+
+  /** Memo of the consensus climb — content-keyed.  See {@link MindContext.climbMemo}. */
+  climbMemo: Map<string, Map<string, AttentionRead>> | null = null;
 
-  /** Per-response memo of recognise() — see {@link MindContext.recogniseMemo}. */
-  recogniseMemo: WeakMap<Uint8Array, Recognition> | null = null;
+  /** Memo of recognise() — content-keyed.  See {@link MindContext.recogniseMemo}. */
+  recogniseMemo: Map<string, Recognition> | null = null;
 
-  /** Per-response memo of perceive() — see {@link MindContext.perceiveMemo}. */
+  /** Memo of perceive() — content-keyed.  See {@link MindContext.perceiveMemo}. */
   perceiveMemo: Map<string, import("../sema.js").Sema> | null = null;
 
+  /** Subtree-resolution cache.  See {@link MindContext._resolvedSubtrees}. */
+  _resolvedSubtrees:
+    | WeakMap<
+      import("../sema.js").Sema,
+      { id: number; len: number }
+    >
+    | null = null;
+
   /** The perceived gist of the query currently being answered.  Set by `think`
    *  before the graph search runs; `chooseNext` consults it as a gate (a null
    *  guide means no query is in flight, so structural walkers keep plain
@@ -176,6 +250,11 @@ export class Mind implements MindContext {
   _depositLens = new Set<number>();
   _internIds = new WeakMap<import("../sema.js").Sema, number>();
 
+  // ── Conversation state ──────────────────────────────────────────────────
+
+  private _nextConvId = 1;
+  private _conversations = new Map<number, ConversationData>();
+
   // ── GraphSearchHost implementation ─────────────────────────────────────
 
   /** Canonical node id of a byte span.  Required by GraphSearchHost & MindContext. */
@@ -226,8 +305,10 @@ export class Mind implements MindContext {
         store: optsStore,
         mechanisms: userMechs,
         mechanismFactories: userFacts,
+        canon: optsCanon,
         ...rest
       } = (optsOrCfg ?? {}) as MindOptions;
+      this._canonOpt = optsCanon ?? null;
       this.cfg = resolveConfig(rest as Partial<MindConfig>);
       this.store = optsStore ?? new SQliteStore({
         maxGroup: this.cfg.geometry.maxGroup,
@@ -319,11 +400,24 @@ export class Mind implements MindContext {
   /** Open one response's transient state — the tracer and the per-response
    *  memos.  Paired with {@link endResponse}; the ONE place this state is
    *  created, so adding a memo cannot forget its reset. */
-  private beginResponse(inspectRationale?: InspectRationale): void {
+  private beginResponse(
+    inspectRationale?: InspectRationale,
+    canon?: Canon | null,
+  ): void {
     this.trace = inspectRationale ? new Rationale(inspectRationale) : null;
-    this.climbMemo = new WeakMap();
-    this.recogniseMemo = new WeakMap();
+    this.climbMemo = new Map();
+    this.recogniseMemo = new Map();
     this.perceiveMemo = new Map();
+    this.canon = canon ?? null;
+    this.canonMemo = canon ? new Map() : null;
+  }
+
+  /** The canonicalizer a response should carry: the Mind-level option when
+   *  set (or none when explicitly disabled), else the entry point's own
+   *  default — text entry points pass {@link textCanon}, binary ones null. */
+  private _canonFor(entryDefault: Canon | null): Canon | null {
+    if (this._canonOpt === false) return null;
+    return this._canonOpt ?? entryDefault;
   }
 
   /** Close one response's transient state — every per-response field, incl.
@@ -333,28 +427,33 @@ export class Mind implements MindContext {
     this.climbMemo = null;
     this.recogniseMemo = null;
     this.perceiveMemo = null;
+    this.canon = null;
+    this.canonMemo = null;
     this._edgeGuide = null;
     this._edgeChoice.clear();
   }
 
-  async respond(
-    input: Input,
+  /** Shared response core — the one path from bytes to voiced answer.
+   *  `respond` calls this directly; `respondTurn` has its own path
+   *  with conversation-persistent memos and incremental perception. */
+  private async _respondImpl(
+    queryBytes: Uint8Array,
     inspectRationale?: InspectRationale,
+    traceLabel = "respond",
+    canon: Canon | null = null,
   ): Promise<Response> {
-    this.beginResponse(inspectRationale);
+    this.beginResponse(inspectRationale, canon);
     try {
-      const inBytes = inputBytes(this, input);
-      const top = this.trace?.enter("respond", [
-        rItem(inBytes, "query"),
+      const top = this.trace?.enter(traceLabel, [
+        rItem(queryBytes, "query"),
       ]);
-
-      const thought = await think(this, inBytes, this.mechanisms);
+      const thought = await think(this, queryBytes, this.mechanisms);
       if (thought === null) {
         top?.done([], "nothing to perceive or an empty store — no answer");
         return { v: null, bytes: new Uint8Array(0) };
       }
 
-      const voiced = await articulate(this, thought.bytes, inBytes);
+      const voiced = await articulate(this, thought.bytes, queryBytes);
       top?.done(
         [rItem(voiced, "answer", resolveImpl(this, voiced) ?? undefined)],
         "the answer, re-voiced in the asker's words",
@@ -369,10 +468,31 @@ export class Mind implements MindContext {
     }
   }
 
+  async respond(
+    input: Input,
+    inspectRationale?: InspectRationale,
+  ): Promise<Response> {
+    // A STRING input is text by nature: it carries the text equivalence even
+    // through the generic entry point.  Raw bytes / grids carry only the
+    // Mind-level canon option, if any.
+    const canon = this._canonFor(typeof input === "string" ? textCanon : null);
+    return this._respondImpl(
+      inputBytes(this, input),
+      inspectRationale,
+      "respond",
+      canon,
+    );
+  }
+
   /** Text view of {@link respond}.  NUL bytes (0x00) are stripped before
    *  decoding — they are structural padding in text answers.  LOSSY for a
    *  binary answer that legitimately contains NULs: use {@link respond} and
-   *  read `bytes` directly for binary/grid modalities. */
+   *  read `bytes` directly for binary/grid modalities.
+   *
+   *  Injects the TEXT canonicalizer (src/canon.ts) so resolution treats
+   *  every character variation of the same text — case, width, whitespace —
+   *  as one form, provided the store's canon index is built
+   *  ({@link buildCanonIndex}). */
   async respondText(
     input: string,
     inspectRationale?: InspectRationale,
@@ -381,6 +501,220 @@ export class Mind implements MindContext {
     return decodeText(r.bytes);
   }
 
+  // ── Conversation API ────────────────────────────────────────────────────
+
+  /** Begin a new conversation, optionally restoring from a previously-saved
+   *  {@link ConversationState}.  The returned handle is required for
+   *  {@link respondTurn} and {@link endConversation}.
+   *
+   *  Conversations are independent — a Mind can manage several concurrently.
+   *  Each tracks the fold pyramid (accumulated internal processing) and
+   *  turn-boundary offsets; the geometry never inspects content to guess
+   *  where one turn ends and the next begins. */
+  beginConversation(state?: ConversationState): Conversation {
+    const id = this._nextConvId++;
+    // Build the initial pyramid: from scratch for a new conversation,
+    // or from the saved context bytes when restoring.
+    const initBytes = state?.context ?? new Uint8Array(0);
+    const { pyramid } = bytesToTreePyramid(
+      this.space,
+      this.alphabet,
+      initBytes,
+    );
+    this._conversations.set(id, {
+      pyramid,
+      bytes: initBytes,
+      boundaries: state?.boundaries ? [...state.boundaries] : [],
+      perceiveMemo: new Map(),
+      recogniseMemo: new Map(),
+      climbMemo: new Map(),
+      resolvedSubtrees: new WeakMap(),
+    });
+    return { id };
+  }
+
+  /** End a conversation, releasing its internal resources (accumulated
+   *  context, boundary offsets, and the fold-pyramid cache).  Idempotent. */
+  endConversation(conv: Conversation): void {
+    this._conversations.delete(conv.id);
+  }
+
+  /** The current serialisable state of an active conversation.  Save this
+   *  to resume the conversation later via {@link beginConversation}. */
+  conversationState(conv: Conversation): ConversationState | null {
+    const data = this._conversations.get(conv.id);
+    if (!data) return null;
+    return {
+      context: data.bytes,
+      boundaries: [...data.boundaries],
+    };
+  }
+
+  /** Append a turn to a conversation's accumulated context WITHOUT
+   *  responding — raw byte append plus a boundary offset, never a
+   *  separator; the fold pyramid advances by O(turn).
+   *
+   *  This is the primitive for turns the Mind should hear but not answer:
+   *  replaying a transcript, feeding the OTHER speaker's line in a
+   *  prediction harness, or restoring context piecewise.  {@link
+   *  respondTurn} = addTurn + think + its own reply appended the same way. */
+  addTurn(conv: Conversation, turn: Input): ConversationState {
+    const data = this._conversations.get(conv.id);
+    if (!data) throw new Error(`Conversation ${conv.id} not found`);
+    const turnBytes = inputBytes(this, turn);
+    this._growContext(data, turnBytes);
+    return this.conversationState(conv)!;
+  }
+
+  /** Grow a conversation's accumulated context by one turn's bytes — raw
+   *  append plus a boundary offset, pyramid advanced by O(turn), the grown
+   *  context's tree seeded into the conversation's perceive memo.  The ONE
+   *  place a context grows ({@link addTurn} and {@link respondTurn} both
+   *  come through here), so the append semantics cannot drift. */
+  private _growContext(data: ConversationData, turnBytes: Uint8Array): Sema {
+    const prevLen = data.pyramid.bytes;
+    // An empty turn neither grows the context nor marks a boundary —
+    // boundaries are documented strictly increasing, and a zero-length
+    // "turn" is no turn.  The pyramid's zero-growth shortcut makes the
+    // refold below O(1), so the existing tree is returned unchanged.
+    const grow = turnBytes.length > 0;
+    const grown = !grow
+      ? data.bytes
+      : prevLen > 0
+      ? concat2(data.bytes, turnBytes)
+      : turnBytes;
+    if (grow && prevLen > 0) data.boundaries.push(prevLen);
+    const { tree, pyramid } = bytesToTreePyramid(
+      this.space,
+      this.alphabet,
+      grown,
+      data.pyramid,
+    );
+    data.pyramid = pyramid;
+    data.bytes = grown;
+    data.perceiveMemo.set(latin1Key(grown), tree);
+    return tree;
+  }
+
+  /** Process one turn of a conversation.
+   *
+   *  `turn` is the raw input for the latest turn — its bytes are appended
+   *  to the accumulated context directly (raw concatenation).  The Mind
+   *  tracks the byte offset where each turn ends; no separator is ever
+   *  inserted or inspected.
+   *
+   *  Returns the response AND the updated {@link ConversationState} so the
+   *  caller can persist it.  The conversation handle's internal state is
+   *  updated in place — the returned state is a snapshot for storage.
+   *
+   *  SINGLE FLIGHT: at most one respondTurn may be in flight per Mind.  The
+   *  conversation's memo caches are swapped into the Mind-level per-response
+   *  pointers for the duration of the turn, so a concurrently-running
+   *  respond()/respondTurn() on the SAME Mind would interleave state.
+   *  Different Minds (or sequential awaits, as in every test) are safe. */
+  async respondTurn(
+    conv: Conversation,
+    turn: Input,
+    inspectRationale?: InspectRationale,
+  ): Promise<{ response: Response; state: ConversationState }> {
+    const data = this._conversations.get(conv.id);
+    if (!data) throw new Error(`Conversation ${conv.id} not found`);
+
+    const turnBytes = inputBytes(this, turn);
+    // Incremental perception — O(turn) instead of O(context).
+    const tree = this._growContext(data, turnBytes);
+    const newContext = data.bytes;
+
+    // Swap in the conversation's persistent state so the inference
+    // pipeline does not re-process the prefix from scratch.
+    //   perceiveMemo / recogniseMemo / climbMemo — content-keyed;
+    //     the prefix's results from the previous turn are found by
+    //     the current turn's sub-span calls.
+    //   _resolvedSubtrees — Sema-node-keyed; when the pyramid reuses
+    //     prefix subtrees (identical objects), foldTree returns their
+    //     ids immediately — O(suffix) instead of O(context).
+    this.perceiveMemo = data.perceiveMemo;
+    this.recogniseMemo = data.recogniseMemo;
+    this.climbMemo = data.climbMemo;
+    this._resolvedSubtrees = data.resolvedSubtrees;
+    this.trace = inspectRationale ? new Rationale(inspectRationale) : null;
+    // A string turn is text by nature — carry the text equivalence, same as
+    // respond() (see _canonFor).
+    this.canon = this._canonFor(typeof turn === "string" ? textCanon : null);
+    this.canonMemo = this.canon ? new Map() : null;
+
+    try {
+      const top = this.trace?.enter("respondTurn", [
+        rItem(newContext, "query"),
+      ]);
+
+      const thought = await think(this, newContext, this.mechanisms);
+      if (thought === null) {
+        top?.done([], "nothing to perceive or an empty store — no answer");
+        return {
+          response: { v: null, bytes: new Uint8Array(0) },
+          state: this.conversationState(conv)!,
+        };
+      }
+
+      const voiced = await articulate(this, thought.bytes, newContext);
+      top?.done(
+        [rItem(voiced, "answer", resolveImpl(this, voiced) ?? undefined)],
+        "the answer, re-voiced in the asker's words",
+      );
+
+      // The REPLY joins the accumulated context the same way a turn does
+      // ({@link addTurn}): raw byte append plus a boundary offset — never a
+      // separator.  A conversation's context is the full exchange, exactly
+      // the cumulative continuous shape multi-turn training deposits, so a
+      // later turn can refer to what was ANSWERED ("which of those two…"),
+      // not only to what was asked.
+      if (voiced.length > 0) this.addTurn(conv, voiced);
+
+      return {
+        response: {
+          v: gistOf(this, voiced),
+          bytes: voiced,
+          provenance: thought.provenance,
+        },
+        state: this.conversationState(conv)!,
+      };
+    } finally {
+      // No save-back: the conversation's memo MAPS were mutated in place —
+      // `data.*` still points at them.  Re-assigning from the Mind-level
+      // pointers here was a no-op in the single-flight case and, under a
+      // concurrently-started respond() (which swaps its own fresh maps into
+      // those pointers), would inject a FOREIGN response's memos into this
+      // conversation.  Clear Mind references — non-conversation respond()
+      // calls get fresh per-response memos.
+      this.trace = null;
+      this.perceiveMemo = null;
+      this.recogniseMemo = null;
+      this.climbMemo = null;
+      this.canon = null;
+      this.canonMemo = null;
+      this._resolvedSubtrees = null;
+      this._edgeGuide = null;
+      this._edgeChoice.clear();
+    }
+  }
+
+  /** Text view of {@link respondTurn}.  See {@link respondText} for the
+   *  NUL-stripping caveat.  For binary or grid turns use {@link respondTurn}
+   *  directly — this is a text-only convenience, like {@link respondText}. */
+  async respondTurnText(
+    conv: Conversation,
+    turn: string,
+    inspectRationale?: InspectRationale,
+  ): Promise<{ response: string; state: ConversationState }> {
+    const { response, state } = await this.respondTurn(
+      conv,
+      turn,
+      inspectRationale,
+    );
+    return { response: decodeText(response.bytes), state };
+  }
+
   async embedding(input: Input): Promise<Vec | null> {
     return (await this.respond(input)).v;
   }
@@ -463,6 +797,46 @@ export class Mind implements MindContext {
     );
   }
 
+  // ── Canonical-form index ───────────────────────────────────────────────
+
+  /** Build (or incrementally refresh) the store's canonical-form index: for
+   *  every content-bearing node, record the hash of its CANONICAL key so
+   *  resolution can find stored forms across surface variation (case, width,
+   *  whitespace — whatever `canon` equates; see src/canon.ts).
+   *
+   *  Incremental and idempotent: the last indexed node id is remembered in
+   *  store meta (`canon.upto`), so a refresh after further training scans
+   *  only the new rows.  Run once after training, and again after ingests —
+   *  the same operational shape as {@link repairContentIndex}.
+   *
+   *  @param canon  the canonicalizer to index under — MUST be the same one
+   *                queries will carry (text queries carry {@link textCanon}
+   *                unless the Mind was constructed with its own)
+   *  @returns number of index rows added */
+  async buildCanonIndex(canon?: Canon): Promise<number> {
+    const c = canon ?? this._canonFor(textCanon);
+    const store = this.store;
+    if (c === null || !store.canonAdd || !store.eachContent) return 0;
+    const from = Number(await store.getMeta("canon.upto") ?? 0);
+    let added = 0;
+    let maxId = from - 1;
+    store.eachContent((id, bytes) => {
+      if (id > maxId) maxId = id;
+      const key = c(bytes);
+      if (key.length === 0) return;
+      // Only index content whose canonical key DIFFERS from its raw bytes —
+      // an already-canonical span is found by the exact lookup (and by the
+      // fallback's own exact probe of the canonical bytes), so indexing it
+      // would only add rows.
+      if (bytesEqual(key, bytes)) return;
+      store.canonAdd!(canonHash(key), id);
+      added++;
+    }, from);
+    await store.setMeta("canon.upto", String(maxId + 1));
+    store.commit();
+    return added;
+  }
+
   // ── Persistence ──────────────────────────────────────────────────────────
 
   async save(): Promise<Uint8Array> {