@hviana/sema 0.1.0

package/src/ingest-cache.ts

@@ -0,0 +1,224 @@
+// ingest-cache.ts — ingest-level cache for Mind.
+//
+// Wraps a Mind and memoises the expensive, content-determined half of ingestion:
+// turning ONE input (a context, a continuation, or a bare experience) into its
+// interned graph nodes.  Perception (riverFold) and interning are a PURE function
+// of the input bytes — the same bytes always cut the same way and hash-cons to
+// the same node ids — so the result can be cached by content and reused wherever
+// those same bytes appear again, in ANY role.
+//
+// This is why the cache keys on the single input, not on the (context,
+// continuation) pair: real training corpora repeat content at the INPUT level far
+// more than at the pair level — a shared system/tool preamble in front of every
+// row, a common answer ("Yes.", a refusal) under many questions, an accumulated
+// dialogue prefix, an exactly-duplicated row.  A pair-level key sees none of these
+// (the pair as a whole is almost always unique); the input-level key catches them
+// all, and a context that recurs as a continuation (or vice versa) hits too.
+//
+// On a cache hit only the cheap relational writes are replayed — the edge and the
+// two halo pours for a pair, or the part-chain links for a single — while the
+// perceive + internTree work (the dominant cost, and on a large store the dominant
+// VECTOR-INDEX cost too) is skipped entirely.  The interned node ids are stable
+// across eviction: a re-perceive of evicted content reproduces the same ids by
+// the store's hash-cons, so correctness never depends on a hit.
+//
+// DEPOSITION IS THE MIND'S OWN: every miss goes through the same
+// {@link deposit} the direct Mind.ingest path uses — perception, interning,
+// sub-span/window indexing, durable CONTAINMENT edges, whole-stream flat
+// branch, changed-node tracking against the Mind's `_prevSeen` — so a store
+// trained through this cache is structurally IDENTICAL to one trained by
+// calling mind.ingest() directly (this class used to re-implement the deposit
+// and drifted: no indexSubSpans/addContainer, wrong part-link stride).  Only
+// the memo layer lives here.  Behaviour is identical to mind.ingest() (links
+// and halos are reinforced on every call), just faster when content repeats.
+
+import { Mind } from "./mind/index.js";
+import {
+  deposit,
+  dispatchIngest,
+  ingestOne as depositOne,
+} from "./mind/learning.js";
+import { bindSeat, companySignature, type Sema } from "./sema.js";
+import type { Input } from "./mind/index.js";
+import { BoundedMap } from "./store.js";
+import type { Vec } from "./vec.js";
+
+/** The interned result of perceiving + interning ONE input.
+ *
+ *  Only the root vector and the node ids are kept — the full Sema tree (hundreds
+ *  of nodes) is GC'd.  `rootV` is needed for the halo binds (a pair seats each
+ *  side's gist into the other's halo) and for the Sema returned by a single
+ *  ingest; `partIds` are the root's immediate children, the chain a single
+ *  experience links together (`[rootId]` when the root is itself a leaf). */
+interface Interned {
+  rootV: Vec;
+  rootId: number;
+  partIds: number[];
+  /** Byte cost of the memo KEY this entry is stored under.  The key of a
+   *  string input is the whole input ("S:" + text), so on long continuations
+   *  it dwarfs the value — charging it to the LRU budget is what keeps the
+   *  memo's real memory at ingestCacheBytes instead of a multiple of it. */
+  keyBytes: number;
+}
+
+/**
+ * An ingest cache layered over a Mind.
+ *
+ * Usage (drop-in for mind.ingest in training loops):
+ *   const ci = new CachedIngest(mind);
+ *   await ci.ingest(ctx, cont);       // caches both sides
+ *   await ci.ingest("some text");     // caches the single text
+ *   await ci.ingest([[ctx, cont]]);   // array form
+ *
+ * All other Mind methods (respond, save, …) are accessed on `ci.mind`.
+ */
+export class CachedIngest {
+  readonly mind: Mind;
+
+  /** One unified, content-addressed memo of interned inputs.  Each value holds
+   *  Float32 root vector (D·4 bytes) plus a small id array, so the byte budget
+   *  is spent on the inputs most worth remembering (LRU). */
+  private _memo!: BoundedMap<string, Interned>;
+
+  hits = 0;
+  misses = 0;
+
+  constructor(mind: Mind) {
+    this.mind = mind;
+    this._memo = this.newMemo();
+  }
+
+  private newMemo(): BoundedMap<string, Interned> {
+    return new BoundedMap<string, Interned>(
+      this.mind.cfg.store.ingestCacheBytes,
+      (c) => c.rootV.byteLength + c.partIds.length * 4 + 8 + c.keyBytes,
+    );
+  }
+
+  // ── cache key ───────────────────────────────────────────────────────────
+  // Content key for an input, or null when the input cannot be keyed reliably
+  // — in which case it BYPASSES the cache (recomputed every time) rather than
+  // risk a collision returning the wrong nodes.  A string is its own key; raw
+  // bytes hash by FNV-1a (+ length, so a hash collision still needs an equal
+  // length to alias); a Grid / Grid[] has no cheap stable key, so it bypasses.
+
+  private keyOf(input: Input): string | null {
+    if (typeof input === "string") return "S:" + input;
+    if (input instanceof Uint8Array) {
+      // FNV-1a — no spread onto the call stack, safe on large binary inputs.
+      let h = 0x811c9dc5 >>> 0;
+      for (let i = 0; i < input.length; i++) {
+        h ^= input[i];
+        h = Math.imul(h, 0x01000193) >>> 0;
+      }
+      return "B:" + (h >>> 0).toString(16) + ":" + input.length;
+    }
+    return null; // Grid / Grid[] — bypass the cache
+  }
+
+  // ── public API ────────────────────────────────────────────────────────
+
+  async ingest(
+    input: Input | (Input | [Input, Input])[],
+    second?: Input,
+  ): Promise<(Sema & { id: number }) | undefined> {
+    // One shape-reading for both ingest paths — see {@link dispatchIngest}.
+    return dispatchIngest(
+      input,
+      second,
+      (i) => this.ingestOne(i),
+      (a, b) => this.ingestPair(a, b),
+    );
+  }
+
+  // ── the one cached step: perceive + intern + index ONE input ──────────────
+
+  /** Resolve an input to its interned nodes, from the memo when its content has
+   *  been seen before, else by the Mind's OWN untracked {@link deposit}
+   *  (perceive + intern + sub-span/containment indexing — identical to the
+   *  direct path; those writes are durable and idempotent, so a later memo hit
+   *  legitimately skips them).  This is the single expensive operation both
+   *  ingest paths share; caching it here is what makes every repeated input —
+   *  in any role — cheap. */
+  private async resolveInput(input: Input): Promise<Interned> {
+    const key = this.keyOf(input);
+    if (key !== null) {
+      const cached = this._memo.get(key);
+      if (cached) {
+        this.hits++;
+        return cached;
+      }
+    }
+    this.misses++;
+
+    // The full deposit, untracked (a continuation must not break the tracked
+    // deposit chain).  `partIds` are the root's immediate children, already
+    // interned by the deposit — free to capture.  Only the root vector and
+    // ids are retained; the tree is GC'd.
+    const { tree, rootId, ids } = await deposit(this.mind, input, false);
+    const partIds: number[] = tree.kids
+      ? tree.kids.map((k) => ids.get(k)!)
+      : [rootId];
+
+    const entry: Interned = {
+      rootV: tree.v,
+      rootId,
+      partIds,
+      // ~2 bytes per UTF-16 code unit — the key is retained by the map.
+      keyBytes: key !== null ? key.length * 2 : 0,
+    };
+    if (key !== null) this._memo.set(key, entry);
+    return entry;
+  }
+
+  // ── pair ingest ───────────────────────────────────────────────────────
+
+  private async ingestPair(ctx: Input, cont: Input): Promise<void> {
+    // The CONTEXT side is TRACKED (it continues the Mind's deposit chain and
+    // is the growing, unique-per-turn side — a cache miss anyway), via the
+    // same {@link deposit} the direct path uses; the continuation resolves
+    // through the memo.
+    const c = await deposit(this.mind, ctx, true);
+    const b = await this.resolveInput(cont);
+
+    // EDGE on the FULL context (disambiguates a shared pivot). HALO only on the
+    // CHANGED NODES — the coherent new subtrees this turn adds — so a recurring
+    // shared prefix never collects halo mass. (See Mind.ingestPair.)
+    await this.mind.store.link(c.rootId, b.rootId);
+    // Halos pour company SIGNATURES (identity), not gists (content) — see
+    // companySignature in sema.ts.
+    const contSeat = bindSeat(
+      this.mind.space,
+      companySignature(this.mind.space, b.rootId),
+      1,
+    );
+    for (const part of c.changed) {
+      const partId = c.ids.get(part)!;
+      await this.mind.store.pourHalo(partId, contSeat);
+      await this.mind.store.pourHalo(
+        b.rootId,
+        bindSeat(this.mind.space, companySignature(this.mind.space, partId), 0),
+      );
+    }
+  }
+
+  // ── one-side ingest ───────────────────────────────────────────────────
+
+  private async ingestOne(input: Input): Promise<Sema & { id: number }> {
+    // A bare experience is the direct path verbatim — tracked deposit, root
+    // marked a resonance target, part chain linked at the maxGroup stride.
+    // (The tracked side is a memo miss by design, so there is nothing for the
+    // cache to add here.)
+    return depositOne(this.mind, input);
+  }
+
+  // ── helpers ───────────────────────────────────────────────────────────
+
+  clear(): void {
+    this._memo = this.newMemo();
+  }
+
+  get size(): number {
+    return this._memo.size;
+  }
+}

package/src/mind/articulation.ts

@@ -0,0 +1,137 @@
+// articulation.ts — re-voice an answer in the asker's wording (Section 5).
+//
+//   articulate — substitute answer forms with the asker's synonyms,
+//                using concept (halo) resonance to match the voices.
+
+import { Vec } from "../vec.js";
+import type { MindContext } from "./types.js";
+import { spliceAll } from "./types.js";
+import { recognise } from "./recognition.js";
+import { contains } from "./traverse.js";
+import { bestHaloMate } from "./match.js";
+import type { Site } from "./graph-search.js";
+import type { CandidateSpan } from "../derive/src/index.js";
+import { coverSequence } from "../derive/src/index.js";
+import { rItem, rNode, traceDerivation } from "./trace.js";
+
+/** Re-voice an answer in the asker's own words.  For each recognised form
+ *  in the answer, find a concept-sibling in the query (by halo resonance)
+ *  and substitute the asker's wording.  The search's own cover mechanism
+ *  splices the substitutes into the answer exactly where the forms sit. */
+export async function articulate(
+  ctx: MindContext,
+  answer: Uint8Array,
+  query: Uint8Array | null,
+): Promise<Uint8Array> {
+  if (!query || query.length < 2 || answer.length < 2) return answer;
+  const store = ctx.store;
+  const t = ctx.trace?.enter("articulate", [
+    rItem(answer, "answer"),
+    rItem(query, "query"),
+  ]);
+  const keep = (note: string): Uint8Array => {
+    t?.done([rItem(answer, "answer")], note);
+    return answer;
+  };
+
+  const qCandidates: Array<
+    CandidateSpan<{ halo: Vec; bytes: Uint8Array; node: number }>
+  > = [];
+  for (const s of recognise(ctx, query).sites) {
+    // payload < 0 is a SINGLE-BYTE leaf (byte leaves occupy the implicit
+    // negative id range −256…−1 — see store.ts).  A one-byte form's halo is
+    // promiscuous (it keeps company with everything it ever neighboured), so
+    // admitting it as a voice or a revoicing target lets e.g. the digit "4"
+    // be "re-voiced" into an unrelated digit the query happens to contain.
+    // Articulation therefore operates on multi-byte forms only.
+    if (s.payload < 0) continue;
+    const h = store.halo(s.payload);
+    if (!h) continue;
+    qCandidates.push({
+      start: s.start,
+      end: s.end,
+      payload: {
+        halo: h,
+        bytes: query.slice(s.start, s.end),
+        node: s.payload,
+      },
+    });
+  }
+  if (qCandidates.length === 0) {
+    return keep("no asker concept to revoice — answer unchanged");
+  }
+  if (coverSequence(query.length, qCandidates).spans.length === 0) {
+    return keep("no asker concept to revoice — answer unchanged");
+  }
+  const voices = qCandidates.map((s) => s.payload);
+
+  const ans = recognise(ctx, answer);
+  const substitutions = new Map<number, Uint8Array>();
+  for (const s of ans.sites) {
+    // Same single-byte-leaf exclusion as the query loop above.
+    if (s.payload < 0) continue;
+    const h = store.halo(s.payload);
+    if (!h) continue;
+    const found = bestHaloMate(ctx, h, voices, (v) => v.halo);
+    if (!found) continue;
+    const voice = found.item;
+    if (voice.node === s.payload || contains(ctx, voice.node, s.payload)) {
+      continue;
+    }
+    substitutions.set(s.payload, voice.bytes);
+  }
+  if (substitutions.size === 0) {
+    return keep("no answer form is a synonym of an asker concept");
+  }
+  ctx.trace?.step(
+    "substitute",
+    [...substitutions.keys()].map((n) => rNode(ctx, n, "answer-form")),
+    [...substitutions.values()].map((b) => rItem(b, "asker-voice")),
+    `revoice ${substitutions.size} answer form(s) in the asker's own words (synonym splice)`,
+  );
+
+  const voicedSites = ans.sites.filter((s) => substitutions.has(s.payload));
+  const tArtCover = ctx.trace?.enter("cover", [
+    ...voicedSites.map((s) =>
+      rItem(answer.subarray(s.start, s.end), "form", s.payload, [
+        s.start,
+        s.end,
+      ])
+    ),
+  ]);
+  const solved = ctx.search.cover(
+    answer.length,
+    voicedSites,
+    new Map(),
+    ans.leaves,
+    ans.splits,
+    substitutions,
+    undefined,
+    undefined,
+    ctx.trace ? (steps) => traceDerivation(ctx, steps) : undefined,
+  );
+  const segs = solved && solved.segs;
+  tArtCover?.done(
+    segs === null
+      ? []
+      : segs.map((s) =>
+        rItem(s.bytes, s.rec ? "voiced" : "kept", s.node, [s.i, s.j])
+      ),
+    segs === null
+      ? "the revoiced cover did not compose"
+      : "lightest derivation of the revoiced answer",
+  );
+  const result = segs && spliceAll(segs);
+  if (result) {
+    t?.done(
+      [rItem(
+        result,
+        "voiced",
+        ctx.store.findLeaf(result.subarray(0, 1)) ?? undefined,
+      )],
+      "splice the asker's wording into the answer where the same concept appears",
+    );
+    return result;
+  }
+  return keep("the revoiced cover did not compose — answer unchanged");
+}