@hviana/sema 0.4.0 → 0.4.2

package/src/mind/traverse.ts

@@ -10,9 +10,9 @@ import { cosine, Vec } from "../vec.js";
 import type { AncestorReach, MindContext, SaturationStop } from "./types.js";
 import { gistOf, read } from "./primitives.js";
 
-// ── Per-response structural memo ────────────────────────────────────────
+// ── Session structural memo ─────────────────────────────────────────────
 //
-// Within one respond() the store is read-only, so structural reads are pure
+// Between ingests the store is read-only, so structural reads are pure
 // functions of the node id.  edgeAncestors climbs different start nodes that
 // share ancestry (regions sharing a chunk, canonicalChunkId's prefix probes)
 // — the reachMemo already caches whole-climb results, but a node visited as an
@@ -42,21 +42,17 @@ const structCaches = new WeakMap<object, StructCache>();
 // every mechanism that prices commonality" — was reached only by confluence.
 // The climb is by far the biggest consumer.
 //
-// Keyed off `ctx.climbMemo`'s OBJECT IDENTITY, exactly like the struct cache
-// above, which buys the right lifetime for free: a plain respond() has a
-// fresh climbMemo, so the memo is response-scoped; a conversation turn has
-// the conversation's persistent one, so it is conversation-scoped.  That
-// matters — the stable-prefix fold makes each turn's subtree independent of
-// what follows, so 59–70% of a later turn's climb regions are byte-identical
-// repeats of an earlier turn's (measured on a 4-turn session), and every one
-// of them used to re-climb from cold.
+// Keyed by the Mind's structural lifecycle identity: ordinary and
+// conversational asks share it, and every ingest invalidates it. A real
+// battery repeatedly reaches the same corpus scaffolding even when its
+// surface questions differ.
 //
 // Budgeted, not unbounded (AGENTS §2.12): past the cap the whole map is
 // dropped and re-derived, costing a cold climb and never a wrong answer.
 const REACH_MEMO_MAX = 100_000;
 const reachCaches = new WeakMap<object, Map<number, AncestorReach>>();
 
-/** The reach memo this response should use — see the note above.
+/** The reach memo this ask should use — see the note above.
  *
  *  A TRACED response always gets a fresh, empty one.  `AncestorReach`'s
  *  `visited`/`maxDepth`/`saturation` fields are populated only when a trace
@@ -70,18 +66,18 @@ export function sharedReachMemo(
   ctx: MindContext,
 ): Map<number, AncestorReach> {
   if (ctx.trace !== null || ctx.climbMemo === null) return new Map();
-  let m = reachCaches.get(ctx.climbMemo);
-  if (m === undefined) reachCaches.set(ctx.climbMemo, m = new Map());
+  let m = reachCaches.get(ctx._structMemoKey);
+  if (m === undefined) reachCaches.set(ctx._structMemoKey, m = new Map());
   else if (m.size >= REACH_MEMO_MAX) m.clear();
   return m;
 }
 
 function getStructCache(ctx: MindContext): StructCache | null {
   if (ctx.climbMemo === null) return null;
-  let c = structCaches.get(ctx.climbMemo);
+  let c = structCaches.get(ctx._structMemoKey);
   if (c === undefined) {
     structCaches.set(
-      ctx.climbMemo,
+      ctx._structMemoKey,
       c = {
         hasNext: new Map(),
         prevCount: new Map(),
@@ -92,6 +88,12 @@ function getStructCache(ctx: MindContext): StructCache | null {
   return c;
 }
 
+/** Invalidate every session-lifetime structural read after a write. */
+export function invalidateStructuralCaches(ctx: MindContext): void {
+  reachCaches.delete(ctx._structMemoKey);
+  structCaches.delete(ctx._structMemoKey);
+}
+
 /** Cached {@link Store.hasNext} — pure during one respond(). */
 function cachedHasNext(
   ctx: MindContext,

package/src/mind/types.ts

@@ -279,6 +279,9 @@ export interface MindContext extends GraphSearchHost {
    *  Null outside respond(); during respondTurn() the conversation's
    *  persistent map is swapped in. */
   climbMemo: Map<string, Map<string, AttentionRead>> | null;
+  /** Stable identity for session-lifetime, write-invalidated structural
+   *  caches. Query-level climb results remain on climbMemo. */
+  _structMemoKey: object;
   /** Memo of {@link recognise} — content-keyed (latin1) so recognised
    *  forms carry forward across conversation turns.  Bypassed while a
    *  trace is attached.  Null outside respond(). */
@@ -295,6 +298,13 @@ export interface MindContext extends GraphSearchHost {
    *  O(suffix) instead of O(context).  Mind-lifetime (WeakMap keys are
    *  the Sema objects the pyramid keeps alive). */
   _resolvedSubtrees: WeakMap<Sema, { id: number; len: number }> | null;
+  /** Completed assistant-turn byte spans in the current cumulative query.
+   *  Empty for ordinary respond(); response-scoped structural context for
+   *  mechanisms that must not re-derive already-produced replies. */
+  answeredSpans: ReadonlyArray<readonly [number, number]>;
+  /** Start offset of the user turn currently being answered. Zero for an
+   *  ordinary respond() and for the first turn of a conversation. */
+  currentTurnStart: number;
   _edgeGuide: Vec | null;
   _edgeChoice: Map<number, number>;
   _prevSeen: Set<number> | null;

package/test/49-natural-units-synonym-bridge.test.mjs

@@ -14,20 +14,25 @@
 // entry point to it — the halo system is real and works, it is simply
 // never asked about a node that was never minted.
 //
-// Fix (units.ts): a derived, corpus-statistics-only notion of a "natural
-// unit" — a run of adjacent chunks whose pairing recurs at least as often
-// as either chunk alone (the same principle behind BPE/content-defined
-// chunking, computed from this store's own reuse/containment counts, no
-// injected modality-specific segmenter).  Interned at deposit time,
-// pairwise halo-poured as each other's company; read at query time via the
-// same derived merge (existingUnits) plus a halo-corroborated substitution
-// fallback in recognise().
+// Fix: compose the distributional meaning of an arbitrary byte span from
+// existing canonical-window containment and episode halos. No lexical unit
+// index is introduced: each stored W-window ascends to the learned episodes
+// containing it, their company halos are VSA-bundled, and the bridge admits
+// the substitution only above the existing significanceBar plus its exact
+// alignment/corroboration gates.
 
 import { test } from "node:test";
 import assert from "node:assert/strict";
-import { Mind } from "../dist/src/index.js";
+import {
+  conceptThreshold,
+  cosine,
+  Mind,
+  significanceBar,
+} from "../dist/src/index.js";
 import { SQliteStore } from "../dist/src/store-sqlite.js";
-import { resolve } from "../dist/src/mind/primitives.js";
+import { substitutionBridge } from "../dist/src/mind/bridge.js";
+import { spanSynonymStrength } from "../dist/src/mind/match.js";
+import { perceive, resolve } from "../dist/src/mind/primitives.js";
 
 const enc = (s) => new TextEncoder().encode(s);
 const dec = (b) => new TextDecoder().decode(b).replace(/\0+$/, "");
@@ -55,18 +60,17 @@ const TRAIN = [
   ["What is the biggest island on Earth?", "The biggest island is Greenland."],
 ];
 
-test("baseline: 'biggest'/'largest' never resolve as independent nodes without natural units", async () => {
+test("bare synonym words remain unaddressable without a lexical unit index", async () => {
   const m = new Mind({ seed: 7, store: new SQliteStore({ path: ":memory:" }) });
   await m.ingest(TRAIN);
-  // This assertion documents the ROOT CAUSE (still true even after the
-  // fix — units.ts does not change what resolve() itself returns for a
-  // bare word; it changes what recognise() can bridge to via halos).
+  // The capability composes meaning at read time; it deliberately does not
+  // mint or index modality-specific word nodes.
   assert.equal(resolve(m, enc("biggest")), null);
   assert.equal(resolve(m, enc("largest")), null);
   await m.store.close();
 });
 
-test("irrefutable failure: a trained fact is unreachable through an untrained near-synonym", async () => {
+test("a trained fact is reachable through an untrained near-synonym", async () => {
   const m = new Mind({ seed: 7, store: new SQliteStore({ path: ":memory:" }) });
   await m.ingest(TRAIN);
 
@@ -173,3 +177,40 @@ test(
     );
   },
 );
+
+test("VSA company grounds a differently-spelled synonym without a lexical unit index", async () => {
+  const m = new Mind({ seed: 7, store: new SQliteStore({ path: ":memory:" }) });
+  await m.ingest([
+    ["A physician treats illness.", "A healer treats illness."],
+    ["A doctor treats illness.", "A healer treats illness."],
+    ["The physician named Alice works today.", "Alice is on duty."],
+    ["Which physician is on duty?", "Alice is on duty."],
+  ]);
+
+  const doctor = enc(" doctor");
+  const physician = enc(" physician");
+  const geometric = cosine(
+    perceive(m, doctor).v,
+    perceive(m, physician).v,
+  );
+  const distributional = spanSynonymStrength(m, doctor, physician);
+  assert.ok(
+    geometric < conceptThreshold(m.store.D),
+    "the fixture must not pass through lexical/geometric similarity",
+  );
+  assert.ok(
+    distributional >= significanceBar(m.store.D),
+    "shared episode company must provide significant VSA synonym evidence",
+  );
+
+  const trained = resolve(m, enc("Which physician is on duty?"));
+  assert.notEqual(trained, null);
+  const hit = await substitutionBridge(
+    m,
+    enc("Which doctor is on duty?"),
+    async () => [trained],
+  );
+  assert.equal(hit?.id, trained);
+  assert.ok(hit?.subs.length > 0);
+  await m.store.close();
+});

package/dist/example/demo.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/example/demo.js

@@ -1,39 +0,0 @@
-// demo.ts — one short session that drives the WHOLE pipeline from one memory.
-//
-// We give Sema a handful of plain notes, then ask things that no single note
-// answers. The headline query is the third one: from three worked examples Sema
-// learns the shape of "X was painted by Y", lifts the painter out of a sentence
-// it has NEVER seen, and then — in the same pass — reasons forward to a separate
-// fact about that painter. The reply contains no word from the question. That is
-// retrieval, generalization, and reasoning composing as a single act, with every
-// step traceable back to the notes behind it.
-import { Mind } from "../src/index.js";
-import { SQliteStore } from "../src/store-sqlite.js";
-async function main() {
-    const mind = new Mind({ store: new SQliteStore({ path: ":memory:" }) });
-    const ask = async (q) => (await mind.respondText(q)).trim();
-    // ── Jot down what we know. Each line is just (context → what follows). ──
-    await mind.ingest([
-        // One relation, shown three times — a pattern taught purely by example:
-        ["The Mona Lisa was painted by Leonardo da Vinci.", "Leonardo da Vinci"],
-        ["The Starry Night was painted by Vincent van Gogh.", "Vincent van Gogh"],
-        [
-            "The Night Watch was painted by Rembrandt van Rijn.",
-            "Rembrandt van Rijn",
-        ],
-        // One stray fact, keyed on a name none of the examples mention:
-        ["Pablo Picasso", "Pablo Picasso co-founded the Cubist movement"],
-    ]);
-    // 1) GENERALIZE — apply the learned pattern to an unseen sentence and read out
-    //    the painter. "Pablo Picasso" was never given as an answer; Sema locates it
-    //    by analogy to the three examples.
-    console.log(await ask("The Weeping Woman was painted by Pablo Picasso."));
-    // → "Pablo Picasso co-founded the Cubist movement"
-    //   …and, having found the painter, it KEEPS GOING: the name bridges into the
-    //   one fact it holds about him. The answer appears in no word of the question.
-    // 2) COMPUTE — exact arithmetic, grounded right where the notes go silent.
-    console.log(await ask("a museum charges 12*4 for a family ticket"));
-    // → "48"
-    await mind.store.close();
-}
-main();

package/dist/example/train_base.d.ts

@@ -1,87 +0,0 @@
-export interface Episode {
-    context: string;
-    continuation: string;
-}
-export type TrainingItem = string | Episode;
-/** Dedup + trim a concept's items: drop empty/degenerate pairs and exact
- *  repeats so a concept never deposits the same form twice. */
-export declare function refineItems(items: TrainingItem[]): TrainingItem[];
-/** One normalized SmolSent row. */
-export interface SmolSentRow {
-    src: string;
-    trg: string;
-    sl: string;
-    tl: string;
-}
-/** Normalize a raw datasets-server row into a SmolSentRow, or null when it lacks
- *  both sides or a side is implausibly large (a dump, not a sentence). */
-export declare function toSmolSentRow(row: unknown): SmolSentRow | null;
-/** Translate ONE SmolSent pair into SEMA facts: the two sentences are one
- *  meaning in two languages, so bind them BOTH ways. refineItems drops the
- *  degenerate case where src === trg. */
-export declare function smolSentRowToItems(row: SmolSentRow): TrainingItem[];
-/** One normalized Aya row. */
-export interface AyaRow {
-    inputs: string;
-    targets: string;
-    language: string;
-}
-/** Normalize a raw datasets-server row object into an AyaRow, or null when it
- *  lacks a usable prompt/answer or a field is implausibly large (a dump, not a
- *  cognitive example). Trims surrounding whitespace; keeps inner text verbatim
- *  (human prose, possibly multi-paragraph). */
-export declare function toAyaRow(row: unknown): AyaRow | null;
-/** Translate ONE Aya row into SEMA training items. A row is a single human
- *  (question → answer) exchange — exactly one FACT, the (inputs → targets) edge.
- *  No standalone-answer experience and no one-exchange "cumulative" walk: a lone
- *  Q→A is not multi-turn, and both would only replicate the same edge. */
-export declare function ayaRowToItems(row: AyaRow): TrainingItem[];
-/** A single oasst2 message node (the fields we use; the tree nests via replies). */
-interface OasstNode {
-    role?: string;
-    text?: string;
-    rank?: number | null;
-    deleted?: boolean;
-    replies?: OasstNode[];
-}
-/** One conversational turn extracted from a tree. */
-export interface OasstTurn {
-    role: string;
-    text: string;
-}
-/** Collapse a conversation tree to ONE linear path: at each node, descend into
- *  its best-ranked, non-deleted reply (rank 0 preferred; unranked sorts last).
- *  Returns the ordered turns (already strictly alternating in this corpus). */
-export declare function bestOasstPath(root: OasstNode): OasstTurn[];
-/** Translate ONE multi-turn oasst2 conversation into SEMA training items.
- *
- *  This is the ONE stage where cumulative continuous context is truly necessary:
- *  the data is a real multi-turn dialogue, and what must be learned is how each
- *  turn follows from the WHOLE conversation so far — not from the previous turn
- *  alone. The conversation is emitted ONLY as the accumulated walk; standalone
- *  turn experiences and local adjacent-pair facts are NOT emitted (they are
- *  subsumed by it and would merely replicate the content).
- *
- *  The walk is byte-for-byte the pattern proven in test/13-conversation.test.mjs
- *  ("teachConversation"): each turn is the continuation of all prior turns joined
- *  by "\n", with BARE turn text — NO "User:/Assistant:" labels. Roles already
- *  alternate by position in an oasst2 best-path (the root is a prompter), so a
- *  label adds nothing the position does not, while a clean continuation matches
- *  the test's recall (predictNext queries bare prior turns) and lets a turn share
- *  its gist with the same text elsewhere (e.g. an Aya question stored bare).
- *
- *  Returns [] for a conversation below the multi-turn threshold, so callers can
- *  simply skip empties. */
-export declare function oasstConversationToItems(turns: OasstTurn[]): TrainingItem[];
-/** One normalized General-Knowledge row. */
-export interface GenKnowRow {
-    question: string;
-    answer: string;
-}
-/** Normalize a raw datasets-server row into a GenKnowRow, or null when it lacks
- *  a usable question/answer or a side is implausibly large (corruption). */
-export declare function toGenKnowRow(row: unknown): GenKnowRow | null;
-/** Translate ONE General-Knowledge row into SEMA items: exactly one
- *  (question → answer) FACT. refineItems drops a degenerate question === answer. */
-export declare function genKnowRowToItems(row: GenKnowRow): TrainingItem[];
-export {};