@hviana/sema 0.4.7 → 0.5.0

package/dist/example/train_base.d.ts

@@ -62,9 +62,15 @@ export declare function bestOasstPath(root: OasstNode): OasstTurn[];
  *  turn experiences and local adjacent-pair facts are NOT emitted (they are
  *  subsumed by it and would merely replicate the content).
  *
- *  The walk is byte-for-byte the pattern proven in test/13-conversation.test.mjs
- *  ("teachConversation"): each turn is the continuation of all prior turns joined
- *  by "\n", with BARE turn text — NO "User:/Assistant:" labels. Roles already
+ *  The walk is the pattern proven in test/13-conversation.test.mjs
+ *  ("teachConversation"): each turn is the continuation of all prior turns,
+ *  with BARE turn text — NO "User:/Assistant:" labels.  The SHAPE is identical
+ *  (cumulative context → next turn); the join string is not, and does not need
+ *  to be — that file joins with nothing and this corpus joins with "\n" (see
+ *  `accumulate`).  Saying "byte-for-byte", as this comment used to, invites the
+ *  reading that the two must agree on a separator.  They must not agree,
+ *  because there is nothing to agree about: turn boundaries are offsets, and
+ *  the join string is just corpus text. Roles already
  *  alternate by position in an oasst2 best-path (the root is a prompter), so a
  *  label adds nothing the position does not, while a clean continuation matches
  *  the test's recall (predictNext queries bare prior turns) and lets a turn share

package/dist/example/train_base.js

@@ -594,7 +594,18 @@ const isEpisode = (it) => typeof it !== "string";
 /** Build the accumulated-context episodes of a turn sequence: each successive
  *  turn is the continuation of ALL the turns before it joined together. This is
  *  the same cumulative-context shape a multi-turn conversation deposits, so the
- *  store learns to continue a growing context. */
+ *  store learns to continue a growing context.
+ *
+ *  The "\n" below is a CORPUS choice, not a protocol.  oasst2 turns are
+ *  paragraphs, and reading them back with the newlines kept is how this corpus
+ *  reads naturally; a different corpus may join with nothing, and
+ *  test/13-conversation.test.mjs does exactly that.  Neither has to match the
+ *  other, because Sema never scans content for turn boundaries — those are
+ *  offsets the Conversation API carries beside the bytes (see Mind.addTurn's
+ *  "ON SEPARATORS" note).  The newline here is simply part of the text this
+ *  store learnt, so anything replaying this corpus feeds it back as part of
+ *  the turn: `addTurn(conv, "\n" + turnText)`.  It is not a convention the
+ *  engine, the API, or the tests have to agree on. */
 function accumulate(turns) {
     const out = [];
     for (let i = 1; i < turns.length; i++) {
@@ -709,9 +720,15 @@ export function bestOasstPath(root) {
  *  turn experiences and local adjacent-pair facts are NOT emitted (they are
  *  subsumed by it and would merely replicate the content).
  *
- *  The walk is byte-for-byte the pattern proven in test/13-conversation.test.mjs
- *  ("teachConversation"): each turn is the continuation of all prior turns joined
- *  by "\n", with BARE turn text — NO "User:/Assistant:" labels. Roles already
+ *  The walk is the pattern proven in test/13-conversation.test.mjs
+ *  ("teachConversation"): each turn is the continuation of all prior turns,
+ *  with BARE turn text — NO "User:/Assistant:" labels.  The SHAPE is identical
+ *  (cumulative context → next turn); the join string is not, and does not need
+ *  to be — that file joins with nothing and this corpus joins with "\n" (see
+ *  `accumulate`).  Saying "byte-for-byte", as this comment used to, invites the
+ *  reading that the two must agree on a separator.  They must not agree,
+ *  because there is nothing to agree about: turn boundaries are offsets, and
+ *  the join string is just corpus text. Roles already
  *  alternate by position in an oasst2 best-path (the root is a prompter), so a
  *  label adds nothing the position does not, while a clean continuation matches
  *  the test's recall (predictNext queries bare prior turns) and lets a turn share

package/dist/src/canon.d.ts

@@ -24,3 +24,22 @@ export declare function textCanon(bytes: Uint8Array): Uint8Array;
  *  is keyed on.  Same construction as the node table's content hash; a
  *  collision is resolved by verifying canon(stored) === key, never trusted. */
 export declare function canonHash(key: Uint8Array): number;
+/** The span of `bytes` between its first and last non-whitespace byte — the
+ *  QUESTION, with the caller's edge spacing dropped.  Returns a subarray (no
+ *  copy), and the original when there is nothing to trim.
+ *
+ *  THIS LIVES HERE, not in the core byte utilities, for the reason stated at
+ *  the top of this file: "nothing in the store or the mind's core knows what
+ *  'case' or 'whitespace' is".  Edge spacing is a TEXT fact — for a binary or
+ *  grid modality 0x20 is content, not presentation — so it belongs beside the
+ *  text canonicalizer, is injected on the same modality test, and never leaks
+ *  into a mechanism.  A modality that supplies its own canon supplies its own
+ *  reading of "edge" too, or none.
+ *
+ *  Why trimming is sound HERE when {@link textCanon} deliberately refuses it:
+ *  canon preserves edge whitespace because the hazard is a recognised SUB-span
+ *  swallowing the boundary byte that separates it from its neighbour (observed:
+ *  "ice " matching the stored "ice").  At the outer edges of a WHOLE input
+ *  there is no neighbour — nothing precedes byte 0, nothing follows the last
+ *  byte — so that hazard cannot arise, and only there. */
+export declare function textEdgeTrim(bytes: Uint8Array): Uint8Array;

package/dist/src/canon.js

@@ -55,3 +55,31 @@ export function canonHash(key) {
     }
     return h >>> 0;
 }
+/** The span of `bytes` between its first and last non-whitespace byte — the
+ *  QUESTION, with the caller's edge spacing dropped.  Returns a subarray (no
+ *  copy), and the original when there is nothing to trim.
+ *
+ *  THIS LIVES HERE, not in the core byte utilities, for the reason stated at
+ *  the top of this file: "nothing in the store or the mind's core knows what
+ *  'case' or 'whitespace' is".  Edge spacing is a TEXT fact — for a binary or
+ *  grid modality 0x20 is content, not presentation — so it belongs beside the
+ *  text canonicalizer, is injected on the same modality test, and never leaks
+ *  into a mechanism.  A modality that supplies its own canon supplies its own
+ *  reading of "edge" too, or none.
+ *
+ *  Why trimming is sound HERE when {@link textCanon} deliberately refuses it:
+ *  canon preserves edge whitespace because the hazard is a recognised SUB-span
+ *  swallowing the boundary byte that separates it from its neighbour (observed:
+ *  "ice " matching the stored "ice").  At the outer edges of a WHOLE input
+ *  there is no neighbour — nothing precedes byte 0, nothing follows the last
+ *  byte — so that hazard cannot arise, and only there. */
+export function textEdgeTrim(bytes) {
+    const space = (b) => b === 0x20 || b === 0x09 || b === 0x0a || b === 0x0d;
+    let from = 0;
+    let to = bytes.length;
+    while (from < to && space(bytes[from]))
+        from++;
+    while (to > from && space(bytes[to - 1]))
+        to--;
+    return from === 0 && to === bytes.length ? bytes : bytes.subarray(from, to);
+}

package/dist/src/geometry.d.ts

@@ -131,6 +131,58 @@ export declare function knownPrefixLength(bytes: Uint8Array, leafAt: (i: number)
  *  turn extend perception instead of refolding it: identical prefixes
  *  produce identical subtrees regardless of what follows them. */
 export declare function bytesToTree(space: Space, alphabet: Alphabet, bytes: Uint8Array, leafAt?: (i: number) => number | null, lookup?: (leafIds: number[]) => number | null, boundaries?: readonly number[]): Sema;
+/** A plain content fold's reusable state: the level-0 cut edges over the whole
+ *  stream and each segment's independently-folded root.  See
+ *  {@link contentFoldIncremental}. */
+export interface ContentFold {
+    edges: number[];
+    segs: Folded[];
+}
+/** {@link contentFoldSpan} over a WHOLE stream, reusing the segments a previous
+ *  fold of a byte-identical prefix already produced.
+ *
+ *  WHY THIS IS SOUND, AND WHY IT NEEDS NO BOUNDARIES.  A level-0 segment is a
+ *  pure function of its own bytes ({@link flatFold} reads nothing else), so
+ *  reusing one whose [start,end) is unchanged is bit-identical to refolding it
+ *  — the cache can never change the tree, only skip work.  And the cuts
+ *  themselves are stable under APPEND: {@link contentLevels} decides each cut
+ *  from a rolling hash over a local window, so bytes added at the right edge
+ *  cannot move a cut to their left (measured over a growing 12-turn context:
+ *  100% of prior cuts survive every append, zero tail churn).  Together those
+ *  two facts are the whole optimisation — a grown stream refolds only the
+ *  segments at its right edge.
+ *
+ *  This is the reuse the conversation path wants, and it costs NOTHING in
+ *  structure: the tree is exactly the tree {@link bytesToTree} builds for the
+ *  same bytes with no boundary set at all.  Turn boundaries buy prefix-ROOT
+ *  identity, which is a different property from incremental reuse; conflating
+ *  the two is what put an imposed boundary set on the inference path and left
+ *  it folding differently from the deposits it was querying.
+ *
+ *  `groupByLevel` above the segments is re-run whole.  It operates on segment
+ *  ROOTS (a few dozen items for a several-hundred-byte context), not on bytes,
+ *  and only its right edge actually changes shape — measured at ~40 rebuilt
+ *  nodes per turn, flat as the context grows sevenfold.
+ *
+ *  PRECONDITION — `prev` MUST have been folded over a BYTE-IDENTICAL PREFIX of
+ *  `bytes`.  Reuse is keyed on a segment's [start,end) OFFSETS, which is what
+ *  makes it O(1) per segment; offsets alone cannot witness that the underlying
+ *  bytes agree.  Hand it a fold of DIFFERENT bytes whose cuts happen to land
+ *  in the same places and it will splice those foreign segments in — measured,
+ *  a deliberately mismatched `prev` produced a wrong tree on 336 of 400 random
+ *  streams.  Verifying the bytes here would cost O(prefix) and defeat the
+ *  whole point, so the obligation sits with the caller, and every caller
+ *  discharges it structurally rather than by care: `perceiveDeposit` looks the
+ *  entry up under `latin1Key(bytes.subarray(0, L))` — the prefix's own bytes
+ *  ARE the cache key — and a conversation's fold state advances only by
+ *  append.  A new caller that cannot make the same structural argument must
+ *  pass no `prev` at all; the cold path is always correct.
+ *  ({@link stablePrefixFoldIncremental} carries the identical precondition for
+ *  the identical reason.) */
+export declare function contentFoldIncremental(space: Space, alphabet: Alphabet, bytes: Uint8Array, prev?: ContentFold): {
+    tree: Sema;
+    fold: ContentFold;
+};
 /** A stable-prefix fold's reusable state: the segment edge offsets and each
  *  segment's independently-folded root ({@link riverFoldRaw} output).  A
  *  grown stream whose boundary set EXTENDS a previous fold's reuses every

package/dist/src/geometry.js

@@ -620,6 +620,84 @@ function contentFoldSpan(space, alphabet, bytes, from, to) {
         return groupByLevel(space, segs, levels, 1);
     return segs[0];
 }
+/** {@link contentFoldSpan} over a WHOLE stream, reusing the segments a previous
+ *  fold of a byte-identical prefix already produced.
+ *
+ *  WHY THIS IS SOUND, AND WHY IT NEEDS NO BOUNDARIES.  A level-0 segment is a
+ *  pure function of its own bytes ({@link flatFold} reads nothing else), so
+ *  reusing one whose [start,end) is unchanged is bit-identical to refolding it
+ *  — the cache can never change the tree, only skip work.  And the cuts
+ *  themselves are stable under APPEND: {@link contentLevels} decides each cut
+ *  from a rolling hash over a local window, so bytes added at the right edge
+ *  cannot move a cut to their left (measured over a growing 12-turn context:
+ *  100% of prior cuts survive every append, zero tail churn).  Together those
+ *  two facts are the whole optimisation — a grown stream refolds only the
+ *  segments at its right edge.
+ *
+ *  This is the reuse the conversation path wants, and it costs NOTHING in
+ *  structure: the tree is exactly the tree {@link bytesToTree} builds for the
+ *  same bytes with no boundary set at all.  Turn boundaries buy prefix-ROOT
+ *  identity, which is a different property from incremental reuse; conflating
+ *  the two is what put an imposed boundary set on the inference path and left
+ *  it folding differently from the deposits it was querying.
+ *
+ *  `groupByLevel` above the segments is re-run whole.  It operates on segment
+ *  ROOTS (a few dozen items for a several-hundred-byte context), not on bytes,
+ *  and only its right edge actually changes shape — measured at ~40 rebuilt
+ *  nodes per turn, flat as the context grows sevenfold.
+ *
+ *  PRECONDITION — `prev` MUST have been folded over a BYTE-IDENTICAL PREFIX of
+ *  `bytes`.  Reuse is keyed on a segment's [start,end) OFFSETS, which is what
+ *  makes it O(1) per segment; offsets alone cannot witness that the underlying
+ *  bytes agree.  Hand it a fold of DIFFERENT bytes whose cuts happen to land
+ *  in the same places and it will splice those foreign segments in — measured,
+ *  a deliberately mismatched `prev` produced a wrong tree on 336 of 400 random
+ *  streams.  Verifying the bytes here would cost O(prefix) and defeat the
+ *  whole point, so the obligation sits with the caller, and every caller
+ *  discharges it structurally rather than by care: `perceiveDeposit` looks the
+ *  entry up under `latin1Key(bytes.subarray(0, L))` — the prefix's own bytes
+ *  ARE the cache key — and a conversation's fold state advances only by
+ *  append.  A new caller that cannot make the same structural argument must
+ *  pass no `prev` at all; the cold path is always correct.
+ *  ({@link stablePrefixFoldIncremental} carries the identical precondition for
+ *  the identical reason.) */
+export function contentFoldIncremental(space, alphabet, bytes, prev) {
+    if (bytes.length === 0) {
+        return {
+            tree: sema(alphabet.vecs[0], new Uint8Array(0), null),
+            fold: { edges: [0], segs: [] },
+        };
+    }
+    const { cuts, levels } = contentLevels(space, bytes);
+    const edges = [0, ...cuts, bytes.length];
+    const segs = [];
+    for (let i = 0; i + 1 < edges.length; i++) {
+        const hit = prev !== undefined && prev.edges[i] === edges[i] &&
+            prev.edges[i + 1] === edges[i + 1]
+            ? prev.segs[i]
+            : undefined;
+        segs.push(hit ?? flatFold(space, alphabet, bytes, edges[i], edges[i + 1]));
+    }
+    const folded = segs.length > 1
+        ? groupByLevel(space, segs, levels, 1)
+        : segs[0];
+    // THE ROOT IS NORMALIZED IN PLACE, A CACHED SEGMENT NEVER IS.  With one
+    // segment — or with a grouping that passes a lone item through — `folded`
+    // IS a cached seg, and a later turn will reuse it as an interior node whose
+    // magnitude must stay byte-proportional.  Copy before normalizing, exactly
+    // as the stable-prefix twin does.  A single LEAF is copied too: its vector
+    // is the shared alphabet entry and must never be written.
+    const aliased = segs.some((s) => s.tree === folded.tree);
+    let tree = folded.tree;
+    if (aliased) {
+        tree = tree.kids === null
+            ? sema(tree.v, tree.leaf, null)
+            : sema(Float32Array.from(tree.v), null, tree.kids);
+    }
+    if (tree.kids !== null)
+        normalize(tree.v);
+    return { tree, fold: { edges, segs } };
+}
 /** Group a row of items by the level of the cut BETWEEN them: items separated
  *  by a cut of level < L belong to the same parent, and a cut of level ≥ L ends
  *  it.  Recurses upward until one root remains, so the shape at every level is
@@ -820,9 +898,17 @@ function stablePrefixFold(space, alphabet, bytes, boundaries) {
  *  folded over a byte-identical prefix (the caller keys the cache by
  *  content).  Requires a non-empty effective boundary set. */
 export function stablePrefixFoldIncremental(space, alphabet, bytes, boundaries, prev) {
+    // SORTED, like {@link bytesToTree} does before calling the non-incremental
+    // twin.  The filter below is sequential (`b > prevB`), so an out-of-order
+    // entry is silently DROPPED rather than rejected — and these two functions
+    // are documented as producing the same cuts, so a caller that hands the
+    // same set to each and gets different trees has hit a trap, not a contract.
+    // Sorting here makes the twins genuinely interchangeable; the set is one
+    // entry per conversation turn, so the cost is nil.
+    const sorted = [...boundaries].sort((a, b) => a - b);
     const cuts = [];
     let prevB = 0;
-    for (const b of boundaries) {
+    for (const b of sorted) {
         if (b > prevB && b < bytes.length) {
             cuts.push(b);
             prevB = b;

package/dist/src/mind/bridge.js

@@ -94,7 +94,7 @@ import { conceptThreshold, dominates, significanceBar } from "../geometry.js";
 import { bytesEqual, indexOf } from "../bytes.js";
 import { foldTree, perceive, read } from "./primitives.js";
 import { chainReach, leafIdRun } from "./canonical.js";
-import { corpusN, edgeAncestors, hubBound, sharedReachMemo, } from "./traverse.js";
+import { allWindowsAreScaffolding, corpusN, edgeAncestors, hubBound, sharedReachMemo, } from "./traverse.js";
 import { rItem, rNode } from "./trace.js";
 import { junctionContainersFrom } from "./junction.js";
 import { spanHalo } from "./match.js";
@@ -329,6 +329,32 @@ async function bridgeImpl(ctx, query, proposed) {
         ctx.trace?.step("substitutionBridge", [rItem(query, "query")], [], "no stored query window can anchor a corroborated substitution", undefined, diagnostics);
         return null;
     }
+    // NO DISCRIMINATING LITERAL EVIDENCE — abstain (§2.13).  A bridge grounds
+    // through the literal spans it did NOT substitute; those anchors are the
+    // whole of its evidence.  When every one of them is SATURATED — containment
+    // clamped at the √N hub bound, i.e. the window is corpus-global scaffolding
+    // — the query's unsubstituted part discriminates nothing, and the single
+    // substituted span is carrying the entire semantic load.  That is not a
+    // corroborated bridge; it is a template match, and it FABRICATES.
+    //
+    // Measured on the trained store (hubBound 571).  "What is the capital of"
+    // has 19 anchors, ALL saturated ("What":572, "hat ":572, "at i":572 …), and
+    // bridged to an unrelated trained context about an integral, voiced
+    // confidently.  Every query the bridge answers CORRECTLY has at least one
+    // unsaturated anchor, by a wide margin and with no near miss:
+    // "Who is the author of Hamlet?" → "let?":12, "How do you say 'thank you'
+    // in French?" → "y 't":3, "…largest planet…" → "tem?":31, "What is the
+    // capital of France?" → "f Fr":114.  The honest-silence probes sit on the
+    // same side as the correct ones ("Zamu":3), so this gate is not what makes
+    // them silent and cannot be credited for them.
+    //
+    // This introduces NO new threshold: `bound` is the same √N reading of "hub"
+    // the anchor scan already clamps its own containment read to (§2.2, §2.7).
+    if (allWindowsAreScaffolding(ctx, query)) {
+        ctx.trace?.step("substitutionBridge", [rItem(query, "query")], [], "every query window that could anchor is corpus-global scaffolding — " +
+            "no literal evidence to corroborate a substitution", undefined, diagnostics);
+        return null;
+    }
     // CORROBORATION (see the module-level doc) over the precomputed window
     // facts: the query span [qs,qe) attests when every full W-window inside
     // it is a stored flat form and at least one is reused across ≥ 2

package/dist/src/mind/frame-filler.d.ts

@@ -0,0 +1,15 @@
+import type { MindContext } from "./types.js";
+/** A grounded frame-filler substitution: the stored form the constructed key
+ *  resolved to, and the spans that explain how it was reached. */
+export interface FrameFillerHit {
+    /** The trained form the key resolved to — grounded through its own edge. */
+    id: number;
+    /** `[start, end)` of the query span the filler stood in for. */
+    described: [number, number];
+    /** The filler's bytes, for the rationale trace. */
+    filler: Uint8Array;
+}
+/** Find the query's own most discriminative unit and the trained contexts that
+ *  hold it, then try the store for the query with a candidate filler in the
+ *  described span's place.  Returns the sole surviving stored form, or null. */
+export declare function frameFillerSubstitution(ctx: MindContext, query: Uint8Array, ranked: ReadonlyArray<number>): FrameFillerHit | null;