@hviana/sema 0.4.7 → 0.5.0

package/dist/src/mind/recognition.js

@@ -8,6 +8,7 @@ import { rItem } from "./trace.js";
 import { canonResolve, foldTree, gistOf, latin1Key, perceive, resolve, } from "./primitives.js";
 import { atomIsHub, corpusN, leadsSomewhere } from "./traverse.js";
 import { chainReach, leafIdAt, leafIdRun } from "./canonical.js";
+import { canonHash } from "../canon.js";
 import { isChunk } from "../sema.js";
 /** Decompose a byte stream into every stored form that leads somewhere
  *  (has a continuation edge or a halo).  Two complementary readings:
@@ -400,7 +401,21 @@ function recogniseImpl(ctx, bytes) {
     // before resolveSpan pays for a fold; approximate evidence never enters.
     // This tier is needed only where atom chains are suppressed. Small stores
     // retain their existing decomposition unchanged.
-    if (atomsAreHubs) {
+    // ALWAYS ON, AND LINEAR.  This used to be gated on `atomsAreHubs` — small
+    // stores were said to "retain their existing decomposition unchanged", which
+    // was true only while the query's fold was told where the turns were: every
+    // turn was then a NODE, so the structural walk found it and this tier had
+    // nothing to add.  The fold no longer imposes turn boundaries (a turn start
+    // is an ordinary interior offset now), so a trained form embedded in a
+    // longer query is reachable ONLY here — the chain caps at chainReach(W)=W²
+    // bytes and cannot span one.  Measured: with the fold imposing boundaries
+    // every turn is a node; without it, none is.
+    //
+    // Ungating it alone made inference QUADRATIC (test/14's constant-KB/s guard
+    // went to 41.8s): every offset near a cut is an endpoint, and each probe
+    // costs O(span) to slice the leaf-id run and hash it.  The budget below is
+    // what makes it affordable — see `spend`.
+    {
         const allLeafIds = singleLeaf.map((x) => x?.id ?? null);
         if (allLeafIds.every((x) => x !== null)) {
             const radius = ctx.space.seats.length;
@@ -410,20 +425,115 @@ function recogniseImpl(ctx, bytes) {
                     endpoints.add(p);
             }
             const ordered = [...endpoints].sort((a, b) => a - b);
-            const probe = (start, end) => {
+            // The leaf-id run is BYTE-EXACT, while `resolveSpan` behind it resolves
+            // exactly OR canonically — so this gate was strictly narrower than its
+            // own resolver, and every embedded form differing from its deposit only
+            // by the response's equivalence (case, width) was dropped before the
+            // resolver ever saw it.  Rebuilding the run over canonicalized bytes
+            // does NOT fix that: a differently-cased deposit's branch kid-ids are
+            // not the query's leaf-id run under ANY canonicalization of the query,
+            // so the second admission route has to be the canon INDEX itself — the
+            // same candidate proposal `canonResolve` makes, and the same
+            // cheap-probe-before-a-fold discipline the exact route already follows
+            // (a hash and an indexed lookup; no fold, no vector, no scan).  Both
+            // routes only PROPOSE; `resolveSpan` still decides, so a hash-bucket
+            // collision costs one fold and can never emit a wrong site (test/71).
+            const canonAdmits = (start, end) => {
+                const canon = ctx.canon;
+                if (canon === null || !store.canonFind)
+                    return false;
+                const key = canon(bytes.subarray(start, end));
+                if (key.length === 0)
+                    return false;
+                return store.canonFind(canonHash(key)).length > 0;
+            };
+            // The byte-exact route probes the SPAN ITSELF (see
+            // Store.findFlatBranch): for a run of single-byte leaves the flat-kid
+            // encoding is the identity, so the span's bytes ARE the branch key.
+            // `subarray` is a view — this allocates nothing per probe, and the
+            // bloom filter answers the misses without touching the database.
+            const flatProbe = (start, end) => store.findFlatBranch
+                ? store.findFlatBranch(bytes.subarray(start, end))
+                : store.findBranch(allLeafIds.slice(start, end));
+            // THE TWO ROUTES COST DIFFERENT THINGS, SO THEY ARE PRICED SEPARATELY.
+            //
+            // The exact route is a bloom-gated hash over a subarray VIEW: no
+            // allocation, and a miss never reaches the database.  It is cheap enough
+            // to run on every endpoint, and that is what makes this tier able to
+            // find a trained form embedded anywhere in the query.
+            //
+            // The canon route is not: it runs the canonicalizer over the span
+            // (NFKC, case-fold, whitespace) and allocates a fresh key for every
+            // probe.  That is the O(span) cost with the heavy constant, and it is
+            // the one worth a budget.  Sharing ONE budget between them made the
+            // cheap route starve on the expensive one's behalf — measured, test/71's
+            // embedded differently-cased form needed 64x the budget to be found,
+            // while the exact route it was competing with needed none of it.
+            const probe = (start, end, canonBudget) => {
                 if (end - start < W || end - start <= chainReach(W))
                     return;
-                const ids = allLeafIds.slice(start, end);
-                if (store.findBranch(ids) === null)
-                    return;
+                if (flatProbe(start, end) === null) {
+                    if (!canonBudget)
+                        return;
+                    if (!canonAdmits(start, end))
+                        return;
+                }
                 const id = resolveSpan(start, end);
                 if (id !== null)
                     emit(start, end, id);
             };
-            for (const end of ordered)
-                probe(0, end);
-            for (const start of ordered)
-                probe(start, bytes.length);
+            // A CUMULATIVE BYTE BUDGET, SPENT SHORTEST-SPAN-FIRST.
+            //
+            // Each probe costs O(span), and there are O(n) endpoints, so probing
+            // them all is O(n²) — the quadratic this tier was gated to avoid.  The
+            // budget caps TOTAL probe bytes at a multiple of the query's own length,
+            // which is what keeps whole-query inference linear.
+            //
+            // Spending it shortest-first is what makes the cap a scale bound rather
+            // than a position bound: the tier recovers embedded forms up to roughly
+            // √(2·budget) bytes ANYWHERE in the endpoint set, instead of walking the
+            // endpoints in order and running out partway along the query.  A form
+            // longer than that is out of this tier's reach — but so is a form the
+            // chain cannot span, and that is exactly the trade the budget prices.
+            // The factor is chainReach(W), the same W² scale the chain already
+            // trusts; no new constant.
+            // The factor is chainReach(W) — the same W² scale the chain itself
+            // trusts — so the cap is derived from the fold's geometry, never tuned.
+            // (It was briefly an environment variable while the cost was being
+            // measured; an env-read here would make inference non-reproducible,
+            // which the determinism contract forbids outright.)
+            // The factor is chainReach(W) — the same W² scale the chain itself
+            // trusts — so the cap is derived from the fold's geometry, never tuned.
+            // (It was briefly an environment variable while the cost was being
+            // measured; an env-read here would make inference non-reproducible,
+            // which the determinism contract forbids outright.)
+            //
+            // It now prices ONLY the canonicalizing route; the exact route runs on
+            // every endpoint regardless, so exhausting this budget narrows which
+            // equivalence-class forms are proposed, never which byte-exact ones.
+            let budget = bytes.length * chainReach(W) * chainReach(W);
+            const spend = (start, end) => {
+                const span = end - start;
+                const afford = span <= budget;
+                if (afford)
+                    budget -= span;
+                probe(start, end, afford);
+                // Always keep walking: the exact route is unbudgeted, so running out
+                // of canon budget must not stop the scan.
+                return true;
+            };
+            const prefixes = ordered.filter((e) => e > 0).sort((a, b) => a - b);
+            const suffixes = ordered
+                .filter((s2) => s2 < bytes.length)
+                .sort((a, b) => b - a);
+            for (let i = 0; i < Math.max(prefixes.length, suffixes.length); i++) {
+                // Interleaved so neither edge starves the other when the budget runs
+                // out — a query can carry a trained form at either end.
+                if (i < prefixes.length && !spend(0, prefixes[i]))
+                    break;
+                if (i < suffixes.length && !spend(suffixes[i], bytes.length))
+                    break;
+            }
         }
     }
     const chunkEnd = new Uint32Array(bytes.length);

package/dist/src/mind/traverse.d.ts

@@ -124,3 +124,35 @@ export declare function chooseAmong(ctx: MindContext, candidates: readonly numbe
     id: number;
     score: number;
 };
+/** True when NO window of `query` discriminates anything — every stored
+ *  W-window it spells is contained by more places than the hub bound allows,
+ *  i.e. the whole query is corpus-global scaffolding.
+ *
+ *  WHAT IT IS FOR.  Several mechanisms ground a query through the literal
+ *  spans it did NOT explain, and those spans are the whole of their evidence.
+ *  When every one of them is a hub, the query says nothing the corpus can be
+ *  held to, and grounding it means picking one of thousands of continuations
+ *  it gives no evidence for — a fabrication whatever the answer happens to be.
+ *  Answering with silence there is the honest degradation contract (§2.13).
+ *
+ *  MEASURED SEPARATION (trained store, hubBound 571) — this is categorical,
+ *  not marginal, and it is why the predicate lives here rather than being
+ *  spelled twice:
+ *    "What is the capital of"  ALL saturated ("What":572)  → fabricated
+ *    "What is the capital "    ALL saturated ("What":572)  → fabricated
+ *    "what is the capital of france"  min "f fr":248       → correct
+ *    "What is the capitol of France?" min "f Fr":114       → correct
+ *    "WHAT IS THE CAPITAL OF FRANCE?" min "HE C":1         → correct
+ *    "What  is   the capital  of France?" min "t  i":4     → correct
+ *    "Who wrote Romeo and Juliet?"    min "iet?":26        → correct
+ *    "What is the capital of Zamunda?" min "Zamu":3        → silent anyway
+ *  Note the last: the honest-silence probes are already refused on other
+ *  evidence and sit on the SAME side as the correct ones, so this predicate
+ *  is not what makes them silent and cannot be credited for them.
+ *
+ *  NO NEW THRESHOLD (§2.2): `hubBound` is the √N reading of "hub" used
+ *  everywhere, and the containment read is clamped to it exactly as every
+ *  other fan-out read is (§2.8).  A query with no stored window at all is NOT
+ *  scaffolding-only — it has no evidence either way, and its callers already
+ *  refuse it on their own terms. */
+export declare function allWindowsAreScaffolding(ctx: MindContext, query: Uint8Array): boolean;

package/dist/src/mind/traverse.js

@@ -7,6 +7,7 @@
 // project) live in match.ts — the elementary match-and-project operation.
 import { cosine } from "../vec.js";
 import { gistOf, read } from "./primitives.js";
+import { leafIdRun } from "./canonical.js";
 const structCaches = new WeakMap();
 // ── The shared ancestor-reach memo ──────────────────────────────────────
 //
@@ -648,3 +649,54 @@ function rItemShort(ctx, id, role, score) {
         score,
     };
 }
+/** True when NO window of `query` discriminates anything — every stored
+ *  W-window it spells is contained by more places than the hub bound allows,
+ *  i.e. the whole query is corpus-global scaffolding.
+ *
+ *  WHAT IT IS FOR.  Several mechanisms ground a query through the literal
+ *  spans it did NOT explain, and those spans are the whole of their evidence.
+ *  When every one of them is a hub, the query says nothing the corpus can be
+ *  held to, and grounding it means picking one of thousands of continuations
+ *  it gives no evidence for — a fabrication whatever the answer happens to be.
+ *  Answering with silence there is the honest degradation contract (§2.13).
+ *
+ *  MEASURED SEPARATION (trained store, hubBound 571) — this is categorical,
+ *  not marginal, and it is why the predicate lives here rather than being
+ *  spelled twice:
+ *    "What is the capital of"  ALL saturated ("What":572)  → fabricated
+ *    "What is the capital "    ALL saturated ("What":572)  → fabricated
+ *    "what is the capital of france"  min "f fr":248       → correct
+ *    "What is the capitol of France?" min "f Fr":114       → correct
+ *    "WHAT IS THE CAPITAL OF FRANCE?" min "HE C":1         → correct
+ *    "What  is   the capital  of France?" min "t  i":4     → correct
+ *    "Who wrote Romeo and Juliet?"    min "iet?":26        → correct
+ *    "What is the capital of Zamunda?" min "Zamu":3        → silent anyway
+ *  Note the last: the honest-silence probes are already refused on other
+ *  evidence and sit on the SAME side as the correct ones, so this predicate
+ *  is not what makes them silent and cannot be credited for them.
+ *
+ *  NO NEW THRESHOLD (§2.2): `hubBound` is the √N reading of "hub" used
+ *  everywhere, and the containment read is clamped to it exactly as every
+ *  other fan-out read is (§2.8).  A query with no stored window at all is NOT
+ *  scaffolding-only — it has no evidence either way, and its callers already
+ *  refuse it on their own terms. */
+export function allWindowsAreScaffolding(ctx, query) {
+    const W = ctx.space.maxGroup;
+    const bound = hubBound(ctx);
+    let sawOne = false;
+    for (let o = 0; o + W <= query.length; o++) {
+        const ids = leafIdRun(ctx, query, o, o + W);
+        if (ids === null)
+            continue;
+        const id = ctx.store.findBranch(ids);
+        if (id === null)
+            continue;
+        const rarity = ctx.store.containersSlice(id, 0, bound + 1).length;
+        if (rarity === 0)
+            continue;
+        if (rarity <= bound)
+            return false;
+        sawOne = true;
+    }
+    return sawOne;
+}

package/dist/src/mind/types.d.ts

@@ -7,22 +7,17 @@ import type { MindConfig } from "../config.js";
 import type { Meter } from "../meter.js";
 import type { GraphSearch, Leaf, Seg, Site } from "./graph-search.js";
 import type { Rationale } from "./rationale.js";
-import type { Grid, StableFold } from "../geometry.js";
-/** One {@link MindContext._depositTrees} entry — see that field's doc. */
+import type { ContentFold, Grid } from "../geometry.js";
+/** One {@link MindContext._depositTrees} entry — see that field's doc.
+ *
+ *  A PURE WORK CACHE.  It carries the already-folded content segments of a
+ *  deposited stream so a longer stream sharing its byte prefix can skip
+ *  refolding them.  It holds no turn boundaries and no continuation proof
+ *  because the deposit fold imposes nothing: reuse is bit-identical to a cold
+ *  fold, so a hit can only save time, never change a tree. */
 export interface DepositCacheEntry {
-    /** Turn boundaries accumulated over this content's deposit chain —
-     *  strictly increasing proper offsets, each a previously-deposited
-     *  whole-context length.  Empty for a first-seen (single-turn) input. */
-    boundaries: number[];
-    /** Stable-prefix segment folds (grown-context inputs only). */
-    stable?: StableFold;
-    /** The continuation bytes this ctxInput was paired with in ingestPair, if
-     *  any — the ONLY thing that makes a later, longer ctxInput a genuine next
-     *  TURN of the same conversation rather than an unrelated fact that
-     *  happens to share this one's byte prefix (e.g. "2+2" vs. "2+2=5").  A
-     *  later deposit only takes this entry as its stable-prefix `prev` when
-     *  its own suffix bytes-equal this exactly. */
-    nextBytes?: Uint8Array;
+    /** The plain content fold's reusable segment state. */
+    content: ContentFold;
 }
 export type Input = string | Uint8Array | Grid | Grid[];
 /** The host capabilities GraphSearch consults during a cover.  MindContext
@@ -291,7 +286,17 @@ export interface MindContext extends GraphSearchHost {
      *  walking children.  When a conversation's pyramid reuses prefix
      *  subtrees, this cache lets {@link recognise} skip them entirely —
      *  O(suffix) instead of O(context).  Mind-lifetime (WeakMap keys are
-     *  the Sema objects the pyramid keeps alive). */
+     *  the Sema objects the pyramid keeps alive).
+     *
+     *  THAT REUSE IS A PRECONDITION, NOT A GIVEN: the keys are node IDENTITIES,
+     *  so it hits only while the conversation's fold hands back the SAME Sema
+     *  objects for the unchanged prefix.  `_growContext` rebuilt the whole tree
+     *  with `bytesToTree` on every turn, so every key was fresh and this cache
+     *  could not hit even once — the O(suffix) claim above described an
+     *  intention rather than the code.  It now grows the context through
+     *  {@link stablePrefixFoldIncremental}, which reuses each already-folded
+     *  segment: measured over four turns, turn 4 shared 69 of its 95 nodes with
+     *  turn 3 (26 new ≈ the new turn's own size). */
     _resolvedSubtrees: WeakMap<Sema, {
         id: number;
         len: number;
@@ -369,6 +374,40 @@ export declare function segRestatesQuery(s: Seg, query: Uint8Array, queryLen: nu
  *  (lo/hi) decision and the final concatenation: it is stale, not a second
  *  answer, but the OTHER spans a derivation chose are independent evidence
  *  and must not be discarded along with it. */
+/** The spans {@link liftAnswer} actually concatenates, in order — the answer
+ *  before it is joined.  Exposed so a caller can ask what the lifted answer is
+ *  MADE OF without re-deriving the selection: in particular how much of it is
+ *  SCAFFOLDING (a `rec: false` span — query bytes carried through verbatim
+ *  because nothing explained them, the same spans the liftAnswer trace labels
+ *  "scaffolding" rather than "chosen").
+ *
+ *  That quantity is load-bearing for the grounding decision.  Two candidates
+ *  can leave the SAME number of query bytes unaccounted and therefore grade
+ *  identically, while one of them pads its answer with those bytes and the
+ *  other does not — measured on test/22's two-fact chain, cover and recall
+ *  both graded 11001 with 11 bytes unexplained, and cover won the tie only on
+ *  consideration order, answering "The capital of France is Paris famous for"
+ *  where recall had crossed the hop.  Carrying an unexplained span into the
+ *  answer is strictly weaker than not explaining it: it manufactures fluency
+ *  out of the asker's own words.  See the tie-break in pipeline.ts. */
+export declare function liftAnswerParts(segs: Seg[], queryLen: number, query: Uint8Array, W: number): Seg[];
+/** The SCAFFOLDING byte count of a lifted answer: how many of its bytes come
+ *  from spans nothing recognised (see {@link liftAnswerParts}).
+ *
+ *  ONLY RUNS OF AT LEAST ONE RIVER WINDOW COUNT.  Not all carried-through
+ *  bytes are a failure to explain: a period, a question mark, the space
+ *  between two fused topics are GLUE — they belong to the answer's surface,
+ *  and dropping them to look better-derived would be a worse answer, not a
+ *  more honest one.  A substantive phrase the derivation never explained
+ *  ("famous for") is a different claim entirely.
+ *
+ *  W is the line between them, and it is the same line the rest of the mind
+ *  already draws: below one river window byte overlap is chance, not evidence
+ *  (see identityBar, the bridge's attestedQ, and recognition's site floor).
+ *  Counting every scaffolding byte instead — which is what this did first —
+ *  made punctuation preservation lose a tie it should win, and test/00's
+ *  "period preserved" / "question mark preserved" caught it immediately. */
+export declare function liftedScaffolding(segs: Seg[], queryLen: number, query: Uint8Array, W: number): number;
 export declare function liftAnswer(segs: Seg[], queryLen: number, query: Uint8Array, W: number): Uint8Array | null;
 /** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
  *  tracked deposit interned (`prevSeen`). */

package/dist/src/mind/types.js

@@ -46,7 +46,23 @@ export function segRestatesQuery(s, query, queryLen, W) {
  *  (lo/hi) decision and the final concatenation: it is stale, not a second
  *  answer, but the OTHER spans a derivation chose are independent evidence
  *  and must not be discarded along with it. */
-export function liftAnswer(segs, queryLen, query, W) {
+/** The spans {@link liftAnswer} actually concatenates, in order — the answer
+ *  before it is joined.  Exposed so a caller can ask what the lifted answer is
+ *  MADE OF without re-deriving the selection: in particular how much of it is
+ *  SCAFFOLDING (a `rec: false` span — query bytes carried through verbatim
+ *  because nothing explained them, the same spans the liftAnswer trace labels
+ *  "scaffolding" rather than "chosen").
+ *
+ *  That quantity is load-bearing for the grounding decision.  Two candidates
+ *  can leave the SAME number of query bytes unaccounted and therefore grade
+ *  identically, while one of them pads its answer with those bytes and the
+ *  other does not — measured on test/22's two-fact chain, cover and recall
+ *  both graded 11001 with 11 bytes unexplained, and cover won the tie only on
+ *  consideration order, answering "The capital of France is Paris famous for"
+ *  where recall had crossed the hop.  Carrying an unexplained span into the
+ *  answer is strictly weaker than not explaining it: it manufactures fluency
+ *  out of the asker's own words.  See the tie-break in pipeline.ts. */
+export function liftAnswerParts(segs, queryLen, query, W) {
     const restated = segs.map((s) => segRestatesQuery(s, query, queryLen, W));
     const recognised = [];
     for (let k = 0; k < segs.length; k++) {
@@ -54,32 +70,65 @@ export function liftAnswer(segs, queryLen, query, W) {
             recognised.push(k);
     }
     if (recognised.length === 0)
-        return null;
+        return [];
     if (recognised.length === 1) {
         const s = segs[recognised[0]];
-        // A COMPUTED span's query-side width is operand digit-count, not
-        // evidence of how much of the query's meaning it accounts for — the
-        // half-dominance check below (built for a genuinely RECOGNISED learned
-        // form) is not a valid framing signal for it (see the `computed` field
-        // doc on Seg/GItem): "1000 - 421" outweighs "what is …?" by width only
-        // because the operands are big, not because the framing matters less.
-        // A LITERAL PREFIX before a computed span is unambiguous framing
-        // regardless of width — an arithmetic expression is never itself
-        // preceded by more literal computed content, so anything literal before
-        // it is question wording ("what is ", "compute ") to lift clear of.
-        // With no prefix (s.i === 0) the span is judged by the ordinary
-        // half-dominance rule below, which already correctly keeps a short
-        // trailing glue byte ("2+2." → "4.", the span dominates a 4-byte query).
         if (s.computed && s.i > 0)
-            return s.bytes;
+            return [s];
         if (dominates(s.j - s.i, queryLen)) {
-            return concatBytes(segs.filter((_, k) => !restated[k]).map((x) => x.bytes));
+            return segs.filter((_, k) => !restated[k]);
         }
-        return s.bytes;
+        return [s];
     }
     const lo = recognised[0];
     const hi = recognised[recognised.length - 1];
-    return concatBytes(segs.slice(lo, hi + 1).filter((_, k) => !restated[lo + k]).map((x) => x.bytes));
+    return segs.slice(lo, hi + 1).filter((_, k) => !restated[lo + k]);
+}
+/** The SCAFFOLDING byte count of a lifted answer: how many of its bytes come
+ *  from spans nothing recognised (see {@link liftAnswerParts}).
+ *
+ *  ONLY RUNS OF AT LEAST ONE RIVER WINDOW COUNT.  Not all carried-through
+ *  bytes are a failure to explain: a period, a question mark, the space
+ *  between two fused topics are GLUE — they belong to the answer's surface,
+ *  and dropping them to look better-derived would be a worse answer, not a
+ *  more honest one.  A substantive phrase the derivation never explained
+ *  ("famous for") is a different claim entirely.
+ *
+ *  W is the line between them, and it is the same line the rest of the mind
+ *  already draws: below one river window byte overlap is chance, not evidence
+ *  (see identityBar, the bridge's attestedQ, and recognition's site floor).
+ *  Counting every scaffolding byte instead — which is what this did first —
+ *  made punctuation preservation lose a tie it should win, and test/00's
+ *  "period preserved" / "question mark preserved" caught it immediately. */
+export function liftedScaffolding(segs, queryLen, query, W) {
+    // MEASURED PER CONTIGUOUS RUN, not per span.  A PASS span is one BYTE — the
+    // cover charges unrecognised bytes individually — so asking whether a single
+    // span reaches W would find no run ever, whatever the query.  " famous for"
+    // arrives as eleven one-byte spans in a row and is one eleven-byte run.
+    let n = 0;
+    let run = 0;
+    const close = () => {
+        if (run >= W)
+            n += run;
+        run = 0;
+    };
+    for (const s of liftAnswerParts(segs, queryLen, query, W)) {
+        if (s.rec)
+            close();
+        else
+            run += s.bytes.length;
+    }
+    close();
+    return n;
+}
+export function liftAnswer(segs, queryLen, query, W) {
+    // ONE selection rule, in {@link liftAnswerParts} — this is its join.  The
+    // two used to be separate copies of the same lo/hi/restated reasoning, which
+    // is exactly how an answer and the accounting OF that answer drift apart.
+    const parts = liftAnswerParts(segs, queryLen, query, W);
+    if (parts.length === 0)
+        return null;
+    return concatBytes(parts.map((x) => x.bytes));
 }
 /** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
  *  tracked deposit interned (`prevSeen`). */

package/dist/src/store.d.ts

@@ -152,6 +152,10 @@ export interface Store {
     contentLen(id: NodeId, cap?: number): number;
     findLeaf(bytes: Uint8Array): NodeId | null;
     findBranch(kids: NodeId[]): NodeId | null;
+    /** {@link findBranch} for a run of single-byte leaves, addressed by the raw
+     *  bytes — the allocation-free probe span scanners use.  Optional: a store
+     *  without it is simply probed through `findBranch`. */
+    findFlatBranch?(bytes: Uint8Array): NodeId | null;
     /** The branch nodes that list `id` among their children — the reverse of
      *  `get(id).kids`. Lets the structural DAG be climbed upward, from a
      *  recognised fragment to the larger learned forms that contain it. */
@@ -585,6 +589,23 @@ export declare abstract class AbstractStore implements Store {
     private _prefix;
     contentLen(id: NodeId, cap?: number): number;
     findLeaf(bytes: Uint8Array): NodeId | null;
+    /** {@link findBranch} for a run of SINGLE-BYTE leaves, addressed by the
+     *  bytes themselves — no kid array, no key string, no copy.
+     *
+     *  A flat branch stores its children as {@link flatKidsBytes}, and that
+     *  encoding is the identity on single-byte leaves: kid id −(b+1) IS byte b.
+     *  So for such a run the kid array and the byte span are the same object in
+     *  two spellings, and `findBranch(leafIds.slice(i, j))` and this call are
+     *  the same lookup — except that the array path allocates the slice, then
+     *  `kids.join(",")`, then the flat bytes, all O(span), for a probe whose
+     *  answer is usually "no".  The bloom filter behind `_dbFindBranchByLeaf`
+     *  answers most of those with no I/O at all, so the allocations dominated.
+     *
+     *  Pass a subarray: it is a view, so a caller scanning spans of a query
+     *  allocates nothing per probe.  Deliberately NOT memoized — its callers
+     *  probe many spans that miss, and a key string per probe is the cost this
+     *  exists to remove. */
+    findFlatBranch(bytes: Uint8Array): NodeId | null;
     findBranch(kids: NodeId[]): NodeId | null;
     parents(id: NodeId): NodeId[];
     parentsFirst(id: NodeId, limit: number): NodeId[];

package/dist/src/store.js

@@ -770,6 +770,27 @@ export class AbstractStore {
             this._leafKey.set(key, id);
         return id;
     }
+    /** {@link findBranch} for a run of SINGLE-BYTE leaves, addressed by the
+     *  bytes themselves — no kid array, no key string, no copy.
+     *
+     *  A flat branch stores its children as {@link flatKidsBytes}, and that
+     *  encoding is the identity on single-byte leaves: kid id −(b+1) IS byte b.
+     *  So for such a run the kid array and the byte span are the same object in
+     *  two spellings, and `findBranch(leafIds.slice(i, j))` and this call are
+     *  the same lookup — except that the array path allocates the slice, then
+     *  `kids.join(",")`, then the flat bytes, all O(span), for a probe whose
+     *  answer is usually "no".  The bloom filter behind `_dbFindBranchByLeaf`
+     *  answers most of those with no I/O at all, so the allocations dominated.
+     *
+     *  Pass a subarray: it is a view, so a caller scanning spans of a query
+     *  allocates nothing per probe.  Deliberately NOT memoized — its callers
+     *  probe many spans that miss, and a key string per probe is the cost this
+     *  exists to remove. */
+    findFlatBranch(bytes) {
+        if (this.meter)
+            this.meter.branchLookups++;
+        return this._dbFindBranchByLeaf(hashOf(bytes), bytes);
+    }
     findBranch(kids) {
         if (this.meter)
             this.meter.branchLookups++;

package/example/train_base.ts

@@ -709,7 +709,18 @@ const isEpisode = (it: TrainingItem): it is Episode => typeof it !== "string";
 /** Build the accumulated-context episodes of a turn sequence: each successive
  *  turn is the continuation of ALL the turns before it joined together. This is
  *  the same cumulative-context shape a multi-turn conversation deposits, so the
- *  store learns to continue a growing context. */
+ *  store learns to continue a growing context.
+ *
+ *  The "\n" below is a CORPUS choice, not a protocol.  oasst2 turns are
+ *  paragraphs, and reading them back with the newlines kept is how this corpus
+ *  reads naturally; a different corpus may join with nothing, and
+ *  test/13-conversation.test.mjs does exactly that.  Neither has to match the
+ *  other, because Sema never scans content for turn boundaries — those are
+ *  offsets the Conversation API carries beside the bytes (see Mind.addTurn's
+ *  "ON SEPARATORS" note).  The newline here is simply part of the text this
+ *  store learnt, so anything replaying this corpus feeds it back as part of
+ *  the turn: `addTurn(conv, "\n" + turnText)`.  It is not a convention the
+ *  engine, the API, or the tests have to agree on. */
 function accumulate(turns: string[]): Episode[] {
   const out: Episode[] = [];
   for (let i = 1; i < turns.length; i++) {
@@ -894,9 +905,15 @@ export function bestOasstPath(root: OasstNode): OasstTurn[] {
  *  turn experiences and local adjacent-pair facts are NOT emitted (they are
  *  subsumed by it and would merely replicate the content).
  *
- *  The walk is byte-for-byte the pattern proven in test/13-conversation.test.mjs
- *  ("teachConversation"): each turn is the continuation of all prior turns joined
- *  by "\n", with BARE turn text — NO "User:/Assistant:" labels. Roles already
+ *  The walk is the pattern proven in test/13-conversation.test.mjs
+ *  ("teachConversation"): each turn is the continuation of all prior turns,
+ *  with BARE turn text — NO "User:/Assistant:" labels.  The SHAPE is identical
+ *  (cumulative context → next turn); the join string is not, and does not need
+ *  to be — that file joins with nothing and this corpus joins with "\n" (see
+ *  `accumulate`).  Saying "byte-for-byte", as this comment used to, invites the
+ *  reading that the two must agree on a separator.  They must not agree,
+ *  because there is nothing to agree about: turn boundaries are offsets, and
+ *  the join string is just corpus text. Roles already
  *  alternate by position in an oasst2 best-path (the root is a prompter), so a
  *  label adds nothing the position does not, while a clean continuation matches
  *  the test's recall (predictNext queries bare prior turns) and lets a turn share

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hviana/sema",
-  "version": "0.4.7",
+  "version": "0.5.0",
   "description": "Sema: a non-parametric, instance-based reasoning system.",
   "type": "module",
   "main": "dist/src/index.js",

package/src/canon.ts

@@ -63,3 +63,31 @@ export function canonHash(key: Uint8Array): number {
   }
   return h >>> 0;
 }
+
+/** The span of `bytes` between its first and last non-whitespace byte — the
+ *  QUESTION, with the caller's edge spacing dropped.  Returns a subarray (no
+ *  copy), and the original when there is nothing to trim.
+ *
+ *  THIS LIVES HERE, not in the core byte utilities, for the reason stated at
+ *  the top of this file: "nothing in the store or the mind's core knows what
+ *  'case' or 'whitespace' is".  Edge spacing is a TEXT fact — for a binary or
+ *  grid modality 0x20 is content, not presentation — so it belongs beside the
+ *  text canonicalizer, is injected on the same modality test, and never leaks
+ *  into a mechanism.  A modality that supplies its own canon supplies its own
+ *  reading of "edge" too, or none.
+ *
+ *  Why trimming is sound HERE when {@link textCanon} deliberately refuses it:
+ *  canon preserves edge whitespace because the hazard is a recognised SUB-span
+ *  swallowing the boundary byte that separates it from its neighbour (observed:
+ *  "ice " matching the stored "ice").  At the outer edges of a WHOLE input
+ *  there is no neighbour — nothing precedes byte 0, nothing follows the last
+ *  byte — so that hazard cannot arise, and only there. */
+export function textEdgeTrim(bytes: Uint8Array): Uint8Array {
+  const space = (b: number) =>
+    b === 0x20 || b === 0x09 || b === 0x0a || b === 0x0d;
+  let from = 0;
+  let to = bytes.length;
+  while (from < to && space(bytes[from])) from++;
+  while (to > from && space(bytes[to - 1])) to--;
+  return from === 0 && to === bytes.length ? bytes : bytes.subarray(from, to);
+}