@hviana/sema 0.4.6 → 0.5.0

package/dist/src/mind/mechanisms/cover.js

@@ -12,7 +12,7 @@ import { read, resolve } from "../primitives.js";
 import { guidedFirst, hubBound } from "../traverse.js";
 import { conceptHop } from "../match.js";
 import { bridge } from "../resonance.js";
-import { liftAnswer, segRestatesQuery } from "../types.js";
+import { liftAnswer, liftedScaffolding, segRestatesQuery } from "../types.js";
 import { decodeText, unexplainedLabel } from "../rationale.js";
 import { indexOf } from "../../bytes.js";
 import { rItem, rNode, traceDerivation } from "../trace.js";
@@ -222,6 +222,11 @@ export const coverMechanism = {
                 moves: 0,
                 weight: solved.cost, // A*LD derivation's g-value IS the weight
                 unexplained: unexplainedLabel(query, accounted),
+                // How much of the composed answer is the asker's own unexplained words
+                // (the spans the liftAnswer trace above labels "scaffolding").  Cover is
+                // the mechanism that can carry them, because a PASS span still lands in
+                // the cover it returns.
+                scaffolding: liftedScaffolding(segs, query.length, query, W),
             }];
     },
 };

package/dist/src/mind/mechanisms/extraction.js

@@ -92,6 +92,33 @@ export async function extractBySkill(ctx, query, pre) {
             subQuantum++;
             continue;
         }
+        // AN UNANCHORED READ IS NOT AN EXTRACTION.  This function's contract (see
+        // the doc above) is that `accounted` carries "the located frames AND any
+        // read span BOUNDED by located frames on both sides", while an open-ended
+        // read "remains a guess about where the span stops — it stays unaccounted".
+        // EMPTY accounted is the degenerate case of that: NO frame of the exemplar
+        // was located in the query at all, so nothing ties the bytes just read to
+        // this question — the skill applied its exemplar's geometry to a query it
+        // never matched.
+        //
+        // The live case (analyze_training.ts F, the battery's ONLY wrong non-silent
+        // answer): "Which city is France's seat of government?" answered "Which ci"
+        // — a fragment of the query itself — from the exemplar "What is dll", with
+        // accounted=[] and pieces=1.  isSpanShaped is deliberately permissive (a
+        // sparse-subsequence check), so it accepts exemplars whose relation to the
+        // query is coincidental gap-matching; requiring at least one LOCATED frame
+        // is the structural evidence that permissiveness leaves out.
+        //
+        // Scoped to extraction ON PURPOSE.  The same test at the pipeline's
+        // post-grounding density check was tried and REVERTED: `accounted` is passed
+        // empty BY CONVENTION on recall's own tiers (recall.ts's ground(…, [], …)),
+        // so a density veto there refused six legitimate reverse-recall groundings.
+        // Here the field is this mechanism's own output and carries its documented
+        // meaning, so the test is sound exactly where the convention does not reach.
+        if (built.accounted.length === 0) {
+            subQuantum++;
+            continue;
+        }
         if (shapeMisses > 0 || subQuantum > 0) {
             ctx.trace?.step("trySkillAnchors", [
                 rItem(query.subarray(0, 0), `skipped ${shapeMisses + subQuantum}`),

package/dist/src/mind/mechanisms/recall.js

@@ -4,15 +4,17 @@
 // index and grounds the nearest learned form.  Four tiers, orderly degrading
 // from exact self-match to honest echo.
 import { cosine } from "../../vec.js";
-import { conceptThreshold, consensusFloor, identityBar, reachThreshold, significanceBar, } from "../../geometry.js";
+import { conceptThreshold, consensusFloor, dominates, identityBar, reachThreshold, significanceBar, } from "../../geometry.js";
 import { gistOf, read, resolve } from "../primitives.js";
 import { bytesEqual, indexOf } from "../../bytes.js";
-import { corpusN, hubBound } from "../traverse.js";
+import { allWindowsAreScaffolding, corpusN, hubBound } from "../traverse.js";
 import { follow, project, reverseContext } from "../match.js";
 import { CONCEPT, STEP } from "../graph-search.js";
 import { unexplainedLabel } from "../rationale.js";
 import { rItem, rNode } from "../trace.js";
 import { substitutionBridge } from "../bridge.js";
+import { frameFillerSubstitution } from "../frame-filler.js";
+import { prefixCandidates, prefixCompletion } from "../prefix-completion.js";
 /** Recall the answer by resonating the whole query against the content index. */
 export async function recallByResonance(ctx, query, pre) {
     const t = ctx.trace?.enter("recallByResonance", [
@@ -180,13 +182,92 @@ export async function recallByResonance(ctx, query, pre) {
         Math.sqrt(otherLen / Math.max(1, query.length)));
     // 2. Scaffolding-dominated.
     if (top.score >= sig) {
-        const N = corpusN(ctx);
-        const minVote = consensusFloor(N);
         // The committed points of attention ARE the shared climb's roots (same
         // query, same k, same DF mode) — read them from Precomputed instead of
         // re-climbing, so even a traced response pays for the climb once.
         const forest = (await pre.attention()).roots;
-        if (forest.length > 0 && forest[0].vote >= minVote) {
+        // TRUST THE ANCHOR ON ITS BREADTH, NOT ON ITS ABSOLUTE VOTE.
+        //
+        // This gate read `forest[0].vote >= consensusFloor(N)`.  Attention.breadth's
+        // own contract (types.ts) says why that is the wrong quantity: the IDF vote
+        // is "an absolute, ln(N)-scaled quantity that means 'strong' on a small
+        // store and 'weak' on a large one for the SAME degree of genuine
+        // consensus", while breadth is the SCALE-INVARIANT reading — "a point whose
+        // breadth clears `dominates` (> half the query's regions corroborate it) is
+        // real consensus; one that does not is a coincidental single-region echo".
+        // Attention.peak's contract makes the same point from the other side:
+        // comparing a POOLED SUM against a floor that prices ONE region's evidence
+        // is a dimensional error.
+        //
+        // Measured on the 15.7M-node store (N=325,615, so the old floor was 13.19).
+        // The absolute vote cannot separate right from wrong at this scale, and the
+        // proof is a probe that must stay SILENT:
+        //
+        //   anchor picked by the climb          vote   breadth  correct?
+        //   "What is the chemical formula …"   10.60    0.556   RIGHT
+        //   "Qual é a capital de França?"       8.19    0.667   RIGHT
+        //   "Who wrote the play Romeo …?"       8.25    0.833   RIGHT
+        //   "How do you say "good morning" …"  10.77    0.800   RIGHT
+        //   "What is the commercial capital …" 12.69    0.333   Zamunda — MUST be silent
+        //   "Menene sunan ginin mafi tsayi …"  12.79    0.214   wrong (Hausa)
+        //   "Today is the 5th of March …"      10.36    0.000   wrong
+        //
+        // Zamunda's junk attractor outvotes every correct anchor, so no vote
+        // threshold admits the right ones without admitting fabrication — while
+        // breadth > ½ admits exactly the four correct anchors and nothing else.
+        // The old floor was simply never cleared on a corpus this large: the tier
+        // was dead code here, which is why 12 probes fell through to silence.
+        //
+        // `dominates(breadth, 1)` is the SAME half-dominance predicate used
+        // throughout, applied to the fraction — no new constant, and the bar the
+        // breadth contract names.  COST: none; breadth is already computed and
+        // carried on every Attention the climb returns.
+        //
+        // The two readings are ALTERNATIVES, never a substitution.  REPLACING the
+        // vote test with the breadth test was tried and broke 7 tests: on a small
+        // store ln(N) is low, so the vote bar is the one that legitimately fires
+        // there, and — as Attention.clusters' own contract warns — "breadth starves
+        // a genuine, evenly-split multi-topic query, since no root in a real N-way
+        // split can exceed half the vote" (the two 3.1 two-topic fusion tests are
+        // exactly that shape).  Each reading is sufficient on its own evidence: a
+        // vote that clears the absolute floor is strong enough wherever the corpus
+        // is small enough for that to mean something, and a breadth past ½ is real
+        // consensus at any scale.  ORing them keeps every admission the floor
+        // already made and adds only the scale-invariant ones it could never see.
+        //
+        // BREADTH ALSO NEEDS DISCRIMINATIVENESS.  Breadth asks how much of the
+        // query corroborates the anchor, never whether the anchor SAYS anything: on
+        // a one-context store every region trivially corroborates the only anchor
+        // there is, so breadth is 1 while the anchor's IDF is 0 — and test/31 A2
+        // ("explain quantum chromodynamics" against a lone cat fact) answered the
+        // cat, which is fabrication.  A region's IDF contribution for an anchor
+        // reached through c of N contexts is ln(N/c), so requiring it to exceed
+        // ln 2 is requiring c·2 < N — the SAME half-dominance reading used
+        // everywhere, expressed in the IDF's own units rather than as a new bar.
+        // `peak` is that per-region contribution, and reading it here is what
+        // Attention.peak's contract asks of a consumer gating on this evidence.
+        //
+        // AND THE QUERY MUST SAY SOMETHING.  Both readings above price the
+        // ANCHOR's evidence; neither asks whether the QUERY discriminates
+        // anything.  A query that is entirely corpus-global scaffolding gives the
+        // corpus nothing to be held to, and this tier — which exists to serve
+        // scaffolding-DOMINATED queries — is exactly where that runs out.
+        // Measured on the trained store: "What is the capital " answered "Colombo
+        // is the commercial capital of Sri Lanka…" on breadth 0.667 / clusters 1,
+        // and every window it spells is a hub ("What":572).  See
+        // allWindowsAreScaffolding for the full separation, including the probes
+        // this tier serves CORRECTLY, which all retain a discriminating window
+        // ("what is the capital of france" → "f fr":248).
+        //
+        // DISPERSION WAS TRIED HERE FIRST AND FALSIFIED — do not retry it: the
+        // fabrication and the no-punctuation robustness probe have the IDENTICAL
+        // profile (breadth 0.667, clusters 1), so requiring clusters >= 2 silenced
+        // "what is the capital of france" too and cost the battery a probe.
+        const minVote = consensusFloor(corpusN(ctx));
+        if (forest.length > 0 &&
+            !allWindowsAreScaffolding(ctx, query) &&
+            (forest[0].vote >= minVote ||
+                (dominates(forest[0].breadth, 1) && forest[0].peak > Math.LN2))) {
             const g = await project(ctx, forest[0].anchor, queryGist);
             // The anchor cleared the consensus floor, but the floor prices the
             // ANCHOR's evidence, not the projection's: a junk attractor can clear
@@ -212,6 +293,42 @@ export async function recallByResonance(ctx, query, pre) {
         }
     }
     // 3b. Corroborated-substitution bridge — refusal-path only (bridge.ts).
+    // MEMOISED ACROSS EVERY REMAINING TIER, and that is load-bearing rather than
+    // tidy: the bridge, prefix completion and the frame filler all read the SAME
+    // candidate list, so the exhaustive branch runs at most once per response.
+    // Without the memo each tier re-issues it — measured at 490 ms median against
+    // 13 ms non-exhaustive (36x).
+    const wideIdsOnce = async () => {
+        // When the top resonance hit is below the concept threshold, the query
+        // gist has no concept-level match to any stored form — an exhaustive √N
+        // ANN would only score more vectors below the bar (profiled at 38K–40K
+        // annVectorReads per refusing query on a 325K-context store).  The
+        // bridge's structural channels (junction walks, anchor climbs) are the
+        // correct proposal source for a query whose gist has no clean match;
+        // the ANN cannot propose what the gist cannot rank.
+        // The condition above is the SCORE of the top hit, not the size of the
+        // corpus.  It used to be spelled `corpusN(ctx) <= (k · W)³`, which asks
+        // a different question and answers it wrongly at exactly the scale the
+        // note was written from: on the trained store N = 325,608 with k = 24
+        // and W = 4 puts the cube at 884,736, so that store took the exhaustive
+        // branch — the very branch measured here as 38K–40K annVectorReads.
+        // Measured cost of the mismatch: substitutionBridge 8,544ms of a
+        // 19,548ms think (44%), against 1,248ms and 14,218ms without it, with
+        // every answer in the battery byte-identical and the suite unchanged
+        // at 445/445.  Corpus size was never the discriminator; whether the
+        // gist ranks ANYTHING at concept level is.
+        //
+        // Reading it as the note states also removes a duplicated (k · W)³ —
+        // the same cube gates crossRegionVotes' walk budget, where it likewise
+        // never engages at real scale (see attention.ts).
+        if (whole.length > 0 && whole[0].score >= conceptThreshold(ctx.store.D)) {
+            const exhaustive = await ctx.store.resonate(queryGist, hubBound(ctx), true);
+            return exhaustive.map((h) => h.id);
+        }
+        return whole.map((h) => h.id);
+    };
+    let wide = null;
+    const wideIds = () => (wide ??= wideIdsOnce());
     // Every gist-based tier has failed; before refusing, align the query
     // byte-for-byte against the trained contexts its own stored windows
     // anchor, accepting mismatches only as corpus-attested, concept-bar
@@ -232,35 +349,6 @@ export async function recallByResonance(ctx, query, pre) {
         // exact co-occurrence and bounded anchor ascent are the bridge's structural
         // proposal channels, while an exhaustive ANN call made every honest
         // refusal cost hundreds of milliseconds regardless of k.
-        const wideIds = async () => {
-            // When the top resonance hit is below the concept threshold, the query
-            // gist has no concept-level match to any stored form — an exhaustive √N
-            // ANN would only score more vectors below the bar (profiled at 38K–40K
-            // annVectorReads per refusing query on a 325K-context store).  The
-            // bridge's structural channels (junction walks, anchor climbs) are the
-            // correct proposal source for a query whose gist has no clean match;
-            // the ANN cannot propose what the gist cannot rank.
-            // The condition above is the SCORE of the top hit, not the size of the
-            // corpus.  It used to be spelled `corpusN(ctx) <= (k · W)³`, which asks
-            // a different question and answers it wrongly at exactly the scale the
-            // note was written from: on the trained store N = 325,608 with k = 24
-            // and W = 4 puts the cube at 884,736, so that store took the exhaustive
-            // branch — the very branch measured here as 38K–40K annVectorReads.
-            // Measured cost of the mismatch: substitutionBridge 8,544ms of a
-            // 19,548ms think (44%), against 1,248ms and 14,218ms without it, with
-            // every answer in the battery byte-identical and the suite unchanged
-            // at 445/445.  Corpus size was never the discriminator; whether the
-            // gist ranks ANYTHING at concept level is.
-            //
-            // Reading it as the note states also removes a duplicated (k · W)³ —
-            // the same cube gates crossRegionVotes' walk budget, where it likewise
-            // never engages at real scale (see attention.ts).
-            if (whole.length > 0 && whole[0].score >= conceptThreshold(ctx.store.D)) {
-                const exhaustive = await ctx.store.resonate(queryGist, hubBound(ctx), true);
-                return exhaustive.map((h) => h.id);
-            }
-            return whole.map((h) => h.id);
-        };
         const bridged = await substitutionBridge(ctx, query, wideIds);
         if (bridged !== null) {
             const g = await project(ctx, bridged.id, queryGist);
@@ -274,7 +362,32 @@ export async function recallByResonance(ctx, query, pre) {
             const cBytes = ctx.store.bytes(bridged.id);
             const manufactured = g !== null &&
                 bridged.subs.some((s) => indexOf(cBytes.subarray(s.cs, s.ce), g, 0) >= 0);
+            // THE PREFIX TRAP IS NOT THIS TIER'S TO SPRING.  With no substitutions
+            // the claim is "a trained context IS this query, up to filler".  When
+            // the query is a STRICT BYTE PREFIX of that context, the claim is false
+            // in the one way that matters: the candidate's extra tail is precisely
+            // the DISCRIMINATING part, and dismissing it as filler asserts a
+            // specification the asker never made.  Measured on a 4,300-fact fixture
+            // of "what is the value of <i>?": the query "what is the value of"
+            // bridged with subs [] to "what is the value of 0?" and answered "the
+            // value of 0 is 0" — one arbitrary pick from 4,300 equally-matching
+            // contexts, every one of which fits the query exactly as well.
+            //
+            // The engine ALREADY has the right machinery for this shape:
+            // prefixCompletion runs a few lines below and carries the three guards
+            // this tier lacks — unreadable-continuation veto, sub-quantum
+            // continuation, and UNIQUENESS (distinct continuations ⇒ refuse), which
+            // is exactly what 4,300 competing values must trip.  So this is not a
+            // new rule and not a new threshold: it is deferring a prefix decision to
+            // the tier that owns it (§2.5, one factored machinery).  Byte-strict on
+            // purpose — a candidate differing by case or punctuation ("what is the
+            // capital of france" → "What is the capital of France?") is NOT a byte
+            // prefix, keeps grounding here, and is unaffected.
+            const strictPrefix = g !== null &&
+                cBytes.length > query.length &&
+                indexOf(cBytes, query, 0) === 0;
             if (g !== null && g.length > 0 && !restates(g) && !manufactured &&
+                !(bridged.subs.length === 0 && strictPrefix) &&
                 !(g.length < query.length && indexOf(query, g, 0) >= 0)) {
                 return ground(g, bridged.subs.length === 0
                     ? `identity bridge — a trained context IS this query, up to ` +
@@ -326,6 +439,73 @@ export async function recallByResonance(ctx, query, pre) {
                 bridged.subs.length === 0);
             }
         }
+        // 3b′. PREFIX COMPLETION — refusal-path only (prefix-completion.ts).
+        // Inside the bridge's block, and deliberately: it consumes `wideIds`, the
+        // list the bridge has already fetched, so it costs a bounded byte compare
+        // per candidate and not one resonance.  The claim it makes is the
+        // strongest in the ladder — every query byte is a LITERAL match from
+        // offset zero of a trained form — so it needs no projection and no reach
+        // gate.  It runs after the bridge only because the bridge answers the
+        // richer relation when it can; a prefix match that the bridge also
+        // explains is the same trained form either way.
+        {
+            // The resonance list first; only when it supplies nothing does the
+            // write side's leaf-id window index propose (prefixCandidates).  That
+            // ordering is the whole cost story: a query the ranked list can already
+            // explain pays not one extra read, and the fallback's bounded walk is
+            // spent only where the alternative is an empty answer.  It is a second
+            // SUPPLY, not a second mechanism — the same three guards decide.
+            const completed = prefixCompletion(ctx, query, await wideIds()) ??
+                prefixCompletion(ctx, query, prefixCandidates(ctx, query));
+            if (completed !== null) {
+                return ground(completed.form, "prefix completion — the query IS the opening of exactly one " +
+                    "trained form, which this grounds whole", 
+                // Every query byte is literally matched against the form.  The
+                // completion is the form's own continuation, not a substitution, so
+                // there is nothing to be humble about in the accounting — the same
+                // reading the IDENTITY bridge above takes.
+                whole_, STEP, false, 
+                // NOT complete: the query is a proper PREFIX, so the form may carry
+                // more past the remainder this tier voiced.
+                false);
+            }
+        }
+    }
+    // 3c. FRAME-FILLER SUBSTITUTION — refusal-path only (frame-filler.ts).
+    // The bridge has failed, and for the shape this tier answers it MUST fail:
+    // a definite description standing where a proper noun stands is not a
+    // similarity relation the bridge can price (raw balance refuses
+    // `dominates(6, 37)`, and correctly — that is the France/Spain trap).  This
+    // tier makes a different claim: not that the two spans resemble each other,
+    // but that the store ALREADY HOLDS this query with the filler in the
+    // description's place, byte-exactly.  A key the store does not hold is
+    // discarded, so the answer is always a trained continuation.
+    {
+        // THE COHORT NEEDS EVIDENCE, AND THE REFUSAL PATH HAS ALREADY BOUGHT IT.
+        // This tier reads constituency from what a cohort of exemplars does NOT
+        // share, so its resolution is bounded by how many instances of the frame it
+        // can see.  The top-k resonance hits are too few — on the two-hop probe the
+        // exemplars holding the query's discriminative content number TWO, and two
+        // structures agree on so little that a whole clause reads as content.  The
+        // exhaustive list the bridge fetched is the same evidence at ~570 wide, and
+        // it is already paid for (memoised above, so this costs no ANN call).
+        const filled = frameFillerSubstitution(ctx, query, await wideIds());
+        if (filled !== null) {
+            const g = await project(ctx, filled.id, queryGist);
+            // The same restated-fragment and manufactured-answer guards every tier
+            // above applies: a projection contained in the FILLER is the
+            // substitution restated as if it were knowledge, not knowledge.
+            if (g !== null && g.length > 0 && !restates(g) &&
+                indexOf(filled.filler, g, 0) < 0 &&
+                !(g.length < query.length && indexOf(query, g, 0) >= 0)) {
+                return ground(g, "frame-filler substitution — a trained form IS this query with a " +
+                    "corroborated filler in the described span's place", 
+                // The frame is literally matched against the resolved form and the
+                // described span is explained by the substitution — the same
+                // matched-plus-substituted accounting the bridge reports.
+                [[0, query.length]], CONCEPT + STEP);
+            }
+        }
     }
     // The refusal/echo decision.  The echo returns a stored form's bytes AS
     // the answer — a near-identity claim about the query — and identity-grade

package/dist/src/mind/mind.d.ts

@@ -219,6 +219,20 @@ export declare class Mind implements MindContext {
      *  they do with the answer afterwards.  It must be called between
      *  {@link beginResponse} and {@link endResponse}. */
     private _groundAndVoice;
+    /** Answer ONE self-contained input.
+     *
+     *  A MULTI-TURN context is not that, and this is the wrong entry point for
+     *  it.  `respond` folds the bytes it is handed with no boundary set, because
+     *  nothing in a flat byte string says where one turn ended — only the caller
+     *  who assembled it knows, which is the whole reason `boundaries` is a
+     *  parameter of {@link perceiveImpl} and never inferred from content.  A
+     *  conversation deposited through {@link ingest} folds its contexts over
+     *  those turn boundaries, so a hand-concatenated transcript passed here
+     *  folds differently from the way it was learnt and reaches the trained
+     *  context node only by luck (measured on a 7-turn conversation: 5/7 here
+     *  against 7/7 through {@link respondTurn}, same bytes).  Use
+     *  {@link beginConversation} + {@link respondTurn}, or {@link addTurn} to
+     *  replay turns the Mind should hear but not answer. */
     respond(input: Input, inspectRationale?: InspectRationale): Promise<Response>;
     /** Text view of {@link respond}.  NUL bytes (0x00) are stripped before
      *  decoding — they are structural padding in text answers.  LOSSY for a
@@ -252,7 +266,41 @@ export declare class Mind implements MindContext {
      *  This is the primitive for turns the Mind should hear but not answer:
      *  replaying a transcript, feeding the OTHER speaker's line in a
      *  prediction harness, or restoring context piecewise.  {@link
-     *  respondTurn} = addTurn + think + its own reply appended the same way. */
+     *  respondTurn} = addTurn + think + its own reply appended the same way.
+     *
+     *  ── ON SEPARATORS: THERE IS NO SEPARATOR QUESTION ────────────────────
+     *
+     *  "Never a separator" above says what this method DOES — it appends the
+     *  bytes you give it and records an OFFSET — not that separator bytes are
+     *  forbidden, unsupported, or something the engine must be taught about.
+     *  Sema is agnostic to them, and reviewers keep mistaking that agnosticism
+     *  for a constraint.  To be explicit, because the mistake is easy:
+     *
+     *  1. A turn boundary is an OFFSET, held here, in `boundaries`.  It is
+     *     never a character the geometry scans for.  Nothing downstream asks
+     *     "what byte separates two turns?" because nothing downstream finds
+     *     boundaries by looking at content at all.
+     *  2. A separator in a CORPUS is ordinary content.  If a trainer joins
+     *     turns with "\n" (example/train_base.ts does), those newlines are
+     *     simply bytes inside the stream, folded like every other byte.  They
+     *     are a property of that corpus, not of this API and not of the fold.
+     *  3. This API can therefore reproduce ANY corpus exactly, with no
+     *     convention to agree on: replaying a "\n"-joined corpus means passing
+     *     `"\n" + turnText` as the turn.  The separator rides along IN the
+     *     turn bytes, where it belongs.  There is nothing to configure and no
+     *     mode to select.
+     *  4. Inference is not exact-match anyway.  Recognition works over
+     *     sub-spans, canonical equivalence and resonance, so a query that
+     *     differs from the trained bytes by punctuation or whitespace still
+     *     reaches the trained forms; it degrades, it does not fail closed.
+     *
+     *  What follows from 1–4: differing separator bytes between a corpus and a
+     *  query is an ordinary CONTENT difference — the same kind as any other
+     *  wording difference — and it is measured the same way.  It is NOT an
+     *  incompatibility between the trainer and this API, and it does NOT
+     *  require choosing a project-wide separator convention.  A review that
+     *  concludes otherwise (this one did, before being corrected) has mistaken
+     *  its own harness feeding untrained bytes for an architectural defect. */
     addTurn(conv: Conversation, turn: Input): ConversationState;
     /** Grow a conversation's accumulated context by one turn's bytes — raw
      *  append plus a boundary offset, pyramid advanced by O(turn), the grown

package/dist/src/mind/mind.js

@@ -10,16 +10,16 @@
 // Implementation split across src/mind/*.ts — this file assembles the Mind class.
 import { makeKeyring, rng, setVecConfig } from "../vec.js";
 import { Alphabet } from "../alphabet.js";
-import { bytesToTree, reachThreshold, } from "../geometry.js";
+import { contentFoldIncremental, reachThreshold, } from "../geometry.js";
 import { BoundedMap } from "../store.js";
 import { SQliteStore } from "../store-sqlite.js";
 import { resolveConfig } from "../config.js";
-import { canonHash, textCanon } from "../canon.js";
+import { canonHash, textCanon, textEdgeTrim } from "../canon.js";
 import { bytesEqual, concat2 } from "../bytes.js";
 import { GraphSearch, } from "./graph-search.js";
 import { Alu } from "../alu/src/index.js";
 import { decodeText, Rationale, } from "./rationale.js";
-import { gistOf, inputBytes, latin1Key, perceive as perceiveImpl, resolve as resolveImpl, } from "./primitives.js";
+import { gistOf, inputBytes, perceive as perceiveImpl, perceiveKey, resolve as resolveImpl, } from "./primitives.js";
 import { chooseNext, edgeAncestors as edgeAncestorsFn, invalidateStructuralCaches, } from "./traverse.js";
 import { invalidateJunctionCache } from "./junction.js";
 import { follow } from "./match.js";
@@ -322,12 +322,58 @@ export class Mind {
             provenance: thought.provenance,
         };
     }
+    /** Answer ONE self-contained input.
+     *
+     *  A MULTI-TURN context is not that, and this is the wrong entry point for
+     *  it.  `respond` folds the bytes it is handed with no boundary set, because
+     *  nothing in a flat byte string says where one turn ended — only the caller
+     *  who assembled it knows, which is the whole reason `boundaries` is a
+     *  parameter of {@link perceiveImpl} and never inferred from content.  A
+     *  conversation deposited through {@link ingest} folds its contexts over
+     *  those turn boundaries, so a hand-concatenated transcript passed here
+     *  folds differently from the way it was learnt and reaches the trained
+     *  context node only by luck (measured on a 7-turn conversation: 5/7 here
+     *  against 7/7 through {@link respondTurn}, same bytes).  Use
+     *  {@link beginConversation} + {@link respondTurn}, or {@link addTurn} to
+     *  replay turns the Mind should hear but not answer. */
     async respond(input, inspectRationale) {
         // A STRING input is text by nature: it carries the text equivalence even
         // through the generic entry point.  Raw bytes / grids carry only the
         // Mind-level canon option, if any.
         const canon = this._canonFor(typeof input === "string" ? textCanon : null);
-        return this._respondImpl(inputBytes(this, input), inspectRationale, "respond", canon);
+        // EDGE WHITESPACE IS NOT PART OF THE QUESTION — trim it once, here, so
+        // every mechanism downstream sees the same question regardless of how the
+        // caller spaced it.  See canon.ts's textEdgeTrim for why the outer edges of a
+        // whole input are exactly where canon.ts's no-trimming hazard cannot arise.
+        // Gated on the SAME modality test as the canonicalizer above: for bytes and
+        // grids 0x20 is content, and nothing is trimmed.
+        //
+        // Measured on the 15.7M-node store: without this, one leading space took
+        // `Who wrote Romeo and Juliet?` and `What is the chemical symbol for
+        // water?` from answered to silent, because a shift re-seats every fold
+        // boundary — the whole of analyze_training.ts's K2 phase-robustness gap.
+        // The caller's EXACT bytes are tried first and the trim is a RETRY, not a
+        // pre-filter.  Trimming up front is asymmetric — it normalises the query but
+        // not the stored forms — so it breaks byte-exact identity for a form trained
+        // WITH edge whitespace: test/04 deposits ["  ice  ", "cold"] and asks
+        // "  ice  ", which must keep answering.  Retrying preserves that (the raw
+        // query resolves on the first pass) while still reaching the padded case
+        // (the raw query grounds nothing, the trimmed one does).
+        //
+        // COST: nothing on any answering path.  The retry needs BOTH silence AND
+        // edge whitespace on the query, the same "only on the already-failed path"
+        // discipline test/44 and the bridge's own trim retry use.  The conversation
+        // entry point (respondTurn) is deliberately NOT trimmed — it tracks
+        // turn-boundary offsets into its accumulated context, and shifting the bytes
+        // under those offsets would desync them.
+        const bytes = inputBytes(this, input);
+        const first = await this._respondImpl(bytes, inspectRationale, "respond", canon);
+        if (first.bytes.length > 0 || typeof input !== "string")
+            return first;
+        const trimmed = textEdgeTrim(bytes);
+        if (trimmed.length === bytes.length || trimmed.length === 0)
+            return first;
+        return this._respondImpl(trimmed, inspectRationale, "respond", canon);
     }
     /** Text view of {@link respond}.  NUL bytes (0x00) are stripped before
      *  decoding — they are structural padding in text answers.  LOSSY for a
@@ -354,15 +400,35 @@ export class Mind {
     beginConversation(state) {
         const id = this._nextConvId++;
         const initBytes = state?.context ?? new Uint8Array(0);
-        const initBoundaries = state?.boundaries ? [...state.boundaries] : [];
+        // NORMALISE CALLER-SUPPLIED BOUNDARIES.  `boundaries` is documented
+        // strictly increasing and every boundary this class produces is (they are
+        // appended as the context grows), but a restored {@link ConversationState}
+        // comes from OUTSIDE — hand-built, migrated, or round-tripped through a
+        // store that did not preserve order.  The folds consume boundaries with a
+        // sequential `b > prev` filter, so an out-of-order entry is silently
+        // DROPPED rather than rejected, and the conversation would then fold over
+        // a different cut set than the one the caller believes it restored.
+        // `bytesToTree` used to sort on the way in and absorbed this; the
+        // incremental fold this now calls does not, so the normalisation belongs
+        // here, at the one public door untrusted boundaries come through.
+        const initBoundaries = state?.boundaries
+            ? [...new Set(state.boundaries)]
+                .filter((b) => b > 0 && b < initBytes.length)
+                .sort((a, b) => a - b)
+            : [];
         const initAnswered = state?.answeredSpans
             ? state.answeredSpans.map(([start, end]) => [start, end])
             : initBoundaries.flatMap((start, i, cuts) => i % 2 === 0 && i + 1 < cuts.length
                 ? [[start, cuts[i + 1]]]
                 : []);
-        const tree = bytesToTree(this.space, this.alphabet, initBytes, undefined, undefined, initBoundaries.length > 0 ? initBoundaries : undefined);
+        // The same incremental fold `_growContext` uses, so a RESTORED
+        // conversation starts with segment state its next turn can reuse — a
+        // resumed conversation is otherwise identical to a live one and must not
+        // pay a full re-fold on every turn for the rest of its life.
+        const restored = contentFoldIncremental(this.space, this.alphabet, initBytes);
         this._conversations.set(id, {
-            tree,
+            tree: restored.tree,
+            content: restored.fold,
             bytes: initBytes,
             boundaries: initBoundaries,
             answeredSpans: initAnswered,
@@ -397,7 +463,41 @@ export class Mind {
      *  This is the primitive for turns the Mind should hear but not answer:
      *  replaying a transcript, feeding the OTHER speaker's line in a
      *  prediction harness, or restoring context piecewise.  {@link
-     *  respondTurn} = addTurn + think + its own reply appended the same way. */
+     *  respondTurn} = addTurn + think + its own reply appended the same way.
+     *
+     *  ── ON SEPARATORS: THERE IS NO SEPARATOR QUESTION ────────────────────
+     *
+     *  "Never a separator" above says what this method DOES — it appends the
+     *  bytes you give it and records an OFFSET — not that separator bytes are
+     *  forbidden, unsupported, or something the engine must be taught about.
+     *  Sema is agnostic to them, and reviewers keep mistaking that agnosticism
+     *  for a constraint.  To be explicit, because the mistake is easy:
+     *
+     *  1. A turn boundary is an OFFSET, held here, in `boundaries`.  It is
+     *     never a character the geometry scans for.  Nothing downstream asks
+     *     "what byte separates two turns?" because nothing downstream finds
+     *     boundaries by looking at content at all.
+     *  2. A separator in a CORPUS is ordinary content.  If a trainer joins
+     *     turns with "\n" (example/train_base.ts does), those newlines are
+     *     simply bytes inside the stream, folded like every other byte.  They
+     *     are a property of that corpus, not of this API and not of the fold.
+     *  3. This API can therefore reproduce ANY corpus exactly, with no
+     *     convention to agree on: replaying a "\n"-joined corpus means passing
+     *     `"\n" + turnText` as the turn.  The separator rides along IN the
+     *     turn bytes, where it belongs.  There is nothing to configure and no
+     *     mode to select.
+     *  4. Inference is not exact-match anyway.  Recognition works over
+     *     sub-spans, canonical equivalence and resonance, so a query that
+     *     differs from the trained bytes by punctuation or whitespace still
+     *     reaches the trained forms; it degrades, it does not fail closed.
+     *
+     *  What follows from 1–4: differing separator bytes between a corpus and a
+     *  query is an ordinary CONTENT difference — the same kind as any other
+     *  wording difference — and it is measured the same way.  It is NOT an
+     *  incompatibility between the trainer and this API, and it does NOT
+     *  require choosing a project-wide separator convention.  A review that
+     *  concludes otherwise (this one did, before being corrected) has mistaken
+     *  its own harness feeding untrained bytes for an architectural defect. */
     addTurn(conv, turn) {
         const data = this._conversations.get(conv.id);
         if (!data)
@@ -422,10 +522,37 @@ export class Mind {
         const grown = prevLen > 0 ? concat2(data.bytes, turnBytes) : turnBytes;
         if (prevLen > 0)
             data.boundaries.push(prevLen);
-        const tree = bytesToTree(this.space, this.alphabet, grown, undefined, undefined, data.boundaries.length > 0 ? data.boundaries : undefined);
+        // THE PLAIN FOLD, INCREMENTALLY.  No boundary set is imposed here: the
+        // tree is exactly the tree `perceive(grown)` builds for these bytes, which
+        // is exactly the tree the DEPOSIT path folded when it learnt them.  That
+        // agreement is the whole point — it is what lets a cumulative context
+        // resolve to its trained node, and when it was absent the alignment family
+        // went quadratic (measured: 5.2M cells on a 476-byte context, against 0
+        // when the two sides agree).
+        //
+        // The optimisation is unaffected by dropping the boundaries, because it
+        // never came from them: content cuts are stable under append, so the
+        // incremental fold reuses every segment left of the new turn as the SAME
+        // object (see contentFoldIncremental).  That object identity is what
+        // `resolvedSubtrees` — a WeakMap keyed by node identity — needs in order
+        // to hit at all.  Measured against the stable-prefix fold it replaces:
+        // ~40 rebuilt nodes per turn either way, flat as the context grows
+        // sevenfold, and ~92% of nodes reused by identity in both.
+        //
+        // `data.boundaries` is still tracked, and is still exact — it is API
+        // metadata (ConversationState, answeredSpans, currentTurnStart), not a
+        // fold instruction.
+        const folded = contentFoldIncremental(this.space, this.alphabet, grown, data.content);
+        const tree = folded.tree;
+        data.content = folded.fold;
         data.tree = tree;
         data.bytes = grown;
-        data.perceiveMemo.set(latin1Key(grown), tree);
+        // Seeded under the PLAIN content key, and that is now the only key there
+        // is: with no boundary set imposed, this tree IS what `perceive(grown)`
+        // computes, so the memo entry is an ordinary cache hit rather than the
+        // deliberate alias it had to be while the two folds differed.  The entry
+        // saves the pipeline re-folding the context it was just handed.
+        data.perceiveMemo.set(perceiveKey(grown), tree);
         return tree;
     }
     /** Process one turn of a conversation.

package/dist/src/mind/pipeline-mechanism.d.ts

@@ -122,6 +122,13 @@ export interface MechanismResult {
     unexplained: string;
     /** Explicit weight override.  When absent, weight = moves + PASS·unaccounted. */
     weight?: number;
+    /** Bytes of `bytes` that came from spans nothing recognised — the asker's
+     *  own words carried through verbatim rather than derived (see
+     *  {@link liftedScaffolding}).  Reported, not priced: the ladder prices what
+     *  a candidate leaves UNACCOUNTED, and this orders candidates that tie on
+     *  exactly that.  Omit when a mechanism composes its answer entirely from
+     *  recognised material, which is the usual case. */
+    scaffolding?: number;
     /** Override the mechanism's default provenance for this result.
      *  When absent, the pipeline uses `mech.provenance`. */
     provenance?: string;