@hviana/sema 0.4.7 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/AGENTS.md +290 -77
  2. package/HOW_IT_WORKS.md +2170 -735
  3. package/dist/example/train_base.d.ts +9 -3
  4. package/dist/example/train_base.js +21 -4
  5. package/dist/src/canon.d.ts +19 -0
  6. package/dist/src/canon.js +28 -0
  7. package/dist/src/geometry.d.ts +52 -0
  8. package/dist/src/geometry.js +87 -1
  9. package/dist/src/mind/attention.d.ts +15 -10
  10. package/dist/src/mind/attention.js +15 -10
  11. package/dist/src/mind/bridge.js +27 -1
  12. package/dist/src/mind/frame-filler.d.ts +15 -0
  13. package/dist/src/mind/frame-filler.js +535 -0
  14. package/dist/src/mind/learning.js +6 -11
  15. package/dist/src/mind/mechanisms/cast.js +72 -2
  16. package/dist/src/mind/mechanisms/cover.js +6 -1
  17. package/dist/src/mind/mechanisms/extraction.js +27 -0
  18. package/dist/src/mind/mechanisms/recall.js +214 -34
  19. package/dist/src/mind/mind.d.ts +52 -3
  20. package/dist/src/mind/mind.js +140 -12
  21. package/dist/src/mind/pipeline-mechanism.d.ts +7 -0
  22. package/dist/src/mind/pipeline.js +29 -1
  23. package/dist/src/mind/prefix-completion.d.ts +59 -0
  24. package/dist/src/mind/prefix-completion.js +270 -0
  25. package/dist/src/mind/primitives.d.ts +29 -10
  26. package/dist/src/mind/primitives.js +98 -71
  27. package/dist/src/mind/recognition.js +153 -26
  28. package/dist/src/mind/traverse.d.ts +32 -0
  29. package/dist/src/mind/traverse.js +52 -0
  30. package/dist/src/mind/types.d.ts +61 -18
  31. package/dist/src/mind/types.js +68 -19
  32. package/dist/src/store.d.ts +21 -0
  33. package/dist/src/store.js +21 -0
  34. package/example/train_base.ts +21 -4
  35. package/package.json +1 -1
  36. package/src/canon.ts +28 -0
  37. package/src/geometry.ts +100 -1
  38. package/src/mind/attention.ts +15 -10
  39. package/src/mind/bridge.ts +34 -0
  40. package/src/mind/frame-filler.ts +604 -0
  41. package/src/mind/learning.ts +5 -9
  42. package/src/mind/mechanisms/cast.ts +70 -2
  43. package/src/mind/mechanisms/cover.ts +6 -1
  44. package/src/mind/mechanisms/extraction.ts +27 -0
  45. package/src/mind/mechanisms/recall.ts +236 -37
  46. package/src/mind/mind.ts +166 -18
  47. package/src/mind/pipeline-mechanism.ts +7 -0
  48. package/src/mind/pipeline.ts +33 -1
  49. package/src/mind/prefix-completion.ts +314 -0
  50. package/src/mind/primitives.ts +105 -80
  51. package/src/mind/recognition.ts +151 -23
  52. package/src/mind/traverse.ts +52 -0
  53. package/src/mind/types.ts +104 -44
  54. package/src/store.ts +25 -0
  55. package/test/13-conversation.test.mjs +13 -0
  56. package/test/57-fusion-order.test.mjs +65 -0
  57. package/test/66-query-edge-whitespace.test.mjs +99 -0
  58. package/test/67-climb-anchor-breadth.test.mjs +113 -0
  59. package/test/68-extraction-unanchored.test.mjs +79 -0
  60. package/test/69-frame-filler.test.mjs +115 -0
  61. package/test/70-prefix-completion.test.mjs +170 -0
  62. package/test/71-embedded-canon-equivalence.test.mjs +121 -0
  63. package/test/72-prefix-candidate-supply.test.mjs +114 -0
  64. package/test/73-scaffolding-only-bridge-abstains.test.mjs +178 -0
  65. package/test/74-prefix-trap-not-sprung-early.test.mjs +114 -0
  66. package/test/75-multiturn-context-optimisation.test.mjs +1334 -0
@@ -7,16 +7,17 @@ import { Vec } from "../vec.js";
7
7
  import { Sema } from "../sema.js";
8
8
  import {
9
9
  bytesToTree,
10
+ contentFoldIncremental,
10
11
  Grid,
11
12
  gridToTree,
12
13
  hilbertBytes,
13
- stablePrefixFoldIncremental,
14
14
  stackGrids,
15
15
  } from "../geometry.js";
16
16
  import { canonHash } from "../canon.js";
17
17
  import { bytesEqual } from "../bytes.js";
18
18
  import { ALL } from "./types.js";
19
- import type { DepositCacheEntry, Input, MindContext } from "./types.js";
19
+ import type { Input, MindContext } from "./types.js";
20
+ import type { ContentFold } from "../geometry.js";
20
21
 
21
22
  // ── Address: bytes → node ──────────────────────────────────────────────
22
23
 
@@ -35,6 +36,26 @@ export function latin1Key(bytes: Uint8Array): string {
35
36
  return s;
36
37
  }
37
38
 
39
+ /** The {@link perceive} memo key: the span's content PLUS the boundary set it
40
+ * was folded under. The tree is a function of BOTH — the same bytes fold
41
+ * plainly with no boundaries and into a left-nested stable-prefix shape with
42
+ * them — so a content-only key returns whichever shape was computed first.
43
+ * That is exactly what happened: a conversation seeded its cumulative context
44
+ * under the content key, and every later plain `perceive` of those bytes was
45
+ * served the boundary tree instead (measured: respondTurn answered where
46
+ * respond() on byte-identical input did not). NUL separates the two parts —
47
+ * the boundary rendering is digits and commas, so no content byte can forge
48
+ * the split. */
49
+ export function perceiveKey(
50
+ bytes: Uint8Array,
51
+ boundaries?: readonly number[],
52
+ ): string {
53
+ const k = latin1Key(bytes);
54
+ return boundaries === undefined || boundaries.length === 0
55
+ ? k
56
+ : k + "\u0000" + boundaries.join(",");
57
+ }
58
+
38
59
  /** Perceive input into a content-defined tree (the river fold).
39
60
  * Deterministic — identical bytes always produce an identical tree.
40
61
  *
@@ -61,7 +82,7 @@ export function perceive(
61
82
  // The tree is shared by reference; Sema nodes are never mutated.
62
83
  const memo = ctx.perceiveMemo;
63
84
  if (memo) {
64
- const key = latin1Key(bytes);
85
+ const key = perceiveKey(bytes, boundaries);
65
86
  const hit = memo.get(key);
66
87
  if (hit !== undefined) {
67
88
  if (ctx.meter) ctx.meter.perceiveHits++;
@@ -104,78 +125,46 @@ export function perceive(
104
125
  }
105
126
 
106
127
  /** The DEPOSIT-shaped perceive. Folds over the stream's own content cuts —
107
- * bit-identical to what inference computes for the same bytes, and that
108
- * train/inference agreement is load-bearing for exact recall. An input that
109
- * EXTENDS a previously deposited one is a conversation context grown by one
110
- * turn; the cached prefix length IS the turn boundary (derived from the deposit
111
- * sequence itself, never from a content convention) and joins the cut set, so
112
- * the trained context node and the query's context subtree are the SAME node.
113
- * Segment folds reuse across deposits ({@link stablePrefixFoldIncremental}) —
114
- * O(turn) instead of O(context) per turn. All of it is purely a cache: an
115
- * evicted chain loses only the turn boundaries, and since the content cuts do
116
- * not depend on the cache, the segments themselves are unchanged. */
128
+ * bit-identical to what inference computes for the same bytes. That
129
+ * train/inference agreement is the whole contract: the trained context node
130
+ * and the node `resolve(query)` reaches must be the SAME node, and the only
131
+ * way to guarantee it is to give this function nothing extra to say. It
132
+ * imposes no boundaries, knows nothing about turns, and reads no convention
133
+ * out of the bytes.
134
+ *
135
+ * An input that EXTENDS a previously deposited one a conversation context
136
+ * grown by a turn, or a resumed replay reuses that deposit's already-folded
137
+ * content segments ({@link contentFoldIncremental}), so it costs O(new bytes)
138
+ * instead of O(context). The reuse is TRANSPARENT by construction: a segment
139
+ * is a pure function of its own bytes, so a reused one is bit-identical to a
140
+ * refolded one. Nothing has to prove that the extending deposit is "really"
141
+ * a next turn — a coincidental byte prefix reuses the same segments and gets
142
+ * the same tree it would have got anyway. (It used to matter: while this
143
+ * path imposed turn BOUNDARIES, a wrong guess changed the tree, so the cache
144
+ * needed a continuation-bytes proof to gate it. Nothing is imposed now, so
145
+ * there is nothing to gate.) */
117
146
  export function perceiveDeposit(
118
147
  ctx: MindContext,
119
148
  bytes: Uint8Array,
120
149
  conversational = false,
121
150
  ): Sema {
122
- let prev: DepositCacheEntry | undefined;
123
- let prefixLen = 0;
124
- // Cache consult (both boundary lookup and stable-prefix reuse) is scoped
125
- // to conversational deposits only a bare, unrelated fact whose bytes
126
- // happen to extend an earlier deposit is NOT a conversation turn, and
127
- // must keep the plain fold so it shares structure with ITS OWN prior
128
- // deposits, not fragment against a coincidental byte-prefix.
129
- if (conversational) {
130
- // Longest cached PROPER prefix first.
131
- const lens = [...ctx._depositLens]
132
- .filter((L) => L >= 2 && L < bytes.length)
133
- .sort((a, b) => b - a);
134
- for (const L of lens) {
135
- const hit = ctx._depositTrees.get(latin1Key(bytes.subarray(0, L)));
136
- // The suffix must bytes-equal the hit's OWN recorded continuation —
137
- // proof this deposit is that turn's actual next turn, not a fact
138
- // that coincidentally shares its byte prefix.
139
- if (
140
- hit !== undefined && hit.nextBytes !== undefined &&
141
- bytesEqual(hit.nextBytes, bytes.subarray(L))
142
- ) {
143
- prev = hit;
144
- prefixLen = L;
145
- break;
146
- }
151
+ // Longest cached PROPER prefix first — the most segments to reuse.
152
+ let prev: ContentFold | undefined;
153
+ const lens = [...ctx._depositLens]
154
+ .filter((L) => L >= 2 && L < bytes.length)
155
+ .sort((a, b) => b - a);
156
+ for (const L of lens) {
157
+ const hit = ctx._depositTrees.get(latin1Key(bytes.subarray(0, L)));
158
+ if (hit !== undefined) {
159
+ prev = hit.content;
160
+ break;
147
161
  }
148
162
  }
149
- // ONLY turn boundaries belong here. The stream's own content cuts are NOT
150
- // passed in: `bytesToTree` and `stablePrefixFoldIncremental` both derive them
151
- // per span, at every level, and handing the level-0 cuts in as stable-prefix
152
- // boundaries instead produces a LEFT-NESTED join of flat segments a
153
- // different tree from the one inference builds for the same bytes. When that
154
- // happened, a deposit's context root and `resolve(question)` were different
155
- // nodes, so the trained edge hung off a node inference never reached and
156
- // recall went silent (test/44 caught it as a site that could not be emitted
157
- // because the resolved node led nowhere). Train and infer must fold
158
- // identically; the way to guarantee that is to give this function nothing
159
- // extra to say.
160
- const cuts = new Set<number>();
161
- if (prev !== undefined) {
162
- for (const b of prev.boundaries) cuts.add(b);
163
- cuts.add(prefixLen);
164
- }
165
- const boundaries = [...cuts].sort((a, b) => a - b);
166
- const folded = stablePrefixFoldIncremental(
167
- ctx.space,
168
- ctx.alphabet,
169
- bytes,
170
- boundaries,
171
- prev?.stable,
172
- );
173
- const tree = folded.tree;
174
- const entry: DepositCacheEntry = { boundaries, stable: folded.fold };
175
- // Only a conversational deposit writes the cache too — otherwise a bare
176
- // fact's plain fold could later be misread as a conversation's turn-zero
177
- // boundary by an unrelated conversational deposit that happens to extend
178
- // its bytes.
163
+ const folded = contentFoldIncremental(ctx.space, ctx.alphabet, bytes, prev);
164
+ // Only a CONVERSATIONAL deposit writes the cache: reuse is sound for any
165
+ // deposit, but the budget is 8 entries and a corpus of unrelated facts would
166
+ // evict the live chains for nothing. Purely a cost decision now, not a
167
+ // correctness one.
179
168
  if (conversational && bytes.length >= 2) {
180
169
  // The lengths set drifts as the map evicts; past the probe budget the
181
170
  // drift itself becomes the cost (each stale length is an O(len) key
@@ -184,10 +173,10 @@ export function perceiveDeposit(
184
173
  ctx._depositLens.clear();
185
174
  ctx._depositTrees.clear();
186
175
  }
187
- ctx._depositTrees.set(latin1Key(bytes), entry);
176
+ ctx._depositTrees.set(latin1Key(bytes), { content: folded.fold });
188
177
  ctx._depositLens.add(bytes.length);
189
178
  }
190
- return tree;
179
+ return folded.tree;
191
180
  }
192
181
 
193
182
  /** The raw bytes of an input — modality-neutral conversion. */
@@ -214,21 +203,48 @@ export function foldTree(
214
203
  start: number,
215
204
  visit?: (n: Sema, start: number, end: number, node: number | null) => void,
216
205
  ): { end: number; node: number | null } {
217
- // Fast path: subtree already resolved (from a previous conversation turn
218
- // or an earlier recognition pass). The pyramid reuses prefix subtrees as
219
- // identical Sema objects, so this cache turns foldTree into O(suffix)
220
- // instead of O(context) for multi-turn recognition.
206
+ // Subtree already resolved (from a previous conversation turn or an earlier
207
+ // recognition pass). The pyramid reuses prefix subtrees as identical Sema
208
+ // objects, so a conversation's prefix is warm from its second turn on.
209
+ // Without a visitor that makes foldTree O(suffix) instead of O(context);
210
+ // with one it stays O(context) and saves the per-node store probes instead
211
+ // (see below for why the distinction is not negotiable).
212
+ //
213
+ // WHAT THE CACHE KNOWS, AND WHAT IT DOES NOT. An entry records this
214
+ // subtree's id and byte length — nothing about its DESCENDANTS' spans.
215
+ // Returning here therefore emits ONE visit() where a cold walk emits one per
216
+ // node, and `visit` is not instrumentation: recognise() emits its sites from
217
+ // it (recognition.ts) and attention's collectRegions votes over what it
218
+ // yields (attention.ts). Skipping the descent silently shrinks the evidence
219
+ // those mechanisms see, purely because the cache happened to be warm.
220
+ //
221
+ // That is not hypothetical and not an edge case — it is every conversation
222
+ // turn after the first. `contentFoldIncremental` deliberately shares prefix
223
+ // segment OBJECTS across turns (~99% reuse), so by turn 2 the prefix is
224
+ // warm; meanwhile recogniseMemo/climbMemo are keyed on exact query BYTES,
225
+ // which a growing context never repeats. Warm subtrees + missed memos is
226
+ // the unprotected quadrant. Measured over real trained conversations,
227
+ // recognising the same context with a warm prefix lost 67-92% of its leaves
228
+ // (772->204, 589->47, 872->291, 377->37) with `sites` unchanged, so the loss
229
+ // is invisible to the coarse counts; a direct foldTree probe on identical
230
+ // bytes and an identical tree object fired visit() 661 times cold and 37
231
+ // warm. respond() is immune only because it never sets _resolvedSubtrees
232
+ // (mind.ts) — the degradation was unique to the multi-turn API.
233
+ //
234
+ // So the fast path is taken only when NOBODY IS WATCHING. With a visitor
235
+ // present we still walk, and the cache degrades to the thing it soundly is:
236
+ // an elision of the store probes (findLeaf/findBranch) at each node, not an
237
+ // elision of the traversal. Ids still come from the cache, so a warm walk
238
+ // is cheaper than a cold one; it is no longer *different* from one.
221
239
  const cached = ctx._resolvedSubtrees?.get(n);
222
- if (cached !== undefined) {
223
- const end = start + cached.len;
224
- visit?.(n, start, end, cached.id);
225
- return { end, node: cached.id };
240
+ if (cached !== undefined && visit === undefined) {
241
+ return { end: start + cached.len, node: cached.id };
226
242
  }
227
243
 
228
244
  if (n.kids === null) {
229
245
  const b = n.leaf ?? new Uint8Array(0);
230
246
  const end = start + b.length;
231
- const node = ctx.store.findLeaf(b);
247
+ const node = cached !== undefined ? cached.id : ctx.store.findLeaf(b);
232
248
  visit?.(n, start, end, node);
233
249
  if (node !== null && ctx._resolvedSubtrees) {
234
250
  ctx._resolvedSubtrees.set(n, { id: node, len: b.length });
@@ -244,7 +260,16 @@ export function foldTree(
244
260
  else if (known) kids.push(r.node);
245
261
  pos = r.end;
246
262
  }
247
- const node = known ? ctx.store.findBranch(kids) : null;
263
+ // Same store-probe elision as the leaf case: a cached entry already names
264
+ // this subtree, so the descent above was for `visit`'s benefit alone and the
265
+ // id need not be re-derived. Using it also keeps a warm walk's ids
266
+ // bit-identical to a cold walk's rather than re-deriving them from children
267
+ // that may themselves have come from cache.
268
+ const node = cached !== undefined
269
+ ? cached.id
270
+ : known
271
+ ? ctx.store.findBranch(kids)
272
+ : null;
248
273
  visit?.(n, start, pos, node);
249
274
  if (node !== null && ctx._resolvedSubtrees) {
250
275
  ctx._resolvedSubtrees.set(n, { id: node, len: pos - start });
@@ -17,6 +17,7 @@ import {
17
17
  } from "./primitives.js";
18
18
  import { atomIsHub, corpusN, leadsSomewhere } from "./traverse.js";
19
19
  import { chainReach, leafIdAt, leafIdRun } from "./canonical.js";
20
+ import { canonHash } from "../canon.js";
20
21
  import { isChunk, type Sema } from "../sema.js";
21
22
  import type { Leaf, Site } from "./graph-search.js";
22
23
 
@@ -36,23 +37,40 @@ export function recognise(ctx: MindContext, bytes: Uint8Array): Recognition {
36
37
  // Content-keyed memo — works for both single-turn respond() and multi-turn
37
38
  // respondTurn() (where the map persists across calls). ALWAYS consulted,
38
39
  // regardless of tracing — matching perceive()'s own memo, which carries no
39
- // trace gate at all. This memo is NOT an optional accelerator: recogniseImpl
40
- // walks the query's perceived tree via foldTree, whose subtree-resolution
41
- // fast path (see primitives.ts) skips invoking `visit` and therefore
42
- // skips EMITTING SITES for any subtree already cached in
43
- // ctx._resolvedSubtrees. A multi-turn conversation's stable-prefix fold
44
- // deliberately shares node OBJECTS across turns, so by the second call on
45
- // the exact same bytes, large swaths of the tree are already cached and
46
- // foldTree stops short of recursing into them a second recogniseImpl
47
- // call on the SAME bytes is not idempotent; it silently finds FEWER sites
48
- // than the first (observed live: 31 sites → 5 on an immediate repeat
49
- // call). Skipping this memo "only while tracing" used to mean every
50
- // traced turn re-ran recogniseImpl from scratch at every one of the many
51
- // call sites that recognise the same query (cover, reason, articulate...),
52
- // each subsequent call silently more incomplete than the last measurably
53
- // changing which mechanism grounds the answer, not just costing time. The
54
- // trace step must still fire on every call regardless (a cache hit is not
55
- // silent), so it is emitted here directly instead of only inside
40
+ // trace gate at all.
41
+ //
42
+ // This memo is an accelerator, and that is now the whole of it: repeated
43
+ // recognition of the same query is ordinary within one response (cover,
44
+ // reason and articulate all recognise it) and recogniseImpl is O(n ·
45
+ // maxGroup) probes each time.
46
+ //
47
+ // IT USED TO BE LOAD-BEARING FOR CORRECTNESS, and the history is worth
48
+ // keeping because it explains why there is no trace gate here. foldTree's
49
+ // subtree-resolution fast path (primitives.ts) once returned on a cache hit
50
+ // WITHOUT recursing, so it skipped invoking `visit` and therefore skipped
51
+ // EMITTING SITES for any subtree already in ctx._resolvedSubtrees. A
52
+ // conversation's incremental fold deliberately shares node OBJECTS across
53
+ // turns, so by the second call on the same bytes large swaths of the tree
54
+ // were already cached and recogniseImpl silently found FEWER sites than the
55
+ // first call (observed live: 31 5). Skipping this memo "only while
56
+ // tracing" therefore meant every traced turn re-ran recogniseImpl at each of
57
+ // those call sites, each result more incomplete than the last — changing
58
+ // which mechanism grounded the answer, not merely costing time.
59
+ //
60
+ // foldTree no longer does that: it takes the fast path only when no `visit`
61
+ // is supplied, so a walk that emits sites always walks in full and the id
62
+ // cache is reduced to eliding store probes (see primitives.ts). recognise()
63
+ // is idempotent on its own now — verified with the memo bypassed, the
64
+ // subtree cache warm and the tree object shared: three consecutive calls on
65
+ // the same 544-byte context returned sites=2 leaves=544 splits=0 starts=88,
66
+ // identical every time.
67
+ //
68
+ // The unconditional consult STAYS regardless. A memo whose absence can only
69
+ // cost time is still not something to gate on whether an audit happens to be
70
+ // attached: tracing must not change what the pipeline computes, and the
71
+ // cheapest way to guarantee that is for the trace flag to touch nothing but
72
+ // the trace. The trace step must still fire on every call (a cache hit is
73
+ // not silent), so it is emitted here directly rather than only inside
56
74
  // recogniseImpl.
57
75
  if (ctx.recogniseMemo) {
58
76
  const key = latin1Key(bytes);
@@ -408,7 +426,21 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
408
426
  // before resolveSpan pays for a fold; approximate evidence never enters.
409
427
  // This tier is needed only where atom chains are suppressed. Small stores
410
428
  // retain their existing decomposition unchanged.
411
- if (atomsAreHubs) {
429
+ // ALWAYS ON, AND LINEAR. This used to be gated on `atomsAreHubs` — small
430
+ // stores were said to "retain their existing decomposition unchanged", which
431
+ // was true only while the query's fold was told where the turns were: every
432
+ // turn was then a NODE, so the structural walk found it and this tier had
433
+ // nothing to add. The fold no longer imposes turn boundaries (a turn start
434
+ // is an ordinary interior offset now), so a trained form embedded in a
435
+ // longer query is reachable ONLY here — the chain caps at chainReach(W)=W²
436
+ // bytes and cannot span one. Measured: with the fold imposing boundaries
437
+ // every turn is a node; without it, none is.
438
+ //
439
+ // Ungating it alone made inference QUADRATIC (test/14's constant-KB/s guard
440
+ // went to 41.8s): every offset near a cut is an endpoint, and each probe
441
+ // costs O(span) to slice the leaf-id run and hash it. The budget below is
442
+ // what makes it affordable — see `spend`.
443
+ {
412
444
  const allLeafIds = singleLeaf.map((x) => x?.id ?? null);
413
445
  if (allLeafIds.every((x): x is number => x !== null)) {
414
446
  const radius = ctx.space.seats.length;
@@ -421,15 +453,111 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
421
453
  ) endpoints.add(p);
422
454
  }
423
455
  const ordered = [...endpoints].sort((a, b) => a - b);
424
- const probe = (start: number, end: number): void => {
456
+ // The leaf-id run is BYTE-EXACT, while `resolveSpan` behind it resolves
457
+ // exactly OR canonically — so this gate was strictly narrower than its
458
+ // own resolver, and every embedded form differing from its deposit only
459
+ // by the response's equivalence (case, width) was dropped before the
460
+ // resolver ever saw it. Rebuilding the run over canonicalized bytes
461
+ // does NOT fix that: a differently-cased deposit's branch kid-ids are
462
+ // not the query's leaf-id run under ANY canonicalization of the query,
463
+ // so the second admission route has to be the canon INDEX itself — the
464
+ // same candidate proposal `canonResolve` makes, and the same
465
+ // cheap-probe-before-a-fold discipline the exact route already follows
466
+ // (a hash and an indexed lookup; no fold, no vector, no scan). Both
467
+ // routes only PROPOSE; `resolveSpan` still decides, so a hash-bucket
468
+ // collision costs one fold and can never emit a wrong site (test/71).
469
+ const canonAdmits = (start: number, end: number): boolean => {
470
+ const canon = ctx.canon;
471
+ if (canon === null || !store.canonFind) return false;
472
+ const key = canon(bytes.subarray(start, end));
473
+ if (key.length === 0) return false;
474
+ return store.canonFind(canonHash(key)).length > 0;
475
+ };
476
+ // The byte-exact route probes the SPAN ITSELF (see
477
+ // Store.findFlatBranch): for a run of single-byte leaves the flat-kid
478
+ // encoding is the identity, so the span's bytes ARE the branch key.
479
+ // `subarray` is a view — this allocates nothing per probe, and the
480
+ // bloom filter answers the misses without touching the database.
481
+ const flatProbe = (start: number, end: number): number | null =>
482
+ store.findFlatBranch
483
+ ? store.findFlatBranch(bytes.subarray(start, end))
484
+ : store.findBranch(allLeafIds.slice(start, end));
485
+ // THE TWO ROUTES COST DIFFERENT THINGS, SO THEY ARE PRICED SEPARATELY.
486
+ //
487
+ // The exact route is a bloom-gated hash over a subarray VIEW: no
488
+ // allocation, and a miss never reaches the database. It is cheap enough
489
+ // to run on every endpoint, and that is what makes this tier able to
490
+ // find a trained form embedded anywhere in the query.
491
+ //
492
+ // The canon route is not: it runs the canonicalizer over the span
493
+ // (NFKC, case-fold, whitespace) and allocates a fresh key for every
494
+ // probe. That is the O(span) cost with the heavy constant, and it is
495
+ // the one worth a budget. Sharing ONE budget between them made the
496
+ // cheap route starve on the expensive one's behalf — measured, test/71's
497
+ // embedded differently-cased form needed 64x the budget to be found,
498
+ // while the exact route it was competing with needed none of it.
499
+ const probe = (
500
+ start: number,
501
+ end: number,
502
+ canonBudget: boolean,
503
+ ): void => {
425
504
  if (end - start < W || end - start <= chainReach(W)) return;
426
- const ids = allLeafIds.slice(start, end);
427
- if (store.findBranch(ids) === null) return;
505
+ if (flatProbe(start, end) === null) {
506
+ if (!canonBudget) return;
507
+ if (!canonAdmits(start, end)) return;
508
+ }
428
509
  const id = resolveSpan(start, end);
429
510
  if (id !== null) emit(start, end, id);
430
511
  };
431
- for (const end of ordered) probe(0, end);
432
- for (const start of ordered) probe(start, bytes.length);
512
+ // A CUMULATIVE BYTE BUDGET, SPENT SHORTEST-SPAN-FIRST.
513
+ //
514
+ // Each probe costs O(span), and there are O(n) endpoints, so probing
515
+ // them all is O(n²) — the quadratic this tier was gated to avoid. The
516
+ // budget caps TOTAL probe bytes at a multiple of the query's own length,
517
+ // which is what keeps whole-query inference linear.
518
+ //
519
+ // Spending it shortest-first is what makes the cap a scale bound rather
520
+ // than a position bound: the tier recovers embedded forms up to roughly
521
+ // √(2·budget) bytes ANYWHERE in the endpoint set, instead of walking the
522
+ // endpoints in order and running out partway along the query. A form
523
+ // longer than that is out of this tier's reach — but so is a form the
524
+ // chain cannot span, and that is exactly the trade the budget prices.
525
+ // The factor is chainReach(W), the same W² scale the chain already
526
+ // trusts; no new constant.
527
+ // The factor is chainReach(W) — the same W² scale the chain itself
528
+ // trusts — so the cap is derived from the fold's geometry, never tuned.
529
+ // (It was briefly an environment variable while the cost was being
530
+ // measured; an env-read here would make inference non-reproducible,
531
+ // which the determinism contract forbids outright.)
532
+ // The factor is chainReach(W) — the same W² scale the chain itself
533
+ // trusts — so the cap is derived from the fold's geometry, never tuned.
534
+ // (It was briefly an environment variable while the cost was being
535
+ // measured; an env-read here would make inference non-reproducible,
536
+ // which the determinism contract forbids outright.)
537
+ //
538
+ // It now prices ONLY the canonicalizing route; the exact route runs on
539
+ // every endpoint regardless, so exhausting this budget narrows which
540
+ // equivalence-class forms are proposed, never which byte-exact ones.
541
+ let budget = bytes.length * chainReach(W) * chainReach(W);
542
+ const spend = (start: number, end: number): boolean => {
543
+ const span = end - start;
544
+ const afford = span <= budget;
545
+ if (afford) budget -= span;
546
+ probe(start, end, afford);
547
+ // Always keep walking: the exact route is unbudgeted, so running out
548
+ // of canon budget must not stop the scan.
549
+ return true;
550
+ };
551
+ const prefixes = ordered.filter((e) => e > 0).sort((a, b) => a - b);
552
+ const suffixes = ordered
553
+ .filter((s2) => s2 < bytes.length)
554
+ .sort((a, b) => b - a);
555
+ for (let i = 0; i < Math.max(prefixes.length, suffixes.length); i++) {
556
+ // Interleaved so neither edge starves the other when the budget runs
557
+ // out — a query can carry a trained form at either end.
558
+ if (i < prefixes.length && !spend(0, prefixes[i])) break;
559
+ if (i < suffixes.length && !spend(suffixes[i], bytes.length)) break;
560
+ }
433
561
  }
434
562
  }
435
563
 
@@ -9,6 +9,7 @@
9
9
  import { cosine, Vec } from "../vec.js";
10
10
  import type { AncestorReach, MindContext, SaturationStop } from "./types.js";
11
11
  import { gistOf, read } from "./primitives.js";
12
+ import { leafIdRun } from "./canonical.js";
12
13
 
13
14
  // ── Session structural memo ─────────────────────────────────────────────
14
15
  //
@@ -769,3 +770,54 @@ function rItemShort(
769
770
  score,
770
771
  };
771
772
  }
773
+
774
+ /** True when NO window of `query` discriminates anything — every stored
775
+ * W-window it spells is contained by more places than the hub bound allows,
776
+ * i.e. the whole query is corpus-global scaffolding.
777
+ *
778
+ * WHAT IT IS FOR. Several mechanisms ground a query through the literal
779
+ * spans it did NOT explain, and those spans are the whole of their evidence.
780
+ * When every one of them is a hub, the query says nothing the corpus can be
781
+ * held to, and grounding it means picking one of thousands of continuations
782
+ * it gives no evidence for — a fabrication whatever the answer happens to be.
783
+ * Answering with silence there is the honest degradation contract (§2.13).
784
+ *
785
+ * MEASURED SEPARATION (trained store, hubBound 571) — this is categorical,
786
+ * not marginal, and it is why the predicate lives here rather than being
787
+ * spelled twice:
788
+ * "What is the capital of" ALL saturated ("What":572) → fabricated
789
+ * "What is the capital " ALL saturated ("What":572) → fabricated
790
+ * "what is the capital of france" min "f fr":248 → correct
791
+ * "What is the capitol of France?" min "f Fr":114 → correct
792
+ * "WHAT IS THE CAPITAL OF FRANCE?" min "HE C":1 → correct
793
+ * "What is the capital of France?" min "t i":4 → correct
794
+ * "Who wrote Romeo and Juliet?" min "iet?":26 → correct
795
+ * "What is the capital of Zamunda?" min "Zamu":3 → silent anyway
796
+ * Note the last: the honest-silence probes are already refused on other
797
+ * evidence and sit on the SAME side as the correct ones, so this predicate
798
+ * is not what makes them silent and cannot be credited for them.
799
+ *
800
+ * NO NEW THRESHOLD (§2.2): `hubBound` is the √N reading of "hub" used
801
+ * everywhere, and the containment read is clamped to it exactly as every
802
+ * other fan-out read is (§2.8). A query with no stored window at all is NOT
803
+ * scaffolding-only — it has no evidence either way, and its callers already
804
+ * refuse it on their own terms. */
805
+ export function allWindowsAreScaffolding(
806
+ ctx: MindContext,
807
+ query: Uint8Array,
808
+ ): boolean {
809
+ const W = ctx.space.maxGroup;
810
+ const bound = hubBound(ctx);
811
+ let sawOne = false;
812
+ for (let o = 0; o + W <= query.length; o++) {
813
+ const ids = leafIdRun(ctx, query, o, o + W);
814
+ if (ids === null) continue;
815
+ const id = ctx.store.findBranch(ids);
816
+ if (id === null) continue;
817
+ const rarity = ctx.store.containersSlice(id, 0, bound + 1).length;
818
+ if (rarity === 0) continue;
819
+ if (rarity <= bound) return false;
820
+ sawOne = true;
821
+ }
822
+ return sawOne;
823
+ }