@hviana/sema 0.4.7 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/AGENTS.md +290 -77
  2. package/HOW_IT_WORKS.md +2170 -735
  3. package/dist/example/train_base.d.ts +9 -3
  4. package/dist/example/train_base.js +21 -4
  5. package/dist/src/canon.d.ts +19 -0
  6. package/dist/src/canon.js +28 -0
  7. package/dist/src/geometry.d.ts +52 -0
  8. package/dist/src/geometry.js +87 -1
  9. package/dist/src/mind/attention.d.ts +15 -10
  10. package/dist/src/mind/attention.js +15 -10
  11. package/dist/src/mind/bridge.js +27 -1
  12. package/dist/src/mind/frame-filler.d.ts +15 -0
  13. package/dist/src/mind/frame-filler.js +535 -0
  14. package/dist/src/mind/learning.js +6 -11
  15. package/dist/src/mind/mechanisms/cast.js +72 -2
  16. package/dist/src/mind/mechanisms/cover.js +6 -1
  17. package/dist/src/mind/mechanisms/extraction.js +27 -0
  18. package/dist/src/mind/mechanisms/recall.js +214 -34
  19. package/dist/src/mind/mind.d.ts +52 -3
  20. package/dist/src/mind/mind.js +140 -12
  21. package/dist/src/mind/pipeline-mechanism.d.ts +7 -0
  22. package/dist/src/mind/pipeline.js +29 -1
  23. package/dist/src/mind/prefix-completion.d.ts +59 -0
  24. package/dist/src/mind/prefix-completion.js +270 -0
  25. package/dist/src/mind/primitives.d.ts +29 -10
  26. package/dist/src/mind/primitives.js +98 -71
  27. package/dist/src/mind/recognition.js +153 -26
  28. package/dist/src/mind/traverse.d.ts +32 -0
  29. package/dist/src/mind/traverse.js +52 -0
  30. package/dist/src/mind/types.d.ts +61 -18
  31. package/dist/src/mind/types.js +68 -19
  32. package/dist/src/store.d.ts +21 -0
  33. package/dist/src/store.js +21 -0
  34. package/example/train_base.ts +21 -4
  35. package/package.json +1 -1
  36. package/src/canon.ts +28 -0
  37. package/src/geometry.ts +100 -1
  38. package/src/mind/attention.ts +15 -10
  39. package/src/mind/bridge.ts +34 -0
  40. package/src/mind/frame-filler.ts +604 -0
  41. package/src/mind/learning.ts +5 -9
  42. package/src/mind/mechanisms/cast.ts +70 -2
  43. package/src/mind/mechanisms/cover.ts +6 -1
  44. package/src/mind/mechanisms/extraction.ts +27 -0
  45. package/src/mind/mechanisms/recall.ts +236 -37
  46. package/src/mind/mind.ts +166 -18
  47. package/src/mind/pipeline-mechanism.ts +7 -0
  48. package/src/mind/pipeline.ts +33 -1
  49. package/src/mind/prefix-completion.ts +314 -0
  50. package/src/mind/primitives.ts +105 -80
  51. package/src/mind/recognition.ts +151 -23
  52. package/src/mind/traverse.ts +52 -0
  53. package/src/mind/types.ts +104 -44
  54. package/src/store.ts +25 -0
  55. package/test/13-conversation.test.mjs +13 -0
  56. package/test/57-fusion-order.test.mjs +65 -0
  57. package/test/66-query-edge-whitespace.test.mjs +99 -0
  58. package/test/67-climb-anchor-breadth.test.mjs +113 -0
  59. package/test/68-extraction-unanchored.test.mjs +79 -0
  60. package/test/69-frame-filler.test.mjs +115 -0
  61. package/test/70-prefix-completion.test.mjs +170 -0
  62. package/test/71-embedded-canon-equivalence.test.mjs +121 -0
  63. package/test/72-prefix-candidate-supply.test.mjs +114 -0
  64. package/test/73-scaffolding-only-bridge-abstains.test.mjs +178 -0
  65. package/test/74-prefix-trap-not-sprung-early.test.mjs +114 -0
  66. package/test/75-multiturn-context-optimisation.test.mjs +1334 -0
@@ -6,6 +6,17 @@ import type { Input, MindContext } from "./types.js";
6
6
  * (windows, regions, candidate spans), so key construction is far cheaper
7
7
  * than the river fold it deduplicates. */
8
8
  export declare function latin1Key(bytes: Uint8Array): string;
9
+ /** The {@link perceive} memo key: the span's content PLUS the boundary set it
10
+ * was folded under. The tree is a function of BOTH — the same bytes fold
11
+ * plainly with no boundaries and into a left-nested stable-prefix shape with
12
+ * them — so a content-only key returns whichever shape was computed first.
13
+ * That is exactly what happened: a conversation seeded its cumulative context
14
+ * under the content key, and every later plain `perceive` of those bytes was
15
+ * served the boundary tree instead (measured: respondTurn answered where
16
+ * respond() on byte-identical input did not). NUL separates the two parts —
17
+ * the boundary rendering is digits and commas, so no content byte can forge
18
+ * the split. */
19
+ export declare function perceiveKey(bytes: Uint8Array, boundaries?: readonly number[]): string;
9
20
  /** Perceive input into a content-defined tree (the river fold).
10
21
  * Deterministic — identical bytes always produce an identical tree.
11
22
  *
@@ -16,16 +27,24 @@ export declare function latin1Key(bytes: Uint8Array): string;
16
27
  * boundaries are; the geometry never guesses them from the bytes. */
17
28
  export declare function perceive(ctx: MindContext, input: Input, leafAt?: (i: number) => number | null, lookup?: (ids: number[]) => number | null, boundaries?: readonly number[]): Sema;
18
29
  /** The DEPOSIT-shaped perceive. Folds over the stream's own content cuts —
19
- * bit-identical to what inference computes for the same bytes, and that
20
- * train/inference agreement is load-bearing for exact recall. An input that
21
- * EXTENDS a previously deposited one is a conversation context grown by one
22
- * turn; the cached prefix length IS the turn boundary (derived from the deposit
23
- * sequence itself, never from a content convention) and joins the cut set, so
24
- * the trained context node and the query's context subtree are the SAME node.
25
- * Segment folds reuse across deposits ({@link stablePrefixFoldIncremental}) —
26
- * O(turn) instead of O(context) per turn. All of it is purely a cache: an
27
- * evicted chain loses only the turn boundaries, and since the content cuts do
28
- * not depend on the cache, the segments themselves are unchanged. */
30
+ * bit-identical to what inference computes for the same bytes. That
31
+ * train/inference agreement is the whole contract: the trained context node
32
+ * and the node `resolve(query)` reaches must be the SAME node, and the only
33
+ * way to guarantee it is to give this function nothing extra to say. It
34
+ * imposes no boundaries, knows nothing about turns, and reads no convention
35
+ * out of the bytes.
36
+ *
37
+ * An input that EXTENDS a previously deposited one a conversation context
38
+ * grown by a turn, or a resumed replay reuses that deposit's already-folded
39
+ * content segments ({@link contentFoldIncremental}), so it costs O(new bytes)
40
+ * instead of O(context). The reuse is TRANSPARENT by construction: a segment
41
+ * is a pure function of its own bytes, so a reused one is bit-identical to a
42
+ * refolded one. Nothing has to prove that the extending deposit is "really"
43
+ * a next turn — a coincidental byte prefix reuses the same segments and gets
44
+ * the same tree it would have got anyway. (It used to matter: while this
45
+ * path imposed turn BOUNDARIES, a wrong guess changed the tree, so the cache
46
+ * needed a continuation-bytes proof to gate it. Nothing is imposed now, so
47
+ * there is nothing to gate.) */
29
48
  export declare function perceiveDeposit(ctx: MindContext, bytes: Uint8Array, conversational?: boolean): Sema;
30
49
  /** The raw bytes of an input — modality-neutral conversion. */
31
50
  export declare function inputBytes(ctx: MindContext, input: Input): Uint8Array;
@@ -2,7 +2,7 @@
2
2
  //
3
3
  // Address — bytes → node (perceive, foldTree, resolve)
4
4
  // Read — node → bytes (read)
5
- import { bytesToTree, gridToTree, hilbertBytes, stablePrefixFoldIncremental, stackGrids, } from "../geometry.js";
5
+ import { bytesToTree, contentFoldIncremental, gridToTree, hilbertBytes, stackGrids, } from "../geometry.js";
6
6
  import { canonHash } from "../canon.js";
7
7
  import { bytesEqual } from "../bytes.js";
8
8
  import { ALL } from "./types.js";
@@ -21,6 +21,22 @@ export function latin1Key(bytes) {
21
21
  }
22
22
  return s;
23
23
  }
24
+ /** The {@link perceive} memo key: the span's content PLUS the boundary set it
25
+ * was folded under. The tree is a function of BOTH — the same bytes fold
26
+ * plainly with no boundaries and into a left-nested stable-prefix shape with
27
+ * them — so a content-only key returns whichever shape was computed first.
28
+ * That is exactly what happened: a conversation seeded its cumulative context
29
+ * under the content key, and every later plain `perceive` of those bytes was
30
+ * served the boundary tree instead (measured: respondTurn answered where
31
+ * respond() on byte-identical input did not). NUL separates the two parts —
32
+ * the boundary rendering is digits and commas, so no content byte can forge
33
+ * the split. */
34
+ export function perceiveKey(bytes, boundaries) {
35
+ const k = latin1Key(bytes);
36
+ return boundaries === undefined || boundaries.length === 0
37
+ ? k
38
+ : k + "\u0000" + boundaries.join(",");
39
+ }
24
40
  /** Perceive input into a content-defined tree (the river fold).
25
41
  * Deterministic — identical bytes always produce an identical tree.
26
42
  *
@@ -41,7 +57,7 @@ export function perceive(ctx, input, leafAt, lookup, boundaries) {
41
57
  // The tree is shared by reference; Sema nodes are never mutated.
42
58
  const memo = ctx.perceiveMemo;
43
59
  if (memo) {
44
- const key = latin1Key(bytes);
60
+ const key = perceiveKey(bytes, boundaries);
45
61
  const hit = memo.get(key);
46
62
  if (hit !== undefined) {
47
63
  if (ctx.meter)
@@ -70,67 +86,42 @@ export function perceive(ctx, input, leafAt, lookup, boundaries) {
70
86
  return gridToTree(ctx.space, ctx.alphabet, input);
71
87
  }
72
88
  /** The DEPOSIT-shaped perceive. Folds over the stream's own content cuts —
73
- * bit-identical to what inference computes for the same bytes, and that
74
- * train/inference agreement is load-bearing for exact recall. An input that
75
- * EXTENDS a previously deposited one is a conversation context grown by one
76
- * turn; the cached prefix length IS the turn boundary (derived from the deposit
77
- * sequence itself, never from a content convention) and joins the cut set, so
78
- * the trained context node and the query's context subtree are the SAME node.
79
- * Segment folds reuse across deposits ({@link stablePrefixFoldIncremental}) —
80
- * O(turn) instead of O(context) per turn. All of it is purely a cache: an
81
- * evicted chain loses only the turn boundaries, and since the content cuts do
82
- * not depend on the cache, the segments themselves are unchanged. */
89
+ * bit-identical to what inference computes for the same bytes. That
90
+ * train/inference agreement is the whole contract: the trained context node
91
+ * and the node `resolve(query)` reaches must be the SAME node, and the only
92
+ * way to guarantee it is to give this function nothing extra to say. It
93
+ * imposes no boundaries, knows nothing about turns, and reads no convention
94
+ * out of the bytes.
95
+ *
96
+ * An input that EXTENDS a previously deposited one a conversation context
97
+ * grown by a turn, or a resumed replay reuses that deposit's already-folded
98
+ * content segments ({@link contentFoldIncremental}), so it costs O(new bytes)
99
+ * instead of O(context). The reuse is TRANSPARENT by construction: a segment
100
+ * is a pure function of its own bytes, so a reused one is bit-identical to a
101
+ * refolded one. Nothing has to prove that the extending deposit is "really"
102
+ * a next turn — a coincidental byte prefix reuses the same segments and gets
103
+ * the same tree it would have got anyway. (It used to matter: while this
104
+ * path imposed turn BOUNDARIES, a wrong guess changed the tree, so the cache
105
+ * needed a continuation-bytes proof to gate it. Nothing is imposed now, so
106
+ * there is nothing to gate.) */
83
107
  export function perceiveDeposit(ctx, bytes, conversational = false) {
108
+ // Longest cached PROPER prefix first — the most segments to reuse.
84
109
  let prev;
85
- let prefixLen = 0;
86
- // Cache consult (both boundary lookup and stable-prefix reuse) is scoped
87
- // to conversational deposits only a bare, unrelated fact whose bytes
88
- // happen to extend an earlier deposit is NOT a conversation turn, and
89
- // must keep the plain fold so it shares structure with ITS OWN prior
90
- // deposits, not fragment against a coincidental byte-prefix.
91
- if (conversational) {
92
- // Longest cached PROPER prefix first.
93
- const lens = [...ctx._depositLens]
94
- .filter((L) => L >= 2 && L < bytes.length)
95
- .sort((a, b) => b - a);
96
- for (const L of lens) {
97
- const hit = ctx._depositTrees.get(latin1Key(bytes.subarray(0, L)));
98
- // The suffix must bytes-equal the hit's OWN recorded continuation —
99
- // proof this deposit is that turn's actual next turn, not a fact
100
- // that coincidentally shares its byte prefix.
101
- if (hit !== undefined && hit.nextBytes !== undefined &&
102
- bytesEqual(hit.nextBytes, bytes.subarray(L))) {
103
- prev = hit;
104
- prefixLen = L;
105
- break;
106
- }
110
+ const lens = [...ctx._depositLens]
111
+ .filter((L) => L >= 2 && L < bytes.length)
112
+ .sort((a, b) => b - a);
113
+ for (const L of lens) {
114
+ const hit = ctx._depositTrees.get(latin1Key(bytes.subarray(0, L)));
115
+ if (hit !== undefined) {
116
+ prev = hit.content;
117
+ break;
107
118
  }
108
119
  }
109
- // ONLY turn boundaries belong here. The stream's own content cuts are NOT
110
- // passed in: `bytesToTree` and `stablePrefixFoldIncremental` both derive them
111
- // per span, at every level, and handing the level-0 cuts in as stable-prefix
112
- // boundaries instead produces a LEFT-NESTED join of flat segments a
113
- // different tree from the one inference builds for the same bytes. When that
114
- // happened, a deposit's context root and `resolve(question)` were different
115
- // nodes, so the trained edge hung off a node inference never reached and
116
- // recall went silent (test/44 caught it as a site that could not be emitted
117
- // because the resolved node led nowhere). Train and infer must fold
118
- // identically; the way to guarantee that is to give this function nothing
119
- // extra to say.
120
- const cuts = new Set();
121
- if (prev !== undefined) {
122
- for (const b of prev.boundaries)
123
- cuts.add(b);
124
- cuts.add(prefixLen);
125
- }
126
- const boundaries = [...cuts].sort((a, b) => a - b);
127
- const folded = stablePrefixFoldIncremental(ctx.space, ctx.alphabet, bytes, boundaries, prev?.stable);
128
- const tree = folded.tree;
129
- const entry = { boundaries, stable: folded.fold };
130
- // Only a conversational deposit writes the cache too — otherwise a bare
131
- // fact's plain fold could later be misread as a conversation's turn-zero
132
- // boundary by an unrelated conversational deposit that happens to extend
133
- // its bytes.
120
+ const folded = contentFoldIncremental(ctx.space, ctx.alphabet, bytes, prev);
121
+ // Only a CONVERSATIONAL deposit writes the cache: reuse is sound for any
122
+ // deposit, but the budget is 8 entries and a corpus of unrelated facts would
123
+ // evict the live chains for nothing. Purely a cost decision now, not a
124
+ // correctness one.
134
125
  if (conversational && bytes.length >= 2) {
135
126
  // The lengths set drifts as the map evicts; past the probe budget the
136
127
  // drift itself becomes the cost (each stale length is an O(len) key
@@ -139,10 +130,10 @@ export function perceiveDeposit(ctx, bytes, conversational = false) {
139
130
  ctx._depositLens.clear();
140
131
  ctx._depositTrees.clear();
141
132
  }
142
- ctx._depositTrees.set(latin1Key(bytes), entry);
133
+ ctx._depositTrees.set(latin1Key(bytes), { content: folded.fold });
143
134
  ctx._depositLens.add(bytes.length);
144
135
  }
145
- return tree;
136
+ return folded.tree;
146
137
  }
147
138
  /** The raw bytes of an input — modality-neutral conversion. */
148
139
  export function inputBytes(ctx, input) {
@@ -164,20 +155,47 @@ export function gistOf(ctx, bytes) {
164
155
  * node with its byte span and resolved id. Returns the node's byte end and
165
156
  * resolved id. */
166
157
  export function foldTree(ctx, n, start, visit) {
167
- // Fast path: subtree already resolved (from a previous conversation turn
168
- // or an earlier recognition pass). The pyramid reuses prefix subtrees as
169
- // identical Sema objects, so this cache turns foldTree into O(suffix)
170
- // instead of O(context) for multi-turn recognition.
158
+ // Subtree already resolved (from a previous conversation turn or an earlier
159
+ // recognition pass). The pyramid reuses prefix subtrees as identical Sema
160
+ // objects, so a conversation's prefix is warm from its second turn on.
161
+ // Without a visitor that makes foldTree O(suffix) instead of O(context);
162
+ // with one it stays O(context) and saves the per-node store probes instead
163
+ // (see below for why the distinction is not negotiable).
164
+ //
165
+ // WHAT THE CACHE KNOWS, AND WHAT IT DOES NOT. An entry records this
166
+ // subtree's id and byte length — nothing about its DESCENDANTS' spans.
167
+ // Returning here therefore emits ONE visit() where a cold walk emits one per
168
+ // node, and `visit` is not instrumentation: recognise() emits its sites from
169
+ // it (recognition.ts) and attention's collectRegions votes over what it
170
+ // yields (attention.ts). Skipping the descent silently shrinks the evidence
171
+ // those mechanisms see, purely because the cache happened to be warm.
172
+ //
173
+ // That is not hypothetical and not an edge case — it is every conversation
174
+ // turn after the first. `contentFoldIncremental` deliberately shares prefix
175
+ // segment OBJECTS across turns (~99% reuse), so by turn 2 the prefix is
176
+ // warm; meanwhile recogniseMemo/climbMemo are keyed on exact query BYTES,
177
+ // which a growing context never repeats. Warm subtrees + missed memos is
178
+ // the unprotected quadrant. Measured over real trained conversations,
179
+ // recognising the same context with a warm prefix lost 67-92% of its leaves
180
+ // (772->204, 589->47, 872->291, 377->37) with `sites` unchanged, so the loss
181
+ // is invisible to the coarse counts; a direct foldTree probe on identical
182
+ // bytes and an identical tree object fired visit() 661 times cold and 37
183
+ // warm. respond() is immune only because it never sets _resolvedSubtrees
184
+ // (mind.ts) — the degradation was unique to the multi-turn API.
185
+ //
186
+ // So the fast path is taken only when NOBODY IS WATCHING. With a visitor
187
+ // present we still walk, and the cache degrades to the thing it soundly is:
188
+ // an elision of the store probes (findLeaf/findBranch) at each node, not an
189
+ // elision of the traversal. Ids still come from the cache, so a warm walk
190
+ // is cheaper than a cold one; it is no longer *different* from one.
171
191
  const cached = ctx._resolvedSubtrees?.get(n);
172
- if (cached !== undefined) {
173
- const end = start + cached.len;
174
- visit?.(n, start, end, cached.id);
175
- return { end, node: cached.id };
192
+ if (cached !== undefined && visit === undefined) {
193
+ return { end: start + cached.len, node: cached.id };
176
194
  }
177
195
  if (n.kids === null) {
178
196
  const b = n.leaf ?? new Uint8Array(0);
179
197
  const end = start + b.length;
180
- const node = ctx.store.findLeaf(b);
198
+ const node = cached !== undefined ? cached.id : ctx.store.findLeaf(b);
181
199
  visit?.(n, start, end, node);
182
200
  if (node !== null && ctx._resolvedSubtrees) {
183
201
  ctx._resolvedSubtrees.set(n, { id: node, len: b.length });
@@ -195,7 +213,16 @@ export function foldTree(ctx, n, start, visit) {
195
213
  kids.push(r.node);
196
214
  pos = r.end;
197
215
  }
198
- const node = known ? ctx.store.findBranch(kids) : null;
216
+ // Same store-probe elision as the leaf case: a cached entry already names
217
+ // this subtree, so the descent above was for `visit`'s benefit alone and the
218
+ // id need not be re-derived. Using it also keeps a warm walk's ids
219
+ // bit-identical to a cold walk's rather than re-deriving them from children
220
+ // that may themselves have come from cache.
221
+ const node = cached !== undefined
222
+ ? cached.id
223
+ : known
224
+ ? ctx.store.findBranch(kids)
225
+ : null;
199
226
  visit?.(n, start, pos, node);
200
227
  if (node !== null && ctx._resolvedSubtrees) {
201
228
  ctx._resolvedSubtrees.set(n, { id: node, len: pos - start });
@@ -8,6 +8,7 @@ import { rItem } from "./trace.js";
8
8
  import { canonResolve, foldTree, gistOf, latin1Key, perceive, resolve, } from "./primitives.js";
9
9
  import { atomIsHub, corpusN, leadsSomewhere } from "./traverse.js";
10
10
  import { chainReach, leafIdAt, leafIdRun } from "./canonical.js";
11
+ import { canonHash } from "../canon.js";
11
12
  import { isChunk } from "../sema.js";
12
13
  /** Decompose a byte stream into every stored form that leads somewhere
13
14
  * (has a continuation edge or a halo). Two complementary readings:
@@ -25,23 +26,40 @@ export function recognise(ctx, bytes) {
25
26
  // Content-keyed memo — works for both single-turn respond() and multi-turn
26
27
  // respondTurn() (where the map persists across calls). ALWAYS consulted,
27
28
  // regardless of tracing — matching perceive()'s own memo, which carries no
28
- // trace gate at all. This memo is NOT an optional accelerator: recogniseImpl
29
- // walks the query's perceived tree via foldTree, whose subtree-resolution
30
- // fast path (see primitives.ts) skips invoking `visit` and therefore
31
- // skips EMITTING SITES for any subtree already cached in
32
- // ctx._resolvedSubtrees. A multi-turn conversation's stable-prefix fold
33
- // deliberately shares node OBJECTS across turns, so by the second call on
34
- // the exact same bytes, large swaths of the tree are already cached and
35
- // foldTree stops short of recursing into them a second recogniseImpl
36
- // call on the SAME bytes is not idempotent; it silently finds FEWER sites
37
- // than the first (observed live: 31 sites → 5 on an immediate repeat
38
- // call). Skipping this memo "only while tracing" used to mean every
39
- // traced turn re-ran recogniseImpl from scratch at every one of the many
40
- // call sites that recognise the same query (cover, reason, articulate...),
41
- // each subsequent call silently more incomplete than the last measurably
42
- // changing which mechanism grounds the answer, not just costing time. The
43
- // trace step must still fire on every call regardless (a cache hit is not
44
- // silent), so it is emitted here directly instead of only inside
29
+ // trace gate at all.
30
+ //
31
+ // This memo is an accelerator, and that is now the whole of it: repeated
32
+ // recognition of the same query is ordinary within one response (cover,
33
+ // reason and articulate all recognise it) and recogniseImpl is O(n ·
34
+ // maxGroup) probes each time.
35
+ //
36
+ // IT USED TO BE LOAD-BEARING FOR CORRECTNESS, and the history is worth
37
+ // keeping because it explains why there is no trace gate here. foldTree's
38
+ // subtree-resolution fast path (primitives.ts) once returned on a cache hit
39
+ // WITHOUT recursing, so it skipped invoking `visit` and therefore skipped
40
+ // EMITTING SITES for any subtree already in ctx._resolvedSubtrees. A
41
+ // conversation's incremental fold deliberately shares node OBJECTS across
42
+ // turns, so by the second call on the same bytes large swaths of the tree
43
+ // were already cached and recogniseImpl silently found FEWER sites than the
44
+ // first call (observed live: 31 5). Skipping this memo "only while
45
+ // tracing" therefore meant every traced turn re-ran recogniseImpl at each of
46
+ // those call sites, each result more incomplete than the last — changing
47
+ // which mechanism grounded the answer, not merely costing time.
48
+ //
49
+ // foldTree no longer does that: it takes the fast path only when no `visit`
50
+ // is supplied, so a walk that emits sites always walks in full and the id
51
+ // cache is reduced to eliding store probes (see primitives.ts). recognise()
52
+ // is idempotent on its own now — verified with the memo bypassed, the
53
+ // subtree cache warm and the tree object shared: three consecutive calls on
54
+ // the same 544-byte context returned sites=2 leaves=544 splits=0 starts=88,
55
+ // identical every time.
56
+ //
57
+ // The unconditional consult STAYS regardless. A memo whose absence can only
58
+ // cost time is still not something to gate on whether an audit happens to be
59
+ // attached: tracing must not change what the pipeline computes, and the
60
+ // cheapest way to guarantee that is for the trace flag to touch nothing but
61
+ // the trace. The trace step must still fire on every call (a cache hit is
62
+ // not silent), so it is emitted here directly rather than only inside
45
63
  // recogniseImpl.
46
64
  if (ctx.recogniseMemo) {
47
65
  const key = latin1Key(bytes);
@@ -400,7 +418,21 @@ function recogniseImpl(ctx, bytes) {
400
418
  // before resolveSpan pays for a fold; approximate evidence never enters.
401
419
  // This tier is needed only where atom chains are suppressed. Small stores
402
420
  // retain their existing decomposition unchanged.
403
- if (atomsAreHubs) {
421
+ // ALWAYS ON, AND LINEAR. This used to be gated on `atomsAreHubs` — small
422
+ // stores were said to "retain their existing decomposition unchanged", which
423
+ // was true only while the query's fold was told where the turns were: every
424
+ // turn was then a NODE, so the structural walk found it and this tier had
425
+ // nothing to add. The fold no longer imposes turn boundaries (a turn start
426
+ // is an ordinary interior offset now), so a trained form embedded in a
427
+ // longer query is reachable ONLY here — the chain caps at chainReach(W)=W²
428
+ // bytes and cannot span one. Measured: with the fold imposing boundaries
429
+ // every turn is a node; without it, none is.
430
+ //
431
+ // Ungating it alone made inference QUADRATIC (test/14's constant-KB/s guard
432
+ // went to 41.8s): every offset near a cut is an endpoint, and each probe
433
+ // costs O(span) to slice the leaf-id run and hash it. The budget below is
434
+ // what makes it affordable — see `spend`.
435
+ {
404
436
  const allLeafIds = singleLeaf.map((x) => x?.id ?? null);
405
437
  if (allLeafIds.every((x) => x !== null)) {
406
438
  const radius = ctx.space.seats.length;
@@ -410,20 +442,115 @@ function recogniseImpl(ctx, bytes) {
410
442
  endpoints.add(p);
411
443
  }
412
444
  const ordered = [...endpoints].sort((a, b) => a - b);
413
- const probe = (start, end) => {
445
+ // The leaf-id run is BYTE-EXACT, while `resolveSpan` behind it resolves
446
+ // exactly OR canonically — so this gate was strictly narrower than its
447
+ // own resolver, and every embedded form differing from its deposit only
448
+ // by the response's equivalence (case, width) was dropped before the
449
+ // resolver ever saw it. Rebuilding the run over canonicalized bytes
450
+ // does NOT fix that: a differently-cased deposit's branch kid-ids are
451
+ // not the query's leaf-id run under ANY canonicalization of the query,
452
+ // so the second admission route has to be the canon INDEX itself — the
453
+ // same candidate proposal `canonResolve` makes, and the same
454
+ // cheap-probe-before-a-fold discipline the exact route already follows
455
+ // (a hash and an indexed lookup; no fold, no vector, no scan). Both
456
+ // routes only PROPOSE; `resolveSpan` still decides, so a hash-bucket
457
+ // collision costs one fold and can never emit a wrong site (test/71).
458
+ const canonAdmits = (start, end) => {
459
+ const canon = ctx.canon;
460
+ if (canon === null || !store.canonFind)
461
+ return false;
462
+ const key = canon(bytes.subarray(start, end));
463
+ if (key.length === 0)
464
+ return false;
465
+ return store.canonFind(canonHash(key)).length > 0;
466
+ };
467
+ // The byte-exact route probes the SPAN ITSELF (see
468
+ // Store.findFlatBranch): for a run of single-byte leaves the flat-kid
469
+ // encoding is the identity, so the span's bytes ARE the branch key.
470
+ // `subarray` is a view — this allocates nothing per probe, and the
471
+ // bloom filter answers the misses without touching the database.
472
+ const flatProbe = (start, end) => store.findFlatBranch
473
+ ? store.findFlatBranch(bytes.subarray(start, end))
474
+ : store.findBranch(allLeafIds.slice(start, end));
475
+ // THE TWO ROUTES COST DIFFERENT THINGS, SO THEY ARE PRICED SEPARATELY.
476
+ //
477
+ // The exact route is a bloom-gated hash over a subarray VIEW: no
478
+ // allocation, and a miss never reaches the database. It is cheap enough
479
+ // to run on every endpoint, and that is what makes this tier able to
480
+ // find a trained form embedded anywhere in the query.
481
+ //
482
+ // The canon route is not: it runs the canonicalizer over the span
483
+ // (NFKC, case-fold, whitespace) and allocates a fresh key for every
484
+ // probe. That is the O(span) cost with the heavy constant, and it is
485
+ // the one worth a budget. Sharing ONE budget between them made the
486
+ // cheap route starve on the expensive one's behalf — measured, test/71's
487
+ // embedded differently-cased form needed 64x the budget to be found,
488
+ // while the exact route it was competing with needed none of it.
489
+ const probe = (start, end, canonBudget) => {
414
490
  if (end - start < W || end - start <= chainReach(W))
415
491
  return;
416
- const ids = allLeafIds.slice(start, end);
417
- if (store.findBranch(ids) === null)
418
- return;
492
+ if (flatProbe(start, end) === null) {
493
+ if (!canonBudget)
494
+ return;
495
+ if (!canonAdmits(start, end))
496
+ return;
497
+ }
419
498
  const id = resolveSpan(start, end);
420
499
  if (id !== null)
421
500
  emit(start, end, id);
422
501
  };
423
- for (const end of ordered)
424
- probe(0, end);
425
- for (const start of ordered)
426
- probe(start, bytes.length);
502
+ // A CUMULATIVE BYTE BUDGET, SPENT SHORTEST-SPAN-FIRST.
503
+ //
504
+ // Each probe costs O(span), and there are O(n) endpoints, so probing
505
+ // them all is O(n²) — the quadratic this tier was gated to avoid. The
506
+ // budget caps TOTAL probe bytes at a multiple of the query's own length,
507
+ // which is what keeps whole-query inference linear.
508
+ //
509
+ // Spending it shortest-first is what makes the cap a scale bound rather
510
+ // than a position bound: the tier recovers embedded forms up to roughly
511
+ // √(2·budget) bytes ANYWHERE in the endpoint set, instead of walking the
512
+ // endpoints in order and running out partway along the query. A form
513
+ // longer than that is out of this tier's reach — but so is a form the
514
+ // chain cannot span, and that is exactly the trade the budget prices.
515
+ // The factor is chainReach(W), the same W² scale the chain already
516
+ // trusts; no new constant.
517
+ // The factor is chainReach(W) — the same W² scale the chain itself
518
+ // trusts — so the cap is derived from the fold's geometry, never tuned.
519
+ // (It was briefly an environment variable while the cost was being
520
+ // measured; an env-read here would make inference non-reproducible,
521
+ // which the determinism contract forbids outright.)
522
+ // The factor is chainReach(W) — the same W² scale the chain itself
523
+ // trusts — so the cap is derived from the fold's geometry, never tuned.
524
+ // (It was briefly an environment variable while the cost was being
525
+ // measured; an env-read here would make inference non-reproducible,
526
+ // which the determinism contract forbids outright.)
527
+ //
528
+ // It now prices ONLY the canonicalizing route; the exact route runs on
529
+ // every endpoint regardless, so exhausting this budget narrows which
530
+ // equivalence-class forms are proposed, never which byte-exact ones.
531
+ let budget = bytes.length * chainReach(W) * chainReach(W);
532
+ const spend = (start, end) => {
533
+ const span = end - start;
534
+ const afford = span <= budget;
535
+ if (afford)
536
+ budget -= span;
537
+ probe(start, end, afford);
538
+ // Always keep walking: the exact route is unbudgeted, so running out
539
+ // of canon budget must not stop the scan.
540
+ return true;
541
+ };
542
+ const prefixes = ordered.filter((e) => e > 0).sort((a, b) => a - b);
543
+ const suffixes = ordered
544
+ .filter((s2) => s2 < bytes.length)
545
+ .sort((a, b) => b - a);
546
+ for (let i = 0; i < Math.max(prefixes.length, suffixes.length); i++) {
547
+ // Interleaved so neither edge starves the other when the budget runs
548
+ // out — a query can carry a trained form at either end.
549
+ if (i < prefixes.length && !spend(0, prefixes[i]))
550
+ break;
551
+ if (i < suffixes.length && !spend(suffixes[i], bytes.length))
552
+ break;
553
+ }
427
554
  }
428
555
  }
429
556
  const chunkEnd = new Uint32Array(bytes.length);
@@ -124,3 +124,35 @@ export declare function chooseAmong(ctx: MindContext, candidates: readonly numbe
124
124
  id: number;
125
125
  score: number;
126
126
  };
127
+ /** True when NO window of `query` discriminates anything — every stored
128
+ * W-window it spells is contained by more places than the hub bound allows,
129
+ * i.e. the whole query is corpus-global scaffolding.
130
+ *
131
+ * WHAT IT IS FOR. Several mechanisms ground a query through the literal
132
+ * spans it did NOT explain, and those spans are the whole of their evidence.
133
+ * When every one of them is a hub, the query says nothing the corpus can be
134
+ * held to, and grounding it means picking one of thousands of continuations
135
+ * it gives no evidence for — a fabrication whatever the answer happens to be.
136
+ * Answering with silence there is the honest degradation contract (§2.13).
137
+ *
138
+ * MEASURED SEPARATION (trained store, hubBound 571) — this is categorical,
139
+ * not marginal, and it is why the predicate lives here rather than being
140
+ * spelled twice:
141
+ * "What is the capital of" ALL saturated ("What":572) → fabricated
142
+ * "What is the capital " ALL saturated ("What":572) → fabricated
143
+ * "what is the capital of france" min "f fr":248 → correct
144
+ * "What is the capitol of France?" min "f Fr":114 → correct
145
+ * "WHAT IS THE CAPITAL OF FRANCE?" min "HE C":1 → correct
146
+ * "What is the capital of France?" min "t i":4 → correct
147
+ * "Who wrote Romeo and Juliet?" min "iet?":26 → correct
148
+ * "What is the capital of Zamunda?" min "Zamu":3 → silent anyway
149
+ * Note the last: the honest-silence probes are already refused on other
150
+ * evidence and sit on the SAME side as the correct ones, so this predicate
151
+ * is not what makes them silent and cannot be credited for them.
152
+ *
153
+ * NO NEW THRESHOLD (§2.2): `hubBound` is the √N reading of "hub" used
154
+ * everywhere, and the containment read is clamped to it exactly as every
155
+ * other fan-out read is (§2.8). A query with no stored window at all is NOT
156
+ * scaffolding-only — it has no evidence either way, and its callers already
157
+ * refuse it on their own terms. */
158
+ export declare function allWindowsAreScaffolding(ctx: MindContext, query: Uint8Array): boolean;
@@ -7,6 +7,7 @@
7
7
  // project) live in match.ts — the elementary match-and-project operation.
8
8
  import { cosine } from "../vec.js";
9
9
  import { gistOf, read } from "./primitives.js";
10
+ import { leafIdRun } from "./canonical.js";
10
11
  const structCaches = new WeakMap();
11
12
  // ── The shared ancestor-reach memo ──────────────────────────────────────
12
13
  //
@@ -648,3 +649,54 @@ function rItemShort(ctx, id, role, score) {
648
649
  score,
649
650
  };
650
651
  }
652
+ /** True when NO window of `query` discriminates anything — every stored
653
+ * W-window it spells is contained by more places than the hub bound allows,
654
+ * i.e. the whole query is corpus-global scaffolding.
655
+ *
656
+ * WHAT IT IS FOR. Several mechanisms ground a query through the literal
657
+ * spans it did NOT explain, and those spans are the whole of their evidence.
658
+ * When every one of them is a hub, the query says nothing the corpus can be
659
+ * held to, and grounding it means picking one of thousands of continuations
660
+ * it gives no evidence for — a fabrication whatever the answer happens to be.
661
+ * Answering with silence there is the honest degradation contract (§2.13).
662
+ *
663
+ * MEASURED SEPARATION (trained store, hubBound 571) — this is categorical,
664
+ * not marginal, and it is why the predicate lives here rather than being
665
+ * spelled twice:
666
+ * "What is the capital of" ALL saturated ("What":572) → fabricated
667
+ * "What is the capital " ALL saturated ("What":572) → fabricated
668
+ * "what is the capital of france" min "f fr":248 → correct
669
+ * "What is the capitol of France?" min "f Fr":114 → correct
670
+ * "WHAT IS THE CAPITAL OF FRANCE?" min "HE C":1 → correct
671
+ * "What is the capital of France?" min "t i":4 → correct
672
+ * "Who wrote Romeo and Juliet?" min "iet?":26 → correct
673
+ * "What is the capital of Zamunda?" min "Zamu":3 → silent anyway
674
+ * Note the last: the honest-silence probes are already refused on other
675
+ * evidence and sit on the SAME side as the correct ones, so this predicate
676
+ * is not what makes them silent and cannot be credited for them.
677
+ *
678
+ * NO NEW THRESHOLD (§2.2): `hubBound` is the √N reading of "hub" used
679
+ * everywhere, and the containment read is clamped to it exactly as every
680
+ * other fan-out read is (§2.8). A query with no stored window at all is NOT
681
+ * scaffolding-only — it has no evidence either way, and its callers already
682
+ * refuse it on their own terms. */
683
+ export function allWindowsAreScaffolding(ctx, query) {
684
+ const W = ctx.space.maxGroup;
685
+ const bound = hubBound(ctx);
686
+ let sawOne = false;
687
+ for (let o = 0; o + W <= query.length; o++) {
688
+ const ids = leafIdRun(ctx, query, o, o + W);
689
+ if (ids === null)
690
+ continue;
691
+ const id = ctx.store.findBranch(ids);
692
+ if (id === null)
693
+ continue;
694
+ const rarity = ctx.store.containersSlice(id, 0, bound + 1).length;
695
+ if (rarity === 0)
696
+ continue;
697
+ if (rarity <= bound)
698
+ return false;
699
+ sawOne = true;
700
+ }
701
+ return sawOne;
702
+ }