@hviana/sema 0.4.7 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +290 -77
- package/HOW_IT_WORKS.md +2170 -735
- package/dist/example/train_base.d.ts +9 -3
- package/dist/example/train_base.js +21 -4
- package/dist/src/canon.d.ts +19 -0
- package/dist/src/canon.js +28 -0
- package/dist/src/geometry.d.ts +52 -0
- package/dist/src/geometry.js +87 -1
- package/dist/src/mind/attention.d.ts +15 -10
- package/dist/src/mind/attention.js +15 -10
- package/dist/src/mind/bridge.js +27 -1
- package/dist/src/mind/frame-filler.d.ts +15 -0
- package/dist/src/mind/frame-filler.js +535 -0
- package/dist/src/mind/learning.js +6 -11
- package/dist/src/mind/mechanisms/cast.js +72 -2
- package/dist/src/mind/mechanisms/cover.js +6 -1
- package/dist/src/mind/mechanisms/extraction.js +27 -0
- package/dist/src/mind/mechanisms/recall.js +214 -34
- package/dist/src/mind/mind.d.ts +52 -3
- package/dist/src/mind/mind.js +140 -12
- package/dist/src/mind/pipeline-mechanism.d.ts +7 -0
- package/dist/src/mind/pipeline.js +29 -1
- package/dist/src/mind/prefix-completion.d.ts +59 -0
- package/dist/src/mind/prefix-completion.js +270 -0
- package/dist/src/mind/primitives.d.ts +29 -10
- package/dist/src/mind/primitives.js +98 -71
- package/dist/src/mind/recognition.js +153 -26
- package/dist/src/mind/traverse.d.ts +32 -0
- package/dist/src/mind/traverse.js +52 -0
- package/dist/src/mind/types.d.ts +61 -18
- package/dist/src/mind/types.js +68 -19
- package/dist/src/store.d.ts +21 -0
- package/dist/src/store.js +21 -0
- package/example/train_base.ts +21 -4
- package/package.json +1 -1
- package/src/canon.ts +28 -0
- package/src/geometry.ts +100 -1
- package/src/mind/attention.ts +15 -10
- package/src/mind/bridge.ts +34 -0
- package/src/mind/frame-filler.ts +604 -0
- package/src/mind/learning.ts +5 -9
- package/src/mind/mechanisms/cast.ts +70 -2
- package/src/mind/mechanisms/cover.ts +6 -1
- package/src/mind/mechanisms/extraction.ts +27 -0
- package/src/mind/mechanisms/recall.ts +236 -37
- package/src/mind/mind.ts +166 -18
- package/src/mind/pipeline-mechanism.ts +7 -0
- package/src/mind/pipeline.ts +33 -1
- package/src/mind/prefix-completion.ts +314 -0
- package/src/mind/primitives.ts +105 -80
- package/src/mind/recognition.ts +151 -23
- package/src/mind/traverse.ts +52 -0
- package/src/mind/types.ts +104 -44
- package/src/store.ts +25 -0
- package/test/13-conversation.test.mjs +13 -0
- package/test/57-fusion-order.test.mjs +65 -0
- package/test/66-query-edge-whitespace.test.mjs +99 -0
- package/test/67-climb-anchor-breadth.test.mjs +113 -0
- package/test/68-extraction-unanchored.test.mjs +79 -0
- package/test/69-frame-filler.test.mjs +115 -0
- package/test/70-prefix-completion.test.mjs +170 -0
- package/test/71-embedded-canon-equivalence.test.mjs +121 -0
- package/test/72-prefix-candidate-supply.test.mjs +114 -0
- package/test/73-scaffolding-only-bridge-abstains.test.mjs +178 -0
- package/test/74-prefix-trap-not-sprung-early.test.mjs +114 -0
- package/test/75-multiturn-context-optimisation.test.mjs +1334 -0
|
@@ -6,6 +6,17 @@ import type { Input, MindContext } from "./types.js";
|
|
|
6
6
|
* (windows, regions, candidate spans), so key construction is far cheaper
|
|
7
7
|
* than the river fold it deduplicates. */
|
|
8
8
|
export declare function latin1Key(bytes: Uint8Array): string;
|
|
9
|
+
/** The {@link perceive} memo key: the span's content PLUS the boundary set it
|
|
10
|
+
* was folded under. The tree is a function of BOTH — the same bytes fold
|
|
11
|
+
* plainly with no boundaries and into a left-nested stable-prefix shape with
|
|
12
|
+
* them — so a content-only key returns whichever shape was computed first.
|
|
13
|
+
* That is exactly what happened: a conversation seeded its cumulative context
|
|
14
|
+
* under the content key, and every later plain `perceive` of those bytes was
|
|
15
|
+
* served the boundary tree instead (measured: respondTurn answered where
|
|
16
|
+
* respond() on byte-identical input did not). NUL separates the two parts —
|
|
17
|
+
* the boundary rendering is digits and commas, so no content byte can forge
|
|
18
|
+
* the split. */
|
|
19
|
+
export declare function perceiveKey(bytes: Uint8Array, boundaries?: readonly number[]): string;
|
|
9
20
|
/** Perceive input into a content-defined tree (the river fold).
|
|
10
21
|
* Deterministic — identical bytes always produce an identical tree.
|
|
11
22
|
*
|
|
@@ -16,16 +27,24 @@ export declare function latin1Key(bytes: Uint8Array): string;
|
|
|
16
27
|
* boundaries are; the geometry never guesses them from the bytes. */
|
|
17
28
|
export declare function perceive(ctx: MindContext, input: Input, leafAt?: (i: number) => number | null, lookup?: (ids: number[]) => number | null, boundaries?: readonly number[]): Sema;
|
|
18
29
|
/** The DEPOSIT-shaped perceive. Folds over the stream's own content cuts —
|
|
19
|
-
* bit-identical to what inference computes for the same bytes
|
|
20
|
-
* train/inference agreement is
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
30
|
+
* bit-identical to what inference computes for the same bytes. That
|
|
31
|
+
* train/inference agreement is the whole contract: the trained context node
|
|
32
|
+
* and the node `resolve(query)` reaches must be the SAME node, and the only
|
|
33
|
+
* way to guarantee it is to give this function nothing extra to say. It
|
|
34
|
+
* imposes no boundaries, knows nothing about turns, and reads no convention
|
|
35
|
+
* out of the bytes.
|
|
36
|
+
*
|
|
37
|
+
* An input that EXTENDS a previously deposited one — a conversation context
|
|
38
|
+
* grown by a turn, or a resumed replay — reuses that deposit's already-folded
|
|
39
|
+
* content segments ({@link contentFoldIncremental}), so it costs O(new bytes)
|
|
40
|
+
* instead of O(context). The reuse is TRANSPARENT by construction: a segment
|
|
41
|
+
* is a pure function of its own bytes, so a reused one is bit-identical to a
|
|
42
|
+
* refolded one. Nothing has to prove that the extending deposit is "really"
|
|
43
|
+
* a next turn — a coincidental byte prefix reuses the same segments and gets
|
|
44
|
+
* the same tree it would have got anyway. (It used to matter: while this
|
|
45
|
+
* path imposed turn BOUNDARIES, a wrong guess changed the tree, so the cache
|
|
46
|
+
* needed a continuation-bytes proof to gate it. Nothing is imposed now, so
|
|
47
|
+
* there is nothing to gate.) */
|
|
29
48
|
export declare function perceiveDeposit(ctx: MindContext, bytes: Uint8Array, conversational?: boolean): Sema;
|
|
30
49
|
/** The raw bytes of an input — modality-neutral conversion. */
|
|
31
50
|
export declare function inputBytes(ctx: MindContext, input: Input): Uint8Array;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
//
|
|
3
3
|
// Address — bytes → node (perceive, foldTree, resolve)
|
|
4
4
|
// Read — node → bytes (read)
|
|
5
|
-
import { bytesToTree, gridToTree, hilbertBytes,
|
|
5
|
+
import { bytesToTree, contentFoldIncremental, gridToTree, hilbertBytes, stackGrids, } from "../geometry.js";
|
|
6
6
|
import { canonHash } from "../canon.js";
|
|
7
7
|
import { bytesEqual } from "../bytes.js";
|
|
8
8
|
import { ALL } from "./types.js";
|
|
@@ -21,6 +21,22 @@ export function latin1Key(bytes) {
|
|
|
21
21
|
}
|
|
22
22
|
return s;
|
|
23
23
|
}
|
|
24
|
+
/** The {@link perceive} memo key: the span's content PLUS the boundary set it
|
|
25
|
+
* was folded under. The tree is a function of BOTH — the same bytes fold
|
|
26
|
+
* plainly with no boundaries and into a left-nested stable-prefix shape with
|
|
27
|
+
* them — so a content-only key returns whichever shape was computed first.
|
|
28
|
+
* That is exactly what happened: a conversation seeded its cumulative context
|
|
29
|
+
* under the content key, and every later plain `perceive` of those bytes was
|
|
30
|
+
* served the boundary tree instead (measured: respondTurn answered where
|
|
31
|
+
* respond() on byte-identical input did not). NUL separates the two parts —
|
|
32
|
+
* the boundary rendering is digits and commas, so no content byte can forge
|
|
33
|
+
* the split. */
|
|
34
|
+
export function perceiveKey(bytes, boundaries) {
|
|
35
|
+
const k = latin1Key(bytes);
|
|
36
|
+
return boundaries === undefined || boundaries.length === 0
|
|
37
|
+
? k
|
|
38
|
+
: k + "\u0000" + boundaries.join(",");
|
|
39
|
+
}
|
|
24
40
|
/** Perceive input into a content-defined tree (the river fold).
|
|
25
41
|
* Deterministic — identical bytes always produce an identical tree.
|
|
26
42
|
*
|
|
@@ -41,7 +57,7 @@ export function perceive(ctx, input, leafAt, lookup, boundaries) {
|
|
|
41
57
|
// The tree is shared by reference; Sema nodes are never mutated.
|
|
42
58
|
const memo = ctx.perceiveMemo;
|
|
43
59
|
if (memo) {
|
|
44
|
-
const key =
|
|
60
|
+
const key = perceiveKey(bytes, boundaries);
|
|
45
61
|
const hit = memo.get(key);
|
|
46
62
|
if (hit !== undefined) {
|
|
47
63
|
if (ctx.meter)
|
|
@@ -70,67 +86,42 @@ export function perceive(ctx, input, leafAt, lookup, boundaries) {
|
|
|
70
86
|
return gridToTree(ctx.space, ctx.alphabet, input);
|
|
71
87
|
}
|
|
72
88
|
/** The DEPOSIT-shaped perceive. Folds over the stream's own content cuts —
|
|
73
|
-
* bit-identical to what inference computes for the same bytes
|
|
74
|
-
* train/inference agreement is
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
89
|
+
* bit-identical to what inference computes for the same bytes. That
|
|
90
|
+
* train/inference agreement is the whole contract: the trained context node
|
|
91
|
+
* and the node `resolve(query)` reaches must be the SAME node, and the only
|
|
92
|
+
* way to guarantee it is to give this function nothing extra to say. It
|
|
93
|
+
* imposes no boundaries, knows nothing about turns, and reads no convention
|
|
94
|
+
* out of the bytes.
|
|
95
|
+
*
|
|
96
|
+
* An input that EXTENDS a previously deposited one — a conversation context
|
|
97
|
+
* grown by a turn, or a resumed replay — reuses that deposit's already-folded
|
|
98
|
+
* content segments ({@link contentFoldIncremental}), so it costs O(new bytes)
|
|
99
|
+
* instead of O(context). The reuse is TRANSPARENT by construction: a segment
|
|
100
|
+
* is a pure function of its own bytes, so a reused one is bit-identical to a
|
|
101
|
+
* refolded one. Nothing has to prove that the extending deposit is "really"
|
|
102
|
+
* a next turn — a coincidental byte prefix reuses the same segments and gets
|
|
103
|
+
* the same tree it would have got anyway. (It used to matter: while this
|
|
104
|
+
* path imposed turn BOUNDARIES, a wrong guess changed the tree, so the cache
|
|
105
|
+
* needed a continuation-bytes proof to gate it. Nothing is imposed now, so
|
|
106
|
+
* there is nothing to gate.) */
|
|
83
107
|
export function perceiveDeposit(ctx, bytes, conversational = false) {
|
|
108
|
+
// Longest cached PROPER prefix first — the most segments to reuse.
|
|
84
109
|
let prev;
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
const lens = [...ctx._depositLens]
|
|
94
|
-
.filter((L) => L >= 2 && L < bytes.length)
|
|
95
|
-
.sort((a, b) => b - a);
|
|
96
|
-
for (const L of lens) {
|
|
97
|
-
const hit = ctx._depositTrees.get(latin1Key(bytes.subarray(0, L)));
|
|
98
|
-
// The suffix must bytes-equal the hit's OWN recorded continuation —
|
|
99
|
-
// proof this deposit is that turn's actual next turn, not a fact
|
|
100
|
-
// that coincidentally shares its byte prefix.
|
|
101
|
-
if (hit !== undefined && hit.nextBytes !== undefined &&
|
|
102
|
-
bytesEqual(hit.nextBytes, bytes.subarray(L))) {
|
|
103
|
-
prev = hit;
|
|
104
|
-
prefixLen = L;
|
|
105
|
-
break;
|
|
106
|
-
}
|
|
110
|
+
const lens = [...ctx._depositLens]
|
|
111
|
+
.filter((L) => L >= 2 && L < bytes.length)
|
|
112
|
+
.sort((a, b) => b - a);
|
|
113
|
+
for (const L of lens) {
|
|
114
|
+
const hit = ctx._depositTrees.get(latin1Key(bytes.subarray(0, L)));
|
|
115
|
+
if (hit !== undefined) {
|
|
116
|
+
prev = hit.content;
|
|
117
|
+
break;
|
|
107
118
|
}
|
|
108
119
|
}
|
|
109
|
-
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
//
|
|
113
|
-
//
|
|
114
|
-
// happened, a deposit's context root and `resolve(question)` were different
|
|
115
|
-
// nodes, so the trained edge hung off a node inference never reached and
|
|
116
|
-
// recall went silent (test/44 caught it as a site that could not be emitted
|
|
117
|
-
// because the resolved node led nowhere). Train and infer must fold
|
|
118
|
-
// identically; the way to guarantee that is to give this function nothing
|
|
119
|
-
// extra to say.
|
|
120
|
-
const cuts = new Set();
|
|
121
|
-
if (prev !== undefined) {
|
|
122
|
-
for (const b of prev.boundaries)
|
|
123
|
-
cuts.add(b);
|
|
124
|
-
cuts.add(prefixLen);
|
|
125
|
-
}
|
|
126
|
-
const boundaries = [...cuts].sort((a, b) => a - b);
|
|
127
|
-
const folded = stablePrefixFoldIncremental(ctx.space, ctx.alphabet, bytes, boundaries, prev?.stable);
|
|
128
|
-
const tree = folded.tree;
|
|
129
|
-
const entry = { boundaries, stable: folded.fold };
|
|
130
|
-
// Only a conversational deposit writes the cache too — otherwise a bare
|
|
131
|
-
// fact's plain fold could later be misread as a conversation's turn-zero
|
|
132
|
-
// boundary by an unrelated conversational deposit that happens to extend
|
|
133
|
-
// its bytes.
|
|
120
|
+
const folded = contentFoldIncremental(ctx.space, ctx.alphabet, bytes, prev);
|
|
121
|
+
// Only a CONVERSATIONAL deposit writes the cache: reuse is sound for any
|
|
122
|
+
// deposit, but the budget is 8 entries and a corpus of unrelated facts would
|
|
123
|
+
// evict the live chains for nothing. Purely a cost decision now, not a
|
|
124
|
+
// correctness one.
|
|
134
125
|
if (conversational && bytes.length >= 2) {
|
|
135
126
|
// The lengths set drifts as the map evicts; past the probe budget the
|
|
136
127
|
// drift itself becomes the cost (each stale length is an O(len) key
|
|
@@ -139,10 +130,10 @@ export function perceiveDeposit(ctx, bytes, conversational = false) {
|
|
|
139
130
|
ctx._depositLens.clear();
|
|
140
131
|
ctx._depositTrees.clear();
|
|
141
132
|
}
|
|
142
|
-
ctx._depositTrees.set(latin1Key(bytes),
|
|
133
|
+
ctx._depositTrees.set(latin1Key(bytes), { content: folded.fold });
|
|
143
134
|
ctx._depositLens.add(bytes.length);
|
|
144
135
|
}
|
|
145
|
-
return tree;
|
|
136
|
+
return folded.tree;
|
|
146
137
|
}
|
|
147
138
|
/** The raw bytes of an input — modality-neutral conversion. */
|
|
148
139
|
export function inputBytes(ctx, input) {
|
|
@@ -164,20 +155,47 @@ export function gistOf(ctx, bytes) {
|
|
|
164
155
|
* node with its byte span and resolved id. Returns the node's byte end and
|
|
165
156
|
* resolved id. */
|
|
166
157
|
export function foldTree(ctx, n, start, visit) {
|
|
167
|
-
//
|
|
168
|
-
//
|
|
169
|
-
//
|
|
170
|
-
// instead of O(context)
|
|
158
|
+
// Subtree already resolved (from a previous conversation turn or an earlier
|
|
159
|
+
// recognition pass). The pyramid reuses prefix subtrees as identical Sema
|
|
160
|
+
// objects, so a conversation's prefix is warm from its second turn on.
|
|
161
|
+
// Without a visitor that makes foldTree O(suffix) instead of O(context);
|
|
162
|
+
// with one it stays O(context) and saves the per-node store probes instead
|
|
163
|
+
// (see below for why the distinction is not negotiable).
|
|
164
|
+
//
|
|
165
|
+
// WHAT THE CACHE KNOWS, AND WHAT IT DOES NOT. An entry records this
|
|
166
|
+
// subtree's id and byte length — nothing about its DESCENDANTS' spans.
|
|
167
|
+
// Returning here therefore emits ONE visit() where a cold walk emits one per
|
|
168
|
+
// node, and `visit` is not instrumentation: recognise() emits its sites from
|
|
169
|
+
// it (recognition.ts) and attention's collectRegions votes over what it
|
|
170
|
+
// yields (attention.ts). Skipping the descent silently shrinks the evidence
|
|
171
|
+
// those mechanisms see, purely because the cache happened to be warm.
|
|
172
|
+
//
|
|
173
|
+
// That is not hypothetical and not an edge case — it is every conversation
|
|
174
|
+
// turn after the first. `contentFoldIncremental` deliberately shares prefix
|
|
175
|
+
// segment OBJECTS across turns (~99% reuse), so by turn 2 the prefix is
|
|
176
|
+
// warm; meanwhile recogniseMemo/climbMemo are keyed on exact query BYTES,
|
|
177
|
+
// which a growing context never repeats. Warm subtrees + missed memos is
|
|
178
|
+
// the unprotected quadrant. Measured over real trained conversations,
|
|
179
|
+
// recognising the same context with a warm prefix lost 67-92% of its leaves
|
|
180
|
+
// (772->204, 589->47, 872->291, 377->37) with `sites` unchanged, so the loss
|
|
181
|
+
// is invisible to the coarse counts; a direct foldTree probe on identical
|
|
182
|
+
// bytes and an identical tree object fired visit() 661 times cold and 37
|
|
183
|
+
// warm. respond() is immune only because it never sets _resolvedSubtrees
|
|
184
|
+
// (mind.ts) — the degradation was unique to the multi-turn API.
|
|
185
|
+
//
|
|
186
|
+
// So the fast path is taken only when NOBODY IS WATCHING. With a visitor
|
|
187
|
+
// present we still walk, and the cache degrades to the thing it soundly is:
|
|
188
|
+
// an elision of the store probes (findLeaf/findBranch) at each node, not an
|
|
189
|
+
// elision of the traversal. Ids still come from the cache, so a warm walk
|
|
190
|
+
// is cheaper than a cold one; it is no longer *different* from one.
|
|
171
191
|
const cached = ctx._resolvedSubtrees?.get(n);
|
|
172
|
-
if (cached !== undefined) {
|
|
173
|
-
|
|
174
|
-
visit?.(n, start, end, cached.id);
|
|
175
|
-
return { end, node: cached.id };
|
|
192
|
+
if (cached !== undefined && visit === undefined) {
|
|
193
|
+
return { end: start + cached.len, node: cached.id };
|
|
176
194
|
}
|
|
177
195
|
if (n.kids === null) {
|
|
178
196
|
const b = n.leaf ?? new Uint8Array(0);
|
|
179
197
|
const end = start + b.length;
|
|
180
|
-
const node = ctx.store.findLeaf(b);
|
|
198
|
+
const node = cached !== undefined ? cached.id : ctx.store.findLeaf(b);
|
|
181
199
|
visit?.(n, start, end, node);
|
|
182
200
|
if (node !== null && ctx._resolvedSubtrees) {
|
|
183
201
|
ctx._resolvedSubtrees.set(n, { id: node, len: b.length });
|
|
@@ -195,7 +213,16 @@ export function foldTree(ctx, n, start, visit) {
|
|
|
195
213
|
kids.push(r.node);
|
|
196
214
|
pos = r.end;
|
|
197
215
|
}
|
|
198
|
-
|
|
216
|
+
// Same store-probe elision as the leaf case: a cached entry already names
|
|
217
|
+
// this subtree, so the descent above was for `visit`'s benefit alone and the
|
|
218
|
+
// id need not be re-derived. Using it also keeps a warm walk's ids
|
|
219
|
+
// bit-identical to a cold walk's rather than re-deriving them from children
|
|
220
|
+
// that may themselves have come from cache.
|
|
221
|
+
const node = cached !== undefined
|
|
222
|
+
? cached.id
|
|
223
|
+
: known
|
|
224
|
+
? ctx.store.findBranch(kids)
|
|
225
|
+
: null;
|
|
199
226
|
visit?.(n, start, pos, node);
|
|
200
227
|
if (node !== null && ctx._resolvedSubtrees) {
|
|
201
228
|
ctx._resolvedSubtrees.set(n, { id: node, len: pos - start });
|
|
@@ -8,6 +8,7 @@ import { rItem } from "./trace.js";
|
|
|
8
8
|
import { canonResolve, foldTree, gistOf, latin1Key, perceive, resolve, } from "./primitives.js";
|
|
9
9
|
import { atomIsHub, corpusN, leadsSomewhere } from "./traverse.js";
|
|
10
10
|
import { chainReach, leafIdAt, leafIdRun } from "./canonical.js";
|
|
11
|
+
import { canonHash } from "../canon.js";
|
|
11
12
|
import { isChunk } from "../sema.js";
|
|
12
13
|
/** Decompose a byte stream into every stored form that leads somewhere
|
|
13
14
|
* (has a continuation edge or a halo). Two complementary readings:
|
|
@@ -25,23 +26,40 @@ export function recognise(ctx, bytes) {
|
|
|
25
26
|
// Content-keyed memo — works for both single-turn respond() and multi-turn
|
|
26
27
|
// respondTurn() (where the map persists across calls). ALWAYS consulted,
|
|
27
28
|
// regardless of tracing — matching perceive()'s own memo, which carries no
|
|
28
|
-
// trace gate at all.
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
//
|
|
32
|
-
//
|
|
33
|
-
//
|
|
34
|
-
//
|
|
35
|
-
//
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
//
|
|
39
|
-
//
|
|
40
|
-
//
|
|
41
|
-
//
|
|
42
|
-
//
|
|
43
|
-
//
|
|
44
|
-
//
|
|
29
|
+
// trace gate at all.
|
|
30
|
+
//
|
|
31
|
+
// This memo is an accelerator, and that is now the whole of it: repeated
|
|
32
|
+
// recognition of the same query is ordinary within one response (cover,
|
|
33
|
+
// reason and articulate all recognise it) and recogniseImpl is O(n ·
|
|
34
|
+
// maxGroup) probes each time.
|
|
35
|
+
//
|
|
36
|
+
// IT USED TO BE LOAD-BEARING FOR CORRECTNESS, and the history is worth
|
|
37
|
+
// keeping because it explains why there is no trace gate here. foldTree's
|
|
38
|
+
// subtree-resolution fast path (primitives.ts) once returned on a cache hit
|
|
39
|
+
// WITHOUT recursing, so it skipped invoking `visit` — and therefore skipped
|
|
40
|
+
// EMITTING SITES — for any subtree already in ctx._resolvedSubtrees. A
|
|
41
|
+
// conversation's incremental fold deliberately shares node OBJECTS across
|
|
42
|
+
// turns, so by the second call on the same bytes large swaths of the tree
|
|
43
|
+
// were already cached and recogniseImpl silently found FEWER sites than the
|
|
44
|
+
// first call (observed live: 31 → 5). Skipping this memo "only while
|
|
45
|
+
// tracing" therefore meant every traced turn re-ran recogniseImpl at each of
|
|
46
|
+
// those call sites, each result more incomplete than the last — changing
|
|
47
|
+
// which mechanism grounded the answer, not merely costing time.
|
|
48
|
+
//
|
|
49
|
+
// foldTree no longer does that: it takes the fast path only when no `visit`
|
|
50
|
+
// is supplied, so a walk that emits sites always walks in full and the id
|
|
51
|
+
// cache is reduced to eliding store probes (see primitives.ts). recognise()
|
|
52
|
+
// is idempotent on its own now — verified with the memo bypassed, the
|
|
53
|
+
// subtree cache warm and the tree object shared: three consecutive calls on
|
|
54
|
+
// the same 544-byte context returned sites=2 leaves=544 splits=0 starts=88,
|
|
55
|
+
// identical every time.
|
|
56
|
+
//
|
|
57
|
+
// The unconditional consult STAYS regardless. A memo whose absence can only
|
|
58
|
+
// cost time is still not something to gate on whether an audit happens to be
|
|
59
|
+
// attached: tracing must not change what the pipeline computes, and the
|
|
60
|
+
// cheapest way to guarantee that is for the trace flag to touch nothing but
|
|
61
|
+
// the trace. The trace step must still fire on every call (a cache hit is
|
|
62
|
+
// not silent), so it is emitted here directly rather than only inside
|
|
45
63
|
// recogniseImpl.
|
|
46
64
|
if (ctx.recogniseMemo) {
|
|
47
65
|
const key = latin1Key(bytes);
|
|
@@ -400,7 +418,21 @@ function recogniseImpl(ctx, bytes) {
|
|
|
400
418
|
// before resolveSpan pays for a fold; approximate evidence never enters.
|
|
401
419
|
// This tier is needed only where atom chains are suppressed. Small stores
|
|
402
420
|
// retain their existing decomposition unchanged.
|
|
403
|
-
|
|
421
|
+
// ALWAYS ON, AND LINEAR. This used to be gated on `atomsAreHubs` — small
|
|
422
|
+
// stores were said to "retain their existing decomposition unchanged", which
|
|
423
|
+
// was true only while the query's fold was told where the turns were: every
|
|
424
|
+
// turn was then a NODE, so the structural walk found it and this tier had
|
|
425
|
+
// nothing to add. The fold no longer imposes turn boundaries (a turn start
|
|
426
|
+
// is an ordinary interior offset now), so a trained form embedded in a
|
|
427
|
+
// longer query is reachable ONLY here — the chain caps at chainReach(W)=W²
|
|
428
|
+
// bytes and cannot span one. Measured: with the fold imposing boundaries
|
|
429
|
+
// every turn is a node; without it, none is.
|
|
430
|
+
//
|
|
431
|
+
// Ungating it alone made inference QUADRATIC (test/14's constant-KB/s guard
|
|
432
|
+
// went to 41.8s): every offset near a cut is an endpoint, and each probe
|
|
433
|
+
// costs O(span) to slice the leaf-id run and hash it. The budget below is
|
|
434
|
+
// what makes it affordable — see `spend`.
|
|
435
|
+
{
|
|
404
436
|
const allLeafIds = singleLeaf.map((x) => x?.id ?? null);
|
|
405
437
|
if (allLeafIds.every((x) => x !== null)) {
|
|
406
438
|
const radius = ctx.space.seats.length;
|
|
@@ -410,20 +442,115 @@ function recogniseImpl(ctx, bytes) {
|
|
|
410
442
|
endpoints.add(p);
|
|
411
443
|
}
|
|
412
444
|
const ordered = [...endpoints].sort((a, b) => a - b);
|
|
413
|
-
|
|
445
|
+
// The leaf-id run is BYTE-EXACT, while `resolveSpan` behind it resolves
|
|
446
|
+
// exactly OR canonically — so this gate was strictly narrower than its
|
|
447
|
+
// own resolver, and every embedded form differing from its deposit only
|
|
448
|
+
// by the response's equivalence (case, width) was dropped before the
|
|
449
|
+
// resolver ever saw it. Rebuilding the run over canonicalized bytes
|
|
450
|
+
// does NOT fix that: a differently-cased deposit's branch kid-ids are
|
|
451
|
+
// not the query's leaf-id run under ANY canonicalization of the query,
|
|
452
|
+
// so the second admission route has to be the canon INDEX itself — the
|
|
453
|
+
// same candidate proposal `canonResolve` makes, and the same
|
|
454
|
+
// cheap-probe-before-a-fold discipline the exact route already follows
|
|
455
|
+
// (a hash and an indexed lookup; no fold, no vector, no scan). Both
|
|
456
|
+
// routes only PROPOSE; `resolveSpan` still decides, so a hash-bucket
|
|
457
|
+
// collision costs one fold and can never emit a wrong site (test/71).
|
|
458
|
+
const canonAdmits = (start, end) => {
|
|
459
|
+
const canon = ctx.canon;
|
|
460
|
+
if (canon === null || !store.canonFind)
|
|
461
|
+
return false;
|
|
462
|
+
const key = canon(bytes.subarray(start, end));
|
|
463
|
+
if (key.length === 0)
|
|
464
|
+
return false;
|
|
465
|
+
return store.canonFind(canonHash(key)).length > 0;
|
|
466
|
+
};
|
|
467
|
+
// The byte-exact route probes the SPAN ITSELF (see
|
|
468
|
+
// Store.findFlatBranch): for a run of single-byte leaves the flat-kid
|
|
469
|
+
// encoding is the identity, so the span's bytes ARE the branch key.
|
|
470
|
+
// `subarray` is a view — this allocates nothing per probe, and the
|
|
471
|
+
// bloom filter answers the misses without touching the database.
|
|
472
|
+
const flatProbe = (start, end) => store.findFlatBranch
|
|
473
|
+
? store.findFlatBranch(bytes.subarray(start, end))
|
|
474
|
+
: store.findBranch(allLeafIds.slice(start, end));
|
|
475
|
+
// THE TWO ROUTES COST DIFFERENT THINGS, SO THEY ARE PRICED SEPARATELY.
|
|
476
|
+
//
|
|
477
|
+
// The exact route is a bloom-gated hash over a subarray VIEW: no
|
|
478
|
+
// allocation, and a miss never reaches the database. It is cheap enough
|
|
479
|
+
// to run on every endpoint, and that is what makes this tier able to
|
|
480
|
+
// find a trained form embedded anywhere in the query.
|
|
481
|
+
//
|
|
482
|
+
// The canon route is not: it runs the canonicalizer over the span
|
|
483
|
+
// (NFKC, case-fold, whitespace) and allocates a fresh key for every
|
|
484
|
+
// probe. That is the O(span) cost with the heavy constant, and it is
|
|
485
|
+
// the one worth a budget. Sharing ONE budget between them made the
|
|
486
|
+
// cheap route starve on the expensive one's behalf — measured, test/71's
|
|
487
|
+
// embedded differently-cased form needed 64x the budget to be found,
|
|
488
|
+
// while the exact route it was competing with needed none of it.
|
|
489
|
+
const probe = (start, end, canonBudget) => {
|
|
414
490
|
if (end - start < W || end - start <= chainReach(W))
|
|
415
491
|
return;
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
492
|
+
if (flatProbe(start, end) === null) {
|
|
493
|
+
if (!canonBudget)
|
|
494
|
+
return;
|
|
495
|
+
if (!canonAdmits(start, end))
|
|
496
|
+
return;
|
|
497
|
+
}
|
|
419
498
|
const id = resolveSpan(start, end);
|
|
420
499
|
if (id !== null)
|
|
421
500
|
emit(start, end, id);
|
|
422
501
|
};
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
502
|
+
// A CUMULATIVE BYTE BUDGET, SPENT SHORTEST-SPAN-FIRST.
|
|
503
|
+
//
|
|
504
|
+
// Each probe costs O(span), and there are O(n) endpoints, so probing
|
|
505
|
+
// them all is O(n²) — the quadratic this tier was gated to avoid. The
|
|
506
|
+
// budget caps TOTAL probe bytes at a multiple of the query's own length,
|
|
507
|
+
// which is what keeps whole-query inference linear.
|
|
508
|
+
//
|
|
509
|
+
// Spending it shortest-first is what makes the cap a scale bound rather
|
|
510
|
+
// than a position bound: the tier recovers embedded forms up to roughly
|
|
511
|
+
// √(2·budget) bytes ANYWHERE in the endpoint set, instead of walking the
|
|
512
|
+
// endpoints in order and running out partway along the query. A form
|
|
513
|
+
// longer than that is out of this tier's reach — but so is a form the
|
|
514
|
+
// chain cannot span, and that is exactly the trade the budget prices.
|
|
515
|
+
// The factor is chainReach(W), the same W² scale the chain already
|
|
516
|
+
// trusts; no new constant.
|
|
517
|
+
// The factor is chainReach(W) — the same W² scale the chain itself
|
|
518
|
+
// trusts — so the cap is derived from the fold's geometry, never tuned.
|
|
519
|
+
// (It was briefly an environment variable while the cost was being
|
|
520
|
+
// measured; an env-read here would make inference non-reproducible,
|
|
521
|
+
// which the determinism contract forbids outright.)
|
|
522
|
+
// The factor is chainReach(W) — the same W² scale the chain itself
|
|
523
|
+
// trusts — so the cap is derived from the fold's geometry, never tuned.
|
|
524
|
+
// (It was briefly an environment variable while the cost was being
|
|
525
|
+
// measured; an env-read here would make inference non-reproducible,
|
|
526
|
+
// which the determinism contract forbids outright.)
|
|
527
|
+
//
|
|
528
|
+
// It now prices ONLY the canonicalizing route; the exact route runs on
|
|
529
|
+
// every endpoint regardless, so exhausting this budget narrows which
|
|
530
|
+
// equivalence-class forms are proposed, never which byte-exact ones.
|
|
531
|
+
let budget = bytes.length * chainReach(W) * chainReach(W);
|
|
532
|
+
const spend = (start, end) => {
|
|
533
|
+
const span = end - start;
|
|
534
|
+
const afford = span <= budget;
|
|
535
|
+
if (afford)
|
|
536
|
+
budget -= span;
|
|
537
|
+
probe(start, end, afford);
|
|
538
|
+
// Always keep walking: the exact route is unbudgeted, so running out
|
|
539
|
+
// of canon budget must not stop the scan.
|
|
540
|
+
return true;
|
|
541
|
+
};
|
|
542
|
+
const prefixes = ordered.filter((e) => e > 0).sort((a, b) => a - b);
|
|
543
|
+
const suffixes = ordered
|
|
544
|
+
.filter((s2) => s2 < bytes.length)
|
|
545
|
+
.sort((a, b) => b - a);
|
|
546
|
+
for (let i = 0; i < Math.max(prefixes.length, suffixes.length); i++) {
|
|
547
|
+
// Interleaved so neither edge starves the other when the budget runs
|
|
548
|
+
// out — a query can carry a trained form at either end.
|
|
549
|
+
if (i < prefixes.length && !spend(0, prefixes[i]))
|
|
550
|
+
break;
|
|
551
|
+
if (i < suffixes.length && !spend(suffixes[i], bytes.length))
|
|
552
|
+
break;
|
|
553
|
+
}
|
|
427
554
|
}
|
|
428
555
|
}
|
|
429
556
|
const chunkEnd = new Uint32Array(bytes.length);
|
|
@@ -124,3 +124,35 @@ export declare function chooseAmong(ctx: MindContext, candidates: readonly numbe
|
|
|
124
124
|
id: number;
|
|
125
125
|
score: number;
|
|
126
126
|
};
|
|
127
|
+
/** True when NO window of `query` discriminates anything — every stored
|
|
128
|
+
* W-window it spells is contained by more places than the hub bound allows,
|
|
129
|
+
* i.e. the whole query is corpus-global scaffolding.
|
|
130
|
+
*
|
|
131
|
+
* WHAT IT IS FOR. Several mechanisms ground a query through the literal
|
|
132
|
+
* spans it did NOT explain, and those spans are the whole of their evidence.
|
|
133
|
+
* When every one of them is a hub, the query says nothing the corpus can be
|
|
134
|
+
* held to, and grounding it means picking one of thousands of continuations
|
|
135
|
+
* it gives no evidence for — a fabrication whatever the answer happens to be.
|
|
136
|
+
* Answering with silence there is the honest degradation contract (§2.13).
|
|
137
|
+
*
|
|
138
|
+
* MEASURED SEPARATION (trained store, hubBound 571) — this is categorical,
|
|
139
|
+
* not marginal, and it is why the predicate lives here rather than being
|
|
140
|
+
* spelled twice:
|
|
141
|
+
* "What is the capital of" ALL saturated ("What":572) → fabricated
|
|
142
|
+
* "What is the capital " ALL saturated ("What":572) → fabricated
|
|
143
|
+
* "what is the capital of france" min "f fr":248 → correct
|
|
144
|
+
* "What is the capitol of France?" min "f Fr":114 → correct
|
|
145
|
+
* "WHAT IS THE CAPITAL OF FRANCE?" min "HE C":1 → correct
|
|
146
|
+
* "What is the capital of France?" min "t i":4 → correct
|
|
147
|
+
* "Who wrote Romeo and Juliet?" min "iet?":26 → correct
|
|
148
|
+
* "What is the capital of Zamunda?" min "Zamu":3 → silent anyway
|
|
149
|
+
* Note the last: the honest-silence probes are already refused on other
|
|
150
|
+
* evidence and sit on the SAME side as the correct ones, so this predicate
|
|
151
|
+
* is not what makes them silent and cannot be credited for them.
|
|
152
|
+
*
|
|
153
|
+
* NO NEW THRESHOLD (§2.2): `hubBound` is the √N reading of "hub" used
|
|
154
|
+
* everywhere, and the containment read is clamped to it exactly as every
|
|
155
|
+
* other fan-out read is (§2.8). A query with no stored window at all is NOT
|
|
156
|
+
* scaffolding-only — it has no evidence either way, and its callers already
|
|
157
|
+
* refuse it on their own terms. */
|
|
158
|
+
export declare function allWindowsAreScaffolding(ctx: MindContext, query: Uint8Array): boolean;
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
// project) live in match.ts — the elementary match-and-project operation.
|
|
8
8
|
import { cosine } from "../vec.js";
|
|
9
9
|
import { gistOf, read } from "./primitives.js";
|
|
10
|
+
import { leafIdRun } from "./canonical.js";
|
|
10
11
|
const structCaches = new WeakMap();
|
|
11
12
|
// ── The shared ancestor-reach memo ──────────────────────────────────────
|
|
12
13
|
//
|
|
@@ -648,3 +649,54 @@ function rItemShort(ctx, id, role, score) {
|
|
|
648
649
|
score,
|
|
649
650
|
};
|
|
650
651
|
}
|
|
652
|
+
/** True when NO window of `query` discriminates anything — every stored
|
|
653
|
+
* W-window it spells is contained by more places than the hub bound allows,
|
|
654
|
+
* i.e. the whole query is corpus-global scaffolding.
|
|
655
|
+
*
|
|
656
|
+
* WHAT IT IS FOR. Several mechanisms ground a query through the literal
|
|
657
|
+
* spans it did NOT explain, and those spans are the whole of their evidence.
|
|
658
|
+
* When every one of them is a hub, the query says nothing the corpus can be
|
|
659
|
+
* held to, and grounding it means picking one of thousands of continuations
|
|
660
|
+
* it gives no evidence for — a fabrication whatever the answer happens to be.
|
|
661
|
+
* Answering with silence there is the honest degradation contract (§2.13).
|
|
662
|
+
*
|
|
663
|
+
* MEASURED SEPARATION (trained store, hubBound 571) — this is categorical,
|
|
664
|
+
* not marginal, and it is why the predicate lives here rather than being
|
|
665
|
+
* spelled twice:
|
|
666
|
+
* "What is the capital of" ALL saturated ("What":572) → fabricated
|
|
667
|
+
* "What is the capital " ALL saturated ("What":572) → fabricated
|
|
668
|
+
* "what is the capital of france" min "f fr":248 → correct
|
|
669
|
+
* "What is the capitol of France?" min "f Fr":114 → correct
|
|
670
|
+
* "WHAT IS THE CAPITAL OF FRANCE?" min "HE C":1 → correct
|
|
671
|
+
* "What is the capital of France?" min "t i":4 → correct
|
|
672
|
+
* "Who wrote Romeo and Juliet?" min "iet?":26 → correct
|
|
673
|
+
* "What is the capital of Zamunda?" min "Zamu":3 → silent anyway
|
|
674
|
+
* Note the last: the honest-silence probes are already refused on other
|
|
675
|
+
* evidence and sit on the SAME side as the correct ones, so this predicate
|
|
676
|
+
* is not what makes them silent and cannot be credited for them.
|
|
677
|
+
*
|
|
678
|
+
* NO NEW THRESHOLD (§2.2): `hubBound` is the √N reading of "hub" used
|
|
679
|
+
* everywhere, and the containment read is clamped to it exactly as every
|
|
680
|
+
* other fan-out read is (§2.8). A query with no stored window at all is NOT
|
|
681
|
+
* scaffolding-only — it has no evidence either way, and its callers already
|
|
682
|
+
* refuse it on their own terms. */
|
|
683
|
+
export function allWindowsAreScaffolding(ctx, query) {
|
|
684
|
+
const W = ctx.space.maxGroup;
|
|
685
|
+
const bound = hubBound(ctx);
|
|
686
|
+
let sawOne = false;
|
|
687
|
+
for (let o = 0; o + W <= query.length; o++) {
|
|
688
|
+
const ids = leafIdRun(ctx, query, o, o + W);
|
|
689
|
+
if (ids === null)
|
|
690
|
+
continue;
|
|
691
|
+
const id = ctx.store.findBranch(ids);
|
|
692
|
+
if (id === null)
|
|
693
|
+
continue;
|
|
694
|
+
const rarity = ctx.store.containersSlice(id, 0, bound + 1).length;
|
|
695
|
+
if (rarity === 0)
|
|
696
|
+
continue;
|
|
697
|
+
if (rarity <= bound)
|
|
698
|
+
return false;
|
|
699
|
+
sawOne = true;
|
|
700
|
+
}
|
|
701
|
+
return sawOne;
|
|
702
|
+
}
|