@hviana/sema 0.4.6 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/AGENTS.md +290 -77
  2. package/HOW_IT_WORKS.md +2170 -735
  3. package/dist/example/train_base.d.ts +9 -3
  4. package/dist/example/train_base.js +21 -4
  5. package/dist/src/canon.d.ts +19 -0
  6. package/dist/src/canon.js +28 -0
  7. package/dist/src/geometry.d.ts +52 -0
  8. package/dist/src/geometry.js +87 -1
  9. package/dist/src/mind/bridge.js +27 -1
  10. package/dist/src/mind/frame-filler.d.ts +15 -0
  11. package/dist/src/mind/frame-filler.js +535 -0
  12. package/dist/src/mind/learning.js +6 -11
  13. package/dist/src/mind/mechanisms/cast.js +72 -2
  14. package/dist/src/mind/mechanisms/cover.js +6 -1
  15. package/dist/src/mind/mechanisms/extraction.js +27 -0
  16. package/dist/src/mind/mechanisms/recall.js +214 -34
  17. package/dist/src/mind/mind.d.ts +49 -1
  18. package/dist/src/mind/mind.js +137 -10
  19. package/dist/src/mind/pipeline-mechanism.d.ts +7 -0
  20. package/dist/src/mind/pipeline.js +29 -1
  21. package/dist/src/mind/prefix-completion.d.ts +59 -0
  22. package/dist/src/mind/prefix-completion.js +270 -0
  23. package/dist/src/mind/primitives.d.ts +29 -10
  24. package/dist/src/mind/primitives.js +52 -61
  25. package/dist/src/mind/recognition.js +119 -9
  26. package/dist/src/mind/traverse.d.ts +32 -0
  27. package/dist/src/mind/traverse.js +52 -0
  28. package/dist/src/mind/types.d.ts +55 -16
  29. package/dist/src/mind/types.js +68 -19
  30. package/dist/src/rabitq-ivf/src/rabitq.js +31 -1
  31. package/dist/src/store.d.ts +21 -0
  32. package/dist/src/store.js +21 -0
  33. package/example/train_base.ts +21 -4
  34. package/package.json +1 -1
  35. package/src/canon.ts +28 -0
  36. package/src/geometry.ts +100 -1
  37. package/src/mind/bridge.ts +34 -0
  38. package/src/mind/frame-filler.ts +604 -0
  39. package/src/mind/learning.ts +5 -9
  40. package/src/mind/mechanisms/cast.ts +70 -2
  41. package/src/mind/mechanisms/cover.ts +6 -1
  42. package/src/mind/mechanisms/extraction.ts +27 -0
  43. package/src/mind/mechanisms/recall.ts +236 -37
  44. package/src/mind/mind.ts +154 -14
  45. package/src/mind/pipeline-mechanism.ts +7 -0
  46. package/src/mind/pipeline.ts +33 -1
  47. package/src/mind/prefix-completion.ts +314 -0
  48. package/src/mind/primitives.ts +59 -70
  49. package/src/mind/recognition.ts +117 -6
  50. package/src/mind/traverse.ts +52 -0
  51. package/src/mind/types.ts +98 -42
  52. package/src/rabitq-ivf/src/rabitq.ts +31 -1
  53. package/src/store.ts +25 -0
  54. package/test/13-conversation.test.mjs +13 -0
  55. package/test/57-fusion-order.test.mjs +65 -0
  56. package/test/65-ann-recall.test.mjs +331 -0
  57. package/test/66-query-edge-whitespace.test.mjs +99 -0
  58. package/test/67-climb-anchor-breadth.test.mjs +113 -0
  59. package/test/68-extraction-unanchored.test.mjs +79 -0
  60. package/test/69-frame-filler.test.mjs +115 -0
  61. package/test/70-prefix-completion.test.mjs +170 -0
  62. package/test/71-embedded-canon-equivalence.test.mjs +121 -0
  63. package/test/72-prefix-candidate-supply.test.mjs +114 -0
  64. package/test/73-scaffolding-only-bridge-abstains.test.mjs +178 -0
  65. package/test/74-prefix-trap-not-sprung-early.test.mjs +114 -0
  66. package/test/75-multiturn-context-optimisation.test.mjs +1082 -0
@@ -108,7 +108,34 @@ export async function think(ctx, query, mechs) {
108
108
  if (ctx.meter)
109
109
  ctx.meter.candidates++;
110
110
  candidates.push(c);
111
- if (best === null || grade(c.weight) < grade(best.weight))
111
+ if (best === null) {
112
+ best = c;
113
+ return;
114
+ }
115
+ const g = grade(c.weight), gb = grade(best.weight);
116
+ if (g < gb) {
117
+ best = c;
118
+ return;
119
+ }
120
+ // TIE-BREAK: AT EQUAL GRADE, PREFER THE ANSWER THAT INVENTS LESS.
121
+ //
122
+ // The ladder prices what a candidate leaves UNACCOUNTED, which is the
123
+ // right primary question but cannot separate two candidates that leave
124
+ // the same bytes unaccounted — and then the winner is whichever mechanism
125
+ // happened to be considered first, which is not a reason.
126
+ //
127
+ // What still separates them is what they DID with those bytes. A
128
+ // candidate that carries an unexplained span into its answer is passing
129
+ // the asker's own words back as if they were derived; one that leaves
130
+ // them out has made a smaller, honest claim. Measured on test/22's
131
+ // two-fact chain: cover and recall both graded 11001 over 11 unexplained
132
+ // bytes, cover answering "The capital of France is Paris famous for" (11
133
+ // bytes of scaffolding) against recall's crossing of the hop (0). Order
134
+ // alone decided it, and the shallower reading won.
135
+ //
136
+ // This never overrides the ladder — it only orders within one grade, so
137
+ // coverage and moves still dominate exactly as before.
138
+ if (g === gb && (c.scaffolding ?? 0) < (best.scaffolding ?? 0))
112
139
  best = c;
113
140
  };
114
141
  const worthRunning = (floor) => best === null || grade(floor) < grade(best.weight);
@@ -150,6 +177,7 @@ export async function think(ctx, query, mechs) {
150
177
  accounted: r.accounted,
151
178
  unexplained: r.unexplained,
152
179
  complete: r.complete,
180
+ scaffolding: r.scaffolding,
153
181
  });
154
182
  }
155
183
  }
@@ -0,0 +1,59 @@
1
+ import type { MindContext } from "./types.js";
2
+ /** Trained forms the query may OPEN, proposed from the write side's own
3
+ * leaf-id window index — the supply of last resort for {@link
4
+ * prefixCompletion}.
5
+ *
6
+ * WHY A SECOND SUPPLY EXISTS. The ranked list this mechanism normally reads
7
+ * is a resonance list, and resonance cannot rank a proper prefix: measured on
8
+ * the trained store, cos(prefix, form) falls from 0.9629 at a one-byte
9
+ * truncation to 0.6206 at three bytes, against a reachThreshold of 0.8750.
10
+ * Three bytes of truncation put the answer out of reach on GEOMETRY, not on a
11
+ * bug, so no k and no re-ranking recovers it.
12
+ *
13
+ * WHY THIS ROUTE WORKS WHERE THE FOLD DOES NOT. A query's own fold is
14
+ * useless here: content addressing is not phrase-position-invariant, so a
15
+ * standalone prefix folds to a DIFFERENT node than the same bytes sitting
16
+ * inside a longer deposit, and neither the prefix's own node nor its
17
+ * ancestors lead to the deposit (measured: the 22-byte prefix of the
18
+ * photosynthesis form resolves, is shared by 6 contexts, and does not have
19
+ * the form among its ancestors). Leaf ids ARE position-invariant — they are
20
+ * content-addressed on single bytes — and `indexSubSpans` already interns a
21
+ * flat branch over every canonical WINDOW of a deposit's leaf-id stream, with
22
+ * containment edges to the chunks that window spans. A query that is a
23
+ * prefix therefore shares those window nodes exactly, and reaches the deposit
24
+ * by climbing containment then parents. Nothing is added to the write side;
25
+ * this reads an index training already built.
26
+ *
27
+ * BOUNDED (§2.8), AND WITH NO NEW THRESHOLD. The window whose containment is
28
+ * SMALLEST carries the most evidence, and one saturated at `hubBound` carries
29
+ * none — that is the same √N reading of "hub" the rest of the mind uses, not
30
+ * a tuned knob. The upward walk spends a budget of `hubBound` nodes and
31
+ * fans out by W, so a hub query enumerates nothing and the caller stays
32
+ * silent rather than guessing (§2.13). Measured on the trained store: the
33
+ * photosynthesis form at a one-byte truncation picks a window with 52
34
+ * containers, visits 446 nodes, and yields exactly ONE candidate that
35
+ * survives the caller's byte compare — the form itself.
36
+ *
37
+ * These are PROPOSALS only. Every candidate still faces the byte-exact
38
+ * prefix compare and all three guards below, so a wrong proposal costs one
39
+ * bounded read and can never be voiced (§2.3). */
40
+ export declare function prefixCandidates(ctx: MindContext, query: Uint8Array): number[];
41
+ /** A trained form the query opens, and the bytes by which it continues. */
42
+ export interface PrefixCompletion {
43
+ /** The trained form whose opening the query is — the answer, voiced whole. */
44
+ id: number;
45
+ /** The form's own bytes. The mechanism grounds a FORM, never a slice of
46
+ * one: slicing at the query's end would cut at an offset the geometry has
47
+ * no reason to treat as a boundary. */
48
+ form: Uint8Array;
49
+ /** The bytes past the query — carried for the rationale and for the
50
+ * uniqueness comparison, not voiced on its own. */
51
+ continuation: Uint8Array;
52
+ }
53
+ /** The sole trained form the query opens — or null when no candidate opens with
54
+ * it, when the continuation is sub-quantum, when a candidate's continuation
55
+ * cannot be read through, or when the candidates disagree.
56
+ *
57
+ * `ranked` must be a list the caller has ALREADY fetched; this mechanism never
58
+ * resonates on its own (see the header's cost note). */
59
+ export declare function prefixCompletion(ctx: MindContext, query: Uint8Array, ranked: ReadonlyArray<number>): PrefixCompletion | null;
@@ -0,0 +1,270 @@
1
+ // prefix-completion.ts — Grounding a query that IS the opening of a trained
2
+ // form.
3
+ //
4
+ // THE SHAPE. `The capital of France is` grounds nothing, while
5
+ // `The capital of France is Paris.` is trained and reads back byte-exact. The
6
+ // query is not SIMILAR to that form, it is a PROPER PREFIX of it: every query
7
+ // byte is a literal match, in order, from offset zero. That is the strongest
8
+ // grounding relation in the store — stronger than the bridge's corroborated
9
+ // substitution, which pays a CONCEPT per substituted span, and stronger than
10
+ // resonance, which only claims an angle. Nothing is invented: the answer IS a
11
+ // trained form, voiced whole.
12
+ //
13
+ // NO NOTION OF TEXT. This mechanism reads bytes and geometry only. It has no
14
+ // separator, no character class, no "word": the only structural quantity it
15
+ // uses is W, the river's grouping window, which is the same capacity the
16
+ // perception tree groups by and the same bar the argument-binding tier holds
17
+ // its constituents to. A completion shorter than one grouping window carries
18
+ // no structure the geometry can perceive, whatever the modality — that is a
19
+ // statement about the fold, not about punctuation. Presentation (what is
20
+ // "spacing", what is "case") belongs to the injected canon and to the modality
21
+ // entry point, never here; see src/canon.ts.
22
+ //
23
+ // WHY THE EARLIER TIERS CANNOT DO IT. Two independent reasons, both measured:
24
+ //
25
+ // 1. `resolve(prefix)` is null. A proper prefix of a deposited stream has no
26
+ // branch of its own unless it was itself deposited, so the exact tiers
27
+ // have nothing to find.
28
+ // 2. The form is not among the resonance candidates AT ALL. Measured on the
29
+ // trained store: cos(query, that form) = 0.5752, yet the form is absent
30
+ // from `resonate(k)` at k = 24, 256 AND 2048 — while forms scoring LOWER
31
+ // (Germany 0.5670, Yemen 0.5591) are returned. `k` only reorders WITHIN
32
+ // the IVF clusters already probed, exactly as Store.resonate's doc warns,
33
+ // so no k recovers it. With `exhaustive` it ranks 8.
34
+ //
35
+ // So this is a RETRIEVABILITY gap, not a semantic one, and it is repaired by
36
+ // reading the candidate list recall's refusal path has ALREADY fetched
37
+ // exhaustively for the substitution bridge — never by resonating on its own.
38
+ // Measured cost of the scan over those 570 candidates: 2.9 ms warm, 20.4 ms
39
+ // cold, against a ~700 ms refusal path. Issuing a FRESH exhaustive call would
40
+ // cost 490 ms median against 13 ms non-exhaustive (36×), which is why this tier
41
+ // takes the candidate list as an argument and adds nothing to it.
42
+ //
43
+ // THREE GUARDS, each falsified into existence by measurement — do not drop any:
44
+ //
45
+ // 1. AN UNREADABLE CONTINUATION VETOES. Reads are bounded (a stored span can
46
+ // run to hundreds of kilobytes), so a candidate that opens with the query
47
+ // but SATURATES the read continues in a way nobody can see. It is a
48
+ // standing disagreement: if any such candidate exists, nothing is grounded.
49
+ // It must NOT be quietly skipped, and that is not a stylistic point — the
50
+ // skip is what MANUFACTURES a fragment. Measured on a one-deposit fixture
51
+ // whose form exceeds the cap: the query matched BOTH the whole 138-byte
52
+ // form (saturating) AND an interior fold node of 34 bytes (unsaturated,
53
+ // continuing `" Paris, an"`). Skipping the saturated candidate removed the
54
+ // only evidence that disagreed, uniqueness then passed on the interior
55
+ // node, and a mid-form slice was voiced as an answer. Suppressing the
56
+ // disagreement is what created the fabrication.
57
+ // (Testing instead whether a candidate is a "complete form" via the fold
58
+ // does NOT work and was measured: content addressing makes an interior
59
+ // node resolve to ITSELF, so self-resolution says nothing about
60
+ // completeness.)
61
+ // 2. THE CONTINUATION MUST REACH ONE GROUPING WINDOW. A trained
62
+ // `What is the capital of France??` opens with `What is the capital of
63
+ // France?` and continues by a single byte. Below W the continuation is
64
+ // sub-quantum — the fold groups nothing from it — and voicing it produces
65
+ // the degenerate reply that is a known failure smell.
66
+ // 3. UNIQUENESS. Several trained forms may open with the query and continue
67
+ // differently, and then the corpus does not say which continuation the
68
+ // asker means. Distinct continuations ⇒ refuse. This is the documented
69
+ // PREFIX TRAP, and it is real — just not for every prefix. Measured: of
70
+ // 15 battery probes exactly ONE yields a unique continuation, and all
71
+ // three honest-silence probes yield none (including `What is the capital
72
+ // of Zamunda?`, whose top hit scores 0.83).
73
+ //
74
+ // Uniqueness is judged on the continuation BYTES, not on the candidate id: the
75
+ // same continuation reached through two trained forms is one answer, not an
76
+ // ambiguity.
77
+ import { bytesEqual } from "../bytes.js";
78
+ import { rItem } from "./trace.js";
79
+ import { canonicalWindows, leafIdPrefix } from "./canonical.js";
80
+ import { hubBound } from "./traverse.js";
81
+ /** Trained forms the query may OPEN, proposed from the write side's own
82
+ * leaf-id window index — the supply of last resort for {@link
83
+ * prefixCompletion}.
84
+ *
85
+ * WHY A SECOND SUPPLY EXISTS. The ranked list this mechanism normally reads
86
+ * is a resonance list, and resonance cannot rank a proper prefix: measured on
87
+ * the trained store, cos(prefix, form) falls from 0.9629 at a one-byte
88
+ * truncation to 0.6206 at three bytes, against a reachThreshold of 0.8750.
89
+ * Three bytes of truncation put the answer out of reach on GEOMETRY, not on a
90
+ * bug, so no k and no re-ranking recovers it.
91
+ *
92
+ * WHY THIS ROUTE WORKS WHERE THE FOLD DOES NOT. A query's own fold is
93
+ * useless here: content addressing is not phrase-position-invariant, so a
94
+ * standalone prefix folds to a DIFFERENT node than the same bytes sitting
95
+ * inside a longer deposit, and neither the prefix's own node nor its
96
+ * ancestors lead to the deposit (measured: the 22-byte prefix of the
97
+ * photosynthesis form resolves, is shared by 6 contexts, and does not have
98
+ * the form among its ancestors). Leaf ids ARE position-invariant — they are
99
+ * content-addressed on single bytes — and `indexSubSpans` already interns a
100
+ * flat branch over every canonical WINDOW of a deposit's leaf-id stream, with
101
+ * containment edges to the chunks that window spans. A query that is a
102
+ * prefix therefore shares those window nodes exactly, and reaches the deposit
103
+ * by climbing containment then parents. Nothing is added to the write side;
104
+ * this reads an index training already built.
105
+ *
106
+ * BOUNDED (§2.8), AND WITH NO NEW THRESHOLD. The window whose containment is
107
+ * SMALLEST carries the most evidence, and one saturated at `hubBound` carries
108
+ * none — that is the same √N reading of "hub" the rest of the mind uses, not
109
+ * a tuned knob. The upward walk spends a budget of `hubBound` nodes and
110
+ * fans out by W, so a hub query enumerates nothing and the caller stays
111
+ * silent rather than guessing (§2.13). Measured on the trained store: the
112
+ * photosynthesis form at a one-byte truncation picks a window with 52
113
+ * containers, visits 446 nodes, and yields exactly ONE candidate that
114
+ * survives the caller's byte compare — the form itself.
115
+ *
116
+ * These are PROPOSALS only. Every candidate still faces the byte-exact
117
+ * prefix compare and all three guards below, so a wrong proposal costs one
118
+ * bounded read and can never be voiced (§2.3). */
119
+ export function prefixCandidates(ctx, query) {
120
+ const store = ctx.store;
121
+ const W = ctx.space.maxGroup;
122
+ const run = leafIdPrefix(ctx, query);
123
+ // The widest canonical window is the most discriminative one the write side
124
+ // ever interned; a query too short to spell one carries no window evidence.
125
+ const len = canonicalWindows(W)[1];
126
+ if (run.length < len)
127
+ return [];
128
+ const bound = hubBound(ctx);
129
+ let best = null;
130
+ let bestN = 0;
131
+ for (let off = 0; off + len <= run.length; off++) {
132
+ const wid = store.findBranch(run.slice(off, off + len));
133
+ if (wid === null)
134
+ continue;
135
+ const n = store.containersSlice(wid, 0, bound).length;
136
+ // Empty says the window spans no chunk; saturated says it is a hub, whose
137
+ // containment discriminates nothing. Neither is evidence.
138
+ if (n === 0 || n >= bound)
139
+ continue;
140
+ if (best === null || n < bestN) {
141
+ best = wid;
142
+ bestN = n;
143
+ }
144
+ }
145
+ if (best === null)
146
+ return [];
147
+ let frontier = store.containersSlice(best, 0, bound);
148
+ const seen = new Set(frontier);
149
+ let budget = bound;
150
+ while (frontier.length > 0 && budget > 0) {
151
+ const next = [];
152
+ for (const f of frontier) {
153
+ if (budget-- <= 0)
154
+ break;
155
+ for (const p of store.parentsFirst(f, W)) {
156
+ if (seen.has(p))
157
+ continue;
158
+ seen.add(p);
159
+ next.push(p);
160
+ }
161
+ }
162
+ frontier = next;
163
+ }
164
+ return [...seen];
165
+ }
166
+ /** The sole trained form the query opens — or null when no candidate opens with
167
+ * it, when the continuation is sub-quantum, when a candidate's continuation
168
+ * cannot be read through, or when the candidates disagree.
169
+ *
170
+ * `ranked` must be a list the caller has ALREADY fetched; this mechanism never
171
+ * resonates on its own (see the header's cost note). */
172
+ export function prefixCompletion(ctx, query, ranked) {
173
+ const W = ctx.space.maxGroup;
174
+ const t = ctx.trace?.enter("prefixCompletion", [rItem(query, "query")]);
175
+ const done = (hit, note, data) => {
176
+ t?.done(hit === null ? [] : [rItem(hit.continuation, "continuation", hit.id)], note, data);
177
+ return hit;
178
+ };
179
+ // Reads are bounded to phrase scale, the same bound the frame filler uses.
180
+ // A query with no room for a whole grouping window past its own length
181
+ // cannot clear guard 2, so it is not worth a single read.
182
+ const cap = query.length * W;
183
+ if (query.length === 0 || cap < query.length + W) {
184
+ return done(null, "no room for a perceivable continuation within the cap");
185
+ }
186
+ // Distinct continuations, each with the first form that offered it. Held as
187
+ // a list, not a byte-keyed map: candidates that open with the query are few
188
+ // (measured: 1 on the trained store's winning query), and a linear byte
189
+ // compare needs no string encoding of content. Uniqueness (guard 3) is
190
+ // decided over this list, so the scan cannot stop early — a second
191
+ // continuation IS the refusal, and finding it is the point.
192
+ const found = [];
193
+ let opened = 0;
194
+ let unreadable = 0;
195
+ let subQuantum = 0;
196
+ for (const id of ranked) {
197
+ const form = ctx.store.bytesPrefix(id, cap);
198
+ if (form.length <= query.length)
199
+ continue;
200
+ let opens = true;
201
+ for (let i = 0; i < query.length; i++) {
202
+ if (form[i] !== query[i]) {
203
+ opens = false;
204
+ break;
205
+ }
206
+ }
207
+ if (!opens)
208
+ continue;
209
+ opened++;
210
+ // Guard 1: a saturated read continues out of sight — a disagreement that
211
+ // cannot be resolved, so it ends the search rather than being skipped.
212
+ if (form.length >= cap) {
213
+ unreadable++;
214
+ continue;
215
+ }
216
+ const rest = form.subarray(query.length);
217
+ // Guard 2: below one grouping window there is no structure to voice.
218
+ if (rest.length < W) {
219
+ subQuantum++;
220
+ continue;
221
+ }
222
+ if (!found.some((f) => bytesEqual(f.continuation, rest))) {
223
+ found.push({ id, form, continuation: rest });
224
+ }
225
+ }
226
+ const data = {
227
+ candidates: ranked.length,
228
+ opened,
229
+ unreadable,
230
+ subQuantum,
231
+ distinctContinuations: found.length,
232
+ };
233
+ if (unreadable > 0 && found.length > 0) {
234
+ return done(null, "a form opens with this query but continues past the read bound — " +
235
+ "its continuation cannot be read, so none is licensed", data);
236
+ }
237
+ // Guard 2b: A SUB-QUANTUM CONTINUATION IS STILL A DISAGREEMENT. Guard 2
238
+ // refuses to VOICE a below-window continuation, and rightly — there is no
239
+ // structure there to speak. But dropping such a candidate from the
240
+ // uniqueness tally silently converts "the corpus offers many continuations,
241
+ // most of them unvoiceable" into "the corpus offers exactly one", and
242
+ // guard 3 then passes VACUOUSLY on the sole survivor. That is precisely
243
+ // the failure guard 1 documents for unreadable continuations — suppressing
244
+ // the disagreement is what manufactures the answer — so it is answered the
245
+ // same way, and for the same reason.
246
+ //
247
+ // Measured on a 4,300-fact fixture of "what is the value of <i>?": the
248
+ // query "what is the value of" drew candidates continuing " 0?", " 4?",
249
+ // " 8?" (3 bytes, sub-quantum at W=4) and " 10?" (4 bytes). The first
250
+ // three were dropped, leaving one survivor, and the mechanism reported
251
+ // "exactly one trained form" and voiced "the value of 10 is 20" — an
252
+ // arbitrary pick from thousands of equally-good readings, with the
253
+ // evidence of ambiguity discarded on the way.
254
+ //
255
+ // Note this can only ever cause SILENCE, never a different answer: it
256
+ // withholds a completion the corpus does not uniquely license.
257
+ if (subQuantum > 0 && found.length > 0) {
258
+ return done(null, "other trained forms open with this query but continue below one " +
259
+ "grouping window — the corpus offers competing readings, so no " +
260
+ "single completion is licensed", data);
261
+ }
262
+ // Guard 3: the corpus must agree on ONE continuation.
263
+ if (found.length !== 1) {
264
+ return done(null, found.length === 0
265
+ ? "no trained form opens with this query and continues perceivably"
266
+ : "trained forms open with this query but continue differently — " +
267
+ "the corpus does not say which continuation is meant", data);
268
+ }
269
+ return done(found[0], "one trained form opens with this query, and continues perceivably", data);
270
+ }
@@ -6,6 +6,17 @@ import type { Input, MindContext } from "./types.js";
6
6
  * (windows, regions, candidate spans), so key construction is far cheaper
7
7
  * than the river fold it deduplicates. */
8
8
  export declare function latin1Key(bytes: Uint8Array): string;
9
+ /** The {@link perceive} memo key: the span's content PLUS the boundary set it
10
+ * was folded under. The tree is a function of BOTH — the same bytes fold
11
+ * plainly with no boundaries and into a left-nested stable-prefix shape with
12
+ * them — so a content-only key returns whichever shape was computed first.
13
+ * That is exactly what happened: a conversation seeded its cumulative context
14
+ * under the content key, and every later plain `perceive` of those bytes was
15
+ * served the boundary tree instead (measured: respondTurn answered where
16
+ * respond() on byte-identical input did not). NUL separates the two parts —
17
+ * the boundary rendering is digits and commas, so no content byte can forge
18
+ * the split. */
19
+ export declare function perceiveKey(bytes: Uint8Array, boundaries?: readonly number[]): string;
9
20
  /** Perceive input into a content-defined tree (the river fold).
10
21
  * Deterministic — identical bytes always produce an identical tree.
11
22
  *
@@ -16,16 +27,24 @@ export declare function latin1Key(bytes: Uint8Array): string;
16
27
  * boundaries are; the geometry never guesses them from the bytes. */
17
28
  export declare function perceive(ctx: MindContext, input: Input, leafAt?: (i: number) => number | null, lookup?: (ids: number[]) => number | null, boundaries?: readonly number[]): Sema;
18
29
  /** The DEPOSIT-shaped perceive. Folds over the stream's own content cuts —
19
- * bit-identical to what inference computes for the same bytes, and that
20
- * train/inference agreement is load-bearing for exact recall. An input that
21
- * EXTENDS a previously deposited one is a conversation context grown by one
22
- * turn; the cached prefix length IS the turn boundary (derived from the deposit
23
- * sequence itself, never from a content convention) and joins the cut set, so
24
- * the trained context node and the query's context subtree are the SAME node.
25
- * Segment folds reuse across deposits ({@link stablePrefixFoldIncremental}) —
26
- * O(turn) instead of O(context) per turn. All of it is purely a cache: an
27
- * evicted chain loses only the turn boundaries, and since the content cuts do
28
- * not depend on the cache, the segments themselves are unchanged. */
30
+ * bit-identical to what inference computes for the same bytes. That
31
+ * train/inference agreement is the whole contract: the trained context node
32
+ * and the node `resolve(query)` reaches must be the SAME node, and the only
33
+ * way to guarantee it is to give this function nothing extra to say. It
34
+ * imposes no boundaries, knows nothing about turns, and reads no convention
35
+ * out of the bytes.
36
+ *
37
+ * An input that EXTENDS a previously deposited one a conversation context
38
+ * grown by a turn, or a resumed replay reuses that deposit's already-folded
39
+ * content segments ({@link contentFoldIncremental}), so it costs O(new bytes)
40
+ * instead of O(context). The reuse is TRANSPARENT by construction: a segment
41
+ * is a pure function of its own bytes, so a reused one is bit-identical to a
42
+ * refolded one. Nothing has to prove that the extending deposit is "really"
43
+ * a next turn — a coincidental byte prefix reuses the same segments and gets
44
+ * the same tree it would have got anyway. (It used to matter: while this
45
+ * path imposed turn BOUNDARIES, a wrong guess changed the tree, so the cache
46
+ * needed a continuation-bytes proof to gate it. Nothing is imposed now, so
47
+ * there is nothing to gate.) */
29
48
  export declare function perceiveDeposit(ctx: MindContext, bytes: Uint8Array, conversational?: boolean): Sema;
30
49
  /** The raw bytes of an input — modality-neutral conversion. */
31
50
  export declare function inputBytes(ctx: MindContext, input: Input): Uint8Array;
@@ -2,7 +2,7 @@
2
2
  //
3
3
  // Address — bytes → node (perceive, foldTree, resolve)
4
4
  // Read — node → bytes (read)
5
- import { bytesToTree, gridToTree, hilbertBytes, stablePrefixFoldIncremental, stackGrids, } from "../geometry.js";
5
+ import { bytesToTree, contentFoldIncremental, gridToTree, hilbertBytes, stackGrids, } from "../geometry.js";
6
6
  import { canonHash } from "../canon.js";
7
7
  import { bytesEqual } from "../bytes.js";
8
8
  import { ALL } from "./types.js";
@@ -21,6 +21,22 @@ export function latin1Key(bytes) {
21
21
  }
22
22
  return s;
23
23
  }
24
+ /** The {@link perceive} memo key: the span's content PLUS the boundary set it
25
+ * was folded under. The tree is a function of BOTH — the same bytes fold
26
+ * plainly with no boundaries and into a left-nested stable-prefix shape with
27
+ * them — so a content-only key returns whichever shape was computed first.
28
+ * That is exactly what happened: a conversation seeded its cumulative context
29
+ * under the content key, and every later plain `perceive` of those bytes was
30
+ * served the boundary tree instead (measured: respondTurn answered where
31
+ * respond() on byte-identical input did not). NUL separates the two parts —
32
+ * the boundary rendering is digits and commas, so no content byte can forge
33
+ * the split. */
34
+ export function perceiveKey(bytes, boundaries) {
35
+ const k = latin1Key(bytes);
36
+ return boundaries === undefined || boundaries.length === 0
37
+ ? k
38
+ : k + "\u0000" + boundaries.join(",");
39
+ }
24
40
  /** Perceive input into a content-defined tree (the river fold).
25
41
  * Deterministic — identical bytes always produce an identical tree.
26
42
  *
@@ -41,7 +57,7 @@ export function perceive(ctx, input, leafAt, lookup, boundaries) {
41
57
  // The tree is shared by reference; Sema nodes are never mutated.
42
58
  const memo = ctx.perceiveMemo;
43
59
  if (memo) {
44
- const key = latin1Key(bytes);
60
+ const key = perceiveKey(bytes, boundaries);
45
61
  const hit = memo.get(key);
46
62
  if (hit !== undefined) {
47
63
  if (ctx.meter)
@@ -70,67 +86,42 @@ export function perceive(ctx, input, leafAt, lookup, boundaries) {
70
86
  return gridToTree(ctx.space, ctx.alphabet, input);
71
87
  }
72
88
  /** The DEPOSIT-shaped perceive. Folds over the stream's own content cuts —
73
- * bit-identical to what inference computes for the same bytes, and that
74
- * train/inference agreement is load-bearing for exact recall. An input that
75
- * EXTENDS a previously deposited one is a conversation context grown by one
76
- * turn; the cached prefix length IS the turn boundary (derived from the deposit
77
- * sequence itself, never from a content convention) and joins the cut set, so
78
- * the trained context node and the query's context subtree are the SAME node.
79
- * Segment folds reuse across deposits ({@link stablePrefixFoldIncremental}) —
80
- * O(turn) instead of O(context) per turn. All of it is purely a cache: an
81
- * evicted chain loses only the turn boundaries, and since the content cuts do
82
- * not depend on the cache, the segments themselves are unchanged. */
89
+ * bit-identical to what inference computes for the same bytes. That
90
+ * train/inference agreement is the whole contract: the trained context node
91
+ * and the node `resolve(query)` reaches must be the SAME node, and the only
92
+ * way to guarantee it is to give this function nothing extra to say. It
93
+ * imposes no boundaries, knows nothing about turns, and reads no convention
94
+ * out of the bytes.
95
+ *
96
+ * An input that EXTENDS a previously deposited one a conversation context
97
+ * grown by a turn, or a resumed replay reuses that deposit's already-folded
98
+ * content segments ({@link contentFoldIncremental}), so it costs O(new bytes)
99
+ * instead of O(context). The reuse is TRANSPARENT by construction: a segment
100
+ * is a pure function of its own bytes, so a reused one is bit-identical to a
101
+ * refolded one. Nothing has to prove that the extending deposit is "really"
102
+ * a next turn — a coincidental byte prefix reuses the same segments and gets
103
+ * the same tree it would have got anyway. (It used to matter: while this
104
+ * path imposed turn BOUNDARIES, a wrong guess changed the tree, so the cache
105
+ * needed a continuation-bytes proof to gate it. Nothing is imposed now, so
106
+ * there is nothing to gate.) */
83
107
  export function perceiveDeposit(ctx, bytes, conversational = false) {
108
+ // Longest cached PROPER prefix first — the most segments to reuse.
84
109
  let prev;
85
- let prefixLen = 0;
86
- // Cache consult (both boundary lookup and stable-prefix reuse) is scoped
87
- // to conversational deposits only a bare, unrelated fact whose bytes
88
- // happen to extend an earlier deposit is NOT a conversation turn, and
89
- // must keep the plain fold so it shares structure with ITS OWN prior
90
- // deposits, not fragment against a coincidental byte-prefix.
91
- if (conversational) {
92
- // Longest cached PROPER prefix first.
93
- const lens = [...ctx._depositLens]
94
- .filter((L) => L >= 2 && L < bytes.length)
95
- .sort((a, b) => b - a);
96
- for (const L of lens) {
97
- const hit = ctx._depositTrees.get(latin1Key(bytes.subarray(0, L)));
98
- // The suffix must bytes-equal the hit's OWN recorded continuation —
99
- // proof this deposit is that turn's actual next turn, not a fact
100
- // that coincidentally shares its byte prefix.
101
- if (hit !== undefined && hit.nextBytes !== undefined &&
102
- bytesEqual(hit.nextBytes, bytes.subarray(L))) {
103
- prev = hit;
104
- prefixLen = L;
105
- break;
106
- }
110
+ const lens = [...ctx._depositLens]
111
+ .filter((L) => L >= 2 && L < bytes.length)
112
+ .sort((a, b) => b - a);
113
+ for (const L of lens) {
114
+ const hit = ctx._depositTrees.get(latin1Key(bytes.subarray(0, L)));
115
+ if (hit !== undefined) {
116
+ prev = hit.content;
117
+ break;
107
118
  }
108
119
  }
109
- // ONLY turn boundaries belong here. The stream's own content cuts are NOT
110
- // passed in: `bytesToTree` and `stablePrefixFoldIncremental` both derive them
111
- // per span, at every level, and handing the level-0 cuts in as stable-prefix
112
- // boundaries instead produces a LEFT-NESTED join of flat segments a
113
- // different tree from the one inference builds for the same bytes. When that
114
- // happened, a deposit's context root and `resolve(question)` were different
115
- // nodes, so the trained edge hung off a node inference never reached and
116
- // recall went silent (test/44 caught it as a site that could not be emitted
117
- // because the resolved node led nowhere). Train and infer must fold
118
- // identically; the way to guarantee that is to give this function nothing
119
- // extra to say.
120
- const cuts = new Set();
121
- if (prev !== undefined) {
122
- for (const b of prev.boundaries)
123
- cuts.add(b);
124
- cuts.add(prefixLen);
125
- }
126
- const boundaries = [...cuts].sort((a, b) => a - b);
127
- const folded = stablePrefixFoldIncremental(ctx.space, ctx.alphabet, bytes, boundaries, prev?.stable);
128
- const tree = folded.tree;
129
- const entry = { boundaries, stable: folded.fold };
130
- // Only a conversational deposit writes the cache too — otherwise a bare
131
- // fact's plain fold could later be misread as a conversation's turn-zero
132
- // boundary by an unrelated conversational deposit that happens to extend
133
- // its bytes.
120
+ const folded = contentFoldIncremental(ctx.space, ctx.alphabet, bytes, prev);
121
+ // Only a CONVERSATIONAL deposit writes the cache: reuse is sound for any
122
+ // deposit, but the budget is 8 entries and a corpus of unrelated facts would
123
+ // evict the live chains for nothing. Purely a cost decision now, not a
124
+ // correctness one.
134
125
  if (conversational && bytes.length >= 2) {
135
126
  // The lengths set drifts as the map evicts; past the probe budget the
136
127
  // drift itself becomes the cost (each stale length is an O(len) key
@@ -139,10 +130,10 @@ export function perceiveDeposit(ctx, bytes, conversational = false) {
139
130
  ctx._depositLens.clear();
140
131
  ctx._depositTrees.clear();
141
132
  }
142
- ctx._depositTrees.set(latin1Key(bytes), entry);
133
+ ctx._depositTrees.set(latin1Key(bytes), { content: folded.fold });
143
134
  ctx._depositLens.add(bytes.length);
144
135
  }
145
- return tree;
136
+ return folded.tree;
146
137
  }
147
138
  /** The raw bytes of an input — modality-neutral conversion. */
148
139
  export function inputBytes(ctx, input) {