@hviana/sema 0.4.7 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/AGENTS.md +290 -77
  2. package/HOW_IT_WORKS.md +2170 -735
  3. package/dist/example/train_base.d.ts +9 -3
  4. package/dist/example/train_base.js +21 -4
  5. package/dist/src/canon.d.ts +19 -0
  6. package/dist/src/canon.js +28 -0
  7. package/dist/src/geometry.d.ts +52 -0
  8. package/dist/src/geometry.js +87 -1
  9. package/dist/src/mind/bridge.js +27 -1
  10. package/dist/src/mind/frame-filler.d.ts +15 -0
  11. package/dist/src/mind/frame-filler.js +535 -0
  12. package/dist/src/mind/learning.js +6 -11
  13. package/dist/src/mind/mechanisms/cast.js +72 -2
  14. package/dist/src/mind/mechanisms/cover.js +6 -1
  15. package/dist/src/mind/mechanisms/extraction.js +27 -0
  16. package/dist/src/mind/mechanisms/recall.js +214 -34
  17. package/dist/src/mind/mind.d.ts +49 -1
  18. package/dist/src/mind/mind.js +137 -10
  19. package/dist/src/mind/pipeline-mechanism.d.ts +7 -0
  20. package/dist/src/mind/pipeline.js +29 -1
  21. package/dist/src/mind/prefix-completion.d.ts +59 -0
  22. package/dist/src/mind/prefix-completion.js +270 -0
  23. package/dist/src/mind/primitives.d.ts +29 -10
  24. package/dist/src/mind/primitives.js +52 -61
  25. package/dist/src/mind/recognition.js +119 -9
  26. package/dist/src/mind/traverse.d.ts +32 -0
  27. package/dist/src/mind/traverse.js +52 -0
  28. package/dist/src/mind/types.d.ts +55 -16
  29. package/dist/src/mind/types.js +68 -19
  30. package/dist/src/store.d.ts +21 -0
  31. package/dist/src/store.js +21 -0
  32. package/example/train_base.ts +21 -4
  33. package/package.json +1 -1
  34. package/src/canon.ts +28 -0
  35. package/src/geometry.ts +100 -1
  36. package/src/mind/bridge.ts +34 -0
  37. package/src/mind/frame-filler.ts +604 -0
  38. package/src/mind/learning.ts +5 -9
  39. package/src/mind/mechanisms/cast.ts +70 -2
  40. package/src/mind/mechanisms/cover.ts +6 -1
  41. package/src/mind/mechanisms/extraction.ts +27 -0
  42. package/src/mind/mechanisms/recall.ts +236 -37
  43. package/src/mind/mind.ts +154 -14
  44. package/src/mind/pipeline-mechanism.ts +7 -0
  45. package/src/mind/pipeline.ts +33 -1
  46. package/src/mind/prefix-completion.ts +314 -0
  47. package/src/mind/primitives.ts +59 -70
  48. package/src/mind/recognition.ts +117 -6
  49. package/src/mind/traverse.ts +52 -0
  50. package/src/mind/types.ts +98 -42
  51. package/src/store.ts +25 -0
  52. package/test/13-conversation.test.mjs +13 -0
  53. package/test/57-fusion-order.test.mjs +65 -0
  54. package/test/66-query-edge-whitespace.test.mjs +99 -0
  55. package/test/67-climb-anchor-breadth.test.mjs +113 -0
  56. package/test/68-extraction-unanchored.test.mjs +79 -0
  57. package/test/69-frame-filler.test.mjs +115 -0
  58. package/test/70-prefix-completion.test.mjs +170 -0
  59. package/test/71-embedded-canon-equivalence.test.mjs +121 -0
  60. package/test/72-prefix-candidate-supply.test.mjs +114 -0
  61. package/test/73-scaffolding-only-bridge-abstains.test.mjs +178 -0
  62. package/test/74-prefix-trap-not-sprung-early.test.mjs +114 -0
  63. package/test/75-multiturn-context-optimisation.test.mjs +1082 -0
@@ -0,0 +1,604 @@
1
+ // frame-filler.ts — compositional substitution grounding (recall's last tier
2
+ // before refusal, after the substitution bridge).
3
+ //
4
+ // THE GAP (analyze_training.ts section G): `What is the capital of the country
5
+ // where the Eiffel Tower is?` returns silence, while the answer sits ONE edge
6
+ // away — `resolve("What is the capital of France?")` is a trained form with a
7
+ // continuation. The query differs from it by a single contiguous span: a
8
+ // DEFINITE DESCRIPTION (`the country where the Eiffel Tower is`, 37 B) stands
9
+ // where a proper noun (`France`, 6 B) stands in the trained question.
10
+ //
11
+ // Every existing tier correctly declines. Recall tier 0b needs the constituent
12
+ // to be an edge SOURCE (`Eiffel Tower` is out=0, in=1). Every gist tier is
13
+ // blind here: cos(query, the trained form) = 0.0076, and `capital of Spain`
14
+ // scores 0.0174 — HIGHER than the correct one — so the target is not even in the
15
+ // ANN top-24. The substitution bridge finds the alignment and then refuses it on
16
+ // RAW BALANCE, `dominates(6, 37)`, which it must: a short span standing for a
17
+ // long one is exactly how `France` → `Spain si(nce)` once voiced a wrong fact.
18
+ //
19
+ // THE REFRAMING. The bridge asks whether the two spans are SIMILAR. A definite
20
+ // description and the noun it denotes are not similar, they are CO-REFERENTIAL,
21
+ // so no similarity threshold can separate this case from that fabrication. So
22
+ // this tier does not try. Instead:
23
+ //
24
+ // We invent a lookup KEY, never an answer.
25
+ //
26
+ // Build the query with the candidate filler in the description's place and
27
+ // require the STORE ITSELF to already hold that key, byte-exactly, by content
28
+ // address. The answer is then the trained continuation of a form the store
29
+ // verifiably has — the same grounding tier 0 performs — so nothing is
30
+ // synthesised. What is constructed is only a lookup key, and a key the store
31
+ // does not hold is discarded.
32
+ //
33
+ // FOUR GUARDS, each falsified into existence on the 15.7M-node store. Dropping
34
+ // any one of them reintroduces a wrong answer or outright fabrication:
35
+ //
36
+ // 1. The evidence hit must literally contain the description's RAREST unit.
37
+ // Pooling fillers from every ranked hit gives the twohop query 9 resolving
38
+ // keys, dominated by `What is the capital of India?` → "New Delhi.". And
39
+ // qualifying on any SHARED unit rather than the rarest gives
40
+ // `Can you write a short poem?` exactly one key,
41
+ // `Can you write hello world in C?` — a confident wrong answer earned on
42
+ // the scaffolding unit "write".
43
+ // 2. The frame must be NON-EMPTY: the description is a proper sub-span.
44
+ // Otherwise two of the twohop keys replace the WHOLE query (`Immanuel
45
+ // Kant`, `Africa`), which is not substitution at all.
46
+ // 3. The key must RESOLVE byte-exactly and lead somewhere.
47
+ // 4. Exactly ONE stored form may survive. `What is the capital of Zamunda?`
48
+ // produces 24 resolving keys in the weaker variants (Chile, India, Japan,
49
+ // Italy …) — fabrication, refused here by ambiguity. This is the same
50
+ // discipline tier 0b applies ("two distinct maximal arguments mean the
51
+ // query asks about neither alone").
52
+ //
53
+ // A NOTE ON WHY RESOLUTION ALONE IS NOT THE SAFETY ARGUMENT. Holding the frame
54
+ // fixed and varying only the filler makes byte-exact resolution look like a
55
+ // perfect filter — every wrong filler tried returned null. That is misleading:
56
+ // when the DESCRIPTION is searched too, 95,836 candidate keys were tried for the
57
+ // twohop query and 9 resolved. Resolution is necessary, never sufficient; the
58
+ // guards above are what make it sound.
59
+ //
60
+ // COST — nothing on any answering path (this runs only where the alternative was
61
+ // silence), and no new retrieval: the ranked hits are the ones recall already
62
+ // computed. On the refusal path, measured over 19 queries (every currently-empty
63
+ // battery probe, all three silence probes, three dialogue turns, and a 125-byte
64
+ // worst case): mean 22.7 ms, worst 452 probes / 102 ms, and 10 of the 19 need
65
+ // ZERO probes because guard 1 exits first. Two things keep it there and both are
66
+ // load-bearing, not optimisations:
67
+ //
68
+ // • fillers are MAXIMAL absent runs, not every sub-span of a hit (8,020 probes
69
+ // → 452). The twohop win survives because a sub-quantum unit (< W)
70
+ // terminates a run: in `The most well-known landmark in France is the Eiffel
71
+ // Tower.`, `in` breaks the run, so `France` IS itself a maximal run.
72
+ // • rarity is memoised per response. On the 125-byte query 3,882 rarity reads
73
+ // collapse to 10 lookups, and that query drops from 309 ms to 22 ms.
74
+ //
75
+ // The probe budget is `hubBound` (√N), the same breadth every other bounded
76
+ // search here self-limits to. Exhausting it REFUSES rather than answers:
77
+ // truncating the search would leave uniqueness (guard 4) unestablished, and an
78
+ // unestablished uniqueness claim is exactly the ambiguity the guard exists to
79
+ // catch.
80
+
81
+ // CONSTITUENCY COMES FROM AGREEMENT, NOT FROM BYTES. This mechanism
82
+ // substitutes one CONSTITUENT for another, so it must know where a constituent
83
+ // begins — and there is no character class here, no "separator", no "word",
84
+ // because Sema has none. A byte value cannot say whether it delimits: the
85
+ // alphabet is 256 learnt directions, one per byte, and asserting a class over
86
+ // it overrides what the corpus is able to state itself.
87
+ //
88
+ // The reading this uses is the store's own, already spelled out twice
89
+ // (pipeline-mechanism.ts's `framed` and cast.ts's frame gate):
90
+ //
91
+ // A byte is FRAME when more than half the aligned structures share it, and a
92
+ // SPAN is frame when more than half its bytes are.
93
+ //
94
+ // Scaffolding is what many exemplars have in common; content is what tells them
95
+ // apart. So the spans come from ALIGNMENT (match.ts's `alignRuns`, the same
96
+ // literal W-gram alignment the weave is built on) and the judgement is
97
+ // `dominates` — both modality-free by construction. In a grid the padding
98
+ // value would be shared by every exemplar and fall out as frame on exactly this
99
+ // test, with nothing rewritten.
100
+ //
101
+ // This is why asking "what are the units of this byte string?" has no answer
102
+ // here and every attempt to derive one failed (measured: the fold's own
103
+ // `segment` cuts mid-constituent, "The ca"/"pital of"; interning is
104
+ // uninformative because EVERY W-window is interned; `recognise` returns only
105
+ // whole learnt forms). All three read ONE string alone. Constituency is
106
+ // RELATIONAL — a property of what the corpus agrees on across exemplars — and
107
+ // only a comparison can expose it.
108
+
109
+ import type { MindContext } from "./types.js";
110
+ import { indexOf } from "../bytes.js";
111
+ import { leafIdRun } from "./canonical.js";
112
+ import { hubBound } from "./traverse.js";
113
+ import { alignRuns } from "./match.js";
114
+ import { dominates } from "../geometry.js";
115
+ import { foldTree, perceive } from "./primitives.js";
116
+ import { rItem } from "./trace.js";
117
+
118
+ /** A grounded frame-filler substitution: the stored form the constructed key
119
+ * resolved to, and the spans that explain how it was reached. */
120
+ export interface FrameFillerHit {
121
+ /** The trained form the key resolved to — grounded through its own edge. */
122
+ id: number;
123
+ /** `[start, end)` of the query span the filler stood in for. */
124
+ described: [number, number];
125
+ /** The filler's bytes, for the rationale trace. */
126
+ filler: Uint8Array;
127
+ }
128
+
129
+ /** A stable memo key for a byte span — latin1, so no UTF-8 validation and no
130
+ * allocation beyond the string itself. */
131
+ function spanKey(bytes: Uint8Array, from: number, to: number): string {
132
+ let s = "";
133
+ for (let i = from; i < to; i++) s += String.fromCharCode(bytes[i]);
134
+ return s;
135
+ }
136
+
137
+ /** Find the query's own most discriminative unit and the trained contexts that
138
+ * hold it, then try the store for the query with a candidate filler in the
139
+ * described span's place. Returns the sole surviving stored form, or null. */
140
+ export function frameFillerSubstitution(
141
+ ctx: MindContext,
142
+ query: Uint8Array,
143
+ ranked: ReadonlyArray<number>,
144
+ ): FrameFillerHit | null {
145
+ const W = ctx.space.maxGroup;
146
+ const _t0 = Date.now();
147
+ const t = ctx.trace?.enter("frameFiller", [rItem(query, "query")]);
148
+ const done = (
149
+ hit: FrameFillerHit | null,
150
+ note: string,
151
+ data?: unknown,
152
+ ): FrameFillerHit | null => {
153
+ t?.done(
154
+ hit === null ? [] : [rItem(hit.filler, "filler", hit.id)],
155
+ note,
156
+ data,
157
+ );
158
+ return hit;
159
+ };
160
+
161
+ // ── CONSTITUENCY BY AGREEMENT ─────────────────────────────────────────
162
+ // Read the candidate contexts once, bounded to phrase scale (a stored span
163
+ // can run to hundreds of kilobytes, and a form an order of magnitude longer
164
+ // than the question is not a candidate for BEING it with one span replaced).
165
+ const capBytes = query.length * W;
166
+ const hitMemo = new Map<number, Uint8Array | null>();
167
+ const hitBytes = (sid: number): Uint8Array | null => {
168
+ const seen = hitMemo.get(sid);
169
+ if (seen !== undefined) return seen;
170
+ const b = ctx.store.bytesPrefix(sid, capBytes + 1);
171
+ const v = b.length === 0 || b.length > capBytes ? null : b;
172
+ hitMemo.set(sid, v);
173
+ return v;
174
+ };
175
+ const contexts: Uint8Array[] = [];
176
+ for (const sid of ranked) {
177
+ const h = hitBytes(sid);
178
+ if (h !== null) contexts.push(h);
179
+ }
180
+
181
+ /** How many of `others` an alignment covers each byte of `subject` with —
182
+ * the same per-byte depth the weave accumulates, over the same literal
183
+ * alignment. */
184
+ const depthOver = (
185
+ subject: Uint8Array,
186
+ others: ReadonlyArray<Uint8Array>,
187
+ ): Uint16Array => {
188
+ const depth = new Uint16Array(subject.length);
189
+ for (const other of others) {
190
+ if (other === subject) continue;
191
+ for (const r of alignRuns(ctx, subject, other)) {
192
+ for (let i = r.qs; i < r.qe && i < depth.length; i++) depth[i]++;
193
+ }
194
+ }
195
+ return depth;
196
+ };
197
+ /** The maximal runs of `subject` that the majority does NOT share — its
198
+ * content, as opposed to the frame. This is the constituent notion: a span
199
+ * no character class produced, only agreement. */
200
+ const contentRuns = (
201
+ subject: Uint8Array,
202
+ others: ReadonlyArray<Uint8Array>,
203
+ from = 0,
204
+ to = subject.length,
205
+ ): Array<[number, number]> => {
206
+ // A single exemplar agrees with nothing, so nothing can be called frame
207
+ // and no constituent is established — the honest reading is "none".
208
+ if (others.length < 2) return [];
209
+ const depth = depthOver(subject, others);
210
+ const out: Array<[number, number]> = [];
211
+ let start = -1;
212
+ for (let i = from; i < to; i++) {
213
+ const frame = dominates(depth[i], others.length);
214
+ if (!frame) {
215
+ if (start < 0) start = i;
216
+ } else if (start >= 0) {
217
+ out.push([start, i]);
218
+ start = -1;
219
+ }
220
+ }
221
+ if (start >= 0) out.push([start, to]);
222
+ return out;
223
+ };
224
+
225
+ // THE FRAME COHORT. A frame is only established among exemplars that are
226
+ // instances of the SAME frame — "more than half the aligned structures share
227
+ // it" says nothing when the structures share nothing to begin with. Two
228
+ // readings were measured and BOTH fail:
229
+ //
230
+ // • ALL resonance candidates. They are merely near in gist, so a majority
231
+ // never forms, every byte reads as content, any span becomes a filler —
232
+ // and it does not merely fail to answer, it FABRICATES ("Tell me the name
233
+ // of the biggest planet orbiting our sun." grounded a list of animals).
234
+ // • candidates holding the query's discriminative content. These are
235
+ // exemplars about the same THING, not instances of the same FRAME: the
236
+ // seven holding "Eiffel" share `" the Eiffel Tower"` and nothing else, so
237
+ // a whole clause reads as content and no constituent is isolated.
238
+ //
239
+ // The cohort of a subject is its STRUCTURAL NEIGHBOURS — the candidates whose
240
+ // alignment covers the most of it — because instances of one frame are
241
+ // exactly the forms that share that frame's bytes. Two weaker readings were
242
+ // measured and both fail: ALL candidates share nothing, so a majority never
243
+ // forms, every byte reads as content and the tier FABRICATES; candidates
244
+ // holding the query's rarest content are exemplars about the same THING, not
245
+ // the same FRAME (the seven holding "Eiffel" share `" the Eiffel Tower"` and
246
+ // nothing else), so no constituent is isolated.
247
+ //
248
+ // THE CUT is half-dominance against the BEST neighbour, not against the
249
+ // subject's length. Coverage is bounded by how much frame two forms can
250
+ // share at all — measured, the best neighbour covered 23 of 59 bytes — so
251
+ // `dominates(n, subject.length)` can never fire and the cohort is always
252
+ // empty. Read against the best coverage the profile actually offers, the
253
+ // same convention becomes scale-free and needs no constant: a form sharing
254
+ // more than half of what the closest instance shares is another instance.
255
+ const coverageOf = (
256
+ subject: Uint8Array,
257
+ other: Uint8Array,
258
+ keepEdges: Set<number> | null = null,
259
+ ): number => {
260
+ const seen = new Uint8Array(subject.length);
261
+ for (const r of alignRuns(ctx, subject, other)) {
262
+ if (keepEdges !== null) {
263
+ keepEdges.add(r.qs);
264
+ keepEdges.add(r.qe);
265
+ }
266
+ for (let i = r.qs; i < r.qe && i < seen.length; i++) seen[i] = 1;
267
+ }
268
+ let n = 0;
269
+ for (const x of seen) n += x;
270
+ return n;
271
+ };
272
+ // WHERE ANY EXEMPLAR'S SHARED MATERIAL STARTS OR STOPS. Collected over EVERY
273
+ // candidate, not just the cohort: the cut decides whose AGREEMENT establishes
274
+ // the frame, which is a different question from where a boundary EXISTS. One
275
+ // exemplar ending a run at an offset is already evidence of an edge there,
276
+ // however unrelated it is otherwise — measured, drawing edges from the cohort
277
+ // alone loses the boundary between `"is"` and `"?"` (2 candidates of 563
278
+ // attest it, 0 of 131 in the cohort), and without it the two-hop description
279
+ // cannot be expressed at all.
280
+ const edgesMemo = new Map<Uint8Array, Set<number>>();
281
+ const cohortMemo = new Map<Uint8Array, Uint8Array[]>();
282
+ const cohortOf = (subject: Uint8Array): Uint8Array[] => {
283
+ const seen = cohortMemo.get(subject);
284
+ if (seen !== undefined) return seen;
285
+ const eset = new Set<number>([0, subject.length]);
286
+ edgesMemo.set(subject, eset);
287
+ let best = 0;
288
+ const scored: Array<{ c: Uint8Array; n: number }> = [];
289
+ for (const c of contexts) {
290
+ if (c === subject) continue;
291
+ const n = coverageOf(subject, c, eset);
292
+ if (n < W) continue;
293
+ if (n > best) best = n;
294
+ scored.push({ c, n });
295
+ }
296
+ const out = scored.filter((x) => dominates(x.n, best)).map((x) => x.c);
297
+ cohortMemo.set(subject, out);
298
+ return out;
299
+ };
300
+ /** Split each run at the boundaries other exemplars attest for `subject` — a
301
+ * run is a stretch the cohort does not share, not itself a constituent, so
302
+ * taking it whole asks the evidence for a span no exemplar holds. */
303
+ const cutAtEdges = (
304
+ subject: Uint8Array,
305
+ runs: Array<[number, number]>,
306
+ ): Array<[number, number]> => {
307
+ const eset = edgesMemo.get(subject);
308
+ if (eset === undefined) return runs;
309
+ const inner = [...eset].sort((a, b) => a - b);
310
+ const out: Array<[number, number]> = [];
311
+ for (const [rs, re] of runs) {
312
+ let prev = rs;
313
+ for (const o of inner) {
314
+ if (o <= rs || o >= re) continue;
315
+ if (o - prev >= W) out.push([prev, o]);
316
+ prev = o;
317
+ }
318
+ if (re - prev >= W) out.push([prev, re]);
319
+ out.push([rs, re]);
320
+ }
321
+ return out;
322
+ };
323
+
324
+ const queryContent = contentRuns(query, cohortOf(query));
325
+
326
+ // Corpus rarity of a unit, by how many trained forms contain its first
327
+ // window — the same container-count reading the bridge's anchor picking uses.
328
+ // Memoised per call: the same units recur across every candidate description.
329
+ const rarityMemo = new Map<string, number>();
330
+ let rarityReads = 0;
331
+ const rarityOf = (from: number, to: number): number => {
332
+ if (to - from < W) return Infinity;
333
+ const key = spanKey(query, from, to);
334
+ const seen = rarityMemo.get(key);
335
+ if (seen !== undefined) return seen;
336
+ rarityReads++;
337
+ let value = Infinity;
338
+ const ids = leafIdRun(ctx, query, from, from + W);
339
+ if (ids !== null) {
340
+ const wid = ctx.store.findBranch(ids);
341
+ if (wid !== null) value = ctx.store.containers(wid).length;
342
+ }
343
+ rarityMemo.set(key, value);
344
+ return value;
345
+ };
346
+ /** The most discriminative unit of `[from, to)`, as its span, or null when it
347
+ * has none. Rarest first; at EQUAL rarity the LONGER unit wins, because the
348
+ * longer form carries more content at the same corpus frequency. The
349
+ * tie-break is load-bearing on a small corpus, where every unit's container
350
+ * count collapses to 1 and first-wins would pick the query's opening unit
351
+ * ("What") over its subject ("Eiffel") — the subject is what a description
352
+ * must be about. On a large corpus the counts separate on their own (54 vs
353
+ * 1,586 for exactly that pair) and the tie-break never engages. */
354
+ const rarestUnit = (
355
+ from: number,
356
+ to: number,
357
+ ): [number, number] | null => {
358
+ let best: [number, number] | null = null;
359
+ let bestRarity = Infinity;
360
+ let bestAttested = 0;
361
+ let bestLen = 0;
362
+ // A content run is a stretch the cohort does not share; it is not itself a
363
+ // constituent, and taking it whole asks the evidence for a span no exemplar
364
+ // holds (measured: `" Eiffel Tower is?"` qualified NO candidate, so the tier
365
+ // never probed). Cut each run at the boundaries other exemplars attest —
366
+ // the same edge set the descriptions are enumerated over — so the unit is a
367
+ // span the corpus has actually seen begin and end.
368
+ //
369
+ // A unit may span ANY two attested edges, not just adjacent ones. Slicing
370
+ // only between consecutive edges makes the unit set depend on how DENSE the
371
+ // evidence is rather than on what it says: measured, at k = 24 the query
372
+ // carries 17 edges and `"Eiffel Tower"` is one slice (elected, attested),
373
+ // while at k = 571 it carries 55 and every consecutive slice is 1-3 bytes —
374
+ // all under the constituent bar — leaving only the whole unattested run and
375
+ // electing nothing at all. More evidence made the tier blinder. Taking
376
+ // every edge-bounded sub-span removes that dependence: the corpus decides
377
+ // which spans exist, never how finely it happened to mark them.
378
+ const pieces: Array<[number, number]> = [];
379
+ for (const [cs0, ce0] of queryContent) {
380
+ const inner = bs.filter((o) => o > cs0 && o < ce0);
381
+ let prev = cs0;
382
+ for (const o of [...inner, ce0]) {
383
+ pieces.push([prev, o]);
384
+ prev = o;
385
+ }
386
+ pieces.push([cs0, ce0]);
387
+ }
388
+ for (const [cs0, ce0] of pieces) {
389
+ const s = Math.max(cs0, from);
390
+ const e = Math.min(ce0, to);
391
+ // ATTESTED, or it is not evidence. Rarity is read over the span's FIRST
392
+ // window, so every longer span sharing that window scores identically —
393
+ // and the longer-wins tie-break below then elects the longest of them,
394
+ // which is a claim about bytes the measurement never looked at. Measured
395
+ // live: the elected unit was `"Eiffel Tower is?"`, held by 0 of the 24
396
+ // candidates, so guard 1 rejected every one and the tier never probed
397
+ // (`descs=14, qualified=0, probes=0`). Requiring the unit to occur in the
398
+ // evidence is not an extra gate — guard 1 demands exactly this — it just
399
+ // has to hold at ELECTION time, or election spends itself on a span no
400
+ // candidate can qualify.
401
+ let attested = 0;
402
+ for (const c of contexts) {
403
+ if (indexOf(c, query.subarray(s, e), 0) >= 0) attested++;
404
+ }
405
+ if (attested === 0) continue;
406
+ // THE CONSTITUENT BAR — two quanta, the same reading argument binding
407
+ // holds its constituents to. Content runs are not units of a modality's
408
+ // making, so a run may be as short as one window, and a single window is
409
+ // too weak to say what a description is ABOUT: measured, the W-byte
410
+ // fragment `phot` qualified `What is photosynthesis?` against an
411
+ // unrelated "photo-sharing app" and fabricated an answer.
412
+ if (e - s < 2 * W) continue;
413
+ const r = rarityOf(s, e);
414
+ if (r === Infinity) continue;
415
+ // Rarity is read over the FIRST window, so spans sharing it are
416
+ // indistinguishable to the measurement; among them the only measured
417
+ // difference is how much evidence actually holds the span. Preferring the
418
+ // LONGER one asserts bytes never looked at — measured, it elected
419
+ // `"Eiffel Tower "` (trailing space, 1 context) over `"Eiffel Tower"`
420
+ // (2 contexts), so the one candidate that could complete the description
421
+ // never qualified. Length breaks only a genuine tie in both.
422
+ if (
423
+ r < bestRarity ||
424
+ (r === bestRarity &&
425
+ (attested > bestAttested ||
426
+ (attested === bestAttested && e - s > bestLen)))
427
+ ) {
428
+ bestRarity = r;
429
+ bestAttested = attested;
430
+ bestLen = e - s;
431
+ best = [s, e];
432
+ }
433
+ }
434
+ return best;
435
+ };
436
+
437
+ const qEdges = edgesMemo.get(query) ?? new Set<number>([0, query.length]);
438
+ for (const [cs0, ce0] of queryContent) {
439
+ qEdges.add(cs0);
440
+ qEdges.add(ce0);
441
+ }
442
+ const bs = [...qEdges].sort((a, b) => a - b);
443
+
444
+ // The query's own discriminative content — every candidate description must
445
+ // contain it, or the substitution is not about what the query is asking.
446
+ const queryRare = rarestUnit(0, query.length);
447
+ if (queryRare === null) {
448
+ return done(
449
+ null,
450
+ "no corpus-attested unit in the query — nothing to describe",
451
+ {
452
+ contexts: contexts.length,
453
+ qc: queryContent.length,
454
+ edges: bs.length,
455
+ runs: queryContent.map(([a, b]) => spanKey(query, a, b)),
456
+ },
457
+ );
458
+ }
459
+
460
+ let probes = 0;
461
+ let descs = 0, qualified = 0, fillers = 0;
462
+ const chosen = new Map<string, number>();
463
+ const fillerSeen = new Set<string>();
464
+ const budget = hubBound(ctx);
465
+ // resolved form id -> the description span and filler that reached it
466
+ const found = new Map<number, FrameFillerHit>();
467
+ // Candidate description edges: every offset some exemplar's shared material
468
+ // begins or ends at, plus the query's own extremes.
469
+
470
+ for (let i = 0; i < bs.length; i++) {
471
+ for (let j = i + 1; j < bs.length; j++) {
472
+ const [dStart, dEnd] = [bs[i], bs[j]];
473
+ // GUARD 2 — the frame must be non-empty: replacing the whole query is
474
+ // not substitution.
475
+ if (dStart === 0 && dEnd === query.length) continue;
476
+ if (dEnd - dStart < W) continue;
477
+ // The description must carry the query's discriminative content.
478
+ if (dStart > queryRare[0] || dEnd < queryRare[1]) continue;
479
+ const dRare = rarestUnit(dStart, dEnd);
480
+ if (dRare === null) continue;
481
+ descs++;
482
+ {
483
+ const u = spanKey(query, dRare[0], dRare[1]);
484
+ chosen.set(u, (chosen.get(u) ?? 0) + 1);
485
+ }
486
+ const rareUnit = query.subarray(dRare[0], dRare[1]);
487
+
488
+ for (const sid of ranked) {
489
+ // PHRASE SCALE — the same bound the bridge puts on a candidate's bytes
490
+ // (`capBytes = query.length * W`): a form an order of magnitude longer
491
+ // than the question is not a candidate for BEING that question with one
492
+ // span replaced. Reading candidates in full instead was measured at
493
+ // +650 ms on the winning query, since a stored span can run to hundreds
494
+ // of kilobytes.
495
+ const hit = hitBytes(sid);
496
+ if (hit === null) continue;
497
+ // GUARD 1 — this hit must literally hold the description's rarest unit.
498
+ if (indexOf(hit, rareUnit, 0) < 0) continue;
499
+ qualified++;
500
+
501
+ // Fillers: this hit's CONTENT — the spans its fellow candidates do not
502
+ // share. What the exemplars have in common is the frame they are all
503
+ // instances of; what is left distinguishes THIS one, and that is the
504
+ // constituent standing where the query's description stands.
505
+ const runs = cutAtEdges(hit, contentRuns(hit, cohortOf(hit)));
506
+
507
+ for (const [fs, fe] of runs) {
508
+ if (fe - fs < W) continue;
509
+ fillers++;
510
+ if (fillerSeen.size < 40) fillerSeen.add(spanKey(hit, fs, fe));
511
+ const filler = hit.subarray(fs, fe);
512
+ if (indexOf(query, filler, 0) >= 0) continue;
513
+ if (probes >= budget) {
514
+ // Uniqueness cannot be established on a truncated search, and an
515
+ // unestablished uniqueness claim is the ambiguity guard 4 exists
516
+ // to refuse.
517
+ return done(
518
+ null,
519
+ `probe budget (${budget}) exhausted — uniqueness unestablished`,
520
+ {
521
+ version: 1,
522
+ probes,
523
+ rarityReads,
524
+ resolved: found.size,
525
+ contexts: contexts.length,
526
+ edges: bs.length,
527
+ qc: queryContent.length,
528
+ descs,
529
+ qualified,
530
+ fillers,
531
+ },
532
+ );
533
+ }
534
+ probes++;
535
+ // The KEY: the query with this filler in the described span's place.
536
+ const key = new Uint8Array(
537
+ dStart + filler.length + (query.length - dEnd),
538
+ );
539
+ key.set(query.subarray(0, dStart), 0);
540
+ key.set(filler, dStart);
541
+ key.set(query.subarray(dEnd), dStart + filler.length);
542
+ // GUARD 3 — the store must already hold it, and it must lead
543
+ // somewhere. EXACT content address only, deliberately not
544
+ // resolve(): that falls through to canonResolve on a miss, and a
545
+ // constructed key misses by design — 451 of the 452 probes on the
546
+ // winning query do — so each miss would pay a canon index query.
547
+ // Measured: +650 ms on that query, for a claim WEAKER than the one
548
+ // this guard wants. A canon hit would mean the store holds a
549
+ // case/width variant of a key we invented; guard 3 asks for the key
550
+ // itself.
551
+ const id = foldTree(ctx, perceive(ctx, key), 0).node;
552
+ if (id === null || !ctx.store.hasNext(id)) continue;
553
+ if (!found.has(id)) {
554
+ found.set(id, {
555
+ id,
556
+ described: [dStart, dEnd],
557
+ filler: filler.slice(),
558
+ });
559
+ }
560
+ }
561
+ }
562
+ }
563
+ }
564
+
565
+ const data = {
566
+ version: 1 as const,
567
+ probes,
568
+ rarityReads,
569
+ resolved: found.size,
570
+ budget,
571
+ contexts: contexts.length,
572
+ edges: bs.length,
573
+ qc: queryContent.length,
574
+ descs,
575
+ qualified,
576
+ fillers,
577
+ fillerSeen: [...fillerSeen],
578
+ chosen: [...chosen.entries()].map(([u, n]) => {
579
+ const b = new Uint8Array(u.length);
580
+ for (let i = 0; i < u.length; i++) b[i] = u.charCodeAt(i);
581
+ const inAny = contexts.filter((c) => indexOf(c, b, 0) >= 0).length;
582
+ return `${JSON.stringify(u)} x${n} inContexts=${inAny}`;
583
+ }),
584
+ ms: Date.now() - _t0,
585
+ };
586
+ // GUARD 4 — exactly one trained form, or the query is ambiguous about its own
587
+ // subject and neither is licensed.
588
+ if (found.size === 1) {
589
+ const hit = [...found.values()][0];
590
+ return done(
591
+ hit,
592
+ "frame-filler substitution — the store holds this query with a " +
593
+ "corroborated filler in the described span's place",
594
+ data,
595
+ );
596
+ }
597
+ return done(
598
+ null,
599
+ found.size === 0
600
+ ? "no filler makes this query a stored form"
601
+ : `${found.size} fillers make this query a stored form — ambiguous subject`,
602
+ data,
603
+ );
604
+ }
@@ -245,15 +245,11 @@ export async function ingestPair(
245
245
  const cont_ = await deposit(ctx, cont, false);
246
246
  const ctxId = c.rootId, contId = cont_.rootId;
247
247
 
248
- // Stamp this turn's continuation onto its own cache entry — the proof a
249
- // FUTURE, longer ctxInput needs (see perceiveDeposit) to recognise itself
250
- // as this conversation's genuine next turn rather than an unrelated fact
251
- // that happens to share this ctxInput's byte prefix.
252
- {
253
- const ctxBytes = inputBytes(ctx, ctxInput);
254
- const entry = ctx._depositTrees.get(latin1Key(ctxBytes));
255
- if (entry !== undefined) entry.nextBytes = inputBytes(ctx, cont);
256
- }
248
+ // NO CONTINUATION STAMP. A longer ctxInput reusing this one's folded
249
+ // segments needs no proof that it is "really" the next turn: the deposit
250
+ // fold imposes no boundaries, so reuse is bit-identical to refolding and a
251
+ // coincidental byte prefix gets the tree it would have got anyway. The
252
+ // stamp existed only to gate a boundary guess that no longer happens.
257
253
 
258
254
  await ctx.store.link(ctxId, contId);
259
255
  await propagateSuffixes(ctx, ctxId, contId);