@hviana/sema 0.7.1 → 0.7.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/HOW_IT_WORKS.md CHANGED
@@ -4434,16 +4434,19 @@ before the conversion. Derived from the existing bars; never tuned.
4434
4434
  ### 21.5 The refusal path — the substitution bridge, before silence
4435
4435
 
4436
4436
  Everything geometric has now failed. One tier remains, making a **structural**
4437
- claim about the query that resonance cannot state. It reads the response's
4438
- **wide candidate list** (`Precomputed.wideResonance`, §14.5) — the ranked hits,
4439
- widened to an exhaustive index scan only when the top hit clears the concept
4440
- threshold. When the query gist has no concept-level match to anything stored, an
4441
- exhaustive scan would only score more vectors below the bar (profiled at 38–40K
4442
- vectors scored per refusing query on a 325K-context store, costing 44% of
4443
- think). Whether the gist ranks _anything_ at concept level is the discriminator
4444
- — corpus size never was. The list is shared response-wide, so whichever
4437
+ claim about the query that resonance cannot state. Its proposal source is the
4438
+ response's **one top-k read** (`Precomputed.resonance`, §14.5) — the same ranked
4439
+ list recall's earlier tiers already consulted, shared response-wide so whichever
4445
4440
  mechanism first-touches it pays once and every later reader is free.
4446
4441
 
4442
+ It used to be a **wide** list, widened to an exhaustive index scan whenever the
4443
+ top hit cleared the concept threshold. That was removed: the bridge's own
4444
+ candidate cap is `2 · recallQueryK`, so the top-k already IS everything it can
4445
+ consume, and every proposal is byte-verified downstream (§4.3) — an exhaustive
4446
+ scan bought an O(k) need at O(index) cost (profiled: 244K vectors scored per
4447
+ refusing query, ~1.5 s, every answer byte-identical to the top-k read). Test/95
4448
+ pins its absence.
4449
+
4447
4450
  #### The substitution bridge
4448
4451
 
4449
4452
  **The gap.** A query phrased through a near-synonym of a trained word ("Name the
@@ -5288,16 +5291,14 @@ recallByResonance(query, pre):
5288
5291
  return { bytes: g, accounted: nothing, moves: STEP }
5289
5292
 
5290
5293
  # ── the REFUSAL PATH ─────────────────────────────────────────────
5291
- # pre.wideResonance() — the response's ONE wide candidate list, shared
5292
- # by every mechanism that must look past the top-k:
5293
- # hits[0].score CONCEPT_BAR
5294
- # ? exhaustive resonate(gistOf(query), hubBound) # ids only
5295
- # : hits # the gist ranks nothing at concept
5296
- # # level, so a wider scan says nothing
5294
+ # pre.resonance() — the response's ONE top-k read, already paid for by
5295
+ # the tiers above. The bridge caps its own candidates at 2·recallQueryK,
5296
+ # so the top-k is exactly the budget it can consume; there is no wider
5297
+ # list, and every proposal is byte-verified below.
5297
5298
 
5298
5299
  # 3b. substitution / identity bridge
5299
- bridged ≔ substitutionBridge(query, pre.wideResonance)
5300
- # anchors: rarest query windows → edgeAncestors, plus wideIds
5300
+ bridged ≔ substitutionBridge(query, ids(pre.resonance))
5301
+ # anchors: rarest query windows → edgeAncestors, plus those ids
5301
5302
  # align byte-for-byte; a mismatch substitutes only under
5302
5303
  # CORROBORATION ∧ GRADED IDENTITY ∧ RAW BALANCE
5303
5304
  # accept when matched+substituted DOMINATES the query, every
@@ -129,6 +129,13 @@ export declare class Meter {
129
129
  /** Nodes popped by those ascents, against their √N·W budget — the counter
130
130
  * that shows whether the walks are deciding early or burning the budget. */
131
131
  junctionPops: number;
132
+ /** Ascents that ended by EXHAUSTING the expansion budget rather than by
133
+ * deciding — the walk abstained and the caller silently fell through to a
134
+ * lower tier of the ladder (§2.13: a degradation nothing else reports).
135
+ * It rises the moment a SHARED budget is drained by an earlier walk, which
136
+ * is what makes "this tier answered nothing" distinguishable from "this
137
+ * tier never got to look". */
138
+ junctionBudgetExhausted: number;
132
139
  /** Arbitrary byte spans whose distributional company was VSA-bundled from
133
140
  * existing episode halos. */
134
141
  spanHalos: number;
@@ -156,6 +163,12 @@ export declare class Meter {
156
163
  /** Charge `ms`, one call, and a counter delta to a named phase.
157
164
  * Insertion-ordered, so a report reads in execution order. */
158
165
  charge(phase: string, ms: number, delta?: Record<string, number>): void;
166
+ /** Time one SYNCHRONOUS phase. The sync/async seam (§2.10) is a real
167
+ * contract — perception, recognition and the graph search are synchronous —
168
+ * so a synchronous layer must not be wrapped in `time`'s promise just to be
169
+ * measured: that would make the profiled path await where the unprofiled
170
+ * one does not, and a meter never changes what a layer computes. */
171
+ timeSync<T>(phase: string, fn: () => T): T;
159
172
  /** Time one async phase and attribute the work done inside it. Returns
160
173
  * the awaited value untouched — a meter never changes what a layer
161
174
  * computes, only what is known about it. */
package/dist/src/meter.js CHANGED
@@ -122,6 +122,13 @@ export class Meter {
122
122
  /** Nodes popped by those ascents, against their √N·W budget — the counter
123
123
  * that shows whether the walks are deciding early or burning the budget. */
124
124
  junctionPops = 0;
125
+ /** Ascents that ended by EXHAUSTING the expansion budget rather than by
126
+ * deciding — the walk abstained and the caller silently fell through to a
127
+ * lower tier of the ladder (§2.13: a degradation nothing else reports).
128
+ * It rises the moment a SHARED budget is drained by an earlier walk, which
129
+ * is what makes "this tier answered nothing" distinguishable from "this
130
+ * tier never got to look". */
131
+ junctionBudgetExhausted = 0;
125
132
  /** Arbitrary byte spans whose distributional company was VSA-bundled from
126
133
  * existing episode halos. */
127
134
  spanHalos = 0;
@@ -173,6 +180,26 @@ export class Meter {
173
180
  }
174
181
  }
175
182
  }
183
+ /** Time one SYNCHRONOUS phase. The sync/async seam (§2.10) is a real
184
+ * contract — perception, recognition and the graph search are synchronous —
185
+ * so a synchronous layer must not be wrapped in `time`'s promise just to be
186
+ * measured: that would make the profiled path await where the unprofiled
187
+ * one does not, and a meter never changes what a layer computes. */
188
+ timeSync(phase, fn) {
189
+ const before = this.snapshot();
190
+ const t = performance.now();
191
+ try {
192
+ return fn();
193
+ }
194
+ finally {
195
+ const ms = performance.now() - t;
196
+ const after = this.snapshot();
197
+ const delta = {};
198
+ for (const k of Object.keys(after))
199
+ delta[k] = after[k] - before[k];
200
+ this.charge(phase, ms, delta);
201
+ }
202
+ }
176
203
  /** Time one async phase and attribute the work done inside it. Returns
177
204
  * the awaited value untouched — a meter never changes what a layer
178
205
  * computes, only what is known about it. */
@@ -185,7 +185,19 @@ unordered = false) {
185
185
  id,
186
186
  d: 0,
187
187
  }));
188
- while (stack.length > 0 && out.length < bound && b.n-- > 0) {
188
+ while (stack.length > 0 && out.length < bound) {
189
+ // BUDGET EXHAUSTION IS AN ABSTENTION, AND IT MUST BE VISIBLE (§2.13). The
190
+ // walk stops with work still on the stack, the caller reads "no container"
191
+ // and falls through to a lower ladder rung — indistinguishable, from the
192
+ // outside, from a walk that looked everywhere and found nothing. With a
193
+ // SHARED budget (cross-region's one k·W allowance per tier) an EARLIER
194
+ // pair can drain it, so a later pair's exact tier may never run at all;
195
+ // this counter is the only thing that says so.
196
+ if (b.n-- <= 0) {
197
+ if (ctx.meter)
198
+ ctx.meter.junctionBudgetExhausted++;
199
+ break;
200
+ }
189
201
  const { id: x, d } = stack.pop();
190
202
  if (ctx.meter)
191
203
  ctx.meter.junctionPops++;
@@ -16,7 +16,8 @@ export interface PrefixCompletion {
16
16
  * it, when the continuation is sub-quantum, when a candidate's continuation
17
17
  * cannot be read through, or when the candidates disagree.
18
18
  *
19
- * `ranked` must be a list the caller has ALREADY fetched; this mechanism never
19
+ * `ranked` must be a list the caller has ALREADY fetched (the write side's
20
+ * window index, or the response's memoised top-k); this mechanism never
20
21
  * resonates on its own (see the header's cost note). */
21
22
  export declare function prefixCompletion(ctx: MindContext, query: Uint8Array, ranked: ReadonlyArray<number>): PrefixCompletion | null;
22
23
  export declare const prefixMechanism: PipelineMechanism;
@@ -45,15 +45,23 @@
45
45
  // from `resonate(k)` at k = 24, 256 AND 2048 — while forms scoring LOWER
46
46
  // (Germany 0.5670, Yemen 0.5591) are returned. `k` only reorders WITHIN
47
47
  // the IVF clusters already probed, exactly as Store.resonate's doc warns,
48
- // so no k recovers it. With `exhaustive` it ranks 8.
48
+ // so no k recovers it.
49
49
  //
50
- // So this is a RETRIEVABILITY gap, not a semantic one, and it is repaired by
51
- // reading the candidate list recall's refusal path has ALREADY fetched
52
- // exhaustively for the substitution bridgenever by resonating on its own.
53
- // Measured cost of the scan over those 570 candidates: 2.9 ms warm, 20.4 ms
54
- // cold, against a ~700 ms refusal path. Issuing a FRESH exhaustive call would
55
- // cost 490 ms median against 13 ms non-exhaustive (36×), which is why this tier
56
- // takes the candidate list as an argument and adds nothing to it.
50
+ // So this is a RETRIEVABILITY gap, not a semantic one, and the ANN is the wrong
51
+ // instrument for it: a proper prefix's gist cannot rank its own continuation.
52
+ // The repair is CONTENT-ADDRESSED (§2.3)`formsOpenedBy` (traverse.ts) reads
53
+ // the leaf-id WINDOW index the write side already maintains and answers "which
54
+ // trained forms does this byte run open?" in a bounded √N walk. That is this
55
+ // mechanism's first supply. The response's memoised top-k `resonance()` is the
56
+ // second, for prefixes long enough that the gist still ranks the form; it is
57
+ // read, never re-issued.
58
+ //
59
+ // AN EXHAUSTIVE ANN LIST IS NOT A SUPPLY HERE, AND WAS REMOVED. This tier once
60
+ // read `Precomputed.wideResonance()` — a full-index `resonate(guide, √N,
61
+ // exhaustive)` — on the argument that the target "ranks 8 with `exhaustive`".
62
+ // It bought an O(k) need at O(index) cost (measured: 244K annVectorReads per
63
+ // refusing query, ~1.5 s) for candidates the window index proposes directly.
64
+ // See pipeline-mechanism.ts's REMOVED note; test/95 pins its absence.
57
65
  //
58
66
  // THREE GUARDS, each falsified into existence by measurement — do not drop any:
59
67
  //
@@ -97,7 +105,8 @@ import { STEP } from "../graph-search.js";
97
105
  * it, when the continuation is sub-quantum, when a candidate's continuation
98
106
  * cannot be read through, or when the candidates disagree.
99
107
  *
100
- * `ranked` must be a list the caller has ALREADY fetched; this mechanism never
108
+ * `ranked` must be a list the caller has ALREADY fetched (the write side's
109
+ * window index, or the response's memoised top-k); this mechanism never
101
110
  * resonates on its own (see the header's cost note). */
102
111
  export function prefixCompletion(ctx, query, ranked) {
103
112
  const W = ctx.space.maxGroup;
@@ -218,16 +227,25 @@ export const prefixMechanism = {
218
227
  return STEP;
219
228
  },
220
229
  async run(ctx, query, pre) {
221
- // The write side's window index proposes FIRST: a proper prefix's gist
222
- // cannot rank its own continuation (cos falls below reachThreshold at a
223
- // few bytes of truncation), so the content-addressed window walk is the
224
- // correct measure for this question (§2.3), and it is a bounded √N walk —
225
- // cheaper than an exhaustive ANN. The top-k resonance list is the SECOND
226
- // supply, for prefixes long enough that the gist still ranks the form. A
227
- // second SUPPLY, not a second mechanism the same three guards decide
228
- // either way.
229
- const completed = prefixCompletion(ctx, query, formsOpenedBy(ctx, query)) ??
230
- prefixCompletion(ctx, query, (await pre.resonance()).map((h) => h.id));
230
+ // ONE SUPPLY PASS, not a two-tier `??`. The window index (exact,
231
+ // content-addressed) and the response's memoised top-k (approximate) are
232
+ // concatenated and the three guards decide ONCE over the union. A
233
+ // first-then-fallback chain would let the APPROXIMATE tier override the
234
+ // EXACT one (§2.3): when formsOpenedBy finds two continuations, guard 3
235
+ // returns null and the fallback re-runs the guards on resonance's top-k
236
+ // alone — which, seeing only one of the two forms, would voice it. That is
237
+ // precisely the disagreement-suppression guard 3 exists to prevent, and it
238
+ // is the exact tier's ambiguity being washed away by the approximate tier.
239
+ // Evaluating the union means a disagreement the window index saw can never
240
+ // be hidden by what the ANN happens to rank. The ANN read is the
241
+ // response's ONE memoised top-k (§2.11), already paid by recall's refusal
242
+ // path on the queries where this mechanism fires, so reading it here is not
243
+ // a second index scan.
244
+ const ids = [
245
+ ...formsOpenedBy(ctx, query),
246
+ ...(await pre.resonance()).map((h) => h.id),
247
+ ];
248
+ const completed = prefixCompletion(ctx, query, ids);
231
249
  if (completed === null)
232
250
  return [];
233
251
  return [{
@@ -52,8 +52,9 @@ export interface RegimePredictionData {
52
52
  /** retrieval | composition — the two regimes R1 measured as a ~100× cost
53
53
  * step. */
54
54
  regime: "retrieval" | "composition";
55
- /** The incumbent's grade right after the first mechanism ran, or null when
56
- * it grounded nothing (best === null composition, with no incumbent). */
55
+ /** The incumbent's grade once the first mechanism's turn is over (it ran, or
56
+ * it was skipped), or null when nothing has grounded `best === null`,
57
+ * which is composition with no incumbent. */
57
58
  incumbentGrade: number | null;
58
59
  /** The cheapest composition floor in grade units (`grade(2 * STEP)` = 2,
59
60
  * CAST's floor) — the bar the incumbent must sit at or below for the
@@ -97,8 +97,11 @@ export async function think(ctx, query, mechs) {
97
97
  // own store work (perceive → foldTree → resolve), which used to land in
98
98
  // `think` and in nothing narrower — the meter's one accounting surface must
99
99
  // charge it to itself, exactly as attention/weave/resonance are charged.
100
+ // SYNCHRONOUS phase: recognition is on the sync side of §2.10's seam, so it
101
+ // is timed with `timeSync` — wrapping it in a promise would make a profiled
102
+ // response await where an unprofiled one does not.
100
103
  const rec = meter
101
- ? await meter.time("recognise", async () => recognise(ctx, query))
104
+ ? meter.timeSync("recognise", () => recognise(ctx, query))
102
105
  : recognise(ctx, query);
103
106
  // Phase 1: collect computed spans from mechanisms that implement parse()
104
107
  const computed = meter
@@ -163,12 +166,59 @@ export async function think(ctx, query, mechs) {
163
166
  best = c;
164
167
  };
165
168
  const worthRunning = (floor) => best === null || grade(floor) < grade(best.weight);
169
+ // REGIME PREDICTION (R8) — observational only. Once the FIRST mechanism has
170
+ // had its turn (cover, which §2.6 places first and floors at 0), the market's
171
+ // outcome is already determined by the one cost ladder: the consensus climb
172
+ // runs exactly when `worthRunning(2 * STEP)` is true — CAST (floor 2·STEP) is
173
+ // the cheapest mechanism that first-touches it, so an incumbent at or below
174
+ // grade 2 prunes CAST and, with it, confluence (3·STEP) and extraction
175
+ // (CONCEPT+STEP) (retrieval); anything above — or no incumbent — runs the
176
+ // full market and the climb (composition). The predicate is `worthRunning`,
177
+ // the same function the loop itself uses — nothing is computed here that the
178
+ // engine had not already computed, and nothing is read back by inference.
179
+ //
180
+ // EMITTED BEFORE THE SECOND MECHANISM'S FLOOR, never after some mechanism's
181
+ // run: a "prediction" published after the fact could assert "the climb will
182
+ // not run" about a climb that already ran — which is what happens whenever
183
+ // the first mechanism is SKIPPED (null floor or pruned) and the block sits at
184
+ // the end of the first mechanism that actually ran. Emitting on entry to
185
+ // iteration 1 makes the claim true by construction, whatever the first
186
+ // mechanism did, and keeps the payload identical on the ordinary path (the
187
+ // incumbent cannot change between the two positions).
188
+ let regimeReported = false;
189
+ const reportRegime = () => {
190
+ if (regimeReported)
191
+ return;
192
+ regimeReported = true;
193
+ const climbFloorGrade = grade(2 * STEP);
194
+ // TS narrows `best` to null in the outer flow (it cannot see the closure
195
+ // assignments in `consider`) — cast back, the same read-back as `decided`
196
+ // below.
197
+ const incumbent = best;
198
+ const incumbentGrade = incumbent === null ? null : grade(incumbent.weight);
199
+ const regime = worthRunning(2 * STEP)
200
+ ? "composition"
201
+ : "retrieval";
202
+ ctx.trace?.step("regimePrediction", [rItem(query, "query")], [], regime === "retrieval"
203
+ ? `retrieval regime — incumbent grade ${incumbentGrade} ≤ climb floor ${climbFloorGrade}, ` +
204
+ `so no mechanism floored above that grade runs; the consensus climb will not run`
205
+ : `composition regime — ${incumbentGrade === null
206
+ ? "no incumbent (nothing grounded)"
207
+ : `incumbent grade ${incumbentGrade}`} above climb floor ${climbFloorGrade}, so the full market and climb run`, undefined, {
208
+ version: 1,
209
+ regime,
210
+ incumbentGrade,
211
+ climbFloorGrade,
212
+ });
213
+ };
166
214
  // Phase 3: grounding loop
167
215
  // Per-mechanism accounting (src/meter.ts). The market's whole premise is
168
216
  // that mechanisms compete on one cost scale — so the profiling read-out is
169
217
  // also per-mechanism, uniformly: the loop never asks which one it holds.
170
- let regimeReported = false;
171
- for (const mech of mechanisms) {
218
+ for (let mi = 0; mi < mechanisms.length; mi++) {
219
+ const mech = mechanisms[mi];
220
+ if (mi > 0)
221
+ reportRegime();
172
222
  const floor = meter
173
223
  ? await meter.time(`${mech.name}.floor`, () => mech.floor(ctx, query, pre, worthRunning))
174
224
  : await mech.floor(ctx, query, pre, worthRunning);
@@ -204,43 +254,11 @@ export async function think(ctx, query, mechs) {
204
254
  scaffolding: r.scaffolding,
205
255
  });
206
256
  }
207
- // REGIME PREDICTION (R8) — observational only. After the FIRST mechanism
208
- // runs (cover, which §2.6 places first and floors at 0), the market's
209
- // outcome is already determined: the consensus climb runs exactly when
210
- // `worthRunning(2 * STEP)` is true — CAST (floor 2·STEP) is the cheapest
211
- // mechanism that first-touches it, so an incumbent at or below grade 2
212
- // prunes CAST and, with it, confluence (3·STEP) and extraction
213
- // (CONCEPT+STEP) (retrieval); anything above — or no incumbent — runs the
214
- // full market and the climb (composition). The predicate is
215
- // `worthRunning`, the same function the loop just used — nothing is
216
- // computed here that the engine had not already computed, and nothing is
217
- // read back by inference.
218
- if (!regimeReported) {
219
- regimeReported = true;
220
- const climbFloorGrade = grade(2 * STEP);
221
- // TS narrows `best` to null in the outer flow (it cannot see the closure
222
- // assignments in `consider`) — cast back, the same read-back as `decided`
223
- // below.
224
- const incumbent = best;
225
- const incumbentGrade = incumbent === null
226
- ? null
227
- : grade(incumbent.weight);
228
- const regime = worthRunning(2 * STEP)
229
- ? "composition"
230
- : "retrieval";
231
- ctx.trace?.step("regimePrediction", [rItem(query, "query")], [], regime === "retrieval"
232
- ? `retrieval regime — incumbent grade ${incumbentGrade} ≤ climb floor ${climbFloorGrade}, so no composition mechanism runs; ` +
233
- `the consensus climb will not run`
234
- : `composition regime — ${incumbentGrade === null
235
- ? "no incumbent (nothing grounded)"
236
- : `incumbent grade ${incumbentGrade}`} above climb floor ${climbFloorGrade}, so the full market and climb run`, undefined, {
237
- version: 1,
238
- regime,
239
- incumbentGrade,
240
- climbFloorGrade,
241
- });
242
- }
243
257
  }
258
+ // A market of ONE mechanism never reaches iteration 1; the step is still
259
+ // emitted exactly once per think(), so a consumer never has to ask whether
260
+ // the list was long enough for the prediction to exist.
261
+ reportRegime();
244
262
  // (TS cannot see the closure assignments into `best` and narrows it to its
245
263
  // initial null, so the read-back needs the assertion.)
246
264
  const decided = best;
@@ -385,7 +403,7 @@ export async function think(ctx, query, mechs) {
385
403
  ? reasoned
386
404
  : meter
387
405
  ? await meter.time("fuse", () => fuseAttention(ctx, query, reasoned, pre, unclimbed, primarySpans))
388
- : await fuseAttention(ctx, query, reasoned, pre, unclimbed, decided.accounted);
406
+ : await fuseAttention(ctx, query, reasoned, pre, unclimbed, primarySpans);
389
407
  done(fused, "grounded, reasoned forward, fused across points of attention");
390
408
  return { bytes: fused, provenance };
391
409
  }
@@ -10,13 +10,22 @@ import type { MindContext, Recognition, Segment } from "./types.js";
10
10
  * the longest known leaf, chained into flat branches. Names forms the
11
11
  * query's own cut cannot, and records sub-leaf boundaries as `splits`.
12
12
  *
13
- * Both O(n · maxGroup) bounded O(1) probes — never a scan of the corpus. */
14
- /** Decompose `bytes` into the learnt forms it contains. `trimmed` skips the
15
- * edge-trim fallbacks (which recover misaligned FRAGMENTS) for callers whose
16
- * own gate rejects fragments anyway (the pivot), so the O(n·W²) trim search is
17
- * paid only where its output can be used. Byte-identical for every caller
18
- * that keeps only top-level forms. */
19
- export declare function recognise(ctx: MindContext, bytes: Uint8Array, trimmed?: boolean): Recognition;
13
+ * Both O(n · maxGroup) bounded O(1) probes — never a scan of the corpus.
14
+ *
15
+ * ONE READING PER BYTE STREAM, deliberately: there is no "cheap mode" that
16
+ * skips the edge-trim fallbacks. A `trimmed` variant was tried and REFUTED
17
+ * twice over. Its premise "the trims only recover misaligned FRAGMENTS, so
18
+ * a consumer whose gate rejects fragments loses nothing" — is false: the
19
+ * left/right trim loops below exist precisely to find WHOLE trained forms
20
+ * embedded at an offset the query's own fold did not cut, and such a form has
21
+ * no structural parents or containers, so it passes the pivot's fragment gate
22
+ * and is exactly the candidate a multi-hop chain steps through. Skipping them
23
+ * narrows the pivot's evidence silently. And a per-caller variant has to key
24
+ * the memo by the variant, which breaks the "computed at most once" property
25
+ * (§2.11): the pipeline recognises a grounded answer untrimmed for
26
+ * `preConsumed`, and the pivot then recognises the same bytes again — the
27
+ * saving inverts into a doubling on the path it was measured for. */
28
+ export declare function recognise(ctx: MindContext, bytes: Uint8Array): Recognition;
20
29
  /** Segment bytes using the geometry's own groupings — leaf-parent
21
30
  * nodes from the perceived tree, with consecutive bare leaves merged
22
31
  * into one segment. Each segment's gist is perceived from its bytes
@@ -21,13 +21,22 @@ import { isChunk } from "../sema.js";
21
21
  * the longest known leaf, chained into flat branches. Names forms the
22
22
  * query's own cut cannot, and records sub-leaf boundaries as `splits`.
23
23
  *
24
- * Both O(n · maxGroup) bounded O(1) probes — never a scan of the corpus. */
25
- /** Decompose `bytes` into the learnt forms it contains. `trimmed` skips the
26
- * edge-trim fallbacks (which recover misaligned FRAGMENTS) for callers whose
27
- * own gate rejects fragments anyway (the pivot), so the O(n·W²) trim search is
28
- * paid only where its output can be used. Byte-identical for every caller
29
- * that keeps only top-level forms. */
30
- export function recognise(ctx, bytes, trimmed = false) {
24
+ * Both O(n · maxGroup) bounded O(1) probes — never a scan of the corpus.
25
+ *
26
+ * ONE READING PER BYTE STREAM, deliberately: there is no "cheap mode" that
27
+ * skips the edge-trim fallbacks. A `trimmed` variant was tried and REFUTED
28
+ * twice over. Its premise "the trims only recover misaligned FRAGMENTS, so
29
+ * a consumer whose gate rejects fragments loses nothing" — is false: the
30
+ * left/right trim loops below exist precisely to find WHOLE trained forms
31
+ * embedded at an offset the query's own fold did not cut, and such a form has
32
+ * no structural parents or containers, so it passes the pivot's fragment gate
33
+ * and is exactly the candidate a multi-hop chain steps through. Skipping them
34
+ * narrows the pivot's evidence silently. And a per-caller variant has to key
35
+ * the memo by the variant, which breaks the "computed at most once" property
36
+ * (§2.11): the pipeline recognises a grounded answer untrimmed for
37
+ * `preConsumed`, and the pivot then recognises the same bytes again — the
38
+ * saving inverts into a doubling on the path it was measured for. */
39
+ export function recognise(ctx, bytes) {
31
40
  // Content-keyed memo — works for both single-turn respond() and multi-turn
32
41
  // respondTurn() (where the map persists across calls). ALWAYS consulted,
33
42
  // regardless of tracing — matching perceive()'s own memo, which carries no
@@ -67,7 +76,7 @@ export function recognise(ctx, bytes, trimmed = false) {
67
76
  // not silent), so it is emitted here directly rather than only inside
68
77
  // recogniseImpl.
69
78
  if (ctx.recogniseMemo) {
70
- const key = (trimmed ? "t" : "f") + latin1Key(bytes);
79
+ const key = latin1Key(bytes);
71
80
  const hit = ctx.recogniseMemo.get(key);
72
81
  if (hit !== undefined) {
73
82
  if (ctx.meter)
@@ -79,13 +88,13 @@ export function recognise(ctx, bytes, trimmed = false) {
79
88
  `lead somewhere (over ${hit.leaves.length} perceived leaves) [cached]`);
80
89
  return hit;
81
90
  }
82
- const fresh = recogniseImpl(ctx, bytes, trimmed);
91
+ const fresh = recogniseImpl(ctx, bytes);
83
92
  ctx.recogniseMemo.set(key, fresh);
84
93
  return fresh;
85
94
  }
86
- return recogniseImpl(ctx, bytes, trimmed);
95
+ return recogniseImpl(ctx, bytes);
87
96
  }
88
- function recogniseImpl(ctx, bytes, trimmed = false) {
97
+ function recogniseImpl(ctx, bytes) {
89
98
  if (ctx.meter) {
90
99
  ctx.meter.recognitions++;
91
100
  ctx.meter.recognisedBytes += bytes.length;
@@ -199,7 +208,7 @@ function recogniseImpl(ctx, bytes, trimmed = false) {
199
208
  // n.kids !== null enforces above) rather than degenerate into
200
209
  // single-byte-atom territory, which atomIsHub already governs
201
210
  // separately.
202
- else if (!trimmed && end - start - 1 >= 2) {
211
+ else if (end - start - 1 >= 2) {
203
212
  // The chunk's own boundary is drawn by content geometry, not by
204
213
  // any notion of "form" — it can include one edge byte the query's
205
214
  // fold happened to attach here that the trained span never had
@@ -288,13 +288,13 @@ export async function pivotInto(ctx, answer, consumed, voiced = []) {
288
288
  for (const c of n.kids)
289
289
  queue.push(c); // breadth-first: larger regions first
290
290
  }
291
- // TRIMMED recognition: the pivot's own filter below rejects fragments
292
- // (`hasParents || hasContainers -Infinity`), and recognition's edge-trim
293
- // fallbacks exist to find exactly those misaligned FRAGMENTS. Skipping them
294
- // (the structural pass + canonResolve still run) is byte-identical for every
295
- // pivot the fallbacks' output is discarded by the filter — and halves the
296
- // O(n·W²) recognition of a long answer (measured: 36KB recognise 4.0s → 2.0s).
297
- const rec = recognise(ctx, answer, true);
291
+ // THE FULL recognition, memo-shared with every other reader of these bytes.
292
+ // A "skip the edge trims here" variant was refuted (see recognise's own
293
+ // note): those trims are what find a WHOLE trained form embedded at an
294
+ // offset the answer's fold did not cut, and such a form is parentless,
295
+ // container-free and edge-bearing i.e. exactly what the filter below
296
+ // ADMITS as a pivot, not what it rejects.
297
+ const rec = recognise(ctx, answer);
298
298
  for (const s of rec.sites) {
299
299
  if (!consumed.has(s.payload) && ctx.store.hasNext(s.payload)) {
300
300
  scored.set(s.payload, Math.max(scored.get(s.payload) ?? 0, 1));
@@ -322,6 +322,13 @@ export async function pivotInto(ctx, answer, consumed, voiced = []) {
322
322
  let pivotId = null;
323
323
  for (const c of ranked) {
324
324
  const id = c.id;
325
+ // A ZERO-LENGTH candidate is not a pivot. `argmaxBy(…, 0, strict)` used to
326
+ // carry this floor in its threshold argument, and dropping it here would
327
+ // admit an empty node: `indexOf(answer, <empty>)` returns 0, so every
328
+ // filter below passes and the chain would hop through nothing (§2.13 —
329
+ // empty bytes are truthy).
330
+ if (c.len === 0)
331
+ continue;
325
332
  // A PIVOT MUST BE A THING THE CORPUS DEPOSITED, NOT A PIECE OF ONE.
326
333
  // "Longest wins" ranks candidates but never asks whether the winner is
327
334
  // an entity at all, and by the time a chain reaches here `consumeAll`
@@ -570,9 +570,6 @@ export declare abstract class AbstractStore implements Store {
570
570
  nodeCount(): number;
571
571
  size(): Promise<number>;
572
572
  get(id: NodeId): NodeRec | null;
573
- /** Reconstruct the bytes a node spans by traversing the DAG bottom-up.
574
- * Iterative post-order on an explicit stack — the call stack never sees the
575
- * tree depth, so even an adversarial chain of nodes stays safe. */
576
573
  /** How many reads hit a MISSING node record this session (a dangling edge
577
574
  * or kid id). Zero in a healthy store; a growing count means references
578
575
  * outlive their records — the read degrades safely to empty bytes, this
@@ -582,6 +579,25 @@ export declare abstract class AbstractStore implements Store {
582
579
  * nothing is profiling. Every read below bumps it through `?.`, so an
583
580
  * unprofiled store pays one null check per read and allocates nothing. */
584
581
  meter: Meter | null;
582
+ /** Reconstruct the bytes a node spans by traversing the DAG bottom-up.
583
+ * Iterative post-order on an explicit stack — the call stack never sees the
584
+ * tree depth, so even an adversarial chain of nodes stays safe.
585
+ *
586
+ * TERMINATION. The walk memoizes into a LOCAL map, and `_bytesCache` is
587
+ * consulted only as a warm hint whose hit is immediately promoted into that
588
+ * map. It used to use `_bytesCache` itself as the memo, which is not a
589
+ * memo at all: it EVICTS, and its `"smallest"` policy prefers precisely the
590
+ * freshly-resolved small children that the pending parents on the stack are
591
+ * waiting for. A parent then finds them uncached again, re-pushes them,
592
+ * they are re-resolved, re-inserted, re-evicted — the loop makes no
593
+ * progress and never exits. Latent until the cache saturates, then
594
+ * unconditional: observed in the wild at 19.9M nodes with the 20 MB cache
595
+ * pinned at 19,999,962/20,000,000 bytes, spinning 8h45m on a node whose
596
+ * whole content was 124 bytes (5 kids, 2 of them perpetually re-evicted).
597
+ * Because the loop is synchronous, no timer could fire — the trainer's stall
598
+ * watchdog never got a turn either. A local map resolves each node at most
599
+ * once per call, so the walk terminates by construction and `_bytesCache`
600
+ * goes back to being a pure speed hint. */
585
601
  bytes(id: NodeId): Uint8Array;
586
602
  /** First `maxLen` bytes of a node. Walks only the leftmost branch,
587
603
  * stopping at `maxLen` — so a 1 MB document root costs the same as a
@@ -665,7 +681,20 @@ export declare abstract class AbstractStore implements Store {
665
681
  * common-prefix / common-suffix trim: whatever remains after both trims is
666
682
  * the single differing span (substitution, insertion or deletion), and both
667
683
  * remainders must fit the budget. Scattered differences leave a wide
668
- * middle and are rejected. */
684
+ * middle and are rejected.
685
+ *
686
+ * Every read here is CAPPED (§2.8). It used to open with
687
+ * `bytesPrefix(k, Number.MAX_SAFE_INTEGER)` — the ALL sentinel, i.e. the
688
+ * full materialising `bytes()` read — on the deposit hot path, and only
689
+ * then compare lengths. So a candidate the length test was about to reject
690
+ * had already been reconstructed byte for byte. The LENGTHS decide first
691
+ * instead, from the `contentLen` memo the interning order has already built
692
+ * bottom-up, and the target's length is itself read under a cap: a target
693
+ * longer than `la + W` is rejected without touching one of its bytes.
694
+ * Same semantics — the old capped `b` read would have produced
695
+ * `a.length + W + 1` here and failed the very same test — strictly fewer
696
+ * byte reads. The `+ 1` on each byte cap keeps `_prefix`'s
697
+ * "complete reconstruction" test true, so the results still cache. */
669
698
  private differsByOneWindow;
670
699
  putLeaf(bytes: Uint8Array, gist: Vec): Promise<NodeId>;
671
700
  putBranch(kids: NodeId[], gist: Vec): Promise<NodeId>;