@hviana/sema 0.2.2 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/dist/src/mind/articulation.js +1 -1
  2. package/dist/src/mind/attention.d.ts +18 -2
  3. package/dist/src/mind/attention.js +88 -18
  4. package/dist/src/mind/bridge.d.ts +30 -0
  5. package/dist/src/mind/bridge.js +569 -0
  6. package/dist/src/mind/graph-search.d.ts +16 -1
  7. package/dist/src/mind/graph-search.js +34 -5
  8. package/dist/src/mind/match.d.ts +15 -2
  9. package/dist/src/mind/match.js +3 -8
  10. package/dist/src/mind/mechanisms/alu.js +8 -1
  11. package/dist/src/mind/mechanisms/cast.d.ts +54 -0
  12. package/dist/src/mind/mechanisms/cast.js +303 -48
  13. package/dist/src/mind/mechanisms/cover.js +24 -32
  14. package/dist/src/mind/mechanisms/extraction.js +75 -30
  15. package/dist/src/mind/mechanisms/recall.js +66 -0
  16. package/dist/src/mind/mind.d.ts +1 -0
  17. package/dist/src/mind/mind.js +6 -1
  18. package/dist/src/mind/pipeline.js +34 -2
  19. package/dist/src/mind/reasoning.d.ts +20 -1
  20. package/dist/src/mind/reasoning.js +84 -6
  21. package/dist/src/mind/recognition.js +157 -13
  22. package/dist/src/mind/traverse.js +16 -0
  23. package/dist/src/mind/types.d.ts +65 -2
  24. package/dist/src/mind/types.js +53 -7
  25. package/dist/src/store.d.ts +12 -3
  26. package/dist/src/store.js +9 -3
  27. package/package.json +1 -1
  28. package/src/mind/articulation.ts +1 -0
  29. package/src/mind/attention.ts +105 -17
  30. package/src/mind/bridge.ts +596 -0
  31. package/src/mind/graph-search.ts +59 -2
  32. package/src/mind/match.ts +19 -5
  33. package/src/mind/mechanisms/alu.ts +8 -1
  34. package/src/mind/mechanisms/cast.ts +336 -46
  35. package/src/mind/mechanisms/cover.ts +31 -36
  36. package/src/mind/mechanisms/extraction.ts +101 -40
  37. package/src/mind/mechanisms/recall.ts +79 -0
  38. package/src/mind/mind.ts +7 -1
  39. package/src/mind/pipeline.ts +37 -2
  40. package/src/mind/reasoning.ts +97 -5
  41. package/src/mind/recognition.ts +160 -12
  42. package/src/mind/traverse.ts +17 -0
  43. package/src/mind/types.ts +110 -6
  44. package/src/store.ts +19 -5
  45. package/test/35-attention-confidence.test.mjs +139 -0
  46. package/test/36-already-answered-fusion.test.mjs +128 -0
  47. package/test/37-cluster-dispersion-fusion.test.mjs +190 -0
  48. package/test/38-reason-restate-guard.test.mjs +94 -0
  49. package/test/39-cast-restate-guard.test.mjs +102 -0
  50. package/test/40-choosenext-scale-guard.test.mjs +75 -0
  51. package/test/41-seatofnode-direction.test.mjs +85 -0
  52. package/test/42-recognise-trace-idempotence.test.mjs +106 -0
  53. package/test/43-cast-analog-seat.test.mjs +244 -0
  54. package/test/44-recognise-edge-whitespace.test.mjs +63 -0
  55. package/test/45-liftanswer-restated-trim.test.mjs +60 -0
  56. package/test/46-recognise-multibyte-edge.test.mjs +85 -0
  57. package/test/47-cast-comparison-coverage.test.mjs +134 -0
  58. package/test/48-recognise-turn-connective.test.mjs +125 -0
  59. package/test/49-natural-units-synonym-bridge.test.mjs +175 -0
  60. package/test/50-cast-analog-consensus-floor.test.mjs +179 -0
@@ -16,7 +16,7 @@ import {
16
16
  resolve,
17
17
  } from "./primitives.js";
18
18
  import { atomIsHub, corpusN, leadsSomewhere } from "./traverse.js";
19
- import { chainReach, leafIdAt } from "./canonical.js";
19
+ import { chainReach, leafIdAt, leafIdRun } from "./canonical.js";
20
20
  import { isChunk, type Sema } from "../sema.js";
21
21
  import type { Leaf, Site } from "./graph-search.js";
22
22
 
@@ -34,12 +34,44 @@ import type { Leaf, Site } from "./graph-search.js";
34
34
  * Both O(n · maxGroup) bounded O(1) probes — never a scan of the corpus. */
35
35
  export function recognise(ctx: MindContext, bytes: Uint8Array): Recognition {
36
36
  // Content-keyed memo — works for both single-turn respond() and multi-turn
37
- // respondTurn() (where the map persists across calls). Skipped while
38
- // tracing so every call still emits its rationale step.
39
- if (ctx.recogniseMemo && !ctx.trace) {
37
+ // respondTurn() (where the map persists across calls). ALWAYS consulted,
38
+ // regardless of tracing matching perceive()'s own memo, which carries no
39
+ // trace gate at all. This memo is NOT an optional accelerator: recogniseImpl
40
+ // walks the query's perceived tree via foldTree, whose subtree-resolution
41
+ // fast path (see primitives.ts) skips invoking `visit` — and therefore
42
+ // skips EMITTING SITES — for any subtree already cached in
43
+ // ctx._resolvedSubtrees. A multi-turn conversation's stable-prefix fold
44
+ // deliberately shares node OBJECTS across turns, so by the second call on
45
+ // the exact same bytes, large swaths of the tree are already cached and
46
+ // foldTree stops short of recursing into them — a second recogniseImpl
47
+ // call on the SAME bytes is not idempotent; it silently finds FEWER sites
48
+ // than the first (observed live: 31 sites → 5 on an immediate repeat
49
+ // call). Skipping this memo "only while tracing" used to mean every
50
+ // traced turn re-ran recogniseImpl from scratch at every one of the many
51
+ // call sites that recognise the same query (cover, reason, articulate...),
52
+ // each subsequent call silently more incomplete than the last — measurably
53
+ // changing which mechanism grounds the answer, not just costing time. The
54
+ // trace step must still fire on every call regardless (a cache hit is not
55
+ // silent), so it is emitted here directly instead of only inside
56
+ // recogniseImpl.
57
+ if (ctx.recogniseMemo) {
40
58
  const key = latin1Key(bytes);
41
59
  const hit = ctx.recogniseMemo.get(key);
42
- if (hit !== undefined) return hit;
60
+ if (hit !== undefined) {
61
+ ctx.trace?.step(
62
+ "recognise",
63
+ [rItem(bytes, "query")],
64
+ hit.sites.map((s) =>
65
+ rItem(bytes.subarray(s.start, s.end), "form", s.payload, [
66
+ s.start,
67
+ s.end,
68
+ ])
69
+ ),
70
+ `decompose the query into ${hit.sites.length} learnt form(s) that ` +
71
+ `lead somewhere (over ${hit.leaves.length} perceived leaves) [cached]`,
72
+ );
73
+ return hit;
74
+ }
43
75
  const fresh = recogniseImpl(ctx, bytes);
44
76
  ctx.recogniseMemo.set(key, fresh);
45
77
  return fresh;
@@ -52,7 +84,8 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
52
84
  const sites: Site[] = [];
53
85
  const leaves: Leaf[] = [];
54
86
  const splits = new Set<number>();
55
- if (bytes.length === 0) return { sites, leaves, splits };
87
+ const starts = new Set<number>();
88
+ if (bytes.length === 0) return { sites, leaves, splits, starts };
56
89
 
57
90
  // Span-resolve memo for THIS call: the structural pass (sub-runs inside
58
91
  // leaf-parents) and the canonical pass (leaf-id chains) probe overlapping
@@ -82,15 +115,24 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
82
115
  // silence. Atoms stay available as leaves (PASS-carried literals) and
83
116
  // through exact tier-0 resolution regardless.
84
117
  const atomsAreHubs = atomIsHub(ctx, corpusN(ctx));
118
+ // Distinct probes (structural exact match, canon fallback, edge trims at
119
+ // several offsets) can legitimately re-derive the SAME (start, end, id)
120
+ // site from different tree nodes — a wide edge-trim search is exactly
121
+ // this on purpose (see below). Duplicate site entries are not wrong
122
+ // evidence, but they double the weight cover's derivation search gives
123
+ // that span, distorting its cost model — the same span must count once.
124
+ const seen = new Set<string>();
85
125
  const emit = (start: number, end: number, id: number) => {
86
126
  if (id < 0 && atomsAreHubs) return;
127
+ const key = start + "," + end + "," + id;
128
+ if (seen.has(key)) return;
129
+ seen.add(key);
87
130
  if (leadsSomewhere(ctx, id)) {
88
131
  sites.push({ start, end, payload: id });
89
132
  }
90
133
  };
91
134
 
92
135
  // ── structural: the query's own perceived tree ──────────────────────
93
- const starts = new Set<number>();
94
136
  starts.add(0);
95
137
  foldTree(ctx, perceive(ctx, bytes), 0, (n, start, end, node) => {
96
138
  if (n.kids === null) {
@@ -101,10 +143,93 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
101
143
  // missed may still be a stored form under the response's equivalence
102
144
  // (case, width, whitespace — whatever the injected canonicalizer says).
103
145
  // O(subtree bytes) per miss, memoised per response; a no-op when no
104
- // canonicalizer was injected or the store has no canon index.
105
- else if (end - start >= 2) {
146
+ // canonicalizer was injected or the store has no canon index. A raw
147
+ // leaf (n.kids === null) is single-byte and handled by the byte-atom
148
+ // path above instead — canon equivalence only applies to composites.
149
+ else if (n.kids !== null) {
106
150
  const cid = canonResolve(ctx, bytes.subarray(start, end));
107
151
  if (cid !== null) emit(start, end, cid);
152
+ // The edge-trim fallbacks below remove 1 byte from a side; the
153
+ // remainder must still be a composite (>= 2 bytes, the same floor
154
+ // n.kids !== null enforces above) rather than degenerate into
155
+ // single-byte-atom territory, which atomIsHub already governs
156
+ // separately.
157
+ else if (end - start - 1 >= 2) {
158
+ // The chunk's own boundary is drawn by content geometry, not by
159
+ // any notion of "form" — it can include one edge byte the query's
160
+ // fold happened to attach here that the trained span never had
161
+ // (e.g. a separator from the preceding chunk). The core has no
162
+ // idea what that byte means; it only knows resolve()/canonResolve
163
+ // are self-verifying (hash-then-verify, same discipline as every
164
+ // content lookup here), so a blind one-byte-shorter guess on
165
+ // either edge costs nothing when wrong and is trustworthy when it
166
+ // hits. Two extra probes, only on the already-failed miss path.
167
+ const left = resolve(ctx, bytes.subarray(start + 1, end));
168
+ if (left !== null) emit(start + 1, end, left);
169
+ const right = resolve(ctx, bytes.subarray(start, end - 1));
170
+ if (right !== null) emit(start, end - 1, right);
171
+ // A misalignment wider than one byte (e.g. more than one edge
172
+ // separator swallowed) is not itself geometry-quantized — the
173
+ // WRITE side's canonical index (canonicalWindows) interns sliding
174
+ // W−1/W-length windows over leaf ids at EVERY offset, not just
175
+ // radix-aligned ones (see canonical.ts) — so the offset that
176
+ // recovers a trained span can be anything, not a multiple of W.
177
+ // What IS bounded is how far it's worth looking: chainReach(W)=W²,
178
+ // the same reach the canonical pass (tryChain) trusts for a chain
179
+ // rebuilt off the query's own fold. Every candidate offset is
180
+ // gated by store.findBranch(leafIds) first — the SAME cheap,
181
+ // fold-free existence check tryChain already uses — so the extra
182
+ // resolve() fold (the real cost) is only paid when a branch could
183
+ // plausibly exist there, not for every offset. The node itself is
184
+ // also bounded to chunk-scale (end - start <= W²): widening this at
185
+ // whole-query/root scale can rediscover a smaller subtree's own
186
+ // content as a second, overlapping site the structural walk's own
187
+ // finer recursion already emits correctly on its own — a duplicate
188
+ // that downstream derivation can stitch into a wrong answer.
189
+ const W = ctx.space.maxGroup;
190
+ for (
191
+ let k = 1;
192
+ end - start <= W * W && k <= W * W && start + k < end - 1;
193
+ k++
194
+ ) {
195
+ const lIds = leafIdRun(ctx, bytes, start + k, end);
196
+ if (lIds !== null && store.findBranch(lIds) !== null) {
197
+ const eLeft = resolve(ctx, bytes.subarray(start + k, end));
198
+ if (eLeft !== null) emit(start + k, end, eLeft);
199
+ }
200
+ const rIds = leafIdRun(ctx, bytes, start, end - k);
201
+ if (rIds !== null && store.findBranch(rIds) !== null) {
202
+ const eRight = resolve(ctx, bytes.subarray(start, end - k));
203
+ if (eRight !== null) emit(start, end - k, eRight);
204
+ }
205
+ }
206
+ // A REAL extra word at the left edge (a discourse connective like
207
+ // "And " prepended to a follow-up turn — not boundary noise, actual
208
+ // content the injected canonicalizer has no equivalence for) shows
209
+ // up as a canon-miss too big for the chunk-scale search above: the
210
+ // turn is its OWN segment (stable-prefix folded independently — see
211
+ // mind.ts's _growContext), so it can be turn/segment-scale, not
212
+ // chunk-scale. Widening the size bound itself reopens the root-
213
+ // scale false-positive this module already fixed once (test/46);
214
+ // widening the SEARCH instead — trying every position up to W
215
+ // chunk-widths from the left edge that the query's OWN fold treats
216
+ // as a chunk boundary (`starts`, the same set the canonical pass
217
+ // privileges with full chain reach) — does not: `starts.has(p)` is
218
+ // fold EVIDENCE the query produced on its own (a leaf-parent chunk
219
+ // like "And " is visited, and its start added to `starts`, before
220
+ // this composite ever runs — foldTree is post-order), never a blind
221
+ // guess. Bounded to W candidates, each an O(1) set lookup before
222
+ // paying for the real canonResolve fold — canonResolve, not
223
+ // resolve()/findBranch, because the gap here is often exactly the
224
+ // kind of equivalence (case, in the live trace) canon exists for,
225
+ // not just an exact-content coincidence.
226
+ for (let k = 1; k <= W; k++) {
227
+ const p = start + k * W;
228
+ if (p >= end - 1 || !starts.has(p)) continue;
229
+ const cid = canonResolve(ctx, bytes.subarray(p, end));
230
+ if (cid !== null) emit(p, end, cid);
231
+ }
232
+ }
108
233
  }
109
234
  if (isChunk(n)) {
110
235
  starts.add(start);
@@ -115,7 +240,15 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
115
240
  leafOffsets.push(off);
116
241
  off += k.leaf?.length ?? 0;
117
242
  }
243
+ // Sub-spans starting at i > 0 begin INSIDE the chunk, at an offset the
244
+ // query's own fold did not itself choose as a boundary — the same
245
+ // opportunistic byte-atom-chain risk `tryChain`'s `boundary` gate
246
+ // guards below (see its comment). Only the chunk's own left edge
247
+ // (i === 0, already registered in `starts` above) carries the fold's
248
+ // evidence; interior sub-starts are exempt from the guard only while
249
+ // atoms themselves still discriminate at this corpus scale.
118
250
  for (let i = 0; i < n.kids.length; i++) {
251
+ if (i > 0 && atomsAreHubs) break;
119
252
  const subIds: number[] = [];
120
253
  for (let j = i; j < n.kids.length; j++) {
121
254
  const kj = n.kids[j];
@@ -158,9 +291,23 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
158
291
  }
159
292
  }
160
293
 
294
+ // A chain rebuilt from a NON-boundary offset (the query's own perceived
295
+ // cut, `starts`, never chose to segment here) is opportunistic: the same
296
+ // byte-atom coincidence the hub guard above already exists for, just
297
+ // spelled over 2+ leaves instead of 1. At small corpus scale that's fine
298
+ // — coincidence is rare and every chain is real evidence (see `atomIsHub`).
299
+ // Past the scale where atoms themselves stop discriminating, the same
300
+ // uniform-expectation argument bounds a CHAIN'S commonality too: it is at
301
+ // least as rare as its rarest atom, so a store where atoms are hubs makes
302
+ // interior chain reconstructions no more trustworthy than the atoms they
303
+ // are built from ("hi" resolving out of "W[hi]ch" is exactly this: two
304
+ // hub-scale atoms, chained at an offset nothing in the query's own fold
305
+ // selected). Chains that start ON a boundary carry the fold's own
306
+ // evidence instead and are exempt.
161
307
  const tryChain = (
162
308
  p: number,
163
309
  maxIds: number,
310
+ boundary: boolean,
164
311
  ): void => {
165
312
  const first = leafFrom(p);
166
313
  if (!first) return;
@@ -174,6 +321,7 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
174
321
  ids.push(nx.id);
175
322
  pos = nx.end;
176
323
  if (store.findBranch(ids) === null) continue;
324
+ if (!boundary && atomsAreHubs) continue;
177
325
  const id = resolveSpan(p, pos);
178
326
  if (id === null || id === prevId) continue;
179
327
  prevId = id;
@@ -183,10 +331,10 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
183
331
 
184
332
  for (let p = 0; p < bytes.length; p++) {
185
333
  if (starts.has(p)) {
186
- tryChain(p, chainReach(W)); // boundary start — full reach
334
+ tryChain(p, chainReach(W), true); // boundary start — full reach
187
335
  } else {
188
336
  const limit = chunkEnd[p] + W;
189
- tryChain(p, Math.min(limit - p, chainReach(W)));
337
+ tryChain(p, Math.min(limit - p, chainReach(W)), false);
190
338
  }
191
339
  }
192
340
 
@@ -211,7 +359,7 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
211
359
  ` (over ${leaves.length} perceived leaves)`,
212
360
  );
213
361
 
214
- return { sites, leaves, splits };
362
+ return { sites, leaves, splits, starts };
215
363
  }
216
364
 
217
365
  /** Segment bytes using the geometry's own groupings — leaf-parent
@@ -527,6 +527,23 @@ export function chooseNext(
527
527
  }
528
528
  }
529
529
 
530
+ // NO consensusFloor gate here (tried and reverted — see
531
+ // test/40-choosenext-scale-guard.test.mjs): that floor is calibrated for
532
+ // POOLED, IDF-weighted CLIMB VOTES (recallByResonance, commitVotes), where
533
+ // each corroborating region contributes at most ln N and the floor grows
534
+ // with N exactly as that per-region ceiling does (HOW_IT_WORKS.md §8.6).
535
+ // `bestSupport` here is a different kind of quantity — a raw prevCount of
536
+ // how many training contexts predicted ONE destination, bounded by how
537
+ // often that specific fact was retold, never by corpus size N. Gating an
538
+ // N-invariant count against an N-growing threshold guarantees failure
539
+ // once N is large enough, discarding genuinely, structurally dominant
540
+ // edges (observed: a fact corroborated 2-to-1-1-1 refused at N≈325K,
541
+ // falling back to a noisy concept-hop). The loop above already IS the
542
+ // "genuinely competing" test: a tie leaves first-inserted as the pick
543
+ // (test/30's own pinned behaviour); a strict winner is real evidence
544
+ // regardless of corpus scale. Matches HOW_IT_WORKS.md §25's own
545
+ // chooseNext pseudocode, which has no such floor.
546
+
530
547
  // Trace is built lazily — the filter + map below only execute when a
531
548
  // trace listener is attached, so the common (no-trace) path pays only
532
549
  // for the prevCount calls in the loop above, never for extra rItemShort
package/src/mind/types.ts CHANGED
@@ -39,7 +39,7 @@ export interface DepositCacheEntry {
39
39
  * its own suffix bytes-equal this exactly. */
40
40
  nextBytes?: Uint8Array;
41
41
  }
42
- import { concatBytes } from "../bytes.js";
42
+ import { bytesEqual, concatBytes, indexOf } from "../bytes.js";
43
43
  import { dominates } from "../geometry.js";
44
44
 
45
45
  // ═══════════════════════════════════════════════════════════════════════════
@@ -64,6 +64,7 @@ export interface GraphSearchHost {
64
64
  sites: ReadonlyArray<Site>;
65
65
  leaves: ReadonlyArray<Leaf>;
66
66
  splits: ReadonlySet<number>;
67
+ starts: ReadonlySet<number>;
67
68
  };
68
69
  chooseNext?(node: number): number | undefined;
69
70
  }
@@ -79,6 +80,13 @@ export interface Recognition {
79
80
  leaves: Leaf[];
80
81
  /** Sub-leaf positions where a form boundary falls between leaf edges. */
81
82
  splits: Set<number>;
83
+ /** Leaf-parent (chunk) start positions from the query's OWN perceived
84
+ * fold — the positions the fold itself chose as a grouping boundary, as
85
+ * opposed to an offset a byte-level scan merely happens to land on. The
86
+ * one boundary signal opportunistic cross-leaf recovery (recognition's
87
+ * own canonical chains, the search's `fuse`) can lean on instead of
88
+ * ASCII/word heuristics: see the `boundary` gate in recognition.ts. */
89
+ starts: Set<number>;
82
90
  }
83
91
 
84
92
  /** How the consensus climb weights a region's Document-Frequency reach. */
@@ -93,6 +101,33 @@ export interface Attention {
93
101
  /** The union of the query byte-spans whose evidence supports this point. */
94
102
  start: number;
95
103
  end: number;
104
+ /** SCALE-INVARIANT confidence: the fraction of the query's OWN regions
105
+ * whose evidence this point accounts for (Σ RegionVote.absorbed among
106
+ * its contributors, over the query's total region count) — read PER-
107
+ * ANCHOR, unlike the raw IDF vote (an absolute, ln(N)-scaled quantity
108
+ * that means "strong" on a small store and "weak" on a large one for
109
+ * the SAME degree of genuine consensus). A point whose breadth clears
110
+ * `dominates` (> half the query's regions corroborate it) is real
111
+ * consensus; one that does not is a coincidental single-region echo —
112
+ * see test/35-attention-confidence.test.mjs. */
113
+ breadth: number;
114
+ /** DISPERSION: the number of distinct clusters this point's contributing
115
+ * regions form, merging any two whose gap is under one river-fold
116
+ * quantum W. Neither breadth NOR raw region count discriminates a
117
+ * genuine further topic from a coincidental echo (both were tried and
118
+ * falsified — breadth starves a genuine, evenly-split multi-topic query,
119
+ * since no root in a real N-way split can exceed half the vote; raw
120
+ * count doesn't separate them either, since a short, structurally simple
121
+ * echo racks up as many corroborating regions as a real topic does).
122
+ * Dispersion asks a different question: not how MUCH evidence, but how
123
+ * many separate PLACES in the query corroborate it. A coincidental
124
+ * match — one local phrase resonating with an unrelated stored form —
125
+ * is structurally confined to ONE cluster no matter how strong its vote;
126
+ * a genuine further topic is named in its own distinctive wording
127
+ * somewhere the query's scaffolding does not reach, always a SEPARATE
128
+ * cluster from whatever else corroborates it. See
129
+ * test/37-cluster-dispersion-fusion.test.mjs. */
130
+ clusters: number;
96
131
  }
97
132
 
98
133
  /** Both read-outs of one consensus climb. */
@@ -130,6 +165,14 @@ export interface RegionVote {
130
165
  roots: readonly number[];
131
166
  w: number;
132
167
  wFocus: number;
168
+ /** How many of the query's ORIGINAL regions this one vote's evidence
169
+ * accounts for. 1 for an ordinary per-region vote (itself); for a
170
+ * cross-region junction vote, 1 (itself) plus however many individual
171
+ * votes it explained away (see crossRegionVotes) — the junction speaks
172
+ * for all of them at once, and breadth accounting must not undercount it
173
+ * to "one region" just because it collapsed to one pooled axiom.
174
+ * Defaults to 1 when absent. */
175
+ absorbed?: number;
133
176
  }
134
177
 
135
178
  /** The edge-bearing contexts reached by climbing from a node, plus saturation info. */
@@ -247,23 +290,84 @@ export function spliceAll(segs: Seg[]): Uint8Array | null {
247
290
  return concatBytes(segs.map((s) => s.bytes));
248
291
  }
249
292
 
293
+ /** Whether a chosen span RESTATES the query rather than answering it: its
294
+ * SUBSTITUTED bytes (an edge followed from a recognised site, not the
295
+ * site's own literal text read back) already occur elsewhere in the query
296
+ * — the same principle recall.ts's tiers apply to a whole-query projection
297
+ * ("a projection that is a proper byte-subspan of the query restates part
298
+ * of the question"). A LITERAL span (the site's own bytes, unchanged) is
299
+ * exempt: naming what's already there at its OWN position is not a
300
+ * substitution. A recognised site that is itself an entire PRIOR TURN of
301
+ * a multi-turn query is exactly this shape: it carries a genuine learnt
302
+ * continuation, but that continuation is something the asker already said
303
+ * moments later in the SAME query, not a new answer. Below one river
304
+ * window, byte overlap is chance, not evidence — the same floor
305
+ * identityBar and reachThreshold hold every other structural-overlap claim
306
+ * to. */
307
+ export function segRestatesQuery(
308
+ s: Seg,
309
+ query: Uint8Array,
310
+ queryLen: number,
311
+ W: number,
312
+ ): boolean {
313
+ if (!s.rec) return false;
314
+ const literal = s.j - s.i === s.bytes.length &&
315
+ bytesEqual(s.bytes, query.subarray(s.i, s.j));
316
+ if (literal) return false;
317
+ return s.bytes.length >= W && s.bytes.length < queryLen &&
318
+ indexOf(query, s.bytes, 0) >= 0;
319
+ }
320
+
250
321
  /** Lift the answer out of the cover for think: the recognised region, free of
251
- * the asker's surrounding (unrecognised) framing. */
252
- export function liftAnswer(segs: Seg[], queryLen: number): Uint8Array | null {
322
+ * the asker's surrounding (unrecognised) framing — and free of any chosen
323
+ * span that only RESTATES content the query already contains (see {@link
324
+ * segRestatesQuery}). A restating span is excluded from both the framing
325
+ * (lo/hi) decision and the final concatenation: it is stale, not a second
326
+ * answer, but the OTHER spans a derivation chose are independent evidence
327
+ * and must not be discarded along with it. */
328
+ export function liftAnswer(
329
+ segs: Seg[],
330
+ queryLen: number,
331
+ query: Uint8Array,
332
+ W: number,
333
+ ): Uint8Array | null {
334
+ const restated = segs.map((s) => segRestatesQuery(s, query, queryLen, W));
253
335
  const recognised: number[] = [];
254
- for (let k = 0; k < segs.length; k++) if (segs[k].rec) recognised.push(k);
336
+ for (let k = 0; k < segs.length; k++) {
337
+ if (segs[k].rec && !restated[k]) recognised.push(k);
338
+ }
255
339
  if (recognised.length === 0) return null;
256
340
 
257
341
  if (recognised.length === 1) {
258
342
  const s = segs[recognised[0]];
343
+ // A COMPUTED span's query-side width is operand digit-count, not
344
+ // evidence of how much of the query's meaning it accounts for — the
345
+ // half-dominance check below (built for a genuinely RECOGNISED learned
346
+ // form) is not a valid framing signal for it (see the `computed` field
347
+ // doc on Seg/GItem): "1000 - 421" outweighs "what is …?" by width only
348
+ // because the operands are big, not because the framing matters less.
349
+ // A LITERAL PREFIX before a computed span is unambiguous framing
350
+ // regardless of width — an arithmetic expression is never itself
351
+ // preceded by more literal computed content, so anything literal before
352
+ // it is question wording ("what is ", "compute ") to lift clear of.
353
+ // With no prefix (s.i === 0) the span is judged by the ordinary
354
+ // half-dominance rule below, which already correctly keeps a short
355
+ // trailing glue byte ("2+2." → "4.", the span dominates a 4-byte query).
356
+ if (s.computed && s.i > 0) return s.bytes;
259
357
  if (dominates(s.j - s.i, queryLen)) {
260
- return concatBytes(segs.map((x) => x.bytes));
358
+ return concatBytes(
359
+ segs.filter((_, k) => !restated[k]).map((x) => x.bytes),
360
+ );
261
361
  }
262
362
  return s.bytes;
263
363
  }
264
364
  const lo = recognised[0];
265
365
  const hi = recognised[recognised.length - 1];
266
- return concatBytes(segs.slice(lo, hi + 1).map((x) => x.bytes));
366
+ return concatBytes(
367
+ segs.slice(lo, hi + 1).filter((_, k) => !restated[lo + k]).map((x) =>
368
+ x.bytes
369
+ ),
370
+ );
267
371
  }
268
372
 
269
373
  /** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
package/src/store.ts CHANGED
@@ -306,8 +306,17 @@ export interface Store {
306
306
  nodeCount(): number;
307
307
 
308
308
  // ── soft resonance over node gists ─────────────────────────────────────
309
- /** The k nodes whose gist resonates most with v. */
310
- resonate(v: Vec, k: number): Promise<Hit[]>;
309
+ /** The k nodes whose gist resonates most with v. `exhaustive` widens the
310
+ * IVF probe to every cluster (see {@link AbstractStore.efFor}'s doc)
311
+ * for refusal-path-only callers where an approximate top-√C-clusters
312
+ * search is not the same discriminator as the caller's own byte-exact
313
+ * verification (the substitution bridge's proposal channel): a rarer
314
+ * paraphrase can score lower than hundreds of unrelated hits by pure
315
+ * fold-geometry structural distance (a middle-of-string mismatch
316
+ * perturbs the tree hash far more than a tail mismatch of the same
317
+ * byte length) and so never even reach a probed cluster, no matter how
318
+ * large k is — k only reorders WITHIN the clusters already probed. */
319
+ resonate(v: Vec, k: number, exhaustive?: boolean): Promise<Hit[]>;
311
320
  /** Mark a node as a RESONANCE TARGET — promote its gist into the content
312
321
  * index so {@link resonate} can find it. A node's gist is captured at intern
313
322
  * but indexed LAZILY (only targets are indexed; the ~99.5% intermediate DAG
@@ -1636,7 +1645,7 @@ export abstract class AbstractStore implements Store {
1636
1645
 
1637
1646
  // ── Soft resonance ─────────────────────────────────────────────────────
1638
1647
 
1639
- async resonate(v: Vec, k: number): Promise<Hit[]> {
1648
+ async resonate(v: Vec, k: number, exhaustive = false): Promise<Hit[]> {
1640
1649
  await this._ensureReady();
1641
1650
  // Synchronous flush of any buffered index writes: the FIRST resonance
1642
1651
  // after a large ingest pays that flush here, so it shows up in respond
@@ -1649,17 +1658,22 @@ export abstract class AbstractStore implements Store {
1649
1658
  // same values still hits. Lazy-init: null after any index write; the
1650
1659
  // first miss after a flush recreates it. When voteRegions resonates
1651
1660
  // identical perceived sub-regions, only the first call descends the ANN.
1652
- const rk = vecKey(v) + ":" + k;
1661
+ const rk = vecKey(v) + ":" + k + (exhaustive ? ":x" : "");
1653
1662
  const cache = this._resonateCache;
1654
1663
  if (cache) {
1655
1664
  const hit = cache.get(rk);
1656
1665
  if (hit !== undefined) return hit;
1657
1666
  }
1658
1667
 
1668
+ const clusters = this._vecContentClusterCount();
1659
1669
  const results = this._vecContentQuery(
1660
1670
  normalize(copy(v)),
1661
1671
  k * this.overfetch,
1662
- this.efFor(this._vecContentClusterCount()),
1672
+ // Exhaustive: probe every cluster (ef ≥ 4·clusters guarantees the
1673
+ // IVF's own ef→nprobe=ceil(ef/4) mapping reaches all of them) — the
1674
+ // natural ceiling for a search that is ALREADY refusal-path-only and
1675
+ // must not miss a candidate hiding in an unprobed cluster.
1676
+ exhaustive ? 4 * clusters : this.efFor(clusters),
1663
1677
  );
1664
1678
  const out: Hit[] = [];
1665
1679
  for (const r of results) {
@@ -0,0 +1,139 @@
1
+ // 35-attention-confidence.test.mjs — SCALE-INVARIANT CONFIDENCE for a point
2
+ // of attention (Attention.breadth).
3
+ //
4
+ // commitVotes always admits the DOMINANT root regardless of its IDF-weighted
5
+ // vote (attention.ts: "roots.length === 0 || ..." — the first root is never
6
+ // floor-gated). That is correct for the common case ("give me your best
7
+ // guess"), but it means a query's SOLE root can be either:
8
+ //
9
+ // (a) genuine consensus — most of the query's own regions independently
10
+ // corroborate it (a real fact, or a real cross-region binding), or
11
+ // (b) a coincidental echo — ONE region's resonance happened to land
12
+ // somewhere, with the rest of the query silent on it.
13
+ //
14
+ // The raw IDF vote cannot tell these apart: it is an ABSOLUTE quantity that
15
+ // scales with ln(corpus size), so the same vote means "strong" on a small
16
+ // store and "weak" on a large one (see the session's earlier finding: a
17
+ // genuine root on a 325K-context store scored BELOW its own consensus floor,
18
+ // while a spurious echo on a 15-fact store scored comfortably above its own —
19
+ // much smaller — floor). A SCALE-INVARIANT measure is needed instead: what
20
+ // FRACTION of the query's own regions this root's evidence accounts for —
21
+ // the "N of M sub-regions voted" the rationale already reports, but read
22
+ // PER-ANCHOR instead of globally, and tested against the same half-dominance
23
+ // convention (`dominates`, part*2 > whole) the rest of the codebase already
24
+ // uses for every other "is this real signal or noise" decision.
25
+
26
+ import { test } from "node:test";
27
+ import assert from "node:assert/strict";
28
+ import { Mind } from "../dist/src/index.js";
29
+ import { SQliteStore } from "../dist/src/store-sqlite.js";
30
+
31
+ const enc = (s) => new TextEncoder().encode(s);
32
+ const mk = (seed) =>
33
+ new Mind({ seed, store: new SQliteStore({ path: ":memory:" }) });
34
+
35
+ // ═══════════════════════════════════════════════════════════════════════════
36
+ // GENUINE CONSENSUS — a real cross-region binding (test/34's own corpus).
37
+ // Most of the query's regions agree on the joint context; breadth must
38
+ // DOMINATE (> half of the query's own regions corroborate it).
39
+ // ═══════════════════════════════════════════════════════════════════════════
40
+
41
+ const BINDING_CORPUS = [
42
+ ["red", "is a color"],
43
+ ["blue", "is a color"],
44
+ ["circle", "is a shape"],
45
+ ["square", "is a shape"],
46
+ ["red circle", "answer alpha"],
47
+ ["red square", "answer beta"],
48
+ ["blue circle", "answer gamma"],
49
+ ["blue square", "answer delta"],
50
+ ];
51
+
52
+ test("breadth: a genuine cross-region binding dominates the query's regions", async () => {
53
+ const m = mk(1);
54
+ await m.ingest(BINDING_CORPUS);
55
+ const roots = await m.climbAttention(enc("red then circle"), 24);
56
+ assert.equal(roots.length, 1, "expected exactly one committed root");
57
+ assert.ok(
58
+ typeof roots[0].breadth === "number",
59
+ "Attention must carry a breadth field",
60
+ );
61
+ assert.ok(
62
+ roots[0].breadth > 0.5,
63
+ `genuine binding must dominate (> half the query's own regions), got breadth=${
64
+ roots[0].breadth
65
+ }`,
66
+ );
67
+ await m.store.close();
68
+ });
69
+
70
+ // ═══════════════════════════════════════════════════════════════════════════
71
+ // SPURIOUS ECHO — a short arithmetic query whose consensus climb lands on an
72
+ // UNRELATED fact by coincidental byte-pattern resonance (observed directly
73
+ // this session: "2+2 equals what?" climbs to "1+1", not "2+2"). Only a
74
+ // minority of the query's regions support it; breadth must NOT dominate.
75
+ // ═══════════════════════════════════════════════════════════════════════════
76
+
77
+ const ARITH_CORPUS = [
78
+ ["1+2", "3"],
79
+ ["2+2", "4"],
80
+ ["2+3", "5"],
81
+ ["3+3", "6"],
82
+ ["3+5", "8"],
83
+ ["4+3", "7"],
84
+ ["2+5", "7"],
85
+ ["1+5", "6"],
86
+ ["6+1", "7"],
87
+ ["4+1", "5"],
88
+ ["3+4", "7"],
89
+ ["5+2", "7"],
90
+ ["1+1", "2"],
91
+ ["5+3", "8"],
92
+ ["7+1", "8"],
93
+ ];
94
+
95
+ test("breadth: a coincidental single-region echo does not dominate", async () => {
96
+ const m = mk(1);
97
+ await m.ingest(ARITH_CORPUS);
98
+ const roots = await m.climbAttention(enc("2+2 equals what?"), 24);
99
+ assert.equal(roots.length, 1, "expected exactly one committed root");
100
+ assert.ok(
101
+ typeof roots[0].breadth === "number",
102
+ "Attention must carry a breadth field",
103
+ );
104
+ assert.ok(
105
+ roots[0].breadth <= 0.5,
106
+ `a coincidental echo must NOT dominate the query's own regions, got breadth=${
107
+ roots[0].breadth
108
+ }`,
109
+ );
110
+ await m.store.close();
111
+ });
112
+
113
+ // ═══════════════════════════════════════════════════════════════════════════
114
+ // SCALE INVARIANCE — the whole point: the SAME breadth bar must separate
115
+ // signal from noise whether the corpus has 15 facts or many more, unlike the
116
+ // raw IDF vote (which is an absolute, ln(N)-scaled quantity — see the header
117
+ // comment). Doubling the corpus (more unrelated arithmetic facts) must not
118
+ // flip either verdict merely by changing N.
119
+ // ═══════════════════════════════════════════════════════════════════════════
120
+
121
+ test("breadth: the same bar holds as the corpus grows (scale invariance)", async () => {
122
+ const bigArith = [...ARITH_CORPUS];
123
+ for (let a = 1; a <= 9; a++) {
124
+ for (let b = 1; b <= 9; b++) {
125
+ bigArith.push([`${a}x${b}`, String(a * b)]);
126
+ }
127
+ }
128
+ const m = mk(1);
129
+ await m.ingest(bigArith);
130
+ const roots = await m.climbAttention(enc("2+2 equals what?"), 24);
131
+ assert.equal(roots.length, 1, "expected exactly one committed root");
132
+ assert.ok(
133
+ roots[0].breadth <= 0.5,
134
+ `a coincidental echo must still not dominate on a larger corpus, got breadth=${
135
+ roots[0].breadth
136
+ }`,
137
+ );
138
+ await m.store.close();
139
+ });