@hviana/sema 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/AGENTS.md +6 -5
  2. package/CITATION.cff +49 -0
  3. package/HOW_IT_WORKS.md +11 -12
  4. package/README.md +7 -5
  5. package/dist/example/demo.js +25 -25
  6. package/dist/example/train_base.d.ts +19 -21
  7. package/dist/example/train_base.js +1700 -1952
  8. package/dist/src/alphabet.d.ts +3 -3
  9. package/dist/src/alphabet.js +24 -27
  10. package/dist/src/alu/src/alu.d.ts +169 -194
  11. package/dist/src/alu/src/alu.js +374 -414
  12. package/dist/src/alu/src/expr.d.ts +36 -43
  13. package/dist/src/alu/src/expr.js +252 -278
  14. package/dist/src/alu/src/index.d.ts +9 -98
  15. package/dist/src/alu/src/index.js +10 -64
  16. package/dist/src/alu/src/kernel-arith.d.ts +1 -5
  17. package/dist/src/alu/src/kernel-arith.js +195 -285
  18. package/dist/src/alu/src/kernel-bits.d.ts +1 -5
  19. package/dist/src/alu/src/kernel-bits.js +81 -115
  20. package/dist/src/alu/src/kernel-logic.js +33 -63
  21. package/dist/src/alu/src/kernel-nd.js +155 -195
  22. package/dist/src/alu/src/kernel-numeric.d.ts +16 -80
  23. package/dist/src/alu/src/kernel-numeric.js +259 -331
  24. package/dist/src/alu/src/operation.d.ts +112 -132
  25. package/dist/src/alu/src/operation.js +149 -163
  26. package/dist/src/alu/src/parser.d.ts +168 -175
  27. package/dist/src/alu/src/parser.js +382 -421
  28. package/dist/src/alu/src/resonance.d.ts +16 -25
  29. package/dist/src/alu/src/resonance.js +49 -56
  30. package/dist/src/alu/src/text.d.ts +3 -7
  31. package/dist/src/alu/src/text.js +31 -37
  32. package/dist/src/alu/src/value.d.ts +14 -14
  33. package/dist/src/alu/src/value.js +150 -160
  34. package/dist/src/alu/test/alu.test.js +543 -656
  35. package/dist/src/bytes.d.ts +1 -5
  36. package/dist/src/bytes.js +33 -40
  37. package/dist/src/config.d.ts +99 -102
  38. package/dist/src/config.js +78 -83
  39. package/dist/src/derive/src/deduction.d.ts +60 -62
  40. package/dist/src/derive/src/deduction.js +103 -109
  41. package/dist/src/derive/src/index.d.ts +1 -7
  42. package/dist/src/derive/src/priority-queue.d.ts +8 -8
  43. package/dist/src/derive/src/priority-queue.js +57 -61
  44. package/dist/src/derive/src/rewrite.d.ts +15 -18
  45. package/dist/src/derive/src/rewrite.js +59 -67
  46. package/dist/src/derive/src/trie.d.ts +56 -56
  47. package/dist/src/derive/src/trie.js +175 -179
  48. package/dist/src/derive/test/derive.test.js +96 -100
  49. package/dist/src/extension.d.ts +13 -16
  50. package/dist/src/geometry.d.ts +29 -43
  51. package/dist/src/geometry.js +324 -268
  52. package/dist/src/index.d.ts +2 -17
  53. package/dist/src/index.js +2 -12
  54. package/dist/src/ingest-cache.d.ts +25 -30
  55. package/dist/src/ingest-cache.js +108 -127
  56. package/dist/src/mind/articulation.d.ts +1 -5
  57. package/dist/src/mind/articulation.js +77 -112
  58. package/dist/src/mind/attention.d.ts +40 -108
  59. package/dist/src/mind/attention.js +754 -872
  60. package/dist/src/mind/canonical.d.ts +4 -19
  61. package/dist/src/mind/canonical.js +27 -31
  62. package/dist/src/mind/graph-search.d.ts +201 -225
  63. package/dist/src/mind/graph-search.js +742 -821
  64. package/dist/src/mind/index.d.ts +2 -10
  65. package/dist/src/mind/junction.d.ts +31 -58
  66. package/dist/src/mind/junction.js +172 -237
  67. package/dist/src/mind/learning.d.ts +16 -52
  68. package/dist/src/mind/learning.js +143 -165
  69. package/dist/src/mind/match.d.ts +20 -69
  70. package/dist/src/mind/match.js +259 -318
  71. package/dist/src/mind/mechanisms/alu.js +16 -16
  72. package/dist/src/mind/mechanisms/cast.d.ts +9 -13
  73. package/dist/src/mind/mechanisms/cast.js +363 -456
  74. package/dist/src/mind/mechanisms/confluence.d.ts +8 -12
  75. package/dist/src/mind/mechanisms/confluence.js +152 -183
  76. package/dist/src/mind/mechanisms/cover.d.ts +2 -8
  77. package/dist/src/mind/mechanisms/cover.js +148 -210
  78. package/dist/src/mind/mechanisms/extraction.d.ts +8 -34
  79. package/dist/src/mind/mechanisms/extraction.js +234 -288
  80. package/dist/src/mind/mechanisms/recall.d.ts +6 -10
  81. package/dist/src/mind/mechanisms/recall.js +141 -185
  82. package/dist/src/mind/mind.d.ts +194 -155
  83. package/dist/src/mind/mind.js +392 -289
  84. package/dist/src/mind/pipeline-mechanism.d.ts +112 -124
  85. package/dist/src/mind/pipeline-mechanism.js +161 -172
  86. package/dist/src/mind/pipeline.d.ts +4 -14
  87. package/dist/src/mind/pipeline.js +125 -189
  88. package/dist/src/mind/primitives.d.ts +20 -33
  89. package/dist/src/mind/primitives.js +124 -120
  90. package/dist/src/mind/rationale.d.ts +86 -98
  91. package/dist/src/mind/rationale.js +113 -121
  92. package/dist/src/mind/reasoning.d.ts +2 -13
  93. package/dist/src/mind/reasoning.js +129 -166
  94. package/dist/src/mind/recognition.d.ts +1 -4
  95. package/dist/src/mind/recognition.js +198 -210
  96. package/dist/src/mind/resonance.d.ts +5 -22
  97. package/dist/src/mind/resonance.js +0 -0
  98. package/dist/src/mind/trace.d.ts +6 -25
  99. package/dist/src/mind/trace.js +50 -58
  100. package/dist/src/mind/traverse.d.ts +29 -58
  101. package/dist/src/mind/traverse.js +344 -356
  102. package/dist/src/mind/types.d.ts +122 -127
  103. package/dist/src/mind/types.js +60 -69
  104. package/dist/src/rabitq-ivf/src/database.d.ts +113 -0
  105. package/dist/src/rabitq-ivf/src/database.js +201 -0
  106. package/dist/src/rabitq-ivf/src/index.d.ts +7 -0
  107. package/dist/src/{rabitq-hnsw → rabitq-ivf}/src/index.js +1 -3
  108. package/dist/src/rabitq-ivf/src/ivf.d.ts +200 -0
  109. package/dist/src/rabitq-ivf/src/ivf.js +1165 -0
  110. package/dist/src/rabitq-ivf/src/prng.d.ts +19 -0
  111. package/dist/src/rabitq-ivf/src/prng.js +36 -0
  112. package/dist/src/rabitq-ivf/src/rabitq.d.ts +95 -0
  113. package/dist/src/rabitq-ivf/src/rabitq.js +283 -0
  114. package/dist/src/sema.d.ts +9 -13
  115. package/dist/src/sema.js +26 -40
  116. package/dist/src/store-sqlite.d.ts +171 -166
  117. package/dist/src/store-sqlite.js +768 -662
  118. package/dist/src/store.d.ts +597 -630
  119. package/dist/src/store.js +1434 -1581
  120. package/dist/src/vec.d.ts +5 -9
  121. package/dist/src/vec.js +61 -73
  122. package/example/train_base.ts +33 -15
  123. package/index.html +65 -0
  124. package/package.json +1 -1
  125. package/src/alu/README.md +1 -1
  126. package/src/alu/src/index.ts +1 -1
  127. package/src/config.ts +19 -27
  128. package/src/geometry.ts +93 -1
  129. package/src/index.ts +6 -11
  130. package/src/mind/attention.ts +11 -6
  131. package/src/mind/mechanisms/recall.ts +29 -13
  132. package/src/mind/mind.ts +250 -19
  133. package/src/mind/primitives.ts +43 -6
  134. package/src/mind/recognition.ts +26 -8
  135. package/src/mind/traverse.ts +47 -0
  136. package/src/mind/types.ts +20 -21
  137. package/src/rabitq-ivf/README.md +56 -0
  138. package/src/rabitq-ivf/src/database.ts +276 -0
  139. package/src/{rabitq-hnsw → rabitq-ivf}/src/index.ts +2 -5
  140. package/src/rabitq-ivf/src/ivf.ts +1330 -0
  141. package/src/{rabitq-hnsw → rabitq-ivf}/src/prng.ts +1 -1
  142. package/src/store-sqlite.ts +204 -9
  143. package/src/store.ts +27 -34
  144. package/test/08-storage.test.mjs +3 -3
  145. package/test/13-conversation.test.mjs +190 -20
  146. package/test/14-scaling.test.mjs +2 -2
  147. package/test/35-ivf.test.mjs +263 -0
  148. package/test/36-bloom.test.mjs +123 -0
  149. package/dist/src/rabitq-hnsw/src/database.d.ts +0 -202
  150. package/dist/src/rabitq-hnsw/src/database.js +0 -405
  151. package/dist/src/rabitq-hnsw/src/heap.d.ts +0 -22
  152. package/dist/src/rabitq-hnsw/src/heap.js +0 -94
  153. package/dist/src/rabitq-hnsw/src/hnsw.d.ts +0 -125
  154. package/dist/src/rabitq-hnsw/src/hnsw.js +0 -500
  155. package/dist/src/rabitq-hnsw/src/index.d.ts +0 -15
  156. package/dist/src/rabitq-hnsw/src/prng.d.ts +0 -19
  157. package/dist/src/rabitq-hnsw/src/prng.js +0 -38
  158. package/dist/src/rabitq-hnsw/src/rabitq.d.ts +0 -95
  159. package/dist/src/rabitq-hnsw/src/rabitq.js +0 -299
  160. package/dist/src/rabitq-hnsw/src/store.d.ts +0 -162
  161. package/dist/src/rabitq-hnsw/src/store.js +0 -916
  162. package/dist/src/rabitq-hnsw/test/hnsw.test.d.ts +0 -1
  163. package/dist/src/rabitq-hnsw/test/hnsw.test.js +0 -1197
  164. package/src/rabitq-hnsw/README.md +0 -303
  165. package/src/rabitq-hnsw/src/database.ts +0 -492
  166. package/src/rabitq-hnsw/src/heap.ts +0 -90
  167. package/src/rabitq-hnsw/src/hnsw.ts +0 -514
  168. package/src/rabitq-hnsw/src/store.ts +0 -994
  169. package/src/rabitq-hnsw/test/hnsw.test.ts +0 -1213
  170. /package/src/{rabitq-hnsw → rabitq-ivf}/src/rabitq.ts +0 -0
@@ -7,19 +7,13 @@
7
7
  // not a hand-rolled tally. The result is one or more independent points of
8
8
  // attention for the rest of the pipeline to follow.
9
9
  import { isChunk } from "../sema.js";
10
- import { lightestDerivation } from "../derive/src/index.js";
10
+ import { lightestDerivation, } from "../derive/src/index.js";
11
11
  import { consensusFloor, dominates, estimatorNoise } from "../geometry.js";
12
- import { foldTree, gistOf, perceive, read } from "./primitives.js";
12
+ import { foldTree, gistOf, latin1Key, perceive, read } from "./primitives.js";
13
13
  import { recognise } from "./recognition.js";
14
14
  import { leafIdRun } from "./canonical.js";
15
15
  import { corpusN, edgeAncestors } from "./traverse.js";
16
- import {
17
- cachedRead,
18
- junctionContainersFrom,
19
- junctionSeeds,
20
- junctionSynonyms,
21
- walkCache,
22
- } from "./junction.js";
16
+ import { cachedRead, junctionContainersFrom, junctionSeeds, junctionSynonyms, walkCache, } from "./junction.js";
23
17
  import { indexOf } from "../bytes.js";
24
18
  import { rNode, traceDerivation } from "./trace.js";
25
19
  // ── Public entry points ───────────────────────────────────────────────────
@@ -27,330 +21,301 @@ import { rNode, traceDerivation } from "./trace.js";
27
21
  * resonance, pool the evidence, and return only the ROOT points of
28
22
  * attention — those that cleared commitVotes' significance floor. */
29
23
  export async function climbAttention(ctx, query, k, mode = "inverse") {
30
- return (await climbAttentionAll(ctx, query, k, mode)).roots;
24
+ return (await climbAttentionAll(ctx, query, k, mode)).roots;
31
25
  }
32
26
  /** Full read-out of one consensus climb: both the roots (dominant points of
33
27
  * attention) and the entire ranked list. Cached via ctx.climbMemo when
34
28
  * ctx.trace is null. */
35
29
  export async function climbAttentionAll(ctx, query, k, mode = "inverse") {
36
- if (ctx.climbMemo && !ctx.trace) {
37
- const key = `${k}:${mode}`;
38
- let byRead = ctx.climbMemo.get(query);
39
- if (byRead === undefined) {
40
- ctx.climbMemo.set(query, byRead = new Map());
41
- }
42
- const hit = byRead.get(key);
43
- if (hit !== undefined) {
44
- return hit;
30
+ // Content-keyed memo — works for both single-turn respond() and multi-turn
31
+ // respondTurn(). Skipped while tracing.
32
+ if (ctx.climbMemo && !ctx.trace) {
33
+ const contentKey = latin1Key(query);
34
+ const modeKey = `${k}:${mode}`;
35
+ let byRead = ctx.climbMemo.get(contentKey);
36
+ if (byRead === undefined) {
37
+ ctx.climbMemo.set(contentKey, byRead = new Map());
38
+ }
39
+ const hit = byRead.get(modeKey);
40
+ if (hit !== undefined)
41
+ return hit;
42
+ const read = await computeAttention(ctx, query, k, mode);
43
+ byRead.set(modeKey, read);
44
+ return read;
45
45
  }
46
- const read = await computeAttention(ctx, query, k, mode);
47
- byRead.set(key, read);
48
- return read;
49
- }
50
- return computeAttention(ctx, query, k, mode);
46
+ return computeAttention(ctx, query, k, mode);
51
47
  }
52
48
  // ── Pipeline ──────────────────────────────────────────────────────────────
53
49
  export async function computeAttention(ctx, query, k, mode) {
54
- const regions = collectRegions(ctx, query);
55
- // Recognised sites carry structural evidence that perceived sub-regions
56
- // miss: a word crossing a W-boundary is split into chunks whose partial
57
- // gists may not resonate distinctively, but the SITE (content-addressed,
58
- // exact) names the whole form. Adding sites as climb regions lets the
59
- // consensus vote with the full word, at zero cost — recognition is already
60
- // memoised per response (ctx.recogniseMemo), and gistOf for short sites is
61
- // O(|span|·D). Sites that overlap perceived regions add corroborating
62
- // evidence; sites in gaps (like cross-boundary words) fill them.
63
- const rec = recognise(ctx, query);
64
- for (const s of rec.sites) {
65
- regions.push({
66
- v: gistOf(ctx, query.subarray(s.start, s.end)),
67
- start: s.start,
68
- end: s.end,
69
- chunk: false,
70
- known: true, // a recognised site IS a stored form
71
- });
72
- }
73
- if (regions.length === 0) {
74
- return { roots: [], ranked: [] };
75
- }
76
- const N = corpusN(ctx);
77
- // One climb per distinct anchor for the WHOLE query: regions sharing a
78
- // chunk, and canonicalChunkId's prefix probes, all hit this memo instead of
79
- // re-reading the anchor's full edge fan-out from the store.
80
- const reachMemo = new Map();
81
- const rvs = await voteRegions(ctx, query, regions, k, mode, N, reachMemo);
82
- // ── Cross-region: DIRECT region-to-region interaction ─────────────────
83
- // Two regions whose individual climbs land on DIFFERENT contexts leave
84
- // their JOINT context — the learnt whole that contains BOTH — with no
85
- // vote. crossRegionVotes recovers it by the bridge's content-addressed
86
- // junction ascent (see the note above the function).
87
- const cross = await crossRegionVotes(
88
- ctx,
89
- query,
90
- regions,
91
- rvs,
92
- k,
93
- N,
94
- reachMemo,
95
- );
96
- // A vote SUPERSEDED by exact joint evidence (its bytes literally live
97
- // inside the joint container, yet it climbed elsewhere — grid aliasing)
98
- // is dropped, not down-weighted: the joint container explains it away.
99
- const allVotes = cross.votes.length > 0
100
- ? [
101
- ...rvs.votes.filter((v) => !cross.superseded.has(v)),
102
- ...cross.votes,
103
- ]
104
- : rvs.votes;
105
- // ──────────────────────────────────────────────────────────────────────
106
- if (allVotes.length === 0) {
107
- traceAttention(ctx, regions, rvs.voters, []);
108
- return { roots: [], ranked: [] };
109
- }
110
- const sat = detectSaturated(ctx, regions, rvs.saturated);
111
- const pooled = poolVotes(ctx, allVotes, sat, N);
112
- return commitVotes(ctx, pooled, sat, regions, rvs.voters, N);
113
- }
114
- export function collectRegions(ctx, query) {
115
- const regions = [];
116
- // A region that DOMINATES the query (covers more than half — the shared
117
- // {@link dominates} test liftAnswer uses for a span that swallows its
118
- // surroundings) can never itself discriminate between several topics the
119
- // query weaves; voting with it only when it is the sole structure (no
120
- // narrower region exists) keeps a flat/short query's single point of
121
- // attention intact without letting a broad, non-discriminative wrapper
122
- // dilute a multi-topic query's vote or masquerade as a genuine second
123
- // point of attention.
124
- // foldTree (not walkTree): the same post-order walk, but each node also
125
- // resolves content-addressed against the store — `known` is what lets the
126
- // climb keep exact evidence at full weight while margin-damping the
127
- // approximate kind (see voteRegions). One findLeaf/findBranch per tree
128
- // node, the same lookups a deposit pays.
129
- foldTree(ctx, perceive(ctx, query), 0, (n, start, end, node) => {
130
- if (n.kids === null) {
131
- return;
50
+ const regions = collectRegions(ctx, query);
51
+ // Recognised sites carry structural evidence that perceived sub-regions
52
+ // miss: a word crossing a W-boundary is split into chunks whose partial
53
+ // gists may not resonate distinctively, but the SITE (content-addressed,
54
+ // exact) names the whole form. Adding sites as climb regions lets the
55
+ // consensus vote with the full word, at zero cost — recognition is already
56
+ // memoised per response (ctx.recogniseMemo), and gistOf for short sites is
57
+ // O(|span|·D). Sites that overlap perceived regions add corroborating
58
+ // evidence; sites in gaps (like cross-boundary words) fill them.
59
+ const rec = recognise(ctx, query);
60
+ for (const s of rec.sites) {
61
+ regions.push({
62
+ v: gistOf(ctx, query.subarray(s.start, s.end)),
63
+ start: s.start,
64
+ end: s.end,
65
+ chunk: false,
66
+ known: true, // a recognised site IS a stored form
67
+ });
132
68
  }
133
- if (!dominates(end - start, query.length) || regions.length === 0) {
134
- regions.push({
135
- v: n.v,
136
- start,
137
- end,
138
- chunk: isChunk(n),
139
- known: node !== null,
140
- });
69
+ if (regions.length === 0)
70
+ return { roots: [], ranked: [] };
71
+ const N = corpusN(ctx);
72
+ // One climb per distinct anchor for the WHOLE query: regions sharing a
73
+ // chunk, and canonicalChunkId's prefix probes, all hit this memo instead of
74
+ // re-reading the anchor's full edge fan-out from the store.
75
+ const reachMemo = new Map();
76
+ const rvs = await voteRegions(ctx, query, regions, k, mode, N, reachMemo);
77
+ // ── Cross-region: DIRECT region-to-region interaction ─────────────────
78
+ // Two regions whose individual climbs land on DIFFERENT contexts leave
79
+ // their JOINT context — the learnt whole that contains BOTH — with no
80
+ // vote. crossRegionVotes recovers it by the bridge's content-addressed
81
+ // junction ascent (see the note above the function).
82
+ const cross = await crossRegionVotes(ctx, query, regions, rvs, k, N, reachMemo);
83
+ // A vote SUPERSEDED by exact joint evidence (its bytes literally live
84
+ // inside the joint container, yet it climbed elsewhere — grid aliasing)
85
+ // is dropped, not down-weighted: the joint container explains it away.
86
+ const allVotes = cross.votes.length > 0
87
+ ? [
88
+ ...rvs.votes.filter((v) => !cross.superseded.has(v)),
89
+ ...cross.votes,
90
+ ]
91
+ : rvs.votes;
92
+ // ──────────────────────────────────────────────────────────────────────
93
+ if (allVotes.length === 0) {
94
+ traceAttention(ctx, regions, rvs.voters, []);
95
+ return { roots: [], ranked: [] };
141
96
  }
142
- });
143
- return regions;
97
+ const sat = detectSaturated(ctx, regions, rvs.saturated);
98
+ const pooled = poolVotes(ctx, allVotes, sat, N);
99
+ return commitVotes(ctx, pooled, sat, regions, rvs.voters, N);
144
100
  }
145
- export async function voteRegions(ctx, query, regions, k, mode, N, reachMemo) {
146
- const regionSaturated = new Array(regions.length).fill(false);
147
- const regionVotes = [];
148
- const regionVoter = ctx.trace ? regions.map(() => null) : [];
149
- for (let ri = 0; ri < regions.length; ri++) {
150
- const { v, start, end, chunk, known } = regions[ri];
151
- // EXACT-FIRST: a chunk whose canonical anchor is content-addressed needs
152
- // no estimator identity is exact, so its score is 1 BY DEFINITION (the
153
- // estimated cosine of a form with itself, minus quantisation noise, and
154
- // the caveat atop geometry.ts forbids trusting the estimate over the
155
- // exact resolution anyway). The ANN query is deferred behind
156
- // `ensureHits` and paid only when actually consulted: the orphan
157
- // fallback, the contrastive margin (approximate regions only), or a
158
- // region with no usable canonical. On chunk-heavy queries this removes
159
- // the resonate() call for most exact regions — the single largest
160
- // remaining inference sink with the anchor choice unchanged (the
161
- // canonical branch already ignored hits[0]).
162
- const canonicalId = chunk
163
- ? canonicalChunkId(ctx, query.subarray(start, end), N, reachMemo)
164
- : null;
165
- const canonicalUsable = canonicalId !== null &&
166
- (ctx.store.hasParents(canonicalId) ||
167
- ctx.store.hasContainers(canonicalId));
168
- let hits = null;
169
- const ensureHits = async () => hits ??= await ctx.store.resonate(v, k);
170
- const canonicalFailed = chunk && canonicalId === null;
171
- let voterId;
172
- let score;
173
- let scoreId; // the node the score was measured against
174
- if (canonicalUsable) {
175
- voterId = canonicalId;
176
- score = 1;
177
- scoreId = canonicalId;
178
- } else {
179
- const h = await ensureHits();
180
- if (h.length === 0) {
181
- continue;
182
- }
183
- voterId = h[0].id;
184
- score = h[0].score;
185
- scoreId = h[0].id;
186
- }
187
- let reach = edgeAncestors(ctx, voterId, N, reachMemo);
188
- // A region's vote must not die with the TOP hit: `hits[1..k]` were
189
- // already fetched, and the top-ranked anchor being a structural orphan
190
- // (no edge-bearing ancestors) is an accident of the approximate ranking,
191
- // not evidence the region relates to nothing. Walk the remaining hits —
192
- // nearest first, climbs memoised — until one climbs. A SATURATED reach
193
- // is not an orphan: it is a deliberate abstention, kept as-is.
194
- if (reach.roots.length === 0 && !reach.saturated) {
195
- for (const h of await ensureHits()) {
196
- if (h.id === voterId) {
197
- continue;
198
- }
199
- const r2 = edgeAncestors(ctx, h.id, N, reachMemo);
200
- if (r2.saturated || r2.roots.length > 0) {
201
- ctx.trace?.step(
202
- "anchorFallback",
203
- [rNode(ctx, voterId, "orphan-anchor", score)],
204
- [rNode(ctx, h.id, "anchor", h.score)],
205
- "the top-ranked anchor climbs to no context — a lower-ranked hit votes instead",
206
- );
207
- reach = r2;
208
- voterId = h.id;
209
- score = h.score;
210
- scoreId = h.id;
211
- break;
101
+ export function collectRegions(ctx, query) {
102
+ const regions = [];
103
+ // A region that DOMINATES the query (covers more than half — the shared
104
+ // {@link dominates} test liftAnswer uses for a span that swallows its
105
+ // surroundings) can never itself discriminate between several topics the
106
+ // query weaves; voting with it only when it is the sole structure (no
107
+ // narrower region exists) keeps a flat/short query's single point of
108
+ // attention intact without letting a broad, non-discriminative wrapper
109
+ // dilute a multi-topic query's vote or masquerade as a genuine second
110
+ // point of attention.
111
+ // foldTree (not walkTree): the same post-order walk, but each node also
112
+ // resolves content-addressed against the store `known` is what lets the
113
+ // climb keep exact evidence at full weight while margin-damping the
114
+ // approximate kind (see voteRegions). One findLeaf/findBranch per tree
115
+ // node, the same lookups a deposit pays.
116
+ foldTree(ctx, perceive(ctx, query), 0, (n, start, end, node) => {
117
+ if (n.kids === null)
118
+ return;
119
+ if (!dominates(end - start, query.length) || regions.length === 0) {
120
+ regions.push({
121
+ v: n.v,
122
+ start,
123
+ end,
124
+ chunk: isChunk(n),
125
+ known: node !== null,
126
+ });
212
127
  }
213
- }
214
- } else if (!canonicalUsable && reach.saturated) {
215
- // TIE-BAND saturation fallback. A saturated top hit abstains the whole
216
- // region (a hub's reach concludes nothing) but the hub may only CLAIM
217
- // that abstention when it is DISTINGUISHABLY the nearest anchor. The
218
- // resonance ranking is an estimate: the difference between two scores
219
- // against the same query carries √2× the estimator's per-score error,
220
- // ≈ 1/√D ({@link estimatorNoise}) so any hit within that band of the
221
- // top is the SAME rank at measurement resolution, and letting the hub
222
- // win the tie decides the region by quantisation accident (observed:
223
- // a 0.1σ rank inversion flipped a pinned behaviour when the query
224
- // estimator sharpened from 4 to 8 bits). Walk the tied hits, nearest
225
- // first; the first that climbs somewhere non-saturated votes for the
226
- // region. Beyond the band the hub is genuinely nearest and its
227
- // abstention stands. A KNOWN (content-addressed) region never enters:
228
- // its anchor is exact, not an estimate.
229
- const band = estimatorNoise(ctx.store.D);
230
- for (const h of await ensureHits()) {
231
- if (h.id === voterId) {
232
- continue;
128
+ });
129
+ return regions;
130
+ }
131
+ export async function voteRegions(ctx, query, regions, k, mode, N, reachMemo) {
132
+ const regionSaturated = new Array(regions.length).fill(false);
133
+ const regionVotes = [];
134
+ const regionVoter = ctx.trace ? regions.map(() => null) : [];
135
+ for (let ri = 0; ri < regions.length; ri++) {
136
+ const { v, start, end, chunk, known } = regions[ri];
137
+ // EXACT-FIRST: a chunk whose canonical anchor is content-addressed needs
138
+ // no estimator identity is exact, so its score is 1 BY DEFINITION (the
139
+ // estimated cosine of a form with itself, minus quantisation noise, and
140
+ // the caveat atop geometry.ts forbids trusting the estimate over the
141
+ // exact resolution anyway). The ANN query is deferred behind
142
+ // `ensureHits` and paid only when actually consulted: the orphan
143
+ // fallback, the contrastive margin (approximate regions only), or a
144
+ // region with no usable canonical. On chunk-heavy queries this removes
145
+ // the resonate() call for most exact regions — the single largest
146
+ // remaining inference sink — with the anchor choice unchanged (the
147
+ // canonical branch already ignored hits[0]).
148
+ const canonicalId = chunk
149
+ ? canonicalChunkId(ctx, query.subarray(start, end), N, reachMemo)
150
+ : null;
151
+ const canonicalUsable = canonicalId !== null &&
152
+ (ctx.store.hasParents(canonicalId) ||
153
+ ctx.store.hasContainers(canonicalId));
154
+ let hits = null;
155
+ const ensureHits = async () => hits ??= await ctx.store.resonate(v, k);
156
+ const canonicalFailed = chunk && canonicalId === null;
157
+ let voterId;
158
+ let score;
159
+ let scoreId; // the node the score was measured against
160
+ if (canonicalUsable) {
161
+ voterId = canonicalId;
162
+ score = 1;
163
+ scoreId = canonicalId;
233
164
  }
234
- if (h.score < score - band) {
235
- break; // hits are nearest-first
165
+ else {
166
+ const h = await ensureHits();
167
+ if (h.length === 0)
168
+ continue;
169
+ voterId = h[0].id;
170
+ score = h[0].score;
171
+ scoreId = h[0].id;
236
172
  }
237
- const r2 = edgeAncestors(ctx, h.id, N, reachMemo);
238
- if (!r2.saturated && r2.roots.length > 0) {
239
- ctx.trace?.step(
240
- "anchorFallback",
241
- [rNode(ctx, voterId, "saturated-anchor", score)],
242
- [rNode(ctx, h.id, "anchor", h.score)],
243
- "the top-ranked anchor is a saturated hub tied within estimator noise the tied hit votes instead",
244
- );
245
- reach = r2;
246
- voterId = h.id;
247
- score = h.score;
248
- scoreId = h.id;
249
- break;
173
+ let reach = edgeAncestors(ctx, voterId, N, reachMemo);
174
+ // A region's vote must not die with the TOP hit: `hits[1..k]` were
175
+ // already fetched, and the top-ranked anchor being a structural orphan
176
+ // (no edge-bearing ancestors) is an accident of the approximate ranking,
177
+ // not evidence the region relates to nothing. Walk the remaining hits —
178
+ // nearest first, climbs memoised — until one climbs. A SATURATED reach
179
+ // is not an orphan: it is a deliberate abstention, kept as-is.
180
+ if (reach.roots.length === 0 && !reach.saturated) {
181
+ for (const h of await ensureHits()) {
182
+ if (h.id === voterId)
183
+ continue;
184
+ const r2 = edgeAncestors(ctx, h.id, N, reachMemo);
185
+ if (r2.saturated || r2.roots.length > 0) {
186
+ ctx.trace?.step("anchorFallback", [rNode(ctx, voterId, "orphan-anchor", score)], [rNode(ctx, h.id, "anchor", h.score)], "the top-ranked anchor climbs to no context — a lower-ranked hit votes instead");
187
+ reach = r2;
188
+ voterId = h.id;
189
+ score = h.score;
190
+ scoreId = h.id;
191
+ break;
192
+ }
193
+ }
250
194
  }
251
- }
252
- }
253
- regionSaturated[ri] = reach.saturated;
254
- if (reach.roots.length === 0) {
255
- continue;
256
- }
257
- if (reach.saturated) {
258
- continue;
259
- }
260
- // One IDF per region dfWeight() and the focus weight used to compute
261
- // the same logarithm independently.
262
- const idf = Math.log(N / Math.max(1, reach.contextsReached));
263
- const df = Math.log(1 + reach.contextsReached);
264
- const wf = mode === "direct" ? df : mode === "combined" ? idf + df : idf;
265
- if (wf <= 0) {
266
- continue;
267
- }
268
- // CONTRASTIVE-MARGIN GATE — the compensation the linear (byte-proportional)
269
- // fold demands, applied to APPROXIMATE evidence only. Under the linear
270
- // fold a resonance score reads "fraction of aligned shared bytes", so a
271
- // NOVEL span sharing a frame with several stored exemplars scores high
272
- // against each of them without being evidence of ANY of them: the shared
273
- // scaffolding, not the span's own content, carries the similarity. Such a
274
- // frame region resonates ~equally to every framed exemplar, so its top hit
275
- // barely beats the best DIFFERENT-conclusion rival (a different climb
276
- // root-set) — its discriminative margin, score MINUS that rival, collapses
277
- // toward zero. A region votes only when that margin clears the estimator's
278
- // own noise floor (1/√D — see {@link estimatorNoise}); below it the margin
279
- // is quantisation noise, not evidence. A KNOWN region (content-addressed,
280
- // exact) skips the contrast: it IS learnt content, not an approximation.
281
- //
282
- // The margin GATES; it does NOT scale the weight. A surviving region votes
283
- // at its genuine strength (score²·wf) — the SAME scale {@link
284
- // consensusFloor} is derived for. Using the margin as a MULTIPLIER
285
- // (score·margin) conflated "discriminative" with "strong": a genuinely
286
- // discriminative span whose frame-rival happened to score close got a tiny
287
- // vote, systematically compressing correct scaffolding-dominated groundings
288
- // (reordered / paraphrased queries) below the floor so they grounded
289
- // nothing. Gating at the noise floor keeps frame-echo suppression (a frame
290
- // region's margin ≈ 0 is gated out) without penalising honest evidence.
291
- if (!known) {
292
- let margin = score;
293
- for (const h of await ensureHits()) {
294
- if (h.id === voterId) {
295
- continue;
195
+ else if (!canonicalUsable && reach.saturated) {
196
+ // TIE-BAND saturation fallback. A saturated top hit abstains the whole
197
+ // region (a hub's reach concludes nothing) — but the hub may only CLAIM
198
+ // that abstention when it is DISTINGUISHABLY the nearest anchor. The
199
+ // resonance ranking is an estimate: the difference between two scores
200
+ // against the same query carries √2× the estimator's per-score error,
201
+ // ≈ 1/√D ({@link estimatorNoise}) — so any hit within that band of the
202
+ // top is the SAME rank at measurement resolution, and letting the hub
203
+ // win the tie decides the region by quantisation accident (observed:
204
+ // a 0.1σ rank inversion flipped a pinned behaviour when the query
205
+ // estimator sharpened from 4 to 8 bits). Walk the tied hits, nearest
206
+ // first; the first that climbs somewhere non-saturated votes for the
207
+ // region. Beyond the band the hub is genuinely nearest and its
208
+ // abstention stands. A KNOWN (content-addressed) region never enters:
209
+ // its anchor is exact, not an estimate.
210
+ const band = estimatorNoise(ctx.store.D);
211
+ for (const h of await ensureHits()) {
212
+ if (h.id === voterId)
213
+ continue;
214
+ if (h.score < score - band)
215
+ break; // hits are nearest-first
216
+ const r2 = edgeAncestors(ctx, h.id, N, reachMemo);
217
+ if (!r2.saturated && r2.roots.length > 0) {
218
+ ctx.trace?.step("anchorFallback", [rNode(ctx, voterId, "saturated-anchor", score)], [rNode(ctx, h.id, "anchor", h.score)], "the top-ranked anchor is a saturated hub tied within estimator noise — the tied hit votes instead");
219
+ reach = r2;
220
+ voterId = h.id;
221
+ score = h.score;
222
+ scoreId = h.id;
223
+ break;
224
+ }
225
+ }
296
226
  }
297
- const r2 = edgeAncestors(ctx, h.id, N, reachMemo);
298
- if (r2.saturated || r2.roots.length === 0) {
299
- continue; // concludes nothing
227
+ regionSaturated[ri] = reach.saturated;
228
+ if (reach.roots.length === 0)
229
+ continue;
230
+ if (reach.saturated)
231
+ continue;
232
+ // One IDF per region — dfWeight() and the focus weight used to compute
233
+ // the same logarithm independently.
234
+ const idf = Math.log(N / Math.max(1, reach.contextsReached));
235
+ const df = Math.log(1 + reach.contextsReached);
236
+ const wf = mode === "direct" ? df : mode === "combined" ? idf + df : idf;
237
+ if (wf <= 0)
238
+ continue;
239
+ // CONTRASTIVE-MARGIN GATE — the compensation the linear (byte-proportional)
240
+ // fold demands, applied to APPROXIMATE evidence only. Under the linear
241
+ // fold a resonance score reads "fraction of aligned shared bytes", so a
242
+ // NOVEL span sharing a frame with several stored exemplars scores high
243
+ // against each of them without being evidence of ANY of them: the shared
244
+ // scaffolding, not the span's own content, carries the similarity. Such a
245
+ // frame region resonates ~equally to every framed exemplar, so its top hit
246
+ // barely beats the best DIFFERENT-conclusion rival (a different climb
247
+ // root-set) — its discriminative margin, score MINUS that rival, collapses
248
+ // toward zero. A region votes only when that margin clears the estimator's
249
+ // own noise floor (1/√D — see {@link estimatorNoise}); below it the margin
250
+ // is quantisation noise, not evidence. A KNOWN region (content-addressed,
251
+ // exact) skips the contrast: it IS learnt content, not an approximation.
252
+ //
253
+ // The margin GATES; it does NOT scale the weight. A surviving region votes
254
+ // at its genuine strength (score²·wf) — the SAME scale {@link
255
+ // consensusFloor} is derived for. Using the margin as a MULTIPLIER
256
+ // (score·margin) conflated "discriminative" with "strong": a genuinely
257
+ // discriminative span whose frame-rival happened to score close got a tiny
258
+ // vote, systematically compressing correct scaffolding-dominated groundings
259
+ // (reordered / paraphrased queries) below the floor so they grounded
260
+ // nothing. Gating at the noise floor keeps frame-echo suppression (a frame
261
+ // region's margin ≈ 0 is gated out) without penalising honest evidence.
262
+ if (!known) {
263
+ let margin = score;
264
+ for (const h of await ensureHits()) {
265
+ if (h.id === voterId)
266
+ continue;
267
+ const r2 = edgeAncestors(ctx, h.id, N, reachMemo);
268
+ if (r2.saturated || r2.roots.length === 0)
269
+ continue; // concludes nothing
270
+ if (sameRoots(r2.roots, reach.roots))
271
+ continue; // same conclusion
272
+ margin = score - h.score; // hits are nearest-first: the best rival
273
+ break;
274
+ }
275
+ if (margin <= estimatorNoise(ctx.store.D))
276
+ continue;
300
277
  }
301
- if (sameRoots(r2.roots, reach.roots)) {
302
- continue; // same conclusion
278
+ // MUTUAL-EXPLANATION WEIGHT (angle + magnitude). Under the linear fold
279
+ // cos = shared/(‖r‖·‖h‖) with ‖·‖² = content bytes, so the old score²
280
+ // was already — implicitly — (shared/len_r)·(shared/len_h): the fraction
281
+ // of the REGION the hit explains times the fraction of the HIT the
282
+ // region pins down. Made explicit, each factor is computed from the two
283
+ // magnitudes (the region's own span; the hit's, read from the store —
284
+ // contentLen, √bytes being the linear fold's gist norm) and CAPPED at 1:
285
+ // the estimated cosine can imply more shared content than the smaller
286
+ // side even holds, and the uncapped square silently credited that
287
+ // impossible surplus — a small region echoing inside a large context, or
288
+ // the reverse, voted above its physical evidence. In the uncapped
289
+ // regime this is exactly score², the scale {@link consensusFloor} is
290
+ // derived for. (The margin gate above deliberately stays in raw cosine
291
+ // units: it tests the ESTIMATOR's noise floor, which lives in cosine
292
+ // space; converting each side by its own hit's magnitude would compare
293
+ // noise floors of different scales.)
294
+ const lenR = Math.max(1, end - start);
295
+ // Cap the magnitude read at lenR·D: past it s/ratio ≤ s/√D — below the
296
+ // estimator's own noise floor — so the mutual weight is ~0 regardless
297
+ // and the clamped value yields exactly that; no full walk of a huge hit.
298
+ const ratio = Math.sqrt(Math.max(1, ctx.store.contentLen(scoreId, lenR * ctx.store.D)) / lenR);
299
+ const mutual = Math.min(1, score * ratio) * Math.min(1, score / ratio);
300
+ const w = (mutual * wf) / reach.roots.length;
301
+ const wFocus = (mutual * idf) / reach.roots.length;
302
+ regionVotes.push({
303
+ start,
304
+ end,
305
+ canonicalFailed,
306
+ roots: reach.roots,
307
+ w,
308
+ wFocus,
309
+ });
310
+ if (ctx.trace) {
311
+ regionVoter[ri] = { id: voterId, score, w: wf };
303
312
  }
304
- margin = score - h.score; // hits are nearest-first: the best rival
305
- break;
306
- }
307
- if (margin <= estimatorNoise(ctx.store.D)) {
308
- continue;
309
- }
310
- }
311
- // MUTUAL-EXPLANATION WEIGHT (angle + magnitude). Under the linear fold
312
- // cos = shared/(‖r‖·‖h‖) with ‖·‖² = content bytes, so the old score²
313
- // was already — implicitly — (shared/len_r)·(shared/len_h): the fraction
314
- // of the REGION the hit explains times the fraction of the HIT the
315
- // region pins down. Made explicit, each factor is computed from the two
316
- // magnitudes (the region's own span; the hit's, read from the store —
317
- // contentLen, √bytes being the linear fold's gist norm) and CAPPED at 1:
318
- // the estimated cosine can imply more shared content than the smaller
319
- // side even holds, and the uncapped square silently credited that
320
- // impossible surplus — a small region echoing inside a large context, or
321
- // the reverse, voted above its physical evidence. In the uncapped
322
- // regime this is exactly score², the scale {@link consensusFloor} is
323
- // derived for. (The margin gate above deliberately stays in raw cosine
324
- // units: it tests the ESTIMATOR's noise floor, which lives in cosine
325
- // space; converting each side by its own hit's magnitude would compare
326
- // noise floors of different scales.)
327
- const lenR = Math.max(1, end - start);
328
- // Cap the magnitude read at lenR·D: past it s/ratio ≤ s/√D — below the
329
- // estimator's own noise floor — so the mutual weight is ~0 regardless
330
- // and the clamped value yields exactly that; no full walk of a huge hit.
331
- const ratio = Math.sqrt(
332
- Math.max(1, ctx.store.contentLen(scoreId, lenR * ctx.store.D)) / lenR,
333
- );
334
- const mutual = Math.min(1, score * ratio) * Math.min(1, score / ratio);
335
- const w = (mutual * wf) / reach.roots.length;
336
- const wFocus = (mutual * idf) / reach.roots.length;
337
- regionVotes.push({
338
- start,
339
- end,
340
- canonicalFailed,
341
- roots: reach.roots,
342
- w,
343
- wFocus,
344
- });
345
- if (ctx.trace) {
346
- regionVoter[ri] = { id: voterId, score, w: wf };
347
313
  }
348
- }
349
- return {
350
- votes: regionVotes,
351
- saturated: regionSaturated,
352
- voters: regionVoter,
353
- };
314
+ return {
315
+ votes: regionVotes,
316
+ saturated: regionSaturated,
317
+ voters: regionVoter,
318
+ };
354
319
  }
355
320
  /** The consensus vote as EVIDENCE POOLING, not shortest path: each surviving
356
321
  * region is an axiom; it contributes to every root it climbed to (or, for a
@@ -365,286 +330,267 @@ export async function voteRegions(ctx, query, regions, k, mode, N, reachMemo) {
365
330
  * read-outs {@link commitVotes} always gated on; only how they accumulate
366
331
  * changed. */
367
332
  export function poolVotes(ctx, regionVotes, sat, N) {
368
- const eligible = [];
369
- for (let ri = 0; ri < regionVotes.length; ri++) {
370
- const rv = regionVotes[ri];
371
- if (
372
- rv.canonicalFailed &&
373
- sat.intervals.some((iv) => rv.start >= iv.start && rv.end <= iv.end)
374
- ) {
375
- continue;
376
- }
377
- eligible.push(ri);
378
- }
379
- const key = (it) =>
380
- it.kind === "region"
381
- ? `r${it.ri}`
382
- : it.kind === "anchor"
383
- ? `a${it.id}`
384
- : `x${it.id}`;
385
- const pool = new Map();
386
- const system = {
387
- key,
388
- *axioms() {
389
- for (const ri of eligible) {
390
- yield { item: { kind: "region", ri }, cost: 0 };
391
- }
392
- },
393
- isGoal: () => false, // exhaust every axiom; there is no single goal to stop at
394
- // Every region axiom ties at cost 0, so the agenda's pop order among them
395
- // is otherwise unspecified; ordering by `ri` here only steers the HEAP
396
- // (never added to a stored cost — see relax's use of h) so pooling fires
397
- // in exactly the regionVotes array order the original loop used, byte-for-
398
- // byte reproducing its accumulation and tie-break order.
399
- heuristic: (it) => it.kind === "region" ? it.ri : 0,
400
- *rules(it) {
401
- if (it.kind !== "region") {
402
- return;
403
- }
404
- const rv = regionVotes[it.ri];
405
- // The same hub bound the rest of the system uses (edgeAncestors' parent
406
- // cutoff, chooseNext's candidate cap): a terminal answer followed by
407
- // more than √N contexts is a non-discriminative hub — spreading a
408
- // region's vote across its FULL corpus-sized fan-in yields O(corpus)
409
- // rule applications per region and near-zero per-target weight anyway.
410
- // Cap the redistribution at the first √N contexts (insertion order,
411
- // the same convention chooseNext caps by).
412
- const hubBound = Math.ceil(Math.sqrt(N));
413
- for (const r of rv.roots) {
414
- // CAPPED read: only the first hubBound targets are ever credited, so
415
- // only they are read — a common continuation's full reverse fan-in
416
- // is corpus-sized and is never materialised.
417
- const pv = ctx.store.prevFirst(r, hubBound);
418
- const isAnswer = pv.length > 0 && !ctx.store.hasNext(r);
419
- const targets = isAnswer ? pv : [r];
420
- for (const t of targets) {
421
- yield {
422
- premises: [it],
423
- conclusion: { kind: "anchor", id: t },
424
- cost: rv.w / targets.length,
425
- combine: "sum",
426
- };
427
- yield {
428
- premises: [it],
429
- conclusion: { kind: "anchorFocus", id: t },
430
- cost: rv.wFocus / targets.length,
431
- combine: "sum",
432
- };
433
- }
434
- }
435
- },
436
- pool,
437
- };
438
- lightestDerivation(system);
439
- const votes = new Map();
440
- const votesIdf = new Map();
441
- const support = new Map();
442
- const steps = [];
443
- let order = 0;
444
- for (const pc of pool.values()) {
445
- if (pc.item.kind === "anchor") {
446
- votes.set(pc.item.id, pc.cost);
447
- const premises = [];
448
- const seenRi = new Set();
449
- for (const c of pc.contributions) {
450
- const p0 = c.premises[0].item;
451
- if (p0.kind !== "region" || seenRi.has(p0.ri)) {
452
- continue;
333
+ const eligible = [];
334
+ for (let ri = 0; ri < regionVotes.length; ri++) {
335
+ const rv = regionVotes[ri];
336
+ if (rv.canonicalFailed &&
337
+ sat.intervals.some((iv) => rv.start >= iv.start && rv.end <= iv.end)) {
338
+ continue;
453
339
  }
454
- seenRi.add(p0.ri);
455
- const rv = regionVotes[p0.ri];
456
- premises.push({ kind: "form", span: [rv.start, rv.end] });
457
- }
458
- steps.push({
459
- order: order++,
460
- move: "pool-vote",
461
- premises,
462
- conclusion: { kind: "form", span: [-1, -1], node: pc.item.id },
463
- cost: pc.cost,
464
- producers: [],
465
- });
466
- } else if (pc.item.kind === "anchorFocus") {
467
- votesIdf.set(pc.item.id, pc.cost);
468
- let bestRv = null;
469
- for (const c of pc.contributions) {
470
- const p0 = c.premises[0].item;
471
- if (p0.kind !== "region") {
472
- continue;
340
+ eligible.push(ri);
341
+ }
342
+ const key = (it) => it.kind === "region"
343
+ ? `r${it.ri}`
344
+ : it.kind === "anchor"
345
+ ? `a${it.id}`
346
+ : `x${it.id}`;
347
+ const pool = new Map();
348
+ const system = {
349
+ key,
350
+ *axioms() {
351
+ for (const ri of eligible) {
352
+ yield { item: { kind: "region", ri }, cost: 0 };
353
+ }
354
+ },
355
+ isGoal: () => false, // exhaust every axiom; there is no single goal to stop at
356
+ // Every region axiom ties at cost 0, so the agenda's pop order among them
357
+ // is otherwise unspecified; ordering by `ri` here only steers the HEAP
358
+ // (never added to a stored cost — see relax's use of h) so pooling fires
359
+ // in exactly the regionVotes array order the original loop used, byte-for-
360
+ // byte reproducing its accumulation and tie-break order.
361
+ heuristic: (it) => it.kind === "region" ? it.ri : 0,
362
+ *rules(it) {
363
+ if (it.kind !== "region")
364
+ return;
365
+ const rv = regionVotes[it.ri];
366
+ // The same hub bound the rest of the system uses (edgeAncestors' parent
367
+ // cutoff, chooseNext's candidate cap): a terminal answer followed by
368
+ // more than √N contexts is a non-discriminative hub — spreading a
369
+ // region's vote across its FULL corpus-sized fan-in yields O(corpus)
370
+ // rule applications per region and near-zero per-target weight anyway.
371
+ // Cap the redistribution at the first √N contexts (insertion order,
372
+ // the same convention chooseNext caps by).
373
+ const hubBound = Math.ceil(Math.sqrt(N));
374
+ for (const r of rv.roots) {
375
+ // CAPPED read: only the first hubBound targets are ever credited, so
376
+ // only they are read — a common continuation's full reverse fan-in
377
+ // is corpus-sized and is never materialised.
378
+ const pv = ctx.store.prevFirst(r, hubBound);
379
+ const isAnswer = pv.length > 0 && !ctx.store.hasNext(r);
380
+ const targets = isAnswer ? pv : [r];
381
+ for (const t of targets) {
382
+ yield {
383
+ premises: [it],
384
+ conclusion: { kind: "anchor", id: t },
385
+ cost: rv.w / targets.length,
386
+ combine: "sum",
387
+ };
388
+ yield {
389
+ premises: [it],
390
+ conclusion: { kind: "anchorFocus", id: t },
391
+ cost: rv.wFocus / targets.length,
392
+ combine: "sum",
393
+ };
394
+ }
395
+ }
396
+ },
397
+ pool,
398
+ };
399
+ lightestDerivation(system);
400
+ const votes = new Map();
401
+ const votesIdf = new Map();
402
+ const support = new Map();
403
+ const steps = [];
404
+ let order = 0;
405
+ for (const pc of pool.values()) {
406
+ if (pc.item.kind === "anchor") {
407
+ votes.set(pc.item.id, pc.cost);
408
+ const premises = [];
409
+ const seenRi = new Set();
410
+ for (const c of pc.contributions) {
411
+ const p0 = c.premises[0].item;
412
+ if (p0.kind !== "region" || seenRi.has(p0.ri))
413
+ continue;
414
+ seenRi.add(p0.ri);
415
+ const rv = regionVotes[p0.ri];
416
+ premises.push({ kind: "form", span: [rv.start, rv.end] });
417
+ }
418
+ steps.push({
419
+ order: order++,
420
+ move: "pool-vote",
421
+ premises,
422
+ conclusion: { kind: "form", span: [-1, -1], node: pc.item.id },
423
+ cost: pc.cost,
424
+ producers: [],
425
+ });
473
426
  }
474
- const rv = regionVotes[p0.ri];
475
- if (!bestRv || rv.wFocus > bestRv.wFocus) {
476
- bestRv = rv;
427
+ else if (pc.item.kind === "anchorFocus") {
428
+ votesIdf.set(pc.item.id, pc.cost);
429
+ let bestRv = null;
430
+ for (const c of pc.contributions) {
431
+ const p0 = c.premises[0].item;
432
+ if (p0.kind !== "region")
433
+ continue;
434
+ const rv = regionVotes[p0.ri];
435
+ if (!bestRv || rv.wFocus > bestRv.wFocus)
436
+ bestRv = rv;
437
+ }
438
+ if (bestRv) {
439
+ support.set(pc.item.id, {
440
+ start: bestRv.start,
441
+ end: bestRv.end,
442
+ w: bestRv.wFocus,
443
+ });
444
+ }
477
445
  }
478
- }
479
- if (bestRv) {
480
- support.set(pc.item.id, {
481
- start: bestRv.start,
482
- end: bestRv.end,
483
- w: bestRv.wFocus,
484
- });
485
- }
486
446
  }
487
- }
488
- return { votes, votesIdf, support, steps };
447
+ return { votes, votesIdf, support, steps };
489
448
  }
490
449
  export function commitVotes(ctx, pooled, sat, regions, regionVoter, N) {
491
- const { votes, votesIdf, support, steps } = pooled;
492
- if (votes.size === 0) {
493
- traceAttention(ctx, regions, regionVoter, [], steps);
494
- return { roots: [], ranked: [] };
495
- }
496
- const ranked = [...votes.entries()]
497
- .map(([anchor, vote]) => {
498
- const s = support.get(anchor);
499
- return { anchor, vote, start: s.start, end: s.end };
450
+ const { votes, votesIdf, support, steps } = pooled;
451
+ if (votes.size === 0) {
452
+ traceAttention(ctx, regions, regionVoter, [], steps);
453
+ return { roots: [], ranked: [] };
454
+ }
455
+ const ranked = [...votes.entries()]
456
+ .map(([anchor, vote]) => {
457
+ const s = support.get(anchor);
458
+ return { anchor, vote, start: s.start, end: s.end };
500
459
  })
501
- .sort((a, b) => b.vote - a.vote);
502
- const overlaps = (a, b) => a.start < b.end && b.start < a.end;
503
- const idfDesc = [...votesIdf.values()].sort((a, b) => b - a);
504
- const rootCut = naturalBreak(idfDesc);
505
- // A FURTHER point of attention (beyond the dominant one, which always
506
- // grounds) must clear the same absolute significance floor
507
- // recallByResonance trusts a climb anchor with — log(N) + 1/2, three-ish
508
- // halvings of confidence above pure chance at this corpus scale — not
509
- // merely beat whatever its immediate neighbour in the ratio happens to be.
510
- // Without it, naturalBreak's ratio is scale-free but not FLOOR-free: on a
511
- // large, topic-diverse corpus the steepest ratio in a long noise tail can
512
- // sit far below any real signal, admitting scaffolding echoes as if they
513
- // were genuine further topics.
514
- const floor = consensusFloor(N);
515
- const placed = [];
516
- const roots = [];
517
- for (const point of ranked) {
518
- const absorbed = placed.some((p) => overlaps(point, p));
519
- if (!absorbed) {
520
- const pastLeading = !sat.hasLeading ||
521
- roots.length === 0 || point.start >= sat.leadingEnd;
522
- const vote = votesIdf.get(point.anchor) ?? 0;
523
- if (
524
- (roots.length === 0 || (vote >= rootCut && vote >= floor)) &&
525
- pastLeading
526
- ) {
527
- roots.push(point);
528
- } else {
529
- continue;
530
- }
460
+ .sort((a, b) => b.vote - a.vote);
461
+ const overlaps = (a, b) => a.start < b.end && b.start < a.end;
462
+ const idfDesc = [...votesIdf.values()].sort((a, b) => b - a);
463
+ const rootCut = naturalBreak(idfDesc);
464
+ // A FURTHER point of attention (beyond the dominant one, which always
465
+ // grounds) must clear the same absolute significance floor
466
+ // recallByResonance trusts a climb anchor with — log(N) + 1/2, three-ish
467
+ // halvings of confidence above pure chance at this corpus scale — not
468
+ // merely beat whatever its immediate neighbour in the ratio happens to be.
469
+ // Without it, naturalBreak's ratio is scale-free but not FLOOR-free: on a
470
+ // large, topic-diverse corpus the steepest ratio in a long noise tail can
471
+ // sit far below any real signal, admitting scaffolding echoes as if they
472
+ // were genuine further topics.
473
+ const floor = consensusFloor(N);
474
+ const placed = [];
475
+ const roots = [];
476
+ for (const point of ranked) {
477
+ const absorbed = placed.some((p) => overlaps(point, p));
478
+ if (!absorbed) {
479
+ const pastLeading = !sat.hasLeading ||
480
+ roots.length === 0 || point.start >= sat.leadingEnd;
481
+ const vote = votesIdf.get(point.anchor) ?? 0;
482
+ if ((roots.length === 0 || (vote >= rootCut && vote >= floor)) &&
483
+ pastLeading) {
484
+ roots.push(point);
485
+ }
486
+ else {
487
+ continue;
488
+ }
489
+ }
490
+ placed.push(point);
531
491
  }
532
- placed.push(point);
533
- }
534
- traceAttention(ctx, regions, regionVoter, roots, steps);
535
- return { roots, ranked };
492
+ traceAttention(ctx, regions, regionVoter, roots, steps);
493
+ return { roots, ranked };
536
494
  }
537
495
  export function detectSaturated(ctx, regions, saturated) {
538
- // Intervals are built from CHUNK regions only. collectRegions emits the
539
- // tree in POST-ORDER — a parent region arrives AFTER its children and
540
- // shares its first child's `start` — so the raw array is not monotone in
541
- // byte position, and a saturated parent would fuse with a later saturated
542
- // chunk into an interval swallowing a NON-saturated child. Chunk regions
543
- // (leaf-parents) are disjoint and already in byte order, and saturation
544
- // masking exists to drop canonicalFailed CHUNK votes (see poolVotes), so
545
- // chunks are both the sufficient and the safe basis. A region without a
546
- // `chunk` flag (a bare {start,end} from a direct caller) is treated as a
547
- // chunk.
548
- const intervals = [];
549
- let intStart = -1;
550
- let intEnd = -1;
551
- let totalLen = 0;
552
- for (let ri = 0; ri < regions.length; ri++) {
553
- const r = regions[ri];
554
- totalLen = Math.max(totalLen, r.end);
555
- if (r.chunk === false) {
556
- continue;
496
+ // Intervals are built from CHUNK regions only. collectRegions emits the
497
+ // tree in POST-ORDER — a parent region arrives AFTER its children and
498
+ // shares its first child's `start` — so the raw array is not monotone in
499
+ // byte position, and a saturated parent would fuse with a later saturated
500
+ // chunk into an interval swallowing a NON-saturated child. Chunk regions
501
+ // (leaf-parents) are disjoint and already in byte order, and saturation
502
+ // masking exists to drop canonicalFailed CHUNK votes (see poolVotes), so
503
+ // chunks are both the sufficient and the safe basis. A region without a
504
+ // `chunk` flag (a bare {start,end} from a direct caller) is treated as a
505
+ // chunk.
506
+ const intervals = [];
507
+ let intStart = -1;
508
+ let intEnd = -1;
509
+ let totalLen = 0;
510
+ for (let ri = 0; ri < regions.length; ri++) {
511
+ const r = regions[ri];
512
+ totalLen = Math.max(totalLen, r.end);
513
+ if (r.chunk === false)
514
+ continue;
515
+ if (saturated[ri]) {
516
+ if (intStart === -1)
517
+ intStart = r.start;
518
+ intEnd = r.end;
519
+ }
520
+ else {
521
+ if (intStart !== -1) {
522
+ intervals.push({ start: intStart, end: intEnd });
523
+ intStart = -1;
524
+ }
525
+ }
557
526
  }
558
- if (saturated[ri]) {
559
- if (intStart === -1) {
560
- intStart = r.start;
561
- }
562
- intEnd = r.end;
563
- } else {
564
- if (intStart !== -1) {
527
+ if (intStart !== -1) {
565
528
  intervals.push({ start: intStart, end: intEnd });
566
- intStart = -1;
567
- }
568
529
  }
569
- }
570
- if (intStart !== -1) {
571
- intervals.push({ start: intStart, end: intEnd });
572
- }
573
- const leading = intervals.length > 0 && intervals[0].start === 0
574
- ? intervals[0]
575
- : null;
576
- const hasLeading = leading !== null &&
577
- leading.end >= ctx.space.maxGroup &&
578
- leading.end < totalLen;
579
- const leadingEnd = leading !== null ? leading.end : 0;
580
- return { leadingEnd, hasLeading, intervals };
530
+ const leading = intervals.length > 0 && intervals[0].start === 0
531
+ ? intervals[0]
532
+ : null;
533
+ const hasLeading = leading !== null &&
534
+ leading.end >= ctx.space.maxGroup &&
535
+ leading.end < totalLen;
536
+ const leadingEnd = leading !== null ? leading.end : 0;
537
+ return { leadingEnd, hasLeading, intervals };
581
538
  }
582
539
  /** Set equality of two climb root lists (the "same conclusion" test the
583
540
  * contrastive margin skips rivals by). */
584
541
  function sameRoots(a, b) {
585
- if (a.length !== b.length) {
586
- return false;
587
- }
588
- const s = new Set(a);
589
- for (const x of b) {
590
- if (!s.has(x)) {
591
- return false;
592
- }
593
- }
594
- return true;
542
+ if (a.length !== b.length)
543
+ return false;
544
+ const s = new Set(a);
545
+ for (const x of b)
546
+ if (!s.has(x))
547
+ return false;
548
+ return true;
595
549
  }
596
550
  export function canonicalChunkId(ctx, regionBytes, N, reachMemo) {
597
- const len = Math.min(regionBytes.length, ctx.space.maxGroup);
598
- for (let off = 0; off + len <= regionBytes.length; off++) {
599
- const ids = leafIdRun(ctx, regionBytes, off, off + len);
600
- if (ids === null) {
601
- return null;
602
- }
603
- const flatId = ctx.store.findBranch(ids);
604
- if (flatId === null) {
605
- continue;
606
- }
607
- if (len < 2) {
608
- return flatId;
609
- }
610
- let bestId = flatId;
611
- let bestReach = edgeAncestors(ctx, flatId, N, reachMemo);
612
- for (let k2 = 1; k2 < len; k2++) {
613
- const shortIds = ids.slice(0, len - k2);
614
- const shortId = ctx.store.findBranch(shortIds);
615
- if (shortId === null) {
616
- continue;
617
- }
618
- const shortReach = edgeAncestors(ctx, shortId, N, reachMemo);
619
- if (
620
- shortReach.saturated ||
621
- shortReach.contextsReached > bestReach.contextsReached
622
- ) {
623
- bestId = shortId;
624
- bestReach = shortReach;
625
- }
551
+ const len = Math.min(regionBytes.length, ctx.space.maxGroup);
552
+ for (let off = 0; off + len <= regionBytes.length; off++) {
553
+ const ids = leafIdRun(ctx, regionBytes, off, off + len);
554
+ if (ids === null)
555
+ return null;
556
+ const flatId = ctx.store.findBranch(ids);
557
+ if (flatId === null)
558
+ continue;
559
+ if (len < 2)
560
+ return flatId;
561
+ let bestId = flatId;
562
+ let bestReach = edgeAncestors(ctx, flatId, N, reachMemo);
563
+ for (let k2 = 1; k2 < len; k2++) {
564
+ const shortIds = ids.slice(0, len - k2);
565
+ const shortId = ctx.store.findBranch(shortIds);
566
+ if (shortId === null)
567
+ continue;
568
+ const shortReach = edgeAncestors(ctx, shortId, N, reachMemo);
569
+ if (shortReach.saturated ||
570
+ shortReach.contextsReached > bestReach.contextsReached) {
571
+ bestId = shortId;
572
+ bestReach = shortReach;
573
+ }
574
+ }
575
+ return bestId;
626
576
  }
627
- return bestId;
628
- }
629
- return null;
577
+ return null;
630
578
  }
631
579
  export function naturalBreak(votes) {
632
- if (votes.length <= 1) {
633
- return votes[0] ?? 0;
634
- }
635
- let breakAt = 1;
636
- let steepest = Infinity;
637
- for (let i = 1; i < votes.length; i++) {
638
- if (votes[i - 1] <= 0) {
639
- break;
640
- }
641
- const ratio = votes[i] / votes[i - 1];
642
- if (ratio < steepest) {
643
- steepest = ratio;
644
- breakAt = i;
580
+ if (votes.length <= 1)
581
+ return votes[0] ?? 0;
582
+ let breakAt = 1;
583
+ let steepest = Infinity;
584
+ for (let i = 1; i < votes.length; i++) {
585
+ if (votes[i - 1] <= 0)
586
+ break;
587
+ const ratio = votes[i] / votes[i - 1];
588
+ if (ratio < steepest) {
589
+ steepest = ratio;
590
+ breakAt = i;
591
+ }
645
592
  }
646
- }
647
- return votes[breakAt - 1];
593
+ return votes[breakAt - 1];
648
594
  }
649
595
  // ═══════════════════════════════════════════════════════════════════════════
650
596
  // Cross-region attention — DIRECT region-to-region interaction.
@@ -703,314 +649,250 @@ export function naturalBreak(votes) {
703
649
  // container does not hold (a genuine second topic) are untouched.
704
650
  // ═══════════════════════════════════════════════════════════════════════════
705
651
  async function crossRegionVotes(ctx, query, regions, rvs, k, N, reachMemo) {
706
- // Candidate regions: every region that ALREADY CAST ITS OWN VOTE in
707
- // voteRegions — individually idf > 0, genuinely discriminative on its own,
708
- // just not necessarily for the SAME context as its partner. This is the
709
- // exact shape of the binding problem: "red" alone votes for `red square`,
710
- // "circle" alone for `circle` — each independently informative, disagreeing
711
- // on the conclusion — and only their CONJUNCTION resolves to the one
712
- // context, `red circle`, that actually holds both.
713
- //
714
- // A region that never voted (idf == 0 — e.g. a repeated system-prompt
715
- // prefix shared by every deposit) carries NO individual signal, and must be
716
- // excluded here too: ascending from a non-discriminative fragment's seeds
717
- // can still land on some deeper, incidentally-unique DESCENDANT container —
718
- // its rarity would come entirely from context OUTSIDE the fragments
719
- // actually composed, manufacturing confidence the query gave no reason to
720
- // have. Requiring a prior individual vote is the same discipline the noise
721
- // drop already applies to single regions, extended to compositions — with
722
- // one graded relaxation: a KNOWN region that did NOT vote (saturated, or
723
- // idf ≤ 0) may still serve as the WEAK side of a pair whose other side DID
724
- // vote. Saturation is an abstention about where the region CLIMBS; its
725
- // content-addressed identity is still exact, and the junction asks a
726
- // different question — "which whole holds both?" — whose conclusion the
727
- // container's own idf gate below still guards. Two non-voting regions
728
- // never pair (that is exactly the shared-prefix trap above), so at least
729
- // one side is always individually discriminative.
730
- //
731
- // Only MAXIMAL spans compose: a span contained in another candidate is a
732
- // fragment of that candidate's evidence, never independent of it.
733
- const votedSpans = new Set();
734
- for (const rv of rvs.votes) {
735
- votedSpans.add(`${rv.start},${rv.end}`);
736
- }
737
- const seen = new Set();
738
- const eligible = [];
739
- const strong = new Set();
740
- for (let ri = 0; ri < regions.length; ri++) {
741
- const r = regions[ri];
742
- const key = `${r.start},${r.end}`;
743
- const isStrong = votedSpans.has(key);
744
- if ((!isStrong && !r.known) || seen.has(key)) {
745
- continue;
746
- }
747
- seen.add(key);
748
- eligible.push(ri);
749
- if (isStrong) {
750
- strong.add(ri);
751
- }
752
- }
753
- const cand = eligible.filter((x) =>
754
- !eligible.some((y) =>
755
- y !== x &&
756
- regions[y].start <= regions[x].start &&
757
- regions[x].end <= regions[y].end &&
758
- regions[y].end - regions[y].start > regions[x].end - regions[x].start
759
- )
760
- );
761
- const none = { votes: [], superseded: new Set() };
762
- if (cand.length < 2) {
763
- return none;
764
- }
765
- cand.sort((x, y) =>
766
- regions[x].start - regions[y].start || regions[x].end - regions[y].end
767
- );
768
- const dec = (b) => new TextDecoder().decode(b).replace(/\s+/g, " ").trim();
769
- const cache = walkCache(ctx);
770
- // One junctionSeeds per candidate for the WHOLE pairing loop — a candidate
771
- // recurs in up to |cand|−1 pairs, and its seeds are a pure function of its
772
- // bytes.
773
- const seedsMemo = new Map();
774
- const seedsOf = (ri) => {
775
- let s = seedsMemo.get(ri);
776
- if (s === undefined) {
777
- const r = regions[ri];
778
- s = junctionSeeds(ctx, query.subarray(r.start, r.end));
779
- seedsMemo.set(ri, s);
780
- }
781
- return s;
782
- };
783
- const overlapsSpan = (e, s) => e.start < s.end && s.start < e.end;
784
- const out = [];
785
- const superseded = new Set();
786
- // A candidate consumed by one junction does not seed another: its evidence
787
- // is already composed at full joint strength, and re-pairing it would vote
788
- // the same container (or a sub-container of it) twice.
789
- const consumed = new Set();
790
- let probes = 0;
791
- for (let a = 0; a < cand.length && probes < k; a++) {
792
- if (consumed.has(cand[a])) {
793
- continue;
652
+ // Candidate regions: every region that ALREADY CAST ITS OWN VOTE in
653
+ // voteRegions — individually idf > 0, genuinely discriminative on its own,
654
+ // just not necessarily for the SAME context as its partner. This is the
655
+ // exact shape of the binding problem: "red" alone votes for `red square`,
656
+ // "circle" alone for `circle` — each independently informative, disagreeing
657
+ // on the conclusion — and only their CONJUNCTION resolves to the one
658
+ // context, `red circle`, that actually holds both.
659
+ //
660
+ // A region that never voted (idf == 0 — e.g. a repeated system-prompt
661
+ // prefix shared by every deposit) carries NO individual signal, and must be
662
+ // excluded here too: ascending from a non-discriminative fragment's seeds
663
+ // can still land on some deeper, incidentally-unique DESCENDANT container —
664
+ // its rarity would come entirely from context OUTSIDE the fragments
665
+ // actually composed, manufacturing confidence the query gave no reason to
666
+ // have. Requiring a prior individual vote is the same discipline the noise
667
+ // drop already applies to single regions, extended to compositions — with
668
+ // one graded relaxation: a KNOWN region that did NOT vote (saturated, or
669
+ // idf ≤ 0) may still serve as the WEAK side of a pair whose other side DID
670
+ // vote. Saturation is an abstention about where the region CLIMBS; its
671
+ // content-addressed identity is still exact, and the junction asks a
672
+ // different question — "which whole holds both?" — whose conclusion the
673
+ // container's own idf gate below still guards. Two non-voting regions
674
+ // never pair (that is exactly the shared-prefix trap above), so at least
675
+ // one side is always individually discriminative.
676
+ //
677
+ // Only MAXIMAL spans compose: a span contained in another candidate is a
678
+ // fragment of that candidate's evidence, never independent of it.
679
+ const votedSpans = new Set();
680
+ for (const rv of rvs.votes)
681
+ votedSpans.add(`${rv.start},${rv.end}`);
682
+ const seen = new Set();
683
+ const eligible = [];
684
+ const strong = new Set();
685
+ for (let ri = 0; ri < regions.length; ri++) {
686
+ const r = regions[ri];
687
+ const key = `${r.start},${r.end}`;
688
+ const isStrong = votedSpans.has(key);
689
+ if ((!isStrong && !r.known) || seen.has(key))
690
+ continue;
691
+ seen.add(key);
692
+ eligible.push(ri);
693
+ if (isStrong)
694
+ strong.add(ri);
794
695
  }
795
- const ra = regions[cand[a]];
796
- for (let b = a + 1; b < cand.length && probes < k; b++) {
797
- if (consumed.has(cand[b])) {
798
- continue;
799
- }
800
- const rb = regions[cand[b]];
801
- if (!strong.has(cand[a]) && !strong.has(cand[b])) {
802
- continue;
803
- }
804
- if (ra.end >= rb.start) {
805
- continue; // overlap or adjacentnothing between
806
- }
807
- // A single KNOWN region covering both: the whole form is already a
808
- // stored identity that votes directly; its pieces add nothing.
809
- if (
810
- regions.some((r) => r.known && r.start <= ra.start && rb.end <= r.end)
811
- ) {
812
- continue;
813
- }
814
- probes++;
815
- const left = query.subarray(ra.start, ra.end);
816
- const right = query.subarray(rb.start, rb.end);
817
- // Phrase-scale contract, exactly as the bridge: the glue between the two
818
- // forms may be up to W× the content it joins.
819
- const maxInterior = (left.length + right.length) * ctx.space.maxGroup;
820
- const cap = left.length + right.length + maxInterior;
821
- // Tier 1 — exact containers (both forms as substrings, either order, by
822
- // DAG ascent). Exact evidence first; only falls through to synonyms
823
- // when the exact ascent finds nothing — the SAME graded ladder the
824
- // bridge uses.
825
- let containers = junctionContainersFrom(
826
- ctx,
827
- left,
828
- right,
829
- cap,
830
- seedsOf(cand[a]),
831
- seedsOf(cand[b]),
832
- undefined,
833
- true,
834
- );
835
- if (containers.length === 0) {
836
- // Tier 2.5 — synonym containers (halo sibling of one side + the other).
837
- containers = await junctionSynonyms(
838
- ctx,
839
- left,
840
- right,
841
- maxInterior,
842
- true,
843
- );
844
- }
845
- if (containers.length === 0) {
846
- continue;
847
- }
848
- // N-ARY selection: the container covering the MOST remaining candidate
849
- // forms wins (then tightest interior, then lowest id). Reads are
850
- // cache hits — every container's bytes were already read by the walk.
851
- //
852
- // SELF-EVIDENCE GUARD: a junction is BINDING evidence only when the
853
- // container joins forms the query mentions APART. When the container's
854
- // own joined occurrence (left..right including its interior) is a
855
- // literal substring of the query, the query already spells that phrase
856
- // out contiguously — perception already voted with it, and grid shards
857
- // of one phrase pairing "around" a gap chunk would merely rediscover
858
- // the phrase they are shards of, then explain away its rivals.
859
- let best = null;
860
- let bestExtras = [];
861
- let bestCov = -1;
862
- for (const c of containers) {
863
- const bytes = cachedRead(ctx, cache, c.id, cap);
864
- const li = indexOf(bytes, left, 0);
865
- const ri = indexOf(bytes, right, 0);
866
- if (li >= 0 && ri >= 0) {
867
- const joined = bytes.subarray(
868
- Math.min(li, ri),
869
- Math.max(li + left.length, ri + right.length),
870
- );
871
- if (indexOf(query, joined, 0) >= 0) {
872
- continue; // query says it itself
873
- }
696
+ const cand = eligible.filter((x) => !eligible.some((y) => y !== x &&
697
+ regions[y].start <= regions[x].start &&
698
+ regions[x].end <= regions[y].end &&
699
+ regions[y].end - regions[y].start > regions[x].end - regions[x].start));
700
+ const none = { votes: [], superseded: new Set() };
701
+ if (cand.length < 2)
702
+ return none;
703
+ cand.sort((x, y) => regions[x].start - regions[y].start || regions[x].end - regions[y].end);
704
+ const dec = (b) => new TextDecoder().decode(b).replace(/\s+/g, " ").trim();
705
+ const cache = walkCache(ctx);
706
+ // One junctionSeeds per candidate for the WHOLE pairing loop a candidate
707
+ // recurs in up to |cand|−1 pairs, and its seeds are a pure function of its
708
+ // bytes.
709
+ const seedsMemo = new Map();
710
+ const seedsOf = (ri) => {
711
+ let s = seedsMemo.get(ri);
712
+ if (s === undefined) {
713
+ const r = regions[ri];
714
+ s = junctionSeeds(ctx, query.subarray(r.start, r.end));
715
+ seedsMemo.set(ri, s);
874
716
  }
875
- let cov = left.length + right.length;
876
- const extras = [];
877
- for (const ei of cand) {
878
- if (ei === cand[a] || ei === cand[b] || consumed.has(ei)) {
717
+ return s;
718
+ };
719
+ const overlapsSpan = (e, s) => e.start < s.end && s.start < e.end;
720
+ const out = [];
721
+ const superseded = new Set();
722
+ // A candidate consumed by one junction does not seed another: its evidence
723
+ // is already composed at full joint strength, and re-pairing it would vote
724
+ // the same container (or a sub-container of it) twice.
725
+ const consumed = new Set();
726
+ let probes = 0;
727
+ for (let a = 0; a < cand.length && probes < k; a++) {
728
+ if (consumed.has(cand[a]))
879
729
  continue;
880
- }
881
- const e = regions[ei];
882
- if (overlapsSpan(e, ra) || overlapsSpan(e, rb)) {
883
- continue;
884
- }
885
- const eb = query.subarray(e.start, e.end);
886
- if (indexOf(bytes, eb, 0) >= 0) {
887
- extras.push(ei);
888
- cov += eb.length;
889
- }
890
- }
891
- if (
892
- cov > bestCov ||
893
- (cov === bestCov && best !== null &&
894
- (c.interior.length < best.interior.length ||
895
- (c.interior.length === best.interior.length && c.id < best.id)))
896
- ) {
897
- best = c;
898
- bestExtras = extras;
899
- bestCov = cov;
900
- }
901
- }
902
- if (best === null) {
903
- continue; // every container was self-evidence
904
- }
905
- const reach = edgeAncestors(ctx, best.id, N, reachMemo);
906
- if (reach.saturated || reach.roots.length === 0) {
907
- continue;
908
- }
909
- const idf = Math.log(N / Math.max(1, reach.contextsReached));
910
- if (idf <= 0) {
911
- continue;
912
- }
913
- // EXACT joint evidence (score = 1): the container literally contains
914
- // every composed form. Mutual-explanation weight over their COMBINED
915
- // byte length the same magnitude reading voteRegions uses, here with
916
- // the estimator collapsed to certainty, so mutual = min(ratio, 1/ratio).
917
- const lenR = Math.max(1, bestCov);
918
- const ratio = Math.sqrt(
919
- Math.max(1, ctx.store.contentLen(best.id, lenR * ctx.store.D)) / lenR,
920
- );
921
- const mutual = Math.min(1, ratio) * Math.min(1, 1 / ratio);
922
- const w = (mutual * idf) / reach.roots.length;
923
- let spanStart = ra.start;
924
- let spanEnd = rb.end;
925
- for (const ei of bestExtras) {
926
- spanStart = Math.min(spanStart, regions[ei].start);
927
- spanEnd = Math.max(spanEnd, regions[ei].end);
928
- }
929
- out.push({
930
- start: spanStart,
931
- end: spanEnd,
932
- canonicalFailed: false, // content-addressed: never saturation-masked
933
- roots: reach.roots,
934
- w,
935
- wFocus: w,
936
- });
937
- consumed.add(cand[a]);
938
- consumed.add(cand[b]);
939
- for (const ei of bestExtras) {
940
- consumed.add(ei);
941
- }
942
- // EXPLAINING AWAY see the block comment above the function. Byte
943
- // containment in the joint container is the relatedness test (the
944
- // vote's bytes are literally part of the learnt whole), and FULL root
945
- // disjointness is the disagreement test: a vote sharing even one root
946
- // with the junction corroborates it and keeps its say elsewhere.
947
- const containerBytes = cachedRead(ctx, cache, best.id, cap);
948
- const jointRoots = new Set(reach.roots);
949
- for (const rv of rvs.votes) {
950
- if (rv.roots.some((r) => jointRoots.has(r))) {
951
- continue;
730
+ const ra = regions[cand[a]];
731
+ for (let b = a + 1; b < cand.length && probes < k; b++) {
732
+ if (consumed.has(cand[b]))
733
+ continue;
734
+ const rb = regions[cand[b]];
735
+ if (!strong.has(cand[a]) && !strong.has(cand[b]))
736
+ continue;
737
+ if (ra.end >= rb.start)
738
+ continue; // overlap or adjacent — nothing between
739
+ // A single KNOWN region covering both: the whole form is already a
740
+ // stored identity that votes directly; its pieces add nothing.
741
+ if (regions.some((r) => r.known && r.start <= ra.start && rb.end <= r.end))
742
+ continue;
743
+ probes++;
744
+ const left = query.subarray(ra.start, ra.end);
745
+ const right = query.subarray(rb.start, rb.end);
746
+ // Phrase-scale contract, exactly as the bridge: the glue between the two
747
+ // forms may be up to W× the content it joins.
748
+ const maxInterior = (left.length + right.length) * ctx.space.maxGroup;
749
+ const cap = left.length + right.length + maxInterior;
750
+ // Tier 1 — exact containers (both forms as substrings, either order, by
751
+ // DAG ascent). Exact evidence first; only falls through to synonyms
752
+ // when the exact ascent finds nothing — the SAME graded ladder the
753
+ // bridge uses.
754
+ let containers = junctionContainersFrom(ctx, left, right, cap, seedsOf(cand[a]), seedsOf(cand[b]), undefined, true);
755
+ if (containers.length === 0) {
756
+ // Tier 2.5 synonym containers (halo sibling of one side + the other).
757
+ containers = await junctionSynonyms(ctx, left, right, maxInterior, true);
758
+ }
759
+ if (containers.length === 0)
760
+ continue;
761
+ // N-ARY selection: the container covering the MOST remaining candidate
762
+ // forms wins (then tightest interior, then lowest id). Reads are
763
+ // cache hits every container's bytes were already read by the walk.
764
+ //
765
+ // SELF-EVIDENCE GUARD: a junction is BINDING evidence only when the
766
+ // container joins forms the query mentions APART. When the container's
767
+ // own joined occurrence (left..right including its interior) is a
768
+ // literal substring of the query, the query already spells that phrase
769
+ // out contiguously perception already voted with it, and grid shards
770
+ // of one phrase pairing "around" a gap chunk would merely rediscover
771
+ // the phrase they are shards of, then explain away its rivals.
772
+ let best = null;
773
+ let bestExtras = [];
774
+ let bestCov = -1;
775
+ for (const c of containers) {
776
+ const bytes = cachedRead(ctx, cache, c.id, cap);
777
+ const li = indexOf(bytes, left, 0);
778
+ const ri = indexOf(bytes, right, 0);
779
+ if (li >= 0 && ri >= 0) {
780
+ const joined = bytes.subarray(Math.min(li, ri), Math.max(li + left.length, ri + right.length));
781
+ if (indexOf(query, joined, 0) >= 0)
782
+ continue; // query says it itself
783
+ }
784
+ let cov = left.length + right.length;
785
+ const extras = [];
786
+ for (const ei of cand) {
787
+ if (ei === cand[a] || ei === cand[b] || consumed.has(ei))
788
+ continue;
789
+ const e = regions[ei];
790
+ if (overlapsSpan(e, ra) || overlapsSpan(e, rb))
791
+ continue;
792
+ const eb = query.subarray(e.start, e.end);
793
+ if (indexOf(bytes, eb, 0) >= 0) {
794
+ extras.push(ei);
795
+ cov += eb.length;
796
+ }
797
+ }
798
+ if (cov > bestCov ||
799
+ (cov === bestCov && best !== null &&
800
+ (c.interior.length < best.interior.length ||
801
+ (c.interior.length === best.interior.length && c.id < best.id)))) {
802
+ best = c;
803
+ bestExtras = extras;
804
+ bestCov = cov;
805
+ }
806
+ }
807
+ if (best === null)
808
+ continue; // every container was self-evidence
809
+ const reach = edgeAncestors(ctx, best.id, N, reachMemo);
810
+ if (reach.saturated || reach.roots.length === 0)
811
+ continue;
812
+ const idf = Math.log(N / Math.max(1, reach.contextsReached));
813
+ if (idf <= 0)
814
+ continue;
815
+ // EXACT joint evidence (score = 1): the container literally contains
816
+ // every composed form. Mutual-explanation weight over their COMBINED
817
+ // byte length — the same magnitude reading voteRegions uses, here with
818
+ // the estimator collapsed to certainty, so mutual = min(ratio, 1/ratio).
819
+ const lenR = Math.max(1, bestCov);
820
+ const ratio = Math.sqrt(Math.max(1, ctx.store.contentLen(best.id, lenR * ctx.store.D)) / lenR);
821
+ const mutual = Math.min(1, ratio) * Math.min(1, 1 / ratio);
822
+ const w = (mutual * idf) / reach.roots.length;
823
+ let spanStart = ra.start;
824
+ let spanEnd = rb.end;
825
+ for (const ei of bestExtras) {
826
+ spanStart = Math.min(spanStart, regions[ei].start);
827
+ spanEnd = Math.max(spanEnd, regions[ei].end);
828
+ }
829
+ out.push({
830
+ start: spanStart,
831
+ end: spanEnd,
832
+ canonicalFailed: false, // content-addressed: never saturation-masked
833
+ roots: reach.roots,
834
+ w,
835
+ wFocus: w,
836
+ });
837
+ consumed.add(cand[a]);
838
+ consumed.add(cand[b]);
839
+ for (const ei of bestExtras)
840
+ consumed.add(ei);
841
+ // EXPLAINING AWAY — see the block comment above the function. Byte
842
+ // containment in the joint container is the relatedness test (the
843
+ // vote's bytes are literally part of the learnt whole), and FULL root
844
+ // disjointness is the disagreement test: a vote sharing even one root
845
+ // with the junction corroborates it and keeps its say elsewhere.
846
+ const containerBytes = cachedRead(ctx, cache, best.id, cap);
847
+ const jointRoots = new Set(reach.roots);
848
+ for (const rv of rvs.votes) {
849
+ if (rv.roots.some((r) => jointRoots.has(r)))
850
+ continue;
851
+ const bytes = query.subarray(rv.start, rv.end);
852
+ if (indexOf(containerBytes, bytes, 0) >= 0)
853
+ superseded.add(rv);
854
+ }
855
+ const label = [cand[a], cand[b], ...bestExtras]
856
+ .sort((x, y) => regions[x].start - regions[y].start)
857
+ .map((ri) => dec(query.subarray(regions[ri].start, regions[ri].end)))
858
+ .join(" ▸ ");
859
+ ctx.trace?.step("crossRegion", [{ text: label, role: "pair" }], reach.roots.map((r) => ({
860
+ text: dec(read(ctx, r)).slice(0, 60),
861
+ node: r,
862
+ role: "joint-context",
863
+ })), `${label} → junction node ${best.id}` +
864
+ (best.interior.length === 0
865
+ ? " (adjacent)"
866
+ : ` (interior "${dec(best.interior)}")`) +
867
+ ` → ${reach.roots.length} context(s), by content-addressed ascent` +
868
+ (superseded.size > 0
869
+ ? `; ${superseded.size} aliasing vote(s) explained away`
870
+ : ""));
871
+ break; // ra is consumed — move to the next unconsumed candidate
952
872
  }
953
- const bytes = query.subarray(rv.start, rv.end);
954
- if (indexOf(containerBytes, bytes, 0) >= 0) {
955
- superseded.add(rv);
956
- }
957
- }
958
- const label = [cand[a], cand[b], ...bestExtras]
959
- .sort((x, y) => regions[x].start - regions[y].start)
960
- .map((ri) => dec(query.subarray(regions[ri].start, regions[ri].end)))
961
- .join(" ▸ ");
962
- ctx.trace?.step(
963
- "crossRegion",
964
- [{ text: label, role: "pair" }],
965
- reach.roots.map((r) => ({
966
- text: dec(read(ctx, r)).slice(0, 60),
967
- node: r,
968
- role: "joint-context",
969
- })),
970
- `${label} → junction node ${best.id}` +
971
- (best.interior.length === 0
972
- ? " (adjacent)"
973
- : ` (interior "${dec(best.interior)}")`) +
974
- ` → ${reach.roots.length} context(s), by content-addressed ascent` +
975
- (superseded.size > 0
976
- ? `; ${superseded.size} aliasing vote(s) explained away`
977
- : ""),
978
- );
979
- break; // ra is consumed — move to the next unconsumed candidate
980
873
  }
981
- }
982
- return { votes: out, superseded };
874
+ return { votes: out, superseded };
983
875
  }
984
876
  export function traceAttention(ctx, regions, regionVoter, roots, steps = []) {
985
- if (!ctx.trace) {
986
- return;
987
- }
988
- const voters = [];
989
- for (let i = 0; i < regions.length; i++) {
990
- const rv = regionVoter[i];
991
- if (rv == null) {
992
- continue;
877
+ if (!ctx.trace)
878
+ return;
879
+ const voters = [];
880
+ for (let i = 0; i < regions.length; i++) {
881
+ const rv = regionVoter[i];
882
+ if (rv == null)
883
+ continue;
884
+ const item = rNode(ctx, rv.id, "sub-region", rv.score);
885
+ item.text = `${item.text} (df-w ${rv.w.toFixed(2)})`;
886
+ voters.push(item);
993
887
  }
994
- const item = rNode(ctx, rv.id, "sub-region", rv.score);
995
- item.text = `${item.text} (df-w ${rv.w.toFixed(2)})`;
996
- voters.push(item);
997
- }
998
- const t = ctx.trace.enter("climbConsensus", voters);
999
- // The pooled-evidence decision, one DerivationStep per anchor — the same
1000
- // shape {@link GraphSearch}'s own cover steps take (see traceDerivation).
1001
- if (steps.length > 0) {
1002
- traceDerivation(ctx, steps);
1003
- }
1004
- t.done(
1005
- roots.map((r) => rNode(ctx, r.anchor, "anchor", r.vote)),
1006
- roots.length === 0
1007
- ? `${regions.length} sub-regions climbed the DAG, but none agreed on a context`
1008
- : roots.length === 1
1009
- ? `${voters.length} of ${regions.length} sub-regions voted; IDF-weighted consensus picked one context (vote ${
1010
- roots[0].vote.toFixed(2)
1011
- })`
1012
- : `${voters.length} of ${regions.length} sub-regions voted; consensus ordered ${roots.length} INDEPENDENT points of attention (votes ${
1013
- roots.map((r) => r.vote.toFixed(2)).join(", ")
1014
- })`,
1015
- );
888
+ const t = ctx.trace.enter("climbConsensus", voters);
889
+ // The pooled-evidence decision, one DerivationStep per anchor — the same
890
+ // shape {@link GraphSearch}'s own cover steps take (see traceDerivation).
891
+ if (steps.length > 0)
892
+ traceDerivation(ctx, steps);
893
+ t.done(roots.map((r) => rNode(ctx, r.anchor, "anchor", r.vote)), roots.length === 0
894
+ ? `${regions.length} sub-regions climbed the DAG, but none agreed on a context`
895
+ : roots.length === 1
896
+ ? `${voters.length} of ${regions.length} sub-regions voted; IDF-weighted consensus picked one context (vote ${roots[0].vote.toFixed(2)})`
897
+ : `${voters.length} of ${regions.length} sub-regions voted; consensus ordered ${roots.length} INDEPENDENT points of attention (votes ${roots.map((r) => r.vote.toFixed(2)).join(", ")})`);
1016
898
  }