@hviana/sema 0.5.2 → 0.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/AGENTS.md +115 -53
  2. package/HOW_IT_WORKS.md +291 -184
  3. package/jsr.json +6 -0
  4. package/package.json +1 -1
  5. package/src/mind/bridge.ts +10 -104
  6. package/src/mind/match.ts +497 -7
  7. package/src/mind/{prefix-completion.ts → mechanisms/prefix-completion.ts} +66 -92
  8. package/src/mind/mechanisms/recall.ts +38 -129
  9. package/src/mind/mechanisms/reference.ts +343 -0
  10. package/src/mind/mind.ts +12 -8
  11. package/src/mind/pipeline-mechanism.ts +120 -3
  12. package/src/mind/pipeline.ts +16 -2
  13. package/src/mind/traverse.ts +92 -1
  14. package/src/store.ts +13 -4
  15. package/test/33-multi-candidate.test.mjs +21 -11
  16. package/test/70-prefix-completion.test.mjs +1 -1
  17. package/test/72-prefix-candidate-supply.test.mjs +7 -9
  18. package/test/74-prefix-trap-not-sprung-early.test.mjs +1 -1
  19. package/test/76-reference-binding.test.mjs +543 -0
  20. package/dist/example/demo.d.ts +0 -1
  21. package/dist/example/demo.js +0 -39
  22. package/dist/example/train_base.d.ts +0 -93
  23. package/dist/example/train_base.js +0 -2269
  24. package/dist/src/alphabet.d.ts +0 -7
  25. package/dist/src/alphabet.js +0 -33
  26. package/dist/src/alu/src/alu.d.ts +0 -185
  27. package/dist/src/alu/src/alu.js +0 -440
  28. package/dist/src/alu/src/expr.d.ts +0 -61
  29. package/dist/src/alu/src/expr.js +0 -318
  30. package/dist/src/alu/src/index.d.ts +0 -11
  31. package/dist/src/alu/src/index.js +0 -19
  32. package/dist/src/alu/src/kernel-arith.d.ts +0 -16
  33. package/dist/src/alu/src/kernel-arith.js +0 -264
  34. package/dist/src/alu/src/kernel-bits.d.ts +0 -19
  35. package/dist/src/alu/src/kernel-bits.js +0 -152
  36. package/dist/src/alu/src/kernel-logic.d.ts +0 -4
  37. package/dist/src/alu/src/kernel-logic.js +0 -60
  38. package/dist/src/alu/src/kernel-nd.d.ts +0 -3
  39. package/dist/src/alu/src/kernel-nd.js +0 -208
  40. package/dist/src/alu/src/kernel-numeric.d.ts +0 -54
  41. package/dist/src/alu/src/kernel-numeric.js +0 -366
  42. package/dist/src/alu/src/operation.d.ts +0 -168
  43. package/dist/src/alu/src/operation.js +0 -189
  44. package/dist/src/alu/src/parser.d.ts +0 -221
  45. package/dist/src/alu/src/parser.js +0 -577
  46. package/dist/src/alu/src/resonance.d.ts +0 -55
  47. package/dist/src/alu/src/resonance.js +0 -126
  48. package/dist/src/alu/src/text.d.ts +0 -31
  49. package/dist/src/alu/src/text.js +0 -73
  50. package/dist/src/alu/src/value.d.ts +0 -109
  51. package/dist/src/alu/src/value.js +0 -300
  52. package/dist/src/alu/test/alu.test.d.ts +0 -1
  53. package/dist/src/alu/test/alu.test.js +0 -764
  54. package/dist/src/bytes.d.ts +0 -14
  55. package/dist/src/bytes.js +0 -59
  56. package/dist/src/canon.d.ts +0 -45
  57. package/dist/src/canon.js +0 -85
  58. package/dist/src/config.d.ts +0 -111
  59. package/dist/src/config.js +0 -91
  60. package/dist/src/derive/src/deduction.d.ts +0 -136
  61. package/dist/src/derive/src/deduction.js +0 -159
  62. package/dist/src/derive/src/index.d.ts +0 -8
  63. package/dist/src/derive/src/index.js +0 -11
  64. package/dist/src/derive/src/priority-queue.d.ts +0 -20
  65. package/dist/src/derive/src/priority-queue.js +0 -73
  66. package/dist/src/derive/src/rewrite.d.ts +0 -56
  67. package/dist/src/derive/src/rewrite.js +0 -100
  68. package/dist/src/derive/src/trie.d.ts +0 -90
  69. package/dist/src/derive/src/trie.js +0 -217
  70. package/dist/src/derive/test/derive.test.d.ts +0 -1
  71. package/dist/src/derive/test/derive.test.js +0 -122
  72. package/dist/src/extension.d.ts +0 -37
  73. package/dist/src/extension.js +0 -7
  74. package/dist/src/geometry.d.ts +0 -233
  75. package/dist/src/geometry.js +0 -1089
  76. package/dist/src/index.d.ts +0 -17
  77. package/dist/src/index.js +0 -19
  78. package/dist/src/ingest-cache.d.ts +0 -41
  79. package/dist/src/ingest-cache.js +0 -165
  80. package/dist/src/meter.d.ts +0 -176
  81. package/dist/src/meter.js +0 -274
  82. package/dist/src/mind/articulation.d.ts +0 -6
  83. package/dist/src/mind/articulation.js +0 -99
  84. package/dist/src/mind/attention.d.ts +0 -430
  85. package/dist/src/mind/attention.js +0 -2418
  86. package/dist/src/mind/bridge.d.ts +0 -39
  87. package/dist/src/mind/bridge.js +0 -998
  88. package/dist/src/mind/canonical.d.ts +0 -34
  89. package/dist/src/mind/canonical.js +0 -93
  90. package/dist/src/mind/frame-filler.d.ts +0 -15
  91. package/dist/src/mind/frame-filler.js +0 -535
  92. package/dist/src/mind/graph-search.d.ts +0 -294
  93. package/dist/src/mind/graph-search.js +0 -996
  94. package/dist/src/mind/index.d.ts +0 -9
  95. package/dist/src/mind/index.js +0 -5
  96. package/dist/src/mind/junction.d.ts +0 -137
  97. package/dist/src/mind/junction.js +0 -358
  98. package/dist/src/mind/learning.d.ts +0 -75
  99. package/dist/src/mind/learning.js +0 -265
  100. package/dist/src/mind/match.d.ts +0 -192
  101. package/dist/src/mind/match.js +0 -666
  102. package/dist/src/mind/mechanisms/alu.d.ts +0 -4
  103. package/dist/src/mind/mechanisms/alu.js +0 -36
  104. package/dist/src/mind/mechanisms/cast.d.ts +0 -89
  105. package/dist/src/mind/mechanisms/cast.js +0 -1186
  106. package/dist/src/mind/mechanisms/confluence.d.ts +0 -24
  107. package/dist/src/mind/mechanisms/confluence.js +0 -271
  108. package/dist/src/mind/mechanisms/cover.d.ts +0 -6
  109. package/dist/src/mind/mechanisms/cover.js +0 -232
  110. package/dist/src/mind/mechanisms/extraction.d.ts +0 -33
  111. package/dist/src/mind/mechanisms/extraction.js +0 -327
  112. package/dist/src/mind/mechanisms/recall.d.ts +0 -16
  113. package/dist/src/mind/mechanisms/recall.js +0 -558
  114. package/dist/src/mind/mind.d.ts +0 -386
  115. package/dist/src/mind/mind.js +0 -754
  116. package/dist/src/mind/pipeline-mechanism.d.ts +0 -183
  117. package/dist/src/mind/pipeline-mechanism.js +0 -521
  118. package/dist/src/mind/pipeline.d.ts +0 -49
  119. package/dist/src/mind/pipeline.js +0 -331
  120. package/dist/src/mind/prefix-completion.d.ts +0 -59
  121. package/dist/src/mind/prefix-completion.js +0 -270
  122. package/dist/src/mind/primitives.d.ts +0 -85
  123. package/dist/src/mind/primitives.js +0 -333
  124. package/dist/src/mind/rationale.d.ts +0 -139
  125. package/dist/src/mind/rationale.js +0 -163
  126. package/dist/src/mind/reasoning.d.ts +0 -42
  127. package/dist/src/mind/reasoning.js +0 -305
  128. package/dist/src/mind/recognition.d.ts +0 -20
  129. package/dist/src/mind/recognition.js +0 -696
  130. package/dist/src/mind/resonance.d.ts +0 -35
  131. package/dist/src/mind/resonance.js +0 -365
  132. package/dist/src/mind/trace.d.ts +0 -15
  133. package/dist/src/mind/trace.js +0 -73
  134. package/dist/src/mind/traverse.d.ts +0 -158
  135. package/dist/src/mind/traverse.js +0 -716
  136. package/dist/src/mind/types.d.ts +0 -418
  137. package/dist/src/mind/types.js +0 -179
  138. package/dist/src/rabitq-ivf/src/database.d.ts +0 -113
  139. package/dist/src/rabitq-ivf/src/database.js +0 -201
  140. package/dist/src/rabitq-ivf/src/index.d.ts +0 -7
  141. package/dist/src/rabitq-ivf/src/index.js +0 -4
  142. package/dist/src/rabitq-ivf/src/ivf.d.ts +0 -200
  143. package/dist/src/rabitq-ivf/src/ivf.js +0 -1165
  144. package/dist/src/rabitq-ivf/src/prng.d.ts +0 -19
  145. package/dist/src/rabitq-ivf/src/prng.js +0 -36
  146. package/dist/src/rabitq-ivf/src/rabitq.d.ts +0 -95
  147. package/dist/src/rabitq-ivf/src/rabitq.js +0 -313
  148. package/dist/src/sema.d.ts +0 -41
  149. package/dist/src/sema.js +0 -77
  150. package/dist/src/store-sqlite.d.ts +0 -184
  151. package/dist/src/store-sqlite.js +0 -942
  152. package/dist/src/store.d.ts +0 -762
  153. package/dist/src/store.js +0 -1823
  154. package/dist/src/vec.d.ts +0 -31
  155. package/dist/src/vec.js +0 -109
  156. package/src/mind/frame-filler.ts +0 -604
  157. package/test/69-frame-filler.test.mjs +0 -115
@@ -1,558 +0,0 @@
1
- // mechanisms/recall.ts — Recall by resonance (Grounding IV).
2
- //
3
- // The recall mechanism resonates the whole query's gist against the content
4
- // index and grounds the nearest learned form. Four tiers, orderly degrading
5
- // from exact self-match to honest echo.
6
- import { cosine } from "../../vec.js";
7
- import { conceptThreshold, consensusFloor, dominates, identityBar, reachThreshold, significanceBar, } from "../../geometry.js";
8
- import { gistOf, read, resolve } from "../primitives.js";
9
- import { bytesEqual, indexOf } from "../../bytes.js";
10
- import { allWindowsAreScaffolding, corpusN, hubBound } from "../traverse.js";
11
- import { follow, project, reverseContext } from "../match.js";
12
- import { CONCEPT, STEP } from "../graph-search.js";
13
- import { unexplainedLabel } from "../rationale.js";
14
- import { rItem, rNode } from "../trace.js";
15
- import { substitutionBridge } from "../bridge.js";
16
- import { frameFillerSubstitution } from "../frame-filler.js";
17
- import { prefixCandidates, prefixCompletion } from "../prefix-completion.js";
18
- /** Recall the answer by resonating the whole query against the content index. */
19
- export async function recallByResonance(ctx, query, pre) {
20
- const t = ctx.trace?.enter("recallByResonance", [
21
- rItem(query, "query"),
22
- ]);
23
- const whole_ = [[0, query.length]];
24
- const nothing = [];
25
- const ground = (bytes, note, accounted, moves, echoed = false, complete = false) => {
26
- t?.done(bytes === null
27
- ? []
28
- : [rItem(bytes, "answer", resolve(ctx, bytes) ?? undefined)], note);
29
- return bytes === null ? null : {
30
- bytes,
31
- echoed,
32
- accounted,
33
- moves,
34
- unexplained: unexplainedLabel(query, accounted),
35
- ...(complete ? { complete } : {}),
36
- };
37
- };
38
- const k = pre.k;
39
- const queryGist = pre.guide;
40
- // 0. Exact self-match — content-addressed, deterministic.
41
- const qId = pre.queryResolved;
42
- if (qId !== null) {
43
- const rev = ctx.store.prevFirst(qId, hubBound(ctx));
44
- const g = reverseContext(ctx, qId, queryGist, rev);
45
- if (g !== null) {
46
- return ground(g, rev.length === 1
47
- ? "exact self-match — reverse recall to the sole predecessor"
48
- : "exact self-match — reverse recall to the best-resonating predecessor", nothing, STEP);
49
- }
50
- }
51
- // 0b. ARGUMENT BINDING (RC8): the query is not itself a stored form, but
52
- // it CONTAINS a recognised constituent that is an edge SOURCE — a learnt
53
- // pair's left side carried inside a wrapper ("How do you say 'thank you'
54
- // in French?"). The wrapper is scaffolding; the argument is the span
55
- // that LEADS somewhere, so its continuation — guided by the whole query's
56
- // gist — is the answer. Matching the wrapper while ignoring the argument
57
- // (the observed "good morning" template failure) is worse than silence,
58
- // so anything short of ONE unambiguous binding falls through: the
59
- // constituent bar is the same two-quanta (2W) reading confluence binds
60
- // under, nested recognitions collapse to their MAXIMAL span, and two
61
- // distinct maximal arguments mean the query asks about neither alone.
62
- if (qId === null) {
63
- const W2 = 2 * ctx.space.maxGroup;
64
- const args = pre.rec.sites.filter((s) => s.end - s.start >= W2 &&
65
- s.end - s.start < query.length &&
66
- ctx.store.hasNext(s.payload));
67
- // Maximal spans by one sorted sweep (starts ascending, ties longest
68
- // first): every earlier span starts at or before s, so s is contained
69
- // exactly when the running max end already covers it. O(m log m) — a
70
- // long input recognises O(|input|) sites, and a pairwise scan here was
71
- // quadratic in the input.
72
- args.sort((a, b) => a.start - b.start || b.end - a.end);
73
- const maximal = [];
74
- let maxEnd = -1;
75
- for (const s of args) {
76
- if (s.end <= maxEnd)
77
- continue;
78
- maximal.push(s);
79
- maxEnd = s.end;
80
- }
81
- // The wrapper must actually BE scaffolding: RC8's own premise is "the
82
- // wrapper is scaffolding; the argument is the span that leads
83
- // somewhere" ("How do you say 'thank you' in French?" — everything
84
- // outside the argument is a small fixed template). When the query
85
- // instead has ANOTHER substantial recognised form (≥ W2, the same
86
- // constituent bar the argument itself must clear) sitting OUTSIDE the
87
- // chosen argument, the query is not one argument in a wrapper — it is
88
- // several complete, independently-meaningful pieces (a multi-turn
89
- // conversation's own accumulated turns are exactly this shape), and
90
- // binding to the argument's continuation would answer past content
91
- // the query itself already carries forward. Derived from the same W2
92
- // bar the argument itself is held to, never a separate tuned number.
93
- const hasSubstantialOutside = maximal.length === 1 &&
94
- pre.rec.sites.some((s) => s.end - s.start >= W2 &&
95
- (s.end <= maximal[0].start || s.start >= maximal[0].end));
96
- if (maximal.length === 1 && !hasSubstantialOutside) {
97
- const arg = maximal[0];
98
- const g = await follow(ctx, arg.payload, queryGist);
99
- // The same "no restated fragment" guard tier 2 applies below (§ "the
100
- // anchor cleared the consensus floor..."): a followed continuation
101
- // that is itself a proper byte-subspan of the QUERY restates part of
102
- // the question — never an answer. A multi-turn query's own later
103
- // turns are exact, content-addressed matches for exactly this reason
104
- // (each turn is its own previously-learnt form), so without this
105
- // guard the argument's OWN later restatement in the same
106
- // conversation reads as if it were the next thing to say.
107
- if (g !== null && g.length > 0 &&
108
- !(g.length < query.length && indexOf(query, g, 0) >= 0)) {
109
- return ground(g, "argument binding — the query's sole edge-source constituent, continuation followed", [[arg.start, arg.end]], STEP);
110
- }
111
- }
112
- }
113
- const whole = await ctx.store.resonate(queryGist, k);
114
- if (whole.length === 0) {
115
- return ground(null, "empty store — nothing to resonate with", [], 0);
116
- }
117
- const top = whole[0];
118
- ctx.trace?.step("resonate", [rItem(query, "query-gist")], whole.map((h) => rNode(ctx, h.id, "hit", h.score)), `resonate the whole-query gist → ${whole.length} nearest learnt form(s)`);
119
- // 1. Clean resonance — the scale-aware identity claim. The ANGLE
120
- // (top.score) carries the shared fraction; the query's MAGNITUDE (√len,
121
- // the linear fold's own norm) converts the tolerated foreign fraction
122
- // into bytes — at most one river window (see {@link identityBar}). A
123
- // fixed cosine bar let long queries claim "near-identical" while whole
124
- // windows — an answer word — differed.
125
- // A hit RESTATES the query when its bytes are the query's own — exactly,
126
- // or under the response's equivalence (a case/width twin). Restating
127
- // hits may only conclude through disciplined reverse recall: voicing
128
- // their bytes echoes the query back at itself (never an answer — the
129
- // same principle that keeps cast from voicing stored questions), and
130
- // projecting them forward is reverse recall's containment failure in the
131
- // other direction — "whatever followed these bytes in some document".
132
- const qKey = ctx.canon ? ctx.canon(query) : query;
133
- const restates = (b) => bytesEqual(b, query) ||
134
- (ctx.canon !== null && bytesEqual(ctx.canon(b), qKey));
135
- const idBar = identityBar(ctx.store.D, ctx.space.maxGroup, query.length);
136
- if (top.score >= idBar) {
137
- for (const h of whole) {
138
- // The identity claim is PER HIT, not per tier: hits are ranked
139
- // nearest-first, and grounding one below the bar under this tier's
140
- // "near-identical" label would launder byte-overlap noise (observed:
141
- // "merci" projecting through the unrelated near hit "meraih").
142
- if (h.score < idBar)
143
- break;
144
- const own = read(ctx, h.id);
145
- if (h.id === qId || restates(own)) {
146
- const rev = ctx.store.prevFirst(h.id, hubBound(ctx));
147
- const g = reverseContext(ctx, h.id, queryGist, rev);
148
- if (g !== null) {
149
- return ground(g, rev.length === 1
150
- ? "perfect self-match — reverse recall to the sole predecessor"
151
- : "perfect self-match — reverse recall to the best-resonating predecessor", nothing, STEP);
152
- }
153
- continue;
154
- }
155
- const g = await project(ctx, h.id, queryGist);
156
- if (g) {
157
- return ground(g, "clean whole-query resonance — ground the nearest hit", whole_, STEP);
158
- }
159
- }
160
- }
161
- // The query-relative grounding fraction, shared by tiers 2–4 — gated on
162
- // the FRACTION OF THE QUERY the grounding explains, not the raw cosine.
163
- // Root gists are unit vectors, but their magnitudes are recoverable from
164
- // the byte lengths (‖·‖ = √len under the linear fold):
165
- // cos = shared/√(lenQ·lenG), so shared/lenQ = cos·√(lenG/lenQ).
166
- // The raw cosine punished honest containment — a query fully inside a
167
- // longer grounded answer scored √(lenQ/lenG) and was refused — and let a
168
- // long answer sharing only scaffolding pass; the query-relative fraction
169
- // measures exactly what the reach bar means: how much of THE QUERY the
170
- // store accounts for.
171
- // Chance similarity survives the length conversion AMPLIFIED: the same
172
- // √(lenG/lenQ) factor that converts an honest shared fraction into a
173
- // query-relative one multiplies the estimator/chance floor too, so a long
174
- // stored form (√(lenG/lenQ) ≈ 10 at 100×) lifted a noise-level cosine past
175
- // the reach bar and grounded pure gibberish (observed). Only the
176
- // ABOVE-CHANCE part of the similarity is evidence of shared content —
177
- // subtract the significance bar (3/√D, §8.3) before converting. Derived
178
- // from the existing bars; never tuned.
179
- const sig = significanceBar(ctx.store.D);
180
- const reach = reachThreshold(ctx.space.maxGroup);
181
- const fracOfQuery = (cos, otherLen) => Math.min(1, Math.max(0, cos - sig) *
182
- Math.sqrt(otherLen / Math.max(1, query.length)));
183
- // 2. Scaffolding-dominated.
184
- if (top.score >= sig) {
185
- // The committed points of attention ARE the shared climb's roots (same
186
- // query, same k, same DF mode) — read them from Precomputed instead of
187
- // re-climbing, so even a traced response pays for the climb once.
188
- const forest = (await pre.attention()).roots;
189
- // TRUST THE ANCHOR ON ITS BREADTH, NOT ON ITS ABSOLUTE VOTE.
190
- //
191
- // This gate read `forest[0].vote >= consensusFloor(N)`. Attention.breadth's
192
- // own contract (types.ts) says why that is the wrong quantity: the IDF vote
193
- // is "an absolute, ln(N)-scaled quantity that means 'strong' on a small
194
- // store and 'weak' on a large one for the SAME degree of genuine
195
- // consensus", while breadth is the SCALE-INVARIANT reading — "a point whose
196
- // breadth clears `dominates` (> half the query's regions corroborate it) is
197
- // real consensus; one that does not is a coincidental single-region echo".
198
- // Attention.peak's contract makes the same point from the other side:
199
- // comparing a POOLED SUM against a floor that prices ONE region's evidence
200
- // is a dimensional error.
201
- //
202
- // Measured on the 15.7M-node store (N=325,615, so the old floor was 13.19).
203
- // The absolute vote cannot separate right from wrong at this scale, and the
204
- // proof is a probe that must stay SILENT:
205
- //
206
- // anchor picked by the climb vote breadth correct?
207
- // "What is the chemical formula …" 10.60 0.556 RIGHT
208
- // "Qual é a capital de França?" 8.19 0.667 RIGHT
209
- // "Who wrote the play Romeo …?" 8.25 0.833 RIGHT
210
- // "How do you say "good morning" …" 10.77 0.800 RIGHT
211
- // "What is the commercial capital …" 12.69 0.333 Zamunda — MUST be silent
212
- // "Menene sunan ginin mafi tsayi …" 12.79 0.214 wrong (Hausa)
213
- // "Today is the 5th of March …" 10.36 0.000 wrong
214
- //
215
- // Zamunda's junk attractor outvotes every correct anchor, so no vote
216
- // threshold admits the right ones without admitting fabrication — while
217
- // breadth > ½ admits exactly the four correct anchors and nothing else.
218
- // The old floor was simply never cleared on a corpus this large: the tier
219
- // was dead code here, which is why 12 probes fell through to silence.
220
- //
221
- // `dominates(breadth, 1)` is the SAME half-dominance predicate used
222
- // throughout, applied to the fraction — no new constant, and the bar the
223
- // breadth contract names. COST: none; breadth is already computed and
224
- // carried on every Attention the climb returns.
225
- //
226
- // The two readings are ALTERNATIVES, never a substitution. REPLACING the
227
- // vote test with the breadth test was tried and broke 7 tests: on a small
228
- // store ln(N) is low, so the vote bar is the one that legitimately fires
229
- // there, and — as Attention.clusters' own contract warns — "breadth starves
230
- // a genuine, evenly-split multi-topic query, since no root in a real N-way
231
- // split can exceed half the vote" (the two 3.1 two-topic fusion tests are
232
- // exactly that shape). Each reading is sufficient on its own evidence: a
233
- // vote that clears the absolute floor is strong enough wherever the corpus
234
- // is small enough for that to mean something, and a breadth past ½ is real
235
- // consensus at any scale. ORing them keeps every admission the floor
236
- // already made and adds only the scale-invariant ones it could never see.
237
- //
238
- // BREADTH ALSO NEEDS DISCRIMINATIVENESS. Breadth asks how much of the
239
- // query corroborates the anchor, never whether the anchor SAYS anything: on
240
- // a one-context store every region trivially corroborates the only anchor
241
- // there is, so breadth is 1 while the anchor's IDF is 0 — and test/31 A2
242
- // ("explain quantum chromodynamics" against a lone cat fact) answered the
243
- // cat, which is fabrication. A region's IDF contribution for an anchor
244
- // reached through c of N contexts is ln(N/c), so requiring it to exceed
245
- // ln 2 is requiring c·2 < N — the SAME half-dominance reading used
246
- // everywhere, expressed in the IDF's own units rather than as a new bar.
247
- // `peak` is that per-region contribution, and reading it here is what
248
- // Attention.peak's contract asks of a consumer gating on this evidence.
249
- //
250
- // AND THE QUERY MUST SAY SOMETHING. Both readings above price the
251
- // ANCHOR's evidence; neither asks whether the QUERY discriminates
252
- // anything. A query that is entirely corpus-global scaffolding gives the
253
- // corpus nothing to be held to, and this tier — which exists to serve
254
- // scaffolding-DOMINATED queries — is exactly where that runs out.
255
- // Measured on the trained store: "What is the capital " answered "Colombo
256
- // is the commercial capital of Sri Lanka…" on breadth 0.667 / clusters 1,
257
- // and every window it spells is a hub ("What":572). See
258
- // allWindowsAreScaffolding for the full separation, including the probes
259
- // this tier serves CORRECTLY, which all retain a discriminating window
260
- // ("what is the capital of france" → "f fr":248).
261
- //
262
- // DISPERSION WAS TRIED HERE FIRST AND FALSIFIED — do not retry it: the
263
- // fabrication and the no-punctuation robustness probe have the IDENTICAL
264
- // profile (breadth 0.667, clusters 1), so requiring clusters >= 2 silenced
265
- // "what is the capital of france" too and cost the battery a probe.
266
- const minVote = consensusFloor(corpusN(ctx));
267
- if (forest.length > 0 &&
268
- !allWindowsAreScaffolding(ctx, query) &&
269
- (forest[0].vote >= minVote ||
270
- (dominates(forest[0].breadth, 1) && forest[0].peak > Math.LN2))) {
271
- const g = await project(ctx, forest[0].anchor, queryGist);
272
- // The anchor cleared the consensus floor, but the floor prices the
273
- // ANCHOR's evidence, not the projection's: a junk attractor can clear
274
- // it and project a PIECE OF THE QUERY back at it (the observed
275
- // "buenos días in English" → "English" fragment). A projection that
276
- // is a proper byte-subspan of the query restates part of the question
277
- // — never an answer (the same principle as `restates` above, extended
278
- // to fragments). Genuine anchor groundings — longer than the query,
279
- // or disjoint from it — pass untouched.
280
- if (g && !(g.length < query.length && indexOf(query, g, 0) >= 0)) {
281
- return ground(g, "scaffolding-dominated query — ground the consensus-climb anchor", [[forest[0].start, forest[0].end]], CONCEPT);
282
- }
283
- }
284
- }
285
- // 3. Last resort — the nearest grounded whole-query hit, same gate.
286
- for (const h of whole) {
287
- const g = await project(ctx, h.id, queryGist);
288
- if (g) {
289
- if (fracOfQuery(cosine(queryGist, gistOf(ctx, g)), g.length) >=
290
- reach) {
291
- return ground(g, "last resort: the nearest grounded whole-query hit", [], STEP);
292
- }
293
- }
294
- }
295
- // 3b. Corroborated-substitution bridge — refusal-path only (bridge.ts).
296
- // MEMOISED ACROSS EVERY REMAINING TIER, and that is load-bearing rather than
297
- // tidy: the bridge, prefix completion and the frame filler all read the SAME
298
- // candidate list, so the exhaustive branch runs at most once per response.
299
- // Without the memo each tier re-issues it — measured at 490 ms median against
300
- // 13 ms non-exhaustive (36x).
301
- const wideIdsOnce = async () => {
302
- // When the top resonance hit is below the concept threshold, the query
303
- // gist has no concept-level match to any stored form — an exhaustive √N
304
- // ANN would only score more vectors below the bar (profiled at 38K–40K
305
- // annVectorReads per refusing query on a 325K-context store). The
306
- // bridge's structural channels (junction walks, anchor climbs) are the
307
- // correct proposal source for a query whose gist has no clean match;
308
- // the ANN cannot propose what the gist cannot rank.
309
- // The condition above is the SCORE of the top hit, not the size of the
310
- // corpus. It used to be spelled `corpusN(ctx) <= (k · W)³`, which asks
311
- // a different question and answers it wrongly at exactly the scale the
312
- // note was written from: on the trained store N = 325,608 with k = 24
313
- // and W = 4 puts the cube at 884,736, so that store took the exhaustive
314
- // branch — the very branch measured here as 38K–40K annVectorReads.
315
- // Measured cost of the mismatch: substitutionBridge 8,544ms of a
316
- // 19,548ms think (44%), against 1,248ms and 14,218ms without it, with
317
- // every answer in the battery byte-identical and the suite unchanged
318
- // at 445/445. Corpus size was never the discriminator; whether the
319
- // gist ranks ANYTHING at concept level is.
320
- //
321
- // Reading it as the note states also removes a duplicated (k · W)³ —
322
- // the same cube gates crossRegionVotes' walk budget, where it likewise
323
- // never engages at real scale (see attention.ts).
324
- if (whole.length > 0 && whole[0].score >= conceptThreshold(ctx.store.D)) {
325
- const exhaustive = await ctx.store.resonate(queryGist, hubBound(ctx), true);
326
- return exhaustive.map((h) => h.id);
327
- }
328
- return whole.map((h) => h.id);
329
- };
330
- let wide = null;
331
- const wideIds = () => (wide ??= wideIdsOnce());
332
- // Every gist-based tier has failed; before refusing, align the query
333
- // byte-for-byte against the trained contexts its own stored windows
334
- // anchor, accepting mismatches only as corpus-attested, concept-bar
335
- // substitutions. A bridged context grounds exactly like any hit —
336
- // projected through its learnt edges — under the same restated-fragment
337
- // guard tiers 0b/2 apply. Costs nothing on any answering path.
338
- {
339
- // The resonance hits already ranked above are handed to the bridge as
340
- // PROPOSED candidates alongside its own anchor climbs: on a corpus this
341
- // size a W-byte window is far too common for the clamped climb to
342
- // single out the right trained context, while the whole-query gist
343
- // already ranked it nearest (observed live: "what is the capital of
344
- // france" resonating straight to "What is the capital of France?" yet
345
- // refusing on the reach bar). Approximate scores propose; the bridge's
346
- // byte-exact alignment and attestation gates decide.
347
- //
348
- // Reuse recall's already-ranked proposals. Never scan every IVF cluster:
349
- // exact co-occurrence and bounded anchor ascent are the bridge's structural
350
- // proposal channels, while an exhaustive ANN call made every honest
351
- // refusal cost hundreds of milliseconds regardless of k.
352
- const bridged = await substitutionBridge(ctx, query, wideIds);
353
- if (bridged !== null) {
354
- const g = await project(ctx, bridged.id, queryGist);
355
- // A projection contained in a substituted candidate-side span is the
356
- // substitution RESTATED as if it were knowledge — the exact failure
357
- // observed live: "Darwin was born in England." bridged to the
358
- // Einstein fact through " England." → " Germany." and would have
359
- // voiced "Germany", an answer the substitution itself manufactured.
360
- // The same principle as the restated-fragment guards above, extended
361
- // to the bridge's own substitutions.
362
- const cBytes = ctx.store.bytes(bridged.id);
363
- const manufactured = g !== null &&
364
- bridged.subs.some((s) => indexOf(cBytes.subarray(s.cs, s.ce), g, 0) >= 0);
365
- // THE PREFIX TRAP IS NOT THIS TIER'S TO SPRING. With no substitutions
366
- // the claim is "a trained context IS this query, up to filler". When
367
- // the query is a STRICT BYTE PREFIX of that context, the claim is false
368
- // in the one way that matters: the candidate's extra tail is precisely
369
- // the DISCRIMINATING part, and dismissing it as filler asserts a
370
- // specification the asker never made. Measured on a 4,300-fact fixture
371
- // of "what is the value of <i>?": the query "what is the value of"
372
- // bridged with subs [] to "what is the value of 0?" and answered "the
373
- // value of 0 is 0" — one arbitrary pick from 4,300 equally-matching
374
- // contexts, every one of which fits the query exactly as well.
375
- //
376
- // The engine ALREADY has the right machinery for this shape:
377
- // prefixCompletion runs a few lines below and carries the three guards
378
- // this tier lacks — unreadable-continuation veto, sub-quantum
379
- // continuation, and UNIQUENESS (distinct continuations ⇒ refuse), which
380
- // is exactly what 4,300 competing values must trip. So this is not a
381
- // new rule and not a new threshold: it is deferring a prefix decision to
382
- // the tier that owns it (§2.5, one factored machinery). Byte-strict on
383
- // purpose — a candidate differing by case or punctuation ("what is the
384
- // capital of france" → "What is the capital of France?") is NOT a byte
385
- // prefix, keeps grounding here, and is unaffected.
386
- const strictPrefix = g !== null &&
387
- cBytes.length > query.length &&
388
- indexOf(cBytes, query, 0) === 0;
389
- if (g !== null && g.length > 0 && !restates(g) && !manufactured &&
390
- !(bridged.subs.length === 0 && strictPrefix) &&
391
- !(g.length < query.length && indexOf(query, g, 0) >= 0)) {
392
- return ground(g, bridged.subs.length === 0
393
- ? `identity bridge — a trained context IS this query, up to ` +
394
- `scaffolding the corpus itself treats as filler`
395
- : `substitution bridge — a trained context accounts for the ` +
396
- `query up to ${bridged.subs.length} corroborated ` +
397
- `substitution(s)`,
398
- // WHAT THIS GROUNDING EXPLAINS — the spans its alignment covers,
399
- // matched AND substituted, for BOTH tiers.
400
- //
401
- // A corroborated substitution is not a gap in the explanation; it is
402
- // an explanation the mechanism PAID for, one CONCEPT each in `moves`
403
- // just below. Leaving its span unaccounted charges the same act
404
- // twice — once as a move, once as PASS-per-unexplained-byte — and the
405
- // second charge is far the larger, so a bridge that matched 28 of 29
406
- // bytes declared the whole query unexplained and lost to any
407
- // mechanism with a smaller honest claim. Measured on test/49's
408
- // paraphrase: it FOUND the trained fact through two corroborated case
409
- // substitutions and was outbid 29011 to 1012 by a CAST comparison
410
- // that voiced the wrong country.
411
- //
412
- // This tier DID once report `[]` for the substituted case, against
413
- // the observation that "pricing the aligned spans outweighed
414
- // extraction's correct answer in the grounding decider". That is no
415
- // longer so and the suite is the witness: with the fold's regions
416
- // content-defined and the junction tiers no longer consuming each
417
- // other's candidates, full accounting here passes every test that
418
- // refutation was recorded for. Reporting only the LITERALLY matched
419
- // spans (accounted minus the substituted ones) was also implemented
420
- // and is a strictly worse reading of the same ladder — it still
421
- // double-charges, just less.
422
- //
423
- // An IDENTITY bridge (zero substitutions) substituted nothing, so
424
- // there is nothing to be humble about: every accounted byte is a
425
- // LITERAL match against a trained form, and the query is that form
426
- // up to scaffolding. Reporting `[]` for it was actively wrong in
427
- // two ways — it priced a full explanation at PASS-per-byte so junk
428
- // outweighed it, and, because the honest-remainder test in think()
429
- // reads the same spans, it left the whole query "unaccounted" and
430
- // forced the multi-topic fusion gate open. Observed live: the
431
- // correct "What is the process of photosynthesis?" grounding was
432
- // fused away into an unrelated "Hello! How can I assist you
433
- // today?" point of attention.
434
- [...bridged.accounted], CONCEPT * bridged.subs.length + STEP, false,
435
- // COMPLETE only for the identity tier: the query IS this trained
436
- // context, so `g` is that context's own continuation — the whole
437
- // read-out. A SUBSTITUTED bridge makes no such claim (it stood a
438
- // different word in the query's place), so it stays extendable.
439
- bridged.subs.length === 0);
440
- }
441
- }
442
- // 3b′. PREFIX COMPLETION — refusal-path only (prefix-completion.ts).
443
- // Inside the bridge's block, and deliberately: it consumes `wideIds`, the
444
- // list the bridge has already fetched, so it costs a bounded byte compare
445
- // per candidate and not one resonance. The claim it makes is the
446
- // strongest in the ladder — every query byte is a LITERAL match from
447
- // offset zero of a trained form — so it needs no projection and no reach
448
- // gate. It runs after the bridge only because the bridge answers the
449
- // richer relation when it can; a prefix match that the bridge also
450
- // explains is the same trained form either way.
451
- {
452
- // The resonance list first; only when it supplies nothing does the
453
- // write side's leaf-id window index propose (prefixCandidates). That
454
- // ordering is the whole cost story: a query the ranked list can already
455
- // explain pays not one extra read, and the fallback's bounded walk is
456
- // spent only where the alternative is an empty answer. It is a second
457
- // SUPPLY, not a second mechanism — the same three guards decide.
458
- const completed = prefixCompletion(ctx, query, await wideIds()) ??
459
- prefixCompletion(ctx, query, prefixCandidates(ctx, query));
460
- if (completed !== null) {
461
- return ground(completed.form, "prefix completion — the query IS the opening of exactly one " +
462
- "trained form, which this grounds whole",
463
- // Every query byte is literally matched against the form. The
464
- // completion is the form's own continuation, not a substitution, so
465
- // there is nothing to be humble about in the accounting — the same
466
- // reading the IDENTITY bridge above takes.
467
- whole_, STEP, false,
468
- // NOT complete: the query is a proper PREFIX, so the form may carry
469
- // more past the remainder this tier voiced.
470
- false);
471
- }
472
- }
473
- }
474
- // 3c. FRAME-FILLER SUBSTITUTION — refusal-path only (frame-filler.ts).
475
- // The bridge has failed, and for the shape this tier answers it MUST fail:
476
- // a definite description standing where a proper noun stands is not a
477
- // similarity relation the bridge can price (raw balance refuses
478
- // `dominates(6, 37)`, and correctly — that is the France/Spain trap). This
479
- // tier makes a different claim: not that the two spans resemble each other,
480
- // but that the store ALREADY HOLDS this query with the filler in the
481
- // description's place, byte-exactly. A key the store does not hold is
482
- // discarded, so the answer is always a trained continuation.
483
- {
484
- // THE COHORT NEEDS EVIDENCE, AND THE REFUSAL PATH HAS ALREADY BOUGHT IT.
485
- // This tier reads constituency from what a cohort of exemplars does NOT
486
- // share, so its resolution is bounded by how many instances of the frame it
487
- // can see. The top-k resonance hits are too few — on the two-hop probe the
488
- // exemplars holding the query's discriminative content number TWO, and two
489
- // structures agree on so little that a whole clause reads as content. The
490
- // exhaustive list the bridge fetched is the same evidence at ~570 wide, and
491
- // it is already paid for (memoised above, so this costs no ANN call).
492
- const filled = frameFillerSubstitution(ctx, query, await wideIds());
493
- if (filled !== null) {
494
- const g = await project(ctx, filled.id, queryGist);
495
- // The same restated-fragment and manufactured-answer guards every tier
496
- // above applies: a projection contained in the FILLER is the
497
- // substitution restated as if it were knowledge, not knowledge.
498
- if (g !== null && g.length > 0 && !restates(g) &&
499
- indexOf(filled.filler, g, 0) < 0 &&
500
- !(g.length < query.length && indexOf(query, g, 0) >= 0)) {
501
- return ground(g, "frame-filler substitution — a trained form IS this query with a " +
502
- "corroborated filler in the described span's place",
503
- // The frame is literally matched against the resolved form and the
504
- // described span is explained by the substitution — the same
505
- // matched-plus-substituted accounting the bridge reports.
506
- [[0, query.length]], CONCEPT + STEP);
507
- }
508
- }
509
- }
510
- // The refusal/echo decision. The echo returns a stored form's bytes AS
511
- // the answer — a near-identity claim about the query — and identity-grade
512
- // decisions are never made on an estimated score ("approximate scores may
513
- // rank and propose; they may never decide", §6.2): the RaBitQ estimate
514
- // overshooting the reach bar echoed a WRONG-entity neighbour ("capital of
515
- // Zamunda?" echoed the Armenia fact, observed). The bytes are read
516
- // anyway to be echoed, so the decision uses their EXACT fold: one river
517
- // fold of the top hit, measured in the same query-relative,
518
- // chance-corrected units as the tier above.
519
- const topBytes = read(ctx, top.id);
520
- const exact = topBytes.length > 0
521
- ? cosine(queryGist, gistOf(ctx, topBytes))
522
- : 0;
523
- if (fracOfQuery(exact, topBytes.length) < reach) {
524
- return ground(null, "below reach threshold — nothing in the store relates to this query", [], 0);
525
- }
526
- // Echoing the query's own bytes back at it is not an echo of a RELATED
527
- // form — it is the query restated, which answers nothing.
528
- if (restates(topBytes)) {
529
- return ground(null, "the nearest form IS the query itself — restating it answers nothing", [], 0);
530
- }
531
- // Honest echo.
532
- return ground(topBytes, "last resort: the nearest resonant form's own bytes (echo, not grounded)", [], 0, true);
533
- }
534
- // ── Pipeline mechanism ──────────────────────────────────────────────────────
535
- export const recallMechanism = {
536
- name: "recall",
537
- provenance: "recall",
538
- // Recall's floor is free to state (one STEP-grade projection) and its run
539
- // gates its own tiers — no expensive investment happens inside floor, so
540
- // there is nothing to guard with worthRunning here: the pipeline's own
541
- // check prunes run() against the incumbent.
542
- async floor(_ctx, _query, _pre, _worthRunning) {
543
- return STEP;
544
- },
545
- async run(ctx, query, pre) {
546
- const r = await recallByResonance(ctx, query, pre);
547
- if (!r)
548
- return [];
549
- return [{
550
- bytes: r.bytes,
551
- accounted: r.accounted,
552
- moves: r.moves,
553
- unexplained: r.unexplained,
554
- provenance: r.echoed ? "recall-echo" : "recall",
555
- ...(r.complete ? { complete: true } : {}),
556
- }];
557
- },
558
- };