@hviana/sema 0.5.3 → 0.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/AGENTS.md +6 -6
  2. package/HOW_IT_WORKS.md +16 -0
  3. package/jsr.json +1 -1
  4. package/package.json +1 -1
  5. package/src/mind/match.ts +81 -0
  6. package/src/mind/mechanisms/recall.ts +29 -3
  7. package/test/76-reference-binding.test.mjs +72 -0
  8. package/dist/example/demo.d.ts +0 -1
  9. package/dist/example/demo.js +0 -39
  10. package/dist/example/train_base.d.ts +0 -93
  11. package/dist/example/train_base.js +0 -2269
  12. package/dist/src/alphabet.d.ts +0 -7
  13. package/dist/src/alphabet.js +0 -33
  14. package/dist/src/alu/src/alu.d.ts +0 -185
  15. package/dist/src/alu/src/alu.js +0 -440
  16. package/dist/src/alu/src/expr.d.ts +0 -61
  17. package/dist/src/alu/src/expr.js +0 -318
  18. package/dist/src/alu/src/index.d.ts +0 -11
  19. package/dist/src/alu/src/index.js +0 -19
  20. package/dist/src/alu/src/kernel-arith.d.ts +0 -16
  21. package/dist/src/alu/src/kernel-arith.js +0 -264
  22. package/dist/src/alu/src/kernel-bits.d.ts +0 -19
  23. package/dist/src/alu/src/kernel-bits.js +0 -152
  24. package/dist/src/alu/src/kernel-logic.d.ts +0 -4
  25. package/dist/src/alu/src/kernel-logic.js +0 -60
  26. package/dist/src/alu/src/kernel-nd.d.ts +0 -3
  27. package/dist/src/alu/src/kernel-nd.js +0 -208
  28. package/dist/src/alu/src/kernel-numeric.d.ts +0 -54
  29. package/dist/src/alu/src/kernel-numeric.js +0 -366
  30. package/dist/src/alu/src/operation.d.ts +0 -168
  31. package/dist/src/alu/src/operation.js +0 -189
  32. package/dist/src/alu/src/parser.d.ts +0 -221
  33. package/dist/src/alu/src/parser.js +0 -577
  34. package/dist/src/alu/src/resonance.d.ts +0 -55
  35. package/dist/src/alu/src/resonance.js +0 -126
  36. package/dist/src/alu/src/text.d.ts +0 -31
  37. package/dist/src/alu/src/text.js +0 -73
  38. package/dist/src/alu/src/value.d.ts +0 -109
  39. package/dist/src/alu/src/value.js +0 -300
  40. package/dist/src/alu/test/alu.test.d.ts +0 -1
  41. package/dist/src/alu/test/alu.test.js +0 -764
  42. package/dist/src/bytes.d.ts +0 -14
  43. package/dist/src/bytes.js +0 -59
  44. package/dist/src/canon.d.ts +0 -45
  45. package/dist/src/canon.js +0 -85
  46. package/dist/src/config.d.ts +0 -111
  47. package/dist/src/config.js +0 -91
  48. package/dist/src/derive/src/deduction.d.ts +0 -136
  49. package/dist/src/derive/src/deduction.js +0 -159
  50. package/dist/src/derive/src/index.d.ts +0 -8
  51. package/dist/src/derive/src/index.js +0 -11
  52. package/dist/src/derive/src/priority-queue.d.ts +0 -20
  53. package/dist/src/derive/src/priority-queue.js +0 -73
  54. package/dist/src/derive/src/rewrite.d.ts +0 -56
  55. package/dist/src/derive/src/rewrite.js +0 -100
  56. package/dist/src/derive/src/trie.d.ts +0 -90
  57. package/dist/src/derive/src/trie.js +0 -217
  58. package/dist/src/derive/test/derive.test.d.ts +0 -1
  59. package/dist/src/derive/test/derive.test.js +0 -122
  60. package/dist/src/extension.d.ts +0 -37
  61. package/dist/src/extension.js +0 -7
  62. package/dist/src/geometry.d.ts +0 -233
  63. package/dist/src/geometry.js +0 -1089
  64. package/dist/src/index.d.ts +0 -17
  65. package/dist/src/index.js +0 -19
  66. package/dist/src/ingest-cache.d.ts +0 -41
  67. package/dist/src/ingest-cache.js +0 -165
  68. package/dist/src/meter.d.ts +0 -176
  69. package/dist/src/meter.js +0 -274
  70. package/dist/src/mind/articulation.d.ts +0 -6
  71. package/dist/src/mind/articulation.js +0 -99
  72. package/dist/src/mind/attention.d.ts +0 -430
  73. package/dist/src/mind/attention.js +0 -2418
  74. package/dist/src/mind/bridge.d.ts +0 -37
  75. package/dist/src/mind/bridge.js +0 -907
  76. package/dist/src/mind/canonical.d.ts +0 -34
  77. package/dist/src/mind/canonical.js +0 -93
  78. package/dist/src/mind/graph-search.d.ts +0 -294
  79. package/dist/src/mind/graph-search.js +0 -996
  80. package/dist/src/mind/index.d.ts +0 -9
  81. package/dist/src/mind/index.js +0 -5
  82. package/dist/src/mind/junction.d.ts +0 -137
  83. package/dist/src/mind/junction.js +0 -358
  84. package/dist/src/mind/learning.d.ts +0 -75
  85. package/dist/src/mind/learning.js +0 -265
  86. package/dist/src/mind/match.d.ts +0 -351
  87. package/dist/src/mind/match.js +0 -959
  88. package/dist/src/mind/mechanisms/alu.d.ts +0 -4
  89. package/dist/src/mind/mechanisms/alu.js +0 -36
  90. package/dist/src/mind/mechanisms/cast.d.ts +0 -89
  91. package/dist/src/mind/mechanisms/cast.js +0 -1186
  92. package/dist/src/mind/mechanisms/confluence.d.ts +0 -24
  93. package/dist/src/mind/mechanisms/confluence.js +0 -271
  94. package/dist/src/mind/mechanisms/cover.d.ts +0 -6
  95. package/dist/src/mind/mechanisms/cover.js +0 -232
  96. package/dist/src/mind/mechanisms/extraction.d.ts +0 -33
  97. package/dist/src/mind/mechanisms/extraction.js +0 -327
  98. package/dist/src/mind/mechanisms/prefix-completion.d.ts +0 -22
  99. package/dist/src/mind/mechanisms/prefix-completion.js +0 -243
  100. package/dist/src/mind/mechanisms/recall.d.ts +0 -16
  101. package/dist/src/mind/mechanisms/recall.js +0 -460
  102. package/dist/src/mind/mechanisms/reference.d.ts +0 -6
  103. package/dist/src/mind/mechanisms/reference.js +0 -296
  104. package/dist/src/mind/mind.d.ts +0 -386
  105. package/dist/src/mind/mind.js +0 -754
  106. package/dist/src/mind/pipeline-mechanism.d.ts +0 -238
  107. package/dist/src/mind/pipeline-mechanism.js +0 -622
  108. package/dist/src/mind/pipeline.d.ts +0 -49
  109. package/dist/src/mind/pipeline.js +0 -343
  110. package/dist/src/mind/primitives.d.ts +0 -85
  111. package/dist/src/mind/primitives.js +0 -333
  112. package/dist/src/mind/rationale.d.ts +0 -139
  113. package/dist/src/mind/rationale.js +0 -163
  114. package/dist/src/mind/reasoning.d.ts +0 -42
  115. package/dist/src/mind/reasoning.js +0 -305
  116. package/dist/src/mind/recognition.d.ts +0 -20
  117. package/dist/src/mind/recognition.js +0 -696
  118. package/dist/src/mind/resonance.d.ts +0 -35
  119. package/dist/src/mind/resonance.js +0 -365
  120. package/dist/src/mind/trace.d.ts +0 -15
  121. package/dist/src/mind/trace.js +0 -73
  122. package/dist/src/mind/traverse.d.ts +0 -196
  123. package/dist/src/mind/traverse.js +0 -806
  124. package/dist/src/mind/types.d.ts +0 -418
  125. package/dist/src/mind/types.js +0 -179
  126. package/dist/src/rabitq-ivf/src/database.d.ts +0 -113
  127. package/dist/src/rabitq-ivf/src/database.js +0 -201
  128. package/dist/src/rabitq-ivf/src/index.d.ts +0 -7
  129. package/dist/src/rabitq-ivf/src/index.js +0 -4
  130. package/dist/src/rabitq-ivf/src/ivf.d.ts +0 -200
  131. package/dist/src/rabitq-ivf/src/ivf.js +0 -1165
  132. package/dist/src/rabitq-ivf/src/prng.d.ts +0 -19
  133. package/dist/src/rabitq-ivf/src/prng.js +0 -36
  134. package/dist/src/rabitq-ivf/src/rabitq.d.ts +0 -95
  135. package/dist/src/rabitq-ivf/src/rabitq.js +0 -313
  136. package/dist/src/sema.d.ts +0 -41
  137. package/dist/src/sema.js +0 -77
  138. package/dist/src/store-sqlite.d.ts +0 -184
  139. package/dist/src/store-sqlite.js +0 -942
  140. package/dist/src/store.d.ts +0 -762
  141. package/dist/src/store.js +0 -1823
  142. package/dist/src/vec.d.ts +0 -31
  143. package/dist/src/vec.js +0 -109
@@ -1,327 +0,0 @@
1
- // extraction.ts — Extraction (Skill) — Section 4 of the mind.
2
- //
3
- // Given a query and its consensus-ranked anchors, find the first span-shaped
4
- // skill exemplar among the ranked anchors and read the analogous span of the
5
- // query. A skill exemplar is a learnt fact whose context and answer together
6
- // form a span-in-context pattern: the answer is a subsequence of the context
7
- // (or one of its pieces is), and the context is the smallest spanning frame
8
- // that contains it.
9
- import { locate } from "../match.js";
10
- import { concatBytes, indexOf } from "../../bytes.js";
11
- import { decodeText, unexplainedLabel } from "../rationale.js";
12
- import { CONCEPT, STEP } from "../graph-search.js";
13
- import { rItem, rNode, traceFail } from "../trace.js";
14
- // ── Extraction ────────────────────────────────────────────────────────────
15
- /** Find the first span-shaped skill exemplar among the ranked anchors from
16
- * climbAttentionAll and read the analogous span from the query. Returns
17
- * the extracted bytes PLUS the query spans the skill ACCOUNTED FOR — the
18
- * located frames AND any read span BOUNDED by located frames on both
19
- * sides, the elementary evidence think's grounding decider weighs. A
20
- * bounded read is explained: the skill located both its borders in the
21
- * query and emitted exactly what sits between them. An OPEN-ENDED read
22
- * (the exemplar's answer reaches the context's end, so the query is read
23
- * to its own end with no located right border) remains a guess about where
24
- * the span stops — it stays unaccounted, priced by exclusion like the
25
- * cover's bridged bytes. (Accounting frames only — the earlier convention
26
- * — let a CAST juxtaposition that merely echoed the query's exact site
27
- * outweigh a correct bounded extraction: the same span counted as
28
- * explained for one mechanism and not the other, and the asymmetry, not
29
- * the answers' merits, decided the grounding.) Null when no skill
30
- * applies. */
31
- export async function extractBySkill(ctx, query, pre) {
32
- const t = ctx.trace?.enter("extractBySkill", [
33
- rItem(query, "query"),
34
- ]);
35
- const fail = traceFail(t);
36
- // Use climbAttentionAll to get the FULL ranked list, not just the
37
- // roots that cleared commitVotes' significance floor. The floor
38
- // gates further points of attention for fusion, but extraction only
39
- // needs ONE anchor that IS a span-shaped skill exemplar — and on
40
- // some seeds the top-voted anchor is not one (e.g. a concept-merge
41
- // nickname outvotes the painting exemplars on shared substrings,
42
- // while the exemplars' votes fall below the floor). Iterating the
43
- // ranked list instead of just the roots lets extraction reach the
44
- // first painting-exemplar anchor regardless of its floor status.
45
- //
46
- const { ranked } = await pre.attention();
47
- if (ranked.length === 0) {
48
- return fail("no consensus anchor — no skill to apply");
49
- }
50
- // Try ranked anchors IN ORDER until one yields a USABLE extraction — not
51
- // merely a span-shaped exemplar, but one whose extracted span clears the
52
- // same one-river-fold quantum (W) cover.ts's restatedSpan gate already
53
- // treats as the floor below which byte overlap is chance, not evidence.
54
- // isSpanShaped (spanShapedOf) is a deliberately permissive sparse-
55
- // subsequence check — see the section note below — so it accepts exemplars
56
- // whose relation to the query is coincidental gap-matching, not genuine
57
- // structure. Stopping at the FIRST such exemplar let a coincidental match
58
- // early in the ranked list win outright and read out a sub-quantum
59
- // fragment (observed: a 3-byte "Hel" pulled from an unrelated exemplar,
60
- // while a later ranked anchor would have read the query's own "Hello…"
61
- // correctly). Trying further anchors when one produces nothing usable is
62
- // the same idiom this loop already uses for non-exemplars — extended to
63
- // cover a bad extraction, not just a structural non-match.
64
- //
65
- // The retry is bounded at pre.k — the SAME evidence-breadth constant every
66
- // other consumer of a ranked list already self-limits to (resonance, the
67
- // weave, the climb itself; see Precomputed.k's own doc comment) — not the
68
- // full ranked list. locate()'s frame match has an EXACT-byte tier with no
69
- // significance correction of its own (short W-byte frames are cheap to
70
- // match by pure chance), so trying every ranked anchor turns that per-
71
- // anchor chance into a near-certainty over enough attempts: on a pure-
72
- // gibberish query, 170 anchors deep found an unrelated Zulu exemplar whose
73
- // short frame happened to byte-match, producing "xyzzy pl" — a coincidence
74
- // no different in kind from the "RaBitQ estimate overshot the reach bar
75
- // and grounded pure gibberish" failure recall.ts's own significance
76
- // correction exists to prevent (see recallByResonance's reach-threshold
77
- // comment). Bounding the search to the ranked list's own top-k restores
78
- // the "genuinely relevant but not root-significant" exemplars this loop
79
- // was built for, without the unbounded tail's chance collisions.
80
- const W = ctx.space.maxGroup;
81
- const searched = ranked.slice(0, pre.k);
82
- let shapeMisses = 0;
83
- let subQuantum = 0;
84
- for (const cand of searched) {
85
- const exemplar = await pre.spanShapedOf(cand.anchor);
86
- if (!exemplar) {
87
- shapeMisses++;
88
- continue;
89
- }
90
- const built = buildFromExemplar(ctx, query, pre, exemplar);
91
- if (built === null || built.bytes.length < W) {
92
- subQuantum++;
93
- continue;
94
- }
95
- // AN UNANCHORED READ IS NOT AN EXTRACTION. This function's contract (see
96
- // the doc above) is that `accounted` carries "the located frames AND any
97
- // read span BOUNDED by located frames on both sides", while an open-ended
98
- // read "remains a guess about where the span stops — it stays unaccounted".
99
- // EMPTY accounted is the degenerate case of that: NO frame of the exemplar
100
- // was located in the query at all, so nothing ties the bytes just read to
101
- // this question — the skill applied its exemplar's geometry to a query it
102
- // never matched.
103
- //
104
- // The live case (analyze_training.ts F, the battery's ONLY wrong non-silent
105
- // answer): "Which city is France's seat of government?" answered "Which ci"
106
- // — a fragment of the query itself — from the exemplar "What is dll", with
107
- // accounted=[] and pieces=1. isSpanShaped is deliberately permissive (a
108
- // sparse-subsequence check), so it accepts exemplars whose relation to the
109
- // query is coincidental gap-matching; requiring at least one LOCATED frame
110
- // is the structural evidence that permissiveness leaves out.
111
- //
112
- // Scoped to extraction ON PURPOSE. The same test at the pipeline's
113
- // post-grounding density check was tried and REVERTED: `accounted` is passed
114
- // empty BY CONVENTION on recall's own tiers (recall.ts's ground(…, [], …)),
115
- // so a density veto there refused six legitimate reverse-recall groundings.
116
- // Here the field is this mechanism's own output and carries its documented
117
- // meaning, so the test is sound exactly where the convention does not reach.
118
- if (built.accounted.length === 0) {
119
- subQuantum++;
120
- continue;
121
- }
122
- if (shapeMisses > 0 || subQuantum > 0) {
123
- ctx.trace?.step("trySkillAnchors", [
124
- rItem(query.subarray(0, 0), `skipped ${shapeMisses + subQuantum}`),
125
- rNode(ctx, cand.anchor, "chosen"),
126
- ], [], `skipped ${shapeMisses} non-exemplar and ${subQuantum} sub-quantum ` +
127
- `anchor(s) before one yielded a usable extraction`);
128
- }
129
- t?.done([rItem(built.bytes, "extracted")], built.pieces === 1
130
- ? `apply a learnt extraction skill — read the analogous span of the query` +
131
- ` framed like "${decodeText(exemplar.answerBytes)}" sits in its exemplar`
132
- : `apply a learnt MULTI-PIECE skill — read ${built.pieces} analogous` +
133
- ` pieces of the query and synthesize them like "${decodeText(exemplar.answerBytes)}"`);
134
- return {
135
- bytes: built.bytes,
136
- accounted: built.accounted,
137
- unexplained: unexplainedLabel(query, built.accounted),
138
- };
139
- }
140
- if (shapeMisses === searched.length) {
141
- ctx.trace?.step("trySkillAnchors", [], [], `none of the top ${searched.length} ranked anchor(s) (of ${ranked.length} total) ` +
142
- `is a span-shaped skill exemplar`);
143
- return fail("no consensus root is a span-shaped skill exemplar");
144
- }
145
- return fail("no ranked anchor yielded an extraction at or above the quantum floor");
146
- }
147
- /** Build the extracted bytes for ONE already-accepted span-shaped exemplar —
148
- * factored out of {@link extractBySkill} so its anchor loop can try
149
- * successive ranked candidates instead of committing to the first
150
- * structural match. Null when the exemplar's answer does not decompose
151
- * against its context, or no piece's frame locates in the query. */
152
- function buildFromExemplar(ctx, query, pre, exemplar) {
153
- const { contextBytes, answerBytes } = exemplar;
154
- const ansCtxRuns = answerRunsInContext(ctx, contextBytes, answerBytes);
155
- if (ansCtxRuns === null || ansCtxRuns.length === 0) {
156
- return null;
157
- }
158
- if (ansCtxRuns.length > 1) {
159
- ctx.trace?.step("decomposeAnswer", [rItem(answerBytes, "multi-piece-answer")], ansCtxRuns.map((r) => rItem(contextBytes.subarray(r.start, r.end), "piece", undefined, [
160
- r.start,
161
- r.end,
162
- ])), `answer splits into ${ansCtxRuns.length} piece(s) within the exemplar context`);
163
- }
164
- const pieces = [];
165
- const accounted = [];
166
- for (let ri = 0; ri < ansCtxRuns.length; ri++) {
167
- const run = ansCtxRuns[ri];
168
- const isLast = ri === ansCtxRuns.length - 1;
169
- const framePreLen = Math.min(run.start, ctx.space.maxGroup);
170
- const framePre = run.start > 0
171
- ? contextBytes.subarray(run.start - framePreLen, run.start)
172
- : null;
173
- const frames = [];
174
- let start = 0;
175
- if (framePre) {
176
- const prePos = locate(ctx, query, framePre, 0, pre.rec.sites);
177
- if (prePos < 0)
178
- continue;
179
- start = prePos + framePre.length;
180
- frames.push([prePos, start]); // the located frame IS matched evidence
181
- }
182
- let end;
183
- if (isLast) {
184
- if (run.end < contextBytes.length) {
185
- const framePostLen = Math.min(contextBytes.length - run.end, ctx.space.maxGroup);
186
- const framePost = contextBytes.subarray(run.end, run.end + framePostLen);
187
- const postPos = locate(ctx, query.subarray(start), framePost, 0, pre.rec.sites);
188
- if (postPos < 0)
189
- continue;
190
- end = start + postPos;
191
- frames.push([end, end + framePost.length]); // matched post-frame
192
- }
193
- else {
194
- end = query.length;
195
- }
196
- }
197
- else {
198
- const nextRun = ansCtxRuns[ri + 1];
199
- const nextPreLen = Math.min(nextRun.start, ctx.space.maxGroup);
200
- const nextPre = contextBytes.subarray(nextRun.start - nextPreLen, nextRun.start);
201
- const nextPos = locate(ctx, query.subarray(start), nextPre, 0, pre.rec.sites);
202
- if (nextPos < 0) {
203
- end = start + run.ansLen;
204
- }
205
- else {
206
- end = start + nextPos;
207
- frames.push([end, end + nextPre.length]); // matched next-frame
208
- }
209
- }
210
- if (start >= end)
211
- continue;
212
- pieces.push(query.subarray(start, end));
213
- accounted.push(...frames);
214
- // Bounded on both sides ⇒ the read span itself is explained (see doc).
215
- // frames carries the pre-border (when the answer is not at the context's
216
- // start) and the located right border (post-frame or next piece's
217
- // pre-frame); only when BOTH borders were located is the read bounded.
218
- const preBounded = run.start === 0 || frames.some(([, e]) => e === start);
219
- const postBounded = frames.some(([b]) => b === end);
220
- if (preBounded && postBounded)
221
- accounted.push([start, end]);
222
- }
223
- if (pieces.length === 0) {
224
- return null;
225
- }
226
- const out = pieces.length === 1 ? pieces[0] : concatBytes(pieces);
227
- return { bytes: out, accounted, pieces: pieces.length };
228
- }
229
- // ── The two span-shape readings: OPEN acceptance vs. STRONG decomposition ──
230
- //
231
- // isSpanShaped and answerRunsInContext read the SAME relation ("the answer is
232
- // drawn from the context") at two deliberately different strengths, and they
233
- // are NOT interchangeable:
234
- //
235
- // • isSpanShaped — the OPEN reading: any in-order embedding (a sparse
236
- // subsequence, arbitrary gaps). O(|context|) byte scan. Used to ACCEPT
237
- // an exemplar candidate.
238
- // • answerRunsInContext — the STRONG reading: a greedy longest-run
239
- // DECOMPOSITION into contiguous pieces. Greedy-longest is strictly
240
- // stronger than subsequence (a long late match can consume context an
241
- // earlier shorter choice needed), so an ACCEPTED exemplar can still fail
242
- // to decompose — extractBySkill then fails with "answer is not a
243
- // subsequence of the context" and think falls through to recall. That
244
- // fall-through is BEHAVIOUR, pinned by the extraction suites: do not
245
- // "unify" the two into one machine — replacing the open reading with the
246
- // strong one silently rejects exemplars extraction today accepts, and
247
- // replacing the strong one with a backtracking embedding changes which
248
- // pieces are read out of the query.
249
- /** Decompose an answer into substrings of its surrounding context, in order —
250
- * the STRONG span-shape reading (see the section note above). Returns null
251
- * when no greedy longest-run decomposition exists. Adjacent runs that
252
- * connect contiguously are merged. */
253
- export function answerRunsInContext(_ctx, context, answer) {
254
- const pos = indexOf(context, answer, 0);
255
- if (pos >= 0) {
256
- return [{ start: pos, end: pos + answer.length, ansLen: answer.length }];
257
- }
258
- const runs = [];
259
- let ai = 0;
260
- let ci = 0;
261
- while (ai < answer.length) {
262
- // Longest match of the remaining answer at any position of the remaining
263
- // context: one direct extend per context position — O(|ctx|·match) per
264
- // run, replacing the previous longest-first indexOf countdown whose
265
- // repeated scans were cubic on long sparse-subsequence answers.
266
- let bestLen = 0;
267
- let bestPos = -1;
268
- for (let p = ci; p < context.length; p++) {
269
- let l = 0;
270
- const maxL = Math.min(context.length - p, answer.length - ai);
271
- if (maxL <= bestLen)
272
- break; // no later position can beat the best
273
- while (l < maxL && context[p + l] === answer[ai + l])
274
- l++;
275
- if (l > bestLen) {
276
- bestLen = l;
277
- bestPos = p;
278
- if (ai + l === answer.length)
279
- break; // the whole remainder matched
280
- }
281
- }
282
- if (bestLen === 0)
283
- return null;
284
- runs.push({ start: bestPos, end: bestPos + bestLen, ansLen: bestLen });
285
- ai += bestLen;
286
- ci = bestPos + bestLen;
287
- }
288
- const merged = [];
289
- for (const r of runs) {
290
- const last = merged[merged.length - 1];
291
- if (last && r.start === last.end) {
292
- last.end = r.end;
293
- last.ansLen += r.ansLen;
294
- }
295
- else {
296
- merged.push({ ...r });
297
- }
298
- }
299
- return merged.length > 0 ? merged : null;
300
- }
301
- // ── Pipeline mechanism ──────────────────────────────────────────────────────
302
- export const extractionMechanism = {
303
- name: "extraction",
304
- provenance: "extract",
305
- async floor(_ctx, _query, pre, worthRunning) {
306
- // Extraction's floor is always exactly CONCEPT+STEP when it exists —
307
- // same investment discipline as CAST's (see cast.ts): when the bound
308
- // already cannot beat the incumbent, return it UNINVESTED (never
309
- // first-touch the climb just to be pruned).
310
- if (!worthRunning(CONCEPT + STEP))
311
- return CONCEPT + STEP;
312
- if ((await pre.attention()).ranked.length === 0)
313
- return null;
314
- return CONCEPT + STEP;
315
- },
316
- async run(ctx, query, pre) {
317
- const ex = await extractBySkill(ctx, query, pre);
318
- if (!ex)
319
- return [];
320
- return [{
321
- bytes: ex.bytes,
322
- accounted: ex.accounted,
323
- moves: CONCEPT + STEP * ex.accounted.length,
324
- unexplained: ex.unexplained,
325
- }];
326
- },
327
- };
@@ -1,22 +0,0 @@
1
- import type { MindContext } from "../types.js";
2
- import type { PipelineMechanism } from "../pipeline-mechanism.js";
3
- /** A trained form the query opens, and the bytes by which it continues. */
4
- export interface PrefixCompletion {
5
- /** The trained form whose opening the query is — the answer, voiced whole. */
6
- id: number;
7
- /** The form's own bytes. The mechanism grounds a FORM, never a slice of
8
- * one: slicing at the query's end would cut at an offset the geometry has
9
- * no reason to treat as a boundary. */
10
- form: Uint8Array;
11
- /** The bytes past the query — carried for the rationale and for the
12
- * uniqueness comparison, not voiced on its own. */
13
- continuation: Uint8Array;
14
- }
15
- /** The sole trained form the query opens — or null when no candidate opens with
16
- * it, when the continuation is sub-quantum, when a candidate's continuation
17
- * cannot be read through, or when the candidates disagree.
18
- *
19
- * `ranked` must be a list the caller has ALREADY fetched; this mechanism never
20
- * resonates on its own (see the header's cost note). */
21
- export declare function prefixCompletion(ctx: MindContext, query: Uint8Array, ranked: ReadonlyArray<number>): PrefixCompletion | null;
22
- export declare const prefixMechanism: PipelineMechanism;
@@ -1,243 +0,0 @@
1
- // mechanisms/prefix-completion.ts — Grounding a query that IS the opening of a
2
- // trained form (Grounding V).
3
- //
4
- // A MECHANISM, NOT A TIER. This used to run inside recall's refusal path, in
5
- // a fixed if-chain that first-match-wins — the shape CAST was refactored away
6
- // from, where placement rather than the cost ladder decided. Its claim is
7
- // maximal (every query byte literally matched, from offset zero, against a
8
- // trained form) at one STEP, so as a market candidate it competes honestly and
9
- // the decider weighs it like everything else. It is registered LAST: recall's
10
- // exact self-match makes an IDENTITY claim about the query while this makes a
11
- // CONTAINMENT one, and on an exact grade tie the identity claim is the
12
- // stronger evidence — the same ordering §2.3's ladders use.
13
- //
14
- // Its SUPPLY moved too, and further: `formsOpenedBy` (traverse.ts) answers a
15
- // question about the STORE — "which trained forms does this byte run open?" —
16
- // so it is retrieval machinery any mechanism may ask, not this one's private
17
- // helper.
18
- //
19
- // THE SHAPE. `The capital of France is` grounds nothing, while
20
- // `The capital of France is Paris.` is trained and reads back byte-exact. The
21
- // query is not SIMILAR to that form, it is a PROPER PREFIX of it: every query
22
- // byte is a literal match, in order, from offset zero. That is the strongest
23
- // grounding relation in the store — stronger than the bridge's corroborated
24
- // substitution, which pays a CONCEPT per substituted span, and stronger than
25
- // resonance, which only claims an angle. Nothing is invented: the answer IS a
26
- // trained form, voiced whole.
27
- //
28
- // NO NOTION OF TEXT. This mechanism reads bytes and geometry only. It has no
29
- // separator, no character class, no "word": the only structural quantity it
30
- // uses is W, the river's grouping window, which is the same capacity the
31
- // perception tree groups by and the same bar the argument-binding tier holds
32
- // its constituents to. A completion shorter than one grouping window carries
33
- // no structure the geometry can perceive, whatever the modality — that is a
34
- // statement about the fold, not about punctuation. Presentation (what is
35
- // "spacing", what is "case") belongs to the injected canon and to the modality
36
- // entry point, never here; see src/canon.ts.
37
- //
38
- // WHY THE EARLIER TIERS CANNOT DO IT. Two independent reasons, both measured:
39
- //
40
- // 1. `resolve(prefix)` is null. A proper prefix of a deposited stream has no
41
- // branch of its own unless it was itself deposited, so the exact tiers
42
- // have nothing to find.
43
- // 2. The form is not among the resonance candidates AT ALL. Measured on the
44
- // trained store: cos(query, that form) = 0.5752, yet the form is absent
45
- // from `resonate(k)` at k = 24, 256 AND 2048 — while forms scoring LOWER
46
- // (Germany 0.5670, Yemen 0.5591) are returned. `k` only reorders WITHIN
47
- // the IVF clusters already probed, exactly as Store.resonate's doc warns,
48
- // so no k recovers it. With `exhaustive` it ranks 8.
49
- //
50
- // So this is a RETRIEVABILITY gap, not a semantic one, and it is repaired by
51
- // reading the candidate list recall's refusal path has ALREADY fetched
52
- // exhaustively for the substitution bridge — never by resonating on its own.
53
- // Measured cost of the scan over those 570 candidates: 2.9 ms warm, 20.4 ms
54
- // cold, against a ~700 ms refusal path. Issuing a FRESH exhaustive call would
55
- // cost 490 ms median against 13 ms non-exhaustive (36×), which is why this tier
56
- // takes the candidate list as an argument and adds nothing to it.
57
- //
58
- // THREE GUARDS, each falsified into existence by measurement — do not drop any:
59
- //
60
- // 1. AN UNREADABLE CONTINUATION VETOES. Reads are bounded (a stored span can
61
- // run to hundreds of kilobytes), so a candidate that opens with the query
62
- // but SATURATES the read continues in a way nobody can see. It is a
63
- // standing disagreement: if any such candidate exists, nothing is grounded.
64
- // It must NOT be quietly skipped, and that is not a stylistic point — the
65
- // skip is what MANUFACTURES a fragment. Measured on a one-deposit fixture
66
- // whose form exceeds the cap: the query matched BOTH the whole 138-byte
67
- // form (saturating) AND an interior fold node of 34 bytes (unsaturated,
68
- // continuing `" Paris, an"`). Skipping the saturated candidate removed the
69
- // only evidence that disagreed, uniqueness then passed on the interior
70
- // node, and a mid-form slice was voiced as an answer. Suppressing the
71
- // disagreement is what created the fabrication.
72
- // (Testing instead whether a candidate is a "complete form" via the fold
73
- // does NOT work and was measured: content addressing makes an interior
74
- // node resolve to ITSELF, so self-resolution says nothing about
75
- // completeness.)
76
- // 2. THE CONTINUATION MUST REACH ONE GROUPING WINDOW. A trained
77
- // `What is the capital of France??` opens with `What is the capital of
78
- // France?` and continues by a single byte. Below W the continuation is
79
- // sub-quantum — the fold groups nothing from it — and voicing it produces
80
- // the degenerate reply that is a known failure smell.
81
- // 3. UNIQUENESS. Several trained forms may open with the query and continue
82
- // differently, and then the corpus does not say which continuation the
83
- // asker means. Distinct continuations ⇒ refuse. This is the documented
84
- // PREFIX TRAP, and it is real — just not for every prefix. Measured: of
85
- // 15 battery probes exactly ONE yields a unique continuation, and all
86
- // three honest-silence probes yield none (including `What is the capital
87
- // of Zamunda?`, whose top hit scores 0.83).
88
- //
89
- // Uniqueness is judged on the continuation BYTES, not on the candidate id: the
90
- // same continuation reached through two trained forms is one answer, not an
91
- // ambiguity.
92
- import { bytesEqual } from "../../bytes.js";
93
- import { rItem } from "../trace.js";
94
- import { formsOpenedBy } from "../traverse.js";
95
- import { STEP } from "../graph-search.js";
96
- /** The sole trained form the query opens — or null when no candidate opens with
97
- * it, when the continuation is sub-quantum, when a candidate's continuation
98
- * cannot be read through, or when the candidates disagree.
99
- *
100
- * `ranked` must be a list the caller has ALREADY fetched; this mechanism never
101
- * resonates on its own (see the header's cost note). */
102
- export function prefixCompletion(ctx, query, ranked) {
103
- const W = ctx.space.maxGroup;
104
- const t = ctx.trace?.enter("prefixCompletion", [rItem(query, "query")]);
105
- const done = (hit, note, data) => {
106
- t?.done(hit === null ? [] : [rItem(hit.continuation, "continuation", hit.id)], note, data);
107
- return hit;
108
- };
109
- // Reads are bounded to phrase scale, the same bound the frame filler uses.
110
- // A query with no room for a whole grouping window past its own length
111
- // cannot clear guard 2, so it is not worth a single read.
112
- const cap = query.length * W;
113
- if (query.length === 0 || cap < query.length + W) {
114
- return done(null, "no room for a perceivable continuation within the cap");
115
- }
116
- // Distinct continuations, each with the first form that offered it. Held as
117
- // a list, not a byte-keyed map: candidates that open with the query are few
118
- // (measured: 1 on the trained store's winning query), and a linear byte
119
- // compare needs no string encoding of content. Uniqueness (guard 3) is
120
- // decided over this list, so the scan cannot stop early — a second
121
- // continuation IS the refusal, and finding it is the point.
122
- const found = [];
123
- let opened = 0;
124
- let unreadable = 0;
125
- let subQuantum = 0;
126
- for (const id of ranked) {
127
- const form = ctx.store.bytesPrefix(id, cap);
128
- if (form.length <= query.length)
129
- continue;
130
- let opens = true;
131
- for (let i = 0; i < query.length; i++) {
132
- if (form[i] !== query[i]) {
133
- opens = false;
134
- break;
135
- }
136
- }
137
- if (!opens)
138
- continue;
139
- opened++;
140
- // Guard 1: a saturated read continues out of sight — a disagreement that
141
- // cannot be resolved, so it ends the search rather than being skipped.
142
- if (form.length >= cap) {
143
- unreadable++;
144
- continue;
145
- }
146
- const rest = form.subarray(query.length);
147
- // Guard 2: below one grouping window there is no structure to voice.
148
- if (rest.length < W) {
149
- subQuantum++;
150
- continue;
151
- }
152
- if (!found.some((f) => bytesEqual(f.continuation, rest))) {
153
- found.push({ id, form, continuation: rest });
154
- }
155
- }
156
- const data = {
157
- candidates: ranked.length,
158
- opened,
159
- unreadable,
160
- subQuantum,
161
- distinctContinuations: found.length,
162
- };
163
- if (unreadable > 0 && found.length > 0) {
164
- return done(null, "a form opens with this query but continues past the read bound — " +
165
- "its continuation cannot be read, so none is licensed", data);
166
- }
167
- // Guard 2b: A SUB-QUANTUM CONTINUATION IS STILL A DISAGREEMENT. Guard 2
168
- // refuses to VOICE a below-window continuation, and rightly — there is no
169
- // structure there to speak. But dropping such a candidate from the
170
- // uniqueness tally silently converts "the corpus offers many continuations,
171
- // most of them unvoiceable" into "the corpus offers exactly one", and
172
- // guard 3 then passes VACUOUSLY on the sole survivor. That is precisely
173
- // the failure guard 1 documents for unreadable continuations — suppressing
174
- // the disagreement is what manufactures the answer — so it is answered the
175
- // same way, and for the same reason.
176
- //
177
- // Measured on a 4,300-fact fixture of "what is the value of <i>?": the
178
- // query "what is the value of" drew candidates continuing " 0?", " 4?",
179
- // " 8?" (3 bytes, sub-quantum at W=4) and " 10?" (4 bytes). The first
180
- // three were dropped, leaving one survivor, and the mechanism reported
181
- // "exactly one trained form" and voiced "the value of 10 is 20" — an
182
- // arbitrary pick from thousands of equally-good readings, with the
183
- // evidence of ambiguity discarded on the way.
184
- //
185
- // Note this can only ever cause SILENCE, never a different answer: it
186
- // withholds a completion the corpus does not uniquely license.
187
- if (subQuantum > 0 && found.length > 0) {
188
- return done(null, "other trained forms open with this query but continue below one " +
189
- "grouping window — the corpus offers competing readings, so no " +
190
- "single completion is licensed", data);
191
- }
192
- // Guard 3: the corpus must agree on ONE continuation.
193
- if (found.length !== 1) {
194
- return done(null, found.length === 0
195
- ? "no trained form opens with this query and continues perceivably"
196
- : "trained forms open with this query but continue differently — " +
197
- "the corpus does not say which continuation is meant", data);
198
- }
199
- return done(found[0], "one trained form opens with this query, and continues perceivably", data);
200
- }
201
- // ── Pipeline mechanism ──────────────────────────────────────────────────────
202
- export const prefixMechanism = {
203
- name: "prefix",
204
- provenance: "prefix",
205
- async floor(ctx, query, _pre, worthRunning) {
206
- // One projection: the form is voiced whole, nothing is substituted.
207
- // INVESTMENT DISCIPLINE — the supplies below are the response's wide
208
- // candidate list and a bounded √N walk, so neither is touched until the
209
- // bound can still beat the incumbent.
210
- if (!worthRunning(STEP))
211
- return STEP;
212
- // A query with no room for a perceivable continuation inside the phrase
213
- // cap cannot clear guard 2, so it is not worth a single read.
214
- const cap = query.length * ctx.space.maxGroup;
215
- if (query.length === 0 || cap < query.length + ctx.space.maxGroup) {
216
- return null;
217
- }
218
- return STEP;
219
- },
220
- async run(ctx, query, pre) {
221
- // The response's shared wide list first; only when it supplies nothing does
222
- // the write side's window index propose. That ordering is the whole cost
223
- // story: a query the ranked list can already explain pays not one extra
224
- // read, and the bounded walk is spent only where the alternative is an
225
- // empty answer. A second SUPPLY, not a second mechanism — the same three
226
- // guards decide either way.
227
- const completed = prefixCompletion(ctx, query, await pre.wideResonance()) ??
228
- prefixCompletion(ctx, query, formsOpenedBy(ctx, query));
229
- if (completed === null)
230
- return [];
231
- return [{
232
- bytes: completed.form,
233
- // Every query byte is literally matched against the form, so there is
234
- // nothing to be humble about in the accounting — the same reading the
235
- // IDENTITY bridge takes.
236
- accounted: [[0, query.length]],
237
- moves: STEP,
238
- unexplained: "",
239
- // NOT complete: the query is a proper PREFIX, so the form may carry more
240
- // past the remainder this voiced.
241
- }];
242
- },
243
- };
@@ -1,16 +0,0 @@
1
- import type { MindContext } from "../types.js";
2
- import type { PipelineMechanism, Precomputed } from "../pipeline-mechanism.js";
3
- /** A recall result. */
4
- export interface RecallResult {
5
- bytes: Uint8Array;
6
- echoed: boolean;
7
- accounted: Array<[number, number]>;
8
- moves: number;
9
- unexplained: string;
10
- /** See {@link import("../pipeline-mechanism.js").MechanismResult.complete}
11
- * — set by the IDENTITY-bridge tier alone. */
12
- complete?: boolean;
13
- }
14
- /** Recall the answer by resonating the whole query against the content index. */
15
- export declare function recallByResonance(ctx: MindContext, query: Uint8Array, pre: Precomputed): Promise<RecallResult | null>;
16
- export declare const recallMechanism: PipelineMechanism;