@hviana/sema 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/AGENTS.md +470 -0
  2. package/AUTHORS.md +12 -0
  3. package/COMMERCIAL-LICENSE.md +19 -0
  4. package/CONTRIBUTING.md +15 -0
  5. package/HOW_IT_WORKS.md +4210 -0
  6. package/LICENSE.md +117 -0
  7. package/README.md +290 -0
  8. package/TRADEMARKS.md +19 -0
  9. package/example/demo.ts +46 -0
  10. package/example/train_base.ts +2675 -0
  11. package/index.html +893 -0
  12. package/package.json +25 -0
  13. package/src/alphabet.ts +34 -0
  14. package/src/alu/README.md +332 -0
  15. package/src/alu/src/alu.ts +541 -0
  16. package/src/alu/src/expr.ts +339 -0
  17. package/src/alu/src/index.ts +115 -0
  18. package/src/alu/src/kernel-arith.ts +377 -0
  19. package/src/alu/src/kernel-bits.ts +199 -0
  20. package/src/alu/src/kernel-logic.ts +102 -0
  21. package/src/alu/src/kernel-nd.ts +235 -0
  22. package/src/alu/src/kernel-numeric.ts +466 -0
  23. package/src/alu/src/operation.ts +344 -0
  24. package/src/alu/src/parser.ts +574 -0
  25. package/src/alu/src/resonance.ts +161 -0
  26. package/src/alu/src/text.ts +83 -0
  27. package/src/alu/src/value.ts +327 -0
  28. package/src/alu/test/alu.test.ts +1004 -0
  29. package/src/bytes.ts +62 -0
  30. package/src/config.ts +227 -0
  31. package/src/derive/README.md +295 -0
  32. package/src/derive/src/deduction.ts +263 -0
  33. package/src/derive/src/index.ts +24 -0
  34. package/src/derive/src/priority-queue.ts +70 -0
  35. package/src/derive/src/rewrite.ts +127 -0
  36. package/src/derive/src/trie.ts +242 -0
  37. package/src/derive/test/derive.test.ts +137 -0
  38. package/src/extension.ts +42 -0
  39. package/src/geometry.ts +511 -0
  40. package/src/index.ts +38 -0
  41. package/src/ingest-cache.ts +224 -0
  42. package/src/mind/articulation.ts +137 -0
  43. package/src/mind/attention.ts +1061 -0
  44. package/src/mind/canonical.ts +107 -0
  45. package/src/mind/graph-search.ts +1100 -0
  46. package/src/mind/index.ts +18 -0
  47. package/src/mind/junction.ts +347 -0
  48. package/src/mind/learning.ts +246 -0
  49. package/src/mind/match.ts +507 -0
  50. package/src/mind/mechanisms/alu.ts +33 -0
  51. package/src/mind/mechanisms/cast.ts +568 -0
  52. package/src/mind/mechanisms/confluence.ts +268 -0
  53. package/src/mind/mechanisms/cover.ts +248 -0
  54. package/src/mind/mechanisms/extraction.ts +422 -0
  55. package/src/mind/mechanisms/recall.ts +241 -0
  56. package/src/mind/mind.ts +483 -0
  57. package/src/mind/pipeline-mechanism.ts +326 -0
  58. package/src/mind/pipeline.ts +300 -0
  59. package/src/mind/primitives.ts +201 -0
  60. package/src/mind/rationale.ts +275 -0
  61. package/src/mind/reasoning.ts +198 -0
  62. package/src/mind/recognition.ts +234 -0
  63. package/src/mind/resonance.ts +0 -0
  64. package/src/mind/trace.ts +108 -0
  65. package/src/mind/traverse.ts +556 -0
  66. package/src/mind/types.ts +281 -0
  67. package/src/rabitq-hnsw/README.md +303 -0
  68. package/src/rabitq-hnsw/src/database.ts +492 -0
  69. package/src/rabitq-hnsw/src/heap.ts +90 -0
  70. package/src/rabitq-hnsw/src/hnsw.ts +514 -0
  71. package/src/rabitq-hnsw/src/index.ts +15 -0
  72. package/src/rabitq-hnsw/src/prng.ts +39 -0
  73. package/src/rabitq-hnsw/src/rabitq.ts +308 -0
  74. package/src/rabitq-hnsw/src/store.ts +994 -0
  75. package/src/rabitq-hnsw/test/hnsw.test.ts +1213 -0
  76. package/src/store-sqlite.ts +928 -0
  77. package/src/store.ts +2148 -0
  78. package/src/vec.ts +124 -0
  79. package/test/00-extract.test.mjs +151 -0
  80. package/test/01-floor.test.mjs +20 -0
  81. package/test/02-roundtrip.test.mjs +83 -0
  82. package/test/03-recall.test.mjs +98 -0
  83. package/test/04-think.test.mjs +627 -0
  84. package/test/05-concepts.test.mjs +84 -0
  85. package/test/08-storage.test.mjs +948 -0
  86. package/test/09-edges.test.mjs +65 -0
  87. package/test/11-universality.test.mjs +180 -0
  88. package/test/13-conversation.test.mjs +228 -0
  89. package/test/14-scaling.test.mjs +503 -0
  90. package/test/15-decomposition-gap.test.mjs +0 -0
  91. package/test/16-bridge.test.mjs +250 -0
  92. package/test/17-intelligence.test.mjs +240 -0
  93. package/test/18-alu.test.mjs +209 -0
  94. package/test/19-nd.test.mjs +254 -0
  95. package/test/20-stability.test.mjs +489 -0
  96. package/test/21-partial.test.mjs +158 -0
  97. package/test/22-multihop.test.mjs +130 -0
  98. package/test/23-rationale.test.mjs +316 -0
  99. package/test/24-generalization.test.mjs +416 -0
  100. package/test/25-embedding.test.mjs +75 -0
  101. package/test/26-cooperative.test.mjs +89 -0
  102. package/test/27-saturation-drop.test.mjs +637 -0
  103. package/test/28-unknowable.test.mjs +88 -0
  104. package/test/29-counterfactual.test.mjs +476 -0
  105. package/test/30-conflict-resolution.test.mjs +166 -0
  106. package/test/31-audit.test.mjs +288 -0
  107. package/test/32-confluence.test.mjs +173 -0
  108. package/test/33-multi-candidate.test.mjs +222 -0
  109. package/test/34-cross-region.test.mjs +252 -0
  110. package/tsconfig.json +16 -0
@@ -0,0 +1,422 @@
1
+ // extraction.ts — Extraction (Skill) — Section 4 of the mind.
2
+ //
3
+ // Given a query and its consensus-ranked anchors, find the first span-shaped
4
+ // skill exemplar among the ranked anchors and read the analogous span of the
5
+ // query. A skill exemplar is a learnt fact whose context and answer together
6
+ // form a span-in-context pattern: the answer is a subsequence of the context
7
+ // (or one of its pieces is), and the context is the smallest spanning frame
8
+ // that contains it.
9
+
10
+ import type { Vec } from "../../vec.js";
11
+ import type { MindContext } from "../types.js";
12
+ import type { Site } from "../graph-search.js";
13
+ import { foldTree, gistOf, perceive, read, resolve } from "../primitives.js";
14
+ import { follow, locate } from "../match.js";
15
+ import { chooseAmong, hubBound } from "../traverse.js";
16
+ import { concatBytes, indexOf } from "../../bytes.js";
17
+ import { decodeText, unexplainedLabel } from "../rationale.js";
18
+ import type {
19
+ PipelineMechanism,
20
+ Precomputed,
21
+ SkillInfo,
22
+ } from "../pipeline-mechanism.js";
23
+ import { CONCEPT, STEP } from "../graph-search.js";
24
+ import { rItem, rNode, traceFail } from "../trace.js";
25
+
26
+ // ── Extraction ────────────────────────────────────────────────────────────
27
+
28
+ /** Find the first span-shaped skill exemplar among the ranked anchors from
29
+ * climbAttentionAll and read the analogous span from the query. Returns
30
+ * the extracted bytes PLUS the query spans the skill ACCOUNTED FOR — the
31
+ * located frames AND any read span BOUNDED by located frames on both
32
+ * sides, the elementary evidence think's grounding decider weighs. A
33
+ * bounded read is explained: the skill located both its borders in the
34
+ * query and emitted exactly what sits between them. An OPEN-ENDED read
35
+ * (the exemplar's answer reaches the context's end, so the query is read
36
+ * to its own end with no located right border) remains a guess about where
37
+ * the span stops — it stays unaccounted, priced by exclusion like the
38
+ * cover's bridged bytes. (Accounting frames only — the earlier convention
39
+ * — let a CAST juxtaposition that merely echoed the query's exact site
40
+ * outweigh a correct bounded extraction: the same span counted as
41
+ * explained for one mechanism and not the other, and the asymmetry, not
42
+ * the answers' merits, decided the grounding.) Null when no skill
43
+ * applies. */
44
+ export async function extractBySkill(
45
+ ctx: MindContext,
46
+ query: Uint8Array,
47
+ pre: Precomputed,
48
+ ): Promise<
49
+ {
50
+ bytes: Uint8Array;
51
+ accounted: Array<[number, number]>;
52
+ unexplained: string;
53
+ } | null
54
+ > {
55
+ const t = ctx.trace?.enter("extractBySkill", [
56
+ rItem(query, "query"),
57
+ ]);
58
+ const fail = traceFail(t);
59
+ // Use climbAttentionAll to get the FULL ranked list, not just the
60
+ // roots that cleared commitVotes' significance floor. The floor
61
+ // gates further points of attention for fusion, but extraction only
62
+ // needs ONE anchor that IS a span-shaped skill exemplar — and on
63
+ // some seeds the top-voted anchor is not one (e.g. a concept-merge
64
+ // nickname outvotes the painting exemplars on shared substrings,
65
+ // while the exemplars' votes fall below the floor). Iterating the
66
+ // ranked list instead of just the roots lets extraction reach the
67
+ // first painting-exemplar anchor regardless of its floor status.
68
+ //
69
+ const { ranked } = await pre.attention();
70
+ if (ranked.length === 0) {
71
+ return fail("no consensus anchor — no skill to apply");
72
+ }
73
+ let exemplar: SkillInfo | null = null;
74
+ let chosenAnchor: number | null = null;
75
+ let skipped = 0;
76
+ for (const cand of ranked) {
77
+ const ex = await pre.spanShapedOf(cand.anchor);
78
+ if (ex) {
79
+ exemplar = ex;
80
+ chosenAnchor = cand.anchor;
81
+ break;
82
+ }
83
+ skipped++;
84
+ }
85
+ if (exemplar === null) {
86
+ ctx.trace?.step(
87
+ "trySkillAnchors",
88
+ [],
89
+ [],
90
+ `none of ${ranked.length} ranked anchor(s) is a span-shaped skill exemplar`,
91
+ );
92
+ return fail("no consensus root is a span-shaped skill exemplar");
93
+ }
94
+ if (skipped > 0) {
95
+ ctx.trace?.step(
96
+ "trySkillAnchors",
97
+ [
98
+ rItem(query.subarray(0, 0), `skipped ${skipped}`),
99
+ rNode(ctx, chosenAnchor!, "chosen"),
100
+ ],
101
+ [],
102
+ `skipped ${skipped} anchor(s) that are not skill exemplars`,
103
+ );
104
+ }
105
+ const { contextBytes, answerBytes } = exemplar;
106
+
107
+ const ansCtxRuns = answerRunsInContext(ctx, contextBytes, answerBytes);
108
+ if (ansCtxRuns === null || ansCtxRuns.length === 0) {
109
+ return fail("answer is not a subsequence of the context");
110
+ }
111
+
112
+ if (ansCtxRuns.length > 1) {
113
+ ctx.trace?.step(
114
+ "decomposeAnswer",
115
+ [rItem(answerBytes, "multi-piece-answer")],
116
+ ansCtxRuns.map((r) =>
117
+ rItem(contextBytes.subarray(r.start, r.end), "piece", undefined, [
118
+ r.start,
119
+ r.end,
120
+ ])
121
+ ),
122
+ `answer splits into ${ansCtxRuns.length} piece(s) within the exemplar context`,
123
+ );
124
+ }
125
+
126
+ const pieces: Uint8Array[] = [];
127
+ const accounted: Array<[number, number]> = [];
128
+ for (let ri = 0; ri < ansCtxRuns.length; ri++) {
129
+ const run = ansCtxRuns[ri];
130
+ const isLast = ri === ansCtxRuns.length - 1;
131
+
132
+ const framePreLen = Math.min(run.start, ctx.space.maxGroup);
133
+ const framePre = run.start > 0
134
+ ? contextBytes.subarray(run.start - framePreLen, run.start)
135
+ : null;
136
+
137
+ const frames: Array<[number, number]> = [];
138
+ let start = 0;
139
+ if (framePre) {
140
+ const prePos = locate(ctx, query, framePre, 0, pre.rec.sites);
141
+ if (prePos < 0) continue;
142
+ start = prePos + framePre.length;
143
+ frames.push([prePos, start]); // the located frame IS matched evidence
144
+ }
145
+
146
+ let end: number;
147
+ if (isLast) {
148
+ if (run.end < contextBytes.length) {
149
+ const framePostLen = Math.min(
150
+ contextBytes.length - run.end,
151
+ ctx.space.maxGroup,
152
+ );
153
+ const framePost = contextBytes.subarray(
154
+ run.end,
155
+ run.end + framePostLen,
156
+ );
157
+ const postPos = locate(
158
+ ctx,
159
+ query.subarray(start),
160
+ framePost,
161
+ 0,
162
+ pre.rec.sites,
163
+ );
164
+ if (postPos < 0) continue;
165
+ end = start + postPos;
166
+ frames.push([end, end + framePost.length]); // matched post-frame
167
+ } else {
168
+ end = query.length;
169
+ }
170
+ } else {
171
+ const nextRun = ansCtxRuns[ri + 1];
172
+ const nextPreLen = Math.min(nextRun.start, ctx.space.maxGroup);
173
+ const nextPre = contextBytes.subarray(
174
+ nextRun.start - nextPreLen,
175
+ nextRun.start,
176
+ );
177
+ const nextPos = locate(
178
+ ctx,
179
+ query.subarray(start),
180
+ nextPre,
181
+ 0,
182
+ pre.rec.sites,
183
+ );
184
+ if (nextPos < 0) {
185
+ end = start + run.ansLen;
186
+ } else {
187
+ end = start + nextPos;
188
+ frames.push([end, end + nextPre.length]); // matched next-frame
189
+ }
190
+ }
191
+ if (start >= end) continue;
192
+ pieces.push(query.subarray(start, end));
193
+ accounted.push(...frames);
194
+ // Bounded on both sides ⇒ the read span itself is explained (see doc).
195
+ // frames carries the pre-border (when the answer is not at the context's
196
+ // start) and the located right border (post-frame or next piece's
197
+ // pre-frame); only when BOTH borders were located is the read bounded.
198
+ const preBounded = run.start === 0 || frames.some(([, e]) => e === start);
199
+ const postBounded = frames.some(([b]) => b === end);
200
+ if (preBounded && postBounded) accounted.push([start, end]);
201
+ }
202
+ if (pieces.length === 0) {
203
+ return fail("no answer piece's frame located in the query");
204
+ }
205
+
206
+ const out = pieces.length === 1 ? pieces[0] : concatBytes(pieces);
207
+ t?.done(
208
+ [rItem(out, "extracted")],
209
+ pieces.length === 1
210
+ ? `apply a learnt extraction skill — read the analogous span of the query` +
211
+ ` framed like "${decodeText(answerBytes)}" sits in its exemplar`
212
+ : `apply a learnt MULTI-PIECE skill — read ${pieces.length} analogous` +
213
+ ` pieces of the query and synthesize them like "${
214
+ decodeText(answerBytes)
215
+ }"`,
216
+ );
217
+ return {
218
+ bytes: out,
219
+ accounted,
220
+ unexplained: unexplainedLabel(query, accounted),
221
+ };
222
+ }
223
+
224
+ // ── The two span-shape readings: OPEN acceptance vs. STRONG decomposition ──
225
+ //
226
+ // isSpanShaped and answerRunsInContext read the SAME relation ("the answer is
227
+ // drawn from the context") at two deliberately different strengths, and they
228
+ // are NOT interchangeable:
229
+ //
230
+ // • isSpanShaped — the OPEN reading: any in-order embedding (a sparse
231
+ // subsequence, arbitrary gaps). O(|context|) byte scan. Used to ACCEPT
232
+ // an exemplar candidate.
233
+ // • answerRunsInContext — the STRONG reading: a greedy longest-run
234
+ // DECOMPOSITION into contiguous pieces. Greedy-longest is strictly
235
+ // stronger than subsequence (a long late match can consume context an
236
+ // earlier shorter choice needed), so an ACCEPTED exemplar can still fail
237
+ // to decompose — extractBySkill then fails with "answer is not a
238
+ // subsequence of the context" and think falls through to recall. That
239
+ // fall-through is BEHAVIOUR, pinned by the extraction suites: do not
240
+ // "unify" the two into one machine — replacing the open reading with the
241
+ // strong one silently rejects exemplars extraction today accepts, and
242
+ // replacing the strong one with a backtracking embedding changes which
243
+ // pieces are read out of the query.
244
+
245
+ /** Decompose an answer into substrings of its surrounding context, in order —
246
+ * the STRONG span-shape reading (see the section note above). Returns null
247
+ * when no greedy longest-run decomposition exists. Adjacent runs that
248
+ * connect contiguously are merged. */
249
+ export function answerRunsInContext(
250
+ _ctx: MindContext,
251
+ context: Uint8Array,
252
+ answer: Uint8Array,
253
+ ): Array<{ start: number; end: number; ansLen: number }> | null {
254
+ const pos = indexOf(context, answer, 0);
255
+ if (pos >= 0) {
256
+ return [{ start: pos, end: pos + answer.length, ansLen: answer.length }];
257
+ }
258
+
259
+ const runs: Array<
260
+ { start: number; end: number; ansLen: number }
261
+ > = [];
262
+ let ai = 0;
263
+ let ci = 0;
264
+ while (ai < answer.length) {
265
+ // Longest match of the remaining answer at any position of the remaining
266
+ // context: one direct extend per context position — O(|ctx|·match) per
267
+ // run, replacing the previous longest-first indexOf countdown whose
268
+ // repeated scans were cubic on long sparse-subsequence answers.
269
+ let bestLen = 0;
270
+ let bestPos = -1;
271
+ for (let p = ci; p < context.length; p++) {
272
+ let l = 0;
273
+ const maxL = Math.min(context.length - p, answer.length - ai);
274
+ if (maxL <= bestLen) break; // no later position can beat the best
275
+ while (l < maxL && context[p + l] === answer[ai + l]) l++;
276
+ if (l > bestLen) {
277
+ bestLen = l;
278
+ bestPos = p;
279
+ if (ai + l === answer.length) break; // the whole remainder matched
280
+ }
281
+ }
282
+ if (bestLen === 0) return null;
283
+ runs.push({ start: bestPos, end: bestPos + bestLen, ansLen: bestLen });
284
+ ai += bestLen;
285
+ ci = bestPos + bestLen;
286
+ }
287
+ const merged: Array<{ start: number; end: number; ansLen: number }> = [];
288
+ for (const r of runs) {
289
+ const last = merged[merged.length - 1];
290
+ if (last && r.start === last.end) {
291
+ last.end = r.end;
292
+ last.ansLen += r.ansLen;
293
+ } else {
294
+ merged.push({ ...r });
295
+ }
296
+ }
297
+ return merged.length > 0 ? merged : null;
298
+ }
299
+
300
+ /** Check whether an anchor is a span-shaped skill exemplar: it represents a
301
+ * fact whose context and answer together form a span-in-context pattern.
302
+ * If the anchor has a nextOf continuation, that is the answer and the anchor
303
+ * itself is the context. Otherwise the anchor's prevOf parents provide
304
+ * candidate contexts, and the longest one whose span is span-shaped wins. */
305
+ export async function skillExemplar(
306
+ ctx: MindContext,
307
+ anchor: number,
308
+ guide?: Vec | null,
309
+ ): Promise<{ contextBytes: Uint8Array; answerBytes: Uint8Array } | null> {
310
+ if (ctx.store.hasNext(anchor)) {
311
+ const contextBytes = read(ctx, anchor);
312
+ const answerBytes = await follow(ctx, anchor, guide);
313
+ if (
314
+ answerBytes !== null && isSpanShaped(ctx, contextBytes, answerBytes)
315
+ ) {
316
+ return { contextBytes, answerBytes };
317
+ }
318
+ return null;
319
+ }
320
+ const answerBytes = read(ctx, anchor);
321
+ // Candidate contexts, capped at the hub bound (a common answer's reverse
322
+ // fan-in is corpus-sized).
323
+ const capped = ctx.store.prevFirst(anchor, hubBound(ctx));
324
+ const spanShaped: Array<{ id: number; bytes: Uint8Array }> = [];
325
+ for (const p of capped) {
326
+ const ctxB = read(ctx, p);
327
+ if (ctxB.length > 0 && isSpanShaped(ctx, ctxB, answerBytes)) {
328
+ spanShaped.push({ id: p, bytes: ctxB });
329
+ }
330
+ }
331
+ if (spanShaped.length === 0) return null;
332
+ // Among span-shaped contexts, the longest wins (the smallest spanning frame
333
+ // heuristic's dual: more frame to locate in the query); the query gist,
334
+ // when given, breaks LENGTH TIES via chooseAmong — the same reverse-regime
335
+ // disambiguator every context pick uses, whose gist cache spares the
336
+ // re-fold this block once paid per tied candidate. Same strict first-seen
337
+ // tie-break as the hand loop it replaces.
338
+ const maxLen = Math.max(...spanShaped.map((s) => s.bytes.length));
339
+ const longest = spanShaped.filter((s) => s.bytes.length === maxLen);
340
+ let contextBytes = longest[0].bytes;
341
+ if (guide && longest.length > 1) {
342
+ const pick = chooseAmong(ctx, longest.map((s) => s.id), guide).id;
343
+ contextBytes = longest.find((s) => s.id === pick)!.bytes;
344
+ }
345
+ return { contextBytes, answerBytes };
346
+ }
347
+
348
+ /** Whether the answer is a SPARSE subsequence of the context (bytes in
349
+ * order, arbitrary gaps) — the OPEN span-shape reading (see the section
350
+ * note above). This is what lets extraction validate a MULTI-PIECE
351
+ * exemplar whose answer is stitched from several context runs — but it is
352
+ * deliberately permissive, so it must never be used as evidence that one
353
+ * span was "drawn from" another (see {@link containsSpan} for that).
354
+ *
355
+ * There is deliberately NO containsSpan pre-check here: strict containment
356
+ * IMPLIES the subsequence embedding (a contiguous run, or a resolved node —
357
+ * whose content-addressed identity means its bytes occur contiguously — is
358
+ * an in-order embedding with zero gaps), so the scan below decides alone,
359
+ * with the same truth value. The old pre-check re-perceived the context
360
+ * (a full river fold) per CANDIDATE in skillExemplar's √N-capped loop —
361
+ * pure cost, no discrimination. */
362
+ export function isSpanShaped(
363
+ _ctx: MindContext,
364
+ context: Uint8Array,
365
+ answer: Uint8Array,
366
+ ): boolean {
367
+ let ai = 0;
368
+ for (let ci = 0; ci < context.length && ai < answer.length; ci++) {
369
+ if (context[ci] === answer[ai]) ai++;
370
+ }
371
+ return ai === answer.length;
372
+ }
373
+
374
+ /** STRICT containment: the answer's resolved node appears in the context's
375
+ * folded tree, or the answer occurs as one CONTIGUOUS byte run of the
376
+ * context. This is real evidence the answer was drawn from the context.
377
+ * Fusion gates on this — the sparse-subsequence reading of
378
+ * {@link isSpanShaped} is trivially satisfied by short answers over long
379
+ * queries ("cold" is a gap-tolerant subsequence of most sentences holding
380
+ * c…o…l…d in order), and gating fusion on it silently starved multi-topic
381
+ * queries of their further points of attention. */
382
+ export function containsSpan(
383
+ ctx: MindContext,
384
+ context: Uint8Array,
385
+ answer: Uint8Array,
386
+ ): boolean {
387
+ const ansId = resolve(ctx, answer);
388
+ if (ansId !== null) {
389
+ let found = false;
390
+ foldTree(ctx, perceive(ctx, context), 0, (_n, _s, _e, node) => {
391
+ if (node === ansId) found = true;
392
+ });
393
+ if (found) return true;
394
+ }
395
+ return indexOf(context, answer, 0) >= 0;
396
+ }
397
+
398
+ // ── Pipeline mechanism ──────────────────────────────────────────────────────
399
+
400
+ export const extractionMechanism: PipelineMechanism = {
401
+ name: "extraction",
402
+ provenance: "extract",
403
+ async floor(_ctx, _query, pre, worthRunning) {
404
+ // Extraction's floor is always exactly CONCEPT+STEP when it exists —
405
+ // same investment discipline as CAST's (see cast.ts): when the bound
406
+ // already cannot beat the incumbent, return it UNINVESTED (never
407
+ // first-touch the climb just to be pruned).
408
+ if (!worthRunning(CONCEPT + STEP)) return CONCEPT + STEP;
409
+ if ((await pre.attention()).ranked.length === 0) return null;
410
+ return CONCEPT + STEP;
411
+ },
412
+ async run(ctx, query, pre) {
413
+ const ex = await extractBySkill(ctx, query, pre);
414
+ if (!ex) return [];
415
+ return [{
416
+ bytes: ex.bytes,
417
+ accounted: ex.accounted,
418
+ moves: CONCEPT + STEP * ex.accounted.length,
419
+ unexplained: ex.unexplained,
420
+ }];
421
+ },
422
+ };
@@ -0,0 +1,241 @@
1
+ // mechanisms/recall.ts — Recall by resonance (Grounding IV).
2
+ //
3
+ // The recall mechanism resonates the whole query's gist against the content
4
+ // index and grounds the nearest learned form. Four tiers, orderly degrading
5
+ // from exact self-match to honest echo.
6
+
7
+ import { cosine } from "../../vec.js";
8
+ import {
9
+ consensusFloor,
10
+ identityBar,
11
+ reachThreshold,
12
+ significanceBar,
13
+ } from "../../geometry.js";
14
+ import type { MindContext } from "../types.js";
15
+ import { gistOf, read, resolve } from "../primitives.js";
16
+ import { corpusN, hubBound } from "../traverse.js";
17
+ import { project, reverseContext } from "../match.js";
18
+ import { CONCEPT, STEP } from "../graph-search.js";
19
+ import { unexplainedLabel } from "../rationale.js";
20
+ import type { PipelineMechanism, Precomputed } from "../pipeline-mechanism.js";
21
+ import { rItem, rNode } from "../trace.js";
22
+
23
+ /** A recall result. */
24
+ export interface RecallResult {
25
+ bytes: Uint8Array;
26
+ echoed: boolean;
27
+ accounted: Array<[number, number]>;
28
+ moves: number;
29
+ unexplained: string;
30
+ }
31
+
32
+ /** Recall the answer by resonating the whole query against the content index. */
33
+ export async function recallByResonance(
34
+ ctx: MindContext,
35
+ query: Uint8Array,
36
+ pre: Precomputed,
37
+ ): Promise<RecallResult | null> {
38
+ const t = ctx.trace?.enter("recallByResonance", [
39
+ rItem(query, "query"),
40
+ ]);
41
+ const whole_: Array<[number, number]> = [[0, query.length]];
42
+ const nothing: Array<[number, number]> = [];
43
+ const ground = (
44
+ bytes: Uint8Array | null,
45
+ note: string,
46
+ accounted: Array<[number, number]>,
47
+ moves: number,
48
+ echoed = false,
49
+ ): RecallResult | null => {
50
+ t?.done(
51
+ bytes === null
52
+ ? []
53
+ : [rItem(bytes, "answer", resolve(ctx, bytes) ?? undefined)],
54
+ note,
55
+ );
56
+ return bytes === null ? null : {
57
+ bytes,
58
+ echoed,
59
+ accounted,
60
+ moves,
61
+ unexplained: unexplainedLabel(query, accounted),
62
+ };
63
+ };
64
+ const k = pre.k;
65
+ const queryGist = pre.guide;
66
+
67
+ // 0. Exact self-match — content-addressed, deterministic.
68
+ const qId = pre.queryResolved;
69
+ if (qId !== null) {
70
+ const rev = ctx.store.prevFirst(qId, hubBound(ctx));
71
+ const g = reverseContext(ctx, qId, queryGist, rev);
72
+ if (g !== null) {
73
+ return ground(
74
+ g,
75
+ rev.length === 1
76
+ ? "exact self-match — reverse recall to the sole predecessor"
77
+ : "exact self-match — reverse recall to the best-resonating predecessor",
78
+ nothing,
79
+ STEP,
80
+ );
81
+ }
82
+ }
83
+
84
+ const whole = await ctx.store.resonate(queryGist, k);
85
+ if (whole.length === 0) {
86
+ return ground(null, "empty store — nothing to resonate with", [], 0);
87
+ }
88
+ const top = whole[0];
89
+ ctx.trace?.step(
90
+ "resonate",
91
+ [rItem(query, "query-gist")],
92
+ whole.map((h) => rNode(ctx, h.id, "hit", h.score)),
93
+ `resonate the whole-query gist → ${whole.length} nearest learnt form(s)`,
94
+ );
95
+
96
+ // 1. Clean resonance — the scale-aware identity claim. The ANGLE
97
+ // (top.score) carries the shared fraction; the query's MAGNITUDE (√len,
98
+ // the linear fold's own norm) converts the tolerated foreign fraction
99
+ // into bytes — at most one river window (see {@link identityBar}). A
100
+ // fixed cosine bar let long queries claim "near-identical" while whole
101
+ // windows — an answer word — differed.
102
+ if (
103
+ top.score >= identityBar(ctx.store.D, ctx.space.maxGroup, query.length)
104
+ ) {
105
+ for (const h of whole) {
106
+ if (h.id === qId || h.score >= 1.0) {
107
+ const rev = ctx.store.prevFirst(h.id, hubBound(ctx));
108
+ const g = reverseContext(ctx, h.id, queryGist, rev);
109
+ if (g !== null) {
110
+ return ground(
111
+ g,
112
+ rev.length === 1
113
+ ? "perfect self-match — reverse recall to the sole predecessor"
114
+ : "perfect self-match — reverse recall to the best-resonating predecessor",
115
+ nothing,
116
+ STEP,
117
+ );
118
+ }
119
+ const own = read(ctx, h.id);
120
+ if (own.length > 0) {
121
+ return ground(
122
+ own,
123
+ "perfect self-match — the query IS this node",
124
+ nothing,
125
+ STEP,
126
+ );
127
+ }
128
+ }
129
+ const g = await project(ctx, h.id, queryGist);
130
+ if (g) {
131
+ return ground(
132
+ g,
133
+ "clean whole-query resonance — ground the nearest hit",
134
+ whole_,
135
+ STEP,
136
+ );
137
+ }
138
+ }
139
+ }
140
+
141
+ // 2. Scaffolding-dominated.
142
+ if (top.score >= significanceBar(ctx.store.D)) {
143
+ const N = corpusN(ctx);
144
+ const minVote = consensusFloor(N);
145
+ // The committed points of attention ARE the shared climb's roots (same
146
+ // query, same k, same DF mode) — read them from Precomputed instead of
147
+ // re-climbing, so even a traced response pays for the climb once.
148
+ const forest = (await pre.attention()).roots;
149
+ if (forest.length > 0 && forest[0].vote >= minVote) {
150
+ const g = await project(ctx, forest[0].anchor, queryGist);
151
+ if (g) {
152
+ return ground(
153
+ g,
154
+ "scaffolding-dominated query — ground the consensus-climb anchor",
155
+ [[forest[0].start, forest[0].end]],
156
+ CONCEPT,
157
+ );
158
+ }
159
+ }
160
+ }
161
+
162
+ // 3. Last resort — gated on the FRACTION OF THE QUERY the grounding
163
+ // explains, not the raw cosine. Root gists are unit vectors, but their
164
+ // magnitudes are recoverable from the byte lengths (‖·‖ = √len under the
165
+ // linear fold): cos = shared/√(lenQ·lenG), so shared/lenQ = cos·√(lenG/lenQ).
166
+ // The raw cosine punished honest containment — a query fully inside a
167
+ // longer grounded answer scored √(lenQ/lenG) and was refused — and let a
168
+ // long answer sharing only scaffolding pass; the query-relative fraction
169
+ // measures exactly what the reach bar means: how much of THE QUERY the
170
+ // store accounts for.
171
+ const fracOfQuery = (cos: number, otherLen: number): number =>
172
+ Math.min(1, cos * Math.sqrt(otherLen / Math.max(1, query.length)));
173
+ for (const h of whole) {
174
+ const g = await project(ctx, h.id, queryGist);
175
+ if (g) {
176
+ if (
177
+ fracOfQuery(cosine(queryGist, gistOf(ctx, g)), g.length) >=
178
+ reachThreshold(ctx.space.maxGroup)
179
+ ) {
180
+ return ground(
181
+ g,
182
+ "last resort: the nearest grounded whole-query hit",
183
+ [],
184
+ STEP,
185
+ );
186
+ }
187
+ }
188
+ }
189
+ // The refusal/echo decision, in the same query-relative units — the top
190
+ // hit's magnitude read from the store (contentLen: √bytes IS the linear
191
+ // fold's gist norm), never from re-reading its bytes.
192
+ // The magnitude read SATURATES at the decision point: frac reaches the
193
+ // reach bar exactly when lenH = lenQ·(reach/score)², so the walk never
194
+ // needs to see past that — a huge conversation root costs a capped read,
195
+ // and a clamped return decides "pass" identically (frac ≥ reach).
196
+ const reach = reachThreshold(ctx.space.maxGroup);
197
+ const lenCap = Math.ceil(
198
+ query.length * (reach / Math.max(top.score, 1e-6)) ** 2,
199
+ ) + 1;
200
+ if (fracOfQuery(top.score, ctx.store.contentLen(top.id, lenCap)) < reach) {
201
+ return ground(
202
+ null,
203
+ "below reach threshold — nothing in the store relates to this query",
204
+ [],
205
+ 0,
206
+ );
207
+ }
208
+ // Honest echo.
209
+ return ground(
210
+ read(ctx, top.id),
211
+ "last resort: the nearest resonant form's own bytes (echo, not grounded)",
212
+ [],
213
+ 0,
214
+ true,
215
+ );
216
+ }
217
+
218
+ // ── Pipeline mechanism ──────────────────────────────────────────────────────
219
+
220
+ export const recallMechanism: PipelineMechanism = {
221
+ name: "recall",
222
+ provenance: "recall",
223
+ // Recall's floor is free to state (one STEP-grade projection) and its run
224
+ // gates its own tiers — no expensive investment happens inside floor, so
225
+ // there is nothing to guard with worthRunning here: the pipeline's own
226
+ // check prunes run() against the incumbent.
227
+ async floor(_ctx, _query, _pre, _worthRunning) {
228
+ return STEP;
229
+ },
230
+ async run(ctx, query, pre) {
231
+ const r = await recallByResonance(ctx, query, pre);
232
+ if (!r) return [];
233
+ return [{
234
+ bytes: r.bytes,
235
+ accounted: r.accounted,
236
+ moves: r.moves,
237
+ unexplained: r.unexplained,
238
+ provenance: r.echoed ? "recall-echo" : "recall",
239
+ }];
240
+ },
241
+ };