@hviana/sema 0.4.6 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/AGENTS.md +290 -77
  2. package/HOW_IT_WORKS.md +2170 -735
  3. package/dist/example/train_base.d.ts +9 -3
  4. package/dist/example/train_base.js +21 -4
  5. package/dist/src/canon.d.ts +19 -0
  6. package/dist/src/canon.js +28 -0
  7. package/dist/src/geometry.d.ts +52 -0
  8. package/dist/src/geometry.js +87 -1
  9. package/dist/src/mind/bridge.js +27 -1
  10. package/dist/src/mind/frame-filler.d.ts +15 -0
  11. package/dist/src/mind/frame-filler.js +535 -0
  12. package/dist/src/mind/learning.js +6 -11
  13. package/dist/src/mind/mechanisms/cast.js +72 -2
  14. package/dist/src/mind/mechanisms/cover.js +6 -1
  15. package/dist/src/mind/mechanisms/extraction.js +27 -0
  16. package/dist/src/mind/mechanisms/recall.js +214 -34
  17. package/dist/src/mind/mind.d.ts +49 -1
  18. package/dist/src/mind/mind.js +137 -10
  19. package/dist/src/mind/pipeline-mechanism.d.ts +7 -0
  20. package/dist/src/mind/pipeline.js +29 -1
  21. package/dist/src/mind/prefix-completion.d.ts +59 -0
  22. package/dist/src/mind/prefix-completion.js +270 -0
  23. package/dist/src/mind/primitives.d.ts +29 -10
  24. package/dist/src/mind/primitives.js +52 -61
  25. package/dist/src/mind/recognition.js +119 -9
  26. package/dist/src/mind/traverse.d.ts +32 -0
  27. package/dist/src/mind/traverse.js +52 -0
  28. package/dist/src/mind/types.d.ts +55 -16
  29. package/dist/src/mind/types.js +68 -19
  30. package/dist/src/rabitq-ivf/src/rabitq.js +31 -1
  31. package/dist/src/store.d.ts +21 -0
  32. package/dist/src/store.js +21 -0
  33. package/example/train_base.ts +21 -4
  34. package/package.json +1 -1
  35. package/src/canon.ts +28 -0
  36. package/src/geometry.ts +100 -1
  37. package/src/mind/bridge.ts +34 -0
  38. package/src/mind/frame-filler.ts +604 -0
  39. package/src/mind/learning.ts +5 -9
  40. package/src/mind/mechanisms/cast.ts +70 -2
  41. package/src/mind/mechanisms/cover.ts +6 -1
  42. package/src/mind/mechanisms/extraction.ts +27 -0
  43. package/src/mind/mechanisms/recall.ts +236 -37
  44. package/src/mind/mind.ts +154 -14
  45. package/src/mind/pipeline-mechanism.ts +7 -0
  46. package/src/mind/pipeline.ts +33 -1
  47. package/src/mind/prefix-completion.ts +314 -0
  48. package/src/mind/primitives.ts +59 -70
  49. package/src/mind/recognition.ts +117 -6
  50. package/src/mind/traverse.ts +52 -0
  51. package/src/mind/types.ts +98 -42
  52. package/src/rabitq-ivf/src/rabitq.ts +31 -1
  53. package/src/store.ts +25 -0
  54. package/test/13-conversation.test.mjs +13 -0
  55. package/test/57-fusion-order.test.mjs +65 -0
  56. package/test/65-ann-recall.test.mjs +331 -0
  57. package/test/66-query-edge-whitespace.test.mjs +99 -0
  58. package/test/67-climb-anchor-breadth.test.mjs +113 -0
  59. package/test/68-extraction-unanchored.test.mjs +79 -0
  60. package/test/69-frame-filler.test.mjs +115 -0
  61. package/test/70-prefix-completion.test.mjs +170 -0
  62. package/test/71-embedded-canon-equivalence.test.mjs +121 -0
  63. package/test/72-prefix-candidate-supply.test.mjs +114 -0
  64. package/test/73-scaffolding-only-bridge-abstains.test.mjs +178 -0
  65. package/test/74-prefix-trap-not-sprung-early.test.mjs +114 -0
  66. package/test/75-multiturn-context-optimisation.test.mjs +1082 -0
@@ -18,6 +18,19 @@
18
18
  // accumulated bytes at inference. The Conversation API tracks turn-boundary
19
19
  // offsets explicitly so no separator character is needed — the geometry never
20
20
  // inspects content to find turn boundaries.
21
+ //
22
+ // "NO SEPARATOR IS NEEDED" ≠ "A SEPARATOR IS A PROBLEM". This file joins its
23
+ // turns with nothing; example/train_base.ts joins its oasst2 turns with "\n".
24
+ // Both are correct, and neither is a convention the other has to match: a
25
+ // separator is CORPUS CONTENT, folded like any other byte, while a turn
26
+ // boundary is an OFFSET the API carries beside the bytes. A harness replaying
27
+ // a "\n"-joined corpus simply passes `"\n" + turnText` to addTurn and gets the
28
+ // trained byte stream back exactly. See Mind.addTurn's "ON SEPARATORS" note
29
+ // for the full statement — it exists because a review read the mismatch
30
+ // between this file's join and the trainer's as an architectural
31
+ // incompatibility, and it is not one. If you are comparing this harness to a
32
+ // corpus and getting poor recall, check that you are feeding the bytes that
33
+ // were actually trained before concluding anything about the engine.
21
34
  // ─────────────────────────────────────────────────────────────────────────
22
35
 
23
36
  import { test } from "node:test";
@@ -82,6 +82,71 @@ test("2. reversing the question reverses the fused answer", async () => {
82
82
  await mind.store.close();
83
83
  });
84
84
 
85
+ test("2b. a topic is never ECHOED back instead of answered", async () => {
86
+ // The failure this pins: "What is the capital of France? And what is the
87
+ // largest planet?" answered "The capital of France is Paris.What is the
88
+ // largest planet?" — one topic answered, the other repeated verbatim.
89
+ //
90
+ // It hinged on CASE. The comparison schema seats a directly-aligned analog
91
+ // by its own bytes rather than chasing a forward edge, which is correct when
92
+ // those bytes are an answer (test/43 pins that) and an echo when they are
93
+ // the question the asker just asked. The guard against that is a restatement
94
+ // check, and a BYTE-EXACT one missed here: the trained node is "What is the
95
+ // largest planet?" while the query says "And what is the largest planet?" —
96
+ // the same words, one capital apart. The check now reads the response's own
97
+ // injected canon, so it sees what the rest of the mind sees.
98
+ //
99
+ // SCOPE: this asserts only that nothing is echoed. Whether BOTH topics get
100
+ // fused is a separate, corpus- and seed-dependent property of the consensus
101
+ // climb — at this file's seed the second point is sometimes not committed at
102
+ // all, which is why test 1 above guards its ordering assertion on both names
103
+ // being present. Answering one topic and staying silent about the other is a
104
+ // coverage limit; answering one and parroting the other is a defect.
105
+ //
106
+ // Asserted in BOTH orders because the echo appeared in only one: which topic
107
+ // got echoed depended on whether the climb landed on the question node or
108
+ // the answer node, so a single-order test passes while the bug is live.
109
+ // ITS OWN CORPUS, DELIBERATELY. The file's shared `trained()` fixture cannot
110
+ // reproduce this: with five same-frame facts the climb often commits only
111
+ // ONE point, so there is no second topic to echo and the test would pass
112
+ // against the unfixed code (verified — it did). The echo needs exactly two
113
+ // topics, each a bare question node whose answer hangs off a forward edge.
114
+ const mind = new Mind({
115
+ seed: 7,
116
+ store: new SQliteStore({ path: ":memory:" }),
117
+ });
118
+ await mind.ingest([
119
+ ["What is the capital of France?", "The capital of France is Paris."],
120
+ ["What is the largest planet?", "The largest planet is Jupiter."],
121
+ ]);
122
+ for (
123
+ const q of [
124
+ "What is the capital of France? And what is the largest planet?",
125
+ "What is the largest planet? And what is the capital of France?",
126
+ ]
127
+ ) {
128
+ const a = await mind.respondText(q);
129
+ assert.ok(
130
+ !/And what is/i.test(a),
131
+ `the query was echoed rather than answered: ${JSON.stringify(a)} for ${
132
+ JSON.stringify(q)
133
+ }`,
134
+ );
135
+ assert.ok(
136
+ a.includes("Paris") && a.includes("Jupiter"),
137
+ `both topics must be ANSWERED, got ${JSON.stringify(a)} for ${
138
+ JSON.stringify(q)
139
+ }`,
140
+ );
141
+ // Nor may an answer be a bare restatement of one of the asked questions.
142
+ assert.ok(
143
+ !/^\s*What is the (largest planet|capital of France)\?\s*$/i.test(a),
144
+ `the answer is just the question restated: ${JSON.stringify(a)}`,
145
+ );
146
+ }
147
+ await mind.store.close();
148
+ });
149
+
85
150
  test("3. a single-topic answer is unchanged by the ordering rule", async () => {
86
151
  const mind = await trained();
87
152
  assert.match(
@@ -0,0 +1,331 @@
1
+ // 65-ann-recall.test.mjs — the ACCURACY half of rabitq-ivf's speed/accuracy
2
+ // trade, pinned to a measured baseline.
3
+ //
4
+ // Every geometry constant in the sub-library (rotation `rounds`, `queryBits`,
5
+ // `efSearch`/nprobe, SPLIT_MAX, and the estimator's own arithmetic) buys speed
6
+ // by giving up recall. Nothing measured the recall, so nothing could be
7
+ // changed: a faster encoder that quietly lost neighbours looked exactly like a
8
+ // faster encoder. These assertions are that missing half — an optimisation is
9
+ // free only if it holds this line.
10
+ //
11
+ // Deliberately SELF-CONTAINED: vectors are real Sema gists, but folded here
12
+ // from generated text rather than read out of sema.*, so the test is
13
+ // deterministic, needs no trained store, and cannot drift when one is
14
+ // retrained.
15
+ //
16
+ // Absolute recall here (~78%) runs higher than the ~71% measured against the
17
+ // trained store (100,000 gists sampled from sema.*, 400 queries, exact-cosine
18
+ // ground truth): generated word-salad folds to an easier distribution than a
19
+ // real corpus. The FLOORS are what matter, and they are set from what this
20
+ // file itself measures, with margin for the estimator's quantisation noise —
21
+ // never from what looked good on a different distribution.
22
+
23
+ import { test } from "node:test";
24
+ import assert from "node:assert/strict";
25
+ import { Mind } from "../dist/src/index.js";
26
+ import { gistOf } from "../dist/src/mind/primitives.js";
27
+ import { IvfIndex } from "../dist/src/rabitq-ivf/src/ivf.js";
28
+ import { RaBitQuantizer } from "../dist/src/rabitq-ivf/src/rabitq.js";
29
+
30
+ const SEED = 7;
31
+ const K = 10;
32
+
33
+ // A deterministic PRNG — the vectors must be identical on every run, or a
34
+ // recall floor is a coin flip.
35
+ const prng = (s) => () => {
36
+ s ^= s << 13;
37
+ s >>>= 0;
38
+ s ^= s >>> 17;
39
+ s ^= s << 5;
40
+ s >>>= 0;
41
+ return s / 4294967296;
42
+ };
43
+
44
+ const WORDS = [
45
+ "the",
46
+ "a",
47
+ "of",
48
+ "in",
49
+ "system",
50
+ "fold",
51
+ "vector",
52
+ "index",
53
+ "query",
54
+ "gist",
55
+ "node",
56
+ "edge",
57
+ "store",
58
+ "cluster",
59
+ "recall",
60
+ "code",
61
+ "byte",
62
+ "span",
63
+ "chunk",
64
+ "seed",
65
+ "paris",
66
+ "france",
67
+ "steel",
68
+ "ice",
69
+ "planet",
70
+ "sun",
71
+ "music",
72
+ "river",
73
+ "light",
74
+ "stone",
75
+ ];
76
+
77
+ /** `n` distinct real Sema gists, folded from generated text. */
78
+ function gists(n) {
79
+ const mind = new Mind({ seed: SEED });
80
+ const rnd = prng(20260729);
81
+ const enc = new TextEncoder();
82
+ const out = [];
83
+ const seen = new Set();
84
+ while (out.length < n) {
85
+ let s = "";
86
+ const len = 3 + ((rnd() * 14) | 0);
87
+ for (let i = 0; i < len; i++) {
88
+ s += (i ? " " : "") + WORDS[(rnd() * WORDS.length) | 0];
89
+ }
90
+ s += ".";
91
+ if (seen.has(s)) continue;
92
+ seen.add(s);
93
+ const g = gistOf(mind, enc.encode(s));
94
+ let sq = 0;
95
+ for (let i = 0; i < g.length; i++) sq += g[i] * g[i];
96
+ if (sq > 0) out.push(g);
97
+ }
98
+ return { vecs: out, D: mind.store.D };
99
+ }
100
+
101
+ function indexOf(vecs, D, rounds = 3) {
102
+ const idx = new IvfIndex(":memory:", {
103
+ dim: D,
104
+ rotationRounds: rounds,
105
+ seed: SEED,
106
+ cacheSizeMb: 64,
107
+ });
108
+ idx.begin();
109
+ for (let i = 0; i < vecs.length; i++) {
110
+ idx.insert(i, idx.encodeToBytes(vecs[i]));
111
+ }
112
+ idx.commit();
113
+ idx.commitFlush();
114
+ return idx;
115
+ }
116
+
117
+ const queryIds = (n, q) => {
118
+ const out = [];
119
+ for (let i = 0; i < q; i++) out.push(Math.floor((i + 0.5) * n / q));
120
+ return out;
121
+ };
122
+
123
+ /** Exact cosine top-k by brute force — the only true ground truth here. */
124
+ function exactTopK(vecs, norms, qi, k) {
125
+ const q = vecs[qi], qn = norms[qi];
126
+ const ds = new Float64Array(vecs.length);
127
+ for (let j = 0; j < vecs.length; j++) {
128
+ const v = vecs[j];
129
+ let dot = 0;
130
+ for (let i = 0; i < q.length; i++) dot += q[i] * v[i];
131
+ const den = qn * norms[j];
132
+ ds[j] = den === 0 ? 1 : 1 - dot / den;
133
+ }
134
+ return new Set(
135
+ Array.from(ds.keys()).sort((a, b) => ds[a] - ds[b]).slice(0, k),
136
+ );
137
+ }
138
+
139
+ // ── QUANTIZATION: what 1-bit coding costs, with routing removed ─────────────
140
+
141
+ test("quantization recall holds its measured floor (routing removed)", () => {
142
+ const { vecs, D } = gists(2000);
143
+ const norms = vecs.map((v) => {
144
+ let s = 0;
145
+ for (let i = 0; i < v.length; i++) s += v[i] * v[i];
146
+ return Math.sqrt(s);
147
+ });
148
+ const qs = queryIds(vecs.length, 100);
149
+ const truth = qs.map((qi) => exactTopK(vecs, norms, qi, K));
150
+
151
+ const idx = indexOf(vecs, D);
152
+ // nprobe = K clusters: every cluster is scanned, so the ONLY thing between
153
+ // the query and exact cosine is the 1-bit code.
154
+ const full = 4 * idx.clusterCount;
155
+ let hit = 0, want = 0;
156
+ for (let t = 0; t < qs.length; t++) {
157
+ for (const h of idx.searchKnn(vecs[qs[t]], K, full)) {
158
+ if (truth[t].has(h.id)) hit++;
159
+ }
160
+ want += truth[t].size;
161
+ }
162
+ idx.close();
163
+ const recall = hit / want;
164
+
165
+ // Measured 79.3% here (2,000 gists, 100 queries, k=10). The floor sits a
166
+ // clear margin below so that ordinary quantisation jitter cannot trip it,
167
+ // while a real loss — dropping a rotation round that mattered, widening the
168
+ // code, changing the estimator's arithmetic — moves recall by far more.
169
+ assert.ok(
170
+ recall >= 0.74,
171
+ `quantization recall@${K} fell to ${(recall * 100).toFixed(1)}% ` +
172
+ `(floor 74.0%, measured baseline 79.3%). The 1-bit code lost ` +
173
+ `neighbours it used to keep — this is the accuracy half of a ` +
174
+ `speed/accuracy trade, so a change that speeds up the encoder or the ` +
175
+ `estimator must NOT land here.`,
176
+ );
177
+ });
178
+
179
+ // ── ROUTING: what probing a subset of clusters costs ───────────────────────
180
+
181
+ test("routing recall degrades monotonically as nprobe shrinks", () => {
182
+ // 12,000 vectors splits into several clusters (SPLIT_MAX = 4096). Routing is
183
+ // then exercised by LOWERING ef — nprobe = ceil(ef/4) — rather than by
184
+ // growing the collection into the tens of thousands, which would cost the
185
+ // suite seconds to say the same thing.
186
+ const { vecs, D } = gists(12000);
187
+ const idx = indexOf(vecs, D);
188
+ const clusters = idx.clusterCount;
189
+ assert.ok(clusters >= 4, `expected several clusters, got ${clusters}`);
190
+
191
+ const qs = queryIds(vecs.length, 100);
192
+ const full = 4 * clusters; // nprobe >= clusters: nothing skipped
193
+ const ref = qs.map((qi) =>
194
+ new Set(idx.searchKnn(vecs[qi], K, full).map((h) => h.id))
195
+ );
196
+
197
+ const at = (ef) => {
198
+ let hit = 0, want = 0;
199
+ for (let t = 0; t < qs.length; t++) {
200
+ for (const h of idx.searchKnn(vecs[qs[t]], K, ef)) {
201
+ if (ref[t].has(h.id)) hit++;
202
+ }
203
+ want += ref[t].size;
204
+ }
205
+ return hit / want;
206
+ };
207
+
208
+ const one = at(4); // nprobe 1
209
+ const two = at(8); // nprobe 2
210
+ const all = at(16 * clusters);
211
+ idx.close();
212
+
213
+ // Probing every cluster must reproduce the reference exactly — this is the
214
+ // same scan, so anything below 1.0 means the probe ORDER dropped a cluster
215
+ // it ranked in, not that the quantizer was imprecise.
216
+ assert.equal(
217
+ all,
218
+ 1,
219
+ "probing every cluster must match the full scan exactly",
220
+ );
221
+ // More clusters probed is never worse.
222
+ assert.ok(
223
+ two >= one,
224
+ `routing recall fell when nprobe grew: nprobe=1 ${
225
+ (one * 100).toFixed(1)
226
+ }% ` +
227
+ `-> nprobe=2 ${(two * 100).toFixed(1)}%`,
228
+ );
229
+ // Measured 52.9% at nprobe=1 and 76.4% at nprobe=2 of 4 clusters. The floors
230
+ // guard the PIVOT quality: routing is only worth anything if the nearest
231
+ // cluster usually holds the nearest vectors, and a pivot chosen badly (a
232
+ // broken split, a majority-code regression) shows up here first.
233
+ assert.ok(
234
+ one >= 0.4,
235
+ `single-cluster routing recall ${(one * 100).toFixed(1)}% is below the ` +
236
+ `0.4 floor (measured 52.9%) — cluster pivots no longer predict where ` +
237
+ `a query's neighbours live.`,
238
+ );
239
+ });
240
+
241
+ // ── ESTIMATOR: the arithmetic itself, pinned exactly ───────────────────────
242
+
243
+ test("the fast estimator is bit-identical to its scalar definition", () => {
244
+ // The scan's inner loop is the hottest code in inference and invites
245
+ // micro-optimisation (it currently folds the popcount into 32-bit SWAR
246
+ // words). Those rewrites are only legitimate while they are EXACT: the
247
+ // estimator's output ranks every candidate, so a last-bit difference is a
248
+ // silent reordering, not a rounding detail. This recomputes the definition
249
+ // straight from the QueryContext and demands equality.
250
+ const rnd = prng(11);
251
+ let checked = 0;
252
+ for (const dim of [8, 64, 100, 256, 1024]) {
253
+ const qz = new RaBitQuantizer(dim, { seed: SEED });
254
+ const nb = qz.paddedDim >>> 3;
255
+ const POP = new Uint8Array(256);
256
+ for (let i = 1; i < 256; i++) POP[i] = POP[i >> 1] + (i & 1);
257
+ for (let t = 0; t < 25; t++) {
258
+ const v = new Float64Array(dim);
259
+ for (let i = 0; i < dim; i++) v[i] = rnd() * 2 - 1;
260
+ const q = qz.prepareQuery(v);
261
+ const code = new Uint8Array(nb * 4);
262
+ for (let i = 0; i < code.length; i++) code[i] = (rnd() * 256) | 0;
263
+ for (let c = 0; c < 4; c++) {
264
+ const off = c * nb;
265
+ let dot = 0, pc = 0;
266
+ for (let p = 0; p < q.nbytes; p++) {
267
+ const b = code[off + p];
268
+ dot += q.qlut[(p << 8) + b];
269
+ pc += POP[b];
270
+ }
271
+ const A = q.vmin * (2 * pc - qz.paddedDim) +
272
+ q.delta * (2 * dot - q.sumQInt);
273
+ const want = q.zero ? 1 : 1 - qz.cosFactor * A;
274
+ assert.ok(
275
+ Object.is(qz.estimate(code, off, q), want),
276
+ `estimate diverged from its definition at dim=${dim}: ` +
277
+ `${qz.estimate(code, off, q)} vs ${want}`,
278
+ );
279
+ checked++;
280
+ }
281
+ }
282
+ }
283
+ assert.ok(checked >= 500, `expected a broad sweep, checked ${checked}`);
284
+ });
285
+
286
+ // ── GEOMETRY: rotation rounds are saturated at 1 ───────────────────────────
287
+
288
+ test("rotation rounds beyond the first buy no recall", () => {
289
+ // Recorded as an executable fact, because it is the one place in this
290
+ // sub-library with real headroom: a round is a sign-flip plus a
291
+ // Walsh-Hadamard transform (the SRHT construction, for which ONE randomised
292
+ // round already approximates a random rotation), and cost is linear in the
293
+ // count — encode runs 28.0us at rounds=3 against 16.8us at rounds=1.
294
+ //
295
+ // Measured against the trained store — 100,000 real gists, routing removed,
296
+ // 400 queries, recall@10: rounds=4 70.5%, rounds=3 71.1%, rounds=2 70.6%,
297
+ // rounds=1 71.1% — indistinguishable across 4,000 ground-truth slots, and
298
+ // reproduced at N=1,000 and N=4,000. If this assertion ever fails, the
299
+ // rotation has stopped being saturated and the default is worth revisiting.
300
+ const { vecs, D } = gists(2000);
301
+ const norms = vecs.map((v) => {
302
+ let s = 0;
303
+ for (let i = 0; i < v.length; i++) s += v[i] * v[i];
304
+ return Math.sqrt(s);
305
+ });
306
+ const qs = queryIds(vecs.length, 100);
307
+ const truth = qs.map((qi) => exactTopK(vecs, norms, qi, K));
308
+
309
+ const recallFor = (rounds) => {
310
+ const idx = indexOf(vecs, D, rounds);
311
+ const full = 4 * idx.clusterCount;
312
+ let hit = 0, want = 0;
313
+ for (let t = 0; t < qs.length; t++) {
314
+ for (const h of idx.searchKnn(vecs[qs[t]], K, full)) {
315
+ if (truth[t].has(h.id)) hit++;
316
+ }
317
+ want += truth[t].size;
318
+ }
319
+ idx.close();
320
+ return hit / want;
321
+ };
322
+
323
+ const three = recallFor(3);
324
+ const one = recallFor(1);
325
+ assert.ok(
326
+ Math.abs(three - one) <= 0.03,
327
+ `one rotation round is no longer equivalent to three: ` +
328
+ `rounds=3 ${(three * 100).toFixed(1)}% vs rounds=1 ` +
329
+ `${(one * 100).toFixed(1)}% (tolerance 3 points)`,
330
+ );
331
+ });
@@ -0,0 +1,99 @@
1
+ // 66-query-edge-whitespace.test.mjs — a query's leading/trailing whitespace is
2
+ // presentation, not part of the question, and must not decide whether a trained
3
+ // fact is reachable.
4
+ //
5
+ // canon.ts's contract: "a span's leading or trailing separator belongs BETWEEN
6
+ // forms, not to the form". canon itself PRESERVES edge whitespace, and must,
7
+ // because the hazard it cites is a recognised SUB-span swallowing the boundary
8
+ // byte that separates it from its neighbour ("ice " matching the stored "ice").
9
+ // At the outer edges of a WHOLE input there is no neighbour, so that hazard
10
+ // cannot arise — which is why respond() may trim there and canon may not.
11
+ // test/44 already relies on the same reading for recognise()'s miss path.
12
+ //
13
+ // THE GAP THIS CLOSES (measured on the 15.7M-node trained store): ONE leading
14
+ // space took `Who wrote Romeo and Juliet?` and `What is the chemical symbol for
15
+ // water?` from answered to silent, because a shift re-seats every fold boundary
16
+ // (cos(query, query shifted 1 byte) = 0.68 against a 0.875 reach bar). That was
17
+ // the whole of analyze_training.ts's K2 phase-robustness gap: 15/18 → 18/18.
18
+ //
19
+ // WHY A RETRY AND NOT A PRE-FILTER — the regression this file pins. Trimming
20
+ // the query up front is ASYMMETRIC: it normalises the query but not the stored
21
+ // forms, so it breaks byte-exact identity for a form trained WITH edge
22
+ // whitespace. Verified: pre-filtering broke test/04's [" ice ", "cold"] case.
23
+ // The exact bytes are therefore tried FIRST and the trim is reached only when
24
+ // they grounded nothing — which also means the retry costs nothing on any
25
+ // answering path.
26
+ //
27
+ // NOTE ON WHAT IS AND IS NOT TESTABLE HERE. The padded-query WIN cannot be
28
+ // reproduced in a miniature fixture: a small store answers a padded query on the
29
+ // first pass anyway (an earlier tier catches it), so the retry never fires and
30
+ // an end-to-end assertion passes with or without the fix — verified, an earlier
31
+ // version of this file did exactly that and guarded nothing. What a fixture CAN
32
+ // pin is the trim's own contract and the asymmetry regression, which is what
33
+ // these tests do; the win itself is evidenced on the real store.
34
+
35
+ import { test } from "node:test";
36
+ import assert from "node:assert/strict";
37
+ import { Mind } from "../dist/src/index.js";
38
+ import { SQliteStore } from "../dist/src/store-sqlite.js";
39
+ import { textEdgeTrim } from "../dist/src/canon.js";
40
+
41
+ const enc = (s) => new TextEncoder().encode(s);
42
+ const dec = new TextDecoder();
43
+ const trim = (s) => dec.decode(textEdgeTrim(enc(s)));
44
+
45
+ test("1. textEdgeTrim drops only the outer spacing run", () => {
46
+ assert.equal(trim(" ice "), "ice");
47
+ assert.equal(trim("\tice\n"), "ice");
48
+ assert.equal(trim("ice"), "ice");
49
+ // INTERIOR whitespace is content and is never touched.
50
+ assert.equal(trim(" a b "), "a b");
51
+ // All-separator and empty inputs collapse to empty rather than throwing.
52
+ assert.equal(trim(" "), "");
53
+ assert.equal(trim(""), "");
54
+ // The untouched case must return the SAME object (no copy on the hot path).
55
+ const b = enc("ice");
56
+ assert.equal(textEdgeTrim(b), b);
57
+ });
58
+
59
+ test("2. a form trained WITH edge whitespace still answers when asked exactly", async () => {
60
+ // The asymmetry regression: trimming the query but not the store would make
61
+ // this query miss its own deposited form.
62
+ const m = new Mind({ seed: 7 });
63
+ await m.ingest([[" ice ", "cold"]]);
64
+ assert.equal(await m.respondText(" ice "), "cold");
65
+ });
66
+
67
+ test("3. whitespace-only and empty queries are silent, not errors", async () => {
68
+ const m = new Mind({ seed: 7, store: new SQliteStore({ path: ":memory:" }) });
69
+ await m.ingest([["what is ice?", "ice is frozen water"]]);
70
+ for (const q of ["", " ", " ", "\t\n"]) {
71
+ assert.equal(
72
+ await m.respondText(q),
73
+ "",
74
+ `expected silence for ${JSON.stringify(q)}`,
75
+ );
76
+ }
77
+ await m.store.close();
78
+ });
79
+
80
+ test("4. a padded query never answers something the unpadded one would not", async () => {
81
+ // The retry may add REACH, never licence: whatever padding does, it must not
82
+ // ground a fact for a question the store cannot answer.
83
+ const m = new Mind({ seed: 7, store: new SQliteStore({ path: ":memory:" }) });
84
+ await m.ingest([
85
+ ["what is the capital of France?", "The capital of France is Paris."],
86
+ ["what is the capital of Spain?", "Madrid is the capital of Spain."],
87
+ ]);
88
+ for (const q of [" Who wrote the Iliad? ", " xyzzy plugh quux "]) {
89
+ const a = await m.respondText(q);
90
+ assert.doesNotMatch(
91
+ a,
92
+ /Paris|Madrid/,
93
+ `padding manufactured an answer for ${JSON.stringify(q)}: ${
94
+ JSON.stringify(a)
95
+ }`,
96
+ );
97
+ }
98
+ await m.store.close();
99
+ });
@@ -0,0 +1,113 @@
1
+ // 67-climb-anchor-breadth.test.mjs — recall's scaffolding-dominated tier
2
+ // trusts a consensus-climb anchor on its SCALE-INVARIANT breadth as well as on
3
+ // its absolute IDF vote, and a breadth-qualified anchor must also be
4
+ // DISCRIMINATIVE.
5
+ //
6
+ // WHY THE ABSOLUTE VOTE IS NOT ENOUGH. Attention.breadth's own contract
7
+ // (types.ts) already says it: the IDF vote is "an absolute, ln(N)-scaled
8
+ // quantity that means 'strong' on a small store and 'weak' on a large one for
9
+ // the SAME degree of genuine consensus", while breadth is "the fraction of the
10
+ // query's OWN regions whose evidence this point accounts for" and "a point
11
+ // whose breadth clears `dominates` … is real consensus". Attention.peak's
12
+ // contract makes the same point from the other side: a floor that prices ONE
13
+ // region's evidence may not be compared against a POOLED SUM.
14
+ //
15
+ // Measured on the 15.7M-node trained store (N=325,615, floor = ln N + ½ =
16
+ // 13.19). The climb picked the RIGHT context and the floor discarded it, while
17
+ // a junk attractor for a query that must stay SILENT outvoted every correct
18
+ // anchor:
19
+ //
20
+ // anchor the climb picked vote breadth correct?
21
+ // "What is the chemical formula …" 10.60 0.556 RIGHT
22
+ // "Qual é a capital de França?" 8.19 0.667 RIGHT
23
+ // "Who wrote the play Romeo …?" 8.25 0.833 RIGHT
24
+ // "How do you say "good morning" …" 10.77 0.800 RIGHT
25
+ // "What is the commercial capital …" 12.69 0.333 Zamunda — MUST be silent
26
+ // "Menene sunan ginin mafi tsayi …" 12.79 0.214 wrong (Hausa)
27
+ //
28
+ // No vote threshold separates those; breadth > ½ separates them exactly. On
29
+ // that store the old floor was never cleared at all, so the tier was dead code
30
+ // and 12 probes fell through to silence.
31
+ //
32
+ // WHAT MUST NOT REGRESS, and why the gate is an OR of two guarded readings:
33
+ //
34
+ // • REPLACING the vote test with the breadth test broke 7 tests. On a small
35
+ // store ln(N) is low, so the vote bar is the reading that legitimately
36
+ // fires there; and Attention.clusters' contract warns that "breadth starves
37
+ // a genuine, evenly-split multi-topic query, since no root in a real N-way
38
+ // split can exceed half the vote" — the two-topic fusion tests are exactly
39
+ // that shape. Each reading is sufficient on its own evidence.
40
+ // • BREADTH ALONE fabricates. On a one-context store every region trivially
41
+ // corroborates the only anchor there is, so breadth is 1 while the anchor's
42
+ // IDF is 0 — test/31 A2 answered a lone cat fact for "explain quantum
43
+ // chromodynamics". Hence the companion condition: a region's IDF for an
44
+ // anchor reached through c of N contexts is ln(N/c), so requiring it past
45
+ // ln 2 requires c·2 < N — the same half-dominance reading in IDF units.
46
+
47
+ import { test } from "node:test";
48
+ import assert from "node:assert/strict";
49
+ import { Mind } from "../dist/src/index.js";
50
+ import { SQliteStore } from "../dist/src/store-sqlite.js";
51
+
52
+ const mk = () =>
53
+ new Mind({ seed: 1, store: new SQliteStore({ path: ":memory:" }) });
54
+
55
+ test("1. a one-context store never grounds an unrelated query (breadth alone must not decide)", async () => {
56
+ // Breadth is trivially 1 when there is only one anchor to corroborate, but
57
+ // that anchor's IDF is 0 — it says nothing. VERIFIED to bite: dropping the
58
+ // `peak > ln 2` companion makes this fail (and test/31 A2 with it). The
59
+ // fixture matches A2's exactly — `new Mind({ seed: 7 })`, the default store —
60
+ // because the same shape over a SQliteStore did NOT reproduce it.
61
+ const m = new Mind({ seed: 7 });
62
+ await m.ingest([["what is a cat?", "a cat is a small feline"]]);
63
+ const r = await m.respond("explain quantum chromodynamics");
64
+ assert.equal(
65
+ r.v,
66
+ null,
67
+ "a lone low-IDF anchor must not ground a foreign query",
68
+ );
69
+ assert.equal(r.provenance, undefined);
70
+ });
71
+
72
+ test("2. an evenly-split multi-topic query still fuses (breadth must not be required)", async () => {
73
+ // The shape Attention.clusters' contract says breadth starves: no root in a
74
+ // real N-way split can hold more than half the query's regions, so a
75
+ // breadth-only gate would refuse both topics.
76
+ const m = mk();
77
+ await m.ingest([
78
+ ["ice", "cold"],
79
+ ["fire", "hot"],
80
+ ["what is ice?", "ice is frozen water"],
81
+ ["what is fire?", "fire is rapid oxidation"],
82
+ ]);
83
+ const a = await m.respondText("ice fire");
84
+ assert.ok(a.length > 0, "a two-topic query must still ground something");
85
+ await m.store.close();
86
+ });
87
+
88
+ test("3. honest silence survives on an unrelated corpus", async () => {
89
+ const m = mk();
90
+ await m.ingest([
91
+ ["what is the capital of France?", "The capital of France is Paris."],
92
+ ["what is the capital of Spain?", "Madrid is the capital of Spain."],
93
+ ["what is the capital of Italy?", "Rome is the capital of Italy."],
94
+ ]);
95
+ for (const q of ["xyzzy plugh quux baz?", "qq8f3kz9 vv2m1x7w?"]) {
96
+ const a = await m.respondText(q);
97
+ assert.equal(a, "", `gibberish must stay silent, got ${JSON.stringify(a)}`);
98
+ }
99
+ await m.store.close();
100
+ });
101
+
102
+ test("4. a trained fact still answers (the tier did not displace an earlier one)", async () => {
103
+ const m = mk();
104
+ await m.ingest([
105
+ ["what is the capital of France?", "The capital of France is Paris."],
106
+ ["what is the capital of Spain?", "Madrid is the capital of Spain."],
107
+ ]);
108
+ assert.match(
109
+ await m.respondText("what is the capital of France?"),
110
+ /Paris/,
111
+ );
112
+ await m.store.close();
113
+ });