@hviana/sema 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/AGENTS.md +6 -5
  2. package/CITATION.cff +49 -0
  3. package/HOW_IT_WORKS.md +11 -12
  4. package/README.md +7 -5
  5. package/dist/example/demo.js +25 -25
  6. package/dist/example/train_base.d.ts +19 -21
  7. package/dist/example/train_base.js +1700 -1952
  8. package/dist/src/alphabet.d.ts +3 -3
  9. package/dist/src/alphabet.js +24 -27
  10. package/dist/src/alu/src/alu.d.ts +169 -194
  11. package/dist/src/alu/src/alu.js +374 -414
  12. package/dist/src/alu/src/expr.d.ts +36 -43
  13. package/dist/src/alu/src/expr.js +252 -278
  14. package/dist/src/alu/src/index.d.ts +9 -98
  15. package/dist/src/alu/src/index.js +10 -64
  16. package/dist/src/alu/src/kernel-arith.d.ts +1 -5
  17. package/dist/src/alu/src/kernel-arith.js +195 -285
  18. package/dist/src/alu/src/kernel-bits.d.ts +1 -5
  19. package/dist/src/alu/src/kernel-bits.js +81 -115
  20. package/dist/src/alu/src/kernel-logic.js +33 -63
  21. package/dist/src/alu/src/kernel-nd.js +155 -195
  22. package/dist/src/alu/src/kernel-numeric.d.ts +16 -80
  23. package/dist/src/alu/src/kernel-numeric.js +259 -331
  24. package/dist/src/alu/src/operation.d.ts +112 -132
  25. package/dist/src/alu/src/operation.js +149 -163
  26. package/dist/src/alu/src/parser.d.ts +168 -175
  27. package/dist/src/alu/src/parser.js +382 -421
  28. package/dist/src/alu/src/resonance.d.ts +16 -25
  29. package/dist/src/alu/src/resonance.js +49 -56
  30. package/dist/src/alu/src/text.d.ts +3 -7
  31. package/dist/src/alu/src/text.js +31 -37
  32. package/dist/src/alu/src/value.d.ts +14 -14
  33. package/dist/src/alu/src/value.js +150 -160
  34. package/dist/src/alu/test/alu.test.js +543 -656
  35. package/dist/src/bytes.d.ts +1 -5
  36. package/dist/src/bytes.js +33 -40
  37. package/dist/src/config.d.ts +99 -102
  38. package/dist/src/config.js +78 -83
  39. package/dist/src/derive/src/deduction.d.ts +60 -62
  40. package/dist/src/derive/src/deduction.js +103 -109
  41. package/dist/src/derive/src/index.d.ts +1 -7
  42. package/dist/src/derive/src/priority-queue.d.ts +8 -8
  43. package/dist/src/derive/src/priority-queue.js +57 -61
  44. package/dist/src/derive/src/rewrite.d.ts +15 -18
  45. package/dist/src/derive/src/rewrite.js +59 -67
  46. package/dist/src/derive/src/trie.d.ts +56 -56
  47. package/dist/src/derive/src/trie.js +175 -179
  48. package/dist/src/derive/test/derive.test.js +96 -100
  49. package/dist/src/extension.d.ts +13 -16
  50. package/dist/src/geometry.d.ts +29 -43
  51. package/dist/src/geometry.js +324 -268
  52. package/dist/src/index.d.ts +2 -17
  53. package/dist/src/index.js +2 -12
  54. package/dist/src/ingest-cache.d.ts +25 -30
  55. package/dist/src/ingest-cache.js +108 -127
  56. package/dist/src/mind/articulation.d.ts +1 -5
  57. package/dist/src/mind/articulation.js +77 -112
  58. package/dist/src/mind/attention.d.ts +40 -108
  59. package/dist/src/mind/attention.js +754 -872
  60. package/dist/src/mind/canonical.d.ts +4 -19
  61. package/dist/src/mind/canonical.js +27 -31
  62. package/dist/src/mind/graph-search.d.ts +201 -225
  63. package/dist/src/mind/graph-search.js +742 -821
  64. package/dist/src/mind/index.d.ts +2 -10
  65. package/dist/src/mind/junction.d.ts +31 -58
  66. package/dist/src/mind/junction.js +172 -237
  67. package/dist/src/mind/learning.d.ts +16 -52
  68. package/dist/src/mind/learning.js +143 -165
  69. package/dist/src/mind/match.d.ts +20 -69
  70. package/dist/src/mind/match.js +259 -318
  71. package/dist/src/mind/mechanisms/alu.js +16 -16
  72. package/dist/src/mind/mechanisms/cast.d.ts +9 -13
  73. package/dist/src/mind/mechanisms/cast.js +363 -456
  74. package/dist/src/mind/mechanisms/confluence.d.ts +8 -12
  75. package/dist/src/mind/mechanisms/confluence.js +152 -183
  76. package/dist/src/mind/mechanisms/cover.d.ts +2 -8
  77. package/dist/src/mind/mechanisms/cover.js +148 -210
  78. package/dist/src/mind/mechanisms/extraction.d.ts +8 -34
  79. package/dist/src/mind/mechanisms/extraction.js +234 -288
  80. package/dist/src/mind/mechanisms/recall.d.ts +6 -10
  81. package/dist/src/mind/mechanisms/recall.js +141 -185
  82. package/dist/src/mind/mind.d.ts +194 -155
  83. package/dist/src/mind/mind.js +392 -289
  84. package/dist/src/mind/pipeline-mechanism.d.ts +112 -124
  85. package/dist/src/mind/pipeline-mechanism.js +161 -172
  86. package/dist/src/mind/pipeline.d.ts +4 -14
  87. package/dist/src/mind/pipeline.js +125 -189
  88. package/dist/src/mind/primitives.d.ts +20 -33
  89. package/dist/src/mind/primitives.js +124 -120
  90. package/dist/src/mind/rationale.d.ts +86 -98
  91. package/dist/src/mind/rationale.js +113 -121
  92. package/dist/src/mind/reasoning.d.ts +2 -13
  93. package/dist/src/mind/reasoning.js +129 -166
  94. package/dist/src/mind/recognition.d.ts +1 -4
  95. package/dist/src/mind/recognition.js +198 -210
  96. package/dist/src/mind/resonance.d.ts +5 -22
  97. package/dist/src/mind/resonance.js +0 -0
  98. package/dist/src/mind/trace.d.ts +6 -25
  99. package/dist/src/mind/trace.js +50 -58
  100. package/dist/src/mind/traverse.d.ts +29 -58
  101. package/dist/src/mind/traverse.js +344 -356
  102. package/dist/src/mind/types.d.ts +122 -127
  103. package/dist/src/mind/types.js +60 -69
  104. package/dist/src/rabitq-ivf/src/database.d.ts +113 -0
  105. package/dist/src/rabitq-ivf/src/database.js +201 -0
  106. package/dist/src/rabitq-ivf/src/index.d.ts +7 -0
  107. package/dist/src/{rabitq-hnsw → rabitq-ivf}/src/index.js +1 -3
  108. package/dist/src/rabitq-ivf/src/ivf.d.ts +200 -0
  109. package/dist/src/rabitq-ivf/src/ivf.js +1165 -0
  110. package/dist/src/rabitq-ivf/src/prng.d.ts +19 -0
  111. package/dist/src/rabitq-ivf/src/prng.js +36 -0
  112. package/dist/src/rabitq-ivf/src/rabitq.d.ts +95 -0
  113. package/dist/src/rabitq-ivf/src/rabitq.js +283 -0
  114. package/dist/src/sema.d.ts +9 -13
  115. package/dist/src/sema.js +26 -40
  116. package/dist/src/store-sqlite.d.ts +171 -166
  117. package/dist/src/store-sqlite.js +768 -662
  118. package/dist/src/store.d.ts +597 -630
  119. package/dist/src/store.js +1434 -1581
  120. package/dist/src/vec.d.ts +5 -9
  121. package/dist/src/vec.js +61 -73
  122. package/example/train_base.ts +33 -15
  123. package/index.html +65 -0
  124. package/package.json +1 -1
  125. package/src/alu/README.md +1 -1
  126. package/src/alu/src/index.ts +1 -1
  127. package/src/config.ts +19 -27
  128. package/src/geometry.ts +93 -1
  129. package/src/index.ts +6 -11
  130. package/src/mind/attention.ts +11 -6
  131. package/src/mind/mechanisms/recall.ts +29 -13
  132. package/src/mind/mind.ts +250 -19
  133. package/src/mind/primitives.ts +43 -6
  134. package/src/mind/recognition.ts +26 -8
  135. package/src/mind/traverse.ts +47 -0
  136. package/src/mind/types.ts +20 -21
  137. package/src/rabitq-ivf/README.md +56 -0
  138. package/src/rabitq-ivf/src/database.ts +276 -0
  139. package/src/{rabitq-hnsw → rabitq-ivf}/src/index.ts +2 -5
  140. package/src/rabitq-ivf/src/ivf.ts +1330 -0
  141. package/src/{rabitq-hnsw → rabitq-ivf}/src/prng.ts +1 -1
  142. package/src/store-sqlite.ts +204 -9
  143. package/src/store.ts +27 -34
  144. package/test/08-storage.test.mjs +3 -3
  145. package/test/13-conversation.test.mjs +190 -20
  146. package/test/14-scaling.test.mjs +2 -2
  147. package/test/35-ivf.test.mjs +263 -0
  148. package/test/36-bloom.test.mjs +123 -0
  149. package/dist/src/rabitq-hnsw/src/database.d.ts +0 -202
  150. package/dist/src/rabitq-hnsw/src/database.js +0 -405
  151. package/dist/src/rabitq-hnsw/src/heap.d.ts +0 -22
  152. package/dist/src/rabitq-hnsw/src/heap.js +0 -94
  153. package/dist/src/rabitq-hnsw/src/hnsw.d.ts +0 -125
  154. package/dist/src/rabitq-hnsw/src/hnsw.js +0 -500
  155. package/dist/src/rabitq-hnsw/src/index.d.ts +0 -15
  156. package/dist/src/rabitq-hnsw/src/prng.d.ts +0 -19
  157. package/dist/src/rabitq-hnsw/src/prng.js +0 -38
  158. package/dist/src/rabitq-hnsw/src/rabitq.d.ts +0 -95
  159. package/dist/src/rabitq-hnsw/src/rabitq.js +0 -299
  160. package/dist/src/rabitq-hnsw/src/store.d.ts +0 -162
  161. package/dist/src/rabitq-hnsw/src/store.js +0 -916
  162. package/dist/src/rabitq-hnsw/test/hnsw.test.d.ts +0 -1
  163. package/dist/src/rabitq-hnsw/test/hnsw.test.js +0 -1197
  164. package/src/rabitq-hnsw/README.md +0 -303
  165. package/src/rabitq-hnsw/src/database.ts +0 -492
  166. package/src/rabitq-hnsw/src/heap.ts +0 -90
  167. package/src/rabitq-hnsw/src/hnsw.ts +0 -514
  168. package/src/rabitq-hnsw/src/store.ts +0 -994
  169. package/src/rabitq-hnsw/test/hnsw.test.ts +0 -1213
  170. /package/src/{rabitq-hnsw → rabitq-ivf}/src/rabitq.ts +0 -0
@@ -14,14 +14,10 @@
14
14
  // The assertions describe BEHAVIOUR only — they never mention how context is
15
15
  // represented — so they stay valid for any implementation.
16
16
  //
17
- // Note: Sema trains on the accumulated context string ("turn0\nturn1\nturn2")
18
- // and queries the same accumulated string at inference. This is not a
19
- // weakness LLMs work the same way: a chat model receives the full
20
- // conversation history as its prompt, and that same history must be
21
- // provided at inference to produce the next turn. The difference is in
22
- // how the answer is produced: Sema composes it from learned graph forms;
23
- // an LLM samples it from a parametric distribution. Neither is "just a
24
- // lookup."
17
+ // Sema trains on the raw concatenation of prior turns and queries the same
18
+ // accumulated bytes at inference. The Conversation API tracks turn-boundary
19
+ // offsets explicitly so no separator character is needed the geometry never
20
+ // inspects content to find turn boundaries.
25
21
  // ─────────────────────────────────────────────────────────────────────────
26
22
 
27
23
  import { test } from "node:test";
@@ -34,16 +30,14 @@ const newMind = () => new Mind({ seed: 7 });
34
30
  // ═══════════════════════════════════════════════════════════════════════
35
31
  // teachConversation — store one conversation (an ordered list of turns).
36
32
  //
37
- // Accumulate every prior turn into the context so each episode carries the
38
- // full conversation history: (t₀ → t₁), (t₀+t₁ → t₂), (t₀+t₁+t₂ → t₃) …
39
- // A pivot turn ("what is its name?") that appears in two conversations now
40
- // produces different episode vectors because the context side encodes which
41
- // conversation it belongs to. All SEMA operations remain sublinear — the
42
- // river cuts any context into O(log N) chunks regardless of length.
33
+ // Accumulate prior turns by raw concatenation no separator character.
34
+ // Each episode carries the full conversation history: (t₀ → t₁),
35
+ // (t₀+t₁ t₂), (t₀+t₁+t₂ t₃) where + is byte concatenation.
43
36
  // ═══════════════════════════════════════════════════════════════════════
44
37
  async function teachConversation(mind, turns) {
38
+ let context = "";
45
39
  for (let i = 0; i + 1 < turns.length; i++) {
46
- const context = turns.slice(0, i + 1).join("\n");
40
+ context += turns[i];
47
41
  await mind.ingest(context, turns[i + 1]);
48
42
  }
49
43
  }
@@ -51,13 +45,22 @@ async function teachConversation(mind, turns) {
51
45
  // ═══════════════════════════════════════════════════════════════════════
52
46
  // predictNext — given the turns spoken so far, predict the next turn.
53
47
  //
54
- // Join every prior turn into one accumulated context, mirroring the storage
55
- // pattern in teachConversation. The resulting query string is the exact
56
- // context side of the matching episode, so recall resolves unambiguously.
48
+ // Uses the Conversation API: beginConversation respondTurn for each
49
+ // prior turn respondTurnText for the query endConversation.
50
+ // Turns are raw strings, concatenated by the Mind no separator.
57
51
  // ═══════════════════════════════════════════════════════════════════════
58
52
  async function predictNext(mind, priorTurns) {
59
- const context = priorTurns.join("\n");
60
- return await mind.respondText(context);
53
+ if (priorTurns.length === 0) return "";
54
+ const conv = mind.beginConversation();
55
+ for (let i = 0; i < priorTurns.length - 1; i++) {
56
+ await mind.respondTurn(conv, priorTurns[i]);
57
+ }
58
+ const { response } = await mind.respondTurnText(
59
+ conv,
60
+ priorTurns[priorTurns.length - 1],
61
+ );
62
+ mind.endConversation(conv);
63
+ return response;
61
64
  }
62
65
 
63
66
  // A small convenience: teach a conversation, then ask for the continuation
@@ -226,3 +229,170 @@ test("D2: prediction is deterministic across runs", async () => {
226
229
  };
227
230
  assert.equal(await run(), await run());
228
231
  });
232
+
233
+ // ═══════════════════════════════════════════════════════════════════════
234
+ // Section E — Conversation API lifecycle & cache management
235
+ //
236
+ // Tests for the beginConversation / respondTurn / endConversation API
237
+ // itself, independent of the conversation-training pattern above.
238
+ // ═══════════════════════════════════════════════════════════════════════
239
+
240
+ test("E1: begin → respondTurn → end completes a single-turn conversation", async () => {
241
+ const mind = newMind();
242
+ await mind.ingest("hello", "world");
243
+
244
+ const conv = mind.beginConversation();
245
+ const { response, state } = await mind.respondTurnText(conv, "hello");
246
+ assert.equal(response, "world");
247
+ // Single turn — no boundaries yet.
248
+ assert.equal(state.boundaries.length, 0);
249
+ mind.endConversation(conv);
250
+ await mind.store.close();
251
+ });
252
+
253
+ test("E2: multi-turn conversation accumulates context and boundaries", async () => {
254
+ const mind = newMind();
255
+ await teachConversation(mind, CAT);
256
+
257
+ const conv = mind.beginConversation();
258
+ // First turn establishes the subject.
259
+ await mind.respondTurn(conv, "I adopted a cat");
260
+
261
+ // Second turn is the pivot query.
262
+ const t2 = await mind.respondTurnText(conv, "what is its name?");
263
+ // Now we have one boundary: the byte position after the first turn.
264
+ assert.equal(t2.state.boundaries.length, 1);
265
+ assert.equal(t2.response, "her name is Whiskers");
266
+
267
+ mind.endConversation(conv);
268
+ await mind.store.close();
269
+ });
270
+
271
+ test("E3: save → end → restore → continue works", async () => {
272
+ const mind = newMind();
273
+ await teachConversation(mind, CAT);
274
+
275
+ // First two turns.
276
+ const conv = mind.beginConversation();
277
+ await mind.respondTurn(conv, "I adopted a cat");
278
+ const { state: saved } = await mind.respondTurnText(
279
+ conv,
280
+ "what is its name?",
281
+ );
282
+ assert.equal(saved.boundaries.length, 1);
283
+
284
+ // Save and end.
285
+ mind.endConversation(conv);
286
+
287
+ // Restore from the saved state into a new handle.
288
+ const conv2 = mind.beginConversation(saved);
289
+ const { response, state: state2 } = await mind.respondTurnText(
290
+ conv2,
291
+ "her name is Whiskers",
292
+ );
293
+ // The restored conversation continues from where it left off.
294
+ assert.ok(response.length > 0 || response === "");
295
+ // Boundaries are intact from the restore.
296
+ assert.equal(state2.boundaries.length, 2);
297
+
298
+ mind.endConversation(conv2);
299
+ await mind.store.close();
300
+ });
301
+
302
+ test("E4: endConversation is idempotent", async () => {
303
+ const mind = newMind();
304
+ const conv = mind.beginConversation();
305
+ mind.endConversation(conv);
306
+ // Second end on the same handle — must not throw.
307
+ mind.endConversation(conv);
308
+ await mind.store.close();
309
+ });
310
+
311
+ test("E5: respondTurn on an ended conversation throws", async () => {
312
+ const mind = newMind();
313
+ const conv = mind.beginConversation();
314
+ mind.endConversation(conv);
315
+ await assert.rejects(
316
+ () => mind.respondTurn(conv, "hello"),
317
+ /not found/,
318
+ );
319
+ await mind.store.close();
320
+ });
321
+
322
+ test("E6: conversationState returns null after endConversation", async () => {
323
+ const mind = newMind();
324
+ const conv = mind.beginConversation();
325
+ mind.endConversation(conv);
326
+ assert.equal(mind.conversationState(conv), null);
327
+ await mind.store.close();
328
+ });
329
+
330
+ test("E7: multiple concurrent conversations are independent", async () => {
331
+ const mind = newMind();
332
+ await teachConversation(mind, CAT);
333
+ await teachConversation(mind, DOG);
334
+
335
+ const catConv = mind.beginConversation();
336
+ const dogConv = mind.beginConversation();
337
+
338
+ // Feed the distinguishing context to each.
339
+ await mind.respondTurn(catConv, "I adopted a cat");
340
+ await mind.respondTurn(dogConv, "I adopted a dog");
341
+
342
+ // The same pivot turn — different answers.
343
+ const catR = await mind.respondTurnText(catConv, "what is its name?");
344
+ const dogR = await mind.respondTurnText(dogConv, "what is its name?");
345
+
346
+ assert.equal(catR.response, "her name is Whiskers");
347
+ assert.equal(dogR.response, "his name is Rex");
348
+ assert.notEqual(catR.response, dogR.response);
349
+
350
+ // Each conversation has its own boundaries.
351
+ assert.equal(catR.state.boundaries.length, 1);
352
+ assert.equal(dogR.state.boundaries.length, 1);
353
+
354
+ // Ending one does not affect the other.
355
+ mind.endConversation(catConv);
356
+ assert.equal(mind.conversationState(catConv), null);
357
+ assert.notEqual(mind.conversationState(dogConv), null);
358
+
359
+ // End the second — both are now gone.
360
+ mind.endConversation(dogConv);
361
+ assert.equal(mind.conversationState(dogConv), null);
362
+
363
+ // Ending again is idempotent.
364
+ mind.endConversation(dogConv);
365
+ await mind.store.close();
366
+ });
367
+
368
+ test("E8: respondTurn with a forged handle throws", async () => {
369
+ const mind = newMind();
370
+ // { id: 999 } was never returned by beginConversation.
371
+ await assert.rejects(
372
+ () => mind.respondTurn({ id: 999 }, "hello"),
373
+ /not found/,
374
+ );
375
+ await mind.store.close();
376
+ });
377
+
378
+ test("E9: respond and respondTurn give the same answer for the same cumulative bytes", async () => {
379
+ const mind = newMind();
380
+ await teachConversation(mind, CAT);
381
+
382
+ // Via respond() — raw concatenation, same as respondTurn.
383
+ const viaRespond = await mind.respondText(
384
+ "I adopted a catwhat is its name?",
385
+ );
386
+
387
+ // Via respondTurn — the new way.
388
+ const conv = mind.beginConversation();
389
+ await mind.respondTurnText(conv, "I adopted a cat");
390
+ const { response: viaConv } = await mind.respondTurnText(
391
+ conv,
392
+ "what is its name?",
393
+ );
394
+ mind.endConversation(conv);
395
+
396
+ assert.equal(viaRespond, viaConv);
397
+ await mind.store.close();
398
+ });
@@ -112,7 +112,7 @@ function unknownInput(targetBytes) {
112
112
  // jitter swamp the signal over a few thousand deposits, so any strict
113
113
  // degradation bound is either flaky or so loose it catches nothing.
114
114
  //
115
- // The real degradation lived in the vector index (rabitq-hnsw) and is now guarded
115
+ // The real degradation lived in the vector index (rabitq-ivf) and is now guarded
116
116
  // DETERMINISTICALLY at its own layer (testScalabilityWithoutCache, on the
117
117
  // cache-independent storage-read count). HERE we assert the same contract end to
118
118
  // end through the Mind, also by STORAGE READS rather than time: the work one
@@ -170,7 +170,7 @@ test("training: recall work does NOT grow with the store (storage reads, not tim
170
170
 
171
171
  // 6x the corpus must not cost anywhere near 6x the reads. A genuinely
172
172
  // sublinear (~log N) lookup barely moves; a per-recall cost that tracked the
173
- // corpus would roughly 6x. Allow a generous 3x band for HNSW's log-N growth
173
+ // corpus would roughly 6x. Allow a generous 3x band for estimator noise
174
174
  // and graph-shape variation, far below the ~6x a true O(N) regression shows.
175
175
  assert.ok(
176
176
  large < small * 3 + 50,
@@ -0,0 +1,263 @@
1
+ // 35-ivf.test.mjs — the partitioned (IVF) vector index, at its own layer.
2
+ //
3
+ // VectorDatabase is an adaptive IVF index over 1-bit RaBitQ codes in SQLite
4
+ // (src/rabitq-ivf). These tests verify the index's OWN contract, below the
5
+ // Sema store:
6
+ //
7
+ // 1. CRUD — insert/has/get/update/delete, id checking.
8
+ // 2. Recall — clustered data resonates to its own cluster; a stored
9
+ // vector finds itself.
10
+ // 3. Splits — the index partitions past SPLIT_MAX and stays correct.
11
+ // 4. Persistence — close/reopen preserves entries, config, and results.
12
+ // 5. Determinism — same insert sequence ⇒ identical results (no RNG).
13
+ // 6. Compaction — tombstones are reclaimed; internal ids (the keysSince
14
+ // watermark) survive.
15
+ // 7. Scaling — per-query STORAGE READS are bounded once the
16
+ // collection is past its first splits: 2x the corpus
17
+ // must not move the reads (the honest witness the old
18
+ // graph index also swore to).
19
+
20
+ import { test } from "node:test";
21
+ import assert from "node:assert/strict";
22
+ import { rmSync } from "node:fs";
23
+ import { VectorDatabase } from "../dist/src/rabitq-ivf/src/index.js";
24
+
25
+ // Deterministic PRNG for test vectors.
26
+ function rng(seed) {
27
+ let s = seed >>> 0;
28
+ return () => {
29
+ s ^= s << 13;
30
+ s ^= s >>> 17;
31
+ s ^= s << 5;
32
+ s >>>= 0;
33
+ return s / 0x100000000;
34
+ };
35
+ }
36
+
37
+ /** A unit vector near `center` with per-coordinate jitter. */
38
+ function near(center, jitter, r) {
39
+ const v = new Float32Array(center.length);
40
+ for (let i = 0; i < v.length; i++) {
41
+ v[i] = center[i] + (r() * 2 - 1) * jitter;
42
+ }
43
+ return v;
44
+ }
45
+
46
+ function randVec(dim, r) {
47
+ const v = new Float32Array(dim);
48
+ for (let i = 0; i < dim; i++) v[i] = r() * 2 - 1;
49
+ return v;
50
+ }
51
+
52
+ test("1: CRUD — insert, has, get, update, delete", () => {
53
+ const db = new VectorDatabase({ dbPath: ":memory:", dim: 64 });
54
+ const r = rng(1);
55
+ const v1 = randVec(64, r);
56
+ const v2 = randVec(64, r);
57
+
58
+ db.insert(10, v1);
59
+ assert.equal(db.size, 1);
60
+ assert.ok(db.has(10));
61
+ assert.ok(!db.has(11));
62
+ assert.throws(() => db.insert(10, v2), /already exists/);
63
+ assert.throws(() => db.insert(1.5, v2), /integer/);
64
+
65
+ const code = db.get(10);
66
+ assert.ok(code instanceof Uint32Array);
67
+
68
+ // update to a genuinely different vector: same ext still resolves
69
+ db.update(10, v2);
70
+ assert.equal(db.size, 1);
71
+ assert.ok(db.has(10));
72
+
73
+ // byte-identical update is a no-op (no new tombstone)
74
+ const phys = db.physicalSize;
75
+ db.update(10, v2);
76
+ assert.equal(db.physicalSize, phys);
77
+
78
+ assert.ok(db.delete(10));
79
+ assert.ok(!db.has(10));
80
+ assert.equal(db.size, 0);
81
+ assert.ok(!db.delete(10));
82
+ db.close();
83
+ });
84
+
85
+ test("2: recall — clustered data finds its own cluster; self-recall", () => {
86
+ const dim = 128;
87
+ const db = new VectorDatabase({ dbPath: ":memory:", dim });
88
+ const r = rng(2);
89
+ // 8 well-separated centers, 100 members each.
90
+ const centers = [];
91
+ for (let c = 0; c < 8; c++) centers.push(randVec(dim, r));
92
+ const vecs = new Map();
93
+ let id = 0;
94
+ const batch = [];
95
+ for (let c = 0; c < 8; c++) {
96
+ for (let m = 0; m < 100; m++) {
97
+ const v = near(centers[c], 0.15, r);
98
+ vecs.set(id, { v, c });
99
+ batch.push({ id, vector: v });
100
+ id++;
101
+ }
102
+ }
103
+ db.upsertMany(batch);
104
+ assert.equal(db.size, 800);
105
+
106
+ // Query each center: top-10 should come from that center's members.
107
+ for (let c = 0; c < 8; c++) {
108
+ const hits = db.query(centers[c], 10);
109
+ assert.equal(hits.length, 10);
110
+ let own = 0;
111
+ for (const h of hits) if (vecs.get(h.id).c === c) own++;
112
+ assert.ok(own >= 8, `center ${c}: only ${own}/10 hits from own cluster`);
113
+ }
114
+
115
+ // Self-recall: a stored vector should find its own ext in the top 5.
116
+ let self = 0;
117
+ for (let probe = 0; probe < 100; probe += 7) {
118
+ const hits = db.query(vecs.get(probe).v, 5);
119
+ if (hits.some((h) => h.id === probe)) self++;
120
+ }
121
+ assert.ok(self >= 12, `self-recall ${self}/15 too low`);
122
+ db.close();
123
+ });
124
+
125
+ test("3: splits — collections past SPLIT_MAX partition and stay correct", () => {
126
+ const dim = 32;
127
+ const db = new VectorDatabase({ dbPath: ":memory:", dim });
128
+ const r = rng(3);
129
+ const N = 10_000; // > 2 × SPLIT_MAX (4096) — forces several splits
130
+ const batch = [];
131
+ for (let i = 0; i < N; i++) batch.push({ id: i, vector: randVec(dim, r) });
132
+ db.upsertMany(batch);
133
+ assert.equal(db.size, N);
134
+ assert.ok(
135
+ db.clusterCount >= 3,
136
+ `expected several clusters after ${N} inserts, got ${db.clusterCount}`,
137
+ );
138
+ // Every ext still resolves and queries still return k results.
139
+ assert.ok(db.has(0) && db.has(N - 1) && db.has(N >> 1));
140
+ const hits = db.query(randVec(dim, r), 20);
141
+ assert.equal(hits.length, 20);
142
+ db.close();
143
+ });
144
+
145
+ test("4: persistence — close/reopen preserves entries and results", (t) => {
146
+ const path = `/tmp/ivf-test-${process.pid}.vec`;
147
+ t.after(() => {
148
+ for (const s of ["", "-wal", "-shm"]) {
149
+ try {
150
+ rmSync(path + s);
151
+ } catch {}
152
+ }
153
+ });
154
+ const dim = 64;
155
+ const r = rng(4);
156
+ const vecs = [];
157
+ for (let i = 0; i < 500; i++) vecs.push(randVec(dim, r));
158
+
159
+ let db = new VectorDatabase({ dbPath: path, dim });
160
+ db.upsertMany(vecs.map((v, i) => ({ id: i, vector: v })));
161
+ const probe = vecs[123];
162
+ const before = db.query(probe, 10);
163
+ db.close();
164
+
165
+ db = new VectorDatabase({ dbPath: path });
166
+ assert.equal(db.dim, dim);
167
+ assert.equal(db.size, 500);
168
+ const after = db.query(probe, 10);
169
+ assert.deepEqual(after, before);
170
+ db.close();
171
+ });
172
+
173
+ test("5: determinism — same insert sequence, identical results", () => {
174
+ const dim = 64;
175
+ const r1 = rng(5), r2 = rng(5);
176
+ const a = new VectorDatabase({ dbPath: ":memory:", dim });
177
+ const b = new VectorDatabase({ dbPath: ":memory:", dim });
178
+ const N = 6000; // crosses a split
179
+ const batchA = [], batchB = [];
180
+ for (let i = 0; i < N; i++) {
181
+ batchA.push({ id: i, vector: randVec(dim, r1) });
182
+ batchB.push({ id: i, vector: randVec(dim, r2) });
183
+ }
184
+ a.upsertMany(batchA);
185
+ b.upsertMany(batchB);
186
+ assert.equal(a.clusterCount, b.clusterCount);
187
+ const probe = randVec(dim, r1);
188
+ assert.deepEqual(a.query(probe, 20), b.query(probe, 20));
189
+ a.close();
190
+ b.close();
191
+ });
192
+
193
+ test("6: compaction — tombstones reclaimed, watermark ids preserved", () => {
194
+ const dim = 32;
195
+ const db = new VectorDatabase({ dbPath: ":memory:", dim });
196
+ const r = rng(6);
197
+ const N = 2000;
198
+ db.upsertMany(
199
+ Array.from({ length: N }, (_, i) => ({ id: i, vector: randVec(dim, r) })),
200
+ );
201
+
202
+ // Watermark before deletion.
203
+ let maxInternal = 0;
204
+ for (const { internal } of db.keysSince(0)) {
205
+ maxInternal = Math.max(maxInternal, internal);
206
+ }
207
+
208
+ // Delete every third entry.
209
+ const dead = [];
210
+ for (let i = 0; i < N; i += 3) dead.push(i);
211
+ assert.equal(db.deleteMany(dead), dead.length);
212
+ assert.equal(db.size, N - dead.length);
213
+ assert.ok(db.physicalSize > db.size);
214
+
215
+ db.compact();
216
+ assert.equal(db.physicalSize, db.size);
217
+ for (const i of dead) assert.ok(!db.has(i));
218
+ assert.ok(db.has(1) && db.has(2));
219
+
220
+ // Internal ids survive compaction: entries added AFTER the old watermark
221
+ // are exactly the new inserts.
222
+ db.upsertMany([{ id: 99999, vector: randVec(dim, r) }]);
223
+ const fresh = [...db.keysSince(maxInternal)];
224
+ assert.equal(fresh.length, 1);
225
+ assert.equal(fresh[0].ext, 99999);
226
+ db.close();
227
+ });
228
+
229
+ test("7: scaling — per-query storage reads are bounded past the first splits", () => {
230
+ const dim = 32;
231
+ // cacheSizeMb 0: no page cache to hide behind; `reads` counts blob rows.
232
+ const db = new VectorDatabase({ dbPath: ":memory:", dim, cacheSizeMb: 0 });
233
+ const r = rng(7);
234
+ const probe = randVec(dim, r);
235
+ const readsNow = () => {
236
+ db.query(probe, 10);
237
+ return db.lastQueryStorageReads;
238
+ };
239
+
240
+ const grow = (n) => {
241
+ const batch = [];
242
+ for (let i = 0; i < n; i++) {
243
+ batch.push({ id: db.size + i, vector: randVec(dim, r) });
244
+ }
245
+ db.upsertMany(batch);
246
+ };
247
+
248
+ // Past ~nprobe × SPLIT_MAX entries the probed set is a fixed number of
249
+ // bounded clusters — reads must plateau.
250
+ grow(80_000);
251
+ const small = readsNow();
252
+ grow(80_000); // 2x the corpus
253
+ const large = readsNow();
254
+ console.log(
255
+ ` ivf reads for one query: N=80k → ${small}, N=160k → ${large}`,
256
+ );
257
+ assert.ok(
258
+ large <= small * 1.5 + 8,
259
+ `reads grew ${small} → ${large} when the corpus doubled — probes are ` +
260
+ `not bounded`,
261
+ );
262
+ db.close();
263
+ });
@@ -0,0 +1,123 @@
1
+ // 36-bloom.test.mjs — the negative dedup filter (store-sqlite's Bloom layer).
2
+ //
3
+ // The filter answers "this content hash is definitely not in the node table"
4
+ // in RAM, sparing the training hot path its dominant SQLite probe (profiled:
5
+ // 38.7% of ingest wall at 2.9M nodes). Its ONE hazard is a false negative,
6
+ // which would mint a duplicate node and silently break hash-consing. These
7
+ // tests pin the invariant on the three lifecycle paths:
8
+ //
9
+ // 1. live session — content minted then re-probed dedups (filter sees
10
+ // every insert synchronously);
11
+ // 2. clean close — reopen loads the persisted filter (or rebuilds) and
12
+ // dedup still holds;
13
+ // 3. CRASH — a process that exits without close() leaves no
14
+ // persisted filter (or a stale one); reopen must
15
+ // rebuild/top-up from the table and STILL dedup all
16
+ // previously-minted content, with zero duplicates.
17
+
18
+ import { test } from "node:test";
19
+ import assert from "node:assert/strict";
20
+ import { execFileSync } from "node:child_process";
21
+ import { rmSync } from "node:fs";
22
+ import { SQliteStore } from "../dist/src/store-sqlite.js";
23
+
24
+ const cleanup = (path) => {
25
+ for (
26
+ const s of [
27
+ ".sqlite",
28
+ ".sqlite-wal",
29
+ ".sqlite-shm",
30
+ ".content.vec",
31
+ ".content.vec-wal",
32
+ ".content.vec-shm",
33
+ ".halo.vec",
34
+ ".halo.vec-wal",
35
+ ".halo.vec-shm",
36
+ ]
37
+ ) {
38
+ try {
39
+ rmSync(path + s);
40
+ } catch {}
41
+ }
42
+ };
43
+
44
+ test("live session: re-probing minted content dedups (no false negatives)", async () => {
45
+ const s = new SQliteStore({ path: ":memory:", D: 64 });
46
+ const enc = new TextEncoder();
47
+ const g = new Float32Array(64).fill(0.1);
48
+ const ids = [];
49
+ for (let i = 0; i < 300; i++) {
50
+ ids.push(await s.putLeaf(enc.encode("bloom-live-" + i), g));
51
+ }
52
+ for (let i = 0; i < 300; i++) {
53
+ assert.equal(await s.putLeaf(enc.encode("bloom-live-" + i), g), ids[i]);
54
+ }
55
+ // Novel probes are answered by the filter without SQLite (the point).
56
+ const skipsBefore = s.bloomSkips;
57
+ s.findLeaf(enc.encode("definitely-not-present-xyzzy"));
58
+ assert.ok(s.bloomSkips >= skipsBefore, "bloomSkips must be observable");
59
+ await s.close();
60
+ });
61
+
62
+ test("clean close: reopen dedups against the persisted/rebuilt filter", async (t) => {
63
+ const path = `/tmp/bloom-clean-${process.pid}`;
64
+ t.after(() => cleanup(path));
65
+ const enc = new TextEncoder();
66
+ const g = new Float32Array(64).fill(0.1);
67
+
68
+ let s = new SQliteStore({ path, D: 64 });
69
+ const ids = [];
70
+ for (let i = 0; i < 300; i++) {
71
+ ids.push(await s.putLeaf(enc.encode("bloom-clean-" + i), g));
72
+ }
73
+ await s.close();
74
+
75
+ s = new SQliteStore({ path, D: 64 });
76
+ const before = s.nodeCount();
77
+ for (let i = 0; i < 300; i++) {
78
+ const id = await s.putLeaf(enc.encode("bloom-clean-" + i), g);
79
+ assert.equal(id, ids[i], `content ${i} re-minted after clean reopen`);
80
+ }
81
+ assert.equal(s.nodeCount(), before, "no duplicate mints after clean reopen");
82
+ await s.close();
83
+ });
84
+
85
+ test("crash: reopen after an unclean exit still dedups everything", (t) => {
86
+ const path = `/tmp/bloom-crash-${process.pid}`;
87
+ t.after(() => cleanup(path));
88
+
89
+ // Phase 1 in a SUBPROCESS that exits WITHOUT close() — a real crash as far
90
+ // as the store is concerned (WAL holds the rows; no filter was persisted).
91
+ const phase1 = `
92
+ import("${process.cwd()}/dist/src/store-sqlite.js").then(async ({ SQliteStore }) => {
93
+ const s = new SQliteStore({ path: "${path}", D: 64 });
94
+ const enc = new TextEncoder();
95
+ const g = new Float32Array(64).fill(0.1);
96
+ for (let i = 0; i < 400; i++) {
97
+ await s.putLeaf(enc.encode("bloom-crash-" + i), g);
98
+ }
99
+ s.commit();
100
+ process.exit(0); // no close()
101
+ });`;
102
+ execFileSync(process.execPath, ["-e", phase1], { stdio: "pipe" });
103
+
104
+ // Phase 2 in a second subprocess: every previously-minted content must
105
+ // resolve to an existing id — zero duplicates.
106
+ const phase2 = `
107
+ import("${process.cwd()}/dist/src/store-sqlite.js").then(async ({ SQliteStore }) => {
108
+ const s = new SQliteStore({ path: "${path}", D: 64 });
109
+ await s.size();
110
+ const enc = new TextEncoder();
111
+ const g = new Float32Array(64).fill(0.1);
112
+ const before = s.nodeCount();
113
+ for (let i = 0; i < 400; i++) {
114
+ const id = await s.putLeaf(enc.encode("bloom-crash-" + i), g);
115
+ if (id >= before) { console.error("DUP " + i); process.exit(1); }
116
+ }
117
+ if (s.nodeCount() !== before) { console.error("GREW"); process.exit(1); }
118
+ await s.close();
119
+ process.exit(0);
120
+ });`;
121
+ execFileSync(process.execPath, ["-e", phase2], { stdio: "pipe" });
122
+ assert.ok(true);
123
+ });