@hviana/sema 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -5
- package/CITATION.cff +49 -0
- package/HOW_IT_WORKS.md +11 -12
- package/README.md +7 -5
- package/dist/example/demo.js +25 -25
- package/dist/example/train_base.d.ts +19 -21
- package/dist/example/train_base.js +1700 -1952
- package/dist/src/alphabet.d.ts +3 -3
- package/dist/src/alphabet.js +24 -27
- package/dist/src/alu/src/alu.d.ts +169 -194
- package/dist/src/alu/src/alu.js +374 -414
- package/dist/src/alu/src/expr.d.ts +36 -43
- package/dist/src/alu/src/expr.js +252 -278
- package/dist/src/alu/src/index.d.ts +9 -98
- package/dist/src/alu/src/index.js +10 -64
- package/dist/src/alu/src/kernel-arith.d.ts +1 -5
- package/dist/src/alu/src/kernel-arith.js +195 -285
- package/dist/src/alu/src/kernel-bits.d.ts +1 -5
- package/dist/src/alu/src/kernel-bits.js +81 -115
- package/dist/src/alu/src/kernel-logic.js +33 -63
- package/dist/src/alu/src/kernel-nd.js +155 -195
- package/dist/src/alu/src/kernel-numeric.d.ts +16 -80
- package/dist/src/alu/src/kernel-numeric.js +259 -331
- package/dist/src/alu/src/operation.d.ts +112 -132
- package/dist/src/alu/src/operation.js +149 -163
- package/dist/src/alu/src/parser.d.ts +168 -175
- package/dist/src/alu/src/parser.js +382 -421
- package/dist/src/alu/src/resonance.d.ts +16 -25
- package/dist/src/alu/src/resonance.js +49 -56
- package/dist/src/alu/src/text.d.ts +3 -7
- package/dist/src/alu/src/text.js +31 -37
- package/dist/src/alu/src/value.d.ts +14 -14
- package/dist/src/alu/src/value.js +150 -160
- package/dist/src/alu/test/alu.test.js +543 -656
- package/dist/src/bytes.d.ts +1 -5
- package/dist/src/bytes.js +33 -40
- package/dist/src/config.d.ts +99 -102
- package/dist/src/config.js +78 -83
- package/dist/src/derive/src/deduction.d.ts +60 -62
- package/dist/src/derive/src/deduction.js +103 -109
- package/dist/src/derive/src/index.d.ts +1 -7
- package/dist/src/derive/src/priority-queue.d.ts +8 -8
- package/dist/src/derive/src/priority-queue.js +57 -61
- package/dist/src/derive/src/rewrite.d.ts +15 -18
- package/dist/src/derive/src/rewrite.js +59 -67
- package/dist/src/derive/src/trie.d.ts +56 -56
- package/dist/src/derive/src/trie.js +175 -179
- package/dist/src/derive/test/derive.test.js +96 -100
- package/dist/src/extension.d.ts +13 -16
- package/dist/src/geometry.d.ts +29 -43
- package/dist/src/geometry.js +324 -268
- package/dist/src/index.d.ts +2 -17
- package/dist/src/index.js +2 -12
- package/dist/src/ingest-cache.d.ts +25 -30
- package/dist/src/ingest-cache.js +108 -127
- package/dist/src/mind/articulation.d.ts +1 -5
- package/dist/src/mind/articulation.js +77 -112
- package/dist/src/mind/attention.d.ts +40 -108
- package/dist/src/mind/attention.js +754 -872
- package/dist/src/mind/canonical.d.ts +4 -19
- package/dist/src/mind/canonical.js +27 -31
- package/dist/src/mind/graph-search.d.ts +201 -225
- package/dist/src/mind/graph-search.js +742 -821
- package/dist/src/mind/index.d.ts +2 -10
- package/dist/src/mind/junction.d.ts +31 -58
- package/dist/src/mind/junction.js +172 -237
- package/dist/src/mind/learning.d.ts +16 -52
- package/dist/src/mind/learning.js +143 -165
- package/dist/src/mind/match.d.ts +20 -69
- package/dist/src/mind/match.js +259 -318
- package/dist/src/mind/mechanisms/alu.js +16 -16
- package/dist/src/mind/mechanisms/cast.d.ts +9 -13
- package/dist/src/mind/mechanisms/cast.js +363 -456
- package/dist/src/mind/mechanisms/confluence.d.ts +8 -12
- package/dist/src/mind/mechanisms/confluence.js +152 -183
- package/dist/src/mind/mechanisms/cover.d.ts +2 -8
- package/dist/src/mind/mechanisms/cover.js +148 -210
- package/dist/src/mind/mechanisms/extraction.d.ts +8 -34
- package/dist/src/mind/mechanisms/extraction.js +234 -288
- package/dist/src/mind/mechanisms/recall.d.ts +6 -10
- package/dist/src/mind/mechanisms/recall.js +141 -185
- package/dist/src/mind/mind.d.ts +194 -155
- package/dist/src/mind/mind.js +392 -289
- package/dist/src/mind/pipeline-mechanism.d.ts +112 -124
- package/dist/src/mind/pipeline-mechanism.js +161 -172
- package/dist/src/mind/pipeline.d.ts +4 -14
- package/dist/src/mind/pipeline.js +125 -189
- package/dist/src/mind/primitives.d.ts +20 -33
- package/dist/src/mind/primitives.js +124 -120
- package/dist/src/mind/rationale.d.ts +86 -98
- package/dist/src/mind/rationale.js +113 -121
- package/dist/src/mind/reasoning.d.ts +2 -13
- package/dist/src/mind/reasoning.js +129 -166
- package/dist/src/mind/recognition.d.ts +1 -4
- package/dist/src/mind/recognition.js +198 -210
- package/dist/src/mind/resonance.d.ts +5 -22
- package/dist/src/mind/resonance.js +0 -0
- package/dist/src/mind/trace.d.ts +6 -25
- package/dist/src/mind/trace.js +50 -58
- package/dist/src/mind/traverse.d.ts +29 -58
- package/dist/src/mind/traverse.js +344 -356
- package/dist/src/mind/types.d.ts +122 -127
- package/dist/src/mind/types.js +60 -69
- package/dist/src/rabitq-ivf/src/database.d.ts +113 -0
- package/dist/src/rabitq-ivf/src/database.js +201 -0
- package/dist/src/rabitq-ivf/src/index.d.ts +7 -0
- package/dist/src/{rabitq-hnsw → rabitq-ivf}/src/index.js +1 -3
- package/dist/src/rabitq-ivf/src/ivf.d.ts +200 -0
- package/dist/src/rabitq-ivf/src/ivf.js +1165 -0
- package/dist/src/rabitq-ivf/src/prng.d.ts +19 -0
- package/dist/src/rabitq-ivf/src/prng.js +36 -0
- package/dist/src/rabitq-ivf/src/rabitq.d.ts +95 -0
- package/dist/src/rabitq-ivf/src/rabitq.js +283 -0
- package/dist/src/sema.d.ts +9 -13
- package/dist/src/sema.js +26 -40
- package/dist/src/store-sqlite.d.ts +171 -166
- package/dist/src/store-sqlite.js +768 -662
- package/dist/src/store.d.ts +597 -630
- package/dist/src/store.js +1434 -1581
- package/dist/src/vec.d.ts +5 -9
- package/dist/src/vec.js +61 -73
- package/example/train_base.ts +33 -15
- package/index.html +65 -0
- package/package.json +1 -1
- package/src/alu/README.md +1 -1
- package/src/alu/src/index.ts +1 -1
- package/src/config.ts +19 -27
- package/src/geometry.ts +93 -1
- package/src/index.ts +6 -11
- package/src/mind/attention.ts +11 -6
- package/src/mind/mechanisms/recall.ts +29 -13
- package/src/mind/mind.ts +250 -19
- package/src/mind/primitives.ts +43 -6
- package/src/mind/recognition.ts +26 -8
- package/src/mind/traverse.ts +47 -0
- package/src/mind/types.ts +20 -21
- package/src/rabitq-ivf/README.md +56 -0
- package/src/rabitq-ivf/src/database.ts +276 -0
- package/src/{rabitq-hnsw → rabitq-ivf}/src/index.ts +2 -5
- package/src/rabitq-ivf/src/ivf.ts +1330 -0
- package/src/{rabitq-hnsw → rabitq-ivf}/src/prng.ts +1 -1
- package/src/store-sqlite.ts +204 -9
- package/src/store.ts +27 -34
- package/test/08-storage.test.mjs +3 -3
- package/test/13-conversation.test.mjs +190 -20
- package/test/14-scaling.test.mjs +2 -2
- package/test/35-ivf.test.mjs +263 -0
- package/test/36-bloom.test.mjs +123 -0
- package/dist/src/rabitq-hnsw/src/database.d.ts +0 -202
- package/dist/src/rabitq-hnsw/src/database.js +0 -405
- package/dist/src/rabitq-hnsw/src/heap.d.ts +0 -22
- package/dist/src/rabitq-hnsw/src/heap.js +0 -94
- package/dist/src/rabitq-hnsw/src/hnsw.d.ts +0 -125
- package/dist/src/rabitq-hnsw/src/hnsw.js +0 -500
- package/dist/src/rabitq-hnsw/src/index.d.ts +0 -15
- package/dist/src/rabitq-hnsw/src/prng.d.ts +0 -19
- package/dist/src/rabitq-hnsw/src/prng.js +0 -38
- package/dist/src/rabitq-hnsw/src/rabitq.d.ts +0 -95
- package/dist/src/rabitq-hnsw/src/rabitq.js +0 -299
- package/dist/src/rabitq-hnsw/src/store.d.ts +0 -162
- package/dist/src/rabitq-hnsw/src/store.js +0 -916
- package/dist/src/rabitq-hnsw/test/hnsw.test.d.ts +0 -1
- package/dist/src/rabitq-hnsw/test/hnsw.test.js +0 -1197
- package/src/rabitq-hnsw/README.md +0 -303
- package/src/rabitq-hnsw/src/database.ts +0 -492
- package/src/rabitq-hnsw/src/heap.ts +0 -90
- package/src/rabitq-hnsw/src/hnsw.ts +0 -514
- package/src/rabitq-hnsw/src/store.ts +0 -994
- package/src/rabitq-hnsw/test/hnsw.test.ts +0 -1213
- /package/src/{rabitq-hnsw → rabitq-ivf}/src/rabitq.ts +0 -0
|
@@ -14,14 +14,10 @@
|
|
|
14
14
|
// The assertions describe BEHAVIOUR only — they never mention how context is
|
|
15
15
|
// represented — so they stay valid for any implementation.
|
|
16
16
|
//
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
// provided at inference to produce the next turn. The difference is in
|
|
22
|
-
// how the answer is produced: Sema composes it from learned graph forms;
|
|
23
|
-
// an LLM samples it from a parametric distribution. Neither is "just a
|
|
24
|
-
// lookup."
|
|
17
|
+
// Sema trains on the raw concatenation of prior turns and queries the same
|
|
18
|
+
// accumulated bytes at inference. The Conversation API tracks turn-boundary
|
|
19
|
+
// offsets explicitly so no separator character is needed — the geometry never
|
|
20
|
+
// inspects content to find turn boundaries.
|
|
25
21
|
// ─────────────────────────────────────────────────────────────────────────
|
|
26
22
|
|
|
27
23
|
import { test } from "node:test";
|
|
@@ -34,16 +30,14 @@ const newMind = () => new Mind({ seed: 7 });
|
|
|
34
30
|
// ═══════════════════════════════════════════════════════════════════════
|
|
35
31
|
// teachConversation — store one conversation (an ordered list of turns).
|
|
36
32
|
//
|
|
37
|
-
// Accumulate
|
|
38
|
-
// full conversation history:
|
|
39
|
-
//
|
|
40
|
-
// produces different episode vectors because the context side encodes which
|
|
41
|
-
// conversation it belongs to. All SEMA operations remain sublinear — the
|
|
42
|
-
// river cuts any context into O(log N) chunks regardless of length.
|
|
33
|
+
// Accumulate prior turns by raw concatenation — no separator character.
|
|
34
|
+
// Each episode carries the full conversation history: (t₀ → t₁),
|
|
35
|
+
// (t₀+t₁ → t₂), (t₀+t₁+t₂ → t₃) … where + is byte concatenation.
|
|
43
36
|
// ═══════════════════════════════════════════════════════════════════════
|
|
44
37
|
async function teachConversation(mind, turns) {
|
|
38
|
+
let context = "";
|
|
45
39
|
for (let i = 0; i + 1 < turns.length; i++) {
|
|
46
|
-
|
|
40
|
+
context += turns[i];
|
|
47
41
|
await mind.ingest(context, turns[i + 1]);
|
|
48
42
|
}
|
|
49
43
|
}
|
|
@@ -51,13 +45,22 @@ async function teachConversation(mind, turns) {
|
|
|
51
45
|
// ═══════════════════════════════════════════════════════════════════════
|
|
52
46
|
// predictNext — given the turns spoken so far, predict the next turn.
|
|
53
47
|
//
|
|
54
|
-
//
|
|
55
|
-
//
|
|
56
|
-
//
|
|
48
|
+
// Uses the Conversation API: beginConversation → respondTurn for each
|
|
49
|
+
// prior turn → respondTurnText for the query → endConversation.
|
|
50
|
+
// Turns are raw strings, concatenated by the Mind — no separator.
|
|
57
51
|
// ═══════════════════════════════════════════════════════════════════════
|
|
58
52
|
async function predictNext(mind, priorTurns) {
|
|
59
|
-
|
|
60
|
-
|
|
53
|
+
if (priorTurns.length === 0) return "";
|
|
54
|
+
const conv = mind.beginConversation();
|
|
55
|
+
for (let i = 0; i < priorTurns.length - 1; i++) {
|
|
56
|
+
await mind.respondTurn(conv, priorTurns[i]);
|
|
57
|
+
}
|
|
58
|
+
const { response } = await mind.respondTurnText(
|
|
59
|
+
conv,
|
|
60
|
+
priorTurns[priorTurns.length - 1],
|
|
61
|
+
);
|
|
62
|
+
mind.endConversation(conv);
|
|
63
|
+
return response;
|
|
61
64
|
}
|
|
62
65
|
|
|
63
66
|
// A small convenience: teach a conversation, then ask for the continuation
|
|
@@ -226,3 +229,170 @@ test("D2: prediction is deterministic across runs", async () => {
|
|
|
226
229
|
};
|
|
227
230
|
assert.equal(await run(), await run());
|
|
228
231
|
});
|
|
232
|
+
|
|
233
|
+
// ═══════════════════════════════════════════════════════════════════════
|
|
234
|
+
// Section E — Conversation API lifecycle & cache management
|
|
235
|
+
//
|
|
236
|
+
// Tests for the beginConversation / respondTurn / endConversation API
|
|
237
|
+
// itself, independent of the conversation-training pattern above.
|
|
238
|
+
// ═══════════════════════════════════════════════════════════════════════
|
|
239
|
+
|
|
240
|
+
test("E1: begin → respondTurn → end completes a single-turn conversation", async () => {
|
|
241
|
+
const mind = newMind();
|
|
242
|
+
await mind.ingest("hello", "world");
|
|
243
|
+
|
|
244
|
+
const conv = mind.beginConversation();
|
|
245
|
+
const { response, state } = await mind.respondTurnText(conv, "hello");
|
|
246
|
+
assert.equal(response, "world");
|
|
247
|
+
// Single turn — no boundaries yet.
|
|
248
|
+
assert.equal(state.boundaries.length, 0);
|
|
249
|
+
mind.endConversation(conv);
|
|
250
|
+
await mind.store.close();
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
test("E2: multi-turn conversation accumulates context and boundaries", async () => {
|
|
254
|
+
const mind = newMind();
|
|
255
|
+
await teachConversation(mind, CAT);
|
|
256
|
+
|
|
257
|
+
const conv = mind.beginConversation();
|
|
258
|
+
// First turn establishes the subject.
|
|
259
|
+
await mind.respondTurn(conv, "I adopted a cat");
|
|
260
|
+
|
|
261
|
+
// Second turn is the pivot query.
|
|
262
|
+
const t2 = await mind.respondTurnText(conv, "what is its name?");
|
|
263
|
+
// Now we have one boundary: the byte position after the first turn.
|
|
264
|
+
assert.equal(t2.state.boundaries.length, 1);
|
|
265
|
+
assert.equal(t2.response, "her name is Whiskers");
|
|
266
|
+
|
|
267
|
+
mind.endConversation(conv);
|
|
268
|
+
await mind.store.close();
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
test("E3: save → end → restore → continue works", async () => {
|
|
272
|
+
const mind = newMind();
|
|
273
|
+
await teachConversation(mind, CAT);
|
|
274
|
+
|
|
275
|
+
// First two turns.
|
|
276
|
+
const conv = mind.beginConversation();
|
|
277
|
+
await mind.respondTurn(conv, "I adopted a cat");
|
|
278
|
+
const { state: saved } = await mind.respondTurnText(
|
|
279
|
+
conv,
|
|
280
|
+
"what is its name?",
|
|
281
|
+
);
|
|
282
|
+
assert.equal(saved.boundaries.length, 1);
|
|
283
|
+
|
|
284
|
+
// Save and end.
|
|
285
|
+
mind.endConversation(conv);
|
|
286
|
+
|
|
287
|
+
// Restore from the saved state into a new handle.
|
|
288
|
+
const conv2 = mind.beginConversation(saved);
|
|
289
|
+
const { response, state: state2 } = await mind.respondTurnText(
|
|
290
|
+
conv2,
|
|
291
|
+
"her name is Whiskers",
|
|
292
|
+
);
|
|
293
|
+
// The restored conversation continues from where it left off.
|
|
294
|
+
assert.ok(response.length > 0 || response === "");
|
|
295
|
+
// Boundaries are intact from the restore.
|
|
296
|
+
assert.equal(state2.boundaries.length, 2);
|
|
297
|
+
|
|
298
|
+
mind.endConversation(conv2);
|
|
299
|
+
await mind.store.close();
|
|
300
|
+
});
|
|
301
|
+
|
|
302
|
+
test("E4: endConversation is idempotent", async () => {
|
|
303
|
+
const mind = newMind();
|
|
304
|
+
const conv = mind.beginConversation();
|
|
305
|
+
mind.endConversation(conv);
|
|
306
|
+
// Second end on the same handle — must not throw.
|
|
307
|
+
mind.endConversation(conv);
|
|
308
|
+
await mind.store.close();
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
test("E5: respondTurn on an ended conversation throws", async () => {
|
|
312
|
+
const mind = newMind();
|
|
313
|
+
const conv = mind.beginConversation();
|
|
314
|
+
mind.endConversation(conv);
|
|
315
|
+
await assert.rejects(
|
|
316
|
+
() => mind.respondTurn(conv, "hello"),
|
|
317
|
+
/not found/,
|
|
318
|
+
);
|
|
319
|
+
await mind.store.close();
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
test("E6: conversationState returns null after endConversation", async () => {
|
|
323
|
+
const mind = newMind();
|
|
324
|
+
const conv = mind.beginConversation();
|
|
325
|
+
mind.endConversation(conv);
|
|
326
|
+
assert.equal(mind.conversationState(conv), null);
|
|
327
|
+
await mind.store.close();
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
test("E7: multiple concurrent conversations are independent", async () => {
|
|
331
|
+
const mind = newMind();
|
|
332
|
+
await teachConversation(mind, CAT);
|
|
333
|
+
await teachConversation(mind, DOG);
|
|
334
|
+
|
|
335
|
+
const catConv = mind.beginConversation();
|
|
336
|
+
const dogConv = mind.beginConversation();
|
|
337
|
+
|
|
338
|
+
// Feed the distinguishing context to each.
|
|
339
|
+
await mind.respondTurn(catConv, "I adopted a cat");
|
|
340
|
+
await mind.respondTurn(dogConv, "I adopted a dog");
|
|
341
|
+
|
|
342
|
+
// The same pivot turn — different answers.
|
|
343
|
+
const catR = await mind.respondTurnText(catConv, "what is its name?");
|
|
344
|
+
const dogR = await mind.respondTurnText(dogConv, "what is its name?");
|
|
345
|
+
|
|
346
|
+
assert.equal(catR.response, "her name is Whiskers");
|
|
347
|
+
assert.equal(dogR.response, "his name is Rex");
|
|
348
|
+
assert.notEqual(catR.response, dogR.response);
|
|
349
|
+
|
|
350
|
+
// Each conversation has its own boundaries.
|
|
351
|
+
assert.equal(catR.state.boundaries.length, 1);
|
|
352
|
+
assert.equal(dogR.state.boundaries.length, 1);
|
|
353
|
+
|
|
354
|
+
// Ending one does not affect the other.
|
|
355
|
+
mind.endConversation(catConv);
|
|
356
|
+
assert.equal(mind.conversationState(catConv), null);
|
|
357
|
+
assert.notEqual(mind.conversationState(dogConv), null);
|
|
358
|
+
|
|
359
|
+
// End the second — both are now gone.
|
|
360
|
+
mind.endConversation(dogConv);
|
|
361
|
+
assert.equal(mind.conversationState(dogConv), null);
|
|
362
|
+
|
|
363
|
+
// Ending again is idempotent.
|
|
364
|
+
mind.endConversation(dogConv);
|
|
365
|
+
await mind.store.close();
|
|
366
|
+
});
|
|
367
|
+
|
|
368
|
+
test("E8: respondTurn with a forged handle throws", async () => {
|
|
369
|
+
const mind = newMind();
|
|
370
|
+
// { id: 999 } was never returned by beginConversation.
|
|
371
|
+
await assert.rejects(
|
|
372
|
+
() => mind.respondTurn({ id: 999 }, "hello"),
|
|
373
|
+
/not found/,
|
|
374
|
+
);
|
|
375
|
+
await mind.store.close();
|
|
376
|
+
});
|
|
377
|
+
|
|
378
|
+
test("E9: respond and respondTurn give the same answer for the same cumulative bytes", async () => {
|
|
379
|
+
const mind = newMind();
|
|
380
|
+
await teachConversation(mind, CAT);
|
|
381
|
+
|
|
382
|
+
// Via respond() — raw concatenation, same as respondTurn.
|
|
383
|
+
const viaRespond = await mind.respondText(
|
|
384
|
+
"I adopted a catwhat is its name?",
|
|
385
|
+
);
|
|
386
|
+
|
|
387
|
+
// Via respondTurn — the new way.
|
|
388
|
+
const conv = mind.beginConversation();
|
|
389
|
+
await mind.respondTurnText(conv, "I adopted a cat");
|
|
390
|
+
const { response: viaConv } = await mind.respondTurnText(
|
|
391
|
+
conv,
|
|
392
|
+
"what is its name?",
|
|
393
|
+
);
|
|
394
|
+
mind.endConversation(conv);
|
|
395
|
+
|
|
396
|
+
assert.equal(viaRespond, viaConv);
|
|
397
|
+
await mind.store.close();
|
|
398
|
+
});
|
package/test/14-scaling.test.mjs
CHANGED
|
@@ -112,7 +112,7 @@ function unknownInput(targetBytes) {
|
|
|
112
112
|
// jitter swamp the signal over a few thousand deposits, so any strict
|
|
113
113
|
// degradation bound is either flaky or so loose it catches nothing.
|
|
114
114
|
//
|
|
115
|
-
// The real degradation lived in the vector index (rabitq-
|
|
115
|
+
// The real degradation lived in the vector index (rabitq-ivf) and is now guarded
|
|
116
116
|
// DETERMINISTICALLY at its own layer (testScalabilityWithoutCache, on the
|
|
117
117
|
// cache-independent storage-read count). HERE we assert the same contract end to
|
|
118
118
|
// end through the Mind, also by STORAGE READS rather than time: the work one
|
|
@@ -170,7 +170,7 @@ test("training: recall work does NOT grow with the store (storage reads, not tim
|
|
|
170
170
|
|
|
171
171
|
// 6x the corpus must not cost anywhere near 6x the reads. A genuinely
|
|
172
172
|
// sublinear (~log N) lookup barely moves; a per-recall cost that tracked the
|
|
173
|
-
// corpus would roughly 6x. Allow a generous 3x band for
|
|
173
|
+
// corpus would roughly 6x. Allow a generous 3x band for estimator noise
|
|
174
174
|
// and graph-shape variation, far below the ~6x a true O(N) regression shows.
|
|
175
175
|
assert.ok(
|
|
176
176
|
large < small * 3 + 50,
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
// 35-ivf.test.mjs — the partitioned (IVF) vector index, at its own layer.
|
|
2
|
+
//
|
|
3
|
+
// VectorDatabase is an adaptive IVF index over 1-bit RaBitQ codes in SQLite
|
|
4
|
+
// (src/rabitq-ivf). These tests verify the index's OWN contract, below the
|
|
5
|
+
// Sema store:
|
|
6
|
+
//
|
|
7
|
+
// 1. CRUD — insert/has/get/update/delete, id checking.
|
|
8
|
+
// 2. Recall — clustered data resonates to its own cluster; a stored
|
|
9
|
+
// vector finds itself.
|
|
10
|
+
// 3. Splits — the index partitions past SPLIT_MAX and stays correct.
|
|
11
|
+
// 4. Persistence — close/reopen preserves entries, config, and results.
|
|
12
|
+
// 5. Determinism — same insert sequence ⇒ identical results (no RNG).
|
|
13
|
+
// 6. Compaction — tombstones are reclaimed; internal ids (the keysSince
|
|
14
|
+
// watermark) survive.
|
|
15
|
+
// 7. Scaling — per-query STORAGE READS are bounded once the
|
|
16
|
+
// collection is past its first splits: 2x the corpus
|
|
17
|
+
// must not move the reads (the honest witness the old
|
|
18
|
+
// graph index also swore to).
|
|
19
|
+
|
|
20
|
+
import { test } from "node:test";
|
|
21
|
+
import assert from "node:assert/strict";
|
|
22
|
+
import { rmSync } from "node:fs";
|
|
23
|
+
import { VectorDatabase } from "../dist/src/rabitq-ivf/src/index.js";
|
|
24
|
+
|
|
25
|
+
// Deterministic PRNG for test vectors.
|
|
26
|
+
function rng(seed) {
|
|
27
|
+
let s = seed >>> 0;
|
|
28
|
+
return () => {
|
|
29
|
+
s ^= s << 13;
|
|
30
|
+
s ^= s >>> 17;
|
|
31
|
+
s ^= s << 5;
|
|
32
|
+
s >>>= 0;
|
|
33
|
+
return s / 0x100000000;
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** A unit vector near `center` with per-coordinate jitter. */
|
|
38
|
+
function near(center, jitter, r) {
|
|
39
|
+
const v = new Float32Array(center.length);
|
|
40
|
+
for (let i = 0; i < v.length; i++) {
|
|
41
|
+
v[i] = center[i] + (r() * 2 - 1) * jitter;
|
|
42
|
+
}
|
|
43
|
+
return v;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function randVec(dim, r) {
|
|
47
|
+
const v = new Float32Array(dim);
|
|
48
|
+
for (let i = 0; i < dim; i++) v[i] = r() * 2 - 1;
|
|
49
|
+
return v;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
test("1: CRUD — insert, has, get, update, delete", () => {
|
|
53
|
+
const db = new VectorDatabase({ dbPath: ":memory:", dim: 64 });
|
|
54
|
+
const r = rng(1);
|
|
55
|
+
const v1 = randVec(64, r);
|
|
56
|
+
const v2 = randVec(64, r);
|
|
57
|
+
|
|
58
|
+
db.insert(10, v1);
|
|
59
|
+
assert.equal(db.size, 1);
|
|
60
|
+
assert.ok(db.has(10));
|
|
61
|
+
assert.ok(!db.has(11));
|
|
62
|
+
assert.throws(() => db.insert(10, v2), /already exists/);
|
|
63
|
+
assert.throws(() => db.insert(1.5, v2), /integer/);
|
|
64
|
+
|
|
65
|
+
const code = db.get(10);
|
|
66
|
+
assert.ok(code instanceof Uint32Array);
|
|
67
|
+
|
|
68
|
+
// update to a genuinely different vector: same ext still resolves
|
|
69
|
+
db.update(10, v2);
|
|
70
|
+
assert.equal(db.size, 1);
|
|
71
|
+
assert.ok(db.has(10));
|
|
72
|
+
|
|
73
|
+
// byte-identical update is a no-op (no new tombstone)
|
|
74
|
+
const phys = db.physicalSize;
|
|
75
|
+
db.update(10, v2);
|
|
76
|
+
assert.equal(db.physicalSize, phys);
|
|
77
|
+
|
|
78
|
+
assert.ok(db.delete(10));
|
|
79
|
+
assert.ok(!db.has(10));
|
|
80
|
+
assert.equal(db.size, 0);
|
|
81
|
+
assert.ok(!db.delete(10));
|
|
82
|
+
db.close();
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test("2: recall — clustered data finds its own cluster; self-recall", () => {
|
|
86
|
+
const dim = 128;
|
|
87
|
+
const db = new VectorDatabase({ dbPath: ":memory:", dim });
|
|
88
|
+
const r = rng(2);
|
|
89
|
+
// 8 well-separated centers, 100 members each.
|
|
90
|
+
const centers = [];
|
|
91
|
+
for (let c = 0; c < 8; c++) centers.push(randVec(dim, r));
|
|
92
|
+
const vecs = new Map();
|
|
93
|
+
let id = 0;
|
|
94
|
+
const batch = [];
|
|
95
|
+
for (let c = 0; c < 8; c++) {
|
|
96
|
+
for (let m = 0; m < 100; m++) {
|
|
97
|
+
const v = near(centers[c], 0.15, r);
|
|
98
|
+
vecs.set(id, { v, c });
|
|
99
|
+
batch.push({ id, vector: v });
|
|
100
|
+
id++;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
db.upsertMany(batch);
|
|
104
|
+
assert.equal(db.size, 800);
|
|
105
|
+
|
|
106
|
+
// Query each center: top-10 should come from that center's members.
|
|
107
|
+
for (let c = 0; c < 8; c++) {
|
|
108
|
+
const hits = db.query(centers[c], 10);
|
|
109
|
+
assert.equal(hits.length, 10);
|
|
110
|
+
let own = 0;
|
|
111
|
+
for (const h of hits) if (vecs.get(h.id).c === c) own++;
|
|
112
|
+
assert.ok(own >= 8, `center ${c}: only ${own}/10 hits from own cluster`);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Self-recall: a stored vector should find its own ext in the top 5.
|
|
116
|
+
let self = 0;
|
|
117
|
+
for (let probe = 0; probe < 100; probe += 7) {
|
|
118
|
+
const hits = db.query(vecs.get(probe).v, 5);
|
|
119
|
+
if (hits.some((h) => h.id === probe)) self++;
|
|
120
|
+
}
|
|
121
|
+
assert.ok(self >= 12, `self-recall ${self}/15 too low`);
|
|
122
|
+
db.close();
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
test("3: splits — collections past SPLIT_MAX partition and stay correct", () => {
|
|
126
|
+
const dim = 32;
|
|
127
|
+
const db = new VectorDatabase({ dbPath: ":memory:", dim });
|
|
128
|
+
const r = rng(3);
|
|
129
|
+
const N = 10_000; // > 2 × SPLIT_MAX (4096) — forces several splits
|
|
130
|
+
const batch = [];
|
|
131
|
+
for (let i = 0; i < N; i++) batch.push({ id: i, vector: randVec(dim, r) });
|
|
132
|
+
db.upsertMany(batch);
|
|
133
|
+
assert.equal(db.size, N);
|
|
134
|
+
assert.ok(
|
|
135
|
+
db.clusterCount >= 3,
|
|
136
|
+
`expected several clusters after ${N} inserts, got ${db.clusterCount}`,
|
|
137
|
+
);
|
|
138
|
+
// Every ext still resolves and queries still return k results.
|
|
139
|
+
assert.ok(db.has(0) && db.has(N - 1) && db.has(N >> 1));
|
|
140
|
+
const hits = db.query(randVec(dim, r), 20);
|
|
141
|
+
assert.equal(hits.length, 20);
|
|
142
|
+
db.close();
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
test("4: persistence — close/reopen preserves entries and results", (t) => {
|
|
146
|
+
const path = `/tmp/ivf-test-${process.pid}.vec`;
|
|
147
|
+
t.after(() => {
|
|
148
|
+
for (const s of ["", "-wal", "-shm"]) {
|
|
149
|
+
try {
|
|
150
|
+
rmSync(path + s);
|
|
151
|
+
} catch {}
|
|
152
|
+
}
|
|
153
|
+
});
|
|
154
|
+
const dim = 64;
|
|
155
|
+
const r = rng(4);
|
|
156
|
+
const vecs = [];
|
|
157
|
+
for (let i = 0; i < 500; i++) vecs.push(randVec(dim, r));
|
|
158
|
+
|
|
159
|
+
let db = new VectorDatabase({ dbPath: path, dim });
|
|
160
|
+
db.upsertMany(vecs.map((v, i) => ({ id: i, vector: v })));
|
|
161
|
+
const probe = vecs[123];
|
|
162
|
+
const before = db.query(probe, 10);
|
|
163
|
+
db.close();
|
|
164
|
+
|
|
165
|
+
db = new VectorDatabase({ dbPath: path });
|
|
166
|
+
assert.equal(db.dim, dim);
|
|
167
|
+
assert.equal(db.size, 500);
|
|
168
|
+
const after = db.query(probe, 10);
|
|
169
|
+
assert.deepEqual(after, before);
|
|
170
|
+
db.close();
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
test("5: determinism — same insert sequence, identical results", () => {
|
|
174
|
+
const dim = 64;
|
|
175
|
+
const r1 = rng(5), r2 = rng(5);
|
|
176
|
+
const a = new VectorDatabase({ dbPath: ":memory:", dim });
|
|
177
|
+
const b = new VectorDatabase({ dbPath: ":memory:", dim });
|
|
178
|
+
const N = 6000; // crosses a split
|
|
179
|
+
const batchA = [], batchB = [];
|
|
180
|
+
for (let i = 0; i < N; i++) {
|
|
181
|
+
batchA.push({ id: i, vector: randVec(dim, r1) });
|
|
182
|
+
batchB.push({ id: i, vector: randVec(dim, r2) });
|
|
183
|
+
}
|
|
184
|
+
a.upsertMany(batchA);
|
|
185
|
+
b.upsertMany(batchB);
|
|
186
|
+
assert.equal(a.clusterCount, b.clusterCount);
|
|
187
|
+
const probe = randVec(dim, r1);
|
|
188
|
+
assert.deepEqual(a.query(probe, 20), b.query(probe, 20));
|
|
189
|
+
a.close();
|
|
190
|
+
b.close();
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
test("6: compaction — tombstones reclaimed, watermark ids preserved", () => {
|
|
194
|
+
const dim = 32;
|
|
195
|
+
const db = new VectorDatabase({ dbPath: ":memory:", dim });
|
|
196
|
+
const r = rng(6);
|
|
197
|
+
const N = 2000;
|
|
198
|
+
db.upsertMany(
|
|
199
|
+
Array.from({ length: N }, (_, i) => ({ id: i, vector: randVec(dim, r) })),
|
|
200
|
+
);
|
|
201
|
+
|
|
202
|
+
// Watermark before deletion.
|
|
203
|
+
let maxInternal = 0;
|
|
204
|
+
for (const { internal } of db.keysSince(0)) {
|
|
205
|
+
maxInternal = Math.max(maxInternal, internal);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// Delete every third entry.
|
|
209
|
+
const dead = [];
|
|
210
|
+
for (let i = 0; i < N; i += 3) dead.push(i);
|
|
211
|
+
assert.equal(db.deleteMany(dead), dead.length);
|
|
212
|
+
assert.equal(db.size, N - dead.length);
|
|
213
|
+
assert.ok(db.physicalSize > db.size);
|
|
214
|
+
|
|
215
|
+
db.compact();
|
|
216
|
+
assert.equal(db.physicalSize, db.size);
|
|
217
|
+
for (const i of dead) assert.ok(!db.has(i));
|
|
218
|
+
assert.ok(db.has(1) && db.has(2));
|
|
219
|
+
|
|
220
|
+
// Internal ids survive compaction: entries added AFTER the old watermark
|
|
221
|
+
// are exactly the new inserts.
|
|
222
|
+
db.upsertMany([{ id: 99999, vector: randVec(dim, r) }]);
|
|
223
|
+
const fresh = [...db.keysSince(maxInternal)];
|
|
224
|
+
assert.equal(fresh.length, 1);
|
|
225
|
+
assert.equal(fresh[0].ext, 99999);
|
|
226
|
+
db.close();
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
test("7: scaling — per-query storage reads are bounded past the first splits", () => {
|
|
230
|
+
const dim = 32;
|
|
231
|
+
// cacheSizeMb 0: no page cache to hide behind; `reads` counts blob rows.
|
|
232
|
+
const db = new VectorDatabase({ dbPath: ":memory:", dim, cacheSizeMb: 0 });
|
|
233
|
+
const r = rng(7);
|
|
234
|
+
const probe = randVec(dim, r);
|
|
235
|
+
const readsNow = () => {
|
|
236
|
+
db.query(probe, 10);
|
|
237
|
+
return db.lastQueryStorageReads;
|
|
238
|
+
};
|
|
239
|
+
|
|
240
|
+
const grow = (n) => {
|
|
241
|
+
const batch = [];
|
|
242
|
+
for (let i = 0; i < n; i++) {
|
|
243
|
+
batch.push({ id: db.size + i, vector: randVec(dim, r) });
|
|
244
|
+
}
|
|
245
|
+
db.upsertMany(batch);
|
|
246
|
+
};
|
|
247
|
+
|
|
248
|
+
// Past ~nprobe × SPLIT_MAX entries the probed set is a fixed number of
|
|
249
|
+
// bounded clusters — reads must plateau.
|
|
250
|
+
grow(80_000);
|
|
251
|
+
const small = readsNow();
|
|
252
|
+
grow(80_000); // 2x the corpus
|
|
253
|
+
const large = readsNow();
|
|
254
|
+
console.log(
|
|
255
|
+
` ivf reads for one query: N=80k → ${small}, N=160k → ${large}`,
|
|
256
|
+
);
|
|
257
|
+
assert.ok(
|
|
258
|
+
large <= small * 1.5 + 8,
|
|
259
|
+
`reads grew ${small} → ${large} when the corpus doubled — probes are ` +
|
|
260
|
+
`not bounded`,
|
|
261
|
+
);
|
|
262
|
+
db.close();
|
|
263
|
+
});
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// 36-bloom.test.mjs — the negative dedup filter (store-sqlite's Bloom layer).
|
|
2
|
+
//
|
|
3
|
+
// The filter answers "this content hash is definitely not in the node table"
|
|
4
|
+
// in RAM, sparing the training hot path its dominant SQLite probe (profiled:
|
|
5
|
+
// 38.7% of ingest wall at 2.9M nodes). Its ONE hazard is a false negative,
|
|
6
|
+
// which would mint a duplicate node and silently break hash-consing. These
|
|
7
|
+
// tests pin the invariant on the three lifecycle paths:
|
|
8
|
+
//
|
|
9
|
+
// 1. live session — content minted then re-probed dedups (filter sees
|
|
10
|
+
// every insert synchronously);
|
|
11
|
+
// 2. clean close — reopen loads the persisted filter (or rebuilds) and
|
|
12
|
+
// dedup still holds;
|
|
13
|
+
// 3. CRASH — a process that exits without close() leaves no
|
|
14
|
+
// persisted filter (or a stale one); reopen must
|
|
15
|
+
// rebuild/top-up from the table and STILL dedup all
|
|
16
|
+
// previously-minted content, with zero duplicates.
|
|
17
|
+
|
|
18
|
+
import { test } from "node:test";
|
|
19
|
+
import assert from "node:assert/strict";
|
|
20
|
+
import { execFileSync } from "node:child_process";
|
|
21
|
+
import { rmSync } from "node:fs";
|
|
22
|
+
import { SQliteStore } from "../dist/src/store-sqlite.js";
|
|
23
|
+
|
|
24
|
+
const cleanup = (path) => {
|
|
25
|
+
for (
|
|
26
|
+
const s of [
|
|
27
|
+
".sqlite",
|
|
28
|
+
".sqlite-wal",
|
|
29
|
+
".sqlite-shm",
|
|
30
|
+
".content.vec",
|
|
31
|
+
".content.vec-wal",
|
|
32
|
+
".content.vec-shm",
|
|
33
|
+
".halo.vec",
|
|
34
|
+
".halo.vec-wal",
|
|
35
|
+
".halo.vec-shm",
|
|
36
|
+
]
|
|
37
|
+
) {
|
|
38
|
+
try {
|
|
39
|
+
rmSync(path + s);
|
|
40
|
+
} catch {}
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
test("live session: re-probing minted content dedups (no false negatives)", async () => {
|
|
45
|
+
const s = new SQliteStore({ path: ":memory:", D: 64 });
|
|
46
|
+
const enc = new TextEncoder();
|
|
47
|
+
const g = new Float32Array(64).fill(0.1);
|
|
48
|
+
const ids = [];
|
|
49
|
+
for (let i = 0; i < 300; i++) {
|
|
50
|
+
ids.push(await s.putLeaf(enc.encode("bloom-live-" + i), g));
|
|
51
|
+
}
|
|
52
|
+
for (let i = 0; i < 300; i++) {
|
|
53
|
+
assert.equal(await s.putLeaf(enc.encode("bloom-live-" + i), g), ids[i]);
|
|
54
|
+
}
|
|
55
|
+
// Novel probes are answered by the filter without SQLite (the point).
|
|
56
|
+
const skipsBefore = s.bloomSkips;
|
|
57
|
+
s.findLeaf(enc.encode("definitely-not-present-xyzzy"));
|
|
58
|
+
assert.ok(s.bloomSkips >= skipsBefore, "bloomSkips must be observable");
|
|
59
|
+
await s.close();
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test("clean close: reopen dedups against the persisted/rebuilt filter", async (t) => {
|
|
63
|
+
const path = `/tmp/bloom-clean-${process.pid}`;
|
|
64
|
+
t.after(() => cleanup(path));
|
|
65
|
+
const enc = new TextEncoder();
|
|
66
|
+
const g = new Float32Array(64).fill(0.1);
|
|
67
|
+
|
|
68
|
+
let s = new SQliteStore({ path, D: 64 });
|
|
69
|
+
const ids = [];
|
|
70
|
+
for (let i = 0; i < 300; i++) {
|
|
71
|
+
ids.push(await s.putLeaf(enc.encode("bloom-clean-" + i), g));
|
|
72
|
+
}
|
|
73
|
+
await s.close();
|
|
74
|
+
|
|
75
|
+
s = new SQliteStore({ path, D: 64 });
|
|
76
|
+
const before = s.nodeCount();
|
|
77
|
+
for (let i = 0; i < 300; i++) {
|
|
78
|
+
const id = await s.putLeaf(enc.encode("bloom-clean-" + i), g);
|
|
79
|
+
assert.equal(id, ids[i], `content ${i} re-minted after clean reopen`);
|
|
80
|
+
}
|
|
81
|
+
assert.equal(s.nodeCount(), before, "no duplicate mints after clean reopen");
|
|
82
|
+
await s.close();
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test("crash: reopen after an unclean exit still dedups everything", (t) => {
|
|
86
|
+
const path = `/tmp/bloom-crash-${process.pid}`;
|
|
87
|
+
t.after(() => cleanup(path));
|
|
88
|
+
|
|
89
|
+
// Phase 1 in a SUBPROCESS that exits WITHOUT close() — a real crash as far
|
|
90
|
+
// as the store is concerned (WAL holds the rows; no filter was persisted).
|
|
91
|
+
const phase1 = `
|
|
92
|
+
import("${process.cwd()}/dist/src/store-sqlite.js").then(async ({ SQliteStore }) => {
|
|
93
|
+
const s = new SQliteStore({ path: "${path}", D: 64 });
|
|
94
|
+
const enc = new TextEncoder();
|
|
95
|
+
const g = new Float32Array(64).fill(0.1);
|
|
96
|
+
for (let i = 0; i < 400; i++) {
|
|
97
|
+
await s.putLeaf(enc.encode("bloom-crash-" + i), g);
|
|
98
|
+
}
|
|
99
|
+
s.commit();
|
|
100
|
+
process.exit(0); // no close()
|
|
101
|
+
});`;
|
|
102
|
+
execFileSync(process.execPath, ["-e", phase1], { stdio: "pipe" });
|
|
103
|
+
|
|
104
|
+
// Phase 2 in a second subprocess: every previously-minted content must
|
|
105
|
+
// resolve to an existing id — zero duplicates.
|
|
106
|
+
const phase2 = `
|
|
107
|
+
import("${process.cwd()}/dist/src/store-sqlite.js").then(async ({ SQliteStore }) => {
|
|
108
|
+
const s = new SQliteStore({ path: "${path}", D: 64 });
|
|
109
|
+
await s.size();
|
|
110
|
+
const enc = new TextEncoder();
|
|
111
|
+
const g = new Float32Array(64).fill(0.1);
|
|
112
|
+
const before = s.nodeCount();
|
|
113
|
+
for (let i = 0; i < 400; i++) {
|
|
114
|
+
const id = await s.putLeaf(enc.encode("bloom-crash-" + i), g);
|
|
115
|
+
if (id >= before) { console.error("DUP " + i); process.exit(1); }
|
|
116
|
+
}
|
|
117
|
+
if (s.nodeCount() !== before) { console.error("GREW"); process.exit(1); }
|
|
118
|
+
await s.close();
|
|
119
|
+
process.exit(0);
|
|
120
|
+
});`;
|
|
121
|
+
execFileSync(process.execPath, ["-e", phase2], { stdio: "pipe" });
|
|
122
|
+
assert.ok(true);
|
|
123
|
+
});
|