@statewalker/db-sqlite-index 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +289 -2
  3. package/dist/ngram.d.mts +93 -0
  4. package/dist/ngram.d.mts.map +1 -0
  5. package/dist/ngram.mjs +615 -0
  6. package/dist/ngram.mjs.map +1 -0
  7. package/dist/vector.d.mts +103 -0
  8. package/dist/vector.d.mts.map +1 -0
  9. package/dist/vector.mjs +1061 -0
  10. package/dist/vector.mjs.map +1 -0
  11. package/package.json +63 -3
  12. package/src/ngram/do-smoke.ts +46 -0
  13. package/src/ngram/fold.test.ts +50 -0
  14. package/src/ngram/fold.ts +21 -0
  15. package/src/ngram/fts5-backend.ts +55 -0
  16. package/src/ngram/index.ts +12 -0
  17. package/src/ngram/json-chunks.test.ts +23 -0
  18. package/src/ngram/json-chunks.ts +23 -0
  19. package/src/ngram/ngram-store.test.ts +291 -0
  20. package/src/ngram/ngram-store.ts +183 -0
  21. package/src/ngram/plan-query.test.ts +141 -0
  22. package/src/ngram/plan-query.ts +42 -0
  23. package/src/ngram/postings-backend.ts +53 -0
  24. package/src/ngram/regex-literals.ts +359 -0
  25. package/src/ngram/trigrams.test.ts +28 -0
  26. package/src/ngram/trigrams.ts +25 -0
  27. package/src/vector/auto-compact.test.ts +28 -0
  28. package/src/vector/do-smoke.ts +39 -0
  29. package/src/vector/index.ts +9 -0
  30. package/src/vector/kmeans.ts +131 -0
  31. package/src/vector/math.ts +185 -0
  32. package/src/vector/segment.test.ts +61 -0
  33. package/src/vector/segment.ts +256 -0
  34. package/src/vector/sql-segment-store.ts +218 -0
  35. package/src/vector/test-fixtures.ts +64 -0
  36. package/src/vector/types.ts +30 -0
  37. package/src/vector/vector-index.test.ts +105 -0
  38. package/src/vector/vector-index.ts +586 -0
  39. package/src/vector/vector-lifecycle.test.ts +171 -0
@@ -0,0 +1,171 @@
1
+ import type { Db } from "@statewalker/db-api";
2
+ import { SQLITE_TRANSACTION } from "@statewalker/db-sqlite";
3
+ import { describe, expect, it } from "vitest";
4
+ import { PART_BYTES } from "./sql-segment-store.js";
5
+ import { BACKENDS, bruteForce, D, dataset } from "./test-fixtures.js";
6
+ import { dropVectorIndex, VectorIndex } from "./vector-index.js";
7
+
8
+ for (const [name, open] of Object.entries(BACKENDS)) {
9
+ describe(`VectorIndex lifecycle over ${name}`, { timeout: 60_000 }, () => {
10
+ async function setup(dims = D) {
11
+ const db = await open();
12
+ const index = new VectorIndex({ db, prefix: "vec_", dims });
13
+ await index.init();
14
+ return { db, index };
15
+ }
16
+
17
+ it("probing every list with exact re-rank equals brute force", async () => {
18
+ const { index } = await setup();
19
+ const items = dataset(3000, 11);
20
+ await index.upsert(items);
21
+ await index.compact();
22
+ expect(index.stats.segments[0]?.nlist).toBeGreaterThan(1);
23
+ for (const it of items.slice(0, 20)) {
24
+ const q = it.vector.map((x) => x + 0.01);
25
+ const got = await index.search(q, { k: 10, nprobe: 1e6, oversample: 1e6, exact: true });
26
+ expect(got.map((h) => h.id)).toEqual(bruteForce(items, q, 10, "cosine"));
27
+ }
28
+ });
29
+
30
+ it("empty index and k > n return what exists", async () => {
31
+ const { index } = await setup();
32
+ const q = dataset(1, 12)[0]?.vector as Float32Array;
33
+ expect(await index.search(q, { k: 10 })).toEqual([]);
34
+ await index.upsert(dataset(5, 12));
35
+ expect((await index.search(q, { k: 10 })).length).toBe(5);
36
+ await index.compact();
37
+ expect((await index.search(q, { k: 10 })).length).toBe(5);
38
+ });
39
+
40
+ it("rejects invalid input and writes nothing", async () => {
41
+ const { index } = await setup();
42
+ await expect(index.upsert([{ id: 1, vector: new Float32Array(D + 1) }])).rejects.toThrow(
43
+ /expected 64 dims/,
44
+ );
45
+ const nan = new Float32Array(D);
46
+ nan[3] = Number.NaN;
47
+ await expect(index.upsert([{ id: 1, vector: nan }])).rejects.toThrow(/non-finite/);
48
+ await expect(index.upsert([{ id: 2 ** 53, vector: new Float32Array(D) }])).rejects.toThrow(
49
+ /safe integer/,
50
+ );
51
+ expect(index.stats.tail).toBe(0);
52
+ });
53
+
54
+ it("deletes many ids in one call", async () => {
55
+ const { index } = await setup();
56
+ const items = dataset(400, 13);
57
+ await index.upsert(items);
58
+ await index.compact();
59
+ const gone = items.slice(0, 150).map((it) => it.id);
60
+ await index.delete(gone);
61
+ const dead = new Set(gone);
62
+ for (const it of items.slice(0, 10)) {
63
+ const hits = await index.search(it.vector, { k: 20 });
64
+ expect(hits.some((h) => dead.has(h.id))).toBe(false);
65
+ }
66
+ });
67
+
68
+ it("splits list blocks into parts no larger than PART_BYTES", async () => {
69
+ const dims = 768;
70
+ const { db, index } = await setup(dims);
71
+ const items = dataset(1500, 14, dims); // flat: one list of ~1.3 MB, floats ~4.6 MB
72
+ await index.upsert(items);
73
+ await index.compact();
74
+ const [seg] = await db.query<{ n: number; max: number }>(
75
+ "SELECT count(*) AS n, max(length(bytes)) AS max FROM vec_seg",
76
+ );
77
+ const [f32] = await db.query<{ n: number; max: number }>(
78
+ "SELECT count(*) AS n, max(length(bytes)) AS max FROM vec_f32",
79
+ );
80
+ expect(Number(seg?.n)).toBeGreaterThanOrEqual(2);
81
+ expect(Number(f32?.n)).toBeGreaterThanOrEqual(5);
82
+ expect(Math.max(Number(seg?.max), Number(f32?.max))).toBeLessThanOrEqual(PART_BYTES);
83
+ for (const it of items.slice(0, 10)) {
84
+ expect((await index.search(it.vector, { k: 1, exact: true }))[0]?.id).toBe(it.id);
85
+ }
86
+ });
87
+
88
+ it("getVectors returns live stored vectors from the tail and from segments", async () => {
89
+ const { index } = await setup();
90
+ const items = dataset(300, 16);
91
+ await index.upsert(items.slice(0, 200));
92
+ await index.compact();
93
+ await index.upsert(items.slice(200));
94
+ await index.delete([5]);
95
+ const got = await index.getVectors([1, 5, 250, 9999]);
96
+ expect([...got.keys()].sort((a, b) => a - b)).toEqual([1, 250]);
97
+ for (const id of [1, 250]) {
98
+ const want = items[id - 1]?.vector as Float32Array;
99
+ const norm = Math.hypot(...want);
100
+ const v = got.get(id) as Float32Array;
101
+ for (let j = 0; j < D; j++) expect(v[j]).toBeCloseTo((want[j] as number) / norm, 5);
102
+ }
103
+ });
104
+
105
+ it("dropVectorIndex removes every table; the prefix can be reused", async () => {
106
+ const { db, index } = await setup();
107
+ await index.upsert(dataset(50, 17));
108
+ await index.compact();
109
+ await dropVectorIndex(db, "vec_");
110
+ const left = await db.query<{ name: string }>(
111
+ "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'vec\\_%' ESCAPE '\\'",
112
+ );
113
+ expect(left).toEqual([]);
114
+ const fresh = new VectorIndex({ db, prefix: "vec_", dims: D * 2 });
115
+ await fresh.init();
116
+ expect(fresh.stats.tail).toBe(0);
117
+ });
118
+
119
+ it("search during merge returns correct hits; retired rows are gone after reopen", async () => {
120
+ const real = await open();
121
+ const gate = holdNextLiveness(real);
122
+ const index = new VectorIndex({ db: gate.db, prefix: "vec_", dims: D });
123
+ await index.init();
124
+ const items = dataset(1200, 15);
125
+ await index.upsert(items.slice(0, 600));
126
+ await index.compact();
127
+ await index.upsert(items.slice(600));
128
+ await index.compact();
129
+ const target = items[700] as (typeof items)[number];
130
+ gate.arm();
131
+ const searching = index.search(target.vector, { k: 1 });
132
+ await gate.held; // the search has read its lists and waits on its liveness read
133
+ expect(await index.merge()).toBe(1200);
134
+ gate.release();
135
+ expect((await searching)[0]?.id).toBe(target.id);
136
+
137
+ const reopened = new VectorIndex({ db: real, prefix: "vec_", dims: D });
138
+ await reopened.init();
139
+ const [row] = await real.query<{ n: number }>("SELECT count(DISTINCT gen) AS n FROM vec_seg");
140
+ expect(Number(row?.n)).toBe(1);
141
+ expect((await reopened.search(target.vector, { k: 1 }))[0]?.id).toBe(target.id);
142
+ });
143
+ });
144
+ }
145
+
146
+ /** Wraps `real` so that, once armed, the next liveness read (`SELECT id, gen`) waits for `release()`. */
147
+ function holdNextLiveness(real: Db) {
148
+ let armed = false;
149
+ let onHeld = () => {};
150
+ let release = () => {};
151
+ const held = new Promise<void>((r) => {
152
+ onHeld = r;
153
+ });
154
+ const released = new Promise<void>((r) => {
155
+ release = r;
156
+ });
157
+ const db = {
158
+ async query<T>(sql: string, params?: unknown[]): Promise<T[]> {
159
+ if (armed && sql.includes("SELECT id, gen")) {
160
+ armed = false;
161
+ onHeld();
162
+ await released;
163
+ }
164
+ return real.query<T>(sql, params);
165
+ },
166
+ exec: (sql: string) => real.exec(sql),
167
+ close: () => real.close(),
168
+ [SQLITE_TRANSACTION]: (real as { [SQLITE_TRANSACTION]?: unknown })[SQLITE_TRANSACTION],
169
+ } as Db;
170
+ return { db, held, release, arm: () => (armed = true) };
171
+ }