@statewalker/db-sqlite-index 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,105 +0,0 @@
1
- import { describe, expect, it } from "vitest";
2
- import { BACKENDS, D, dataset, recall } from "./test-fixtures.js";
3
- import type { Metric } from "./types.js";
4
- import { VectorIndex } from "./vector-index.js";
5
-
6
- for (const [name, open] of Object.entries(BACKENDS)) {
7
- describe(`VectorIndex over ${name}`, { timeout: 60_000 }, () => {
8
- async function setup(metric: Metric = "cosine") {
9
- const db = await open();
10
- const index = new VectorIndex({
11
- db,
12
- prefix: "vec_",
13
- dims: D,
14
- metric,
15
- autoCompactAt: Infinity,
16
- });
17
- await index.init();
18
- return { db, index };
19
- }
20
-
21
- it("tail search is exact", async () => {
22
- const { index } = await setup();
23
- const items = dataset(300, 1);
24
- await index.upsert(items);
25
- expect(await recall(index, items, "cosine")).toBe(1);
26
- });
27
-
28
- for (const metric of ["cosine", "dot", "l2"] as const) {
29
- it(`compacted IVF segment has high recall (${metric})`, async () => {
30
- const { index } = await setup(metric);
31
- const items = dataset(5000, 2);
32
- await index.upsert(items);
33
- expect(await index.compact()).toBe(5000);
34
- expect(index.stats.tail).toBe(0);
35
- const approx = await recall(index, items, metric, { nprobe: 12 });
36
- const exact = await recall(index, items, metric, { nprobe: 12, exact: true });
37
- console.log(
38
- ` ${name} ${metric}: recall@10 int8=${approx.toFixed(3)} exact=${exact.toFixed(3)}`,
39
- );
40
- expect(approx).toBeGreaterThanOrEqual(0.85);
41
- expect(exact).toBeGreaterThanOrEqual(approx - 0.01);
42
- });
43
- }
44
-
45
- it("deletes and overwrites hide segment copies; merge drops them", async () => {
46
- const { index, db } = await setup();
47
- const items = dataset(2000, 3);
48
- await index.upsert(items);
49
- await index.compact();
50
-
51
- const target = items[10];
52
- expect((await index.search(target.vector, { k: 1 }))[0]?.id).toBe(target.id);
53
-
54
- await index.delete([target.id]);
55
- const after = await index.search(target.vector, { k: 5 });
56
- expect(after.some((h) => h.id === target.id)).toBe(false);
57
-
58
- // Overwrite id 20 with the vector of id 30: a search for 30's vector finds both.
59
- await index.upsert([{ id: 20, vector: items[29].vector }]);
60
- const both = (await index.search(items[29].vector, { k: 2 }))
61
- .map((h) => h.id)
62
- .sort((a, b) => a - b);
63
- expect(both).toEqual([20, 30]);
64
- // The old copy of id 20 (still in the segment) must not show up for its old vector.
65
- const old = await index.search(items[19].vector, { k: 3 });
66
- expect(old.every((h) => h.id !== 20 || h.distance > 1e-4)).toBe(true);
67
-
68
- await index.compact(); // a second segment holds the new id 20
69
- expect(index.stats.segments.length).toBe(2);
70
- expect(await index.merge()).toBe(1999);
71
- expect(index.stats.segments.length).toBe(1);
72
- const both2 = (await index.search(items[29].vector, { k: 2 }))
73
- .map((h) => h.id)
74
- .sort((a, b) => a - b);
75
- expect(both2).toEqual([20, 30]);
76
- const [row] = await db.query<{ n: number }>("SELECT count(*) AS n FROM vec_ids");
77
- expect(Number(row?.n)).toBe(1999);
78
- });
79
-
80
- it("state survives a reopen", async () => {
81
- const { index, db } = await setup();
82
- const items = dataset(1500, 4);
83
- await index.upsert(items.slice(0, 1200));
84
- await index.compact();
85
- await index.upsert(items.slice(1200));
86
- const reopened = new VectorIndex({ db, prefix: "vec_", dims: D });
87
- await reopened.init();
88
- expect(reopened.stats.tail).toBe(300);
89
- expect(reopened.stats.segments.length).toBe(1);
90
- expect(await recall(reopened, items, "cosine", { nprobe: 16 })).toBeGreaterThanOrEqual(0.85);
91
- });
92
-
93
- it("filters and rejects a mismatched configuration", async () => {
94
- const { index, db } = await setup();
95
- const items = dataset(1200, 5);
96
- await index.upsert(items);
97
- await index.compact();
98
- const even = await index.search(items[0].vector, { k: 10, filter: (id) => id % 2 === 0 });
99
- expect(even.length).toBe(10);
100
- expect(even.every((h) => h.id % 2 === 0)).toBe(true);
101
- const wrong = new VectorIndex({ db, prefix: "vec_", dims: D + 1 });
102
- await expect(wrong.init()).rejects.toThrow(/created with/);
103
- });
104
- });
105
- }
@@ -1,171 +0,0 @@
1
- import type { Db } from "@statewalker/db-api";
2
- import { SQLITE_TRANSACTION } from "@statewalker/db-sqlite";
3
- import { describe, expect, it } from "vitest";
4
- import { PART_BYTES } from "./sql-segment-store.js";
5
- import { BACKENDS, bruteForce, D, dataset } from "./test-fixtures.js";
6
- import { dropVectorIndex, VectorIndex } from "./vector-index.js";
7
-
8
- for (const [name, open] of Object.entries(BACKENDS)) {
9
- describe(`VectorIndex lifecycle over ${name}`, { timeout: 60_000 }, () => {
10
- async function setup(dims = D) {
11
- const db = await open();
12
- const index = new VectorIndex({ db, prefix: "vec_", dims });
13
- await index.init();
14
- return { db, index };
15
- }
16
-
17
- it("probing every list with exact re-rank equals brute force", async () => {
18
- const { index } = await setup();
19
- const items = dataset(3000, 11);
20
- await index.upsert(items);
21
- await index.compact();
22
- expect(index.stats.segments[0]?.nlist).toBeGreaterThan(1);
23
- for (const it of items.slice(0, 20)) {
24
- const q = it.vector.map((x) => x + 0.01);
25
- const got = await index.search(q, { k: 10, nprobe: 1e6, oversample: 1e6, exact: true });
26
- expect(got.map((h) => h.id)).toEqual(bruteForce(items, q, 10, "cosine"));
27
- }
28
- });
29
-
30
- it("empty index and k > n return what exists", async () => {
31
- const { index } = await setup();
32
- const q = dataset(1, 12)[0]?.vector as Float32Array;
33
- expect(await index.search(q, { k: 10 })).toEqual([]);
34
- await index.upsert(dataset(5, 12));
35
- expect((await index.search(q, { k: 10 })).length).toBe(5);
36
- await index.compact();
37
- expect((await index.search(q, { k: 10 })).length).toBe(5);
38
- });
39
-
40
- it("rejects invalid input and writes nothing", async () => {
41
- const { index } = await setup();
42
- await expect(index.upsert([{ id: 1, vector: new Float32Array(D + 1) }])).rejects.toThrow(
43
- /expected 64 dims/,
44
- );
45
- const nan = new Float32Array(D);
46
- nan[3] = Number.NaN;
47
- await expect(index.upsert([{ id: 1, vector: nan }])).rejects.toThrow(/non-finite/);
48
- await expect(index.upsert([{ id: 2 ** 53, vector: new Float32Array(D) }])).rejects.toThrow(
49
- /safe integer/,
50
- );
51
- expect(index.stats.tail).toBe(0);
52
- });
53
-
54
- it("deletes many ids in one call", async () => {
55
- const { index } = await setup();
56
- const items = dataset(400, 13);
57
- await index.upsert(items);
58
- await index.compact();
59
- const gone = items.slice(0, 150).map((it) => it.id);
60
- await index.delete(gone);
61
- const dead = new Set(gone);
62
- for (const it of items.slice(0, 10)) {
63
- const hits = await index.search(it.vector, { k: 20 });
64
- expect(hits.some((h) => dead.has(h.id))).toBe(false);
65
- }
66
- });
67
-
68
- it("splits list blocks into parts no larger than PART_BYTES", async () => {
69
- const dims = 768;
70
- const { db, index } = await setup(dims);
71
- const items = dataset(1500, 14, dims); // flat: one list of ~1.3 MB, floats ~4.6 MB
72
- await index.upsert(items);
73
- await index.compact();
74
- const [seg] = await db.query<{ n: number; max: number }>(
75
- "SELECT count(*) AS n, max(length(bytes)) AS max FROM vec_seg",
76
- );
77
- const [f32] = await db.query<{ n: number; max: number }>(
78
- "SELECT count(*) AS n, max(length(bytes)) AS max FROM vec_f32",
79
- );
80
- expect(Number(seg?.n)).toBeGreaterThanOrEqual(2);
81
- expect(Number(f32?.n)).toBeGreaterThanOrEqual(5);
82
- expect(Math.max(Number(seg?.max), Number(f32?.max))).toBeLessThanOrEqual(PART_BYTES);
83
- for (const it of items.slice(0, 10)) {
84
- expect((await index.search(it.vector, { k: 1, exact: true }))[0]?.id).toBe(it.id);
85
- }
86
- });
87
-
88
- it("getVectors returns live stored vectors from the tail and from segments", async () => {
89
- const { index } = await setup();
90
- const items = dataset(300, 16);
91
- await index.upsert(items.slice(0, 200));
92
- await index.compact();
93
- await index.upsert(items.slice(200));
94
- await index.delete([5]);
95
- const got = await index.getVectors([1, 5, 250, 9999]);
96
- expect([...got.keys()].sort((a, b) => a - b)).toEqual([1, 250]);
97
- for (const id of [1, 250]) {
98
- const want = items[id - 1]?.vector as Float32Array;
99
- const norm = Math.hypot(...want);
100
- const v = got.get(id) as Float32Array;
101
- for (let j = 0; j < D; j++) expect(v[j]).toBeCloseTo((want[j] as number) / norm, 5);
102
- }
103
- });
104
-
105
- it("dropVectorIndex removes every table; the prefix can be reused", async () => {
106
- const { db, index } = await setup();
107
- await index.upsert(dataset(50, 17));
108
- await index.compact();
109
- await dropVectorIndex(db, "vec_");
110
- const left = await db.query<{ name: string }>(
111
- "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'vec\\_%' ESCAPE '\\'",
112
- );
113
- expect(left).toEqual([]);
114
- const fresh = new VectorIndex({ db, prefix: "vec_", dims: D * 2 });
115
- await fresh.init();
116
- expect(fresh.stats.tail).toBe(0);
117
- });
118
-
119
- it("search during merge returns correct hits; retired rows are gone after reopen", async () => {
120
- const real = await open();
121
- const gate = holdNextLiveness(real);
122
- const index = new VectorIndex({ db: gate.db, prefix: "vec_", dims: D });
123
- await index.init();
124
- const items = dataset(1200, 15);
125
- await index.upsert(items.slice(0, 600));
126
- await index.compact();
127
- await index.upsert(items.slice(600));
128
- await index.compact();
129
- const target = items[700] as (typeof items)[number];
130
- gate.arm();
131
- const searching = index.search(target.vector, { k: 1 });
132
- await gate.held; // the search has read its lists and waits on its liveness read
133
- expect(await index.merge()).toBe(1200);
134
- gate.release();
135
- expect((await searching)[0]?.id).toBe(target.id);
136
-
137
- const reopened = new VectorIndex({ db: real, prefix: "vec_", dims: D });
138
- await reopened.init();
139
- const [row] = await real.query<{ n: number }>("SELECT count(DISTINCT gen) AS n FROM vec_seg");
140
- expect(Number(row?.n)).toBe(1);
141
- expect((await reopened.search(target.vector, { k: 1 }))[0]?.id).toBe(target.id);
142
- });
143
- });
144
- }
145
-
146
- /** Wraps `real` so that, once armed, the next liveness read (`SELECT id, gen`) waits for `release()`. */
147
- function holdNextLiveness(real: Db) {
148
- let armed = false;
149
- let onHeld = () => {};
150
- let release = () => {};
151
- const held = new Promise<void>((r) => {
152
- onHeld = r;
153
- });
154
- const released = new Promise<void>((r) => {
155
- release = r;
156
- });
157
- const db = {
158
- async query<T>(sql: string, params?: unknown[]): Promise<T[]> {
159
- if (armed && sql.includes("SELECT id, gen")) {
160
- armed = false;
161
- onHeld();
162
- await released;
163
- }
164
- return real.query<T>(sql, params);
165
- },
166
- exec: (sql: string) => real.exec(sql),
167
- close: () => real.close(),
168
- [SQLITE_TRANSACTION]: (real as { [SQLITE_TRANSACTION]?: unknown })[SQLITE_TRANSACTION],
169
- } as Db;
170
- return { db, held, release, arm: () => (armed = true) };
171
- }