@statewalker/db-sqlite-index 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +289 -2
  3. package/dist/ngram.d.mts +93 -0
  4. package/dist/ngram.d.mts.map +1 -0
  5. package/dist/ngram.mjs +616 -0
  6. package/dist/ngram.mjs.map +1 -0
  7. package/dist/prefix-CTeW1EwH.mjs +7 -0
  8. package/dist/prefix-CTeW1EwH.mjs.map +1 -0
  9. package/dist/vector.d.mts +103 -0
  10. package/dist/vector.d.mts.map +1 -0
  11. package/dist/vector.mjs +1062 -0
  12. package/dist/vector.mjs.map +1 -0
  13. package/package.json +63 -3
  14. package/src/ngram/do-smoke.ts +46 -0
  15. package/src/ngram/fold.test.ts +50 -0
  16. package/src/ngram/fold.ts +21 -0
  17. package/src/ngram/fts5-backend.ts +55 -0
  18. package/src/ngram/index.ts +12 -0
  19. package/src/ngram/json-chunks.test.ts +23 -0
  20. package/src/ngram/json-chunks.ts +23 -0
  21. package/src/ngram/ngram-store.test.ts +291 -0
  22. package/src/ngram/ngram-store.ts +184 -0
  23. package/src/ngram/plan-query.test.ts +141 -0
  24. package/src/ngram/plan-query.ts +42 -0
  25. package/src/ngram/postings-backend.ts +53 -0
  26. package/src/ngram/regex-literals.ts +359 -0
  27. package/src/ngram/trigrams.test.ts +28 -0
  28. package/src/ngram/trigrams.ts +25 -0
  29. package/src/prefix.ts +2 -0
  30. package/src/vector/auto-compact.test.ts +28 -0
  31. package/src/vector/do-smoke.ts +39 -0
  32. package/src/vector/index.ts +9 -0
  33. package/src/vector/kmeans.ts +131 -0
  34. package/src/vector/math.ts +178 -0
  35. package/src/vector/segment.test.ts +61 -0
  36. package/src/vector/segment.ts +256 -0
  37. package/src/vector/sql-segment-store.ts +218 -0
  38. package/src/vector/test-fixtures.ts +64 -0
  39. package/src/vector/types.ts +30 -0
  40. package/src/vector/vector-index.test.ts +105 -0
  41. package/src/vector/vector-index.ts +586 -0
  42. package/src/vector/vector-lifecycle.test.ts +171 -0
@@ -0,0 +1,64 @@
1
+ import type { Db } from "@statewalker/db-api";
2
+ import { newDoSqliteDb } from "@statewalker/db-sqlite-do";
3
+ import { newFakeDoStorage } from "@statewalker/db-sqlite-do/fake";
4
+ import { newNodeSqliteDb } from "@statewalker/db-sqlite-node";
5
+ import { mulberry32 } from "./kmeans.js";
6
+ import { distance, prepare } from "./math.js";
7
+ import type { Metric, SearchOptions } from "./types.js";
8
+ import type { VectorIndex } from "./vector-index.js";
9
+
10
+ /** Every suite runs over each of these. */
11
+ export const BACKENDS: Record<string, () => Promise<Db>> = {
12
+ "node:sqlite": () => newNodeSqliteDb(),
13
+ "durable object (fake storage)": async () => newDoSqliteDb(newFakeDoStorage()),
14
+ };
15
+
16
+ export const D = 64;
17
+
18
+ export interface Item {
19
+ id: number;
20
+ vector: Float32Array;
21
+ }
22
+
23
+ /** Clustered data: realistic for IVF (uniform noise makes every ANN look bad). */
24
+ export function dataset(n: number, seed: number, dims = D, clusters = 40): Item[] {
25
+ const r = mulberry32(seed);
26
+ const centers = Array.from({ length: clusters }, () =>
27
+ Float32Array.from({ length: dims }, () => r() * 2 - 1),
28
+ );
29
+ return Array.from({ length: n }, (_, i) => {
30
+ const c = centers[Math.floor(r() * clusters)] as Float32Array;
31
+ return { id: i + 1, vector: Float32Array.from(c, (x) => x + (r() - 0.5) * 0.6) };
32
+ });
33
+ }
34
+
35
+ export function bruteForce(items: Item[], q: Float32Array, k: number, metric: Metric, dims = D) {
36
+ const qq = prepare(q, dims, metric);
37
+ return items
38
+ .map((it) => ({
39
+ id: it.id,
40
+ d: distance(metric, qq, 0, prepare(it.vector, dims, metric), 0, dims),
41
+ }))
42
+ .sort((a, b) => a.d - b.d)
43
+ .slice(0, k)
44
+ .map((x) => x.id);
45
+ }
46
+
47
+ export async function recall(
48
+ index: VectorIndex,
49
+ items: Item[],
50
+ metric: Metric,
51
+ opts: SearchOptions = {},
52
+ ): Promise<number> {
53
+ const r = mulberry32(7);
54
+ let hit = 0;
55
+ const Q = 30;
56
+ for (let i = 0; i < Q; i++) {
57
+ const base = items[Math.floor(r() * items.length)] as Item;
58
+ const q = base.vector.map((x) => x + (r() - 0.5) * 0.2);
59
+ const truth = new Set(bruteForce(items, q, 10, metric));
60
+ const got = await index.search(q, { k: 10, ...opts });
61
+ hit += got.filter((h) => truth.has(h.id)).length;
62
+ }
63
+ return hit / (Q * 10);
64
+ }
@@ -0,0 +1,30 @@
1
+ export type Metric = "cosine" | "dot" | "l2";
2
+
3
+ export interface SearchHit {
4
+ id: number;
5
+ /** Lower is closer: 1 - cos for cosine, -dot for dot, squared L2 for l2. */
6
+ distance: number;
7
+ }
8
+
9
+ export interface SearchOptions {
10
+ k?: number;
11
+ /** IVF lists probed per segment. Default 8. */
12
+ nprobe?: number;
13
+ /** Candidates kept per probed list by the 1-bit filter = k * oversample. Default 8. */
14
+ oversample?: number;
15
+ /** Re-rank the int8 winners with the stored float32 vectors. Default false. */
16
+ exact?: boolean;
17
+ /** Keep only ids for which this returns true (applied before re-ranking). */
18
+ filter?: (id: number) => boolean;
19
+ signal?: AbortSignal;
20
+ }
21
+
22
+ /** An aligned Float32Array copy of BLOB bytes (views over arbitrary offsets are unsafe). */
23
+ export function toFloat32(bytes: Uint8Array): Float32Array {
24
+ const copy = bytes.slice();
25
+ return new Float32Array(copy.buffer, 0, copy.byteLength >> 2);
26
+ }
27
+
28
+ export function float32Bytes(v: Float32Array): Uint8Array {
29
+ return new Uint8Array(v.buffer, v.byteOffset, v.byteLength).slice();
30
+ }
@@ -0,0 +1,105 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import { BACKENDS, D, dataset, recall } from "./test-fixtures.js";
3
+ import type { Metric } from "./types.js";
4
+ import { VectorIndex } from "./vector-index.js";
5
+
6
+ for (const [name, open] of Object.entries(BACKENDS)) {
7
+ describe(`VectorIndex over ${name}`, { timeout: 60_000 }, () => {
8
+ async function setup(metric: Metric = "cosine") {
9
+ const db = await open();
10
+ const index = new VectorIndex({
11
+ db,
12
+ prefix: "vec_",
13
+ dims: D,
14
+ metric,
15
+ autoCompactAt: Infinity,
16
+ });
17
+ await index.init();
18
+ return { db, index };
19
+ }
20
+
21
+ it("tail search is exact", async () => {
22
+ const { index } = await setup();
23
+ const items = dataset(300, 1);
24
+ await index.upsert(items);
25
+ expect(await recall(index, items, "cosine")).toBe(1);
26
+ });
27
+
28
+ for (const metric of ["cosine", "dot", "l2"] as const) {
29
+ it(`compacted IVF segment has high recall (${metric})`, async () => {
30
+ const { index } = await setup(metric);
31
+ const items = dataset(5000, 2);
32
+ await index.upsert(items);
33
+ expect(await index.compact()).toBe(5000);
34
+ expect(index.stats.tail).toBe(0);
35
+ const approx = await recall(index, items, metric, { nprobe: 12 });
36
+ const exact = await recall(index, items, metric, { nprobe: 12, exact: true });
37
+ console.log(
38
+ ` ${name} ${metric}: recall@10 int8=${approx.toFixed(3)} exact=${exact.toFixed(3)}`,
39
+ );
40
+ expect(approx).toBeGreaterThanOrEqual(0.85);
41
+ expect(exact).toBeGreaterThanOrEqual(approx - 0.01);
42
+ });
43
+ }
44
+
45
+ it("deletes and overwrites hide segment copies; merge drops them", async () => {
46
+ const { index, db } = await setup();
47
+ const items = dataset(2000, 3);
48
+ await index.upsert(items);
49
+ await index.compact();
50
+
51
+ const target = items[10];
52
+ expect((await index.search(target.vector, { k: 1 }))[0]?.id).toBe(target.id);
53
+
54
+ await index.delete([target.id]);
55
+ const after = await index.search(target.vector, { k: 5 });
56
+ expect(after.some((h) => h.id === target.id)).toBe(false);
57
+
58
+ // Overwrite id 20 with the vector of id 30: a search for 30's vector finds both.
59
+ await index.upsert([{ id: 20, vector: items[29].vector }]);
60
+ const both = (await index.search(items[29].vector, { k: 2 }))
61
+ .map((h) => h.id)
62
+ .sort((a, b) => a - b);
63
+ expect(both).toEqual([20, 30]);
64
+ // The old copy of id 20 (still in the segment) must not show up for its old vector.
65
+ const old = await index.search(items[19].vector, { k: 3 });
66
+ expect(old.every((h) => h.id !== 20 || h.distance > 1e-4)).toBe(true);
67
+
68
+ await index.compact(); // a second segment holds the new id 20
69
+ expect(index.stats.segments.length).toBe(2);
70
+ expect(await index.merge()).toBe(1999);
71
+ expect(index.stats.segments.length).toBe(1);
72
+ const both2 = (await index.search(items[29].vector, { k: 2 }))
73
+ .map((h) => h.id)
74
+ .sort((a, b) => a - b);
75
+ expect(both2).toEqual([20, 30]);
76
+ const [row] = await db.query<{ n: number }>("SELECT count(*) AS n FROM vec_ids");
77
+ expect(Number(row?.n)).toBe(1999);
78
+ });
79
+
80
+ it("state survives a reopen", async () => {
81
+ const { index, db } = await setup();
82
+ const items = dataset(1500, 4);
83
+ await index.upsert(items.slice(0, 1200));
84
+ await index.compact();
85
+ await index.upsert(items.slice(1200));
86
+ const reopened = new VectorIndex({ db, prefix: "vec_", dims: D });
87
+ await reopened.init();
88
+ expect(reopened.stats.tail).toBe(300);
89
+ expect(reopened.stats.segments.length).toBe(1);
90
+ expect(await recall(reopened, items, "cosine", { nprobe: 16 })).toBeGreaterThanOrEqual(0.85);
91
+ });
92
+
93
+ it("filters and rejects a mismatched configuration", async () => {
94
+ const { index, db } = await setup();
95
+ const items = dataset(1200, 5);
96
+ await index.upsert(items);
97
+ await index.compact();
98
+ const even = await index.search(items[0].vector, { k: 10, filter: (id) => id % 2 === 0 });
99
+ expect(even.length).toBe(10);
100
+ expect(even.every((h) => h.id % 2 === 0)).toBe(true);
101
+ const wrong = new VectorIndex({ db, prefix: "vec_", dims: D + 1 });
102
+ await expect(wrong.init()).rejects.toThrow(/created with/);
103
+ });
104
+ });
105
+ }