@statewalker/db-sqlite-index 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +289 -2
  3. package/dist/ngram.d.mts +93 -0
  4. package/dist/ngram.d.mts.map +1 -0
  5. package/dist/ngram.mjs +616 -0
  6. package/dist/ngram.mjs.map +1 -0
  7. package/dist/prefix-CTeW1EwH.mjs +7 -0
  8. package/dist/prefix-CTeW1EwH.mjs.map +1 -0
  9. package/dist/vector.d.mts +103 -0
  10. package/dist/vector.d.mts.map +1 -0
  11. package/dist/vector.mjs +1062 -0
  12. package/dist/vector.mjs.map +1 -0
  13. package/package.json +63 -3
  14. package/src/ngram/do-smoke.ts +46 -0
  15. package/src/ngram/fold.test.ts +50 -0
  16. package/src/ngram/fold.ts +21 -0
  17. package/src/ngram/fts5-backend.ts +55 -0
  18. package/src/ngram/index.ts +12 -0
  19. package/src/ngram/json-chunks.test.ts +23 -0
  20. package/src/ngram/json-chunks.ts +23 -0
  21. package/src/ngram/ngram-store.test.ts +291 -0
  22. package/src/ngram/ngram-store.ts +184 -0
  23. package/src/ngram/plan-query.test.ts +141 -0
  24. package/src/ngram/plan-query.ts +42 -0
  25. package/src/ngram/postings-backend.ts +53 -0
  26. package/src/ngram/regex-literals.ts +359 -0
  27. package/src/ngram/trigrams.test.ts +28 -0
  28. package/src/ngram/trigrams.ts +25 -0
  29. package/src/prefix.ts +2 -0
  30. package/src/vector/auto-compact.test.ts +28 -0
  31. package/src/vector/do-smoke.ts +39 -0
  32. package/src/vector/index.ts +9 -0
  33. package/src/vector/kmeans.ts +131 -0
  34. package/src/vector/math.ts +178 -0
  35. package/src/vector/segment.test.ts +61 -0
  36. package/src/vector/segment.ts +256 -0
  37. package/src/vector/sql-segment-store.ts +218 -0
  38. package/src/vector/test-fixtures.ts +64 -0
  39. package/src/vector/types.ts +30 -0
  40. package/src/vector/vector-index.test.ts +105 -0
  41. package/src/vector/vector-index.ts +586 -0
  42. package/src/vector/vector-lifecycle.test.ts +171 -0
@@ -0,0 +1,53 @@
1
+ import type { GramBackend, NgramItem } from "./fts5-backend.js";
2
+ import { chunkJson } from "./json-chunks.js";
3
+ import { encodeGram, gramsOf, trigrams } from "./trigrams.js";
4
+
5
+ const PAIR = "json_extract(value, '$[0]'), json_extract(value, '$[1]')";
6
+
7
+ /**
8
+ * One row per (gram, id) in a WITHOUT ROWID table: the primary key is the
9
+ * posting list, clustered by gram. A text is a candidate for a branch when it
10
+ * holds every distinct gram of the branch's literals.
11
+ */
12
+ export function newPostingsBackend(prefix: string): GramBackend {
13
+ const table = `${prefix}grams`;
14
+ const pairs = (items: NgramItem[]) =>
15
+ chunkJson(
16
+ items.flatMap((item) => trigrams(item.text).map((gram) => [encodeGram(gram), item.id])),
17
+ );
18
+ return {
19
+ create: (tx) =>
20
+ tx.exec(
21
+ `CREATE TABLE IF NOT EXISTS ${table}(gram INTEGER NOT NULL, id INTEGER NOT NULL, PRIMARY KEY (gram, id)) WITHOUT ROWID`,
22
+ ),
23
+ async add(tx, items) {
24
+ for (const chunk of pairs(items)) {
25
+ await tx.query(
26
+ `INSERT OR IGNORE INTO ${table}(gram, id) SELECT ${PAIR} FROM json_each(?)`,
27
+ [chunk],
28
+ );
29
+ }
30
+ },
31
+ async remove(tx, items) {
32
+ for (const chunk of pairs(items)) {
33
+ await tx.query(
34
+ `DELETE FROM ${table} WHERE (gram, id) IN (SELECT ${PAIR} FROM json_each(?))`,
35
+ [chunk],
36
+ );
37
+ }
38
+ },
39
+ async candidates(db, branches, max) {
40
+ const ids = new Set<number>();
41
+ for (const branch of branches) {
42
+ // Literals are already folded: split them without folding again.
43
+ const grams = [...new Set(branch.flatMap((literal) => gramsOf(literal).map(encodeGram)))];
44
+ const rows = await db.query<{ id: number }>(
45
+ `SELECT id FROM ${table} WHERE gram IN (SELECT value FROM json_each(?)) GROUP BY id HAVING count(*) = ? ORDER BY id LIMIT ?`,
46
+ [JSON.stringify(grams), grams.length, max],
47
+ );
48
+ for (const row of rows) ids.add(Number(row.id));
49
+ }
50
+ return [...ids].sort((a, b) => a - b).slice(0, max);
51
+ },
52
+ };
53
+ }
@@ -0,0 +1,359 @@
1
+ /*
2
+ * Mandatory-literal extraction from a regex, after Russ Cox's trigram method.
3
+ * Ported from hypgrep 0.5.3 src/regex.js (Hyperparam), MIT License:
4
+ *
5
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
6
+ * software and associated documentation files (the "Software"), to deal in the Software
7
+ * without restriction, including without limitation the rights to use, copy, modify,
8
+ * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
9
+ * permit persons to whom the Software is furnished to do so, subject to the following
10
+ * conditions: The above copyright notice and this permission notice shall be included in
11
+ * all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS",
12
+ * WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED.
13
+ */
14
+
15
+ /** Cross-product cap: past it, the construct counts as a wildcard. */
16
+ const MAX_DNF = 32;
17
+
18
+ type LiveBranch = { literals: string[]; run: string };
19
+ type Quantifier = { copies: number; openEnded: boolean; end: number };
20
+ type GroupPrefix = { kind: "consuming"; innerStart: number } | { kind: "assertion" | "unknown" };
21
+
22
+ /**
23
+ * DNF of literals that every match must contain: a string can match only if all
24
+ * literals of at least one branch occur in it. An empty branch matches anything.
25
+ * Conservative: a construct it cannot reason about becomes a wildcard, never a literal.
26
+ *
27
+ * /abc(def|ghi)/ → [["abcdef"], ["abcghi"]] /foo|.+/ → [["foo"], []]
28
+ */
29
+ export function regexLiterals(regex: RegExp): string[][] {
30
+ // v-mode class syntax desyncs skipClass and would invent literals.
31
+ if (regex.flags.includes("v")) return [[]];
32
+ return expandRegion(regex.source, 0, regex.source.length).dnf;
33
+ }
34
+
35
+ /** `exact`: every arm is one contiguous literal that is the arm's whole match, with nothing dropped. */
36
+ type Region = { dnf: string[][]; exact: boolean };
37
+
38
+ function expandRegion(src: string, start: number, end: number): Region {
39
+ const dnf: string[][] = [];
40
+ let exact = true;
41
+ for (const [bStart, bEnd] of splitTopLevelAlternation(src, start, end)) {
42
+ const arm = expandBranch(src, bStart, bEnd);
43
+ for (const branch of arm.dnf) dnf.push(branch);
44
+ exact = exact && arm.exact;
45
+ }
46
+ return { dnf, exact };
47
+ }
48
+
49
+ function splitTopLevelAlternation(
50
+ src: string,
51
+ start: number,
52
+ end: number,
53
+ ): Array<[number, number]> {
54
+ const spans: Array<[number, number]> = [];
55
+ let bStart = start;
56
+ let depth = 0;
57
+ let i = start;
58
+ while (i < end) {
59
+ const c = src[i];
60
+ if (c === "\\") {
61
+ i += 2;
62
+ continue;
63
+ }
64
+ if (c === "[") {
65
+ i = skipClass(src, i);
66
+ continue;
67
+ }
68
+ if (c === "(") depth += 1;
69
+ else if (c === ")") depth -= 1;
70
+ else if (c === "|" && depth === 0) {
71
+ spans.push([bStart, i]);
72
+ bStart = i + 1;
73
+ }
74
+ i += 1;
75
+ }
76
+ spans.push([bStart, end]);
77
+ return spans;
78
+ }
79
+
80
+ function expandBranch(src: string, start: number, end: number): Region {
81
+ let cross: LiveBranch[] = [{ literals: [], run: "" }];
82
+ // Set when part of the match is not captured in a literal (wildcard, optional or repeated
83
+ // atom, assertion): gluing this branch to neighbouring text would be unsound.
84
+ let lossy = false;
85
+ const flushRuns = () => {
86
+ for (const b of cross) {
87
+ if (b.run.length > 0) {
88
+ b.literals.push(b.run);
89
+ b.run = "";
90
+ }
91
+ }
92
+ };
93
+ // Append `text` repeated by the quantifier's mandatory copies to every live run.
94
+ const appendRun = (text: string, q: Quantifier) => {
95
+ if (q.copies === 0) {
96
+ lossy = true;
97
+ flushRuns();
98
+ return;
99
+ }
100
+ for (const b of cross) b.run += text.repeat(q.copies);
101
+ if (q.openEnded) {
102
+ lossy = true;
103
+ flushRuns();
104
+ }
105
+ };
106
+
107
+ let i = start;
108
+ while (i < end) {
109
+ const c = src[i] ?? "";
110
+
111
+ if (c === "^" || c === "$") {
112
+ i += 1;
113
+ continue;
114
+ }
115
+
116
+ if (c === "\\") {
117
+ const next = src[i + 1];
118
+ if (next === undefined) {
119
+ i += 1;
120
+ continue;
121
+ }
122
+ const escaped = parseEscapedLiteral(src, i);
123
+ if (escaped) {
124
+ const q = peekQuantifier(src, escaped.end);
125
+ i = q.end;
126
+ appendRun(escaped.char, q);
127
+ continue;
128
+ }
129
+ // \d \w \s \b, backrefs, \p{…}: a wildcard; consume its payload.
130
+ if (/[a-zA-Z0-9]/.test(next)) {
131
+ lossy = true;
132
+ flushRuns();
133
+ i = peekQuantifier(src, skipSpecialEscape(src, i)).end;
134
+ continue;
135
+ }
136
+ // \. \/ \( …: the literal character.
137
+ const q = peekQuantifier(src, i + 2);
138
+ i = q.end;
139
+ appendRun(next, q);
140
+ continue;
141
+ }
142
+
143
+ if (c === "[") {
144
+ const classEnd = skipClass(src, i);
145
+ const chars = parseLiteralClass(src, i, classEnd);
146
+ const q = peekQuantifier(src, classEnd);
147
+ i = q.end;
148
+ if (
149
+ q.copies === 0 ||
150
+ chars === null ||
151
+ q.copies > 1 ||
152
+ cross.length * chars.length > MAX_DNF
153
+ ) {
154
+ lossy = true;
155
+ flushRuns();
156
+ continue;
157
+ }
158
+ cross = cross.flatMap((b) =>
159
+ chars.map((ch) => ({ literals: b.literals.slice(), run: b.run + ch })),
160
+ );
161
+ if (q.openEnded) {
162
+ lossy = true;
163
+ flushRuns();
164
+ }
165
+ continue;
166
+ }
167
+
168
+ if (c === "(") {
169
+ const groupEnd = skipGroup(src, i);
170
+ const group = parseGroupPrefix(src, i);
171
+ const q = peekQuantifier(src, groupEnd);
172
+ i = q.end;
173
+ if (group.kind !== "consuming" || q.copies === 0) {
174
+ lossy = true;
175
+ flushRuns();
176
+ continue;
177
+ }
178
+ const { dnf: inner, exact } = expandRegion(src, group.innerStart, groupEnd - 1);
179
+ // Fold into the run only when every arm is one contiguous literal used once and is the
180
+ // arm's whole match: a repeated group (`(a|b){3}`) may pick different arms per copy, and
181
+ // a wildcard at an arm's edge (`x(.abc)`) must not be glued away.
182
+ const foldable =
183
+ exact && q.copies === 1 && inner.length > 0 && inner.every((b) => b.length === 1);
184
+ lossy = lossy || q.openEnded || !foldable;
185
+ if (foldable && cross.length * inner.length <= MAX_DNF) {
186
+ cross = cross.flatMap((b) =>
187
+ inner.map((arm) => ({ literals: b.literals.slice(), run: b.run + (arm[0] ?? "") })),
188
+ );
189
+ if (q.openEnded) flushRuns();
190
+ continue;
191
+ }
192
+ lossy = true;
193
+ flushRuns();
194
+ if (cross.length * inner.length > MAX_DNF) continue;
195
+ cross = cross.flatMap((b) =>
196
+ inner.map((arm) => ({ literals: b.literals.concat(arm), run: "" })),
197
+ );
198
+ continue;
199
+ }
200
+
201
+ if (c === ".") {
202
+ lossy = true;
203
+ flushRuns();
204
+ i = peekQuantifier(src, i + 1).end;
205
+ continue;
206
+ }
207
+
208
+ const q = peekQuantifier(src, i + 1);
209
+ i = q.end;
210
+ appendRun(c, q);
211
+ }
212
+ flushRuns();
213
+ return {
214
+ dnf: cross.map((b) => b.literals),
215
+ exact: !lossy && cross.every((b) => b.literals.length === 1),
216
+ };
217
+ }
218
+
219
+ /** Where a group's consuming body starts; lookarounds and unknown `(?…)` are not consuming. */
220
+ function parseGroupPrefix(src: string, start: number): GroupPrefix {
221
+ if (src[start + 1] !== "?") return { kind: "consuming", innerStart: start + 1 };
222
+ const marker = src[start + 2];
223
+ if (marker === ":") return { kind: "consuming", innerStart: start + 3 };
224
+ if (marker === "=" || marker === "!") return { kind: "assertion" };
225
+ if (marker === "<") {
226
+ const next = src[start + 3];
227
+ if (next === "=" || next === "!") return { kind: "assertion" };
228
+ const nameEnd = src.indexOf(">", start + 3);
229
+ if (nameEnd >= 0) return { kind: "consuming", innerStart: nameEnd + 1 };
230
+ }
231
+ return { kind: "unknown" };
232
+ }
233
+
234
+ /**
235
+ * `\xHH`, `\uHHHH`: escapes that stand for one literal character. `\u{…}` is not one:
236
+ * the planner compiles without the `u` flag, where it means "u" repeated or the text "u{…}".
237
+ */
238
+ function parseEscapedLiteral(
239
+ src: string,
240
+ start: number,
241
+ ): { char: string; end: number } | undefined {
242
+ const kind = src[start + 1];
243
+ if (kind === "x") {
244
+ const hex = src.slice(start + 2, start + 4);
245
+ if (!/^[0-9a-fA-F]{2}$/.test(hex)) return undefined;
246
+ return { char: String.fromCharCode(Number.parseInt(hex, 16)), end: start + 4 };
247
+ }
248
+ if (kind === "u") {
249
+ const hex = src.slice(start + 2, start + 6);
250
+ if (!/^[0-9a-fA-F]{4}$/.test(hex)) return undefined;
251
+ return { char: String.fromCharCode(Number.parseInt(hex, 16)), end: start + 6 };
252
+ }
253
+ return undefined;
254
+ }
255
+
256
+ /**
257
+ * Skip an escape that is not a literal punctuation character, so its payload
258
+ * (`\x41`, `\p{L}`, `\k<name>`, backrefs) never leaks into a literal. Malformed
259
+ * `\x`, `\u`, `\c` payloads are Annex-B literals and are rescanned as atoms.
260
+ */
261
+ function skipSpecialEscape(src: string, start: number): number {
262
+ const next = src[start + 1];
263
+ if (next === "x") {
264
+ return /^[0-9a-fA-F]{2}$/.test(src.slice(start + 2, start + 4)) ? start + 4 : start + 2;
265
+ }
266
+ if (next === "u") {
267
+ if (src[start + 2] === "{") {
268
+ const close = src.indexOf("}", start + 3);
269
+ return close >= 0 ? close + 1 : src.length;
270
+ }
271
+ return /^[0-9a-fA-F]{4}$/.test(src.slice(start + 2, start + 6)) ? start + 6 : start + 2;
272
+ }
273
+ if (next === "c") return /^[a-zA-Z]$/.test(src[start + 2] ?? "") ? start + 3 : start + 2;
274
+ if ((next === "p" || next === "P") && src[start + 2] === "{") {
275
+ const close = src.indexOf("}", start + 3);
276
+ return close >= 0 ? close + 1 : src.length;
277
+ }
278
+ if (next === "k" && src[start + 2] === "<") {
279
+ const close = src.indexOf(">", start + 3);
280
+ return close >= 0 ? close + 1 : src.length;
281
+ }
282
+ if (/[0-9]/.test(next ?? "")) {
283
+ let i = start + 2;
284
+ while (/[0-9]/.test(src[i] ?? "")) i += 1;
285
+ return i;
286
+ }
287
+ return Math.min(start + 2, src.length);
288
+ }
289
+
290
+ /** A `[...]` class as literal alternatives, or null if negated, ranged, special or empty. */
291
+ function parseLiteralClass(src: string, start: number, end: number): string[] | null {
292
+ let i = start + 1;
293
+ const lastInside = end - 1;
294
+ if (src[i] === "^") return null;
295
+ const chars = new Set<string>();
296
+ while (i < lastInside) {
297
+ const c = src[i] ?? "";
298
+ if (c === "\\") {
299
+ const next = src[i + 1];
300
+ if (next === undefined || /[a-zA-Z]/.test(next)) return null;
301
+ chars.add(next);
302
+ i += 2;
303
+ continue;
304
+ }
305
+ if (i + 2 < lastInside && src[i + 1] === "-") return null;
306
+ chars.add(c);
307
+ i += 1;
308
+ }
309
+ // A surrogate half would split an astral character the index sees as one code point.
310
+ if ([...chars].some((ch) => /[\ud800-\udfff]/.test(ch))) return null;
311
+ return chars.size === 0 ? null : [...chars];
312
+ }
313
+
314
+ /** Mandatory copies of the preceding atom, whether more may follow, and where the quantifier ends. */
315
+ function peekQuantifier(src: string, i: number): Quantifier {
316
+ const c = src[i];
317
+ if (c === "?" || c === "*" || c === "+") {
318
+ const end = src[i + 1] === "?" ? i + 2 : i + 1;
319
+ return { copies: c === "+" ? 1 : 0, openEnded: true, end };
320
+ }
321
+ if (c === "{") {
322
+ const close = src.indexOf("}", i);
323
+ if (close < 0) return { copies: 1, openEnded: false, end: i };
324
+ const m = /^(\d+)(?:,(\d*))?$/.exec(src.slice(i + 1, close));
325
+ if (!m) return { copies: 1, openEnded: false, end: i };
326
+ const min = Number(m[1]);
327
+ const max = m[2] === undefined ? min : m[2] === "" ? Number.POSITIVE_INFINITY : Number(m[2]);
328
+ const end = src[close + 1] === "?" ? close + 2 : close + 1;
329
+ return { copies: min, openEnded: max > min, end };
330
+ }
331
+ return { copies: 1, openEnded: false, end: i };
332
+ }
333
+
334
+ function skipClass(src: string, i: number): number {
335
+ let j = i + 1;
336
+ if (src[j] === "^") j += 1;
337
+ while (j < src.length && src[j] !== "]") j += src[j] === "\\" && j + 1 < src.length ? 2 : 1;
338
+ return j < src.length ? j + 1 : j;
339
+ }
340
+
341
+ function skipGroup(src: string, i: number): number {
342
+ let depth = 1;
343
+ let j = i + 1;
344
+ while (j < src.length && depth > 0) {
345
+ const c = src[j];
346
+ if (c === "\\" && j + 1 < src.length) {
347
+ j += 2;
348
+ continue;
349
+ }
350
+ if (c === "[") {
351
+ j = skipClass(src, j);
352
+ continue;
353
+ }
354
+ if (c === "(") depth += 1;
355
+ else if (c === ")") depth -= 1;
356
+ j += 1;
357
+ }
358
+ return j;
359
+ }
@@ -0,0 +1,28 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import { encodeGram, gramsOf, trigrams } from "./trigrams.js";
3
+
4
+ describe("trigrams", () => {
5
+ it("returns distinct folded 3-code-point grams in order of first appearance", () => {
6
+ expect(trigrams("Hello")).toEqual(["hel", "ell", "llo"]);
7
+ expect(trigrams("aaaa")).toEqual(["aaa"]);
8
+ });
9
+
10
+ it("returns nothing for texts shorter than 3 code points", () => {
11
+ expect(trigrams("ab")).toEqual([]);
12
+ expect(trigrams("😀😀")).toEqual([]);
13
+ });
14
+
15
+ it("never splits a surrogate pair", () => {
16
+ expect(trigrams("😀abc")).toEqual(["😀ab", "abc"]);
17
+ });
18
+ });
19
+
20
+ describe("encodeGram", () => {
21
+ it("encodes BMP grams exactly and astral grams above 2^48, all as safe integers", () => {
22
+ const bmp = encodeGram("abc");
23
+ expect(bmp).toBe(0x61 * 2 ** 32 + 0x62 * 2 ** 16 + 0x63);
24
+ const astral = encodeGram(gramsOf("😀ab")[0] ?? "");
25
+ expect(astral).toBeGreaterThanOrEqual(2 ** 48);
26
+ expect(Number.isSafeInteger(astral)).toBe(true);
27
+ });
28
+ });
@@ -0,0 +1,25 @@
1
+ import { fold } from "./fold.js";
2
+
3
+ /** Distinct 3-code-point grams of an already folded string, in first-seen order. */
4
+ export function gramsOf(folded: string): string[] {
5
+ const cps = Array.from(folded);
6
+ const out = new Set<string>();
7
+ for (let i = 0; i + 3 <= cps.length; i++) out.add(cps.slice(i, i + 3).join(""));
8
+ return [...out];
9
+ }
10
+
11
+ /** Distinct trigrams of `text` as the index sees it: case-folded, by code point. */
12
+ export function trigrams(text: string): string[] {
13
+ return gramsOf(fold(text));
14
+ }
15
+
16
+ /**
17
+ * One SQLite INTEGER per gram. Three BMP code points pack exactly into 48 bits.
18
+ * A gram with an astral code point keeps the low 16 bits of each code point
19
+ * above 2^48: a collision only adds candidates, which verification removes.
20
+ */
21
+ export function encodeGram(gram: string): number {
22
+ const [a = 0, b = 0, c = 0] = Array.from(gram, (ch) => ch.codePointAt(0) ?? 0);
23
+ if (a < 0x10000 && b < 0x10000 && c < 0x10000) return a * 2 ** 32 + b * 2 ** 16 + c;
24
+ return 2 ** 48 + (a % 0x10000) * 2 ** 32 + (b % 0x10000) * 2 ** 16 + (c % 0x10000);
25
+ }
package/src/prefix.ts ADDED
@@ -0,0 +1,2 @@
1
+ /** Table-name prefixes the indexes accept: an SQL identifier, so it is safe to interpolate. */
2
+ export const PREFIX_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
@@ -0,0 +1,28 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import { BACKENDS, D, dataset, recall } from "./test-fixtures.js";
3
+ import { VectorIndex } from "./vector-index.js";
4
+
5
+ for (const [name, open] of Object.entries(BACKENDS)) {
6
+ describe(`auto-compaction over ${name}`, { timeout: 60_000 }, () => {
7
+ it("compacts once the tail reaches 4096 vectors by default", async () => {
8
+ const db = await open();
9
+ const index = new VectorIndex({ db, prefix: "vec_", dims: D });
10
+ await index.init();
11
+ const items = dataset(4500, 21);
12
+ await index.upsert(items);
13
+ // Batches of 200: the tail reaches 4200 >= 4096 after 21 batches, then 300 more arrive.
14
+ expect(index.stats.segments.map((s) => s.count)).toEqual([4200]);
15
+ expect(index.stats.tail).toBe(300);
16
+ expect(await recall(index, items, "cosine", { nprobe: 16 })).toBeGreaterThanOrEqual(0.85);
17
+ });
18
+
19
+ it("never compacts on its own when autoCompactAt is Infinity", async () => {
20
+ const db = await open();
21
+ const index = new VectorIndex({ db, prefix: "vec_", dims: D, autoCompactAt: Infinity });
22
+ await index.init();
23
+ await index.upsert(dataset(4500, 22));
24
+ expect(index.stats.segments).toEqual([]);
25
+ expect(index.stats.tail).toBe(4500);
26
+ });
27
+ });
28
+ }
@@ -0,0 +1,39 @@
1
+ import type { Db } from "@statewalker/db-api";
2
+ import { VectorIndex } from "./vector-index.js";
3
+
4
+ const check = (ok: boolean, message: string) => {
5
+ if (!ok) throw new Error(`vector smoke: ${message}`);
6
+ };
7
+
8
+ /** Run inside a real Durable Object by db-sqlite-do's `pnpm test:do`. */
9
+ export const vectorDoScenarios = [
10
+ {
11
+ name: "vector: multi-part segment rows, 150 ids in one delete, read-back",
12
+ async run(db: Db) {
13
+ const dims = 256;
14
+ const n = 1100; // flat list; float block 4 × 1100 × 256 ≈ 1.1 MB → 2 parts
15
+ const index = new VectorIndex({
16
+ db,
17
+ prefix: "smoke_",
18
+ dims,
19
+ autoCompactAt: Number.POSITIVE_INFINITY,
20
+ });
21
+ await index.init();
22
+ // Well-spread vectors: a sin(i * 7 + j) phase ramp makes ids ~150 apart near-duplicates.
23
+ const vector = (i: number) =>
24
+ Float32Array.from({ length: dims }, (_, j) => Math.sin((i * 7919 + j * 104729) ** 2));
25
+ await index.upsert(
26
+ Array.from({ length: n }, (_, i) => ({ id: i + 1, vector: vector(i + 1) })),
27
+ );
28
+ check((await index.compact()) === n, "compact sealed every vector");
29
+ const [parts] = await db.query<{ n: number }>("SELECT count(*) AS n FROM smoke_f32");
30
+ check(Number(parts?.n) >= 2, `expected ≥ 2 float parts, got ${parts?.n}`);
31
+ const gone = Array.from({ length: 150 }, (_, i) => i + 1);
32
+ await index.delete(gone);
33
+ const hit = await index.search(vector(500), { k: 1, exact: true });
34
+ check(hit[0]?.id === 500, `nearest of 500 is ${hit[0]?.id}`);
35
+ const back = await index.getVectors([1, 500]);
36
+ check(!back.has(1) && back.has(500), "getVectors returns only live ids");
37
+ },
38
+ },
39
+ ];
@@ -0,0 +1,9 @@
1
+ export type { SegmentRef } from "./sql-segment-store.js";
2
+ export type { Metric, SearchHit, SearchOptions } from "./types.js";
3
+ export {
4
+ dropVectorIndex,
5
+ VectorIndex,
6
+ type VectorIndexOptions,
7
+ type VectorIndexStats,
8
+ type VectorItem,
9
+ } from "./vector-index.js";
@@ -0,0 +1,131 @@
1
+ import { distance, normalize } from "./math.js";
2
+ import type { Metric } from "./types.js";
3
+
4
+ export interface KMeansOptions {
5
+ iterations?: number;
6
+ /** Training points drawn from the data. Default min(n, 32 * k, 25_000). */
7
+ sampleSize?: number;
8
+ seed?: number;
9
+ }
10
+
11
+ /** Small deterministic PRNG so segment builds are reproducible. */
12
+ export function mulberry32(seed: number) {
13
+ let a = seed >>> 0;
14
+ return () => {
15
+ a = (a + 0x6d2b79f5) >>> 0;
16
+ let t = a;
17
+ t = Math.imul(t ^ (t >>> 15), t | 1);
18
+ t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
19
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
20
+ };
21
+ }
22
+
23
+ /** Index of the closest centroid. */
24
+ export function nearest(
25
+ metric: Metric,
26
+ data: Float32Array,
27
+ offset: number,
28
+ centroids: Float32Array,
29
+ k: number,
30
+ d: number,
31
+ ): number {
32
+ let best = 0;
33
+ let bestD = Number.POSITIVE_INFINITY;
34
+ // Centroids are compared with l2 for l2, and with dot otherwise (spherical k-means).
35
+ const m: Metric = metric === "l2" ? "l2" : "dot";
36
+ for (let c = 0; c < k; c++) {
37
+ const dist = distance(m, data, offset, centroids, c * d, d);
38
+ if (dist < bestD) {
39
+ bestD = dist;
40
+ best = c;
41
+ }
42
+ }
43
+ return best;
44
+ }
45
+
46
+ /**
47
+ * k-means++ seeding plus Lloyd iterations on a sample. `data` is n × d, flat.
48
+ * For cosine and dot, centroids are re-normalized (spherical k-means).
49
+ */
50
+ export function kmeans(
51
+ data: Float32Array,
52
+ n: number,
53
+ d: number,
54
+ k: number,
55
+ metric: Metric,
56
+ opts: KMeansOptions = {},
57
+ ): Float32Array {
58
+ const rand = mulberry32(opts.seed ?? 42);
59
+ k = Math.max(1, Math.min(k, n));
60
+ const sampleSize = Math.min(n, opts.sampleSize ?? Math.min(32 * k, 25_000));
61
+
62
+ // Reservoir-free sample: a partial Fisher–Yates over indices.
63
+ const idx = new Uint32Array(n);
64
+ for (let i = 0; i < n; i++) idx[i] = i;
65
+ for (let i = 0; i < sampleSize; i++) {
66
+ const j = i + Math.floor(rand() * (n - i));
67
+ [idx[i], idx[j]] = [idx[j], idx[i]];
68
+ }
69
+ const sample = new Float32Array(sampleSize * d);
70
+ for (let i = 0; i < sampleSize; i++) sample.set(data.subarray(idx[i] * d, idx[i] * d + d), i * d);
71
+
72
+ const spherical = metric !== "l2";
73
+ const centroids = new Float32Array(k * d);
74
+
75
+ // k-means++ with squared l2 on the sample.
76
+ centroids.set(sample.subarray(0, d), 0);
77
+ const minD = new Float64Array(sampleSize).fill(Number.POSITIVE_INFINITY);
78
+ for (let c = 1; c < k; c++) {
79
+ let total = 0;
80
+ for (let i = 0; i < sampleSize; i++) {
81
+ const dd = distance("l2", sample, i * d, centroids, (c - 1) * d, d);
82
+ if (dd < minD[i]) minD[i] = dd;
83
+ total += minD[i];
84
+ }
85
+ let r = rand() * total;
86
+ let pick = sampleSize - 1;
87
+ for (let i = 0; i < sampleSize; i++) {
88
+ r -= minD[i];
89
+ if (r <= 0) {
90
+ pick = i;
91
+ break;
92
+ }
93
+ }
94
+ centroids.set(sample.subarray(pick * d, pick * d + d), c * d);
95
+ }
96
+
97
+ const assign = new Uint32Array(sampleSize);
98
+ const counts = new Uint32Array(k);
99
+ const iterations = opts.iterations ?? 10;
100
+ for (let it = 0; it < iterations; it++) {
101
+ let moved = 0;
102
+ for (let i = 0; i < sampleSize; i++) {
103
+ const c = nearest(metric, sample, i * d, centroids, k, d);
104
+ if (c !== assign[i] || it === 0) moved++;
105
+ assign[i] = c;
106
+ }
107
+ const sums = new Float64Array(k * d);
108
+ counts.fill(0);
109
+ for (let i = 0; i < sampleSize; i++) {
110
+ const c = assign[i];
111
+ counts[c]++;
112
+ for (let j = 0; j < d; j++) sums[c * d + j] += sample[i * d + j];
113
+ }
114
+ for (let c = 0; c < k; c++) {
115
+ if (counts[c] === 0) {
116
+ // Re-seed an empty cluster with a random sample point.
117
+ const p = Math.floor(rand() * sampleSize);
118
+ centroids.set(sample.subarray(p * d, p * d + d), c * d);
119
+ continue;
120
+ }
121
+ for (let j = 0; j < d; j++) centroids[c * d + j] = sums[c * d + j] / counts[c];
122
+ }
123
+ if (spherical) {
124
+ for (let c = 0; c < k; c++) {
125
+ centroids.set(normalize(centroids.subarray(c * d, c * d + d)), c * d);
126
+ }
127
+ }
128
+ if (it > 0 && moved === 0) break;
129
+ }
130
+ return centroids;
131
+ }