@statewalker/db-sqlite-index 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +289 -2
  3. package/dist/ngram.d.mts +93 -0
  4. package/dist/ngram.d.mts.map +1 -0
  5. package/dist/ngram.mjs +615 -0
  6. package/dist/ngram.mjs.map +1 -0
  7. package/dist/vector.d.mts +103 -0
  8. package/dist/vector.d.mts.map +1 -0
  9. package/dist/vector.mjs +1061 -0
  10. package/dist/vector.mjs.map +1 -0
  11. package/package.json +63 -3
  12. package/src/ngram/do-smoke.ts +46 -0
  13. package/src/ngram/fold.test.ts +50 -0
  14. package/src/ngram/fold.ts +21 -0
  15. package/src/ngram/fts5-backend.ts +55 -0
  16. package/src/ngram/index.ts +12 -0
  17. package/src/ngram/json-chunks.test.ts +23 -0
  18. package/src/ngram/json-chunks.ts +23 -0
  19. package/src/ngram/ngram-store.test.ts +291 -0
  20. package/src/ngram/ngram-store.ts +183 -0
  21. package/src/ngram/plan-query.test.ts +141 -0
  22. package/src/ngram/plan-query.ts +42 -0
  23. package/src/ngram/postings-backend.ts +53 -0
  24. package/src/ngram/regex-literals.ts +359 -0
  25. package/src/ngram/trigrams.test.ts +28 -0
  26. package/src/ngram/trigrams.ts +25 -0
  27. package/src/vector/auto-compact.test.ts +28 -0
  28. package/src/vector/do-smoke.ts +39 -0
  29. package/src/vector/index.ts +9 -0
  30. package/src/vector/kmeans.ts +131 -0
  31. package/src/vector/math.ts +185 -0
  32. package/src/vector/segment.test.ts +61 -0
  33. package/src/vector/segment.ts +256 -0
  34. package/src/vector/sql-segment-store.ts +218 -0
  35. package/src/vector/test-fixtures.ts +64 -0
  36. package/src/vector/types.ts +30 -0
  37. package/src/vector/vector-index.test.ts +105 -0
  38. package/src/vector/vector-index.ts +586 -0
  39. package/src/vector/vector-lifecycle.test.ts +171 -0
@@ -0,0 +1,586 @@
1
+ import type { Db } from "@statewalker/db-api";
2
+ import { inTransaction } from "@statewalker/db-sqlite";
3
+ import { distance, prepare, TopK } from "./math.js";
4
+ import {
5
+ type BuiltSegment,
6
+ buildSegment,
7
+ type Candidate,
8
+ type ListData,
9
+ SegmentIndex,
10
+ } from "./segment.js";
11
+ import { type SegmentRef, SqlSegmentStore } from "./sql-segment-store.js";
12
+ import {
13
+ float32Bytes,
14
+ type Metric,
15
+ type SearchHit,
16
+ type SearchOptions,
17
+ toFloat32,
18
+ } from "./types.js";
19
+
20
+ export interface VectorIndexOptions {
21
+ db: Db;
22
+ /** Table name prefix, e.g. "vec_". Letters, digits and underscores. */
23
+ prefix: string;
24
+ dims: number;
25
+ metric?: Metric;
26
+ /** IVF lists for a new segment of n vectors. Default ~sqrt(n), flat below 2048. */
27
+ nlist?: (n: number) => number;
28
+ /** Cache budget for list blocks read from SQLite, in bytes. Default 64 MiB. */
29
+ cacheBytes?: number;
30
+ /**
31
+ * upsert() seals the tail into a segment once it holds this many vectors.
32
+ * Default 4096: compacting a larger tail risks the 15 s operation deadline of
33
+ * a Durable Object on celld. Infinity turns it off.
34
+ */
35
+ autoCompactAt?: number;
36
+ }
37
+
38
+ export interface VectorItem {
39
+ /** A safe integer: typically the rowid of your documents table. */
40
+ id: number;
41
+ vector: ArrayLike<number>;
42
+ }
43
+
44
+ export interface VectorIndexStats {
45
+ tail: number;
46
+ segments: SegmentRef[];
47
+ seq: number;
48
+ }
49
+
50
+ interface State {
51
+ seq: number;
52
+ nextGen: number;
53
+ segments: SegmentRef[];
54
+ }
55
+
56
+ interface TailEntry {
57
+ seq: number;
58
+ vector: Float32Array;
59
+ }
60
+
61
+ /** Ids per lookup (bound as one JSON parameter). */
62
+ const VALIDATE_BATCH = 2000;
63
+ /** Upserted rows per transaction. */
64
+ const WRITE_BATCH = 200;
65
+
66
+ /**
67
+ * An LSM-style vector index over a `Db`.
68
+ *
69
+ * - The mutable tail lives in `{p}tail`, mirrored in memory and searched exactly.
70
+ * - `compact()` seals the tail into an immutable IVF segment (1-bit residual
71
+ * filter → int8 re-rank → float32 re-rank on `exact`), written as rows in the
72
+ * same transaction that moves the ids out of the tail.
73
+ * - `{p}ids(id, gen)` says which generation holds the live copy of each id
74
+ * (0 = tail). A segment hit counts only if its generation matches, so deletes
75
+ * and overwrites never touch segment rows. `merge()` rewrites all segments
76
+ * into one, dropping dead rows.
77
+ *
78
+ * One writer per database is assumed (a Durable Object, a worker, one tab);
79
+ * within an instance, writes are serialized.
80
+ */
81
+ export class VectorIndex {
82
+ readonly dims: number;
83
+ readonly metric: Metric;
84
+ /** `{prefix}ids`: its `id` column holds exactly the live ids. Callers may join it in SQL. */
85
+ readonly idTable: string;
86
+ readonly #db: Db;
87
+ readonly #store: SqlSegmentStore;
88
+ readonly #nlist?: (n: number) => number;
89
+ readonly #autoCompactAt: number;
90
+ readonly #t: { meta: string; ids: string; tail: string };
91
+
92
+ #state: State = { seq: 0, nextGen: 1, segments: [] };
93
+ #tail = new Map<number, TailEntry>();
94
+ #segments: SegmentIndex[] = [];
95
+ #refs = new Map<SegmentIndex, number>();
96
+ #retired: SegmentIndex[] = [];
97
+ #queue: Promise<unknown> = Promise.resolve();
98
+ #maintaining = false;
99
+ #ready = false;
100
+ /** Bumped before and after each commit that changes the segment list: odd while one runs. */
101
+ #epoch = 0;
102
+
103
+ constructor(opts: VectorIndexOptions) {
104
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(opts.prefix)) {
105
+ throw new Error(`vector-index: bad prefix ${opts.prefix}`);
106
+ }
107
+ if (!Number.isInteger(opts.dims) || opts.dims < 1) {
108
+ throw new Error("vector-index: dims must be a positive integer");
109
+ }
110
+ this.dims = opts.dims;
111
+ this.metric = opts.metric ?? "cosine";
112
+ this.#db = opts.db;
113
+ this.#store = new SqlSegmentStore(opts.db, opts.prefix, opts.dims, opts.cacheBytes);
114
+ this.#nlist = opts.nlist;
115
+ this.#autoCompactAt = opts.autoCompactAt ?? 4096;
116
+ const p = opts.prefix;
117
+ this.#t = { meta: `${p}meta`, ids: `${p}ids`, tail: `${p}tail` };
118
+ this.idTable = this.#t.ids;
119
+ }
120
+
121
+ // ------------------------------------------------------------ lifecycle
122
+
123
+ /** Creates tables if missing, loads state and sweeps rows of crashed maintenance. */
124
+ async init(): Promise<void> {
125
+ const { meta, ids, tail } = this.#t;
126
+ for (const s of [
127
+ `CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,
128
+ `CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,
129
+ `CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,
130
+ `CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,
131
+ `CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,
132
+ ...this.#store.schema(),
133
+ ]) {
134
+ await this.#db.exec(s);
135
+ }
136
+
137
+ const config = JSON.stringify({ dims: this.dims, metric: this.metric });
138
+ await this.#db.query(
139
+ `INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`,
140
+ [config],
141
+ );
142
+ const [stored] = await this.#db.query<{ v: string }>(
143
+ `SELECT v FROM ${meta} WHERE k = 'config'`,
144
+ );
145
+ if (stored?.v !== config) {
146
+ throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);
147
+ }
148
+ await this.refresh();
149
+ const live = this.#state.segments.map((s) => s.gen);
150
+ await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));
151
+ this.#ready = true;
152
+ }
153
+
154
+ /** Reloads state, tail and segments from storage (e.g. after another writer changed them). */
155
+ async refresh(): Promise<void> {
156
+ const [row] = await this.#db.query<{ v: string }>(
157
+ `SELECT v FROM ${this.#t.meta} WHERE k = 'state'`,
158
+ );
159
+ const state: State = row ? JSON.parse(row.v) : { seq: 0, nextGen: 1, segments: [] };
160
+
161
+ const tail = new Map<number, TailEntry>();
162
+ const rows = await this.#db.query<{ id: number; seq: number; vec: Uint8Array }>(
163
+ `SELECT id, seq, vec FROM ${this.#t.tail}`,
164
+ );
165
+ for (const r of rows) tail.set(Number(r.id), { seq: Number(r.seq), vector: toFloat32(r.vec) });
166
+
167
+ const known = new Map(this.#segments.map((s) => [s.gen, s]));
168
+ const segments = await Promise.all(
169
+ state.segments.map((s) => known.get(s.gen) ?? this.#store.open(s, this.metric)),
170
+ );
171
+ this.#state = state;
172
+ this.#tail = tail;
173
+ this.#segments = segments;
174
+ }
175
+
176
+ get stats(): VectorIndexStats {
177
+ return {
178
+ tail: this.#tail.size,
179
+ segments: this.#state.segments.map((s) => ({ ...s })),
180
+ seq: this.#state.seq,
181
+ };
182
+ }
183
+
184
+ // ------------------------------------------------------------ writes
185
+
186
+ async upsert(items: VectorItem[]): Promise<void> {
187
+ this.#assertReady();
188
+ const prepared = items.map((it) => {
189
+ assertId(it.id);
190
+ return { id: it.id, vector: prepare(it.vector, this.dims, this.metric) };
191
+ });
192
+ const { tail, ids } = this.#t;
193
+ for (let i = 0; i < prepared.length; i += WRITE_BATCH) {
194
+ const batch = prepared.slice(i, i + WRITE_BATCH);
195
+ await this.#exclusive(async () => {
196
+ const seq = this.#state.seq + 1;
197
+ const next = { ...this.#state, seq };
198
+ await inTransaction(this.#db, async (tx) => {
199
+ for (const { id, vector } of batch) {
200
+ await tx.query(
201
+ `INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)
202
+ ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`,
203
+ [id, seq, float32Bytes(vector)],
204
+ );
205
+ await tx.query(
206
+ `INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`,
207
+ [id],
208
+ );
209
+ }
210
+ await this.#saveState(tx, next);
211
+ });
212
+ this.#state = next;
213
+ for (const { id, vector } of batch) this.#tail.set(id, { seq, vector });
214
+ });
215
+ if (this.#tail.size >= this.#autoCompactAt && !this.#maintaining) await this.compact();
216
+ }
217
+ }
218
+
219
+ async delete(ids: number[]): Promise<void> {
220
+ this.#assertReady();
221
+ ids.forEach(assertId);
222
+ await this.#exclusive(async () => {
223
+ for (let i = 0; i < ids.length; i += VALIDATE_BATCH) {
224
+ const chunk = ids.slice(i, i + VALIDATE_BATCH);
225
+ const json = JSON.stringify(chunk);
226
+ await inTransaction(this.#db, async (tx) => {
227
+ await tx.query(
228
+ `DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`,
229
+ [json],
230
+ );
231
+ await tx.query(
232
+ `DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`,
233
+ [json],
234
+ );
235
+ });
236
+ for (const id of chunk) this.#tail.delete(id);
237
+ }
238
+ });
239
+ }
240
+
241
+ // ------------------------------------------------------------ search
242
+
243
+ async search(query: ArrayLike<number>, opts: SearchOptions = {}): Promise<SearchHit[]> {
244
+ this.#assertReady();
245
+ const k = opts.k ?? 10;
246
+ const nprobe = opts.nprobe ?? 8;
247
+ const perList = k * (opts.oversample ?? 8);
248
+ const q = prepare(query, this.dims, this.metric);
249
+ const d = this.dims;
250
+ const filter = opts.filter;
251
+
252
+ return this.#read(async (segments) => {
253
+ // Tail: exact, from memory.
254
+ const tailTop = new TopK<number>(k);
255
+ for (const [id, { vector }] of this.#tail) {
256
+ if (filter && !filter(id)) continue;
257
+ tailTop.push(distance(this.metric, q, 0, vector, 0, d), id);
258
+ }
259
+
260
+ // Stage 1: probe lists, keep the best rows of each by estimated residual distance.
261
+ type Hit = Candidate & { seg: SegmentIndex; data: ListData };
262
+ const hits: Hit[] = [];
263
+ // Tail ids are never live in a segment; skip them early.
264
+ const keep = (id: number) => !this.#tail.has(id) && (!filter || filter(id));
265
+ await Promise.all(
266
+ segments.map(async (seg) => {
267
+ const lists = seg.probe(q, nprobe);
268
+ const datas = await this.#store.lists(seg, lists);
269
+ opts.signal?.throwIfAborted();
270
+ lists.forEach((c, i) => {
271
+ const data = datas[i] as ListData;
272
+ for (const cand of seg.filterList(q, c, data, perList, keep))
273
+ hits.push({ ...cand, seg, data });
274
+ });
275
+ }),
276
+ );
277
+
278
+ // Liveness: a segment row counts only if its generation holds the live copy.
279
+ const live = await this.#generations(hits.map((h) => h.id));
280
+
281
+ // Stage 2: int8 re-rank.
282
+ const rerank = opts.exact ? k * 4 : k;
283
+ const segTop = new TopK<Hit>(rerank);
284
+ for (const h of hits) {
285
+ if (live.get(h.id) !== h.seg.gen) continue;
286
+ segTop.push(h.seg.scoreInt8(q, h.data, h.row), h);
287
+ }
288
+ let segResults = segTop
289
+ .sorted()
290
+ .map((e) => ({ id: e.item.id, distance: e.score, hit: e.item }));
291
+
292
+ // Stage 3: exact re-rank from stored floats.
293
+ if (opts.exact) {
294
+ segResults = await Promise.all(
295
+ segResults.map(async (r) => {
296
+ const all = await this.#store.floats(r.hit.seg, r.hit.list);
297
+ const v = all.subarray(r.hit.row * d, (r.hit.row + 1) * d);
298
+ return { ...r, distance: distance(this.metric, q, 0, v, 0, d) };
299
+ }),
300
+ );
301
+ }
302
+
303
+ const final = new TopK<number>(k);
304
+ for (const r of segResults) final.push(r.distance, r.id);
305
+ for (const e of tailTop.sorted()) final.push(e.score, e.item);
306
+ return final.sorted().map((e) => ({ id: e.item, distance: e.score }));
307
+ });
308
+ }
309
+
310
+ /** The stored vectors of the live `ids` (normalised for cosine); unknown or deleted ids are absent. */
311
+ async getVectors(ids: number[]): Promise<Map<number, Float32Array>> {
312
+ this.#assertReady();
313
+ ids.forEach(assertId);
314
+ return this.#read(async (segments) => {
315
+ const out = new Map<number, Float32Array>();
316
+ const gens = await this.#generations(ids);
317
+ const wanted = new Map<number, Set<number>>(); // gen -> ids
318
+ for (const [id, gen] of gens) {
319
+ if (gen === 0) {
320
+ const entry = this.#tail.get(id);
321
+ if (entry) out.set(id, entry.vector.slice());
322
+ continue;
323
+ }
324
+ const set = wanted.get(gen) ?? new Set<number>();
325
+ set.add(id);
326
+ wanted.set(gen, set);
327
+ }
328
+ for (const seg of segments) {
329
+ const set = wanted.get(seg.gen);
330
+ if (!set) continue;
331
+ for (let c = 0; c < seg.nlist && set.size > 0; c++) {
332
+ if (seg.counts[c] === 0) continue;
333
+ const [data] = (await this.#store.lists(seg, [c])) as [ListData];
334
+ let floats: Float32Array | null = null;
335
+ for (let r = 0; r < data.ids.length; r++) {
336
+ const id = data.ids[r] as number;
337
+ if (!set.has(id)) continue;
338
+ floats ??= await this.#store.floats(seg, c);
339
+ out.set(id, floats.slice(r * this.dims, (r + 1) * this.dims));
340
+ set.delete(id);
341
+ }
342
+ }
343
+ }
344
+ return out;
345
+ });
346
+ }
347
+
348
+ // ------------------------------------------------------------ maintenance
349
+
350
+ /** Seals the current tail into a new segment. Returns the number of vectors sealed. */
351
+ async compact(): Promise<number> {
352
+ this.#assertReady();
353
+ return this.#maintenance(async () => {
354
+ const snapshot = await this.#exclusive(async () => ({
355
+ maxSeq: this.#state.seq,
356
+ entries: [...this.#tail.entries()],
357
+ }));
358
+ if (snapshot.entries.length === 0) return 0;
359
+
360
+ const n = snapshot.entries.length;
361
+ const ids = new Float64Array(n);
362
+ const vectors = new Float32Array(n * this.dims);
363
+ snapshot.entries.forEach(([id, e], i) => {
364
+ ids[i] = id;
365
+ vectors.set(e.vector, i * this.dims);
366
+ });
367
+ const built = this.#build(ids, vectors);
368
+
369
+ await this.#exclusive(() =>
370
+ this.#swapSegments(async () => {
371
+ const ref = this.#refOf(built, n);
372
+ const next: State = {
373
+ ...this.#state,
374
+ nextGen: ref.gen + 1,
375
+ segments: [...this.#state.segments, ref],
376
+ };
377
+ const { ids: idsT, tail } = this.#t;
378
+ await inTransaction(this.#db, async (tx) => {
379
+ await this.#store.write(tx, ref.gen, built);
380
+ await tx.query(
381
+ `UPDATE ${idsT} SET gen = ? WHERE gen = 0
382
+ AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`,
383
+ [ref.gen, snapshot.maxSeq],
384
+ );
385
+ await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [snapshot.maxSeq]);
386
+ await this.#saveState(tx, next);
387
+ });
388
+ this.#state = next;
389
+ for (const [id, e] of this.#tail) if (e.seq <= snapshot.maxSeq) this.#tail.delete(id);
390
+ this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
391
+ }),
392
+ );
393
+ return n;
394
+ });
395
+ }
396
+
397
+ /**
398
+ * Rewrites all segments into one (retraining the IVF lists) and drops dead
399
+ * rows. Holds all live segment vectors in memory while it runs.
400
+ */
401
+ async merge(): Promise<number> {
402
+ this.#assertReady();
403
+ return this.#maintenance(async () => {
404
+ const old = this.#segments;
405
+ if (old.length === 0) return 0;
406
+ const oldGens = old.map((s) => s.gen);
407
+
408
+ const idList: number[] = [];
409
+ const chunks: Float32Array[] = [];
410
+ for (const seg of old) {
411
+ for (let c = 0; c < seg.nlist; c++) {
412
+ if (seg.counts[c] === 0) continue;
413
+ const [data] = (await this.#store.lists(seg, [c])) as [ListData];
414
+ const floats = await this.#store.floats(seg, c);
415
+ const live = await this.#generations(Array.from(data.ids));
416
+ for (let r = 0; r < data.ids.length; r++) {
417
+ const id = data.ids[r];
418
+ if (live.get(id) !== seg.gen) continue;
419
+ idList.push(id);
420
+ chunks.push(floats.subarray(r * this.dims, (r + 1) * this.dims));
421
+ }
422
+ }
423
+ }
424
+
425
+ const n = idList.length;
426
+ const vectors = new Float32Array(n * this.dims);
427
+ chunks.forEach((v, i) => {
428
+ vectors.set(v, i * this.dims);
429
+ });
430
+ const built = n > 0 ? this.#build(Float64Array.from(idList), vectors) : null;
431
+
432
+ await this.#exclusive(() =>
433
+ this.#swapSegments(async () => {
434
+ const ref = built ? this.#refOf(built, n) : null;
435
+ const kept = this.#state.segments.filter((s) => !oldGens.includes(s.gen));
436
+ const next: State = {
437
+ ...this.#state,
438
+ nextGen: ref ? ref.gen + 1 : this.#state.nextGen,
439
+ segments: ref ? [...kept, ref] : kept,
440
+ };
441
+ await inTransaction(this.#db, async (tx) => {
442
+ if (ref && built) {
443
+ await this.#store.write(tx, ref.gen, built);
444
+ await tx.query(
445
+ `UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`,
446
+ [ref.gen, JSON.stringify(oldGens)],
447
+ );
448
+ }
449
+ await this.#saveState(tx, next);
450
+ });
451
+ this.#state = next;
452
+ this.#segments = [
453
+ ...this.#segments.filter((s) => !oldGens.includes(s.gen)),
454
+ ...(ref && built ? [this.#segmentOf(ref, built)] : []),
455
+ ];
456
+ this.#retired.push(...old);
457
+ }),
458
+ );
459
+ await this.#reap();
460
+ return n;
461
+ });
462
+ }
463
+
464
+ /** compact() then merge(). */
465
+ async optimize(): Promise<void> {
466
+ await this.compact();
467
+ await this.merge();
468
+ }
469
+
470
+ // ------------------------------------------------------------ private
471
+
472
+ #build(ids: Float64Array, vectors: Float32Array): BuiltSegment {
473
+ return buildSegment({
474
+ dims: this.dims,
475
+ metric: this.metric,
476
+ ids,
477
+ vectors,
478
+ nlist: this.#nlist?.(ids.length),
479
+ });
480
+ }
481
+
482
+ #refOf(built: BuiltSegment, count: number): SegmentRef {
483
+ return { gen: this.#state.nextGen, count, nlist: built.nlist };
484
+ }
485
+
486
+ #segmentOf(ref: SegmentRef, built: BuiltSegment): SegmentIndex {
487
+ return new SegmentIndex(ref.gen, this.dims, this.metric, built.centroids, built.counts);
488
+ }
489
+
490
+ async #generations(ids: number[]): Promise<Map<number, number>> {
491
+ const out = new Map<number, number>();
492
+ const unique = [...new Set(ids)];
493
+ for (let i = 0; i < unique.length; i += VALIDATE_BATCH) {
494
+ const rows = await this.#db.query<{ id: number; gen: number }>(
495
+ `SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`,
496
+ [JSON.stringify(unique.slice(i, i + VALIDATE_BATCH))],
497
+ );
498
+ for (const r of rows) out.set(Number(r.id), Number(r.gen));
499
+ }
500
+ return out;
501
+ }
502
+
503
+ async #saveState(tx: Db, state: State): Promise<void> {
504
+ await tx.query(
505
+ `INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)
506
+ ON CONFLICT(k) DO UPDATE SET v = excluded.v`,
507
+ [JSON.stringify(state)],
508
+ );
509
+ }
510
+
511
+ /** Deletes the rows of retired segments once no search is reading them. */
512
+ async #reap(): Promise<void> {
513
+ const ready = this.#retired.filter((s) => (this.#refs.get(s) ?? 0) === 0);
514
+ if (ready.length === 0) return;
515
+ this.#retired = this.#retired.filter((s) => !ready.includes(s));
516
+ for (const s of ready) this.#refs.delete(s);
517
+ const gens = ready.map((s) => s.gen);
518
+ await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));
519
+ }
520
+
521
+ /**
522
+ * Runs `read` over a snapshot of the segments, held against reaping. A read
523
+ * whose liveness lookup straddles a compact or merge commit may not match its
524
+ * snapshot, so it runs again on the new one; the third attempt stands.
525
+ */
526
+ async #read<T>(read: (segments: SegmentIndex[]) => Promise<T>): Promise<T> {
527
+ for (let attempt = 1; ; attempt++) {
528
+ const epoch = this.#epoch;
529
+ const segments = this.#segments;
530
+ for (const s of segments) this.#refs.set(s, (this.#refs.get(s) ?? 0) + 1);
531
+ try {
532
+ const result = await read(segments);
533
+ if (attempt === 3 || (epoch % 2 === 0 && epoch === this.#epoch)) return result;
534
+ } finally {
535
+ for (const s of segments) this.#refs.set(s, (this.#refs.get(s) ?? 1) - 1);
536
+ void this.#reap().catch(() => {}); // a failed reap leaves rows that init() sweeps
537
+ }
538
+ }
539
+ }
540
+
541
+ /** Commits a segment-list change; readers that overlap it retry (see `#read`). */
542
+ async #swapSegments(fn: () => Promise<void>): Promise<void> {
543
+ this.#epoch++;
544
+ try {
545
+ await fn();
546
+ } finally {
547
+ this.#epoch++;
548
+ }
549
+ }
550
+
551
+ #exclusive<T>(fn: () => Promise<T>): Promise<T> {
552
+ const run = this.#queue.then(fn, fn);
553
+ this.#queue = run.catch(() => {});
554
+ return run;
555
+ }
556
+
557
+ async #maintenance<T>(fn: () => Promise<T>): Promise<T> {
558
+ if (this.#maintaining) throw new Error("vector-index: compact or merge already running");
559
+ this.#maintaining = true;
560
+ try {
561
+ return await fn();
562
+ } finally {
563
+ this.#maintaining = false;
564
+ }
565
+ }
566
+
567
+ #assertReady() {
568
+ if (!this.#ready) throw new Error("vector-index: call init() first");
569
+ }
570
+ }
571
+
572
+ function assertId(id: number) {
573
+ if (!Number.isSafeInteger(id))
574
+ throw new Error(`vector-index: id must be a safe integer, got ${id}`);
575
+ }
576
+
577
+ /** Drops every table of the index under `prefix`, in one transaction. Needs no dims or metric. */
578
+ export async function dropVectorIndex(db: Db, prefix: string): Promise<void> {
579
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(prefix))
580
+ throw new Error(`vector-index: bad prefix ${prefix}`);
581
+ await inTransaction(db, async (tx) => {
582
+ for (const t of ["meta", "ids", "tail", "centroids", "seg", "f32"]) {
583
+ await tx.exec(`DROP TABLE IF EXISTS ${prefix}${t}`);
584
+ }
585
+ });
586
+ }