@statewalker/db-sqlite-index 0.1.2 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  import type { Db } from "@statewalker/db-api";
2
- import { inTransaction } from "@statewalker/db-sqlite";
3
2
  import { PREFIX_PATTERN } from "../prefix.js";
3
+ import { dropCatalog, GenerationCatalog } from "./catalog.js";
4
4
  import { distance, prepare, TopK } from "./math.js";
5
5
  import {
6
6
  type BuiltSegment,
@@ -10,13 +10,7 @@ import {
10
10
  SegmentIndex,
11
11
  } from "./segment.js";
12
12
  import { type SegmentRef, SqlSegmentStore } from "./sql-segment-store.js";
13
- import {
14
- float32Bytes,
15
- type Metric,
16
- type SearchHit,
17
- type SearchOptions,
18
- toFloat32,
19
- } from "./types.js";
13
+ import type { Metric, SearchHit, SearchOptions } from "./types.js";
20
14
 
21
15
  export interface VectorIndexOptions {
22
16
  db: Db;
@@ -48,19 +42,6 @@ export interface VectorIndexStats {
48
42
  seq: number;
49
43
  }
50
44
 
51
- interface State {
52
- seq: number;
53
- nextGen: number;
54
- segments: SegmentRef[];
55
- }
56
-
57
- interface TailEntry {
58
- seq: number;
59
- vector: Float32Array;
60
- }
61
-
62
- /** Ids per lookup (bound as one JSON parameter). */
63
- const VALIDATE_BATCH = 2000;
64
45
  /** Lists probed per segment when the caller passes no `nprobe`. */
65
46
  const DEFAULT_NPROBE = 8;
66
47
  /**
@@ -84,26 +65,27 @@ const WRITE_BATCH = 200;
84
65
  * into one, dropping dead rows.
85
66
  *
86
67
  * One writer per database is assumed (a Durable Object, a worker, one tab);
87
- * within an instance, writes are serialized.
68
+ * within an instance, writes are serialized. The tables, the committed state and
69
+ * the in-memory tail belong to the generation catalog (`catalog.ts`); this class decides
70
+ * what to commit and when, builds segments and searches them.
88
71
  */
89
72
  export class VectorIndex {
90
73
  readonly dims: number;
91
74
  readonly metric: Metric;
92
75
  /** `{prefix}ids`: its `id` column holds exactly the live ids. Callers may join it in SQL. */
93
76
  readonly idTable: string;
94
- readonly #db: Db;
95
77
  readonly #store: SqlSegmentStore;
78
+ readonly #catalog: GenerationCatalog;
96
79
  readonly #nlist?: (n: number) => number;
97
80
  readonly #autoCompactAt: number;
98
- readonly #t: { meta: string; ids: string; tail: string };
99
81
 
100
- #state: State = { seq: 0, nextGen: 1, segments: [] };
101
- #tail = new Map<number, TailEntry>();
102
82
  #segments: SegmentIndex[] = [];
103
83
  #refs = new Map<SegmentIndex, number>();
104
84
  #retired: SegmentIndex[] = [];
105
85
  #queue: Promise<unknown> = Promise.resolve();
106
86
  #maintaining = false;
87
+ /** The running compact() or merge(), for refresh() to wait on. */
88
+ #maintenanceRun: Promise<unknown> | null = null;
107
89
  #ready = false;
108
90
  /** Bumped before and after each commit that changes the segment list: odd while one runs. */
109
91
  #epoch = 0;
@@ -117,75 +99,54 @@ export class VectorIndex {
117
99
  }
118
100
  this.dims = opts.dims;
119
101
  this.metric = opts.metric ?? "cosine";
120
- this.#db = opts.db;
121
102
  this.#store = new SqlSegmentStore(opts.db, opts.prefix, opts.dims, opts.cacheBytes);
103
+ this.#catalog = new GenerationCatalog(opts.db, opts.prefix, this.#store);
122
104
  this.#nlist = opts.nlist;
123
105
  this.#autoCompactAt = opts.autoCompactAt ?? 4096;
124
- const p = opts.prefix;
125
- this.#t = { meta: `${p}meta`, ids: `${p}ids`, tail: `${p}tail` };
126
- this.idTable = this.#t.ids;
106
+ this.idTable = this.#catalog.idTable;
127
107
  }
128
108
 
129
109
  // ------------------------------------------------------------ lifecycle
130
110
 
131
111
  /** Creates tables if missing, loads state and sweeps rows of crashed maintenance. */
132
112
  async init(): Promise<void> {
133
- const { meta, ids, tail } = this.#t;
134
- for (const s of [
135
- `CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,
136
- `CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,
137
- `CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,
138
- `CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,
139
- `CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,
140
- ...this.#store.schema(),
141
- ]) {
142
- await this.#db.exec(s);
143
- }
144
-
145
- const config = JSON.stringify({ dims: this.dims, metric: this.metric });
146
- await this.#db.query(
147
- `INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`,
148
- [config],
149
- );
150
- const [stored] = await this.#db.query<{ v: string }>(
151
- `SELECT v FROM ${meta} WHERE k = 'config'`,
152
- );
153
- if (stored?.v !== config) {
154
- throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);
155
- }
113
+ await this.#catalog.create(JSON.stringify({ dims: this.dims, metric: this.metric }));
156
114
  await this.refresh();
157
- const live = this.#state.segments.map((s) => s.gen);
158
- await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));
115
+ await this.#catalog.sweep();
159
116
  this.#ready = true;
160
117
  }
161
118
 
162
- /** Reloads state, tail and segments from storage (e.g. after another writer changed them). */
119
+ /**
120
+ * Reloads state, tail and segments from storage (e.g. after another writer changed them).
121
+ * Waits for a running compact() or merge() and runs between writes, so it never mixes the
122
+ * state before a commit with the rows after it (#26).
123
+ */
163
124
  async refresh(): Promise<void> {
164
- const [row] = await this.#db.query<{ v: string }>(
165
- `SELECT v FROM ${this.#t.meta} WHERE k = 'state'`,
166
- );
167
- const state: State = row ? JSON.parse(row.v) : { seq: 0, nextGen: 1, segments: [] };
168
-
169
- const tail = new Map<number, TailEntry>();
170
- const rows = await this.#db.query<{ id: number; seq: number; vec: Uint8Array }>(
171
- `SELECT id, seq, vec FROM ${this.#t.tail}`,
172
- );
173
- for (const r of rows) tail.set(Number(r.id), { seq: Number(r.seq), vector: toFloat32(r.vec) });
174
-
175
- const known = new Map(this.#segments.map((s) => [s.gen, s]));
176
- const segments = await Promise.all(
177
- state.segments.map((s) => known.get(s.gen) ?? this.#store.open(s, this.metric)),
178
- );
179
- this.#state = state;
180
- this.#tail = tail;
181
- this.#segments = segments;
125
+ for (;;) {
126
+ while (this.#maintenanceRun) await this.#maintenanceRun.catch(() => {});
127
+ const done = await this.#exclusive(async () => {
128
+ if (this.#maintaining) return false; // one started while we queued: wait for it too
129
+ const snapshot = await this.#catalog.read();
130
+ const known = new Map(this.#segments.map((s) => [s.gen, s]));
131
+ const segments = await Promise.all(
132
+ snapshot.state.segments.map((s) => known.get(s.gen) ?? this.#store.open(s, this.metric)),
133
+ );
134
+ await this.#swapSegments(async () => {
135
+ this.#catalog.adopt(snapshot);
136
+ this.#segments = segments;
137
+ });
138
+ return true;
139
+ });
140
+ if (done) return;
141
+ }
182
142
  }
183
143
 
184
144
  get stats(): VectorIndexStats {
145
+ const { state, tail } = this.#catalog;
185
146
  return {
186
- tail: this.#tail.size,
187
- segments: this.#state.segments.map((s) => ({ ...s })),
188
- seq: this.#state.seq,
147
+ tail: tail.size,
148
+ segments: state.segments.map((s) => ({ ...s })),
149
+ seq: state.seq,
189
150
  };
190
151
  }
191
152
 
@@ -197,53 +158,18 @@ export class VectorIndex {
197
158
  assertId(it.id);
198
159
  return { id: it.id, vector: prepare(it.vector, this.dims, this.metric) };
199
160
  });
200
- const { tail, ids } = this.#t;
201
161
  for (let i = 0; i < prepared.length; i += WRITE_BATCH) {
202
162
  const batch = prepared.slice(i, i + WRITE_BATCH);
203
- await this.#exclusive(async () => {
204
- const seq = this.#state.seq + 1;
205
- const next = { ...this.#state, seq };
206
- await inTransaction(this.#db, async (tx) => {
207
- for (const { id, vector } of batch) {
208
- await tx.query(
209
- `INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)
210
- ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`,
211
- [id, seq, float32Bytes(vector)],
212
- );
213
- await tx.query(
214
- `INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`,
215
- [id],
216
- );
217
- }
218
- await this.#saveState(tx, next);
219
- });
220
- this.#state = next;
221
- for (const { id, vector } of batch) this.#tail.set(id, { seq, vector });
222
- });
223
- if (this.#tail.size >= this.#autoCompactAt && !this.#maintaining) await this.compact();
163
+ await this.#exclusive(() => this.#catalog.append(batch));
164
+ if (this.#catalog.tail.size >= this.#autoCompactAt && !this.#maintaining)
165
+ await this.compact();
224
166
  }
225
167
  }
226
168
 
227
169
  async delete(ids: number[]): Promise<void> {
228
170
  this.#assertReady();
229
171
  ids.forEach(assertId);
230
- await this.#exclusive(async () => {
231
- for (let i = 0; i < ids.length; i += VALIDATE_BATCH) {
232
- const chunk = ids.slice(i, i + VALIDATE_BATCH);
233
- const json = JSON.stringify(chunk);
234
- await inTransaction(this.#db, async (tx) => {
235
- await tx.query(
236
- `DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`,
237
- [json],
238
- );
239
- await tx.query(
240
- `DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`,
241
- [json],
242
- );
243
- });
244
- for (const id of chunk) this.#tail.delete(id);
245
- }
246
- });
172
+ await this.#exclusive(() => this.#catalog.remove(ids));
247
173
  }
248
174
 
249
175
  // ------------------------------------------------------------ search
@@ -268,7 +194,7 @@ export class VectorIndex {
268
194
  opts.signal?.throwIfAborted();
269
195
  // Tail: exact, from memory.
270
196
  const tailTop = new TopK<number>(k);
271
- for (const [id, { vector }] of this.#tail) {
197
+ for (const [id, { vector }] of this.#catalog.tail) {
272
198
  if (filter && !filter(id)) continue;
273
199
  tailTop.push(distance(this.metric, q, 0, vector, 0, d), id);
274
200
  }
@@ -300,7 +226,7 @@ export class VectorIndex {
300
226
  const rounds = pending.map((p) => {
301
227
  // Tail ids are never live in a segment; known-dead rows are skipped too.
302
228
  const keep = (id: number) =>
303
- !this.#tail.has(id) &&
229
+ !this.#catalog.tail.has(id) &&
304
230
  !(gens.has(id) && gens.get(id) !== p.seg.gen) &&
305
231
  (!filter || filter(id));
306
232
  return { p, cands: p.seg.filterList(q, p.list, p.data, p.limit, keep) };
@@ -309,7 +235,7 @@ export class VectorIndex {
309
235
  .flatMap((r) => r.cands.map((c) => c.id))
310
236
  .filter((id) => !gens.has(id));
311
237
  if (unknown.length > 0) {
312
- const found = await this.#generations(unknown);
238
+ const found = await this.#catalog.generations(unknown);
313
239
  for (const id of unknown) gens.set(id, found.get(id));
314
240
  opts.signal?.throwIfAborted();
315
241
  }
@@ -360,11 +286,11 @@ export class VectorIndex {
360
286
  ids.forEach(assertId);
361
287
  return this.#read(async (segments) => {
362
288
  const out = new Map<number, Float32Array>();
363
- const gens = await this.#generations(ids);
289
+ const gens = await this.#catalog.generations(ids);
364
290
  const wanted = new Map<number, Set<number>>(); // gen -> ids
365
291
  for (const [id, gen] of gens) {
366
292
  if (gen === 0) {
367
- const entry = this.#tail.get(id);
293
+ const entry = this.#catalog.tail.get(id);
368
294
  if (entry) out.set(id, entry.vector.slice());
369
295
  continue;
370
296
  }
@@ -399,8 +325,8 @@ export class VectorIndex {
399
325
  this.#assertReady();
400
326
  return this.#maintenance(async () => {
401
327
  const snapshot = await this.#exclusive(async () => ({
402
- maxSeq: this.#state.seq,
403
- entries: [...this.#tail.entries()],
328
+ maxSeq: this.#catalog.state.seq,
329
+ entries: [...this.#catalog.tail.entries()],
404
330
  }));
405
331
  if (snapshot.entries.length === 0) return 0;
406
332
 
@@ -415,26 +341,9 @@ export class VectorIndex {
415
341
 
416
342
  await this.#exclusive(() =>
417
343
  this.#swapSegments(async () => {
418
- const ref = this.#refOf(built, n);
419
- const next: State = {
420
- ...this.#state,
421
- nextGen: ref.gen + 1,
422
- segments: [...this.#state.segments, ref],
423
- };
424
- const { ids: idsT, tail } = this.#t;
425
- await inTransaction(this.#db, async (tx) => {
426
- await this.#store.write(tx, ref.gen, built);
427
- await tx.query(
428
- `UPDATE ${idsT} SET gen = ? WHERE gen = 0
429
- AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`,
430
- [ref.gen, snapshot.maxSeq],
431
- );
432
- await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [snapshot.maxSeq]);
433
- await this.#saveState(tx, next);
344
+ await this.#catalog.seal(snapshot.maxSeq, built, n, (ref) => {
345
+ this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
434
346
  });
435
- this.#state = next;
436
- for (const [id, e] of this.#tail) if (e.seq <= snapshot.maxSeq) this.#tail.delete(id);
437
- this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
438
347
  }),
439
348
  );
440
349
  return n;
@@ -459,7 +368,7 @@ export class VectorIndex {
459
368
  if (seg.counts[c] === 0) continue;
460
369
  const [data] = (await this.#store.lists(seg, [c])) as [ListData];
461
370
  const floats = await this.#store.floats(seg, c);
462
- const live = await this.#generations(Array.from(data.ids));
371
+ const live = await this.#catalog.generations(Array.from(data.ids));
463
372
  for (let r = 0; r < data.ids.length; r++) {
464
373
  const id = data.ids[r];
465
374
  if (live.get(id) !== seg.gen) continue;
@@ -478,29 +387,13 @@ export class VectorIndex {
478
387
 
479
388
  await this.#exclusive(() =>
480
389
  this.#swapSegments(async () => {
481
- const ref = built ? this.#refOf(built, n) : null;
482
- const kept = this.#state.segments.filter((s) => !oldGens.includes(s.gen));
483
- const next: State = {
484
- ...this.#state,
485
- nextGen: ref ? ref.gen + 1 : this.#state.nextGen,
486
- segments: ref ? [...kept, ref] : kept,
487
- };
488
- await inTransaction(this.#db, async (tx) => {
489
- if (ref && built) {
490
- await this.#store.write(tx, ref.gen, built);
491
- await tx.query(
492
- `UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`,
493
- [ref.gen, JSON.stringify(oldGens)],
494
- );
495
- }
496
- await this.#saveState(tx, next);
390
+ await this.#catalog.replace(oldGens, built, n, (ref) => {
391
+ this.#segments = [
392
+ ...this.#segments.filter((s) => !oldGens.includes(s.gen)),
393
+ ...(ref && built ? [this.#segmentOf(ref, built)] : []),
394
+ ];
395
+ this.#retired.push(...old);
497
396
  });
498
- this.#state = next;
499
- this.#segments = [
500
- ...this.#segments.filter((s) => !oldGens.includes(s.gen)),
501
- ...(ref && built ? [this.#segmentOf(ref, built)] : []),
502
- ];
503
- this.#retired.push(...old);
504
397
  }),
505
398
  );
506
399
  await this.#reap();
@@ -526,43 +419,17 @@ export class VectorIndex {
526
419
  });
527
420
  }
528
421
 
529
- #refOf(built: BuiltSegment, count: number): SegmentRef {
530
- return { gen: this.#state.nextGen, count, nlist: built.nlist };
531
- }
532
-
533
422
  #segmentOf(ref: SegmentRef, built: BuiltSegment): SegmentIndex {
534
423
  return new SegmentIndex(ref.gen, this.dims, this.metric, built.centroids, built.counts);
535
424
  }
536
425
 
537
- async #generations(ids: number[]): Promise<Map<number, number>> {
538
- const out = new Map<number, number>();
539
- const unique = [...new Set(ids)];
540
- for (let i = 0; i < unique.length; i += VALIDATE_BATCH) {
541
- const rows = await this.#db.query<{ id: number; gen: number }>(
542
- `SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`,
543
- [JSON.stringify(unique.slice(i, i + VALIDATE_BATCH))],
544
- );
545
- for (const r of rows) out.set(Number(r.id), Number(r.gen));
546
- }
547
- return out;
548
- }
549
-
550
- async #saveState(tx: Db, state: State): Promise<void> {
551
- await tx.query(
552
- `INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)
553
- ON CONFLICT(k) DO UPDATE SET v = excluded.v`,
554
- [JSON.stringify(state)],
555
- );
556
- }
557
-
558
426
  /** Deletes the rows of retired segments once no search is reading them. */
559
427
  async #reap(): Promise<void> {
560
428
  const ready = this.#retired.filter((s) => (this.#refs.get(s) ?? 0) === 0);
561
429
  if (ready.length === 0) return;
562
430
  this.#retired = this.#retired.filter((s) => !ready.includes(s));
563
431
  for (const s of ready) this.#refs.delete(s);
564
- const gens = ready.map((s) => s.gen);
565
- await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));
432
+ await this.#catalog.dropGenerations(ready.map((s) => s.gen));
566
433
  }
567
434
 
568
435
  /**
@@ -604,11 +471,16 @@ export class VectorIndex {
604
471
  async #maintenance<T>(fn: () => Promise<T>): Promise<T> {
605
472
  if (this.#maintaining) throw new Error("vector-index: compact or merge already running");
606
473
  this.#maintaining = true;
607
- try {
608
- return await fn();
609
- } finally {
610
- this.#maintaining = false;
611
- }
474
+ const run = (async () => {
475
+ try {
476
+ return await fn();
477
+ } finally {
478
+ this.#maintaining = false;
479
+ this.#maintenanceRun = null;
480
+ }
481
+ })();
482
+ this.#maintenanceRun = run;
483
+ return run;
612
484
  }
613
485
 
614
486
  #assertReady() {
@@ -624,9 +496,5 @@ function assertId(id: number) {
624
496
  /** Drops every table of the index under `prefix`, in one transaction. Needs no dims or metric. */
625
497
  export async function dropVectorIndex(db: Db, prefix: string): Promise<void> {
626
498
  if (!PREFIX_PATTERN.test(prefix)) throw new Error(`vector-index: bad prefix ${prefix}`);
627
- await inTransaction(db, async (tx) => {
628
- for (const t of ["meta", "ids", "tail", "centroids", "seg", "f32"]) {
629
- await tx.exec(`DROP TABLE IF EXISTS ${prefix}${t}`);
630
- }
631
- });
499
+ await dropCatalog(db, prefix);
632
500
  }