@statewalker/db-sqlite-index 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  import type { Db } from "@statewalker/db-api";
2
- import { inTransaction } from "@statewalker/db-sqlite";
3
2
  import { PREFIX_PATTERN } from "../prefix.js";
3
+ import { dropCatalog, GenerationCatalog } from "./catalog.js";
4
4
  import { distance, prepare, TopK } from "./math.js";
5
5
  import {
6
6
  type BuiltSegment,
@@ -10,13 +10,7 @@ import {
10
10
  SegmentIndex,
11
11
  } from "./segment.js";
12
12
  import { type SegmentRef, SqlSegmentStore } from "./sql-segment-store.js";
13
- import {
14
- float32Bytes,
15
- type Metric,
16
- type SearchHit,
17
- type SearchOptions,
18
- toFloat32,
19
- } from "./types.js";
13
+ import type { Metric, SearchHit, SearchOptions } from "./types.js";
20
14
 
21
15
  export interface VectorIndexOptions {
22
16
  db: Db;
@@ -48,19 +42,13 @@ export interface VectorIndexStats {
48
42
  seq: number;
49
43
  }
50
44
 
51
- interface State {
52
- seq: number;
53
- nextGen: number;
54
- segments: SegmentRef[];
55
- }
56
-
57
- interface TailEntry {
58
- seq: number;
59
- vector: Float32Array;
60
- }
61
-
62
- /** Ids per lookup (bound as one JSON parameter). */
63
- const VALIDATE_BATCH = 2000;
45
+ /** Lists probed per segment when the caller passes no `nprobe`. */
46
+ const DEFAULT_NPROBE = 8;
47
+ /**
48
+ * Without an explicit `nprobe`, probe lists until they hold this many times
49
+ * `k × oversample` rows (#67).
50
+ */
51
+ const PROBE_ROWS_FACTOR = 16;
64
52
  /** Upserted rows per transaction. */
65
53
  const WRITE_BATCH = 200;
66
54
 
@@ -77,26 +65,27 @@ const WRITE_BATCH = 200;
77
65
  * into one, dropping dead rows.
78
66
  *
79
67
  * One writer per database is assumed (a Durable Object, a worker, one tab);
80
- * within an instance, writes are serialized.
68
+ * within an instance, writes are serialized. The tables, the committed state and
69
+ * the in-memory tail belong to the generation catalog (`catalog.ts`); this class decides
70
+ * what to commit and when, builds segments and searches them.
81
71
  */
82
72
  export class VectorIndex {
83
73
  readonly dims: number;
84
74
  readonly metric: Metric;
85
75
  /** `{prefix}ids`: its `id` column holds exactly the live ids. Callers may join it in SQL. */
86
76
  readonly idTable: string;
87
- readonly #db: Db;
88
77
  readonly #store: SqlSegmentStore;
78
+ readonly #catalog: GenerationCatalog;
89
79
  readonly #nlist?: (n: number) => number;
90
80
  readonly #autoCompactAt: number;
91
- readonly #t: { meta: string; ids: string; tail: string };
92
81
 
93
- #state: State = { seq: 0, nextGen: 1, segments: [] };
94
- #tail = new Map<number, TailEntry>();
95
82
  #segments: SegmentIndex[] = [];
96
83
  #refs = new Map<SegmentIndex, number>();
97
84
  #retired: SegmentIndex[] = [];
98
85
  #queue: Promise<unknown> = Promise.resolve();
99
86
  #maintaining = false;
87
+ /** The running compact() or merge(), for refresh() to wait on. */
88
+ #maintenanceRun: Promise<unknown> | null = null;
100
89
  #ready = false;
101
90
  /** Bumped before and after each commit that changes the segment list: odd while one runs. */
102
91
  #epoch = 0;
@@ -110,75 +99,54 @@ export class VectorIndex {
110
99
  }
111
100
  this.dims = opts.dims;
112
101
  this.metric = opts.metric ?? "cosine";
113
- this.#db = opts.db;
114
102
  this.#store = new SqlSegmentStore(opts.db, opts.prefix, opts.dims, opts.cacheBytes);
103
+ this.#catalog = new GenerationCatalog(opts.db, opts.prefix, this.#store);
115
104
  this.#nlist = opts.nlist;
116
105
  this.#autoCompactAt = opts.autoCompactAt ?? 4096;
117
- const p = opts.prefix;
118
- this.#t = { meta: `${p}meta`, ids: `${p}ids`, tail: `${p}tail` };
119
- this.idTable = this.#t.ids;
106
+ this.idTable = this.#catalog.idTable;
120
107
  }
121
108
 
122
109
  // ------------------------------------------------------------ lifecycle
123
110
 
124
111
  /** Creates tables if missing, loads state and sweeps rows of crashed maintenance. */
125
112
  async init(): Promise<void> {
126
- const { meta, ids, tail } = this.#t;
127
- for (const s of [
128
- `CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,
129
- `CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,
130
- `CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,
131
- `CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,
132
- `CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,
133
- ...this.#store.schema(),
134
- ]) {
135
- await this.#db.exec(s);
136
- }
137
-
138
- const config = JSON.stringify({ dims: this.dims, metric: this.metric });
139
- await this.#db.query(
140
- `INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`,
141
- [config],
142
- );
143
- const [stored] = await this.#db.query<{ v: string }>(
144
- `SELECT v FROM ${meta} WHERE k = 'config'`,
145
- );
146
- if (stored?.v !== config) {
147
- throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);
148
- }
113
+ await this.#catalog.create(JSON.stringify({ dims: this.dims, metric: this.metric }));
149
114
  await this.refresh();
150
- const live = this.#state.segments.map((s) => s.gen);
151
- await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));
115
+ await this.#catalog.sweep();
152
116
  this.#ready = true;
153
117
  }
154
118
 
155
- /** Reloads state, tail and segments from storage (e.g. after another writer changed them). */
119
+ /**
120
+ * Reloads state, tail and segments from storage (e.g. after another writer changed them).
121
+ * Waits for a running compact() or merge() and runs between writes, so it never mixes the
122
+ * state before a commit with the rows after it (#26).
123
+ */
156
124
  async refresh(): Promise<void> {
157
- const [row] = await this.#db.query<{ v: string }>(
158
- `SELECT v FROM ${this.#t.meta} WHERE k = 'state'`,
159
- );
160
- const state: State = row ? JSON.parse(row.v) : { seq: 0, nextGen: 1, segments: [] };
161
-
162
- const tail = new Map<number, TailEntry>();
163
- const rows = await this.#db.query<{ id: number; seq: number; vec: Uint8Array }>(
164
- `SELECT id, seq, vec FROM ${this.#t.tail}`,
165
- );
166
- for (const r of rows) tail.set(Number(r.id), { seq: Number(r.seq), vector: toFloat32(r.vec) });
167
-
168
- const known = new Map(this.#segments.map((s) => [s.gen, s]));
169
- const segments = await Promise.all(
170
- state.segments.map((s) => known.get(s.gen) ?? this.#store.open(s, this.metric)),
171
- );
172
- this.#state = state;
173
- this.#tail = tail;
174
- this.#segments = segments;
125
+ for (;;) {
126
+ while (this.#maintenanceRun) await this.#maintenanceRun.catch(() => {});
127
+ const done = await this.#exclusive(async () => {
128
+ if (this.#maintaining) return false; // one started while we queued: wait for it too
129
+ const snapshot = await this.#catalog.read();
130
+ const known = new Map(this.#segments.map((s) => [s.gen, s]));
131
+ const segments = await Promise.all(
132
+ snapshot.state.segments.map((s) => known.get(s.gen) ?? this.#store.open(s, this.metric)),
133
+ );
134
+ await this.#swapSegments(async () => {
135
+ this.#catalog.adopt(snapshot);
136
+ this.#segments = segments;
137
+ });
138
+ return true;
139
+ });
140
+ if (done) return;
141
+ }
175
142
  }
176
143
 
177
144
  get stats(): VectorIndexStats {
145
+ const { state, tail } = this.#catalog;
178
146
  return {
179
- tail: this.#tail.size,
180
- segments: this.#state.segments.map((s) => ({ ...s })),
181
- seq: this.#state.seq,
147
+ tail: tail.size,
148
+ segments: state.segments.map((s) => ({ ...s })),
149
+ seq: state.seq,
182
150
  };
183
151
  }
184
152
 
@@ -190,100 +158,104 @@ export class VectorIndex {
190
158
  assertId(it.id);
191
159
  return { id: it.id, vector: prepare(it.vector, this.dims, this.metric) };
192
160
  });
193
- const { tail, ids } = this.#t;
194
161
  for (let i = 0; i < prepared.length; i += WRITE_BATCH) {
195
162
  const batch = prepared.slice(i, i + WRITE_BATCH);
196
- await this.#exclusive(async () => {
197
- const seq = this.#state.seq + 1;
198
- const next = { ...this.#state, seq };
199
- await inTransaction(this.#db, async (tx) => {
200
- for (const { id, vector } of batch) {
201
- await tx.query(
202
- `INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)
203
- ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`,
204
- [id, seq, float32Bytes(vector)],
205
- );
206
- await tx.query(
207
- `INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`,
208
- [id],
209
- );
210
- }
211
- await this.#saveState(tx, next);
212
- });
213
- this.#state = next;
214
- for (const { id, vector } of batch) this.#tail.set(id, { seq, vector });
215
- });
216
- if (this.#tail.size >= this.#autoCompactAt && !this.#maintaining) await this.compact();
163
+ await this.#exclusive(() => this.#catalog.append(batch));
164
+ if (this.#catalog.tail.size >= this.#autoCompactAt && !this.#maintaining)
165
+ await this.compact();
217
166
  }
218
167
  }
219
168
 
220
169
  async delete(ids: number[]): Promise<void> {
221
170
  this.#assertReady();
222
171
  ids.forEach(assertId);
223
- await this.#exclusive(async () => {
224
- for (let i = 0; i < ids.length; i += VALIDATE_BATCH) {
225
- const chunk = ids.slice(i, i + VALIDATE_BATCH);
226
- const json = JSON.stringify(chunk);
227
- await inTransaction(this.#db, async (tx) => {
228
- await tx.query(
229
- `DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`,
230
- [json],
231
- );
232
- await tx.query(
233
- `DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`,
234
- [json],
235
- );
236
- });
237
- for (const id of chunk) this.#tail.delete(id);
238
- }
239
- });
172
+ await this.#exclusive(() => this.#catalog.remove(ids));
240
173
  }
241
174
 
242
175
  // ------------------------------------------------------------ search
243
176
 
244
177
  async search(query: ArrayLike<number>, opts: SearchOptions = {}): Promise<SearchHit[]> {
245
178
  this.#assertReady();
179
+ opts.signal?.throwIfAborted();
246
180
  const k = opts.k ?? 10;
247
- const nprobe = opts.nprobe ?? 8;
248
181
  const perList = k * (opts.oversample ?? 8);
182
+ // An explicit nprobe probes exactly that many lists. The default probes at least
183
+ // DEFAULT_NPROBE lists and keeps going while they hold fewer than
184
+ // PROBE_ROWS_FACTOR × perList rows: after auto-compaction lists are small
185
+ // (~65 rows at 4200 vectors), and 8 of them miss neighbours spread over many (#67).
186
+ const nprobe = opts.nprobe ?? DEFAULT_NPROBE;
187
+ const minRows = opts.nprobe === undefined ? PROBE_ROWS_FACTOR * perList : 0;
249
188
  const q = prepare(query, this.dims, this.metric);
250
189
  const d = this.dims;
251
190
  const filter = opts.filter;
191
+ const rerank = opts.exact ? k * 4 : k;
252
192
 
253
193
  return this.#read(async (segments) => {
194
+ opts.signal?.throwIfAborted();
254
195
  // Tail: exact, from memory.
255
196
  const tailTop = new TopK<number>(k);
256
- for (const [id, { vector }] of this.#tail) {
197
+ for (const [id, { vector }] of this.#catalog.tail) {
257
198
  if (filter && !filter(id)) continue;
258
199
  tailTop.push(distance(this.metric, q, 0, vector, 0, d), id);
259
200
  }
260
201
 
261
- // Stage 1: probe lists, keep the best rows of each by estimated residual distance.
262
- type Hit = Candidate & { seg: SegmentIndex; data: ListData };
263
- const hits: Hit[] = [];
264
- // Tail ids are never live in a segment; skip them early.
265
- const keep = (id: number) => !this.#tail.has(id) && (!filter || filter(id));
202
+ // Probe lists and load their blocks.
203
+ type Probed = { seg: SegmentIndex; list: number; data: ListData };
204
+ const probed: Probed[] = [];
266
205
  await Promise.all(
267
206
  segments.map(async (seg) => {
268
- const lists = seg.probe(q, nprobe);
207
+ const lists = seg.probe(q, nprobe, minRows);
269
208
  const datas = await this.#store.lists(seg, lists);
270
- opts.signal?.throwIfAborted();
271
- lists.forEach((c, i) => {
272
- const data = datas[i] as ListData;
273
- for (const cand of seg.filterList(q, c, data, perList, keep))
274
- hits.push({ ...cand, seg, data });
209
+ lists.forEach((list, i) => {
210
+ probed.push({ seg, list, data: datas[i] as ListData });
275
211
  });
276
212
  }),
277
213
  );
214
+ opts.signal?.throwIfAborted();
278
215
 
279
- // Liveness: a segment row counts only if its generation holds the live copy.
280
- const live = await this.#generations(hits.map((h) => h.id));
216
+ // Stage 1: keep the best `perList` rows of each list by estimated residual
217
+ // distance. A segment row counts only if its generation holds the live copy
218
+ // of its id. Rows found dead (deleted, or overwritten in another generation)
219
+ // are excluded and the list is filtered again with a doubled limit, so dead
220
+ // rows never take the slots of live ones (#33).
221
+ type Hit = Candidate & { seg: SegmentIndex; data: ListData };
222
+ const gens = new Map<number, number | undefined>(); // id -> live gen; undefined = deleted
223
+ const hits: Hit[] = [];
224
+ let pending = probed.map((p) => ({ ...p, limit: perList }));
225
+ while (pending.length > 0) {
226
+ const rounds = pending.map((p) => {
227
+ // Tail ids are never live in a segment; known-dead rows are skipped too.
228
+ const keep = (id: number) =>
229
+ !this.#catalog.tail.has(id) &&
230
+ !(gens.has(id) && gens.get(id) !== p.seg.gen) &&
231
+ (!filter || filter(id));
232
+ return { p, cands: p.seg.filterList(q, p.list, p.data, p.limit, keep) };
233
+ });
234
+ const unknown = rounds
235
+ .flatMap((r) => r.cands.map((c) => c.id))
236
+ .filter((id) => !gens.has(id));
237
+ if (unknown.length > 0) {
238
+ const found = await this.#catalog.generations(unknown);
239
+ for (const id of unknown) gens.set(id, found.get(id));
240
+ opts.signal?.throwIfAborted();
241
+ }
242
+ pending = [];
243
+ for (const { p, cands } of rounds) {
244
+ const live = cands.filter((c) => gens.get(c.id) === p.seg.gen);
245
+ // A list passed whole comes in row order: keep all of it, the int8 re-rank ranks it.
246
+ const whole = p.seg.passesWhole(p.data.ids.length, p.limit);
247
+ if (!whole && live.length < perList && cands.length === p.limit) {
248
+ pending.push({ ...p, limit: p.limit * 2 }); // some were dead and more rows remain
249
+ continue;
250
+ }
251
+ for (const cand of whole ? live : live.slice(0, perList))
252
+ hits.push({ ...cand, seg: p.seg, data: p.data });
253
+ }
254
+ }
281
255
 
282
256
  // Stage 2: int8 re-rank.
283
- const rerank = opts.exact ? k * 4 : k;
284
257
  const segTop = new TopK<Hit>(rerank);
285
258
  for (const h of hits) {
286
- if (live.get(h.id) !== h.seg.gen) continue;
287
259
  segTop.push(h.seg.scoreInt8(q, h.data, h.row), h);
288
260
  }
289
261
  let segResults = segTop
@@ -314,11 +286,11 @@ export class VectorIndex {
314
286
  ids.forEach(assertId);
315
287
  return this.#read(async (segments) => {
316
288
  const out = new Map<number, Float32Array>();
317
- const gens = await this.#generations(ids);
289
+ const gens = await this.#catalog.generations(ids);
318
290
  const wanted = new Map<number, Set<number>>(); // gen -> ids
319
291
  for (const [id, gen] of gens) {
320
292
  if (gen === 0) {
321
- const entry = this.#tail.get(id);
293
+ const entry = this.#catalog.tail.get(id);
322
294
  if (entry) out.set(id, entry.vector.slice());
323
295
  continue;
324
296
  }
@@ -353,8 +325,8 @@ export class VectorIndex {
353
325
  this.#assertReady();
354
326
  return this.#maintenance(async () => {
355
327
  const snapshot = await this.#exclusive(async () => ({
356
- maxSeq: this.#state.seq,
357
- entries: [...this.#tail.entries()],
328
+ maxSeq: this.#catalog.state.seq,
329
+ entries: [...this.#catalog.tail.entries()],
358
330
  }));
359
331
  if (snapshot.entries.length === 0) return 0;
360
332
 
@@ -369,26 +341,9 @@ export class VectorIndex {
369
341
 
370
342
  await this.#exclusive(() =>
371
343
  this.#swapSegments(async () => {
372
- const ref = this.#refOf(built, n);
373
- const next: State = {
374
- ...this.#state,
375
- nextGen: ref.gen + 1,
376
- segments: [...this.#state.segments, ref],
377
- };
378
- const { ids: idsT, tail } = this.#t;
379
- await inTransaction(this.#db, async (tx) => {
380
- await this.#store.write(tx, ref.gen, built);
381
- await tx.query(
382
- `UPDATE ${idsT} SET gen = ? WHERE gen = 0
383
- AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`,
384
- [ref.gen, snapshot.maxSeq],
385
- );
386
- await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [snapshot.maxSeq]);
387
- await this.#saveState(tx, next);
344
+ await this.#catalog.seal(snapshot.maxSeq, built, n, (ref) => {
345
+ this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
388
346
  });
389
- this.#state = next;
390
- for (const [id, e] of this.#tail) if (e.seq <= snapshot.maxSeq) this.#tail.delete(id);
391
- this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
392
347
  }),
393
348
  );
394
349
  return n;
@@ -413,7 +368,7 @@ export class VectorIndex {
413
368
  if (seg.counts[c] === 0) continue;
414
369
  const [data] = (await this.#store.lists(seg, [c])) as [ListData];
415
370
  const floats = await this.#store.floats(seg, c);
416
- const live = await this.#generations(Array.from(data.ids));
371
+ const live = await this.#catalog.generations(Array.from(data.ids));
417
372
  for (let r = 0; r < data.ids.length; r++) {
418
373
  const id = data.ids[r];
419
374
  if (live.get(id) !== seg.gen) continue;
@@ -432,29 +387,13 @@ export class VectorIndex {
432
387
 
433
388
  await this.#exclusive(() =>
434
389
  this.#swapSegments(async () => {
435
- const ref = built ? this.#refOf(built, n) : null;
436
- const kept = this.#state.segments.filter((s) => !oldGens.includes(s.gen));
437
- const next: State = {
438
- ...this.#state,
439
- nextGen: ref ? ref.gen + 1 : this.#state.nextGen,
440
- segments: ref ? [...kept, ref] : kept,
441
- };
442
- await inTransaction(this.#db, async (tx) => {
443
- if (ref && built) {
444
- await this.#store.write(tx, ref.gen, built);
445
- await tx.query(
446
- `UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`,
447
- [ref.gen, JSON.stringify(oldGens)],
448
- );
449
- }
450
- await this.#saveState(tx, next);
390
+ await this.#catalog.replace(oldGens, built, n, (ref) => {
391
+ this.#segments = [
392
+ ...this.#segments.filter((s) => !oldGens.includes(s.gen)),
393
+ ...(ref && built ? [this.#segmentOf(ref, built)] : []),
394
+ ];
395
+ this.#retired.push(...old);
451
396
  });
452
- this.#state = next;
453
- this.#segments = [
454
- ...this.#segments.filter((s) => !oldGens.includes(s.gen)),
455
- ...(ref && built ? [this.#segmentOf(ref, built)] : []),
456
- ];
457
- this.#retired.push(...old);
458
397
  }),
459
398
  );
460
399
  await this.#reap();
@@ -480,43 +419,17 @@ export class VectorIndex {
480
419
  });
481
420
  }
482
421
 
483
- #refOf(built: BuiltSegment, count: number): SegmentRef {
484
- return { gen: this.#state.nextGen, count, nlist: built.nlist };
485
- }
486
-
487
422
  #segmentOf(ref: SegmentRef, built: BuiltSegment): SegmentIndex {
488
423
  return new SegmentIndex(ref.gen, this.dims, this.metric, built.centroids, built.counts);
489
424
  }
490
425
 
491
- async #generations(ids: number[]): Promise<Map<number, number>> {
492
- const out = new Map<number, number>();
493
- const unique = [...new Set(ids)];
494
- for (let i = 0; i < unique.length; i += VALIDATE_BATCH) {
495
- const rows = await this.#db.query<{ id: number; gen: number }>(
496
- `SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`,
497
- [JSON.stringify(unique.slice(i, i + VALIDATE_BATCH))],
498
- );
499
- for (const r of rows) out.set(Number(r.id), Number(r.gen));
500
- }
501
- return out;
502
- }
503
-
504
- async #saveState(tx: Db, state: State): Promise<void> {
505
- await tx.query(
506
- `INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)
507
- ON CONFLICT(k) DO UPDATE SET v = excluded.v`,
508
- [JSON.stringify(state)],
509
- );
510
- }
511
-
512
426
  /** Deletes the rows of retired segments once no search is reading them. */
513
427
  async #reap(): Promise<void> {
514
428
  const ready = this.#retired.filter((s) => (this.#refs.get(s) ?? 0) === 0);
515
429
  if (ready.length === 0) return;
516
430
  this.#retired = this.#retired.filter((s) => !ready.includes(s));
517
431
  for (const s of ready) this.#refs.delete(s);
518
- const gens = ready.map((s) => s.gen);
519
- await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));
432
+ await this.#catalog.dropGenerations(ready.map((s) => s.gen));
520
433
  }
521
434
 
522
435
  /**
@@ -558,11 +471,16 @@ export class VectorIndex {
558
471
  async #maintenance<T>(fn: () => Promise<T>): Promise<T> {
559
472
  if (this.#maintaining) throw new Error("vector-index: compact or merge already running");
560
473
  this.#maintaining = true;
561
- try {
562
- return await fn();
563
- } finally {
564
- this.#maintaining = false;
565
- }
474
+ const run = (async () => {
475
+ try {
476
+ return await fn();
477
+ } finally {
478
+ this.#maintaining = false;
479
+ this.#maintenanceRun = null;
480
+ }
481
+ })();
482
+ this.#maintenanceRun = run;
483
+ return run;
566
484
  }
567
485
 
568
486
  #assertReady() {
@@ -578,9 +496,5 @@ function assertId(id: number) {
578
496
  /** Drops every table of the index under `prefix`, in one transaction. Needs no dims or metric. */
579
497
  export async function dropVectorIndex(db: Db, prefix: string): Promise<void> {
580
498
  if (!PREFIX_PATTERN.test(prefix)) throw new Error(`vector-index: bad prefix ${prefix}`);
581
- await inTransaction(db, async (tx) => {
582
- for (const t of ["meta", "ids", "tail", "centroids", "seg", "f32"]) {
583
- await tx.exec(`DROP TABLE IF EXISTS ${prefix}${t}`);
584
- }
585
- });
499
+ await dropCatalog(db, prefix);
586
500
  }
@@ -1,46 +0,0 @@
1
- import type { Db } from "@statewalker/db-api";
2
- import { NgramStore } from "./ngram-store.js";
3
-
4
- const check = (ok: boolean, message: string) => {
5
- if (!ok) throw new Error(`ngram smoke: ${message}`);
6
- };
7
-
8
- /** True when the index itself still offers candidates (candidateBudget 0 skips the verifier). */
9
- const indexed = async (index: NgramStore, pattern: string) =>
10
- (await index.search(pattern, { candidateBudget: 0 })).truncated;
11
-
12
- const ids = (result: { hits: { id: number }[] }) => result.hits.map((h) => h.id).join();
13
-
14
- /** Run inside a real Durable Object by db-sqlite-do's `pnpm test:do`. */
15
- export const ngramDoScenarios = (["fts5", "postings"] as const).map((store) => ({
16
- name: `ngram (${store}): upsert, overwrite, lone surrogate, drop inside a transaction`,
17
- async run(db: Db) {
18
- const prefix = `smoke_${store}_`;
19
- const index = new NgramStore({ db, prefix, store });
20
- await index.init(); // CREATE of the index tables (a contentless trigram table for fts5) in a transaction
21
- await index.upsert([
22
- { id: 1, text: "Hello World" },
23
- { id: 2, text: "hello there" },
24
- { id: 3, text: "lone \ud800 surrogate" },
25
- ]);
26
- check(
27
- ids(await index.search("hello", { caseSensitive: false })) === "1,2",
28
- "case-insensitive hits",
29
- );
30
- check(ids(await index.search("Hello")) === "1", "case-sensitive hits");
31
- check(ids(await index.search("surrogate")) === "3", "text with a lone surrogate is indexed");
32
- check((await index.getTexts([3])).get(3) === "lone \ufffd surrogate", "text made well-formed");
33
- check(await indexed(index, "World"), "index lacks the grams of 'World'");
34
- check(await indexed(index, "surrogate"), "index lacks the grams of 'surrogate'");
35
- await index.upsert([{ id: 1, text: "goodbye" }]); // replays the old text to remove its grams
36
- check(!(await indexed(index, "World")), "stale grams after overwrite");
37
- await index.upsert([{ id: 3, text: "lone \ud800 changed" }]);
38
- check(!(await indexed(index, "surrogate")), "stale grams after surrogate overwrite");
39
- await index.drop(); // DROP of the tables inside a transaction
40
- const [row] = await db.query<{ n: number }>(
41
- "SELECT count(*) AS n FROM sqlite_master WHERE name LIKE ?",
42
- [`${prefix}%`],
43
- );
44
- check(Number(row?.n) === 0, `tables left after drop: ${row?.n}`);
45
- },
46
- }));
@@ -1,50 +0,0 @@
1
- import { describe, expect, it } from "vitest";
2
- import { fold } from "./fold.js";
3
-
4
- const escapeRegex = (s: string) => s.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&");
5
-
6
- describe("fold", () => {
7
- it("lower-cases ASCII and keeps length", () => {
8
- expect(fold("Hello WORLD")).toBe("hello world");
9
- });
10
-
11
- it("maps every case form of sigma to one letter", () => {
12
- expect(new Set([fold("Σ"), fold("σ"), fold("ς")]).size).toBe(1);
13
- });
14
-
15
- it("never separates two BMP characters that a case-insensitive regex treats as equal", () => {
16
- const broken: string[] = [];
17
- for (let cp = 0; cp <= 0xffff; cp++) {
18
- if (cp >= 0xd800 && cp <= 0xdfff) continue;
19
- const c = String.fromCharCode(cp);
20
- const re = new RegExp(`^${escapeRegex(c)}$`, "i");
21
- for (const d of [c.toUpperCase(), c.toLowerCase()]) {
22
- if (d === c || d.length !== 1 || !re.test(d)) continue;
23
- if (fold(c) !== fold(d)) broken.push(`U+${cp.toString(16)} ${c}/${d}`);
24
- }
25
- }
26
- expect(broken).toEqual([]);
27
- });
28
-
29
- it("keeps the UTF-16 length of every code point, so offsets stay valid", () => {
30
- const changed: string[] = [];
31
- for (let cp = 0; cp <= 0x10ffff; cp++) {
32
- if (cp >= 0xd800 && cp <= 0xdfff) continue;
33
- const c = String.fromCodePoint(cp);
34
- if (fold(c).length !== c.length) changed.push(`U+${cp.toString(16)}`);
35
- }
36
- expect(changed).toEqual([]);
37
- });
38
-
39
- it("keeps the length of a text with U+0130 and an astral character", () => {
40
- const folded = fold("İ😀A");
41
- expect(folded.length).toBe("İ😀A".length);
42
- expect(folded.endsWith("a")).toBe(true);
43
- });
44
-
45
- it("is a homomorphism over concatenation, so substrings stay substrings", () => {
46
- const a = "ΟΔΟΣ ſtraße";
47
- const b = "İK";
48
- expect(fold(a + b)).toBe(fold(a) + fold(b));
49
- });
50
- });
@@ -1,23 +0,0 @@
1
- import { describe, expect, it } from "vitest";
2
- import { chunkJson } from "./json-chunks.js";
3
-
4
- describe("chunkJson", () => {
5
- const rows = Array.from({ length: 30 }, (_, i) => [i, `t${i}`]);
6
-
7
- it("splits into chunks of at most maxChars that rebuild the rows in order", () => {
8
- const chunks = chunkJson(rows, 20);
9
- expect(chunks.length).toBeGreaterThan(1);
10
- for (const chunk of chunks) expect(chunk.length).toBeLessThanOrEqual(20);
11
- expect(chunks.flatMap((chunk) => JSON.parse(chunk))).toEqual(rows);
12
- });
13
-
14
- it("keeps a row larger than maxChars as one chunk of its own", () => {
15
- const big = [1, "x".repeat(50)];
16
- const chunks = chunkJson([[0, "a"], big, [2, "b"]], 20);
17
- expect(chunks).toEqual(['[[0,"a"]]', JSON.stringify([big]), '[[2,"b"]]']);
18
- });
19
-
20
- it("returns no chunks for no rows", () => {
21
- expect(chunkJson([], 20)).toEqual([]);
22
- });
23
- });