@statewalker/db-sqlite-index 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/vector.d.mts +8 -2
- package/dist/vector.d.mts.map +1 -1
- package/dist/vector.mjs +286 -175
- package/dist/vector.mjs.map +1 -1
- package/package.json +6 -6
- package/src/vector/catalog.ts +285 -0
- package/src/vector/sql-segment-store.ts +6 -1
- package/src/vector/vector-index.ts +70 -202
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Db } from "@statewalker/db-api";
|
|
2
|
-
import { inTransaction } from "@statewalker/db-sqlite";
|
|
3
2
|
import { PREFIX_PATTERN } from "../prefix.js";
|
|
3
|
+
import { dropCatalog, GenerationCatalog } from "./catalog.js";
|
|
4
4
|
import { distance, prepare, TopK } from "./math.js";
|
|
5
5
|
import {
|
|
6
6
|
type BuiltSegment,
|
|
@@ -10,13 +10,7 @@ import {
|
|
|
10
10
|
SegmentIndex,
|
|
11
11
|
} from "./segment.js";
|
|
12
12
|
import { type SegmentRef, SqlSegmentStore } from "./sql-segment-store.js";
|
|
13
|
-
import {
|
|
14
|
-
float32Bytes,
|
|
15
|
-
type Metric,
|
|
16
|
-
type SearchHit,
|
|
17
|
-
type SearchOptions,
|
|
18
|
-
toFloat32,
|
|
19
|
-
} from "./types.js";
|
|
13
|
+
import type { Metric, SearchHit, SearchOptions } from "./types.js";
|
|
20
14
|
|
|
21
15
|
export interface VectorIndexOptions {
|
|
22
16
|
db: Db;
|
|
@@ -48,19 +42,6 @@ export interface VectorIndexStats {
|
|
|
48
42
|
seq: number;
|
|
49
43
|
}
|
|
50
44
|
|
|
51
|
-
interface State {
|
|
52
|
-
seq: number;
|
|
53
|
-
nextGen: number;
|
|
54
|
-
segments: SegmentRef[];
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
interface TailEntry {
|
|
58
|
-
seq: number;
|
|
59
|
-
vector: Float32Array;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
/** Ids per lookup (bound as one JSON parameter). */
|
|
63
|
-
const VALIDATE_BATCH = 2000;
|
|
64
45
|
/** Lists probed per segment when the caller passes no `nprobe`. */
|
|
65
46
|
const DEFAULT_NPROBE = 8;
|
|
66
47
|
/**
|
|
@@ -84,26 +65,27 @@ const WRITE_BATCH = 200;
|
|
|
84
65
|
* into one, dropping dead rows.
|
|
85
66
|
*
|
|
86
67
|
* One writer per database is assumed (a Durable Object, a worker, one tab);
|
|
87
|
-
* within an instance, writes are serialized.
|
|
68
|
+
* within an instance, writes are serialized. The tables, the committed state and
|
|
69
|
+
* the in-memory tail belong to the generation catalog (`catalog.ts`); this class decides
|
|
70
|
+
* what to commit and when, builds segments and searches them.
|
|
88
71
|
*/
|
|
89
72
|
export class VectorIndex {
|
|
90
73
|
readonly dims: number;
|
|
91
74
|
readonly metric: Metric;
|
|
92
75
|
/** `{prefix}ids`: its `id` column holds exactly the live ids. Callers may join it in SQL. */
|
|
93
76
|
readonly idTable: string;
|
|
94
|
-
readonly #db: Db;
|
|
95
77
|
readonly #store: SqlSegmentStore;
|
|
78
|
+
readonly #catalog: GenerationCatalog;
|
|
96
79
|
readonly #nlist?: (n: number) => number;
|
|
97
80
|
readonly #autoCompactAt: number;
|
|
98
|
-
readonly #t: { meta: string; ids: string; tail: string };
|
|
99
81
|
|
|
100
|
-
#state: State = { seq: 0, nextGen: 1, segments: [] };
|
|
101
|
-
#tail = new Map<number, TailEntry>();
|
|
102
82
|
#segments: SegmentIndex[] = [];
|
|
103
83
|
#refs = new Map<SegmentIndex, number>();
|
|
104
84
|
#retired: SegmentIndex[] = [];
|
|
105
85
|
#queue: Promise<unknown> = Promise.resolve();
|
|
106
86
|
#maintaining = false;
|
|
87
|
+
/** The running compact() or merge(), for refresh() to wait on. */
|
|
88
|
+
#maintenanceRun: Promise<unknown> | null = null;
|
|
107
89
|
#ready = false;
|
|
108
90
|
/** Bumped before and after each commit that changes the segment list: odd while one runs. */
|
|
109
91
|
#epoch = 0;
|
|
@@ -117,75 +99,54 @@ export class VectorIndex {
|
|
|
117
99
|
}
|
|
118
100
|
this.dims = opts.dims;
|
|
119
101
|
this.metric = opts.metric ?? "cosine";
|
|
120
|
-
this.#db = opts.db;
|
|
121
102
|
this.#store = new SqlSegmentStore(opts.db, opts.prefix, opts.dims, opts.cacheBytes);
|
|
103
|
+
this.#catalog = new GenerationCatalog(opts.db, opts.prefix, this.#store);
|
|
122
104
|
this.#nlist = opts.nlist;
|
|
123
105
|
this.#autoCompactAt = opts.autoCompactAt ?? 4096;
|
|
124
|
-
|
|
125
|
-
this.#t = { meta: `${p}meta`, ids: `${p}ids`, tail: `${p}tail` };
|
|
126
|
-
this.idTable = this.#t.ids;
|
|
106
|
+
this.idTable = this.#catalog.idTable;
|
|
127
107
|
}
|
|
128
108
|
|
|
129
109
|
// ------------------------------------------------------------ lifecycle
|
|
130
110
|
|
|
131
111
|
/** Creates tables if missing, loads state and sweeps rows of crashed maintenance. */
|
|
132
112
|
async init(): Promise<void> {
|
|
133
|
-
|
|
134
|
-
for (const s of [
|
|
135
|
-
`CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,
|
|
136
|
-
`CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,
|
|
137
|
-
`CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,
|
|
138
|
-
`CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,
|
|
139
|
-
`CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,
|
|
140
|
-
...this.#store.schema(),
|
|
141
|
-
]) {
|
|
142
|
-
await this.#db.exec(s);
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
const config = JSON.stringify({ dims: this.dims, metric: this.metric });
|
|
146
|
-
await this.#db.query(
|
|
147
|
-
`INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`,
|
|
148
|
-
[config],
|
|
149
|
-
);
|
|
150
|
-
const [stored] = await this.#db.query<{ v: string }>(
|
|
151
|
-
`SELECT v FROM ${meta} WHERE k = 'config'`,
|
|
152
|
-
);
|
|
153
|
-
if (stored?.v !== config) {
|
|
154
|
-
throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);
|
|
155
|
-
}
|
|
113
|
+
await this.#catalog.create(JSON.stringify({ dims: this.dims, metric: this.metric }));
|
|
156
114
|
await this.refresh();
|
|
157
|
-
|
|
158
|
-
await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));
|
|
115
|
+
await this.#catalog.sweep();
|
|
159
116
|
this.#ready = true;
|
|
160
117
|
}
|
|
161
118
|
|
|
162
|
-
/**
|
|
119
|
+
/**
|
|
120
|
+
* Reloads state, tail and segments from storage (e.g. after another writer changed them).
|
|
121
|
+
* Waits for a running compact() or merge() and runs between writes, so it never mixes the
|
|
122
|
+
* state before a commit with the rows after it (#26).
|
|
123
|
+
*/
|
|
163
124
|
async refresh(): Promise<void> {
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
this.#segments = segments;
|
|
125
|
+
for (;;) {
|
|
126
|
+
while (this.#maintenanceRun) await this.#maintenanceRun.catch(() => {});
|
|
127
|
+
const done = await this.#exclusive(async () => {
|
|
128
|
+
if (this.#maintaining) return false; // one started while we queued: wait for it too
|
|
129
|
+
const snapshot = await this.#catalog.read();
|
|
130
|
+
const known = new Map(this.#segments.map((s) => [s.gen, s]));
|
|
131
|
+
const segments = await Promise.all(
|
|
132
|
+
snapshot.state.segments.map((s) => known.get(s.gen) ?? this.#store.open(s, this.metric)),
|
|
133
|
+
);
|
|
134
|
+
await this.#swapSegments(async () => {
|
|
135
|
+
this.#catalog.adopt(snapshot);
|
|
136
|
+
this.#segments = segments;
|
|
137
|
+
});
|
|
138
|
+
return true;
|
|
139
|
+
});
|
|
140
|
+
if (done) return;
|
|
141
|
+
}
|
|
182
142
|
}
|
|
183
143
|
|
|
184
144
|
get stats(): VectorIndexStats {
|
|
145
|
+
const { state, tail } = this.#catalog;
|
|
185
146
|
return {
|
|
186
|
-
tail:
|
|
187
|
-
segments:
|
|
188
|
-
seq:
|
|
147
|
+
tail: tail.size,
|
|
148
|
+
segments: state.segments.map((s) => ({ ...s })),
|
|
149
|
+
seq: state.seq,
|
|
189
150
|
};
|
|
190
151
|
}
|
|
191
152
|
|
|
@@ -197,53 +158,18 @@ export class VectorIndex {
|
|
|
197
158
|
assertId(it.id);
|
|
198
159
|
return { id: it.id, vector: prepare(it.vector, this.dims, this.metric) };
|
|
199
160
|
});
|
|
200
|
-
const { tail, ids } = this.#t;
|
|
201
161
|
for (let i = 0; i < prepared.length; i += WRITE_BATCH) {
|
|
202
162
|
const batch = prepared.slice(i, i + WRITE_BATCH);
|
|
203
|
-
await this.#exclusive(
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
await inTransaction(this.#db, async (tx) => {
|
|
207
|
-
for (const { id, vector } of batch) {
|
|
208
|
-
await tx.query(
|
|
209
|
-
`INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)
|
|
210
|
-
ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`,
|
|
211
|
-
[id, seq, float32Bytes(vector)],
|
|
212
|
-
);
|
|
213
|
-
await tx.query(
|
|
214
|
-
`INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`,
|
|
215
|
-
[id],
|
|
216
|
-
);
|
|
217
|
-
}
|
|
218
|
-
await this.#saveState(tx, next);
|
|
219
|
-
});
|
|
220
|
-
this.#state = next;
|
|
221
|
-
for (const { id, vector } of batch) this.#tail.set(id, { seq, vector });
|
|
222
|
-
});
|
|
223
|
-
if (this.#tail.size >= this.#autoCompactAt && !this.#maintaining) await this.compact();
|
|
163
|
+
await this.#exclusive(() => this.#catalog.append(batch));
|
|
164
|
+
if (this.#catalog.tail.size >= this.#autoCompactAt && !this.#maintaining)
|
|
165
|
+
await this.compact();
|
|
224
166
|
}
|
|
225
167
|
}
|
|
226
168
|
|
|
227
169
|
async delete(ids: number[]): Promise<void> {
|
|
228
170
|
this.#assertReady();
|
|
229
171
|
ids.forEach(assertId);
|
|
230
|
-
await this.#exclusive(
|
|
231
|
-
for (let i = 0; i < ids.length; i += VALIDATE_BATCH) {
|
|
232
|
-
const chunk = ids.slice(i, i + VALIDATE_BATCH);
|
|
233
|
-
const json = JSON.stringify(chunk);
|
|
234
|
-
await inTransaction(this.#db, async (tx) => {
|
|
235
|
-
await tx.query(
|
|
236
|
-
`DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`,
|
|
237
|
-
[json],
|
|
238
|
-
);
|
|
239
|
-
await tx.query(
|
|
240
|
-
`DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`,
|
|
241
|
-
[json],
|
|
242
|
-
);
|
|
243
|
-
});
|
|
244
|
-
for (const id of chunk) this.#tail.delete(id);
|
|
245
|
-
}
|
|
246
|
-
});
|
|
172
|
+
await this.#exclusive(() => this.#catalog.remove(ids));
|
|
247
173
|
}
|
|
248
174
|
|
|
249
175
|
// ------------------------------------------------------------ search
|
|
@@ -268,7 +194,7 @@ export class VectorIndex {
|
|
|
268
194
|
opts.signal?.throwIfAborted();
|
|
269
195
|
// Tail: exact, from memory.
|
|
270
196
|
const tailTop = new TopK<number>(k);
|
|
271
|
-
for (const [id, { vector }] of this.#tail) {
|
|
197
|
+
for (const [id, { vector }] of this.#catalog.tail) {
|
|
272
198
|
if (filter && !filter(id)) continue;
|
|
273
199
|
tailTop.push(distance(this.metric, q, 0, vector, 0, d), id);
|
|
274
200
|
}
|
|
@@ -300,7 +226,7 @@ export class VectorIndex {
|
|
|
300
226
|
const rounds = pending.map((p) => {
|
|
301
227
|
// Tail ids are never live in a segment; known-dead rows are skipped too.
|
|
302
228
|
const keep = (id: number) =>
|
|
303
|
-
!this.#tail.has(id) &&
|
|
229
|
+
!this.#catalog.tail.has(id) &&
|
|
304
230
|
!(gens.has(id) && gens.get(id) !== p.seg.gen) &&
|
|
305
231
|
(!filter || filter(id));
|
|
306
232
|
return { p, cands: p.seg.filterList(q, p.list, p.data, p.limit, keep) };
|
|
@@ -309,7 +235,7 @@ export class VectorIndex {
|
|
|
309
235
|
.flatMap((r) => r.cands.map((c) => c.id))
|
|
310
236
|
.filter((id) => !gens.has(id));
|
|
311
237
|
if (unknown.length > 0) {
|
|
312
|
-
const found = await this.#generations(unknown);
|
|
238
|
+
const found = await this.#catalog.generations(unknown);
|
|
313
239
|
for (const id of unknown) gens.set(id, found.get(id));
|
|
314
240
|
opts.signal?.throwIfAborted();
|
|
315
241
|
}
|
|
@@ -360,11 +286,11 @@ export class VectorIndex {
|
|
|
360
286
|
ids.forEach(assertId);
|
|
361
287
|
return this.#read(async (segments) => {
|
|
362
288
|
const out = new Map<number, Float32Array>();
|
|
363
|
-
const gens = await this.#generations(ids);
|
|
289
|
+
const gens = await this.#catalog.generations(ids);
|
|
364
290
|
const wanted = new Map<number, Set<number>>(); // gen -> ids
|
|
365
291
|
for (const [id, gen] of gens) {
|
|
366
292
|
if (gen === 0) {
|
|
367
|
-
const entry = this.#tail.get(id);
|
|
293
|
+
const entry = this.#catalog.tail.get(id);
|
|
368
294
|
if (entry) out.set(id, entry.vector.slice());
|
|
369
295
|
continue;
|
|
370
296
|
}
|
|
@@ -399,8 +325,8 @@ export class VectorIndex {
|
|
|
399
325
|
this.#assertReady();
|
|
400
326
|
return this.#maintenance(async () => {
|
|
401
327
|
const snapshot = await this.#exclusive(async () => ({
|
|
402
|
-
maxSeq: this.#state.seq,
|
|
403
|
-
entries: [...this.#tail.entries()],
|
|
328
|
+
maxSeq: this.#catalog.state.seq,
|
|
329
|
+
entries: [...this.#catalog.tail.entries()],
|
|
404
330
|
}));
|
|
405
331
|
if (snapshot.entries.length === 0) return 0;
|
|
406
332
|
|
|
@@ -415,26 +341,9 @@ export class VectorIndex {
|
|
|
415
341
|
|
|
416
342
|
await this.#exclusive(() =>
|
|
417
343
|
this.#swapSegments(async () => {
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
...this.#state,
|
|
421
|
-
nextGen: ref.gen + 1,
|
|
422
|
-
segments: [...this.#state.segments, ref],
|
|
423
|
-
};
|
|
424
|
-
const { ids: idsT, tail } = this.#t;
|
|
425
|
-
await inTransaction(this.#db, async (tx) => {
|
|
426
|
-
await this.#store.write(tx, ref.gen, built);
|
|
427
|
-
await tx.query(
|
|
428
|
-
`UPDATE ${idsT} SET gen = ? WHERE gen = 0
|
|
429
|
-
AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`,
|
|
430
|
-
[ref.gen, snapshot.maxSeq],
|
|
431
|
-
);
|
|
432
|
-
await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [snapshot.maxSeq]);
|
|
433
|
-
await this.#saveState(tx, next);
|
|
344
|
+
await this.#catalog.seal(snapshot.maxSeq, built, n, (ref) => {
|
|
345
|
+
this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
|
|
434
346
|
});
|
|
435
|
-
this.#state = next;
|
|
436
|
-
for (const [id, e] of this.#tail) if (e.seq <= snapshot.maxSeq) this.#tail.delete(id);
|
|
437
|
-
this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
|
|
438
347
|
}),
|
|
439
348
|
);
|
|
440
349
|
return n;
|
|
@@ -459,7 +368,7 @@ export class VectorIndex {
|
|
|
459
368
|
if (seg.counts[c] === 0) continue;
|
|
460
369
|
const [data] = (await this.#store.lists(seg, [c])) as [ListData];
|
|
461
370
|
const floats = await this.#store.floats(seg, c);
|
|
462
|
-
const live = await this.#generations(Array.from(data.ids));
|
|
371
|
+
const live = await this.#catalog.generations(Array.from(data.ids));
|
|
463
372
|
for (let r = 0; r < data.ids.length; r++) {
|
|
464
373
|
const id = data.ids[r];
|
|
465
374
|
if (live.get(id) !== seg.gen) continue;
|
|
@@ -478,29 +387,13 @@ export class VectorIndex {
|
|
|
478
387
|
|
|
479
388
|
await this.#exclusive(() =>
|
|
480
389
|
this.#swapSegments(async () => {
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
};
|
|
488
|
-
await inTransaction(this.#db, async (tx) => {
|
|
489
|
-
if (ref && built) {
|
|
490
|
-
await this.#store.write(tx, ref.gen, built);
|
|
491
|
-
await tx.query(
|
|
492
|
-
`UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`,
|
|
493
|
-
[ref.gen, JSON.stringify(oldGens)],
|
|
494
|
-
);
|
|
495
|
-
}
|
|
496
|
-
await this.#saveState(tx, next);
|
|
390
|
+
await this.#catalog.replace(oldGens, built, n, (ref) => {
|
|
391
|
+
this.#segments = [
|
|
392
|
+
...this.#segments.filter((s) => !oldGens.includes(s.gen)),
|
|
393
|
+
...(ref && built ? [this.#segmentOf(ref, built)] : []),
|
|
394
|
+
];
|
|
395
|
+
this.#retired.push(...old);
|
|
497
396
|
});
|
|
498
|
-
this.#state = next;
|
|
499
|
-
this.#segments = [
|
|
500
|
-
...this.#segments.filter((s) => !oldGens.includes(s.gen)),
|
|
501
|
-
...(ref && built ? [this.#segmentOf(ref, built)] : []),
|
|
502
|
-
];
|
|
503
|
-
this.#retired.push(...old);
|
|
504
397
|
}),
|
|
505
398
|
);
|
|
506
399
|
await this.#reap();
|
|
@@ -526,43 +419,17 @@ export class VectorIndex {
|
|
|
526
419
|
});
|
|
527
420
|
}
|
|
528
421
|
|
|
529
|
-
#refOf(built: BuiltSegment, count: number): SegmentRef {
|
|
530
|
-
return { gen: this.#state.nextGen, count, nlist: built.nlist };
|
|
531
|
-
}
|
|
532
|
-
|
|
533
422
|
#segmentOf(ref: SegmentRef, built: BuiltSegment): SegmentIndex {
|
|
534
423
|
return new SegmentIndex(ref.gen, this.dims, this.metric, built.centroids, built.counts);
|
|
535
424
|
}
|
|
536
425
|
|
|
537
|
-
async #generations(ids: number[]): Promise<Map<number, number>> {
|
|
538
|
-
const out = new Map<number, number>();
|
|
539
|
-
const unique = [...new Set(ids)];
|
|
540
|
-
for (let i = 0; i < unique.length; i += VALIDATE_BATCH) {
|
|
541
|
-
const rows = await this.#db.query<{ id: number; gen: number }>(
|
|
542
|
-
`SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`,
|
|
543
|
-
[JSON.stringify(unique.slice(i, i + VALIDATE_BATCH))],
|
|
544
|
-
);
|
|
545
|
-
for (const r of rows) out.set(Number(r.id), Number(r.gen));
|
|
546
|
-
}
|
|
547
|
-
return out;
|
|
548
|
-
}
|
|
549
|
-
|
|
550
|
-
async #saveState(tx: Db, state: State): Promise<void> {
|
|
551
|
-
await tx.query(
|
|
552
|
-
`INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)
|
|
553
|
-
ON CONFLICT(k) DO UPDATE SET v = excluded.v`,
|
|
554
|
-
[JSON.stringify(state)],
|
|
555
|
-
);
|
|
556
|
-
}
|
|
557
|
-
|
|
558
426
|
/** Deletes the rows of retired segments once no search is reading them. */
|
|
559
427
|
async #reap(): Promise<void> {
|
|
560
428
|
const ready = this.#retired.filter((s) => (this.#refs.get(s) ?? 0) === 0);
|
|
561
429
|
if (ready.length === 0) return;
|
|
562
430
|
this.#retired = this.#retired.filter((s) => !ready.includes(s));
|
|
563
431
|
for (const s of ready) this.#refs.delete(s);
|
|
564
|
-
|
|
565
|
-
await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));
|
|
432
|
+
await this.#catalog.dropGenerations(ready.map((s) => s.gen));
|
|
566
433
|
}
|
|
567
434
|
|
|
568
435
|
/**
|
|
@@ -604,11 +471,16 @@ export class VectorIndex {
|
|
|
604
471
|
async #maintenance<T>(fn: () => Promise<T>): Promise<T> {
|
|
605
472
|
if (this.#maintaining) throw new Error("vector-index: compact or merge already running");
|
|
606
473
|
this.#maintaining = true;
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
474
|
+
const run = (async () => {
|
|
475
|
+
try {
|
|
476
|
+
return await fn();
|
|
477
|
+
} finally {
|
|
478
|
+
this.#maintaining = false;
|
|
479
|
+
this.#maintenanceRun = null;
|
|
480
|
+
}
|
|
481
|
+
})();
|
|
482
|
+
this.#maintenanceRun = run;
|
|
483
|
+
return run;
|
|
612
484
|
}
|
|
613
485
|
|
|
614
486
|
#assertReady() {
|
|
@@ -624,9 +496,5 @@ function assertId(id: number) {
|
|
|
624
496
|
/** Drops every table of the index under `prefix`, in one transaction. Needs no dims or metric. */
|
|
625
497
|
export async function dropVectorIndex(db: Db, prefix: string): Promise<void> {
|
|
626
498
|
if (!PREFIX_PATTERN.test(prefix)) throw new Error(`vector-index: bad prefix ${prefix}`);
|
|
627
|
-
await
|
|
628
|
-
for (const t of ["meta", "ids", "tail", "centroids", "seg", "f32"]) {
|
|
629
|
-
await tx.exec(`DROP TABLE IF EXISTS ${prefix}${t}`);
|
|
630
|
-
}
|
|
631
|
-
});
|
|
499
|
+
await dropCatalog(db, prefix);
|
|
632
500
|
}
|