@statewalker/db-sqlite-index 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -4
- package/dist/ngram.d.mts.map +1 -1
- package/dist/ngram.mjs +83 -3
- package/dist/ngram.mjs.map +1 -1
- package/dist/vector.d.mts +12 -3
- package/dist/vector.d.mts.map +1 -1
- package/dist/vector.mjs +399 -195
- package/dist/vector.mjs.map +1 -1
- package/package.json +16 -8
- package/src/ngram/fold.ts +44 -0
- package/src/ngram/fts5-backend.ts +4 -0
- package/src/ngram/ngram-store.ts +37 -0
- package/src/ngram/plan-query.ts +8 -1
- package/src/ngram/postings-backend.ts +1 -0
- package/src/ngram/regex-literals.ts +5 -2
- package/src/vector/catalog.ts +285 -0
- package/src/vector/math.ts +6 -1
- package/src/vector/segment.ts +48 -6
- package/src/vector/sql-segment-store.ts +6 -1
- package/src/vector/types.ts +4 -1
- package/src/vector/vector-index.ts +130 -216
- package/src/ngram/do-smoke.ts +0 -46
- package/src/ngram/fold.test.ts +0 -50
- package/src/ngram/json-chunks.test.ts +0 -23
- package/src/ngram/ngram-store.test.ts +0 -291
- package/src/ngram/plan-query.test.ts +0 -141
- package/src/ngram/trigrams.test.ts +0 -28
- package/src/vector/auto-compact.test.ts +0 -28
- package/src/vector/do-smoke.ts +0 -39
- package/src/vector/segment.test.ts +0 -61
- package/src/vector/test-fixtures.ts +0 -64
- package/src/vector/vector-index.test.ts +0 -105
- package/src/vector/vector-lifecycle.test.ts +0 -171
package/dist/vector.mjs
CHANGED
|
@@ -15,7 +15,10 @@ function prepare(v, dims, metric) {
|
|
|
15
15
|
if (v.length !== dims) throw new Error(`vector-index: expected ${dims} dims, got ${v.length}`);
|
|
16
16
|
const f = v instanceof Float32Array ? v : Float32Array.from(v);
|
|
17
17
|
for (let i = 0; i < f.length; i++) if (!Number.isFinite(f[i])) throw new Error("vector-index: vector contains a non-finite value");
|
|
18
|
-
|
|
18
|
+
if (metric !== "cosine") return f.slice();
|
|
19
|
+
const out = normalize(f);
|
|
20
|
+
if (!out.some((x) => x !== 0)) throw new Error("vector-index: a zero vector has no direction under the cosine metric");
|
|
21
|
+
return out;
|
|
19
22
|
}
|
|
20
23
|
/** Distance between two float vectors at offsets; lower is closer. */
|
|
21
24
|
function distance(metric, a, ao, b, bo, d) {
|
|
@@ -348,6 +351,8 @@ function parseList(bytes, count, dims) {
|
|
|
348
351
|
};
|
|
349
352
|
}
|
|
350
353
|
/** The in-memory part of a segment: centroids and list sizes. List blocks are fetched by the caller. */
|
|
354
|
+
/** The share of a segment's lists that {@link SegmentIndex.probe} may widen to for `minRows`. */
|
|
355
|
+
const MAX_WIDEN_FRACTION = 1 / 3;
|
|
351
356
|
var SegmentIndex = class {
|
|
352
357
|
gen;
|
|
353
358
|
dims;
|
|
@@ -364,21 +369,66 @@ var SegmentIndex = class {
|
|
|
364
369
|
get nlist() {
|
|
365
370
|
return this.counts.length;
|
|
366
371
|
}
|
|
367
|
-
/**
|
|
368
|
-
|
|
372
|
+
/**
|
|
373
|
+
* The non-empty lists closest to the query, in centroid order: at least the
|
|
374
|
+
* `nprobe` nearest, then more until the probed lists hold `minRows` rows (or
|
|
375
|
+
* none are left). `minRows` 0 probes exactly `nprobe` lists. The widening
|
|
376
|
+
* stops at a third of the lists ({@link MAX_WIDEN_FRACTION}), so a large `k`
|
|
377
|
+
* never turns a search into a scan of every list.
|
|
378
|
+
*/
|
|
379
|
+
probe(query, nprobe, minRows = 0) {
|
|
380
|
+
const maxLists = Math.max(nprobe, Math.ceil(this.nlist * MAX_WIDEN_FRACTION));
|
|
369
381
|
const m = this.metric === "l2" ? "l2" : "dot";
|
|
370
|
-
const
|
|
382
|
+
const scored = [];
|
|
371
383
|
for (let c = 0; c < this.nlist; c++) {
|
|
372
384
|
if (this.counts[c] === 0) continue;
|
|
373
|
-
|
|
385
|
+
scored.push({
|
|
386
|
+
d: distance(m, query, 0, this.centroids, c * this.dims, this.dims),
|
|
387
|
+
c
|
|
388
|
+
});
|
|
374
389
|
}
|
|
375
|
-
|
|
390
|
+
scored.sort((a, b) => a.d - b.d || a.c - b.c);
|
|
391
|
+
const out = [];
|
|
392
|
+
let rows = 0;
|
|
393
|
+
for (const { c } of scored) {
|
|
394
|
+
if (out.length >= nprobe && (rows >= minRows || out.length >= maxLists)) break;
|
|
395
|
+
out.push(c);
|
|
396
|
+
rows += this.counts[c];
|
|
397
|
+
}
|
|
398
|
+
return out;
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* Whether stage 1 passes a list of `n` rows whole (in row order, not ranked):
|
|
402
|
+
* a list no longer than `limit`, or a flat segment's only list of at most
|
|
403
|
+
* {@link MIN_TRAIN} rows. Callers must not cut such a list by position.
|
|
404
|
+
*/
|
|
405
|
+
passesWhole(n, limit) {
|
|
406
|
+
return n <= limit || this.nlist === 1 && n <= 2048;
|
|
376
407
|
}
|
|
377
408
|
/**
|
|
378
409
|
* Stage 1: the `limit` rows of a list with the best estimated distance, from
|
|
379
410
|
* residual sign bits plus the stored residual norm and centroid projection.
|
|
411
|
+
*
|
|
412
|
+
* A flat segment (one list of at most {@link MIN_TRAIN} rows) passes every row:
|
|
413
|
+
* its centroid is the mean of all vectors, far from most of them on clustered
|
|
414
|
+
* data, so the 1-bit estimate is weakest exactly there (issue #40), while
|
|
415
|
+
* int8-scoring the whole list costs no more than scanning the tail. A list
|
|
416
|
+
* that does not exceed `limit` also passes whole, without estimating.
|
|
380
417
|
*/
|
|
381
418
|
filterList(query, c, data, limit, keep) {
|
|
419
|
+
const n = data.ids.length;
|
|
420
|
+
if (this.passesWhole(n, limit)) {
|
|
421
|
+
const all = [];
|
|
422
|
+
for (let r = 0; r < n; r++) {
|
|
423
|
+
const id = data.ids[r];
|
|
424
|
+
if (!keep || keep(id)) all.push({
|
|
425
|
+
list: c,
|
|
426
|
+
row: r,
|
|
427
|
+
id
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
return all;
|
|
431
|
+
}
|
|
382
432
|
const d = this.dims;
|
|
383
433
|
const w = wordsFor(d);
|
|
384
434
|
const co = c * d;
|
|
@@ -391,7 +441,7 @@ var SegmentIndex = class {
|
|
|
391
441
|
const angle = Math.PI / d;
|
|
392
442
|
const l2 = this.metric === "l2";
|
|
393
443
|
const top = new TopK(limit);
|
|
394
|
-
for (let r = 0; r <
|
|
444
|
+
for (let r = 0; r < n; r++) {
|
|
395
445
|
const id = data.ids[r];
|
|
396
446
|
if (keep && !keep(id)) continue;
|
|
397
447
|
const rr = qn * data.rnorm[r] * Math.cos(angle * hamming(qb, data.bits, r * w, w));
|
|
@@ -431,19 +481,23 @@ const PART_BYTES = 1048576;
|
|
|
431
481
|
* float32 blocks split into parts. Reads go through an LRU cache: a
|
|
432
482
|
* generation is immutable, so cached blocks never go stale.
|
|
433
483
|
*/
|
|
434
|
-
var SqlSegmentStore = class {
|
|
484
|
+
var SqlSegmentStore = class SqlSegmentStore {
|
|
435
485
|
#db;
|
|
436
486
|
#dims;
|
|
437
487
|
#t;
|
|
438
488
|
#cache;
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
this.#t = {
|
|
489
|
+
/** The names of the tables segment rows live in, under `prefix`. */
|
|
490
|
+
static tables(prefix) {
|
|
491
|
+
return {
|
|
443
492
|
centroids: `${prefix}centroids`,
|
|
444
493
|
seg: `${prefix}seg`,
|
|
445
494
|
f32: `${prefix}f32`
|
|
446
495
|
};
|
|
496
|
+
}
|
|
497
|
+
constructor(db, prefix, dims, cacheBytes = 67108864) {
|
|
498
|
+
this.#db = db;
|
|
499
|
+
this.#dims = dims;
|
|
500
|
+
this.#t = SqlSegmentStore.tables(prefix);
|
|
447
501
|
this.#cache = new Lru(cacheBytes);
|
|
448
502
|
}
|
|
449
503
|
schema() {
|
|
@@ -597,9 +651,229 @@ var Lru = class {
|
|
|
597
651
|
}
|
|
598
652
|
};
|
|
599
653
|
//#endregion
|
|
654
|
+
//#region src/vector/catalog.ts
|
|
655
|
+
/** Ids per statement (bound as one JSON parameter). */
|
|
656
|
+
const ID_BATCH = 2e3;
|
|
657
|
+
const EMPTY = {
|
|
658
|
+
seq: 0,
|
|
659
|
+
nextGen: 1,
|
|
660
|
+
segments: []
|
|
661
|
+
};
|
|
662
|
+
/**
|
|
663
|
+
* The generation catalog of a vector index: the committed state, the `{p}ids` table (which
|
|
664
|
+
* generation holds the live copy of each id, 0 = the tail), the `{p}tail` rows, and the in-memory
|
|
665
|
+
* copy of the state and the tail.
|
|
666
|
+
*
|
|
667
|
+
* It is the only place that commits a state change, and every commit updates the copy in the
|
|
668
|
+
* same synchronous step as it returns from the transaction, so the copy always matches storage.
|
|
669
|
+
* It also creates, sweeps and drops every table under the prefix, asking the segment store for
|
|
670
|
+
* its own.
|
|
671
|
+
*
|
|
672
|
+
* Callers serialize the commits (one at a time per instance); reads of `state` and `tail` may
|
|
673
|
+
* happen at any time and see either the copy before a commit or the copy after it.
|
|
674
|
+
*/
|
|
675
|
+
var GenerationCatalog = class {
|
|
676
|
+
/** `{p}ids`: its `id` column holds exactly the live ids. */
|
|
677
|
+
idTable;
|
|
678
|
+
#db;
|
|
679
|
+
#store;
|
|
680
|
+
#t;
|
|
681
|
+
#state = EMPTY;
|
|
682
|
+
#tail = /* @__PURE__ */ new Map();
|
|
683
|
+
constructor(db, prefix, store) {
|
|
684
|
+
this.#db = db;
|
|
685
|
+
this.#store = store;
|
|
686
|
+
this.#t = catalogTables(prefix);
|
|
687
|
+
this.idTable = this.#t.ids;
|
|
688
|
+
}
|
|
689
|
+
/** The committed state, as of the last commit or `adopt()`. Never mutated: replaced. */
|
|
690
|
+
get state() {
|
|
691
|
+
return this.#state;
|
|
692
|
+
}
|
|
693
|
+
/** The tail rows in memory, by id. */
|
|
694
|
+
get tail() {
|
|
695
|
+
return this.#tail;
|
|
696
|
+
}
|
|
697
|
+
/**
|
|
698
|
+
* Creates every table if missing and checks the stored config (dims and metric) is `config`:
|
|
699
|
+
* the first open stores it, later opens must match.
|
|
700
|
+
*/
|
|
701
|
+
async create(config) {
|
|
702
|
+
const { meta, ids, tail } = this.#t;
|
|
703
|
+
for (const s of [
|
|
704
|
+
`CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,
|
|
705
|
+
`CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,
|
|
706
|
+
`CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,
|
|
707
|
+
`CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,
|
|
708
|
+
`CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,
|
|
709
|
+
...this.#store.schema()
|
|
710
|
+
]) await this.#db.exec(s);
|
|
711
|
+
await this.#db.query(`INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`, [config]);
|
|
712
|
+
const [stored] = await this.#db.query(`SELECT v FROM ${meta} WHERE k = 'config'`);
|
|
713
|
+
if (stored?.v !== config) throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);
|
|
714
|
+
}
|
|
715
|
+
/** Deletes segment rows of every generation the committed state does not list (crash leftovers). */
|
|
716
|
+
async sweep() {
|
|
717
|
+
const live = this.#state.segments.map((s) => s.gen);
|
|
718
|
+
await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));
|
|
719
|
+
}
|
|
720
|
+
/** Deletes the segment rows of retired generations. */
|
|
721
|
+
async dropGenerations(gens) {
|
|
722
|
+
await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));
|
|
723
|
+
}
|
|
724
|
+
/** Reads the committed state and the tail rows, without installing them. */
|
|
725
|
+
async read() {
|
|
726
|
+
const [row] = await this.#db.query(`SELECT v FROM ${this.#t.meta} WHERE k = 'state'`);
|
|
727
|
+
const state = row ? JSON.parse(row.v) : EMPTY;
|
|
728
|
+
const tail = /* @__PURE__ */ new Map();
|
|
729
|
+
const rows = await this.#db.query(`SELECT id, seq, vec FROM ${this.#t.tail}`);
|
|
730
|
+
for (const r of rows) tail.set(Number(r.id), {
|
|
731
|
+
seq: Number(r.seq),
|
|
732
|
+
vector: toFloat32(r.vec)
|
|
733
|
+
});
|
|
734
|
+
return {
|
|
735
|
+
state,
|
|
736
|
+
tail
|
|
737
|
+
};
|
|
738
|
+
}
|
|
739
|
+
/** Installs a snapshot from `read()` as the copy. No commit may run between the two. */
|
|
740
|
+
adopt(snapshot) {
|
|
741
|
+
this.#state = snapshot.state;
|
|
742
|
+
this.#tail = snapshot.tail;
|
|
743
|
+
}
|
|
744
|
+
/** The generation holding the live copy of each id; deleted or unknown ids are absent. */
|
|
745
|
+
async generations(ids) {
|
|
746
|
+
const out = /* @__PURE__ */ new Map();
|
|
747
|
+
const unique = [...new Set(ids)];
|
|
748
|
+
for (let i = 0; i < unique.length; i += ID_BATCH) {
|
|
749
|
+
const rows = await this.#db.query(`SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [JSON.stringify(unique.slice(i, i + ID_BATCH))]);
|
|
750
|
+
for (const r of rows) out.set(Number(r.id), Number(r.gen));
|
|
751
|
+
}
|
|
752
|
+
return out;
|
|
753
|
+
}
|
|
754
|
+
/** Writes `items` to the tail as one new batch (the next seq), in one transaction. */
|
|
755
|
+
async append(items) {
|
|
756
|
+
const { tail, ids } = this.#t;
|
|
757
|
+
const seq = this.#state.seq + 1;
|
|
758
|
+
const next = {
|
|
759
|
+
...this.#state,
|
|
760
|
+
seq
|
|
761
|
+
};
|
|
762
|
+
await inTransaction(this.#db, async (tx) => {
|
|
763
|
+
for (const { id, vector } of items) {
|
|
764
|
+
await tx.query(`INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)
|
|
765
|
+
ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`, [
|
|
766
|
+
id,
|
|
767
|
+
seq,
|
|
768
|
+
float32Bytes(vector)
|
|
769
|
+
]);
|
|
770
|
+
await tx.query(`INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`, [id]);
|
|
771
|
+
}
|
|
772
|
+
await this.#save(tx, next);
|
|
773
|
+
});
|
|
774
|
+
this.#state = next;
|
|
775
|
+
for (const { id, vector } of items) this.#tail.set(id, {
|
|
776
|
+
seq,
|
|
777
|
+
vector
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
/** Removes `ids` from the tail and from the live ids, one transaction per batch. */
|
|
781
|
+
async remove(ids) {
|
|
782
|
+
for (let i = 0; i < ids.length; i += ID_BATCH) {
|
|
783
|
+
const chunk = ids.slice(i, i + ID_BATCH);
|
|
784
|
+
const json = JSON.stringify(chunk);
|
|
785
|
+
await inTransaction(this.#db, async (tx) => {
|
|
786
|
+
await tx.query(`DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
|
|
787
|
+
await tx.query(`DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
|
|
788
|
+
});
|
|
789
|
+
for (const id of chunk) this.#tail.delete(id);
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
/**
|
|
793
|
+
* Commits `built` (sealed from the tail rows up to `maxSeq`, `count` vectors) as a new
|
|
794
|
+
* generation: writes its rows, moves those ids to it and deletes those tail rows. Tail rows of
|
|
795
|
+
* later batches stay. `install` runs with the new ref in the same synchronous step as the copy
|
|
796
|
+
* update, for the caller's own in-memory segment list.
|
|
797
|
+
*/
|
|
798
|
+
async seal(maxSeq, built, count, install) {
|
|
799
|
+
const ref = this.#refOf(built, count);
|
|
800
|
+
const next = {
|
|
801
|
+
...this.#state,
|
|
802
|
+
nextGen: ref.gen + 1,
|
|
803
|
+
segments: [...this.#state.segments, ref]
|
|
804
|
+
};
|
|
805
|
+
const { ids, tail } = this.#t;
|
|
806
|
+
await inTransaction(this.#db, async (tx) => {
|
|
807
|
+
await this.#store.write(tx, ref.gen, built);
|
|
808
|
+
await tx.query(`UPDATE ${ids} SET gen = ? WHERE gen = 0
|
|
809
|
+
AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`, [ref.gen, maxSeq]);
|
|
810
|
+
await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [maxSeq]);
|
|
811
|
+
await this.#save(tx, next);
|
|
812
|
+
});
|
|
813
|
+
this.#state = next;
|
|
814
|
+
for (const [id, e] of this.#tail) if (e.seq <= maxSeq) this.#tail.delete(id);
|
|
815
|
+
install(ref);
|
|
816
|
+
return ref;
|
|
817
|
+
}
|
|
818
|
+
/**
|
|
819
|
+
* Replaces the generations `oldGens` with `built` (`count` live vectors), or with nothing when
|
|
820
|
+
* `built` is null: writes its rows, moves the ids of the old generations to it and drops them
|
|
821
|
+
* from the state. Their rows stay until `dropGenerations()`. `install` runs as in `seal()`.
|
|
822
|
+
*/
|
|
823
|
+
async replace(oldGens, built, count, install) {
|
|
824
|
+
const ref = built ? this.#refOf(built, count) : null;
|
|
825
|
+
const kept = this.#state.segments.filter((s) => !oldGens.includes(s.gen));
|
|
826
|
+
const next = {
|
|
827
|
+
...this.#state,
|
|
828
|
+
nextGen: ref ? ref.gen + 1 : this.#state.nextGen,
|
|
829
|
+
segments: ref ? [...kept, ref] : kept
|
|
830
|
+
};
|
|
831
|
+
await inTransaction(this.#db, async (tx) => {
|
|
832
|
+
if (ref && built) {
|
|
833
|
+
await this.#store.write(tx, ref.gen, built);
|
|
834
|
+
await tx.query(`UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`, [ref.gen, JSON.stringify(oldGens)]);
|
|
835
|
+
}
|
|
836
|
+
await this.#save(tx, next);
|
|
837
|
+
});
|
|
838
|
+
this.#state = next;
|
|
839
|
+
install(ref);
|
|
840
|
+
return ref;
|
|
841
|
+
}
|
|
842
|
+
#refOf(built, count) {
|
|
843
|
+
return {
|
|
844
|
+
gen: this.#state.nextGen,
|
|
845
|
+
count,
|
|
846
|
+
nlist: built.nlist
|
|
847
|
+
};
|
|
848
|
+
}
|
|
849
|
+
async #save(tx, state) {
|
|
850
|
+
await tx.query(`INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)
|
|
851
|
+
ON CONFLICT(k) DO UPDATE SET v = excluded.v`, [JSON.stringify(state)]);
|
|
852
|
+
}
|
|
853
|
+
};
|
|
854
|
+
function catalogTables(prefix) {
|
|
855
|
+
return {
|
|
856
|
+
meta: `${prefix}meta`,
|
|
857
|
+
ids: `${prefix}ids`,
|
|
858
|
+
tail: `${prefix}tail`
|
|
859
|
+
};
|
|
860
|
+
}
|
|
861
|
+
/** Drops every table under `prefix` (the catalog's and the segment store's), in one transaction. */
|
|
862
|
+
async function dropCatalog(db, prefix) {
|
|
863
|
+
const tables = [...Object.values(catalogTables(prefix)), ...Object.values(SqlSegmentStore.tables(prefix))];
|
|
864
|
+
await inTransaction(db, async (tx) => {
|
|
865
|
+
for (const t of tables) await tx.exec(`DROP TABLE IF EXISTS ${t}`);
|
|
866
|
+
});
|
|
867
|
+
}
|
|
868
|
+
//#endregion
|
|
600
869
|
//#region src/vector/vector-index.ts
|
|
601
|
-
/**
|
|
602
|
-
const
|
|
870
|
+
/** Lists probed per segment when the caller passes no `nprobe`. */
|
|
871
|
+
const DEFAULT_NPROBE = 8;
|
|
872
|
+
/**
|
|
873
|
+
* Without an explicit `nprobe`, probe lists until they hold this many times
|
|
874
|
+
* `k × oversample` rows (#67).
|
|
875
|
+
*/
|
|
876
|
+
const PROBE_ROWS_FACTOR = 16;
|
|
603
877
|
/** Upserted rows per transaction. */
|
|
604
878
|
const WRITE_BATCH = 200;
|
|
605
879
|
/**
|
|
@@ -615,29 +889,26 @@ const WRITE_BATCH = 200;
|
|
|
615
889
|
* into one, dropping dead rows.
|
|
616
890
|
*
|
|
617
891
|
* One writer per database is assumed (a Durable Object, a worker, one tab);
|
|
618
|
-
* within an instance, writes are serialized.
|
|
892
|
+
* within an instance, writes are serialized. The tables, the committed state and
|
|
893
|
+
* the in-memory tail belong to the generation catalog (`catalog.ts`); this class decides
|
|
894
|
+
* what to commit and when, builds segments and searches them.
|
|
619
895
|
*/
|
|
620
896
|
var VectorIndex = class {
|
|
621
897
|
dims;
|
|
622
898
|
metric;
|
|
623
899
|
/** `{prefix}ids`: its `id` column holds exactly the live ids. Callers may join it in SQL. */
|
|
624
900
|
idTable;
|
|
625
|
-
#db;
|
|
626
901
|
#store;
|
|
902
|
+
#catalog;
|
|
627
903
|
#nlist;
|
|
628
904
|
#autoCompactAt;
|
|
629
|
-
#t;
|
|
630
|
-
#state = {
|
|
631
|
-
seq: 0,
|
|
632
|
-
nextGen: 1,
|
|
633
|
-
segments: []
|
|
634
|
-
};
|
|
635
|
-
#tail = /* @__PURE__ */ new Map();
|
|
636
905
|
#segments = [];
|
|
637
906
|
#refs = /* @__PURE__ */ new Map();
|
|
638
907
|
#retired = [];
|
|
639
908
|
#queue = Promise.resolve();
|
|
640
909
|
#maintaining = false;
|
|
910
|
+
/** The running compact() or merge(), for refresh() to wait on. */
|
|
911
|
+
#maintenanceRun = null;
|
|
641
912
|
#ready = false;
|
|
642
913
|
/** Bumped before and after each commit that changes the segment list: odd while one runs. */
|
|
643
914
|
#epoch = 0;
|
|
@@ -646,66 +917,49 @@ var VectorIndex = class {
|
|
|
646
917
|
if (!Number.isInteger(opts.dims) || opts.dims < 1) throw new Error("vector-index: dims must be a positive integer");
|
|
647
918
|
this.dims = opts.dims;
|
|
648
919
|
this.metric = opts.metric ?? "cosine";
|
|
649
|
-
this.#db = opts.db;
|
|
650
920
|
this.#store = new SqlSegmentStore(opts.db, opts.prefix, opts.dims, opts.cacheBytes);
|
|
921
|
+
this.#catalog = new GenerationCatalog(opts.db, opts.prefix, this.#store);
|
|
651
922
|
this.#nlist = opts.nlist;
|
|
652
923
|
this.#autoCompactAt = opts.autoCompactAt ?? 4096;
|
|
653
|
-
|
|
654
|
-
this.#t = {
|
|
655
|
-
meta: `${p}meta`,
|
|
656
|
-
ids: `${p}ids`,
|
|
657
|
-
tail: `${p}tail`
|
|
658
|
-
};
|
|
659
|
-
this.idTable = this.#t.ids;
|
|
924
|
+
this.idTable = this.#catalog.idTable;
|
|
660
925
|
}
|
|
661
926
|
/** Creates tables if missing, loads state and sweeps rows of crashed maintenance. */
|
|
662
927
|
async init() {
|
|
663
|
-
|
|
664
|
-
for (const s of [
|
|
665
|
-
`CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,
|
|
666
|
-
`CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,
|
|
667
|
-
`CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,
|
|
668
|
-
`CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,
|
|
669
|
-
`CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,
|
|
670
|
-
...this.#store.schema()
|
|
671
|
-
]) await this.#db.exec(s);
|
|
672
|
-
const config = JSON.stringify({
|
|
928
|
+
await this.#catalog.create(JSON.stringify({
|
|
673
929
|
dims: this.dims,
|
|
674
930
|
metric: this.metric
|
|
675
|
-
});
|
|
676
|
-
await this.#db.query(`INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`, [config]);
|
|
677
|
-
const [stored] = await this.#db.query(`SELECT v FROM ${meta} WHERE k = 'config'`);
|
|
678
|
-
if (stored?.v !== config) throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);
|
|
931
|
+
}));
|
|
679
932
|
await this.refresh();
|
|
680
|
-
|
|
681
|
-
await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));
|
|
933
|
+
await this.#catalog.sweep();
|
|
682
934
|
this.#ready = true;
|
|
683
935
|
}
|
|
684
|
-
/**
|
|
936
|
+
/**
|
|
937
|
+
* Reloads state, tail and segments from storage (e.g. after another writer changed them).
|
|
938
|
+
* Waits for a running compact() or merge() and runs between writes, so it never mixes the
|
|
939
|
+
* state before a commit with the rows after it (#26).
|
|
940
|
+
*/
|
|
685
941
|
async refresh() {
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
this.#state = state;
|
|
701
|
-
this.#tail = tail;
|
|
702
|
-
this.#segments = segments;
|
|
942
|
+
for (;;) {
|
|
943
|
+
while (this.#maintenanceRun) await this.#maintenanceRun.catch(() => {});
|
|
944
|
+
if (await this.#exclusive(async () => {
|
|
945
|
+
if (this.#maintaining) return false;
|
|
946
|
+
const snapshot = await this.#catalog.read();
|
|
947
|
+
const known = new Map(this.#segments.map((s) => [s.gen, s]));
|
|
948
|
+
const segments = await Promise.all(snapshot.state.segments.map((s) => known.get(s.gen) ?? this.#store.open(s, this.metric)));
|
|
949
|
+
await this.#swapSegments(async () => {
|
|
950
|
+
this.#catalog.adopt(snapshot);
|
|
951
|
+
this.#segments = segments;
|
|
952
|
+
});
|
|
953
|
+
return true;
|
|
954
|
+
})) return;
|
|
955
|
+
}
|
|
703
956
|
}
|
|
704
957
|
get stats() {
|
|
958
|
+
const { state, tail } = this.#catalog;
|
|
705
959
|
return {
|
|
706
|
-
tail:
|
|
707
|
-
segments:
|
|
708
|
-
seq:
|
|
960
|
+
tail: tail.size,
|
|
961
|
+
segments: state.segments.map((s) => ({ ...s })),
|
|
962
|
+
seq: state.seq
|
|
709
963
|
};
|
|
710
964
|
}
|
|
711
965
|
async upsert(items) {
|
|
@@ -717,86 +971,88 @@ var VectorIndex = class {
|
|
|
717
971
|
vector: prepare(it.vector, this.dims, this.metric)
|
|
718
972
|
};
|
|
719
973
|
});
|
|
720
|
-
const { tail, ids } = this.#t;
|
|
721
974
|
for (let i = 0; i < prepared.length; i += WRITE_BATCH) {
|
|
722
975
|
const batch = prepared.slice(i, i + WRITE_BATCH);
|
|
723
|
-
await this.#exclusive(
|
|
724
|
-
|
|
725
|
-
const next = {
|
|
726
|
-
...this.#state,
|
|
727
|
-
seq
|
|
728
|
-
};
|
|
729
|
-
await inTransaction(this.#db, async (tx) => {
|
|
730
|
-
for (const { id, vector } of batch) {
|
|
731
|
-
await tx.query(`INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)
|
|
732
|
-
ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`, [
|
|
733
|
-
id,
|
|
734
|
-
seq,
|
|
735
|
-
float32Bytes(vector)
|
|
736
|
-
]);
|
|
737
|
-
await tx.query(`INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`, [id]);
|
|
738
|
-
}
|
|
739
|
-
await this.#saveState(tx, next);
|
|
740
|
-
});
|
|
741
|
-
this.#state = next;
|
|
742
|
-
for (const { id, vector } of batch) this.#tail.set(id, {
|
|
743
|
-
seq,
|
|
744
|
-
vector
|
|
745
|
-
});
|
|
746
|
-
});
|
|
747
|
-
if (this.#tail.size >= this.#autoCompactAt && !this.#maintaining) await this.compact();
|
|
976
|
+
await this.#exclusive(() => this.#catalog.append(batch));
|
|
977
|
+
if (this.#catalog.tail.size >= this.#autoCompactAt && !this.#maintaining) await this.compact();
|
|
748
978
|
}
|
|
749
979
|
}
|
|
750
980
|
async delete(ids) {
|
|
751
981
|
this.#assertReady();
|
|
752
982
|
ids.forEach(assertId);
|
|
753
|
-
await this.#exclusive(
|
|
754
|
-
for (let i = 0; i < ids.length; i += VALIDATE_BATCH) {
|
|
755
|
-
const chunk = ids.slice(i, i + VALIDATE_BATCH);
|
|
756
|
-
const json = JSON.stringify(chunk);
|
|
757
|
-
await inTransaction(this.#db, async (tx) => {
|
|
758
|
-
await tx.query(`DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
|
|
759
|
-
await tx.query(`DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
|
|
760
|
-
});
|
|
761
|
-
for (const id of chunk) this.#tail.delete(id);
|
|
762
|
-
}
|
|
763
|
-
});
|
|
983
|
+
await this.#exclusive(() => this.#catalog.remove(ids));
|
|
764
984
|
}
|
|
765
985
|
async search(query, opts = {}) {
|
|
766
986
|
this.#assertReady();
|
|
987
|
+
opts.signal?.throwIfAborted();
|
|
767
988
|
const k = opts.k ?? 10;
|
|
768
|
-
const nprobe = opts.nprobe ?? 8;
|
|
769
989
|
const perList = k * (opts.oversample ?? 8);
|
|
990
|
+
const nprobe = opts.nprobe ?? DEFAULT_NPROBE;
|
|
991
|
+
const minRows = opts.nprobe === void 0 ? PROBE_ROWS_FACTOR * perList : 0;
|
|
770
992
|
const q = prepare(query, this.dims, this.metric);
|
|
771
993
|
const d = this.dims;
|
|
772
994
|
const filter = opts.filter;
|
|
995
|
+
const rerank = opts.exact ? k * 4 : k;
|
|
773
996
|
return this.#read(async (segments) => {
|
|
997
|
+
opts.signal?.throwIfAborted();
|
|
774
998
|
const tailTop = new TopK(k);
|
|
775
|
-
for (const [id, { vector }] of this.#tail) {
|
|
999
|
+
for (const [id, { vector }] of this.#catalog.tail) {
|
|
776
1000
|
if (filter && !filter(id)) continue;
|
|
777
1001
|
tailTop.push(distance(this.metric, q, 0, vector, 0, d), id);
|
|
778
1002
|
}
|
|
779
|
-
const
|
|
780
|
-
const keep = (id) => !this.#tail.has(id) && (!filter || filter(id));
|
|
1003
|
+
const probed = [];
|
|
781
1004
|
await Promise.all(segments.map(async (seg) => {
|
|
782
|
-
const lists = seg.probe(q, nprobe);
|
|
1005
|
+
const lists = seg.probe(q, nprobe, minRows);
|
|
783
1006
|
const datas = await this.#store.lists(seg, lists);
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
const data = datas[i];
|
|
787
|
-
for (const cand of seg.filterList(q, c, data, perList, keep)) hits.push({
|
|
788
|
-
...cand,
|
|
1007
|
+
lists.forEach((list, i) => {
|
|
1008
|
+
probed.push({
|
|
789
1009
|
seg,
|
|
790
|
-
|
|
1010
|
+
list,
|
|
1011
|
+
data: datas[i]
|
|
791
1012
|
});
|
|
792
1013
|
});
|
|
793
1014
|
}));
|
|
794
|
-
|
|
795
|
-
const
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
1015
|
+
opts.signal?.throwIfAborted();
|
|
1016
|
+
const gens = /* @__PURE__ */ new Map();
|
|
1017
|
+
const hits = [];
|
|
1018
|
+
let pending = probed.map((p) => ({
|
|
1019
|
+
...p,
|
|
1020
|
+
limit: perList
|
|
1021
|
+
}));
|
|
1022
|
+
while (pending.length > 0) {
|
|
1023
|
+
const rounds = pending.map((p) => {
|
|
1024
|
+
const keep = (id) => !this.#catalog.tail.has(id) && !(gens.has(id) && gens.get(id) !== p.seg.gen) && (!filter || filter(id));
|
|
1025
|
+
return {
|
|
1026
|
+
p,
|
|
1027
|
+
cands: p.seg.filterList(q, p.list, p.data, p.limit, keep)
|
|
1028
|
+
};
|
|
1029
|
+
});
|
|
1030
|
+
const unknown = rounds.flatMap((r) => r.cands.map((c) => c.id)).filter((id) => !gens.has(id));
|
|
1031
|
+
if (unknown.length > 0) {
|
|
1032
|
+
const found = await this.#catalog.generations(unknown);
|
|
1033
|
+
for (const id of unknown) gens.set(id, found.get(id));
|
|
1034
|
+
opts.signal?.throwIfAborted();
|
|
1035
|
+
}
|
|
1036
|
+
pending = [];
|
|
1037
|
+
for (const { p, cands } of rounds) {
|
|
1038
|
+
const live = cands.filter((c) => gens.get(c.id) === p.seg.gen);
|
|
1039
|
+
const whole = p.seg.passesWhole(p.data.ids.length, p.limit);
|
|
1040
|
+
if (!whole && live.length < perList && cands.length === p.limit) {
|
|
1041
|
+
pending.push({
|
|
1042
|
+
...p,
|
|
1043
|
+
limit: p.limit * 2
|
|
1044
|
+
});
|
|
1045
|
+
continue;
|
|
1046
|
+
}
|
|
1047
|
+
for (const cand of whole ? live : live.slice(0, perList)) hits.push({
|
|
1048
|
+
...cand,
|
|
1049
|
+
seg: p.seg,
|
|
1050
|
+
data: p.data
|
|
1051
|
+
});
|
|
1052
|
+
}
|
|
799
1053
|
}
|
|
1054
|
+
const segTop = new TopK(rerank);
|
|
1055
|
+
for (const h of hits) segTop.push(h.seg.scoreInt8(q, h.data, h.row), h);
|
|
800
1056
|
let segResults = segTop.sorted().map((e) => ({
|
|
801
1057
|
id: e.item.id,
|
|
802
1058
|
distance: e.score,
|
|
@@ -824,11 +1080,11 @@ var VectorIndex = class {
|
|
|
824
1080
|
ids.forEach(assertId);
|
|
825
1081
|
return this.#read(async (segments) => {
|
|
826
1082
|
const out = /* @__PURE__ */ new Map();
|
|
827
|
-
const gens = await this.#generations(ids);
|
|
1083
|
+
const gens = await this.#catalog.generations(ids);
|
|
828
1084
|
const wanted = /* @__PURE__ */ new Map();
|
|
829
1085
|
for (const [id, gen] of gens) {
|
|
830
1086
|
if (gen === 0) {
|
|
831
|
-
const entry = this.#tail.get(id);
|
|
1087
|
+
const entry = this.#catalog.tail.get(id);
|
|
832
1088
|
if (entry) out.set(id, entry.vector.slice());
|
|
833
1089
|
continue;
|
|
834
1090
|
}
|
|
@@ -860,8 +1116,8 @@ var VectorIndex = class {
|
|
|
860
1116
|
this.#assertReady();
|
|
861
1117
|
return this.#maintenance(async () => {
|
|
862
1118
|
const snapshot = await this.#exclusive(async () => ({
|
|
863
|
-
maxSeq: this.#state.seq,
|
|
864
|
-
entries: [...this.#tail.entries()]
|
|
1119
|
+
maxSeq: this.#catalog.state.seq,
|
|
1120
|
+
entries: [...this.#catalog.tail.entries()]
|
|
865
1121
|
}));
|
|
866
1122
|
if (snapshot.entries.length === 0) return 0;
|
|
867
1123
|
const n = snapshot.entries.length;
|
|
@@ -873,23 +1129,9 @@ var VectorIndex = class {
|
|
|
873
1129
|
});
|
|
874
1130
|
const built = this.#build(ids, vectors);
|
|
875
1131
|
await this.#exclusive(() => this.#swapSegments(async () => {
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
...this.#state,
|
|
879
|
-
nextGen: ref.gen + 1,
|
|
880
|
-
segments: [...this.#state.segments, ref]
|
|
881
|
-
};
|
|
882
|
-
const { ids: idsT, tail } = this.#t;
|
|
883
|
-
await inTransaction(this.#db, async (tx) => {
|
|
884
|
-
await this.#store.write(tx, ref.gen, built);
|
|
885
|
-
await tx.query(`UPDATE ${idsT} SET gen = ? WHERE gen = 0
|
|
886
|
-
AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`, [ref.gen, snapshot.maxSeq]);
|
|
887
|
-
await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [snapshot.maxSeq]);
|
|
888
|
-
await this.#saveState(tx, next);
|
|
1132
|
+
await this.#catalog.seal(snapshot.maxSeq, built, n, (ref) => {
|
|
1133
|
+
this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
|
|
889
1134
|
});
|
|
890
|
-
this.#state = next;
|
|
891
|
-
for (const [id, e] of this.#tail) if (e.seq <= snapshot.maxSeq) this.#tail.delete(id);
|
|
892
|
-
this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
|
|
893
1135
|
}));
|
|
894
1136
|
return n;
|
|
895
1137
|
});
|
|
@@ -910,7 +1152,7 @@ var VectorIndex = class {
|
|
|
910
1152
|
if (seg.counts[c] === 0) continue;
|
|
911
1153
|
const [data] = await this.#store.lists(seg, [c]);
|
|
912
1154
|
const floats = await this.#store.floats(seg, c);
|
|
913
|
-
const live = await this.#generations(Array.from(data.ids));
|
|
1155
|
+
const live = await this.#catalog.generations(Array.from(data.ids));
|
|
914
1156
|
for (let r = 0; r < data.ids.length; r++) {
|
|
915
1157
|
const id = data.ids[r];
|
|
916
1158
|
if (live.get(id) !== seg.gen) continue;
|
|
@@ -925,23 +1167,10 @@ var VectorIndex = class {
|
|
|
925
1167
|
});
|
|
926
1168
|
const built = n > 0 ? this.#build(Float64Array.from(idList), vectors) : null;
|
|
927
1169
|
await this.#exclusive(() => this.#swapSegments(async () => {
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
...this.#state,
|
|
932
|
-
nextGen: ref ? ref.gen + 1 : this.#state.nextGen,
|
|
933
|
-
segments: ref ? [...kept, ref] : kept
|
|
934
|
-
};
|
|
935
|
-
await inTransaction(this.#db, async (tx) => {
|
|
936
|
-
if (ref && built) {
|
|
937
|
-
await this.#store.write(tx, ref.gen, built);
|
|
938
|
-
await tx.query(`UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`, [ref.gen, JSON.stringify(oldGens)]);
|
|
939
|
-
}
|
|
940
|
-
await this.#saveState(tx, next);
|
|
1170
|
+
await this.#catalog.replace(oldGens, built, n, (ref) => {
|
|
1171
|
+
this.#segments = [...this.#segments.filter((s) => !oldGens.includes(s.gen)), ...ref && built ? [this.#segmentOf(ref, built)] : []];
|
|
1172
|
+
this.#retired.push(...old);
|
|
941
1173
|
});
|
|
942
|
-
this.#state = next;
|
|
943
|
-
this.#segments = [...this.#segments.filter((s) => !oldGens.includes(s.gen)), ...ref && built ? [this.#segmentOf(ref, built)] : []];
|
|
944
|
-
this.#retired.push(...old);
|
|
945
1174
|
}));
|
|
946
1175
|
await this.#reap();
|
|
947
1176
|
return n;
|
|
@@ -961,37 +1190,16 @@ var VectorIndex = class {
|
|
|
961
1190
|
nlist: this.#nlist?.(ids.length)
|
|
962
1191
|
});
|
|
963
1192
|
}
|
|
964
|
-
#refOf(built, count) {
|
|
965
|
-
return {
|
|
966
|
-
gen: this.#state.nextGen,
|
|
967
|
-
count,
|
|
968
|
-
nlist: built.nlist
|
|
969
|
-
};
|
|
970
|
-
}
|
|
971
1193
|
#segmentOf(ref, built) {
|
|
972
1194
|
return new SegmentIndex(ref.gen, this.dims, this.metric, built.centroids, built.counts);
|
|
973
1195
|
}
|
|
974
|
-
async #generations(ids) {
|
|
975
|
-
const out = /* @__PURE__ */ new Map();
|
|
976
|
-
const unique = [...new Set(ids)];
|
|
977
|
-
for (let i = 0; i < unique.length; i += VALIDATE_BATCH) {
|
|
978
|
-
const rows = await this.#db.query(`SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [JSON.stringify(unique.slice(i, i + VALIDATE_BATCH))]);
|
|
979
|
-
for (const r of rows) out.set(Number(r.id), Number(r.gen));
|
|
980
|
-
}
|
|
981
|
-
return out;
|
|
982
|
-
}
|
|
983
|
-
async #saveState(tx, state) {
|
|
984
|
-
await tx.query(`INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)
|
|
985
|
-
ON CONFLICT(k) DO UPDATE SET v = excluded.v`, [JSON.stringify(state)]);
|
|
986
|
-
}
|
|
987
1196
|
/** Deletes the rows of retired segments once no search is reading them. */
|
|
988
1197
|
async #reap() {
|
|
989
1198
|
const ready = this.#retired.filter((s) => (this.#refs.get(s) ?? 0) === 0);
|
|
990
1199
|
if (ready.length === 0) return;
|
|
991
1200
|
this.#retired = this.#retired.filter((s) => !ready.includes(s));
|
|
992
1201
|
for (const s of ready) this.#refs.delete(s);
|
|
993
|
-
|
|
994
|
-
await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));
|
|
1202
|
+
await this.#catalog.dropGenerations(ready.map((s) => s.gen));
|
|
995
1203
|
}
|
|
996
1204
|
/**
|
|
997
1205
|
* Runs `read` over a snapshot of the segments, held against reaping. A read
|
|
@@ -1029,11 +1237,16 @@ var VectorIndex = class {
|
|
|
1029
1237
|
async #maintenance(fn) {
|
|
1030
1238
|
if (this.#maintaining) throw new Error("vector-index: compact or merge already running");
|
|
1031
1239
|
this.#maintaining = true;
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1240
|
+
const run = (async () => {
|
|
1241
|
+
try {
|
|
1242
|
+
return await fn();
|
|
1243
|
+
} finally {
|
|
1244
|
+
this.#maintaining = false;
|
|
1245
|
+
this.#maintenanceRun = null;
|
|
1246
|
+
}
|
|
1247
|
+
})();
|
|
1248
|
+
this.#maintenanceRun = run;
|
|
1249
|
+
return run;
|
|
1037
1250
|
}
|
|
1038
1251
|
#assertReady() {
|
|
1039
1252
|
if (!this.#ready) throw new Error("vector-index: call init() first");
|
|
@@ -1045,16 +1258,7 @@ function assertId(id) {
|
|
|
1045
1258
|
/** Drops every table of the index under `prefix`, in one transaction. Needs no dims or metric. */
|
|
1046
1259
|
async function dropVectorIndex(db, prefix) {
|
|
1047
1260
|
if (!PREFIX_PATTERN.test(prefix)) throw new Error(`vector-index: bad prefix ${prefix}`);
|
|
1048
|
-
await
|
|
1049
|
-
for (const t of [
|
|
1050
|
-
"meta",
|
|
1051
|
-
"ids",
|
|
1052
|
-
"tail",
|
|
1053
|
-
"centroids",
|
|
1054
|
-
"seg",
|
|
1055
|
-
"f32"
|
|
1056
|
-
]) await tx.exec(`DROP TABLE IF EXISTS ${prefix}${t}`);
|
|
1057
|
-
});
|
|
1261
|
+
await dropCatalog(db, prefix);
|
|
1058
1262
|
}
|
|
1059
1263
|
//#endregion
|
|
1060
1264
|
export { VectorIndex, dropVectorIndex };
|