@statewalker/db-sqlite-index 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/vector.mjs CHANGED
@@ -15,7 +15,10 @@ function prepare(v, dims, metric) {
15
15
  if (v.length !== dims) throw new Error(`vector-index: expected ${dims} dims, got ${v.length}`);
16
16
  const f = v instanceof Float32Array ? v : Float32Array.from(v);
17
17
  for (let i = 0; i < f.length; i++) if (!Number.isFinite(f[i])) throw new Error("vector-index: vector contains a non-finite value");
18
- return metric === "cosine" ? normalize(f) : f.slice();
18
+ if (metric !== "cosine") return f.slice();
19
+ const out = normalize(f);
20
+ if (!out.some((x) => x !== 0)) throw new Error("vector-index: a zero vector has no direction under the cosine metric");
21
+ return out;
19
22
  }
20
23
  /** Distance between two float vectors at offsets; lower is closer. */
21
24
  function distance(metric, a, ao, b, bo, d) {
@@ -348,6 +351,8 @@ function parseList(bytes, count, dims) {
348
351
  };
349
352
  }
350
353
  /** The in-memory part of a segment: centroids and list sizes. List blocks are fetched by the caller. */
354
+ /** The share of a segment's lists that {@link SegmentIndex.probe} may widen to for `minRows`. */
355
+ const MAX_WIDEN_FRACTION = 1 / 3;
351
356
  var SegmentIndex = class {
352
357
  gen;
353
358
  dims;
@@ -364,21 +369,66 @@ var SegmentIndex = class {
364
369
  get nlist() {
365
370
  return this.counts.length;
366
371
  }
367
- /** The `nprobe` non-empty lists whose centroids are closest to the query. */
368
- probe(query, nprobe) {
372
+ /**
373
+ * The non-empty lists closest to the query, in centroid order: at least the
374
+ * `nprobe` nearest, then more until the probed lists hold `minRows` rows (or
375
+ * none are left). `minRows` 0 probes exactly `nprobe` lists. The widening
376
+ * stops at a third of the lists ({@link MAX_WIDEN_FRACTION}), so a large `k`
377
+ * never turns a search into a scan of every list.
378
+ */
379
+ probe(query, nprobe, minRows = 0) {
380
+ const maxLists = Math.max(nprobe, Math.ceil(this.nlist * MAX_WIDEN_FRACTION));
369
381
  const m = this.metric === "l2" ? "l2" : "dot";
370
- const top = new TopK(Math.min(nprobe, this.nlist));
382
+ const scored = [];
371
383
  for (let c = 0; c < this.nlist; c++) {
372
384
  if (this.counts[c] === 0) continue;
373
- top.push(distance(m, query, 0, this.centroids, c * this.dims, this.dims), c);
385
+ scored.push({
386
+ d: distance(m, query, 0, this.centroids, c * this.dims, this.dims),
387
+ c
388
+ });
374
389
  }
375
- return top.sorted().map((e) => e.item);
390
+ scored.sort((a, b) => a.d - b.d || a.c - b.c);
391
+ const out = [];
392
+ let rows = 0;
393
+ for (const { c } of scored) {
394
+ if (out.length >= nprobe && (rows >= minRows || out.length >= maxLists)) break;
395
+ out.push(c);
396
+ rows += this.counts[c];
397
+ }
398
+ return out;
399
+ }
400
+ /**
401
+ * Whether stage 1 passes a list of `n` rows whole (in row order, not ranked):
402
+ * a list no longer than `limit`, or a flat segment's only list of at most
403
+ * {@link MIN_TRAIN} rows. Callers must not cut such a list by position.
404
+ */
405
+ passesWhole(n, limit) {
406
+ return n <= limit || this.nlist === 1 && n <= 2048;
376
407
  }
377
408
  /**
378
409
  * Stage 1: the `limit` rows of a list with the best estimated distance, from
379
410
  * residual sign bits plus the stored residual norm and centroid projection.
411
+ *
412
+ * A flat segment (one list of at most {@link MIN_TRAIN} rows) passes every row:
413
+ * its centroid is the mean of all vectors, far from most of them on clustered
414
+ * data, so the 1-bit estimate is weakest exactly there (issue #40), while
415
+ * int8-scoring the whole list costs no more than scanning the tail. A list
416
+ * that does not exceed `limit` also passes whole, without estimating.
380
417
  */
381
418
  filterList(query, c, data, limit, keep) {
419
+ const n = data.ids.length;
420
+ if (this.passesWhole(n, limit)) {
421
+ const all = [];
422
+ for (let r = 0; r < n; r++) {
423
+ const id = data.ids[r];
424
+ if (!keep || keep(id)) all.push({
425
+ list: c,
426
+ row: r,
427
+ id
428
+ });
429
+ }
430
+ return all;
431
+ }
382
432
  const d = this.dims;
383
433
  const w = wordsFor(d);
384
434
  const co = c * d;
@@ -391,7 +441,7 @@ var SegmentIndex = class {
391
441
  const angle = Math.PI / d;
392
442
  const l2 = this.metric === "l2";
393
443
  const top = new TopK(limit);
394
- for (let r = 0; r < data.ids.length; r++) {
444
+ for (let r = 0; r < n; r++) {
395
445
  const id = data.ids[r];
396
446
  if (keep && !keep(id)) continue;
397
447
  const rr = qn * data.rnorm[r] * Math.cos(angle * hamming(qb, data.bits, r * w, w));
@@ -431,19 +481,23 @@ const PART_BYTES = 1048576;
431
481
  * float32 blocks split into parts. Reads go through an LRU cache: a
432
482
  * generation is immutable, so cached blocks never go stale.
433
483
  */
434
- var SqlSegmentStore = class {
484
+ var SqlSegmentStore = class SqlSegmentStore {
435
485
  #db;
436
486
  #dims;
437
487
  #t;
438
488
  #cache;
439
- constructor(db, prefix, dims, cacheBytes = 67108864) {
440
- this.#db = db;
441
- this.#dims = dims;
442
- this.#t = {
489
+ /** The names of the tables segment rows live in, under `prefix`. */
490
+ static tables(prefix) {
491
+ return {
443
492
  centroids: `${prefix}centroids`,
444
493
  seg: `${prefix}seg`,
445
494
  f32: `${prefix}f32`
446
495
  };
496
+ }
497
+ constructor(db, prefix, dims, cacheBytes = 67108864) {
498
+ this.#db = db;
499
+ this.#dims = dims;
500
+ this.#t = SqlSegmentStore.tables(prefix);
447
501
  this.#cache = new Lru(cacheBytes);
448
502
  }
449
503
  schema() {
@@ -597,9 +651,229 @@ var Lru = class {
597
651
  }
598
652
  };
599
653
  //#endregion
654
+ //#region src/vector/catalog.ts
655
+ /** Ids per statement (bound as one JSON parameter). */
656
+ const ID_BATCH = 2e3;
657
+ const EMPTY = {
658
+ seq: 0,
659
+ nextGen: 1,
660
+ segments: []
661
+ };
662
+ /**
663
+ * The generation catalog of a vector index: the committed state, the `{p}ids` table (which
664
+ * generation holds the live copy of each id, 0 = the tail), the `{p}tail` rows, and the in-memory
665
+ * copy of the state and the tail.
666
+ *
667
+ * It is the only place that commits a state change, and every commit updates the copy in the
668
+ * same synchronous step as it returns from the transaction, so the copy always matches storage.
669
+ * It also creates, sweeps and drops every table under the prefix, asking the segment store for
670
+ * its own.
671
+ *
672
+ * Callers serialize the commits (one at a time per instance); reads of `state` and `tail` may
673
+ * happen at any time and see either the copy before a commit or the copy after it.
674
+ */
675
+ var GenerationCatalog = class {
676
+ /** `{p}ids`: its `id` column holds exactly the live ids. */
677
+ idTable;
678
+ #db;
679
+ #store;
680
+ #t;
681
+ #state = EMPTY;
682
+ #tail = /* @__PURE__ */ new Map();
683
+ constructor(db, prefix, store) {
684
+ this.#db = db;
685
+ this.#store = store;
686
+ this.#t = catalogTables(prefix);
687
+ this.idTable = this.#t.ids;
688
+ }
689
+ /** The committed state, as of the last commit or `adopt()`. Never mutated: replaced. */
690
+ get state() {
691
+ return this.#state;
692
+ }
693
+ /** The tail rows in memory, by id. */
694
+ get tail() {
695
+ return this.#tail;
696
+ }
697
+ /**
698
+ * Creates every table if missing and checks the stored config (dims and metric) is `config`:
699
+ * the first open stores it, later opens must match.
700
+ */
701
+ async create(config) {
702
+ const { meta, ids, tail } = this.#t;
703
+ for (const s of [
704
+ `CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,
705
+ `CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,
706
+ `CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,
707
+ `CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,
708
+ `CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,
709
+ ...this.#store.schema()
710
+ ]) await this.#db.exec(s);
711
+ await this.#db.query(`INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`, [config]);
712
+ const [stored] = await this.#db.query(`SELECT v FROM ${meta} WHERE k = 'config'`);
713
+ if (stored?.v !== config) throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);
714
+ }
715
+ /** Deletes segment rows of every generation the committed state does not list (crash leftovers). */
716
+ async sweep() {
717
+ const live = this.#state.segments.map((s) => s.gen);
718
+ await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));
719
+ }
720
+ /** Deletes the segment rows of retired generations. */
721
+ async dropGenerations(gens) {
722
+ await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));
723
+ }
724
+ /** Reads the committed state and the tail rows, without installing them. */
725
+ async read() {
726
+ const [row] = await this.#db.query(`SELECT v FROM ${this.#t.meta} WHERE k = 'state'`);
727
+ const state = row ? JSON.parse(row.v) : EMPTY;
728
+ const tail = /* @__PURE__ */ new Map();
729
+ const rows = await this.#db.query(`SELECT id, seq, vec FROM ${this.#t.tail}`);
730
+ for (const r of rows) tail.set(Number(r.id), {
731
+ seq: Number(r.seq),
732
+ vector: toFloat32(r.vec)
733
+ });
734
+ return {
735
+ state,
736
+ tail
737
+ };
738
+ }
739
+ /** Installs a snapshot from `read()` as the copy. No commit may run between the two. */
740
+ adopt(snapshot) {
741
+ this.#state = snapshot.state;
742
+ this.#tail = snapshot.tail;
743
+ }
744
+ /** The generation holding the live copy of each id; deleted or unknown ids are absent. */
745
+ async generations(ids) {
746
+ const out = /* @__PURE__ */ new Map();
747
+ const unique = [...new Set(ids)];
748
+ for (let i = 0; i < unique.length; i += ID_BATCH) {
749
+ const rows = await this.#db.query(`SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [JSON.stringify(unique.slice(i, i + ID_BATCH))]);
750
+ for (const r of rows) out.set(Number(r.id), Number(r.gen));
751
+ }
752
+ return out;
753
+ }
754
+ /** Writes `items` to the tail as one new batch (the next seq), in one transaction. */
755
+ async append(items) {
756
+ const { tail, ids } = this.#t;
757
+ const seq = this.#state.seq + 1;
758
+ const next = {
759
+ ...this.#state,
760
+ seq
761
+ };
762
+ await inTransaction(this.#db, async (tx) => {
763
+ for (const { id, vector } of items) {
764
+ await tx.query(`INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)
765
+ ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`, [
766
+ id,
767
+ seq,
768
+ float32Bytes(vector)
769
+ ]);
770
+ await tx.query(`INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`, [id]);
771
+ }
772
+ await this.#save(tx, next);
773
+ });
774
+ this.#state = next;
775
+ for (const { id, vector } of items) this.#tail.set(id, {
776
+ seq,
777
+ vector
778
+ });
779
+ }
780
+ /** Removes `ids` from the tail and from the live ids, one transaction per batch. */
781
+ async remove(ids) {
782
+ for (let i = 0; i < ids.length; i += ID_BATCH) {
783
+ const chunk = ids.slice(i, i + ID_BATCH);
784
+ const json = JSON.stringify(chunk);
785
+ await inTransaction(this.#db, async (tx) => {
786
+ await tx.query(`DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
787
+ await tx.query(`DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
788
+ });
789
+ for (const id of chunk) this.#tail.delete(id);
790
+ }
791
+ }
792
+ /**
793
+ * Commits `built` (sealed from the tail rows up to `maxSeq`, `count` vectors) as a new
794
+ * generation: writes its rows, moves those ids to it and deletes those tail rows. Tail rows of
795
+ * later batches stay. `install` runs with the new ref in the same synchronous step as the copy
796
+ * update, for the caller's own in-memory segment list.
797
+ */
798
+ async seal(maxSeq, built, count, install) {
799
+ const ref = this.#refOf(built, count);
800
+ const next = {
801
+ ...this.#state,
802
+ nextGen: ref.gen + 1,
803
+ segments: [...this.#state.segments, ref]
804
+ };
805
+ const { ids, tail } = this.#t;
806
+ await inTransaction(this.#db, async (tx) => {
807
+ await this.#store.write(tx, ref.gen, built);
808
+ await tx.query(`UPDATE ${ids} SET gen = ? WHERE gen = 0
809
+ AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`, [ref.gen, maxSeq]);
810
+ await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [maxSeq]);
811
+ await this.#save(tx, next);
812
+ });
813
+ this.#state = next;
814
+ for (const [id, e] of this.#tail) if (e.seq <= maxSeq) this.#tail.delete(id);
815
+ install(ref);
816
+ return ref;
817
+ }
818
+ /**
819
+ * Replaces the generations `oldGens` with `built` (`count` live vectors), or with nothing when
820
+ * `built` is null: writes its rows, moves the ids of the old generations to it and drops them
821
+ * from the state. Their rows stay until `dropGenerations()`. `install` runs as in `seal()`.
822
+ */
823
+ async replace(oldGens, built, count, install) {
824
+ const ref = built ? this.#refOf(built, count) : null;
825
+ const kept = this.#state.segments.filter((s) => !oldGens.includes(s.gen));
826
+ const next = {
827
+ ...this.#state,
828
+ nextGen: ref ? ref.gen + 1 : this.#state.nextGen,
829
+ segments: ref ? [...kept, ref] : kept
830
+ };
831
+ await inTransaction(this.#db, async (tx) => {
832
+ if (ref && built) {
833
+ await this.#store.write(tx, ref.gen, built);
834
+ await tx.query(`UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`, [ref.gen, JSON.stringify(oldGens)]);
835
+ }
836
+ await this.#save(tx, next);
837
+ });
838
+ this.#state = next;
839
+ install(ref);
840
+ return ref;
841
+ }
842
+ #refOf(built, count) {
843
+ return {
844
+ gen: this.#state.nextGen,
845
+ count,
846
+ nlist: built.nlist
847
+ };
848
+ }
849
+ async #save(tx, state) {
850
+ await tx.query(`INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)
851
+ ON CONFLICT(k) DO UPDATE SET v = excluded.v`, [JSON.stringify(state)]);
852
+ }
853
+ };
854
+ function catalogTables(prefix) {
855
+ return {
856
+ meta: `${prefix}meta`,
857
+ ids: `${prefix}ids`,
858
+ tail: `${prefix}tail`
859
+ };
860
+ }
861
+ /** Drops every table under `prefix` (the catalog's and the segment store's), in one transaction. */
862
+ async function dropCatalog(db, prefix) {
863
+ const tables = [...Object.values(catalogTables(prefix)), ...Object.values(SqlSegmentStore.tables(prefix))];
864
+ await inTransaction(db, async (tx) => {
865
+ for (const t of tables) await tx.exec(`DROP TABLE IF EXISTS ${t}`);
866
+ });
867
+ }
868
+ //#endregion
600
869
  //#region src/vector/vector-index.ts
601
- /** Ids per lookup (bound as one JSON parameter). */
602
- const VALIDATE_BATCH = 2e3;
870
+ /** Lists probed per segment when the caller passes no `nprobe`. */
871
+ const DEFAULT_NPROBE = 8;
872
+ /**
873
+ * Without an explicit `nprobe`, probe lists until they hold this many times
874
+ * `k × oversample` rows (#67).
875
+ */
876
+ const PROBE_ROWS_FACTOR = 16;
603
877
  /** Upserted rows per transaction. */
604
878
  const WRITE_BATCH = 200;
605
879
  /**
@@ -615,29 +889,26 @@ const WRITE_BATCH = 200;
615
889
  * into one, dropping dead rows.
616
890
  *
617
891
  * One writer per database is assumed (a Durable Object, a worker, one tab);
618
- * within an instance, writes are serialized.
892
+ * within an instance, writes are serialized. The tables, the committed state and
893
+ * the in-memory tail belong to the generation catalog (`catalog.ts`); this class decides
894
+ * what to commit and when, builds segments and searches them.
619
895
  */
620
896
  var VectorIndex = class {
621
897
  dims;
622
898
  metric;
623
899
  /** `{prefix}ids`: its `id` column holds exactly the live ids. Callers may join it in SQL. */
624
900
  idTable;
625
- #db;
626
901
  #store;
902
+ #catalog;
627
903
  #nlist;
628
904
  #autoCompactAt;
629
- #t;
630
- #state = {
631
- seq: 0,
632
- nextGen: 1,
633
- segments: []
634
- };
635
- #tail = /* @__PURE__ */ new Map();
636
905
  #segments = [];
637
906
  #refs = /* @__PURE__ */ new Map();
638
907
  #retired = [];
639
908
  #queue = Promise.resolve();
640
909
  #maintaining = false;
910
+ /** The running compact() or merge(), for refresh() to wait on. */
911
+ #maintenanceRun = null;
641
912
  #ready = false;
642
913
  /** Bumped before and after each commit that changes the segment list: odd while one runs. */
643
914
  #epoch = 0;
@@ -646,66 +917,49 @@ var VectorIndex = class {
646
917
  if (!Number.isInteger(opts.dims) || opts.dims < 1) throw new Error("vector-index: dims must be a positive integer");
647
918
  this.dims = opts.dims;
648
919
  this.metric = opts.metric ?? "cosine";
649
- this.#db = opts.db;
650
920
  this.#store = new SqlSegmentStore(opts.db, opts.prefix, opts.dims, opts.cacheBytes);
921
+ this.#catalog = new GenerationCatalog(opts.db, opts.prefix, this.#store);
651
922
  this.#nlist = opts.nlist;
652
923
  this.#autoCompactAt = opts.autoCompactAt ?? 4096;
653
- const p = opts.prefix;
654
- this.#t = {
655
- meta: `${p}meta`,
656
- ids: `${p}ids`,
657
- tail: `${p}tail`
658
- };
659
- this.idTable = this.#t.ids;
924
+ this.idTable = this.#catalog.idTable;
660
925
  }
661
926
  /** Creates tables if missing, loads state and sweeps rows of crashed maintenance. */
662
927
  async init() {
663
- const { meta, ids, tail } = this.#t;
664
- for (const s of [
665
- `CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,
666
- `CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,
667
- `CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,
668
- `CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,
669
- `CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,
670
- ...this.#store.schema()
671
- ]) await this.#db.exec(s);
672
- const config = JSON.stringify({
928
+ await this.#catalog.create(JSON.stringify({
673
929
  dims: this.dims,
674
930
  metric: this.metric
675
- });
676
- await this.#db.query(`INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`, [config]);
677
- const [stored] = await this.#db.query(`SELECT v FROM ${meta} WHERE k = 'config'`);
678
- if (stored?.v !== config) throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);
931
+ }));
679
932
  await this.refresh();
680
- const live = this.#state.segments.map((s) => s.gen);
681
- await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));
933
+ await this.#catalog.sweep();
682
934
  this.#ready = true;
683
935
  }
684
- /** Reloads state, tail and segments from storage (e.g. after another writer changed them). */
936
+ /**
937
+ * Reloads state, tail and segments from storage (e.g. after another writer changed them).
938
+ * Waits for a running compact() or merge() and runs between writes, so it never mixes the
939
+ * state before a commit with the rows after it (#26).
940
+ */
685
941
  async refresh() {
686
- const [row] = await this.#db.query(`SELECT v FROM ${this.#t.meta} WHERE k = 'state'`);
687
- const state = row ? JSON.parse(row.v) : {
688
- seq: 0,
689
- nextGen: 1,
690
- segments: []
691
- };
692
- const tail = /* @__PURE__ */ new Map();
693
- const rows = await this.#db.query(`SELECT id, seq, vec FROM ${this.#t.tail}`);
694
- for (const r of rows) tail.set(Number(r.id), {
695
- seq: Number(r.seq),
696
- vector: toFloat32(r.vec)
697
- });
698
- const known = new Map(this.#segments.map((s) => [s.gen, s]));
699
- const segments = await Promise.all(state.segments.map((s) => known.get(s.gen) ?? this.#store.open(s, this.metric)));
700
- this.#state = state;
701
- this.#tail = tail;
702
- this.#segments = segments;
942
+ for (;;) {
943
+ while (this.#maintenanceRun) await this.#maintenanceRun.catch(() => {});
944
+ if (await this.#exclusive(async () => {
945
+ if (this.#maintaining) return false;
946
+ const snapshot = await this.#catalog.read();
947
+ const known = new Map(this.#segments.map((s) => [s.gen, s]));
948
+ const segments = await Promise.all(snapshot.state.segments.map((s) => known.get(s.gen) ?? this.#store.open(s, this.metric)));
949
+ await this.#swapSegments(async () => {
950
+ this.#catalog.adopt(snapshot);
951
+ this.#segments = segments;
952
+ });
953
+ return true;
954
+ })) return;
955
+ }
703
956
  }
704
957
  get stats() {
958
+ const { state, tail } = this.#catalog;
705
959
  return {
706
- tail: this.#tail.size,
707
- segments: this.#state.segments.map((s) => ({ ...s })),
708
- seq: this.#state.seq
960
+ tail: tail.size,
961
+ segments: state.segments.map((s) => ({ ...s })),
962
+ seq: state.seq
709
963
  };
710
964
  }
711
965
  async upsert(items) {
@@ -717,86 +971,88 @@ var VectorIndex = class {
717
971
  vector: prepare(it.vector, this.dims, this.metric)
718
972
  };
719
973
  });
720
- const { tail, ids } = this.#t;
721
974
  for (let i = 0; i < prepared.length; i += WRITE_BATCH) {
722
975
  const batch = prepared.slice(i, i + WRITE_BATCH);
723
- await this.#exclusive(async () => {
724
- const seq = this.#state.seq + 1;
725
- const next = {
726
- ...this.#state,
727
- seq
728
- };
729
- await inTransaction(this.#db, async (tx) => {
730
- for (const { id, vector } of batch) {
731
- await tx.query(`INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)
732
- ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`, [
733
- id,
734
- seq,
735
- float32Bytes(vector)
736
- ]);
737
- await tx.query(`INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`, [id]);
738
- }
739
- await this.#saveState(tx, next);
740
- });
741
- this.#state = next;
742
- for (const { id, vector } of batch) this.#tail.set(id, {
743
- seq,
744
- vector
745
- });
746
- });
747
- if (this.#tail.size >= this.#autoCompactAt && !this.#maintaining) await this.compact();
976
+ await this.#exclusive(() => this.#catalog.append(batch));
977
+ if (this.#catalog.tail.size >= this.#autoCompactAt && !this.#maintaining) await this.compact();
748
978
  }
749
979
  }
750
980
  async delete(ids) {
751
981
  this.#assertReady();
752
982
  ids.forEach(assertId);
753
- await this.#exclusive(async () => {
754
- for (let i = 0; i < ids.length; i += VALIDATE_BATCH) {
755
- const chunk = ids.slice(i, i + VALIDATE_BATCH);
756
- const json = JSON.stringify(chunk);
757
- await inTransaction(this.#db, async (tx) => {
758
- await tx.query(`DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
759
- await tx.query(`DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
760
- });
761
- for (const id of chunk) this.#tail.delete(id);
762
- }
763
- });
983
+ await this.#exclusive(() => this.#catalog.remove(ids));
764
984
  }
765
985
  async search(query, opts = {}) {
766
986
  this.#assertReady();
987
+ opts.signal?.throwIfAborted();
767
988
  const k = opts.k ?? 10;
768
- const nprobe = opts.nprobe ?? 8;
769
989
  const perList = k * (opts.oversample ?? 8);
990
+ const nprobe = opts.nprobe ?? DEFAULT_NPROBE;
991
+ const minRows = opts.nprobe === void 0 ? PROBE_ROWS_FACTOR * perList : 0;
770
992
  const q = prepare(query, this.dims, this.metric);
771
993
  const d = this.dims;
772
994
  const filter = opts.filter;
995
+ const rerank = opts.exact ? k * 4 : k;
773
996
  return this.#read(async (segments) => {
997
+ opts.signal?.throwIfAborted();
774
998
  const tailTop = new TopK(k);
775
- for (const [id, { vector }] of this.#tail) {
999
+ for (const [id, { vector }] of this.#catalog.tail) {
776
1000
  if (filter && !filter(id)) continue;
777
1001
  tailTop.push(distance(this.metric, q, 0, vector, 0, d), id);
778
1002
  }
779
- const hits = [];
780
- const keep = (id) => !this.#tail.has(id) && (!filter || filter(id));
1003
+ const probed = [];
781
1004
  await Promise.all(segments.map(async (seg) => {
782
- const lists = seg.probe(q, nprobe);
1005
+ const lists = seg.probe(q, nprobe, minRows);
783
1006
  const datas = await this.#store.lists(seg, lists);
784
- opts.signal?.throwIfAborted();
785
- lists.forEach((c, i) => {
786
- const data = datas[i];
787
- for (const cand of seg.filterList(q, c, data, perList, keep)) hits.push({
788
- ...cand,
1007
+ lists.forEach((list, i) => {
1008
+ probed.push({
789
1009
  seg,
790
- data
1010
+ list,
1011
+ data: datas[i]
791
1012
  });
792
1013
  });
793
1014
  }));
794
- const live = await this.#generations(hits.map((h) => h.id));
795
- const segTop = new TopK(opts.exact ? k * 4 : k);
796
- for (const h of hits) {
797
- if (live.get(h.id) !== h.seg.gen) continue;
798
- segTop.push(h.seg.scoreInt8(q, h.data, h.row), h);
1015
+ opts.signal?.throwIfAborted();
1016
+ const gens = /* @__PURE__ */ new Map();
1017
+ const hits = [];
1018
+ let pending = probed.map((p) => ({
1019
+ ...p,
1020
+ limit: perList
1021
+ }));
1022
+ while (pending.length > 0) {
1023
+ const rounds = pending.map((p) => {
1024
+ const keep = (id) => !this.#catalog.tail.has(id) && !(gens.has(id) && gens.get(id) !== p.seg.gen) && (!filter || filter(id));
1025
+ return {
1026
+ p,
1027
+ cands: p.seg.filterList(q, p.list, p.data, p.limit, keep)
1028
+ };
1029
+ });
1030
+ const unknown = rounds.flatMap((r) => r.cands.map((c) => c.id)).filter((id) => !gens.has(id));
1031
+ if (unknown.length > 0) {
1032
+ const found = await this.#catalog.generations(unknown);
1033
+ for (const id of unknown) gens.set(id, found.get(id));
1034
+ opts.signal?.throwIfAborted();
1035
+ }
1036
+ pending = [];
1037
+ for (const { p, cands } of rounds) {
1038
+ const live = cands.filter((c) => gens.get(c.id) === p.seg.gen);
1039
+ const whole = p.seg.passesWhole(p.data.ids.length, p.limit);
1040
+ if (!whole && live.length < perList && cands.length === p.limit) {
1041
+ pending.push({
1042
+ ...p,
1043
+ limit: p.limit * 2
1044
+ });
1045
+ continue;
1046
+ }
1047
+ for (const cand of whole ? live : live.slice(0, perList)) hits.push({
1048
+ ...cand,
1049
+ seg: p.seg,
1050
+ data: p.data
1051
+ });
1052
+ }
799
1053
  }
1054
+ const segTop = new TopK(rerank);
1055
+ for (const h of hits) segTop.push(h.seg.scoreInt8(q, h.data, h.row), h);
800
1056
  let segResults = segTop.sorted().map((e) => ({
801
1057
  id: e.item.id,
802
1058
  distance: e.score,
@@ -824,11 +1080,11 @@ var VectorIndex = class {
824
1080
  ids.forEach(assertId);
825
1081
  return this.#read(async (segments) => {
826
1082
  const out = /* @__PURE__ */ new Map();
827
- const gens = await this.#generations(ids);
1083
+ const gens = await this.#catalog.generations(ids);
828
1084
  const wanted = /* @__PURE__ */ new Map();
829
1085
  for (const [id, gen] of gens) {
830
1086
  if (gen === 0) {
831
- const entry = this.#tail.get(id);
1087
+ const entry = this.#catalog.tail.get(id);
832
1088
  if (entry) out.set(id, entry.vector.slice());
833
1089
  continue;
834
1090
  }
@@ -860,8 +1116,8 @@ var VectorIndex = class {
860
1116
  this.#assertReady();
861
1117
  return this.#maintenance(async () => {
862
1118
  const snapshot = await this.#exclusive(async () => ({
863
- maxSeq: this.#state.seq,
864
- entries: [...this.#tail.entries()]
1119
+ maxSeq: this.#catalog.state.seq,
1120
+ entries: [...this.#catalog.tail.entries()]
865
1121
  }));
866
1122
  if (snapshot.entries.length === 0) return 0;
867
1123
  const n = snapshot.entries.length;
@@ -873,23 +1129,9 @@ var VectorIndex = class {
873
1129
  });
874
1130
  const built = this.#build(ids, vectors);
875
1131
  await this.#exclusive(() => this.#swapSegments(async () => {
876
- const ref = this.#refOf(built, n);
877
- const next = {
878
- ...this.#state,
879
- nextGen: ref.gen + 1,
880
- segments: [...this.#state.segments, ref]
881
- };
882
- const { ids: idsT, tail } = this.#t;
883
- await inTransaction(this.#db, async (tx) => {
884
- await this.#store.write(tx, ref.gen, built);
885
- await tx.query(`UPDATE ${idsT} SET gen = ? WHERE gen = 0
886
- AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`, [ref.gen, snapshot.maxSeq]);
887
- await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [snapshot.maxSeq]);
888
- await this.#saveState(tx, next);
1132
+ await this.#catalog.seal(snapshot.maxSeq, built, n, (ref) => {
1133
+ this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
889
1134
  });
890
- this.#state = next;
891
- for (const [id, e] of this.#tail) if (e.seq <= snapshot.maxSeq) this.#tail.delete(id);
892
- this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
893
1135
  }));
894
1136
  return n;
895
1137
  });
@@ -910,7 +1152,7 @@ var VectorIndex = class {
910
1152
  if (seg.counts[c] === 0) continue;
911
1153
  const [data] = await this.#store.lists(seg, [c]);
912
1154
  const floats = await this.#store.floats(seg, c);
913
- const live = await this.#generations(Array.from(data.ids));
1155
+ const live = await this.#catalog.generations(Array.from(data.ids));
914
1156
  for (let r = 0; r < data.ids.length; r++) {
915
1157
  const id = data.ids[r];
916
1158
  if (live.get(id) !== seg.gen) continue;
@@ -925,23 +1167,10 @@ var VectorIndex = class {
925
1167
  });
926
1168
  const built = n > 0 ? this.#build(Float64Array.from(idList), vectors) : null;
927
1169
  await this.#exclusive(() => this.#swapSegments(async () => {
928
- const ref = built ? this.#refOf(built, n) : null;
929
- const kept = this.#state.segments.filter((s) => !oldGens.includes(s.gen));
930
- const next = {
931
- ...this.#state,
932
- nextGen: ref ? ref.gen + 1 : this.#state.nextGen,
933
- segments: ref ? [...kept, ref] : kept
934
- };
935
- await inTransaction(this.#db, async (tx) => {
936
- if (ref && built) {
937
- await this.#store.write(tx, ref.gen, built);
938
- await tx.query(`UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`, [ref.gen, JSON.stringify(oldGens)]);
939
- }
940
- await this.#saveState(tx, next);
1170
+ await this.#catalog.replace(oldGens, built, n, (ref) => {
1171
+ this.#segments = [...this.#segments.filter((s) => !oldGens.includes(s.gen)), ...ref && built ? [this.#segmentOf(ref, built)] : []];
1172
+ this.#retired.push(...old);
941
1173
  });
942
- this.#state = next;
943
- this.#segments = [...this.#segments.filter((s) => !oldGens.includes(s.gen)), ...ref && built ? [this.#segmentOf(ref, built)] : []];
944
- this.#retired.push(...old);
945
1174
  }));
946
1175
  await this.#reap();
947
1176
  return n;
@@ -961,37 +1190,16 @@ var VectorIndex = class {
961
1190
  nlist: this.#nlist?.(ids.length)
962
1191
  });
963
1192
  }
964
- #refOf(built, count) {
965
- return {
966
- gen: this.#state.nextGen,
967
- count,
968
- nlist: built.nlist
969
- };
970
- }
971
1193
  #segmentOf(ref, built) {
972
1194
  return new SegmentIndex(ref.gen, this.dims, this.metric, built.centroids, built.counts);
973
1195
  }
974
- async #generations(ids) {
975
- const out = /* @__PURE__ */ new Map();
976
- const unique = [...new Set(ids)];
977
- for (let i = 0; i < unique.length; i += VALIDATE_BATCH) {
978
- const rows = await this.#db.query(`SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [JSON.stringify(unique.slice(i, i + VALIDATE_BATCH))]);
979
- for (const r of rows) out.set(Number(r.id), Number(r.gen));
980
- }
981
- return out;
982
- }
983
- async #saveState(tx, state) {
984
- await tx.query(`INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)
985
- ON CONFLICT(k) DO UPDATE SET v = excluded.v`, [JSON.stringify(state)]);
986
- }
987
1196
  /** Deletes the rows of retired segments once no search is reading them. */
988
1197
  async #reap() {
989
1198
  const ready = this.#retired.filter((s) => (this.#refs.get(s) ?? 0) === 0);
990
1199
  if (ready.length === 0) return;
991
1200
  this.#retired = this.#retired.filter((s) => !ready.includes(s));
992
1201
  for (const s of ready) this.#refs.delete(s);
993
- const gens = ready.map((s) => s.gen);
994
- await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));
1202
+ await this.#catalog.dropGenerations(ready.map((s) => s.gen));
995
1203
  }
996
1204
  /**
997
1205
  * Runs `read` over a snapshot of the segments, held against reaping. A read
@@ -1029,11 +1237,16 @@ var VectorIndex = class {
1029
1237
  async #maintenance(fn) {
1030
1238
  if (this.#maintaining) throw new Error("vector-index: compact or merge already running");
1031
1239
  this.#maintaining = true;
1032
- try {
1033
- return await fn();
1034
- } finally {
1035
- this.#maintaining = false;
1036
- }
1240
+ const run = (async () => {
1241
+ try {
1242
+ return await fn();
1243
+ } finally {
1244
+ this.#maintaining = false;
1245
+ this.#maintenanceRun = null;
1246
+ }
1247
+ })();
1248
+ this.#maintenanceRun = run;
1249
+ return run;
1037
1250
  }
1038
1251
  #assertReady() {
1039
1252
  if (!this.#ready) throw new Error("vector-index: call init() first");
@@ -1045,16 +1258,7 @@ function assertId(id) {
1045
1258
  /** Drops every table of the index under `prefix`, in one transaction. Needs no dims or metric. */
1046
1259
  async function dropVectorIndex(db, prefix) {
1047
1260
  if (!PREFIX_PATTERN.test(prefix)) throw new Error(`vector-index: bad prefix ${prefix}`);
1048
- await inTransaction(db, async (tx) => {
1049
- for (const t of [
1050
- "meta",
1051
- "ids",
1052
- "tail",
1053
- "centroids",
1054
- "seg",
1055
- "f32"
1056
- ]) await tx.exec(`DROP TABLE IF EXISTS ${prefix}${t}`);
1057
- });
1261
+ await dropCatalog(db, prefix);
1058
1262
  }
1059
1263
  //#endregion
1060
1264
  export { VectorIndex, dropVectorIndex };