@statewalker/db-sqlite-index 0.1.2 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/vector.d.mts CHANGED
@@ -70,7 +70,9 @@ interface VectorIndexStats {
70
70
  * into one, dropping dead rows.
71
71
  *
72
72
  * One writer per database is assumed (a Durable Object, a worker, one tab);
73
- * within an instance, writes are serialized.
73
+ * within an instance, writes are serialized. The tables, the committed state and
74
+ * the in-memory tail belong to the generation catalog (`catalog.ts`); this class decides
75
+ * what to commit and when, builds segments and searches them.
74
76
  */
75
77
  export declare class VectorIndex {
76
78
  #private;
@@ -81,7 +83,11 @@ export declare class VectorIndex {
81
83
  constructor(opts: VectorIndexOptions);
82
84
  /** Creates tables if missing, loads state and sweeps rows of crashed maintenance. */
83
85
  init(): Promise<void>;
84
- /** Reloads state, tail and segments from storage (e.g. after another writer changed them). */
86
+ /**
87
+ * Reloads state, tail and segments from storage (e.g. after another writer changed them).
88
+ * Waits for a running compact() or merge() and runs between writes, so it never mixes the
89
+ * state before a commit with the rows after it (#26).
90
+ */
85
91
  refresh(): Promise<void>;
86
92
  get stats(): VectorIndexStats;
87
93
  upsert(items: VectorItem[]): Promise<void>;
@@ -1 +1 @@
1
- {"version":3,"file":"vector.d.mts","names":[],"sources":["../src/vector/types.ts","../src/vector/sql-segment-store.ts","../src/vector/vector-index.ts"],"mappings":";;KAAY;UAEK;EACf;;EAEA;;UAGe;EACf;;;;;EAKA;;EAEA;;EAEA;;EAEA,UAAU;EACV,SAAS;;;;UCXM;EACf;EACA;EACA;;;;UCOe;EACf,IAAI;;EAEJ;EACA;EACA,SAAS;;EAET,SAAS;;EAET;;;;;;EAMA;;UAGe;;EAEf;EACA,QAAQ;;UAGO;EACf;EACA,UAAU;EACV;;;;;;;;;;;;;;;;;qBAyCW;;WACF;WACA,QAAQ;;WAER;EAkBT,YAAY,MAAM;;EAqBZ,QAAQ;;EA+BR,WAAW;MAqBb,SAAS;EAUP,OAAO,OAAO,eAAe;EAiC7B,OAAO,gBAAgB;EAwBvB,OAAO,OAAO,mBAAmB,OAAM,gBAAqB,QAAQ;;EA2GpE,WAAW,gBAAgB,QAAQ,YAAY;;EAwC/C,WAAW;;;;;EAkDX,SAAS;;EAgET,YAAY;;;wBAiHE,gBAAgB,IAAI,IAAI,iBAAiB"}
1
+ {"version":3,"file":"vector.d.mts","names":[],"sources":["../src/vector/types.ts","../src/vector/sql-segment-store.ts","../src/vector/vector-index.ts"],"mappings":";;KAAY;UAEK;EACf;;EAEA;;UAGe;EACf;;;;;EAKA;;EAEA;;EAEA;;EAEA,UAAU;EACV,SAAS;;;;UCXM;EACf;EACA;EACA;;;;UCCe;EACf,IAAI;;EAEJ;EACA;EACA,SAAS;;EAET,SAAS;;EAET;;;;;;EAMA;;UAGe;;EAEf;EACA,QAAQ;;UAGO;EACf;EACA,UAAU;EACV;;;;;;;;;;;;;;;;;;;qBA8BW;;WACF;WACA,QAAQ;;WAER;EAiBT,YAAY,MAAM;;EAmBZ,QAAQ;;;;;;EAYR,WAAW;MAoBb,SAAS;EAWP,OAAO,OAAO,eAAe;EAc7B,OAAO,gBAAgB;EAQvB,OAAO,OAAO,mBAAmB,OAAM,gBAAqB,QAAQ;;EA2GpE,WAAW,gBAAgB,QAAQ,YAAY;;EAwC/C,WAAW;;;;;EAiCX,SAAS;;EAgDT,YAAY;;;wBA4FE,gBAAgB,IAAI,IAAI,iBAAiB"}
package/dist/vector.mjs CHANGED
@@ -481,19 +481,23 @@ const PART_BYTES = 1048576;
481
481
  * float32 blocks split into parts. Reads go through an LRU cache: a
482
482
  * generation is immutable, so cached blocks never go stale.
483
483
  */
484
- var SqlSegmentStore = class {
484
+ var SqlSegmentStore = class SqlSegmentStore {
485
485
  #db;
486
486
  #dims;
487
487
  #t;
488
488
  #cache;
489
- constructor(db, prefix, dims, cacheBytes = 67108864) {
490
- this.#db = db;
491
- this.#dims = dims;
492
- this.#t = {
489
+ /** The names of the tables segment rows live in, under `prefix`. */
490
+ static tables(prefix) {
491
+ return {
493
492
  centroids: `${prefix}centroids`,
494
493
  seg: `${prefix}seg`,
495
494
  f32: `${prefix}f32`
496
495
  };
496
+ }
497
+ constructor(db, prefix, dims, cacheBytes = 67108864) {
498
+ this.#db = db;
499
+ this.#dims = dims;
500
+ this.#t = SqlSegmentStore.tables(prefix);
497
501
  this.#cache = new Lru(cacheBytes);
498
502
  }
499
503
  schema() {
@@ -647,9 +651,222 @@ var Lru = class {
647
651
  }
648
652
  };
649
653
  //#endregion
654
+ //#region src/vector/catalog.ts
655
+ /** Ids per statement (bound as one JSON parameter). */
656
+ const ID_BATCH = 2e3;
657
+ const EMPTY = {
658
+ seq: 0,
659
+ nextGen: 1,
660
+ segments: []
661
+ };
662
+ /**
663
+ * The generation catalog of a vector index: the committed state, the `{p}ids` table (which
664
+ * generation holds the live copy of each id, 0 = the tail), the `{p}tail` rows, and the in-memory
665
+ * copy of the state and the tail.
666
+ *
667
+ * It is the only place that commits a state change, and every commit updates the copy in the
668
+ * same synchronous step as it returns from the transaction, so the copy always matches storage.
669
+ * It also creates, sweeps and drops every table under the prefix, asking the segment store for
670
+ * its own.
671
+ *
672
+ * Callers serialize the commits (one at a time per instance); reads of `state` and `tail` may
673
+ * happen at any time and see either the copy before a commit or the copy after it.
674
+ */
675
+ var GenerationCatalog = class {
676
+ /** `{p}ids`: its `id` column holds exactly the live ids. */
677
+ idTable;
678
+ #db;
679
+ #store;
680
+ #t;
681
+ #state = EMPTY;
682
+ #tail = /* @__PURE__ */ new Map();
683
+ constructor(db, prefix, store) {
684
+ this.#db = db;
685
+ this.#store = store;
686
+ this.#t = catalogTables(prefix);
687
+ this.idTable = this.#t.ids;
688
+ }
689
+ /** The committed state, as of the last commit or `adopt()`. Never mutated: replaced. */
690
+ get state() {
691
+ return this.#state;
692
+ }
693
+ /** The tail rows in memory, by id. */
694
+ get tail() {
695
+ return this.#tail;
696
+ }
697
+ /**
698
+ * Creates every table if missing and checks the stored config (dims and metric) is `config`:
699
+ * the first open stores it, later opens must match.
700
+ */
701
+ async create(config) {
702
+ const { meta, ids, tail } = this.#t;
703
+ for (const s of [
704
+ `CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,
705
+ `CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,
706
+ `CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,
707
+ `CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,
708
+ `CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,
709
+ ...this.#store.schema()
710
+ ]) await this.#db.exec(s);
711
+ await this.#db.query(`INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`, [config]);
712
+ const [stored] = await this.#db.query(`SELECT v FROM ${meta} WHERE k = 'config'`);
713
+ if (stored?.v !== config) throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);
714
+ }
715
+ /** Deletes segment rows of every generation the committed state does not list (crash leftovers). */
716
+ async sweep() {
717
+ const live = this.#state.segments.map((s) => s.gen);
718
+ await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));
719
+ }
720
+ /** Deletes the segment rows of retired generations. */
721
+ async dropGenerations(gens) {
722
+ await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));
723
+ }
724
+ /** Reads the committed state and the tail rows, without installing them. */
725
+ async read() {
726
+ const [row] = await this.#db.query(`SELECT v FROM ${this.#t.meta} WHERE k = 'state'`);
727
+ const state = row ? JSON.parse(row.v) : EMPTY;
728
+ const tail = /* @__PURE__ */ new Map();
729
+ const rows = await this.#db.query(`SELECT id, seq, vec FROM ${this.#t.tail}`);
730
+ for (const r of rows) tail.set(Number(r.id), {
731
+ seq: Number(r.seq),
732
+ vector: toFloat32(r.vec)
733
+ });
734
+ return {
735
+ state,
736
+ tail
737
+ };
738
+ }
739
+ /** Installs a snapshot from `read()` as the copy. No commit may run between the two. */
740
+ adopt(snapshot) {
741
+ this.#state = snapshot.state;
742
+ this.#tail = snapshot.tail;
743
+ }
744
+ /** The generation holding the live copy of each id; deleted or unknown ids are absent. */
745
+ async generations(ids) {
746
+ const out = /* @__PURE__ */ new Map();
747
+ const unique = [...new Set(ids)];
748
+ for (let i = 0; i < unique.length; i += ID_BATCH) {
749
+ const rows = await this.#db.query(`SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [JSON.stringify(unique.slice(i, i + ID_BATCH))]);
750
+ for (const r of rows) out.set(Number(r.id), Number(r.gen));
751
+ }
752
+ return out;
753
+ }
754
+ /** Writes `items` to the tail as one new batch (the next seq), in one transaction. */
755
+ async append(items) {
756
+ const { tail, ids } = this.#t;
757
+ const seq = this.#state.seq + 1;
758
+ const next = {
759
+ ...this.#state,
760
+ seq
761
+ };
762
+ await inTransaction(this.#db, async (tx) => {
763
+ for (const { id, vector } of items) {
764
+ await tx.query(`INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)
765
+ ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`, [
766
+ id,
767
+ seq,
768
+ float32Bytes(vector)
769
+ ]);
770
+ await tx.query(`INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`, [id]);
771
+ }
772
+ await this.#save(tx, next);
773
+ });
774
+ this.#state = next;
775
+ for (const { id, vector } of items) this.#tail.set(id, {
776
+ seq,
777
+ vector
778
+ });
779
+ }
780
+ /** Removes `ids` from the tail and from the live ids, one transaction per batch. */
781
+ async remove(ids) {
782
+ for (let i = 0; i < ids.length; i += ID_BATCH) {
783
+ const chunk = ids.slice(i, i + ID_BATCH);
784
+ const json = JSON.stringify(chunk);
785
+ await inTransaction(this.#db, async (tx) => {
786
+ await tx.query(`DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
787
+ await tx.query(`DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
788
+ });
789
+ for (const id of chunk) this.#tail.delete(id);
790
+ }
791
+ }
792
+ /**
793
+ * Commits `built` (sealed from the tail rows up to `maxSeq`, `count` vectors) as a new
794
+ * generation: writes its rows, moves those ids to it and deletes those tail rows. Tail rows of
795
+ * later batches stay. `install` runs with the new ref in the same synchronous step as the copy
796
+ * update, for the caller's own in-memory segment list.
797
+ */
798
+ async seal(maxSeq, built, count, install) {
799
+ const ref = this.#refOf(built, count);
800
+ const next = {
801
+ ...this.#state,
802
+ nextGen: ref.gen + 1,
803
+ segments: [...this.#state.segments, ref]
804
+ };
805
+ const { ids, tail } = this.#t;
806
+ await inTransaction(this.#db, async (tx) => {
807
+ await this.#store.write(tx, ref.gen, built);
808
+ await tx.query(`UPDATE ${ids} SET gen = ? WHERE gen = 0
809
+ AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`, [ref.gen, maxSeq]);
810
+ await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [maxSeq]);
811
+ await this.#save(tx, next);
812
+ });
813
+ this.#state = next;
814
+ for (const [id, e] of this.#tail) if (e.seq <= maxSeq) this.#tail.delete(id);
815
+ install(ref);
816
+ return ref;
817
+ }
818
+ /**
819
+ * Replaces the generations `oldGens` with `built` (`count` live vectors), or with nothing when
820
+ * `built` is null: writes its rows, moves the ids of the old generations to it and drops them
821
+ * from the state. Their rows stay until `dropGenerations()`. `install` runs as in `seal()`.
822
+ */
823
+ async replace(oldGens, built, count, install) {
824
+ const ref = built ? this.#refOf(built, count) : null;
825
+ const kept = this.#state.segments.filter((s) => !oldGens.includes(s.gen));
826
+ const next = {
827
+ ...this.#state,
828
+ nextGen: ref ? ref.gen + 1 : this.#state.nextGen,
829
+ segments: ref ? [...kept, ref] : kept
830
+ };
831
+ await inTransaction(this.#db, async (tx) => {
832
+ if (ref && built) {
833
+ await this.#store.write(tx, ref.gen, built);
834
+ await tx.query(`UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`, [ref.gen, JSON.stringify(oldGens)]);
835
+ }
836
+ await this.#save(tx, next);
837
+ });
838
+ this.#state = next;
839
+ install(ref);
840
+ return ref;
841
+ }
842
+ #refOf(built, count) {
843
+ return {
844
+ gen: this.#state.nextGen,
845
+ count,
846
+ nlist: built.nlist
847
+ };
848
+ }
849
+ async #save(tx, state) {
850
+ await tx.query(`INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)
851
+ ON CONFLICT(k) DO UPDATE SET v = excluded.v`, [JSON.stringify(state)]);
852
+ }
853
+ };
854
+ function catalogTables(prefix) {
855
+ return {
856
+ meta: `${prefix}meta`,
857
+ ids: `${prefix}ids`,
858
+ tail: `${prefix}tail`
859
+ };
860
+ }
861
+ /** Drops every table under `prefix` (the catalog's and the segment store's), in one transaction. */
862
+ async function dropCatalog(db, prefix) {
863
+ const tables = [...Object.values(catalogTables(prefix)), ...Object.values(SqlSegmentStore.tables(prefix))];
864
+ await inTransaction(db, async (tx) => {
865
+ for (const t of tables) await tx.exec(`DROP TABLE IF EXISTS ${t}`);
866
+ });
867
+ }
868
+ //#endregion
650
869
  //#region src/vector/vector-index.ts
651
- /** Ids per lookup (bound as one JSON parameter). */
652
- const VALIDATE_BATCH = 2e3;
653
870
  /** Lists probed per segment when the caller passes no `nprobe`. */
654
871
  const DEFAULT_NPROBE = 8;
655
872
  /**
@@ -672,29 +889,26 @@ const WRITE_BATCH = 200;
672
889
  * into one, dropping dead rows.
673
890
  *
674
891
  * One writer per database is assumed (a Durable Object, a worker, one tab);
675
- * within an instance, writes are serialized.
892
+ * within an instance, writes are serialized. The tables, the committed state and
893
+ * the in-memory tail belong to the generation catalog (`catalog.ts`); this class decides
894
+ * what to commit and when, builds segments and searches them.
676
895
  */
677
896
  var VectorIndex = class {
678
897
  dims;
679
898
  metric;
680
899
  /** `{prefix}ids`: its `id` column holds exactly the live ids. Callers may join it in SQL. */
681
900
  idTable;
682
- #db;
683
901
  #store;
902
+ #catalog;
684
903
  #nlist;
685
904
  #autoCompactAt;
686
- #t;
687
- #state = {
688
- seq: 0,
689
- nextGen: 1,
690
- segments: []
691
- };
692
- #tail = /* @__PURE__ */ new Map();
693
905
  #segments = [];
694
906
  #refs = /* @__PURE__ */ new Map();
695
907
  #retired = [];
696
908
  #queue = Promise.resolve();
697
909
  #maintaining = false;
910
+ /** The running compact() or merge(), for refresh() to wait on. */
911
+ #maintenanceRun = null;
698
912
  #ready = false;
699
913
  /** Bumped before and after each commit that changes the segment list: odd while one runs. */
700
914
  #epoch = 0;
@@ -703,66 +917,49 @@ var VectorIndex = class {
703
917
  if (!Number.isInteger(opts.dims) || opts.dims < 1) throw new Error("vector-index: dims must be a positive integer");
704
918
  this.dims = opts.dims;
705
919
  this.metric = opts.metric ?? "cosine";
706
- this.#db = opts.db;
707
920
  this.#store = new SqlSegmentStore(opts.db, opts.prefix, opts.dims, opts.cacheBytes);
921
+ this.#catalog = new GenerationCatalog(opts.db, opts.prefix, this.#store);
708
922
  this.#nlist = opts.nlist;
709
923
  this.#autoCompactAt = opts.autoCompactAt ?? 4096;
710
- const p = opts.prefix;
711
- this.#t = {
712
- meta: `${p}meta`,
713
- ids: `${p}ids`,
714
- tail: `${p}tail`
715
- };
716
- this.idTable = this.#t.ids;
924
+ this.idTable = this.#catalog.idTable;
717
925
  }
718
926
  /** Creates tables if missing, loads state and sweeps rows of crashed maintenance. */
719
927
  async init() {
720
- const { meta, ids, tail } = this.#t;
721
- for (const s of [
722
- `CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,
723
- `CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,
724
- `CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,
725
- `CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,
726
- `CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,
727
- ...this.#store.schema()
728
- ]) await this.#db.exec(s);
729
- const config = JSON.stringify({
928
+ await this.#catalog.create(JSON.stringify({
730
929
  dims: this.dims,
731
930
  metric: this.metric
732
- });
733
- await this.#db.query(`INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`, [config]);
734
- const [stored] = await this.#db.query(`SELECT v FROM ${meta} WHERE k = 'config'`);
735
- if (stored?.v !== config) throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);
931
+ }));
736
932
  await this.refresh();
737
- const live = this.#state.segments.map((s) => s.gen);
738
- await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));
933
+ await this.#catalog.sweep();
739
934
  this.#ready = true;
740
935
  }
741
- /** Reloads state, tail and segments from storage (e.g. after another writer changed them). */
936
+ /**
937
+ * Reloads state, tail and segments from storage (e.g. after another writer changed them).
938
+ * Waits for a running compact() or merge() and runs between writes, so it never mixes the
939
+ * state before a commit with the rows after it (#26).
940
+ */
742
941
  async refresh() {
743
- const [row] = await this.#db.query(`SELECT v FROM ${this.#t.meta} WHERE k = 'state'`);
744
- const state = row ? JSON.parse(row.v) : {
745
- seq: 0,
746
- nextGen: 1,
747
- segments: []
748
- };
749
- const tail = /* @__PURE__ */ new Map();
750
- const rows = await this.#db.query(`SELECT id, seq, vec FROM ${this.#t.tail}`);
751
- for (const r of rows) tail.set(Number(r.id), {
752
- seq: Number(r.seq),
753
- vector: toFloat32(r.vec)
754
- });
755
- const known = new Map(this.#segments.map((s) => [s.gen, s]));
756
- const segments = await Promise.all(state.segments.map((s) => known.get(s.gen) ?? this.#store.open(s, this.metric)));
757
- this.#state = state;
758
- this.#tail = tail;
759
- this.#segments = segments;
942
+ for (;;) {
943
+ while (this.#maintenanceRun) await this.#maintenanceRun.catch(() => {});
944
+ if (await this.#exclusive(async () => {
945
+ if (this.#maintaining) return false;
946
+ const snapshot = await this.#catalog.read();
947
+ const known = new Map(this.#segments.map((s) => [s.gen, s]));
948
+ const segments = await Promise.all(snapshot.state.segments.map((s) => known.get(s.gen) ?? this.#store.open(s, this.metric)));
949
+ await this.#swapSegments(async () => {
950
+ this.#catalog.adopt(snapshot);
951
+ this.#segments = segments;
952
+ });
953
+ return true;
954
+ })) return;
955
+ }
760
956
  }
761
957
  get stats() {
958
+ const { state, tail } = this.#catalog;
762
959
  return {
763
- tail: this.#tail.size,
764
- segments: this.#state.segments.map((s) => ({ ...s })),
765
- seq: this.#state.seq
960
+ tail: tail.size,
961
+ segments: state.segments.map((s) => ({ ...s })),
962
+ seq: state.seq
766
963
  };
767
964
  }
768
965
  async upsert(items) {
@@ -774,50 +971,16 @@ var VectorIndex = class {
774
971
  vector: prepare(it.vector, this.dims, this.metric)
775
972
  };
776
973
  });
777
- const { tail, ids } = this.#t;
778
974
  for (let i = 0; i < prepared.length; i += WRITE_BATCH) {
779
975
  const batch = prepared.slice(i, i + WRITE_BATCH);
780
- await this.#exclusive(async () => {
781
- const seq = this.#state.seq + 1;
782
- const next = {
783
- ...this.#state,
784
- seq
785
- };
786
- await inTransaction(this.#db, async (tx) => {
787
- for (const { id, vector } of batch) {
788
- await tx.query(`INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)
789
- ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`, [
790
- id,
791
- seq,
792
- float32Bytes(vector)
793
- ]);
794
- await tx.query(`INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`, [id]);
795
- }
796
- await this.#saveState(tx, next);
797
- });
798
- this.#state = next;
799
- for (const { id, vector } of batch) this.#tail.set(id, {
800
- seq,
801
- vector
802
- });
803
- });
804
- if (this.#tail.size >= this.#autoCompactAt && !this.#maintaining) await this.compact();
976
+ await this.#exclusive(() => this.#catalog.append(batch));
977
+ if (this.#catalog.tail.size >= this.#autoCompactAt && !this.#maintaining) await this.compact();
805
978
  }
806
979
  }
807
980
  async delete(ids) {
808
981
  this.#assertReady();
809
982
  ids.forEach(assertId);
810
- await this.#exclusive(async () => {
811
- for (let i = 0; i < ids.length; i += VALIDATE_BATCH) {
812
- const chunk = ids.slice(i, i + VALIDATE_BATCH);
813
- const json = JSON.stringify(chunk);
814
- await inTransaction(this.#db, async (tx) => {
815
- await tx.query(`DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
816
- await tx.query(`DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
817
- });
818
- for (const id of chunk) this.#tail.delete(id);
819
- }
820
- });
983
+ await this.#exclusive(() => this.#catalog.remove(ids));
821
984
  }
822
985
  async search(query, opts = {}) {
823
986
  this.#assertReady();
@@ -833,7 +996,7 @@ var VectorIndex = class {
833
996
  return this.#read(async (segments) => {
834
997
  opts.signal?.throwIfAborted();
835
998
  const tailTop = new TopK(k);
836
- for (const [id, { vector }] of this.#tail) {
999
+ for (const [id, { vector }] of this.#catalog.tail) {
837
1000
  if (filter && !filter(id)) continue;
838
1001
  tailTop.push(distance(this.metric, q, 0, vector, 0, d), id);
839
1002
  }
@@ -858,7 +1021,7 @@ var VectorIndex = class {
858
1021
  }));
859
1022
  while (pending.length > 0) {
860
1023
  const rounds = pending.map((p) => {
861
- const keep = (id) => !this.#tail.has(id) && !(gens.has(id) && gens.get(id) !== p.seg.gen) && (!filter || filter(id));
1024
+ const keep = (id) => !this.#catalog.tail.has(id) && !(gens.has(id) && gens.get(id) !== p.seg.gen) && (!filter || filter(id));
862
1025
  return {
863
1026
  p,
864
1027
  cands: p.seg.filterList(q, p.list, p.data, p.limit, keep)
@@ -866,7 +1029,7 @@ var VectorIndex = class {
866
1029
  });
867
1030
  const unknown = rounds.flatMap((r) => r.cands.map((c) => c.id)).filter((id) => !gens.has(id));
868
1031
  if (unknown.length > 0) {
869
- const found = await this.#generations(unknown);
1032
+ const found = await this.#catalog.generations(unknown);
870
1033
  for (const id of unknown) gens.set(id, found.get(id));
871
1034
  opts.signal?.throwIfAborted();
872
1035
  }
@@ -917,11 +1080,11 @@ var VectorIndex = class {
917
1080
  ids.forEach(assertId);
918
1081
  return this.#read(async (segments) => {
919
1082
  const out = /* @__PURE__ */ new Map();
920
- const gens = await this.#generations(ids);
1083
+ const gens = await this.#catalog.generations(ids);
921
1084
  const wanted = /* @__PURE__ */ new Map();
922
1085
  for (const [id, gen] of gens) {
923
1086
  if (gen === 0) {
924
- const entry = this.#tail.get(id);
1087
+ const entry = this.#catalog.tail.get(id);
925
1088
  if (entry) out.set(id, entry.vector.slice());
926
1089
  continue;
927
1090
  }
@@ -953,8 +1116,8 @@ var VectorIndex = class {
953
1116
  this.#assertReady();
954
1117
  return this.#maintenance(async () => {
955
1118
  const snapshot = await this.#exclusive(async () => ({
956
- maxSeq: this.#state.seq,
957
- entries: [...this.#tail.entries()]
1119
+ maxSeq: this.#catalog.state.seq,
1120
+ entries: [...this.#catalog.tail.entries()]
958
1121
  }));
959
1122
  if (snapshot.entries.length === 0) return 0;
960
1123
  const n = snapshot.entries.length;
@@ -966,23 +1129,9 @@ var VectorIndex = class {
966
1129
  });
967
1130
  const built = this.#build(ids, vectors);
968
1131
  await this.#exclusive(() => this.#swapSegments(async () => {
969
- const ref = this.#refOf(built, n);
970
- const next = {
971
- ...this.#state,
972
- nextGen: ref.gen + 1,
973
- segments: [...this.#state.segments, ref]
974
- };
975
- const { ids: idsT, tail } = this.#t;
976
- await inTransaction(this.#db, async (tx) => {
977
- await this.#store.write(tx, ref.gen, built);
978
- await tx.query(`UPDATE ${idsT} SET gen = ? WHERE gen = 0
979
- AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`, [ref.gen, snapshot.maxSeq]);
980
- await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [snapshot.maxSeq]);
981
- await this.#saveState(tx, next);
1132
+ await this.#catalog.seal(snapshot.maxSeq, built, n, (ref) => {
1133
+ this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
982
1134
  });
983
- this.#state = next;
984
- for (const [id, e] of this.#tail) if (e.seq <= snapshot.maxSeq) this.#tail.delete(id);
985
- this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
986
1135
  }));
987
1136
  return n;
988
1137
  });
@@ -1003,7 +1152,7 @@ var VectorIndex = class {
1003
1152
  if (seg.counts[c] === 0) continue;
1004
1153
  const [data] = await this.#store.lists(seg, [c]);
1005
1154
  const floats = await this.#store.floats(seg, c);
1006
- const live = await this.#generations(Array.from(data.ids));
1155
+ const live = await this.#catalog.generations(Array.from(data.ids));
1007
1156
  for (let r = 0; r < data.ids.length; r++) {
1008
1157
  const id = data.ids[r];
1009
1158
  if (live.get(id) !== seg.gen) continue;
@@ -1018,23 +1167,10 @@ var VectorIndex = class {
1018
1167
  });
1019
1168
  const built = n > 0 ? this.#build(Float64Array.from(idList), vectors) : null;
1020
1169
  await this.#exclusive(() => this.#swapSegments(async () => {
1021
- const ref = built ? this.#refOf(built, n) : null;
1022
- const kept = this.#state.segments.filter((s) => !oldGens.includes(s.gen));
1023
- const next = {
1024
- ...this.#state,
1025
- nextGen: ref ? ref.gen + 1 : this.#state.nextGen,
1026
- segments: ref ? [...kept, ref] : kept
1027
- };
1028
- await inTransaction(this.#db, async (tx) => {
1029
- if (ref && built) {
1030
- await this.#store.write(tx, ref.gen, built);
1031
- await tx.query(`UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`, [ref.gen, JSON.stringify(oldGens)]);
1032
- }
1033
- await this.#saveState(tx, next);
1170
+ await this.#catalog.replace(oldGens, built, n, (ref) => {
1171
+ this.#segments = [...this.#segments.filter((s) => !oldGens.includes(s.gen)), ...ref && built ? [this.#segmentOf(ref, built)] : []];
1172
+ this.#retired.push(...old);
1034
1173
  });
1035
- this.#state = next;
1036
- this.#segments = [...this.#segments.filter((s) => !oldGens.includes(s.gen)), ...ref && built ? [this.#segmentOf(ref, built)] : []];
1037
- this.#retired.push(...old);
1038
1174
  }));
1039
1175
  await this.#reap();
1040
1176
  return n;
@@ -1054,37 +1190,16 @@ var VectorIndex = class {
1054
1190
  nlist: this.#nlist?.(ids.length)
1055
1191
  });
1056
1192
  }
1057
- #refOf(built, count) {
1058
- return {
1059
- gen: this.#state.nextGen,
1060
- count,
1061
- nlist: built.nlist
1062
- };
1063
- }
1064
1193
  #segmentOf(ref, built) {
1065
1194
  return new SegmentIndex(ref.gen, this.dims, this.metric, built.centroids, built.counts);
1066
1195
  }
1067
- async #generations(ids) {
1068
- const out = /* @__PURE__ */ new Map();
1069
- const unique = [...new Set(ids)];
1070
- for (let i = 0; i < unique.length; i += VALIDATE_BATCH) {
1071
- const rows = await this.#db.query(`SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [JSON.stringify(unique.slice(i, i + VALIDATE_BATCH))]);
1072
- for (const r of rows) out.set(Number(r.id), Number(r.gen));
1073
- }
1074
- return out;
1075
- }
1076
- async #saveState(tx, state) {
1077
- await tx.query(`INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)
1078
- ON CONFLICT(k) DO UPDATE SET v = excluded.v`, [JSON.stringify(state)]);
1079
- }
1080
1196
  /** Deletes the rows of retired segments once no search is reading them. */
1081
1197
  async #reap() {
1082
1198
  const ready = this.#retired.filter((s) => (this.#refs.get(s) ?? 0) === 0);
1083
1199
  if (ready.length === 0) return;
1084
1200
  this.#retired = this.#retired.filter((s) => !ready.includes(s));
1085
1201
  for (const s of ready) this.#refs.delete(s);
1086
- const gens = ready.map((s) => s.gen);
1087
- await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));
1202
+ await this.#catalog.dropGenerations(ready.map((s) => s.gen));
1088
1203
  }
1089
1204
  /**
1090
1205
  * Runs `read` over a snapshot of the segments, held against reaping. A read
@@ -1122,11 +1237,16 @@ var VectorIndex = class {
1122
1237
  async #maintenance(fn) {
1123
1238
  if (this.#maintaining) throw new Error("vector-index: compact or merge already running");
1124
1239
  this.#maintaining = true;
1125
- try {
1126
- return await fn();
1127
- } finally {
1128
- this.#maintaining = false;
1129
- }
1240
+ const run = (async () => {
1241
+ try {
1242
+ return await fn();
1243
+ } finally {
1244
+ this.#maintaining = false;
1245
+ this.#maintenanceRun = null;
1246
+ }
1247
+ })();
1248
+ this.#maintenanceRun = run;
1249
+ return run;
1130
1250
  }
1131
1251
  #assertReady() {
1132
1252
  if (!this.#ready) throw new Error("vector-index: call init() first");
@@ -1138,16 +1258,7 @@ function assertId(id) {
1138
1258
  /** Drops every table of the index under `prefix`, in one transaction. Needs no dims or metric. */
1139
1259
  async function dropVectorIndex(db, prefix) {
1140
1260
  if (!PREFIX_PATTERN.test(prefix)) throw new Error(`vector-index: bad prefix ${prefix}`);
1141
- await inTransaction(db, async (tx) => {
1142
- for (const t of [
1143
- "meta",
1144
- "ids",
1145
- "tail",
1146
- "centroids",
1147
- "seg",
1148
- "f32"
1149
- ]) await tx.exec(`DROP TABLE IF EXISTS ${prefix}${t}`);
1150
- });
1261
+ await dropCatalog(db, prefix);
1151
1262
  }
1152
1263
  //#endregion
1153
1264
  export { VectorIndex, dropVectorIndex };