@statewalker/db-sqlite-index 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/vector.d.mts +8 -2
- package/dist/vector.d.mts.map +1 -1
- package/dist/vector.mjs +286 -175
- package/dist/vector.mjs.map +1 -1
- package/package.json +6 -6
- package/src/vector/catalog.ts +285 -0
- package/src/vector/sql-segment-store.ts +6 -1
- package/src/vector/vector-index.ts +70 -202
package/dist/vector.d.mts
CHANGED
|
@@ -70,7 +70,9 @@ interface VectorIndexStats {
|
|
|
70
70
|
* into one, dropping dead rows.
|
|
71
71
|
*
|
|
72
72
|
* One writer per database is assumed (a Durable Object, a worker, one tab);
|
|
73
|
-
* within an instance, writes are serialized.
|
|
73
|
+
* within an instance, writes are serialized. The tables, the committed state and
|
|
74
|
+
* the in-memory tail belong to the generation catalog (`catalog.ts`); this class decides
|
|
75
|
+
* what to commit and when, builds segments and searches them.
|
|
74
76
|
*/
|
|
75
77
|
export declare class VectorIndex {
|
|
76
78
|
#private;
|
|
@@ -81,7 +83,11 @@ export declare class VectorIndex {
|
|
|
81
83
|
constructor(opts: VectorIndexOptions);
|
|
82
84
|
/** Creates tables if missing, loads state and sweeps rows of crashed maintenance. */
|
|
83
85
|
init(): Promise<void>;
|
|
84
|
-
/**
|
|
86
|
+
/**
|
|
87
|
+
* Reloads state, tail and segments from storage (e.g. after another writer changed them).
|
|
88
|
+
* Waits for a running compact() or merge() and runs between writes, so it never mixes the
|
|
89
|
+
* state before a commit with the rows after it (#26).
|
|
90
|
+
*/
|
|
85
91
|
refresh(): Promise<void>;
|
|
86
92
|
get stats(): VectorIndexStats;
|
|
87
93
|
upsert(items: VectorItem[]): Promise<void>;
|
package/dist/vector.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector.d.mts","names":[],"sources":["../src/vector/types.ts","../src/vector/sql-segment-store.ts","../src/vector/vector-index.ts"],"mappings":";;KAAY;UAEK;EACf;;EAEA;;UAGe;EACf;;;;;EAKA;;EAEA;;EAEA;;EAEA,UAAU;EACV,SAAS;;;;UCXM;EACf;EACA;EACA;;;;
|
|
1
|
+
{"version":3,"file":"vector.d.mts","names":[],"sources":["../src/vector/types.ts","../src/vector/sql-segment-store.ts","../src/vector/vector-index.ts"],"mappings":";;KAAY;UAEK;EACf;;EAEA;;UAGe;EACf;;;;;EAKA;;EAEA;;EAEA;;EAEA,UAAU;EACV,SAAS;;;;UCXM;EACf;EACA;EACA;;;;UCCe;EACf,IAAI;;EAEJ;EACA;EACA,SAAS;;EAET,SAAS;;EAET;;;;;;EAMA;;UAGe;;EAEf;EACA,QAAQ;;UAGO;EACf;EACA,UAAU;EACV;;;;;;;;;;;;;;;;;;;qBA8BW;;WACF;WACA,QAAQ;;WAER;EAiBT,YAAY,MAAM;;EAmBZ,QAAQ;;;;;;EAYR,WAAW;MAoBb,SAAS;EAWP,OAAO,OAAO,eAAe;EAc7B,OAAO,gBAAgB;EAQvB,OAAO,OAAO,mBAAmB,OAAM,gBAAqB,QAAQ;;EA2GpE,WAAW,gBAAgB,QAAQ,YAAY;;EAwC/C,WAAW;;;;;EAiCX,SAAS;;EAgDT,YAAY;;;wBA4FE,gBAAgB,IAAI,IAAI,iBAAiB"}
|
package/dist/vector.mjs
CHANGED
|
@@ -481,19 +481,23 @@ const PART_BYTES = 1048576;
|
|
|
481
481
|
* float32 blocks split into parts. Reads go through an LRU cache: a
|
|
482
482
|
* generation is immutable, so cached blocks never go stale.
|
|
483
483
|
*/
|
|
484
|
-
var SqlSegmentStore = class {
|
|
484
|
+
var SqlSegmentStore = class SqlSegmentStore {
|
|
485
485
|
#db;
|
|
486
486
|
#dims;
|
|
487
487
|
#t;
|
|
488
488
|
#cache;
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
this.#t = {
|
|
489
|
+
/** The names of the tables segment rows live in, under `prefix`. */
|
|
490
|
+
static tables(prefix) {
|
|
491
|
+
return {
|
|
493
492
|
centroids: `${prefix}centroids`,
|
|
494
493
|
seg: `${prefix}seg`,
|
|
495
494
|
f32: `${prefix}f32`
|
|
496
495
|
};
|
|
496
|
+
}
|
|
497
|
+
constructor(db, prefix, dims, cacheBytes = 67108864) {
|
|
498
|
+
this.#db = db;
|
|
499
|
+
this.#dims = dims;
|
|
500
|
+
this.#t = SqlSegmentStore.tables(prefix);
|
|
497
501
|
this.#cache = new Lru(cacheBytes);
|
|
498
502
|
}
|
|
499
503
|
schema() {
|
|
@@ -647,9 +651,222 @@ var Lru = class {
|
|
|
647
651
|
}
|
|
648
652
|
};
|
|
649
653
|
//#endregion
|
|
654
|
+
//#region src/vector/catalog.ts
|
|
655
|
+
/** Ids per statement (bound as one JSON parameter). */
|
|
656
|
+
const ID_BATCH = 2e3;
|
|
657
|
+
const EMPTY = {
|
|
658
|
+
seq: 0,
|
|
659
|
+
nextGen: 1,
|
|
660
|
+
segments: []
|
|
661
|
+
};
|
|
662
|
+
/**
|
|
663
|
+
* The generation catalog of a vector index: the committed state, the `{p}ids` table (which
|
|
664
|
+
* generation holds the live copy of each id, 0 = the tail), the `{p}tail` rows, and the in-memory
|
|
665
|
+
* copy of the state and the tail.
|
|
666
|
+
*
|
|
667
|
+
* It is the only place that commits a state change, and every commit updates the copy in the
|
|
668
|
+
* same synchronous step as it returns from the transaction, so the copy always matches storage.
|
|
669
|
+
* It also creates, sweeps and drops every table under the prefix, asking the segment store for
|
|
670
|
+
* its own.
|
|
671
|
+
*
|
|
672
|
+
* Callers serialize the commits (one at a time per instance); reads of `state` and `tail` may
|
|
673
|
+
* happen at any time and see either the copy before a commit or the copy after it.
|
|
674
|
+
*/
|
|
675
|
+
var GenerationCatalog = class {
|
|
676
|
+
/** `{p}ids`: its `id` column holds exactly the live ids. */
|
|
677
|
+
idTable;
|
|
678
|
+
#db;
|
|
679
|
+
#store;
|
|
680
|
+
#t;
|
|
681
|
+
#state = EMPTY;
|
|
682
|
+
#tail = /* @__PURE__ */ new Map();
|
|
683
|
+
constructor(db, prefix, store) {
|
|
684
|
+
this.#db = db;
|
|
685
|
+
this.#store = store;
|
|
686
|
+
this.#t = catalogTables(prefix);
|
|
687
|
+
this.idTable = this.#t.ids;
|
|
688
|
+
}
|
|
689
|
+
/** The committed state, as of the last commit or `adopt()`. Never mutated: replaced. */
|
|
690
|
+
get state() {
|
|
691
|
+
return this.#state;
|
|
692
|
+
}
|
|
693
|
+
/** The tail rows in memory, by id. */
|
|
694
|
+
get tail() {
|
|
695
|
+
return this.#tail;
|
|
696
|
+
}
|
|
697
|
+
/**
|
|
698
|
+
* Creates every table if missing and checks the stored config (dims and metric) is `config`:
|
|
699
|
+
* the first open stores it, later opens must match.
|
|
700
|
+
*/
|
|
701
|
+
async create(config) {
|
|
702
|
+
const { meta, ids, tail } = this.#t;
|
|
703
|
+
for (const s of [
|
|
704
|
+
`CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,
|
|
705
|
+
`CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,
|
|
706
|
+
`CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,
|
|
707
|
+
`CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,
|
|
708
|
+
`CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,
|
|
709
|
+
...this.#store.schema()
|
|
710
|
+
]) await this.#db.exec(s);
|
|
711
|
+
await this.#db.query(`INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`, [config]);
|
|
712
|
+
const [stored] = await this.#db.query(`SELECT v FROM ${meta} WHERE k = 'config'`);
|
|
713
|
+
if (stored?.v !== config) throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);
|
|
714
|
+
}
|
|
715
|
+
/** Deletes segment rows of every generation the committed state does not list (crash leftovers). */
|
|
716
|
+
async sweep() {
|
|
717
|
+
const live = this.#state.segments.map((s) => s.gen);
|
|
718
|
+
await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));
|
|
719
|
+
}
|
|
720
|
+
/** Deletes the segment rows of retired generations. */
|
|
721
|
+
async dropGenerations(gens) {
|
|
722
|
+
await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));
|
|
723
|
+
}
|
|
724
|
+
/** Reads the committed state and the tail rows, without installing them. */
|
|
725
|
+
async read() {
|
|
726
|
+
const [row] = await this.#db.query(`SELECT v FROM ${this.#t.meta} WHERE k = 'state'`);
|
|
727
|
+
const state = row ? JSON.parse(row.v) : EMPTY;
|
|
728
|
+
const tail = /* @__PURE__ */ new Map();
|
|
729
|
+
const rows = await this.#db.query(`SELECT id, seq, vec FROM ${this.#t.tail}`);
|
|
730
|
+
for (const r of rows) tail.set(Number(r.id), {
|
|
731
|
+
seq: Number(r.seq),
|
|
732
|
+
vector: toFloat32(r.vec)
|
|
733
|
+
});
|
|
734
|
+
return {
|
|
735
|
+
state,
|
|
736
|
+
tail
|
|
737
|
+
};
|
|
738
|
+
}
|
|
739
|
+
/** Installs a snapshot from `read()` as the copy. No commit may run between the two. */
|
|
740
|
+
adopt(snapshot) {
|
|
741
|
+
this.#state = snapshot.state;
|
|
742
|
+
this.#tail = snapshot.tail;
|
|
743
|
+
}
|
|
744
|
+
/** The generation holding the live copy of each id; deleted or unknown ids are absent. */
|
|
745
|
+
async generations(ids) {
|
|
746
|
+
const out = /* @__PURE__ */ new Map();
|
|
747
|
+
const unique = [...new Set(ids)];
|
|
748
|
+
for (let i = 0; i < unique.length; i += ID_BATCH) {
|
|
749
|
+
const rows = await this.#db.query(`SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [JSON.stringify(unique.slice(i, i + ID_BATCH))]);
|
|
750
|
+
for (const r of rows) out.set(Number(r.id), Number(r.gen));
|
|
751
|
+
}
|
|
752
|
+
return out;
|
|
753
|
+
}
|
|
754
|
+
/** Writes `items` to the tail as one new batch (the next seq), in one transaction. */
|
|
755
|
+
async append(items) {
|
|
756
|
+
const { tail, ids } = this.#t;
|
|
757
|
+
const seq = this.#state.seq + 1;
|
|
758
|
+
const next = {
|
|
759
|
+
...this.#state,
|
|
760
|
+
seq
|
|
761
|
+
};
|
|
762
|
+
await inTransaction(this.#db, async (tx) => {
|
|
763
|
+
for (const { id, vector } of items) {
|
|
764
|
+
await tx.query(`INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)
|
|
765
|
+
ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`, [
|
|
766
|
+
id,
|
|
767
|
+
seq,
|
|
768
|
+
float32Bytes(vector)
|
|
769
|
+
]);
|
|
770
|
+
await tx.query(`INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`, [id]);
|
|
771
|
+
}
|
|
772
|
+
await this.#save(tx, next);
|
|
773
|
+
});
|
|
774
|
+
this.#state = next;
|
|
775
|
+
for (const { id, vector } of items) this.#tail.set(id, {
|
|
776
|
+
seq,
|
|
777
|
+
vector
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
/** Removes `ids` from the tail and from the live ids, one transaction per batch. */
|
|
781
|
+
async remove(ids) {
|
|
782
|
+
for (let i = 0; i < ids.length; i += ID_BATCH) {
|
|
783
|
+
const chunk = ids.slice(i, i + ID_BATCH);
|
|
784
|
+
const json = JSON.stringify(chunk);
|
|
785
|
+
await inTransaction(this.#db, async (tx) => {
|
|
786
|
+
await tx.query(`DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
|
|
787
|
+
await tx.query(`DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
|
|
788
|
+
});
|
|
789
|
+
for (const id of chunk) this.#tail.delete(id);
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
/**
|
|
793
|
+
* Commits `built` (sealed from the tail rows up to `maxSeq`, `count` vectors) as a new
|
|
794
|
+
* generation: writes its rows, moves those ids to it and deletes those tail rows. Tail rows of
|
|
795
|
+
* later batches stay. `install` runs with the new ref in the same synchronous step as the copy
|
|
796
|
+
* update, for the caller's own in-memory segment list.
|
|
797
|
+
*/
|
|
798
|
+
async seal(maxSeq, built, count, install) {
|
|
799
|
+
const ref = this.#refOf(built, count);
|
|
800
|
+
const next = {
|
|
801
|
+
...this.#state,
|
|
802
|
+
nextGen: ref.gen + 1,
|
|
803
|
+
segments: [...this.#state.segments, ref]
|
|
804
|
+
};
|
|
805
|
+
const { ids, tail } = this.#t;
|
|
806
|
+
await inTransaction(this.#db, async (tx) => {
|
|
807
|
+
await this.#store.write(tx, ref.gen, built);
|
|
808
|
+
await tx.query(`UPDATE ${ids} SET gen = ? WHERE gen = 0
|
|
809
|
+
AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`, [ref.gen, maxSeq]);
|
|
810
|
+
await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [maxSeq]);
|
|
811
|
+
await this.#save(tx, next);
|
|
812
|
+
});
|
|
813
|
+
this.#state = next;
|
|
814
|
+
for (const [id, e] of this.#tail) if (e.seq <= maxSeq) this.#tail.delete(id);
|
|
815
|
+
install(ref);
|
|
816
|
+
return ref;
|
|
817
|
+
}
|
|
818
|
+
/**
|
|
819
|
+
* Replaces the generations `oldGens` with `built` (`count` live vectors), or with nothing when
|
|
820
|
+
* `built` is null: writes its rows, moves the ids of the old generations to it and drops them
|
|
821
|
+
* from the state. Their rows stay until `dropGenerations()`. `install` runs as in `seal()`.
|
|
822
|
+
*/
|
|
823
|
+
async replace(oldGens, built, count, install) {
|
|
824
|
+
const ref = built ? this.#refOf(built, count) : null;
|
|
825
|
+
const kept = this.#state.segments.filter((s) => !oldGens.includes(s.gen));
|
|
826
|
+
const next = {
|
|
827
|
+
...this.#state,
|
|
828
|
+
nextGen: ref ? ref.gen + 1 : this.#state.nextGen,
|
|
829
|
+
segments: ref ? [...kept, ref] : kept
|
|
830
|
+
};
|
|
831
|
+
await inTransaction(this.#db, async (tx) => {
|
|
832
|
+
if (ref && built) {
|
|
833
|
+
await this.#store.write(tx, ref.gen, built);
|
|
834
|
+
await tx.query(`UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`, [ref.gen, JSON.stringify(oldGens)]);
|
|
835
|
+
}
|
|
836
|
+
await this.#save(tx, next);
|
|
837
|
+
});
|
|
838
|
+
this.#state = next;
|
|
839
|
+
install(ref);
|
|
840
|
+
return ref;
|
|
841
|
+
}
|
|
842
|
+
#refOf(built, count) {
|
|
843
|
+
return {
|
|
844
|
+
gen: this.#state.nextGen,
|
|
845
|
+
count,
|
|
846
|
+
nlist: built.nlist
|
|
847
|
+
};
|
|
848
|
+
}
|
|
849
|
+
async #save(tx, state) {
|
|
850
|
+
await tx.query(`INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)
|
|
851
|
+
ON CONFLICT(k) DO UPDATE SET v = excluded.v`, [JSON.stringify(state)]);
|
|
852
|
+
}
|
|
853
|
+
};
|
|
854
|
+
function catalogTables(prefix) {
|
|
855
|
+
return {
|
|
856
|
+
meta: `${prefix}meta`,
|
|
857
|
+
ids: `${prefix}ids`,
|
|
858
|
+
tail: `${prefix}tail`
|
|
859
|
+
};
|
|
860
|
+
}
|
|
861
|
+
/** Drops every table under `prefix` (the catalog's and the segment store's), in one transaction. */
|
|
862
|
+
async function dropCatalog(db, prefix) {
|
|
863
|
+
const tables = [...Object.values(catalogTables(prefix)), ...Object.values(SqlSegmentStore.tables(prefix))];
|
|
864
|
+
await inTransaction(db, async (tx) => {
|
|
865
|
+
for (const t of tables) await tx.exec(`DROP TABLE IF EXISTS ${t}`);
|
|
866
|
+
});
|
|
867
|
+
}
|
|
868
|
+
//#endregion
|
|
650
869
|
//#region src/vector/vector-index.ts
|
|
651
|
-
/** Ids per lookup (bound as one JSON parameter). */
|
|
652
|
-
const VALIDATE_BATCH = 2e3;
|
|
653
870
|
/** Lists probed per segment when the caller passes no `nprobe`. */
|
|
654
871
|
const DEFAULT_NPROBE = 8;
|
|
655
872
|
/**
|
|
@@ -672,29 +889,26 @@ const WRITE_BATCH = 200;
|
|
|
672
889
|
* into one, dropping dead rows.
|
|
673
890
|
*
|
|
674
891
|
* One writer per database is assumed (a Durable Object, a worker, one tab);
|
|
675
|
-
* within an instance, writes are serialized.
|
|
892
|
+
* within an instance, writes are serialized. The tables, the committed state and
|
|
893
|
+
* the in-memory tail belong to the generation catalog (`catalog.ts`); this class decides
|
|
894
|
+
* what to commit and when, builds segments and searches them.
|
|
676
895
|
*/
|
|
677
896
|
var VectorIndex = class {
|
|
678
897
|
dims;
|
|
679
898
|
metric;
|
|
680
899
|
/** `{prefix}ids`: its `id` column holds exactly the live ids. Callers may join it in SQL. */
|
|
681
900
|
idTable;
|
|
682
|
-
#db;
|
|
683
901
|
#store;
|
|
902
|
+
#catalog;
|
|
684
903
|
#nlist;
|
|
685
904
|
#autoCompactAt;
|
|
686
|
-
#t;
|
|
687
|
-
#state = {
|
|
688
|
-
seq: 0,
|
|
689
|
-
nextGen: 1,
|
|
690
|
-
segments: []
|
|
691
|
-
};
|
|
692
|
-
#tail = /* @__PURE__ */ new Map();
|
|
693
905
|
#segments = [];
|
|
694
906
|
#refs = /* @__PURE__ */ new Map();
|
|
695
907
|
#retired = [];
|
|
696
908
|
#queue = Promise.resolve();
|
|
697
909
|
#maintaining = false;
|
|
910
|
+
/** The running compact() or merge(), for refresh() to wait on. */
|
|
911
|
+
#maintenanceRun = null;
|
|
698
912
|
#ready = false;
|
|
699
913
|
/** Bumped before and after each commit that changes the segment list: odd while one runs. */
|
|
700
914
|
#epoch = 0;
|
|
@@ -703,66 +917,49 @@ var VectorIndex = class {
|
|
|
703
917
|
if (!Number.isInteger(opts.dims) || opts.dims < 1) throw new Error("vector-index: dims must be a positive integer");
|
|
704
918
|
this.dims = opts.dims;
|
|
705
919
|
this.metric = opts.metric ?? "cosine";
|
|
706
|
-
this.#db = opts.db;
|
|
707
920
|
this.#store = new SqlSegmentStore(opts.db, opts.prefix, opts.dims, opts.cacheBytes);
|
|
921
|
+
this.#catalog = new GenerationCatalog(opts.db, opts.prefix, this.#store);
|
|
708
922
|
this.#nlist = opts.nlist;
|
|
709
923
|
this.#autoCompactAt = opts.autoCompactAt ?? 4096;
|
|
710
|
-
|
|
711
|
-
this.#t = {
|
|
712
|
-
meta: `${p}meta`,
|
|
713
|
-
ids: `${p}ids`,
|
|
714
|
-
tail: `${p}tail`
|
|
715
|
-
};
|
|
716
|
-
this.idTable = this.#t.ids;
|
|
924
|
+
this.idTable = this.#catalog.idTable;
|
|
717
925
|
}
|
|
718
926
|
/** Creates tables if missing, loads state and sweeps rows of crashed maintenance. */
|
|
719
927
|
async init() {
|
|
720
|
-
|
|
721
|
-
for (const s of [
|
|
722
|
-
`CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,
|
|
723
|
-
`CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,
|
|
724
|
-
`CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,
|
|
725
|
-
`CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,
|
|
726
|
-
`CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,
|
|
727
|
-
...this.#store.schema()
|
|
728
|
-
]) await this.#db.exec(s);
|
|
729
|
-
const config = JSON.stringify({
|
|
928
|
+
await this.#catalog.create(JSON.stringify({
|
|
730
929
|
dims: this.dims,
|
|
731
930
|
metric: this.metric
|
|
732
|
-
});
|
|
733
|
-
await this.#db.query(`INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`, [config]);
|
|
734
|
-
const [stored] = await this.#db.query(`SELECT v FROM ${meta} WHERE k = 'config'`);
|
|
735
|
-
if (stored?.v !== config) throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);
|
|
931
|
+
}));
|
|
736
932
|
await this.refresh();
|
|
737
|
-
|
|
738
|
-
await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));
|
|
933
|
+
await this.#catalog.sweep();
|
|
739
934
|
this.#ready = true;
|
|
740
935
|
}
|
|
741
|
-
/**
|
|
936
|
+
/**
|
|
937
|
+
* Reloads state, tail and segments from storage (e.g. after another writer changed them).
|
|
938
|
+
* Waits for a running compact() or merge() and runs between writes, so it never mixes the
|
|
939
|
+
* state before a commit with the rows after it (#26).
|
|
940
|
+
*/
|
|
742
941
|
async refresh() {
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
this.#state = state;
|
|
758
|
-
this.#tail = tail;
|
|
759
|
-
this.#segments = segments;
|
|
942
|
+
for (;;) {
|
|
943
|
+
while (this.#maintenanceRun) await this.#maintenanceRun.catch(() => {});
|
|
944
|
+
if (await this.#exclusive(async () => {
|
|
945
|
+
if (this.#maintaining) return false;
|
|
946
|
+
const snapshot = await this.#catalog.read();
|
|
947
|
+
const known = new Map(this.#segments.map((s) => [s.gen, s]));
|
|
948
|
+
const segments = await Promise.all(snapshot.state.segments.map((s) => known.get(s.gen) ?? this.#store.open(s, this.metric)));
|
|
949
|
+
await this.#swapSegments(async () => {
|
|
950
|
+
this.#catalog.adopt(snapshot);
|
|
951
|
+
this.#segments = segments;
|
|
952
|
+
});
|
|
953
|
+
return true;
|
|
954
|
+
})) return;
|
|
955
|
+
}
|
|
760
956
|
}
|
|
761
957
|
get stats() {
|
|
958
|
+
const { state, tail } = this.#catalog;
|
|
762
959
|
return {
|
|
763
|
-
tail:
|
|
764
|
-
segments:
|
|
765
|
-
seq:
|
|
960
|
+
tail: tail.size,
|
|
961
|
+
segments: state.segments.map((s) => ({ ...s })),
|
|
962
|
+
seq: state.seq
|
|
766
963
|
};
|
|
767
964
|
}
|
|
768
965
|
async upsert(items) {
|
|
@@ -774,50 +971,16 @@ var VectorIndex = class {
|
|
|
774
971
|
vector: prepare(it.vector, this.dims, this.metric)
|
|
775
972
|
};
|
|
776
973
|
});
|
|
777
|
-
const { tail, ids } = this.#t;
|
|
778
974
|
for (let i = 0; i < prepared.length; i += WRITE_BATCH) {
|
|
779
975
|
const batch = prepared.slice(i, i + WRITE_BATCH);
|
|
780
|
-
await this.#exclusive(
|
|
781
|
-
|
|
782
|
-
const next = {
|
|
783
|
-
...this.#state,
|
|
784
|
-
seq
|
|
785
|
-
};
|
|
786
|
-
await inTransaction(this.#db, async (tx) => {
|
|
787
|
-
for (const { id, vector } of batch) {
|
|
788
|
-
await tx.query(`INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)
|
|
789
|
-
ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`, [
|
|
790
|
-
id,
|
|
791
|
-
seq,
|
|
792
|
-
float32Bytes(vector)
|
|
793
|
-
]);
|
|
794
|
-
await tx.query(`INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`, [id]);
|
|
795
|
-
}
|
|
796
|
-
await this.#saveState(tx, next);
|
|
797
|
-
});
|
|
798
|
-
this.#state = next;
|
|
799
|
-
for (const { id, vector } of batch) this.#tail.set(id, {
|
|
800
|
-
seq,
|
|
801
|
-
vector
|
|
802
|
-
});
|
|
803
|
-
});
|
|
804
|
-
if (this.#tail.size >= this.#autoCompactAt && !this.#maintaining) await this.compact();
|
|
976
|
+
await this.#exclusive(() => this.#catalog.append(batch));
|
|
977
|
+
if (this.#catalog.tail.size >= this.#autoCompactAt && !this.#maintaining) await this.compact();
|
|
805
978
|
}
|
|
806
979
|
}
|
|
807
980
|
async delete(ids) {
|
|
808
981
|
this.#assertReady();
|
|
809
982
|
ids.forEach(assertId);
|
|
810
|
-
await this.#exclusive(
|
|
811
|
-
for (let i = 0; i < ids.length; i += VALIDATE_BATCH) {
|
|
812
|
-
const chunk = ids.slice(i, i + VALIDATE_BATCH);
|
|
813
|
-
const json = JSON.stringify(chunk);
|
|
814
|
-
await inTransaction(this.#db, async (tx) => {
|
|
815
|
-
await tx.query(`DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
|
|
816
|
-
await tx.query(`DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [json]);
|
|
817
|
-
});
|
|
818
|
-
for (const id of chunk) this.#tail.delete(id);
|
|
819
|
-
}
|
|
820
|
-
});
|
|
983
|
+
await this.#exclusive(() => this.#catalog.remove(ids));
|
|
821
984
|
}
|
|
822
985
|
async search(query, opts = {}) {
|
|
823
986
|
this.#assertReady();
|
|
@@ -833,7 +996,7 @@ var VectorIndex = class {
|
|
|
833
996
|
return this.#read(async (segments) => {
|
|
834
997
|
opts.signal?.throwIfAborted();
|
|
835
998
|
const tailTop = new TopK(k);
|
|
836
|
-
for (const [id, { vector }] of this.#tail) {
|
|
999
|
+
for (const [id, { vector }] of this.#catalog.tail) {
|
|
837
1000
|
if (filter && !filter(id)) continue;
|
|
838
1001
|
tailTop.push(distance(this.metric, q, 0, vector, 0, d), id);
|
|
839
1002
|
}
|
|
@@ -858,7 +1021,7 @@ var VectorIndex = class {
|
|
|
858
1021
|
}));
|
|
859
1022
|
while (pending.length > 0) {
|
|
860
1023
|
const rounds = pending.map((p) => {
|
|
861
|
-
const keep = (id) => !this.#tail.has(id) && !(gens.has(id) && gens.get(id) !== p.seg.gen) && (!filter || filter(id));
|
|
1024
|
+
const keep = (id) => !this.#catalog.tail.has(id) && !(gens.has(id) && gens.get(id) !== p.seg.gen) && (!filter || filter(id));
|
|
862
1025
|
return {
|
|
863
1026
|
p,
|
|
864
1027
|
cands: p.seg.filterList(q, p.list, p.data, p.limit, keep)
|
|
@@ -866,7 +1029,7 @@ var VectorIndex = class {
|
|
|
866
1029
|
});
|
|
867
1030
|
const unknown = rounds.flatMap((r) => r.cands.map((c) => c.id)).filter((id) => !gens.has(id));
|
|
868
1031
|
if (unknown.length > 0) {
|
|
869
|
-
const found = await this.#generations(unknown);
|
|
1032
|
+
const found = await this.#catalog.generations(unknown);
|
|
870
1033
|
for (const id of unknown) gens.set(id, found.get(id));
|
|
871
1034
|
opts.signal?.throwIfAborted();
|
|
872
1035
|
}
|
|
@@ -917,11 +1080,11 @@ var VectorIndex = class {
|
|
|
917
1080
|
ids.forEach(assertId);
|
|
918
1081
|
return this.#read(async (segments) => {
|
|
919
1082
|
const out = /* @__PURE__ */ new Map();
|
|
920
|
-
const gens = await this.#generations(ids);
|
|
1083
|
+
const gens = await this.#catalog.generations(ids);
|
|
921
1084
|
const wanted = /* @__PURE__ */ new Map();
|
|
922
1085
|
for (const [id, gen] of gens) {
|
|
923
1086
|
if (gen === 0) {
|
|
924
|
-
const entry = this.#tail.get(id);
|
|
1087
|
+
const entry = this.#catalog.tail.get(id);
|
|
925
1088
|
if (entry) out.set(id, entry.vector.slice());
|
|
926
1089
|
continue;
|
|
927
1090
|
}
|
|
@@ -953,8 +1116,8 @@ var VectorIndex = class {
|
|
|
953
1116
|
this.#assertReady();
|
|
954
1117
|
return this.#maintenance(async () => {
|
|
955
1118
|
const snapshot = await this.#exclusive(async () => ({
|
|
956
|
-
maxSeq: this.#state.seq,
|
|
957
|
-
entries: [...this.#tail.entries()]
|
|
1119
|
+
maxSeq: this.#catalog.state.seq,
|
|
1120
|
+
entries: [...this.#catalog.tail.entries()]
|
|
958
1121
|
}));
|
|
959
1122
|
if (snapshot.entries.length === 0) return 0;
|
|
960
1123
|
const n = snapshot.entries.length;
|
|
@@ -966,23 +1129,9 @@ var VectorIndex = class {
|
|
|
966
1129
|
});
|
|
967
1130
|
const built = this.#build(ids, vectors);
|
|
968
1131
|
await this.#exclusive(() => this.#swapSegments(async () => {
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
...this.#state,
|
|
972
|
-
nextGen: ref.gen + 1,
|
|
973
|
-
segments: [...this.#state.segments, ref]
|
|
974
|
-
};
|
|
975
|
-
const { ids: idsT, tail } = this.#t;
|
|
976
|
-
await inTransaction(this.#db, async (tx) => {
|
|
977
|
-
await this.#store.write(tx, ref.gen, built);
|
|
978
|
-
await tx.query(`UPDATE ${idsT} SET gen = ? WHERE gen = 0
|
|
979
|
-
AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`, [ref.gen, snapshot.maxSeq]);
|
|
980
|
-
await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [snapshot.maxSeq]);
|
|
981
|
-
await this.#saveState(tx, next);
|
|
1132
|
+
await this.#catalog.seal(snapshot.maxSeq, built, n, (ref) => {
|
|
1133
|
+
this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
|
|
982
1134
|
});
|
|
983
|
-
this.#state = next;
|
|
984
|
-
for (const [id, e] of this.#tail) if (e.seq <= snapshot.maxSeq) this.#tail.delete(id);
|
|
985
|
-
this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
|
|
986
1135
|
}));
|
|
987
1136
|
return n;
|
|
988
1137
|
});
|
|
@@ -1003,7 +1152,7 @@ var VectorIndex = class {
|
|
|
1003
1152
|
if (seg.counts[c] === 0) continue;
|
|
1004
1153
|
const [data] = await this.#store.lists(seg, [c]);
|
|
1005
1154
|
const floats = await this.#store.floats(seg, c);
|
|
1006
|
-
const live = await this.#generations(Array.from(data.ids));
|
|
1155
|
+
const live = await this.#catalog.generations(Array.from(data.ids));
|
|
1007
1156
|
for (let r = 0; r < data.ids.length; r++) {
|
|
1008
1157
|
const id = data.ids[r];
|
|
1009
1158
|
if (live.get(id) !== seg.gen) continue;
|
|
@@ -1018,23 +1167,10 @@ var VectorIndex = class {
|
|
|
1018
1167
|
});
|
|
1019
1168
|
const built = n > 0 ? this.#build(Float64Array.from(idList), vectors) : null;
|
|
1020
1169
|
await this.#exclusive(() => this.#swapSegments(async () => {
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
...this.#state,
|
|
1025
|
-
nextGen: ref ? ref.gen + 1 : this.#state.nextGen,
|
|
1026
|
-
segments: ref ? [...kept, ref] : kept
|
|
1027
|
-
};
|
|
1028
|
-
await inTransaction(this.#db, async (tx) => {
|
|
1029
|
-
if (ref && built) {
|
|
1030
|
-
await this.#store.write(tx, ref.gen, built);
|
|
1031
|
-
await tx.query(`UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`, [ref.gen, JSON.stringify(oldGens)]);
|
|
1032
|
-
}
|
|
1033
|
-
await this.#saveState(tx, next);
|
|
1170
|
+
await this.#catalog.replace(oldGens, built, n, (ref) => {
|
|
1171
|
+
this.#segments = [...this.#segments.filter((s) => !oldGens.includes(s.gen)), ...ref && built ? [this.#segmentOf(ref, built)] : []];
|
|
1172
|
+
this.#retired.push(...old);
|
|
1034
1173
|
});
|
|
1035
|
-
this.#state = next;
|
|
1036
|
-
this.#segments = [...this.#segments.filter((s) => !oldGens.includes(s.gen)), ...ref && built ? [this.#segmentOf(ref, built)] : []];
|
|
1037
|
-
this.#retired.push(...old);
|
|
1038
1174
|
}));
|
|
1039
1175
|
await this.#reap();
|
|
1040
1176
|
return n;
|
|
@@ -1054,37 +1190,16 @@ var VectorIndex = class {
|
|
|
1054
1190
|
nlist: this.#nlist?.(ids.length)
|
|
1055
1191
|
});
|
|
1056
1192
|
}
|
|
1057
|
-
#refOf(built, count) {
|
|
1058
|
-
return {
|
|
1059
|
-
gen: this.#state.nextGen,
|
|
1060
|
-
count,
|
|
1061
|
-
nlist: built.nlist
|
|
1062
|
-
};
|
|
1063
|
-
}
|
|
1064
1193
|
#segmentOf(ref, built) {
|
|
1065
1194
|
return new SegmentIndex(ref.gen, this.dims, this.metric, built.centroids, built.counts);
|
|
1066
1195
|
}
|
|
1067
|
-
async #generations(ids) {
|
|
1068
|
-
const out = /* @__PURE__ */ new Map();
|
|
1069
|
-
const unique = [...new Set(ids)];
|
|
1070
|
-
for (let i = 0; i < unique.length; i += VALIDATE_BATCH) {
|
|
1071
|
-
const rows = await this.#db.query(`SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [JSON.stringify(unique.slice(i, i + VALIDATE_BATCH))]);
|
|
1072
|
-
for (const r of rows) out.set(Number(r.id), Number(r.gen));
|
|
1073
|
-
}
|
|
1074
|
-
return out;
|
|
1075
|
-
}
|
|
1076
|
-
async #saveState(tx, state) {
|
|
1077
|
-
await tx.query(`INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)
|
|
1078
|
-
ON CONFLICT(k) DO UPDATE SET v = excluded.v`, [JSON.stringify(state)]);
|
|
1079
|
-
}
|
|
1080
1196
|
/** Deletes the rows of retired segments once no search is reading them. */
|
|
1081
1197
|
async #reap() {
|
|
1082
1198
|
const ready = this.#retired.filter((s) => (this.#refs.get(s) ?? 0) === 0);
|
|
1083
1199
|
if (ready.length === 0) return;
|
|
1084
1200
|
this.#retired = this.#retired.filter((s) => !ready.includes(s));
|
|
1085
1201
|
for (const s of ready) this.#refs.delete(s);
|
|
1086
|
-
|
|
1087
|
-
await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));
|
|
1202
|
+
await this.#catalog.dropGenerations(ready.map((s) => s.gen));
|
|
1088
1203
|
}
|
|
1089
1204
|
/**
|
|
1090
1205
|
* Runs `read` over a snapshot of the segments, held against reaping. A read
|
|
@@ -1122,11 +1237,16 @@ var VectorIndex = class {
|
|
|
1122
1237
|
async #maintenance(fn) {
|
|
1123
1238
|
if (this.#maintaining) throw new Error("vector-index: compact or merge already running");
|
|
1124
1239
|
this.#maintaining = true;
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1240
|
+
const run = (async () => {
|
|
1241
|
+
try {
|
|
1242
|
+
return await fn();
|
|
1243
|
+
} finally {
|
|
1244
|
+
this.#maintaining = false;
|
|
1245
|
+
this.#maintenanceRun = null;
|
|
1246
|
+
}
|
|
1247
|
+
})();
|
|
1248
|
+
this.#maintenanceRun = run;
|
|
1249
|
+
return run;
|
|
1130
1250
|
}
|
|
1131
1251
|
#assertReady() {
|
|
1132
1252
|
if (!this.#ready) throw new Error("vector-index: call init() first");
|
|
@@ -1138,16 +1258,7 @@ function assertId(id) {
|
|
|
1138
1258
|
/** Drops every table of the index under `prefix`, in one transaction. Needs no dims or metric. */
|
|
1139
1259
|
async function dropVectorIndex(db, prefix) {
|
|
1140
1260
|
if (!PREFIX_PATTERN.test(prefix)) throw new Error(`vector-index: bad prefix ${prefix}`);
|
|
1141
|
-
await
|
|
1142
|
-
for (const t of [
|
|
1143
|
-
"meta",
|
|
1144
|
-
"ids",
|
|
1145
|
-
"tail",
|
|
1146
|
-
"centroids",
|
|
1147
|
-
"seg",
|
|
1148
|
-
"f32"
|
|
1149
|
-
]) await tx.exec(`DROP TABLE IF EXISTS ${prefix}${t}`);
|
|
1150
|
-
});
|
|
1261
|
+
await dropCatalog(db, prefix);
|
|
1151
1262
|
}
|
|
1152
1263
|
//#endregion
|
|
1153
1264
|
export { VectorIndex, dropVectorIndex };
|