@statewalker/db-sqlite-index 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/vector.d.mts +8 -2
- package/dist/vector.d.mts.map +1 -1
- package/dist/vector.mjs +286 -175
- package/dist/vector.mjs.map +1 -1
- package/package.json +6 -6
- package/src/vector/catalog.ts +285 -0
- package/src/vector/sql-segment-store.ts +6 -1
- package/src/vector/vector-index.ts +70 -202
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
import type { Db } from "@statewalker/db-api";
|
|
2
|
+
import { inTransaction } from "@statewalker/db-sqlite";
|
|
3
|
+
import type { BuiltSegment } from "./segment.js";
|
|
4
|
+
import { type SegmentRef, SqlSegmentStore } from "./sql-segment-store.js";
|
|
5
|
+
import { float32Bytes, toFloat32 } from "./types.js";
|
|
6
|
+
|
|
7
|
+
/** The committed state, stored as JSON in `{p}meta` under `state`. */
|
|
8
|
+
export interface CatalogState {
|
|
9
|
+
/** The last upsert batch: tail rows carry the seq of the batch that wrote them. */
|
|
10
|
+
readonly seq: number;
|
|
11
|
+
/** The generation the next segment gets. Generation 0 is the tail. */
|
|
12
|
+
readonly nextGen: number;
|
|
13
|
+
/** The live segments, oldest first. */
|
|
14
|
+
readonly segments: readonly SegmentRef[];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface TailEntry {
|
|
18
|
+
seq: number;
|
|
19
|
+
vector: Float32Array;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** What `read()` found in storage, for `adopt()` to install. */
|
|
23
|
+
export interface CatalogSnapshot {
|
|
24
|
+
readonly state: CatalogState;
|
|
25
|
+
readonly tail: Map<number, TailEntry>;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Ids per statement (bound as one JSON parameter). */
|
|
29
|
+
const ID_BATCH = 2000;
|
|
30
|
+
|
|
31
|
+
const EMPTY: CatalogState = { seq: 0, nextGen: 1, segments: [] };
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The generation catalog of a vector index: the committed state, the `{p}ids` table (which
|
|
35
|
+
* generation holds the live copy of each id, 0 = the tail), the `{p}tail` rows, and the in-memory
|
|
36
|
+
* copy of the state and the tail.
|
|
37
|
+
*
|
|
38
|
+
* It is the only place that commits a state change, and every commit updates the copy in the
|
|
39
|
+
* same synchronous step as it returns from the transaction, so the copy always matches storage.
|
|
40
|
+
* It also creates, sweeps and drops every table under the prefix, asking the segment store for
|
|
41
|
+
* its own.
|
|
42
|
+
*
|
|
43
|
+
* Callers serialize the commits (one at a time per instance); reads of `state` and `tail` may
|
|
44
|
+
* happen at any time and see either the copy before a commit or the copy after it.
|
|
45
|
+
*/
|
|
46
|
+
export class GenerationCatalog {
|
|
47
|
+
/** `{p}ids`: its `id` column holds exactly the live ids. */
|
|
48
|
+
readonly idTable: string;
|
|
49
|
+
readonly #db: Db;
|
|
50
|
+
readonly #store: SqlSegmentStore;
|
|
51
|
+
readonly #t: { meta: string; ids: string; tail: string };
|
|
52
|
+
#state: CatalogState = EMPTY;
|
|
53
|
+
#tail = new Map<number, TailEntry>();
|
|
54
|
+
|
|
55
|
+
constructor(db: Db, prefix: string, store: SqlSegmentStore) {
|
|
56
|
+
this.#db = db;
|
|
57
|
+
this.#store = store;
|
|
58
|
+
this.#t = catalogTables(prefix);
|
|
59
|
+
this.idTable = this.#t.ids;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** The committed state, as of the last commit or `adopt()`. Never mutated: replaced. */
|
|
63
|
+
get state(): CatalogState {
|
|
64
|
+
return this.#state;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** The tail rows in memory, by id. */
|
|
68
|
+
get tail(): ReadonlyMap<number, TailEntry> {
|
|
69
|
+
return this.#tail;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// ------------------------------------------------------------ tables
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Creates every table if missing and checks the stored config (dims and metric) is `config`:
|
|
76
|
+
* the first open stores it, later opens must match.
|
|
77
|
+
*/
|
|
78
|
+
async create(config: string): Promise<void> {
|
|
79
|
+
const { meta, ids, tail } = this.#t;
|
|
80
|
+
for (const s of [
|
|
81
|
+
`CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,
|
|
82
|
+
`CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,
|
|
83
|
+
`CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,
|
|
84
|
+
`CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,
|
|
85
|
+
`CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,
|
|
86
|
+
...this.#store.schema(),
|
|
87
|
+
]) {
|
|
88
|
+
await this.#db.exec(s);
|
|
89
|
+
}
|
|
90
|
+
await this.#db.query(
|
|
91
|
+
`INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`,
|
|
92
|
+
[config],
|
|
93
|
+
);
|
|
94
|
+
const [stored] = await this.#db.query<{ v: string }>(
|
|
95
|
+
`SELECT v FROM ${meta} WHERE k = 'config'`,
|
|
96
|
+
);
|
|
97
|
+
if (stored?.v !== config) {
|
|
98
|
+
throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Deletes segment rows of every generation the committed state does not list (crash leftovers). */
|
|
103
|
+
async sweep(): Promise<void> {
|
|
104
|
+
const live = this.#state.segments.map((s) => s.gen);
|
|
105
|
+
await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Deletes the segment rows of retired generations. */
|
|
109
|
+
async dropGenerations(gens: number[]): Promise<void> {
|
|
110
|
+
await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// ------------------------------------------------------------ reading storage
|
|
114
|
+
|
|
115
|
+
/** Reads the committed state and the tail rows, without installing them. */
|
|
116
|
+
async read(): Promise<CatalogSnapshot> {
|
|
117
|
+
const [row] = await this.#db.query<{ v: string }>(
|
|
118
|
+
`SELECT v FROM ${this.#t.meta} WHERE k = 'state'`,
|
|
119
|
+
);
|
|
120
|
+
const state: CatalogState = row ? JSON.parse(row.v) : EMPTY;
|
|
121
|
+
const tail = new Map<number, TailEntry>();
|
|
122
|
+
const rows = await this.#db.query<{ id: number; seq: number; vec: Uint8Array }>(
|
|
123
|
+
`SELECT id, seq, vec FROM ${this.#t.tail}`,
|
|
124
|
+
);
|
|
125
|
+
for (const r of rows) tail.set(Number(r.id), { seq: Number(r.seq), vector: toFloat32(r.vec) });
|
|
126
|
+
return { state, tail };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Installs a snapshot from `read()` as the copy. No commit may run between the two. */
|
|
130
|
+
adopt(snapshot: CatalogSnapshot): void {
|
|
131
|
+
this.#state = snapshot.state;
|
|
132
|
+
this.#tail = snapshot.tail;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** The generation holding the live copy of each id; deleted or unknown ids are absent. */
|
|
136
|
+
async generations(ids: number[]): Promise<Map<number, number>> {
|
|
137
|
+
const out = new Map<number, number>();
|
|
138
|
+
const unique = [...new Set(ids)];
|
|
139
|
+
for (let i = 0; i < unique.length; i += ID_BATCH) {
|
|
140
|
+
const rows = await this.#db.query<{ id: number; gen: number }>(
|
|
141
|
+
`SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`,
|
|
142
|
+
[JSON.stringify(unique.slice(i, i + ID_BATCH))],
|
|
143
|
+
);
|
|
144
|
+
for (const r of rows) out.set(Number(r.id), Number(r.gen));
|
|
145
|
+
}
|
|
146
|
+
return out;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// ------------------------------------------------------------ commits
|
|
150
|
+
|
|
151
|
+
/** Writes `items` to the tail as one new batch (the next seq), in one transaction. */
|
|
152
|
+
async append(items: { id: number; vector: Float32Array }[]): Promise<void> {
|
|
153
|
+
const { tail, ids } = this.#t;
|
|
154
|
+
const seq = this.#state.seq + 1;
|
|
155
|
+
const next: CatalogState = { ...this.#state, seq };
|
|
156
|
+
await inTransaction(this.#db, async (tx) => {
|
|
157
|
+
for (const { id, vector } of items) {
|
|
158
|
+
await tx.query(
|
|
159
|
+
`INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)
|
|
160
|
+
ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`,
|
|
161
|
+
[id, seq, float32Bytes(vector)],
|
|
162
|
+
);
|
|
163
|
+
await tx.query(
|
|
164
|
+
`INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`,
|
|
165
|
+
[id],
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
await this.#save(tx, next);
|
|
169
|
+
});
|
|
170
|
+
this.#state = next;
|
|
171
|
+
for (const { id, vector } of items) this.#tail.set(id, { seq, vector });
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** Removes `ids` from the tail and from the live ids, one transaction per batch. */
|
|
175
|
+
async remove(ids: number[]): Promise<void> {
|
|
176
|
+
for (let i = 0; i < ids.length; i += ID_BATCH) {
|
|
177
|
+
const chunk = ids.slice(i, i + ID_BATCH);
|
|
178
|
+
const json = JSON.stringify(chunk);
|
|
179
|
+
await inTransaction(this.#db, async (tx) => {
|
|
180
|
+
await tx.query(`DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`, [
|
|
181
|
+
json,
|
|
182
|
+
]);
|
|
183
|
+
await tx.query(`DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [
|
|
184
|
+
json,
|
|
185
|
+
]);
|
|
186
|
+
});
|
|
187
|
+
for (const id of chunk) this.#tail.delete(id);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Commits `built` (sealed from the tail rows up to `maxSeq`, `count` vectors) as a new
|
|
193
|
+
* generation: writes its rows, moves those ids to it and deletes those tail rows. Tail rows of
|
|
194
|
+
* later batches stay. `install` runs with the new ref in the same synchronous step as the copy
|
|
195
|
+
* update, for the caller's own in-memory segment list.
|
|
196
|
+
*/
|
|
197
|
+
async seal(
|
|
198
|
+
maxSeq: number,
|
|
199
|
+
built: BuiltSegment,
|
|
200
|
+
count: number,
|
|
201
|
+
install: (ref: SegmentRef) => void,
|
|
202
|
+
): Promise<SegmentRef> {
|
|
203
|
+
const ref = this.#refOf(built, count);
|
|
204
|
+
const next: CatalogState = {
|
|
205
|
+
...this.#state,
|
|
206
|
+
nextGen: ref.gen + 1,
|
|
207
|
+
segments: [...this.#state.segments, ref],
|
|
208
|
+
};
|
|
209
|
+
const { ids, tail } = this.#t;
|
|
210
|
+
await inTransaction(this.#db, async (tx) => {
|
|
211
|
+
await this.#store.write(tx, ref.gen, built);
|
|
212
|
+
await tx.query(
|
|
213
|
+
`UPDATE ${ids} SET gen = ? WHERE gen = 0
|
|
214
|
+
AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`,
|
|
215
|
+
[ref.gen, maxSeq],
|
|
216
|
+
);
|
|
217
|
+
await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [maxSeq]);
|
|
218
|
+
await this.#save(tx, next);
|
|
219
|
+
});
|
|
220
|
+
this.#state = next;
|
|
221
|
+
for (const [id, e] of this.#tail) if (e.seq <= maxSeq) this.#tail.delete(id);
|
|
222
|
+
install(ref);
|
|
223
|
+
return ref;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Replaces the generations `oldGens` with `built` (`count` live vectors), or with nothing when
|
|
228
|
+
* `built` is null: writes its rows, moves the ids of the old generations to it and drops them
|
|
229
|
+
* from the state. Their rows stay until `dropGenerations()`. `install` runs as in `seal()`.
|
|
230
|
+
*/
|
|
231
|
+
async replace(
|
|
232
|
+
oldGens: number[],
|
|
233
|
+
built: BuiltSegment | null,
|
|
234
|
+
count: number,
|
|
235
|
+
install: (ref: SegmentRef | null) => void,
|
|
236
|
+
): Promise<SegmentRef | null> {
|
|
237
|
+
const ref = built ? this.#refOf(built, count) : null;
|
|
238
|
+
const kept = this.#state.segments.filter((s) => !oldGens.includes(s.gen));
|
|
239
|
+
const next: CatalogState = {
|
|
240
|
+
...this.#state,
|
|
241
|
+
nextGen: ref ? ref.gen + 1 : this.#state.nextGen,
|
|
242
|
+
segments: ref ? [...kept, ref] : kept,
|
|
243
|
+
};
|
|
244
|
+
await inTransaction(this.#db, async (tx) => {
|
|
245
|
+
if (ref && built) {
|
|
246
|
+
await this.#store.write(tx, ref.gen, built);
|
|
247
|
+
await tx.query(
|
|
248
|
+
`UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`,
|
|
249
|
+
[ref.gen, JSON.stringify(oldGens)],
|
|
250
|
+
);
|
|
251
|
+
}
|
|
252
|
+
await this.#save(tx, next);
|
|
253
|
+
});
|
|
254
|
+
this.#state = next;
|
|
255
|
+
install(ref);
|
|
256
|
+
return ref;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
#refOf(built: BuiltSegment, count: number): SegmentRef {
|
|
260
|
+
return { gen: this.#state.nextGen, count, nlist: built.nlist };
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
async #save(tx: Db, state: CatalogState): Promise<void> {
|
|
264
|
+
await tx.query(
|
|
265
|
+
`INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)
|
|
266
|
+
ON CONFLICT(k) DO UPDATE SET v = excluded.v`,
|
|
267
|
+
[JSON.stringify(state)],
|
|
268
|
+
);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function catalogTables(prefix: string) {
|
|
273
|
+
return { meta: `${prefix}meta`, ids: `${prefix}ids`, tail: `${prefix}tail` };
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/** Drops every table under `prefix` (the catalog's and the segment store's), in one transaction. */
|
|
277
|
+
export async function dropCatalog(db: Db, prefix: string): Promise<void> {
|
|
278
|
+
const tables = [
|
|
279
|
+
...Object.values(catalogTables(prefix)),
|
|
280
|
+
...Object.values(SqlSegmentStore.tables(prefix)),
|
|
281
|
+
];
|
|
282
|
+
await inTransaction(db, async (tx) => {
|
|
283
|
+
for (const t of tables) await tx.exec(`DROP TABLE IF EXISTS ${t}`);
|
|
284
|
+
});
|
|
285
|
+
}
|
|
@@ -25,10 +25,15 @@ export class SqlSegmentStore {
|
|
|
25
25
|
readonly #t: { centroids: string; seg: string; f32: string };
|
|
26
26
|
readonly #cache: Lru<ListData | Float32Array>;
|
|
27
27
|
|
|
28
|
+
/** The names of the tables segment rows live in, under `prefix`. */
|
|
29
|
+
static tables(prefix: string): { centroids: string; seg: string; f32: string } {
|
|
30
|
+
return { centroids: `${prefix}centroids`, seg: `${prefix}seg`, f32: `${prefix}f32` };
|
|
31
|
+
}
|
|
32
|
+
|
|
28
33
|
constructor(db: Db, prefix: string, dims: number, cacheBytes = 64 * 1024 * 1024) {
|
|
29
34
|
this.#db = db;
|
|
30
35
|
this.#dims = dims;
|
|
31
|
-
this.#t =
|
|
36
|
+
this.#t = SqlSegmentStore.tables(prefix);
|
|
32
37
|
this.#cache = new Lru(cacheBytes);
|
|
33
38
|
}
|
|
34
39
|
|