@statewalker/db-sqlite-index 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -4
- package/dist/ngram.d.mts.map +1 -1
- package/dist/ngram.mjs +83 -3
- package/dist/ngram.mjs.map +1 -1
- package/dist/vector.d.mts +12 -3
- package/dist/vector.d.mts.map +1 -1
- package/dist/vector.mjs +399 -195
- package/dist/vector.mjs.map +1 -1
- package/package.json +16 -8
- package/src/ngram/fold.ts +44 -0
- package/src/ngram/fts5-backend.ts +4 -0
- package/src/ngram/ngram-store.ts +37 -0
- package/src/ngram/plan-query.ts +8 -1
- package/src/ngram/postings-backend.ts +1 -0
- package/src/ngram/regex-literals.ts +5 -2
- package/src/vector/catalog.ts +285 -0
- package/src/vector/math.ts +6 -1
- package/src/vector/segment.ts +48 -6
- package/src/vector/sql-segment-store.ts +6 -1
- package/src/vector/types.ts +4 -1
- package/src/vector/vector-index.ts +130 -216
- package/src/ngram/do-smoke.ts +0 -46
- package/src/ngram/fold.test.ts +0 -50
- package/src/ngram/json-chunks.test.ts +0 -23
- package/src/ngram/ngram-store.test.ts +0 -291
- package/src/ngram/plan-query.test.ts +0 -141
- package/src/ngram/trigrams.test.ts +0 -28
- package/src/vector/auto-compact.test.ts +0 -28
- package/src/vector/do-smoke.ts +0 -39
- package/src/vector/segment.test.ts +0 -61
- package/src/vector/test-fixtures.ts +0 -64
- package/src/vector/vector-index.test.ts +0 -105
- package/src/vector/vector-lifecycle.test.ts +0 -171
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Db } from "@statewalker/db-api";
|
|
2
|
-
import { inTransaction } from "@statewalker/db-sqlite";
|
|
3
2
|
import { PREFIX_PATTERN } from "../prefix.js";
|
|
3
|
+
import { dropCatalog, GenerationCatalog } from "./catalog.js";
|
|
4
4
|
import { distance, prepare, TopK } from "./math.js";
|
|
5
5
|
import {
|
|
6
6
|
type BuiltSegment,
|
|
@@ -10,13 +10,7 @@ import {
|
|
|
10
10
|
SegmentIndex,
|
|
11
11
|
} from "./segment.js";
|
|
12
12
|
import { type SegmentRef, SqlSegmentStore } from "./sql-segment-store.js";
|
|
13
|
-
import {
|
|
14
|
-
float32Bytes,
|
|
15
|
-
type Metric,
|
|
16
|
-
type SearchHit,
|
|
17
|
-
type SearchOptions,
|
|
18
|
-
toFloat32,
|
|
19
|
-
} from "./types.js";
|
|
13
|
+
import type { Metric, SearchHit, SearchOptions } from "./types.js";
|
|
20
14
|
|
|
21
15
|
export interface VectorIndexOptions {
|
|
22
16
|
db: Db;
|
|
@@ -48,19 +42,13 @@ export interface VectorIndexStats {
|
|
|
48
42
|
seq: number;
|
|
49
43
|
}
|
|
50
44
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
seq: number;
|
|
59
|
-
vector: Float32Array;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
/** Ids per lookup (bound as one JSON parameter). */
|
|
63
|
-
const VALIDATE_BATCH = 2000;
|
|
45
|
+
/** Lists probed per segment when the caller passes no `nprobe`. */
|
|
46
|
+
const DEFAULT_NPROBE = 8;
|
|
47
|
+
/**
|
|
48
|
+
* Without an explicit `nprobe`, probe lists until they hold this many times
|
|
49
|
+
* `k × oversample` rows (#67).
|
|
50
|
+
*/
|
|
51
|
+
const PROBE_ROWS_FACTOR = 16;
|
|
64
52
|
/** Upserted rows per transaction. */
|
|
65
53
|
const WRITE_BATCH = 200;
|
|
66
54
|
|
|
@@ -77,26 +65,27 @@ const WRITE_BATCH = 200;
|
|
|
77
65
|
* into one, dropping dead rows.
|
|
78
66
|
*
|
|
79
67
|
* One writer per database is assumed (a Durable Object, a worker, one tab);
|
|
80
|
-
* within an instance, writes are serialized.
|
|
68
|
+
* within an instance, writes are serialized. The tables, the committed state and
|
|
69
|
+
* the in-memory tail belong to the generation catalog (`catalog.ts`); this class decides
|
|
70
|
+
* what to commit and when, builds segments and searches them.
|
|
81
71
|
*/
|
|
82
72
|
export class VectorIndex {
|
|
83
73
|
readonly dims: number;
|
|
84
74
|
readonly metric: Metric;
|
|
85
75
|
/** `{prefix}ids`: its `id` column holds exactly the live ids. Callers may join it in SQL. */
|
|
86
76
|
readonly idTable: string;
|
|
87
|
-
readonly #db: Db;
|
|
88
77
|
readonly #store: SqlSegmentStore;
|
|
78
|
+
readonly #catalog: GenerationCatalog;
|
|
89
79
|
readonly #nlist?: (n: number) => number;
|
|
90
80
|
readonly #autoCompactAt: number;
|
|
91
|
-
readonly #t: { meta: string; ids: string; tail: string };
|
|
92
81
|
|
|
93
|
-
#state: State = { seq: 0, nextGen: 1, segments: [] };
|
|
94
|
-
#tail = new Map<number, TailEntry>();
|
|
95
82
|
#segments: SegmentIndex[] = [];
|
|
96
83
|
#refs = new Map<SegmentIndex, number>();
|
|
97
84
|
#retired: SegmentIndex[] = [];
|
|
98
85
|
#queue: Promise<unknown> = Promise.resolve();
|
|
99
86
|
#maintaining = false;
|
|
87
|
+
/** The running compact() or merge(), for refresh() to wait on. */
|
|
88
|
+
#maintenanceRun: Promise<unknown> | null = null;
|
|
100
89
|
#ready = false;
|
|
101
90
|
/** Bumped before and after each commit that changes the segment list: odd while one runs. */
|
|
102
91
|
#epoch = 0;
|
|
@@ -110,75 +99,54 @@ export class VectorIndex {
|
|
|
110
99
|
}
|
|
111
100
|
this.dims = opts.dims;
|
|
112
101
|
this.metric = opts.metric ?? "cosine";
|
|
113
|
-
this.#db = opts.db;
|
|
114
102
|
this.#store = new SqlSegmentStore(opts.db, opts.prefix, opts.dims, opts.cacheBytes);
|
|
103
|
+
this.#catalog = new GenerationCatalog(opts.db, opts.prefix, this.#store);
|
|
115
104
|
this.#nlist = opts.nlist;
|
|
116
105
|
this.#autoCompactAt = opts.autoCompactAt ?? 4096;
|
|
117
|
-
|
|
118
|
-
this.#t = { meta: `${p}meta`, ids: `${p}ids`, tail: `${p}tail` };
|
|
119
|
-
this.idTable = this.#t.ids;
|
|
106
|
+
this.idTable = this.#catalog.idTable;
|
|
120
107
|
}
|
|
121
108
|
|
|
122
109
|
// ------------------------------------------------------------ lifecycle
|
|
123
110
|
|
|
124
111
|
/** Creates tables if missing, loads state and sweeps rows of crashed maintenance. */
|
|
125
112
|
async init(): Promise<void> {
|
|
126
|
-
|
|
127
|
-
for (const s of [
|
|
128
|
-
`CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,
|
|
129
|
-
`CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,
|
|
130
|
-
`CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,
|
|
131
|
-
`CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,
|
|
132
|
-
`CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,
|
|
133
|
-
...this.#store.schema(),
|
|
134
|
-
]) {
|
|
135
|
-
await this.#db.exec(s);
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
const config = JSON.stringify({ dims: this.dims, metric: this.metric });
|
|
139
|
-
await this.#db.query(
|
|
140
|
-
`INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`,
|
|
141
|
-
[config],
|
|
142
|
-
);
|
|
143
|
-
const [stored] = await this.#db.query<{ v: string }>(
|
|
144
|
-
`SELECT v FROM ${meta} WHERE k = 'config'`,
|
|
145
|
-
);
|
|
146
|
-
if (stored?.v !== config) {
|
|
147
|
-
throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);
|
|
148
|
-
}
|
|
113
|
+
await this.#catalog.create(JSON.stringify({ dims: this.dims, metric: this.metric }));
|
|
149
114
|
await this.refresh();
|
|
150
|
-
|
|
151
|
-
await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));
|
|
115
|
+
await this.#catalog.sweep();
|
|
152
116
|
this.#ready = true;
|
|
153
117
|
}
|
|
154
118
|
|
|
155
|
-
/**
|
|
119
|
+
/**
|
|
120
|
+
* Reloads state, tail and segments from storage (e.g. after another writer changed them).
|
|
121
|
+
* Waits for a running compact() or merge() and runs between writes, so it never mixes the
|
|
122
|
+
* state before a commit with the rows after it (#26).
|
|
123
|
+
*/
|
|
156
124
|
async refresh(): Promise<void> {
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
this.#segments = segments;
|
|
125
|
+
for (;;) {
|
|
126
|
+
while (this.#maintenanceRun) await this.#maintenanceRun.catch(() => {});
|
|
127
|
+
const done = await this.#exclusive(async () => {
|
|
128
|
+
if (this.#maintaining) return false; // one started while we queued: wait for it too
|
|
129
|
+
const snapshot = await this.#catalog.read();
|
|
130
|
+
const known = new Map(this.#segments.map((s) => [s.gen, s]));
|
|
131
|
+
const segments = await Promise.all(
|
|
132
|
+
snapshot.state.segments.map((s) => known.get(s.gen) ?? this.#store.open(s, this.metric)),
|
|
133
|
+
);
|
|
134
|
+
await this.#swapSegments(async () => {
|
|
135
|
+
this.#catalog.adopt(snapshot);
|
|
136
|
+
this.#segments = segments;
|
|
137
|
+
});
|
|
138
|
+
return true;
|
|
139
|
+
});
|
|
140
|
+
if (done) return;
|
|
141
|
+
}
|
|
175
142
|
}
|
|
176
143
|
|
|
177
144
|
get stats(): VectorIndexStats {
|
|
145
|
+
const { state, tail } = this.#catalog;
|
|
178
146
|
return {
|
|
179
|
-
tail:
|
|
180
|
-
segments:
|
|
181
|
-
seq:
|
|
147
|
+
tail: tail.size,
|
|
148
|
+
segments: state.segments.map((s) => ({ ...s })),
|
|
149
|
+
seq: state.seq,
|
|
182
150
|
};
|
|
183
151
|
}
|
|
184
152
|
|
|
@@ -190,100 +158,104 @@ export class VectorIndex {
|
|
|
190
158
|
assertId(it.id);
|
|
191
159
|
return { id: it.id, vector: prepare(it.vector, this.dims, this.metric) };
|
|
192
160
|
});
|
|
193
|
-
const { tail, ids } = this.#t;
|
|
194
161
|
for (let i = 0; i < prepared.length; i += WRITE_BATCH) {
|
|
195
162
|
const batch = prepared.slice(i, i + WRITE_BATCH);
|
|
196
|
-
await this.#exclusive(
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
await inTransaction(this.#db, async (tx) => {
|
|
200
|
-
for (const { id, vector } of batch) {
|
|
201
|
-
await tx.query(
|
|
202
|
-
`INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)
|
|
203
|
-
ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`,
|
|
204
|
-
[id, seq, float32Bytes(vector)],
|
|
205
|
-
);
|
|
206
|
-
await tx.query(
|
|
207
|
-
`INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`,
|
|
208
|
-
[id],
|
|
209
|
-
);
|
|
210
|
-
}
|
|
211
|
-
await this.#saveState(tx, next);
|
|
212
|
-
});
|
|
213
|
-
this.#state = next;
|
|
214
|
-
for (const { id, vector } of batch) this.#tail.set(id, { seq, vector });
|
|
215
|
-
});
|
|
216
|
-
if (this.#tail.size >= this.#autoCompactAt && !this.#maintaining) await this.compact();
|
|
163
|
+
await this.#exclusive(() => this.#catalog.append(batch));
|
|
164
|
+
if (this.#catalog.tail.size >= this.#autoCompactAt && !this.#maintaining)
|
|
165
|
+
await this.compact();
|
|
217
166
|
}
|
|
218
167
|
}
|
|
219
168
|
|
|
220
169
|
async delete(ids: number[]): Promise<void> {
|
|
221
170
|
this.#assertReady();
|
|
222
171
|
ids.forEach(assertId);
|
|
223
|
-
await this.#exclusive(
|
|
224
|
-
for (let i = 0; i < ids.length; i += VALIDATE_BATCH) {
|
|
225
|
-
const chunk = ids.slice(i, i + VALIDATE_BATCH);
|
|
226
|
-
const json = JSON.stringify(chunk);
|
|
227
|
-
await inTransaction(this.#db, async (tx) => {
|
|
228
|
-
await tx.query(
|
|
229
|
-
`DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`,
|
|
230
|
-
[json],
|
|
231
|
-
);
|
|
232
|
-
await tx.query(
|
|
233
|
-
`DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`,
|
|
234
|
-
[json],
|
|
235
|
-
);
|
|
236
|
-
});
|
|
237
|
-
for (const id of chunk) this.#tail.delete(id);
|
|
238
|
-
}
|
|
239
|
-
});
|
|
172
|
+
await this.#exclusive(() => this.#catalog.remove(ids));
|
|
240
173
|
}
|
|
241
174
|
|
|
242
175
|
// ------------------------------------------------------------ search
|
|
243
176
|
|
|
244
177
|
async search(query: ArrayLike<number>, opts: SearchOptions = {}): Promise<SearchHit[]> {
|
|
245
178
|
this.#assertReady();
|
|
179
|
+
opts.signal?.throwIfAborted();
|
|
246
180
|
const k = opts.k ?? 10;
|
|
247
|
-
const nprobe = opts.nprobe ?? 8;
|
|
248
181
|
const perList = k * (opts.oversample ?? 8);
|
|
182
|
+
// An explicit nprobe probes exactly that many lists. The default probes at least
|
|
183
|
+
// DEFAULT_NPROBE lists and keeps going while they hold fewer than
|
|
184
|
+
// PROBE_ROWS_FACTOR × perList rows: after auto-compaction lists are small
|
|
185
|
+
// (~65 rows at 4200 vectors), and 8 of them miss neighbours spread over many (#67).
|
|
186
|
+
const nprobe = opts.nprobe ?? DEFAULT_NPROBE;
|
|
187
|
+
const minRows = opts.nprobe === undefined ? PROBE_ROWS_FACTOR * perList : 0;
|
|
249
188
|
const q = prepare(query, this.dims, this.metric);
|
|
250
189
|
const d = this.dims;
|
|
251
190
|
const filter = opts.filter;
|
|
191
|
+
const rerank = opts.exact ? k * 4 : k;
|
|
252
192
|
|
|
253
193
|
return this.#read(async (segments) => {
|
|
194
|
+
opts.signal?.throwIfAborted();
|
|
254
195
|
// Tail: exact, from memory.
|
|
255
196
|
const tailTop = new TopK<number>(k);
|
|
256
|
-
for (const [id, { vector }] of this.#tail) {
|
|
197
|
+
for (const [id, { vector }] of this.#catalog.tail) {
|
|
257
198
|
if (filter && !filter(id)) continue;
|
|
258
199
|
tailTop.push(distance(this.metric, q, 0, vector, 0, d), id);
|
|
259
200
|
}
|
|
260
201
|
|
|
261
|
-
//
|
|
262
|
-
type
|
|
263
|
-
const
|
|
264
|
-
// Tail ids are never live in a segment; skip them early.
|
|
265
|
-
const keep = (id: number) => !this.#tail.has(id) && (!filter || filter(id));
|
|
202
|
+
// Probe lists and load their blocks.
|
|
203
|
+
type Probed = { seg: SegmentIndex; list: number; data: ListData };
|
|
204
|
+
const probed: Probed[] = [];
|
|
266
205
|
await Promise.all(
|
|
267
206
|
segments.map(async (seg) => {
|
|
268
|
-
const lists = seg.probe(q, nprobe);
|
|
207
|
+
const lists = seg.probe(q, nprobe, minRows);
|
|
269
208
|
const datas = await this.#store.lists(seg, lists);
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
const data = datas[i] as ListData;
|
|
273
|
-
for (const cand of seg.filterList(q, c, data, perList, keep))
|
|
274
|
-
hits.push({ ...cand, seg, data });
|
|
209
|
+
lists.forEach((list, i) => {
|
|
210
|
+
probed.push({ seg, list, data: datas[i] as ListData });
|
|
275
211
|
});
|
|
276
212
|
}),
|
|
277
213
|
);
|
|
214
|
+
opts.signal?.throwIfAborted();
|
|
278
215
|
|
|
279
|
-
//
|
|
280
|
-
|
|
216
|
+
// Stage 1: keep the best `perList` rows of each list by estimated residual
|
|
217
|
+
// distance. A segment row counts only if its generation holds the live copy
|
|
218
|
+
// of its id. Rows found dead (deleted, or overwritten in another generation)
|
|
219
|
+
// are excluded and the list is filtered again with a doubled limit, so dead
|
|
220
|
+
// rows never take the slots of live ones (#33).
|
|
221
|
+
type Hit = Candidate & { seg: SegmentIndex; data: ListData };
|
|
222
|
+
const gens = new Map<number, number | undefined>(); // id -> live gen; undefined = deleted
|
|
223
|
+
const hits: Hit[] = [];
|
|
224
|
+
let pending = probed.map((p) => ({ ...p, limit: perList }));
|
|
225
|
+
while (pending.length > 0) {
|
|
226
|
+
const rounds = pending.map((p) => {
|
|
227
|
+
// Tail ids are never live in a segment; known-dead rows are skipped too.
|
|
228
|
+
const keep = (id: number) =>
|
|
229
|
+
!this.#catalog.tail.has(id) &&
|
|
230
|
+
!(gens.has(id) && gens.get(id) !== p.seg.gen) &&
|
|
231
|
+
(!filter || filter(id));
|
|
232
|
+
return { p, cands: p.seg.filterList(q, p.list, p.data, p.limit, keep) };
|
|
233
|
+
});
|
|
234
|
+
const unknown = rounds
|
|
235
|
+
.flatMap((r) => r.cands.map((c) => c.id))
|
|
236
|
+
.filter((id) => !gens.has(id));
|
|
237
|
+
if (unknown.length > 0) {
|
|
238
|
+
const found = await this.#catalog.generations(unknown);
|
|
239
|
+
for (const id of unknown) gens.set(id, found.get(id));
|
|
240
|
+
opts.signal?.throwIfAborted();
|
|
241
|
+
}
|
|
242
|
+
pending = [];
|
|
243
|
+
for (const { p, cands } of rounds) {
|
|
244
|
+
const live = cands.filter((c) => gens.get(c.id) === p.seg.gen);
|
|
245
|
+
// A list passed whole comes in row order: keep all of it, the int8 re-rank ranks it.
|
|
246
|
+
const whole = p.seg.passesWhole(p.data.ids.length, p.limit);
|
|
247
|
+
if (!whole && live.length < perList && cands.length === p.limit) {
|
|
248
|
+
pending.push({ ...p, limit: p.limit * 2 }); // some were dead and more rows remain
|
|
249
|
+
continue;
|
|
250
|
+
}
|
|
251
|
+
for (const cand of whole ? live : live.slice(0, perList))
|
|
252
|
+
hits.push({ ...cand, seg: p.seg, data: p.data });
|
|
253
|
+
}
|
|
254
|
+
}
|
|
281
255
|
|
|
282
256
|
// Stage 2: int8 re-rank.
|
|
283
|
-
const rerank = opts.exact ? k * 4 : k;
|
|
284
257
|
const segTop = new TopK<Hit>(rerank);
|
|
285
258
|
for (const h of hits) {
|
|
286
|
-
if (live.get(h.id) !== h.seg.gen) continue;
|
|
287
259
|
segTop.push(h.seg.scoreInt8(q, h.data, h.row), h);
|
|
288
260
|
}
|
|
289
261
|
let segResults = segTop
|
|
@@ -314,11 +286,11 @@ export class VectorIndex {
|
|
|
314
286
|
ids.forEach(assertId);
|
|
315
287
|
return this.#read(async (segments) => {
|
|
316
288
|
const out = new Map<number, Float32Array>();
|
|
317
|
-
const gens = await this.#generations(ids);
|
|
289
|
+
const gens = await this.#catalog.generations(ids);
|
|
318
290
|
const wanted = new Map<number, Set<number>>(); // gen -> ids
|
|
319
291
|
for (const [id, gen] of gens) {
|
|
320
292
|
if (gen === 0) {
|
|
321
|
-
const entry = this.#tail.get(id);
|
|
293
|
+
const entry = this.#catalog.tail.get(id);
|
|
322
294
|
if (entry) out.set(id, entry.vector.slice());
|
|
323
295
|
continue;
|
|
324
296
|
}
|
|
@@ -353,8 +325,8 @@ export class VectorIndex {
|
|
|
353
325
|
this.#assertReady();
|
|
354
326
|
return this.#maintenance(async () => {
|
|
355
327
|
const snapshot = await this.#exclusive(async () => ({
|
|
356
|
-
maxSeq: this.#state.seq,
|
|
357
|
-
entries: [...this.#tail.entries()],
|
|
328
|
+
maxSeq: this.#catalog.state.seq,
|
|
329
|
+
entries: [...this.#catalog.tail.entries()],
|
|
358
330
|
}));
|
|
359
331
|
if (snapshot.entries.length === 0) return 0;
|
|
360
332
|
|
|
@@ -369,26 +341,9 @@ export class VectorIndex {
|
|
|
369
341
|
|
|
370
342
|
await this.#exclusive(() =>
|
|
371
343
|
this.#swapSegments(async () => {
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
...this.#state,
|
|
375
|
-
nextGen: ref.gen + 1,
|
|
376
|
-
segments: [...this.#state.segments, ref],
|
|
377
|
-
};
|
|
378
|
-
const { ids: idsT, tail } = this.#t;
|
|
379
|
-
await inTransaction(this.#db, async (tx) => {
|
|
380
|
-
await this.#store.write(tx, ref.gen, built);
|
|
381
|
-
await tx.query(
|
|
382
|
-
`UPDATE ${idsT} SET gen = ? WHERE gen = 0
|
|
383
|
-
AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`,
|
|
384
|
-
[ref.gen, snapshot.maxSeq],
|
|
385
|
-
);
|
|
386
|
-
await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [snapshot.maxSeq]);
|
|
387
|
-
await this.#saveState(tx, next);
|
|
344
|
+
await this.#catalog.seal(snapshot.maxSeq, built, n, (ref) => {
|
|
345
|
+
this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
|
|
388
346
|
});
|
|
389
|
-
this.#state = next;
|
|
390
|
-
for (const [id, e] of this.#tail) if (e.seq <= snapshot.maxSeq) this.#tail.delete(id);
|
|
391
|
-
this.#segments = [...this.#segments, this.#segmentOf(ref, built)];
|
|
392
347
|
}),
|
|
393
348
|
);
|
|
394
349
|
return n;
|
|
@@ -413,7 +368,7 @@ export class VectorIndex {
|
|
|
413
368
|
if (seg.counts[c] === 0) continue;
|
|
414
369
|
const [data] = (await this.#store.lists(seg, [c])) as [ListData];
|
|
415
370
|
const floats = await this.#store.floats(seg, c);
|
|
416
|
-
const live = await this.#generations(Array.from(data.ids));
|
|
371
|
+
const live = await this.#catalog.generations(Array.from(data.ids));
|
|
417
372
|
for (let r = 0; r < data.ids.length; r++) {
|
|
418
373
|
const id = data.ids[r];
|
|
419
374
|
if (live.get(id) !== seg.gen) continue;
|
|
@@ -432,29 +387,13 @@ export class VectorIndex {
|
|
|
432
387
|
|
|
433
388
|
await this.#exclusive(() =>
|
|
434
389
|
this.#swapSegments(async () => {
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
};
|
|
442
|
-
await inTransaction(this.#db, async (tx) => {
|
|
443
|
-
if (ref && built) {
|
|
444
|
-
await this.#store.write(tx, ref.gen, built);
|
|
445
|
-
await tx.query(
|
|
446
|
-
`UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`,
|
|
447
|
-
[ref.gen, JSON.stringify(oldGens)],
|
|
448
|
-
);
|
|
449
|
-
}
|
|
450
|
-
await this.#saveState(tx, next);
|
|
390
|
+
await this.#catalog.replace(oldGens, built, n, (ref) => {
|
|
391
|
+
this.#segments = [
|
|
392
|
+
...this.#segments.filter((s) => !oldGens.includes(s.gen)),
|
|
393
|
+
...(ref && built ? [this.#segmentOf(ref, built)] : []),
|
|
394
|
+
];
|
|
395
|
+
this.#retired.push(...old);
|
|
451
396
|
});
|
|
452
|
-
this.#state = next;
|
|
453
|
-
this.#segments = [
|
|
454
|
-
...this.#segments.filter((s) => !oldGens.includes(s.gen)),
|
|
455
|
-
...(ref && built ? [this.#segmentOf(ref, built)] : []),
|
|
456
|
-
];
|
|
457
|
-
this.#retired.push(...old);
|
|
458
397
|
}),
|
|
459
398
|
);
|
|
460
399
|
await this.#reap();
|
|
@@ -480,43 +419,17 @@ export class VectorIndex {
|
|
|
480
419
|
});
|
|
481
420
|
}
|
|
482
421
|
|
|
483
|
-
#refOf(built: BuiltSegment, count: number): SegmentRef {
|
|
484
|
-
return { gen: this.#state.nextGen, count, nlist: built.nlist };
|
|
485
|
-
}
|
|
486
|
-
|
|
487
422
|
#segmentOf(ref: SegmentRef, built: BuiltSegment): SegmentIndex {
|
|
488
423
|
return new SegmentIndex(ref.gen, this.dims, this.metric, built.centroids, built.counts);
|
|
489
424
|
}
|
|
490
425
|
|
|
491
|
-
async #generations(ids: number[]): Promise<Map<number, number>> {
|
|
492
|
-
const out = new Map<number, number>();
|
|
493
|
-
const unique = [...new Set(ids)];
|
|
494
|
-
for (let i = 0; i < unique.length; i += VALIDATE_BATCH) {
|
|
495
|
-
const rows = await this.#db.query<{ id: number; gen: number }>(
|
|
496
|
-
`SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`,
|
|
497
|
-
[JSON.stringify(unique.slice(i, i + VALIDATE_BATCH))],
|
|
498
|
-
);
|
|
499
|
-
for (const r of rows) out.set(Number(r.id), Number(r.gen));
|
|
500
|
-
}
|
|
501
|
-
return out;
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
async #saveState(tx: Db, state: State): Promise<void> {
|
|
505
|
-
await tx.query(
|
|
506
|
-
`INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)
|
|
507
|
-
ON CONFLICT(k) DO UPDATE SET v = excluded.v`,
|
|
508
|
-
[JSON.stringify(state)],
|
|
509
|
-
);
|
|
510
|
-
}
|
|
511
|
-
|
|
512
426
|
/** Deletes the rows of retired segments once no search is reading them. */
|
|
513
427
|
async #reap(): Promise<void> {
|
|
514
428
|
const ready = this.#retired.filter((s) => (this.#refs.get(s) ?? 0) === 0);
|
|
515
429
|
if (ready.length === 0) return;
|
|
516
430
|
this.#retired = this.#retired.filter((s) => !ready.includes(s));
|
|
517
431
|
for (const s of ready) this.#refs.delete(s);
|
|
518
|
-
|
|
519
|
-
await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));
|
|
432
|
+
await this.#catalog.dropGenerations(ready.map((s) => s.gen));
|
|
520
433
|
}
|
|
521
434
|
|
|
522
435
|
/**
|
|
@@ -558,11 +471,16 @@ export class VectorIndex {
|
|
|
558
471
|
async #maintenance<T>(fn: () => Promise<T>): Promise<T> {
|
|
559
472
|
if (this.#maintaining) throw new Error("vector-index: compact or merge already running");
|
|
560
473
|
this.#maintaining = true;
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
474
|
+
const run = (async () => {
|
|
475
|
+
try {
|
|
476
|
+
return await fn();
|
|
477
|
+
} finally {
|
|
478
|
+
this.#maintaining = false;
|
|
479
|
+
this.#maintenanceRun = null;
|
|
480
|
+
}
|
|
481
|
+
})();
|
|
482
|
+
this.#maintenanceRun = run;
|
|
483
|
+
return run;
|
|
566
484
|
}
|
|
567
485
|
|
|
568
486
|
#assertReady() {
|
|
@@ -578,9 +496,5 @@ function assertId(id: number) {
|
|
|
578
496
|
/** Drops every table of the index under `prefix`, in one transaction. Needs no dims or metric. */
|
|
579
497
|
export async function dropVectorIndex(db: Db, prefix: string): Promise<void> {
|
|
580
498
|
if (!PREFIX_PATTERN.test(prefix)) throw new Error(`vector-index: bad prefix ${prefix}`);
|
|
581
|
-
await
|
|
582
|
-
for (const t of ["meta", "ids", "tail", "centroids", "seg", "f32"]) {
|
|
583
|
-
await tx.exec(`DROP TABLE IF EXISTS ${prefix}${t}`);
|
|
584
|
-
}
|
|
585
|
-
});
|
|
499
|
+
await dropCatalog(db, prefix);
|
|
586
500
|
}
|
package/src/ngram/do-smoke.ts
DELETED
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
import type { Db } from "@statewalker/db-api";
|
|
2
|
-
import { NgramStore } from "./ngram-store.js";
|
|
3
|
-
|
|
4
|
-
const check = (ok: boolean, message: string) => {
|
|
5
|
-
if (!ok) throw new Error(`ngram smoke: ${message}`);
|
|
6
|
-
};
|
|
7
|
-
|
|
8
|
-
/** True when the index itself still offers candidates (candidateBudget 0 skips the verifier). */
|
|
9
|
-
const indexed = async (index: NgramStore, pattern: string) =>
|
|
10
|
-
(await index.search(pattern, { candidateBudget: 0 })).truncated;
|
|
11
|
-
|
|
12
|
-
const ids = (result: { hits: { id: number }[] }) => result.hits.map((h) => h.id).join();
|
|
13
|
-
|
|
14
|
-
/** Run inside a real Durable Object by db-sqlite-do's `pnpm test:do`. */
|
|
15
|
-
export const ngramDoScenarios = (["fts5", "postings"] as const).map((store) => ({
|
|
16
|
-
name: `ngram (${store}): upsert, overwrite, lone surrogate, drop inside a transaction`,
|
|
17
|
-
async run(db: Db) {
|
|
18
|
-
const prefix = `smoke_${store}_`;
|
|
19
|
-
const index = new NgramStore({ db, prefix, store });
|
|
20
|
-
await index.init(); // CREATE of the index tables (a contentless trigram table for fts5) in a transaction
|
|
21
|
-
await index.upsert([
|
|
22
|
-
{ id: 1, text: "Hello World" },
|
|
23
|
-
{ id: 2, text: "hello there" },
|
|
24
|
-
{ id: 3, text: "lone \ud800 surrogate" },
|
|
25
|
-
]);
|
|
26
|
-
check(
|
|
27
|
-
ids(await index.search("hello", { caseSensitive: false })) === "1,2",
|
|
28
|
-
"case-insensitive hits",
|
|
29
|
-
);
|
|
30
|
-
check(ids(await index.search("Hello")) === "1", "case-sensitive hits");
|
|
31
|
-
check(ids(await index.search("surrogate")) === "3", "text with a lone surrogate is indexed");
|
|
32
|
-
check((await index.getTexts([3])).get(3) === "lone \ufffd surrogate", "text made well-formed");
|
|
33
|
-
check(await indexed(index, "World"), "index lacks the grams of 'World'");
|
|
34
|
-
check(await indexed(index, "surrogate"), "index lacks the grams of 'surrogate'");
|
|
35
|
-
await index.upsert([{ id: 1, text: "goodbye" }]); // replays the old text to remove its grams
|
|
36
|
-
check(!(await indexed(index, "World")), "stale grams after overwrite");
|
|
37
|
-
await index.upsert([{ id: 3, text: "lone \ud800 changed" }]);
|
|
38
|
-
check(!(await indexed(index, "surrogate")), "stale grams after surrogate overwrite");
|
|
39
|
-
await index.drop(); // DROP of the tables inside a transaction
|
|
40
|
-
const [row] = await db.query<{ n: number }>(
|
|
41
|
-
"SELECT count(*) AS n FROM sqlite_master WHERE name LIKE ?",
|
|
42
|
-
[`${prefix}%`],
|
|
43
|
-
);
|
|
44
|
-
check(Number(row?.n) === 0, `tables left after drop: ${row?.n}`);
|
|
45
|
-
},
|
|
46
|
-
}));
|
package/src/ngram/fold.test.ts
DELETED
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { fold } from "./fold.js";
|
|
3
|
-
|
|
4
|
-
const escapeRegex = (s: string) => s.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&");
|
|
5
|
-
|
|
6
|
-
describe("fold", () => {
|
|
7
|
-
it("lower-cases ASCII and keeps length", () => {
|
|
8
|
-
expect(fold("Hello WORLD")).toBe("hello world");
|
|
9
|
-
});
|
|
10
|
-
|
|
11
|
-
it("maps every case form of sigma to one letter", () => {
|
|
12
|
-
expect(new Set([fold("Σ"), fold("σ"), fold("ς")]).size).toBe(1);
|
|
13
|
-
});
|
|
14
|
-
|
|
15
|
-
it("never separates two BMP characters that a case-insensitive regex treats as equal", () => {
|
|
16
|
-
const broken: string[] = [];
|
|
17
|
-
for (let cp = 0; cp <= 0xffff; cp++) {
|
|
18
|
-
if (cp >= 0xd800 && cp <= 0xdfff) continue;
|
|
19
|
-
const c = String.fromCharCode(cp);
|
|
20
|
-
const re = new RegExp(`^${escapeRegex(c)}$`, "i");
|
|
21
|
-
for (const d of [c.toUpperCase(), c.toLowerCase()]) {
|
|
22
|
-
if (d === c || d.length !== 1 || !re.test(d)) continue;
|
|
23
|
-
if (fold(c) !== fold(d)) broken.push(`U+${cp.toString(16)} ${c}/${d}`);
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
expect(broken).toEqual([]);
|
|
27
|
-
});
|
|
28
|
-
|
|
29
|
-
it("keeps the UTF-16 length of every code point, so offsets stay valid", () => {
|
|
30
|
-
const changed: string[] = [];
|
|
31
|
-
for (let cp = 0; cp <= 0x10ffff; cp++) {
|
|
32
|
-
if (cp >= 0xd800 && cp <= 0xdfff) continue;
|
|
33
|
-
const c = String.fromCodePoint(cp);
|
|
34
|
-
if (fold(c).length !== c.length) changed.push(`U+${cp.toString(16)}`);
|
|
35
|
-
}
|
|
36
|
-
expect(changed).toEqual([]);
|
|
37
|
-
});
|
|
38
|
-
|
|
39
|
-
it("keeps the length of a text with U+0130 and an astral character", () => {
|
|
40
|
-
const folded = fold("İ😀A");
|
|
41
|
-
expect(folded.length).toBe("İ😀A".length);
|
|
42
|
-
expect(folded.endsWith("a")).toBe(true);
|
|
43
|
-
});
|
|
44
|
-
|
|
45
|
-
it("is a homomorphism over concatenation, so substrings stay substrings", () => {
|
|
46
|
-
const a = "ΟΔΟΣ ſtraße";
|
|
47
|
-
const b = "İK";
|
|
48
|
-
expect(fold(a + b)).toBe(fold(a) + fold(b));
|
|
49
|
-
});
|
|
50
|
-
});
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { chunkJson } from "./json-chunks.js";
|
|
3
|
-
|
|
4
|
-
describe("chunkJson", () => {
|
|
5
|
-
const rows = Array.from({ length: 30 }, (_, i) => [i, `t${i}`]);
|
|
6
|
-
|
|
7
|
-
it("splits into chunks of at most maxChars that rebuild the rows in order", () => {
|
|
8
|
-
const chunks = chunkJson(rows, 20);
|
|
9
|
-
expect(chunks.length).toBeGreaterThan(1);
|
|
10
|
-
for (const chunk of chunks) expect(chunk.length).toBeLessThanOrEqual(20);
|
|
11
|
-
expect(chunks.flatMap((chunk) => JSON.parse(chunk))).toEqual(rows);
|
|
12
|
-
});
|
|
13
|
-
|
|
14
|
-
it("keeps a row larger than maxChars as one chunk of its own", () => {
|
|
15
|
-
const big = [1, "x".repeat(50)];
|
|
16
|
-
const chunks = chunkJson([[0, "a"], big, [2, "b"]], 20);
|
|
17
|
-
expect(chunks).toEqual(['[[0,"a"]]', JSON.stringify([big]), '[[2,"b"]]']);
|
|
18
|
-
});
|
|
19
|
-
|
|
20
|
-
it("returns no chunks for no rows", () => {
|
|
21
|
-
expect(chunkJson([], 20)).toEqual([]);
|
|
22
|
-
});
|
|
23
|
-
});
|