@statewalker/db-sqlite-index 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -4
- package/dist/ngram.d.mts.map +1 -1
- package/dist/ngram.mjs +83 -3
- package/dist/ngram.mjs.map +1 -1
- package/dist/vector.d.mts +12 -3
- package/dist/vector.d.mts.map +1 -1
- package/dist/vector.mjs +399 -195
- package/dist/vector.mjs.map +1 -1
- package/package.json +16 -8
- package/src/ngram/fold.ts +44 -0
- package/src/ngram/fts5-backend.ts +4 -0
- package/src/ngram/ngram-store.ts +37 -0
- package/src/ngram/plan-query.ts +8 -1
- package/src/ngram/postings-backend.ts +1 -0
- package/src/ngram/regex-literals.ts +5 -2
- package/src/vector/catalog.ts +285 -0
- package/src/vector/math.ts +6 -1
- package/src/vector/segment.ts +48 -6
- package/src/vector/sql-segment-store.ts +6 -1
- package/src/vector/types.ts +4 -1
- package/src/vector/vector-index.ts +130 -216
- package/src/ngram/do-smoke.ts +0 -46
- package/src/ngram/fold.test.ts +0 -50
- package/src/ngram/json-chunks.test.ts +0 -23
- package/src/ngram/ngram-store.test.ts +0 -291
- package/src/ngram/plan-query.test.ts +0 -141
- package/src/ngram/trigrams.test.ts +0 -28
- package/src/vector/auto-compact.test.ts +0 -28
- package/src/vector/do-smoke.ts +0 -39
- package/src/vector/segment.test.ts +0 -61
- package/src/vector/test-fixtures.ts +0 -64
- package/src/vector/vector-index.test.ts +0 -105
- package/src/vector/vector-lifecycle.test.ts +0 -171
|
@@ -297,9 +297,12 @@ function parseLiteralClass(src: string, start: number, end: number): string[] |
|
|
|
297
297
|
const c = src[i] ?? "";
|
|
298
298
|
if (c === "\\") {
|
|
299
299
|
const next = src[i + 1];
|
|
300
|
-
|
|
301
|
-
|
|
300
|
+
// Letters are classes or code escapes; digits are octal escapes without the `u` flag.
|
|
301
|
+
if (next === undefined || /[a-zA-Z0-9]/.test(next)) return null;
|
|
302
302
|
i += 2;
|
|
303
|
+
// `[\.-z]`: the escape starts a range, not three literals.
|
|
304
|
+
if (src[i] === "-" && i + 1 < lastInside) return null;
|
|
305
|
+
chars.add(next);
|
|
303
306
|
continue;
|
|
304
307
|
}
|
|
305
308
|
if (i + 2 < lastInside && src[i + 1] === "-") return null;
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
import type { Db } from "@statewalker/db-api";
|
|
2
|
+
import { inTransaction } from "@statewalker/db-sqlite";
|
|
3
|
+
import type { BuiltSegment } from "./segment.js";
|
|
4
|
+
import { type SegmentRef, SqlSegmentStore } from "./sql-segment-store.js";
|
|
5
|
+
import { float32Bytes, toFloat32 } from "./types.js";
|
|
6
|
+
|
|
7
|
+
/** The committed state, stored as JSON in `{p}meta` under `state`. */
|
|
8
|
+
export interface CatalogState {
|
|
9
|
+
/** The last upsert batch: tail rows carry the seq of the batch that wrote them. */
|
|
10
|
+
readonly seq: number;
|
|
11
|
+
/** The generation the next segment gets. Generation 0 is the tail. */
|
|
12
|
+
readonly nextGen: number;
|
|
13
|
+
/** The live segments, oldest first. */
|
|
14
|
+
readonly segments: readonly SegmentRef[];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface TailEntry {
|
|
18
|
+
seq: number;
|
|
19
|
+
vector: Float32Array;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** What `read()` found in storage, for `adopt()` to install. */
|
|
23
|
+
export interface CatalogSnapshot {
|
|
24
|
+
readonly state: CatalogState;
|
|
25
|
+
readonly tail: Map<number, TailEntry>;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Ids per statement (bound as one JSON parameter). */
|
|
29
|
+
const ID_BATCH = 2000;
|
|
30
|
+
|
|
31
|
+
const EMPTY: CatalogState = { seq: 0, nextGen: 1, segments: [] };
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The generation catalog of a vector index: the committed state, the `{p}ids` table (which
|
|
35
|
+
* generation holds the live copy of each id, 0 = the tail), the `{p}tail` rows, and the in-memory
|
|
36
|
+
* copy of the state and the tail.
|
|
37
|
+
*
|
|
38
|
+
* It is the only place that commits a state change, and every commit updates the copy in the
|
|
39
|
+
* same synchronous step as it returns from the transaction, so the copy always matches storage.
|
|
40
|
+
* It also creates, sweeps and drops every table under the prefix, asking the segment store for
|
|
41
|
+
* its own.
|
|
42
|
+
*
|
|
43
|
+
* Callers serialize the commits (one at a time per instance); reads of `state` and `tail` may
|
|
44
|
+
* happen at any time and see either the copy before a commit or the copy after it.
|
|
45
|
+
*/
|
|
46
|
+
export class GenerationCatalog {
|
|
47
|
+
/** `{p}ids`: its `id` column holds exactly the live ids. */
|
|
48
|
+
readonly idTable: string;
|
|
49
|
+
readonly #db: Db;
|
|
50
|
+
readonly #store: SqlSegmentStore;
|
|
51
|
+
readonly #t: { meta: string; ids: string; tail: string };
|
|
52
|
+
#state: CatalogState = EMPTY;
|
|
53
|
+
#tail = new Map<number, TailEntry>();
|
|
54
|
+
|
|
55
|
+
constructor(db: Db, prefix: string, store: SqlSegmentStore) {
|
|
56
|
+
this.#db = db;
|
|
57
|
+
this.#store = store;
|
|
58
|
+
this.#t = catalogTables(prefix);
|
|
59
|
+
this.idTable = this.#t.ids;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** The committed state, as of the last commit or `adopt()`. Never mutated: replaced. */
|
|
63
|
+
get state(): CatalogState {
|
|
64
|
+
return this.#state;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** The tail rows in memory, by id. */
|
|
68
|
+
get tail(): ReadonlyMap<number, TailEntry> {
|
|
69
|
+
return this.#tail;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// ------------------------------------------------------------ tables
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Creates every table if missing and checks the stored config (dims and metric) is `config`:
|
|
76
|
+
* the first open stores it, later opens must match.
|
|
77
|
+
*/
|
|
78
|
+
async create(config: string): Promise<void> {
|
|
79
|
+
const { meta, ids, tail } = this.#t;
|
|
80
|
+
for (const s of [
|
|
81
|
+
`CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,
|
|
82
|
+
`CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,
|
|
83
|
+
`CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,
|
|
84
|
+
`CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,
|
|
85
|
+
`CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,
|
|
86
|
+
...this.#store.schema(),
|
|
87
|
+
]) {
|
|
88
|
+
await this.#db.exec(s);
|
|
89
|
+
}
|
|
90
|
+
await this.#db.query(
|
|
91
|
+
`INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`,
|
|
92
|
+
[config],
|
|
93
|
+
);
|
|
94
|
+
const [stored] = await this.#db.query<{ v: string }>(
|
|
95
|
+
`SELECT v FROM ${meta} WHERE k = 'config'`,
|
|
96
|
+
);
|
|
97
|
+
if (stored?.v !== config) {
|
|
98
|
+
throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Deletes segment rows of every generation the committed state does not list (crash leftovers). */
|
|
103
|
+
async sweep(): Promise<void> {
|
|
104
|
+
const live = this.#state.segments.map((s) => s.gen);
|
|
105
|
+
await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Deletes the segment rows of retired generations. */
|
|
109
|
+
async dropGenerations(gens: number[]): Promise<void> {
|
|
110
|
+
await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// ------------------------------------------------------------ reading storage
|
|
114
|
+
|
|
115
|
+
/** Reads the committed state and the tail rows, without installing them. */
|
|
116
|
+
async read(): Promise<CatalogSnapshot> {
|
|
117
|
+
const [row] = await this.#db.query<{ v: string }>(
|
|
118
|
+
`SELECT v FROM ${this.#t.meta} WHERE k = 'state'`,
|
|
119
|
+
);
|
|
120
|
+
const state: CatalogState = row ? JSON.parse(row.v) : EMPTY;
|
|
121
|
+
const tail = new Map<number, TailEntry>();
|
|
122
|
+
const rows = await this.#db.query<{ id: number; seq: number; vec: Uint8Array }>(
|
|
123
|
+
`SELECT id, seq, vec FROM ${this.#t.tail}`,
|
|
124
|
+
);
|
|
125
|
+
for (const r of rows) tail.set(Number(r.id), { seq: Number(r.seq), vector: toFloat32(r.vec) });
|
|
126
|
+
return { state, tail };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Installs a snapshot from `read()` as the copy. No commit may run between the two. */
|
|
130
|
+
adopt(snapshot: CatalogSnapshot): void {
|
|
131
|
+
this.#state = snapshot.state;
|
|
132
|
+
this.#tail = snapshot.tail;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** The generation holding the live copy of each id; deleted or unknown ids are absent. */
|
|
136
|
+
async generations(ids: number[]): Promise<Map<number, number>> {
|
|
137
|
+
const out = new Map<number, number>();
|
|
138
|
+
const unique = [...new Set(ids)];
|
|
139
|
+
for (let i = 0; i < unique.length; i += ID_BATCH) {
|
|
140
|
+
const rows = await this.#db.query<{ id: number; gen: number }>(
|
|
141
|
+
`SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`,
|
|
142
|
+
[JSON.stringify(unique.slice(i, i + ID_BATCH))],
|
|
143
|
+
);
|
|
144
|
+
for (const r of rows) out.set(Number(r.id), Number(r.gen));
|
|
145
|
+
}
|
|
146
|
+
return out;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// ------------------------------------------------------------ commits
|
|
150
|
+
|
|
151
|
+
/** Writes `items` to the tail as one new batch (the next seq), in one transaction. */
|
|
152
|
+
async append(items: { id: number; vector: Float32Array }[]): Promise<void> {
|
|
153
|
+
const { tail, ids } = this.#t;
|
|
154
|
+
const seq = this.#state.seq + 1;
|
|
155
|
+
const next: CatalogState = { ...this.#state, seq };
|
|
156
|
+
await inTransaction(this.#db, async (tx) => {
|
|
157
|
+
for (const { id, vector } of items) {
|
|
158
|
+
await tx.query(
|
|
159
|
+
`INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)
|
|
160
|
+
ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`,
|
|
161
|
+
[id, seq, float32Bytes(vector)],
|
|
162
|
+
);
|
|
163
|
+
await tx.query(
|
|
164
|
+
`INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`,
|
|
165
|
+
[id],
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
await this.#save(tx, next);
|
|
169
|
+
});
|
|
170
|
+
this.#state = next;
|
|
171
|
+
for (const { id, vector } of items) this.#tail.set(id, { seq, vector });
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** Removes `ids` from the tail and from the live ids, one transaction per batch. */
|
|
175
|
+
async remove(ids: number[]): Promise<void> {
|
|
176
|
+
for (let i = 0; i < ids.length; i += ID_BATCH) {
|
|
177
|
+
const chunk = ids.slice(i, i + ID_BATCH);
|
|
178
|
+
const json = JSON.stringify(chunk);
|
|
179
|
+
await inTransaction(this.#db, async (tx) => {
|
|
180
|
+
await tx.query(`DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`, [
|
|
181
|
+
json,
|
|
182
|
+
]);
|
|
183
|
+
await tx.query(`DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`, [
|
|
184
|
+
json,
|
|
185
|
+
]);
|
|
186
|
+
});
|
|
187
|
+
for (const id of chunk) this.#tail.delete(id);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Commits `built` (sealed from the tail rows up to `maxSeq`, `count` vectors) as a new
|
|
193
|
+
* generation: writes its rows, moves those ids to it and deletes those tail rows. Tail rows of
|
|
194
|
+
* later batches stay. `install` runs with the new ref in the same synchronous step as the copy
|
|
195
|
+
* update, for the caller's own in-memory segment list.
|
|
196
|
+
*/
|
|
197
|
+
async seal(
|
|
198
|
+
maxSeq: number,
|
|
199
|
+
built: BuiltSegment,
|
|
200
|
+
count: number,
|
|
201
|
+
install: (ref: SegmentRef) => void,
|
|
202
|
+
): Promise<SegmentRef> {
|
|
203
|
+
const ref = this.#refOf(built, count);
|
|
204
|
+
const next: CatalogState = {
|
|
205
|
+
...this.#state,
|
|
206
|
+
nextGen: ref.gen + 1,
|
|
207
|
+
segments: [...this.#state.segments, ref],
|
|
208
|
+
};
|
|
209
|
+
const { ids, tail } = this.#t;
|
|
210
|
+
await inTransaction(this.#db, async (tx) => {
|
|
211
|
+
await this.#store.write(tx, ref.gen, built);
|
|
212
|
+
await tx.query(
|
|
213
|
+
`UPDATE ${ids} SET gen = ? WHERE gen = 0
|
|
214
|
+
AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`,
|
|
215
|
+
[ref.gen, maxSeq],
|
|
216
|
+
);
|
|
217
|
+
await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [maxSeq]);
|
|
218
|
+
await this.#save(tx, next);
|
|
219
|
+
});
|
|
220
|
+
this.#state = next;
|
|
221
|
+
for (const [id, e] of this.#tail) if (e.seq <= maxSeq) this.#tail.delete(id);
|
|
222
|
+
install(ref);
|
|
223
|
+
return ref;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Replaces the generations `oldGens` with `built` (`count` live vectors), or with nothing when
|
|
228
|
+
* `built` is null: writes its rows, moves the ids of the old generations to it and drops them
|
|
229
|
+
* from the state. Their rows stay until `dropGenerations()`. `install` runs as in `seal()`.
|
|
230
|
+
*/
|
|
231
|
+
async replace(
|
|
232
|
+
oldGens: number[],
|
|
233
|
+
built: BuiltSegment | null,
|
|
234
|
+
count: number,
|
|
235
|
+
install: (ref: SegmentRef | null) => void,
|
|
236
|
+
): Promise<SegmentRef | null> {
|
|
237
|
+
const ref = built ? this.#refOf(built, count) : null;
|
|
238
|
+
const kept = this.#state.segments.filter((s) => !oldGens.includes(s.gen));
|
|
239
|
+
const next: CatalogState = {
|
|
240
|
+
...this.#state,
|
|
241
|
+
nextGen: ref ? ref.gen + 1 : this.#state.nextGen,
|
|
242
|
+
segments: ref ? [...kept, ref] : kept,
|
|
243
|
+
};
|
|
244
|
+
await inTransaction(this.#db, async (tx) => {
|
|
245
|
+
if (ref && built) {
|
|
246
|
+
await this.#store.write(tx, ref.gen, built);
|
|
247
|
+
await tx.query(
|
|
248
|
+
`UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`,
|
|
249
|
+
[ref.gen, JSON.stringify(oldGens)],
|
|
250
|
+
);
|
|
251
|
+
}
|
|
252
|
+
await this.#save(tx, next);
|
|
253
|
+
});
|
|
254
|
+
this.#state = next;
|
|
255
|
+
install(ref);
|
|
256
|
+
return ref;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
#refOf(built: BuiltSegment, count: number): SegmentRef {
|
|
260
|
+
return { gen: this.#state.nextGen, count, nlist: built.nlist };
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
async #save(tx: Db, state: CatalogState): Promise<void> {
|
|
264
|
+
await tx.query(
|
|
265
|
+
`INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)
|
|
266
|
+
ON CONFLICT(k) DO UPDATE SET v = excluded.v`,
|
|
267
|
+
[JSON.stringify(state)],
|
|
268
|
+
);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function catalogTables(prefix: string) {
|
|
273
|
+
return { meta: `${prefix}meta`, ids: `${prefix}ids`, tail: `${prefix}tail` };
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/** Drops every table under `prefix` (the catalog's and the segment store's), in one transaction. */
|
|
277
|
+
export async function dropCatalog(db: Db, prefix: string): Promise<void> {
|
|
278
|
+
const tables = [
|
|
279
|
+
...Object.values(catalogTables(prefix)),
|
|
280
|
+
...Object.values(SqlSegmentStore.tables(prefix)),
|
|
281
|
+
];
|
|
282
|
+
await inTransaction(db, async (tx) => {
|
|
283
|
+
for (const t of tables) await tx.exec(`DROP TABLE IF EXISTS ${t}`);
|
|
284
|
+
});
|
|
285
|
+
}
|
package/src/vector/math.ts
CHANGED
|
@@ -17,7 +17,12 @@ export function prepare(v: ArrayLike<number>, dims: number, metric: Metric): Flo
|
|
|
17
17
|
for (let i = 0; i < f.length; i++) {
|
|
18
18
|
if (!Number.isFinite(f[i])) throw new Error("vector-index: vector contains a non-finite value");
|
|
19
19
|
}
|
|
20
|
-
|
|
20
|
+
if (metric !== "cosine") return f.slice();
|
|
21
|
+
const out = normalize(f);
|
|
22
|
+
if (!out.some((x) => x !== 0)) {
|
|
23
|
+
throw new Error("vector-index: a zero vector has no direction under the cosine metric");
|
|
24
|
+
}
|
|
25
|
+
return out;
|
|
21
26
|
}
|
|
22
27
|
|
|
23
28
|
/** Distance between two float vectors at offsets; lower is closer. */
|
package/src/vector/segment.ts
CHANGED
|
@@ -184,6 +184,9 @@ export interface Candidate {
|
|
|
184
184
|
}
|
|
185
185
|
|
|
186
186
|
/** The in-memory part of a segment: centroids and list sizes. List blocks are fetched by the caller. */
|
|
187
|
+
/** The share of a segment's lists that {@link SegmentIndex.probe} may widen to for `minRows`. */
|
|
188
|
+
export const MAX_WIDEN_FRACTION = 1 / 3;
|
|
189
|
+
|
|
187
190
|
export class SegmentIndex {
|
|
188
191
|
constructor(
|
|
189
192
|
readonly gen: number,
|
|
@@ -197,20 +200,50 @@ export class SegmentIndex {
|
|
|
197
200
|
return this.counts.length;
|
|
198
201
|
}
|
|
199
202
|
|
|
200
|
-
/**
|
|
201
|
-
|
|
203
|
+
/**
|
|
204
|
+
* The non-empty lists closest to the query, in centroid order: at least the
|
|
205
|
+
* `nprobe` nearest, then more until the probed lists hold `minRows` rows (or
|
|
206
|
+
* none are left). `minRows` 0 probes exactly `nprobe` lists. The widening
|
|
207
|
+
* stops at a third of the lists ({@link MAX_WIDEN_FRACTION}), so a large `k`
|
|
208
|
+
* never turns a search into a scan of every list.
|
|
209
|
+
*/
|
|
210
|
+
probe(query: Float32Array, nprobe: number, minRows = 0): number[] {
|
|
211
|
+
const maxLists = Math.max(nprobe, Math.ceil(this.nlist * MAX_WIDEN_FRACTION));
|
|
202
212
|
const m: Metric = this.metric === "l2" ? "l2" : "dot";
|
|
203
|
-
const
|
|
213
|
+
const scored: { d: number; c: number }[] = [];
|
|
204
214
|
for (let c = 0; c < this.nlist; c++) {
|
|
205
215
|
if (this.counts[c] === 0) continue;
|
|
206
|
-
|
|
216
|
+
scored.push({ d: distance(m, query, 0, this.centroids, c * this.dims, this.dims), c });
|
|
207
217
|
}
|
|
208
|
-
|
|
218
|
+
scored.sort((a, b) => a.d - b.d || a.c - b.c);
|
|
219
|
+
const out: number[] = [];
|
|
220
|
+
let rows = 0;
|
|
221
|
+
for (const { c } of scored) {
|
|
222
|
+
if (out.length >= nprobe && (rows >= minRows || out.length >= maxLists)) break;
|
|
223
|
+
out.push(c);
|
|
224
|
+
rows += this.counts[c];
|
|
225
|
+
}
|
|
226
|
+
return out;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Whether stage 1 passes a list of `n` rows whole (in row order, not ranked):
|
|
231
|
+
* a list no longer than `limit`, or a flat segment's only list of at most
|
|
232
|
+
* {@link MIN_TRAIN} rows. Callers must not cut such a list by position.
|
|
233
|
+
*/
|
|
234
|
+
passesWhole(n: number, limit: number): boolean {
|
|
235
|
+
return n <= limit || (this.nlist === 1 && n <= MIN_TRAIN);
|
|
209
236
|
}
|
|
210
237
|
|
|
211
238
|
/**
|
|
212
239
|
* Stage 1: the `limit` rows of a list with the best estimated distance, from
|
|
213
240
|
* residual sign bits plus the stored residual norm and centroid projection.
|
|
241
|
+
*
|
|
242
|
+
* A flat segment (one list of at most {@link MIN_TRAIN} rows) passes every row:
|
|
243
|
+
* its centroid is the mean of all vectors, far from most of them on clustered
|
|
244
|
+
* data, so the 1-bit estimate is weakest exactly there (issue #40), while
|
|
245
|
+
* int8-scoring the whole list costs no more than scanning the tail. A list
|
|
246
|
+
* that does not exceed `limit` also passes whole, without estimating.
|
|
214
247
|
*/
|
|
215
248
|
filterList(
|
|
216
249
|
query: Float32Array,
|
|
@@ -219,6 +252,15 @@ export class SegmentIndex {
|
|
|
219
252
|
limit: number,
|
|
220
253
|
keep?: (id: number) => boolean,
|
|
221
254
|
): Candidate[] {
|
|
255
|
+
const n = data.ids.length;
|
|
256
|
+
if (this.passesWhole(n, limit)) {
|
|
257
|
+
const all: Candidate[] = [];
|
|
258
|
+
for (let r = 0; r < n; r++) {
|
|
259
|
+
const id = data.ids[r];
|
|
260
|
+
if (!keep || keep(id)) all.push({ list: c, row: r, id });
|
|
261
|
+
}
|
|
262
|
+
return all;
|
|
263
|
+
}
|
|
222
264
|
const d = this.dims;
|
|
223
265
|
const w = wordsFor(d);
|
|
224
266
|
const co = c * d;
|
|
@@ -231,7 +273,7 @@ export class SegmentIndex {
|
|
|
231
273
|
const angle = Math.PI / d;
|
|
232
274
|
const l2 = this.metric === "l2";
|
|
233
275
|
const top = new TopK<Candidate>(limit);
|
|
234
|
-
for (let r = 0; r <
|
|
276
|
+
for (let r = 0; r < n; r++) {
|
|
235
277
|
const id = data.ids[r];
|
|
236
278
|
if (keep && !keep(id)) continue;
|
|
237
279
|
const rr = qn * data.rnorm[r] * Math.cos(angle * hamming(qb, data.bits, r * w, w));
|
|
@@ -25,10 +25,15 @@ export class SqlSegmentStore {
|
|
|
25
25
|
readonly #t: { centroids: string; seg: string; f32: string };
|
|
26
26
|
readonly #cache: Lru<ListData | Float32Array>;
|
|
27
27
|
|
|
28
|
+
/** The names of the tables segment rows live in, under `prefix`. */
|
|
29
|
+
static tables(prefix: string): { centroids: string; seg: string; f32: string } {
|
|
30
|
+
return { centroids: `${prefix}centroids`, seg: `${prefix}seg`, f32: `${prefix}f32` };
|
|
31
|
+
}
|
|
32
|
+
|
|
28
33
|
constructor(db: Db, prefix: string, dims: number, cacheBytes = 64 * 1024 * 1024) {
|
|
29
34
|
this.#db = db;
|
|
30
35
|
this.#dims = dims;
|
|
31
|
-
this.#t =
|
|
36
|
+
this.#t = SqlSegmentStore.tables(prefix);
|
|
32
37
|
this.#cache = new Lru(cacheBytes);
|
|
33
38
|
}
|
|
34
39
|
|
package/src/vector/types.ts
CHANGED
|
@@ -8,7 +8,10 @@ export interface SearchHit {
|
|
|
8
8
|
|
|
9
9
|
export interface SearchOptions {
|
|
10
10
|
k?: number;
|
|
11
|
-
/**
|
|
11
|
+
/**
|
|
12
|
+
* IVF lists probed per segment, exactly. Default: at least 8, more until the
|
|
13
|
+
* probed lists hold 16 × k × oversample rows (small lists after auto-compaction).
|
|
14
|
+
*/
|
|
12
15
|
nprobe?: number;
|
|
13
16
|
/** Candidates kept per probed list by the 1-bit filter = k * oversample. Default 8. */
|
|
14
17
|
oversample?: number;
|