@statewalker/db-sqlite-index 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -4
- package/dist/ngram.d.mts.map +1 -1
- package/dist/ngram.mjs +83 -3
- package/dist/ngram.mjs.map +1 -1
- package/dist/vector.d.mts +4 -1
- package/dist/vector.d.mts.map +1 -1
- package/dist/vector.mjs +115 -22
- package/dist/vector.mjs.map +1 -1
- package/package.json +16 -8
- package/src/ngram/fold.ts +44 -0
- package/src/ngram/fts5-backend.ts +4 -0
- package/src/ngram/ngram-store.ts +37 -0
- package/src/ngram/plan-query.ts +8 -1
- package/src/ngram/postings-backend.ts +1 -0
- package/src/ngram/regex-literals.ts +5 -2
- package/src/vector/math.ts +6 -1
- package/src/vector/segment.ts +48 -6
- package/src/vector/types.ts +4 -1
- package/src/vector/vector-index.ts +62 -16
- package/src/ngram/do-smoke.ts +0 -46
- package/src/ngram/fold.test.ts +0 -50
- package/src/ngram/json-chunks.test.ts +0 -23
- package/src/ngram/ngram-store.test.ts +0 -291
- package/src/ngram/plan-query.test.ts +0 -141
- package/src/ngram/trigrams.test.ts +0 -28
- package/src/vector/auto-compact.test.ts +0 -28
- package/src/vector/do-smoke.ts +0 -39
- package/src/vector/segment.test.ts +0 -61
- package/src/vector/test-fixtures.ts +0 -64
- package/src/vector/vector-index.test.ts +0 -105
- package/src/vector/vector-lifecycle.test.ts +0 -171
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@statewalker/db-sqlite-index",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Id-based vector (IVF) and n-gram indexes stored in SQLite through @statewalker/db-api: Node, browser and Durable Objects.",
|
|
@@ -22,31 +22,37 @@
|
|
|
22
22
|
"./vector": {
|
|
23
23
|
"source": "./src/vector/index.ts",
|
|
24
24
|
"types": "./dist/vector.d.mts",
|
|
25
|
-
"import": "./dist/vector.mjs"
|
|
25
|
+
"import": "./dist/vector.mjs",
|
|
26
|
+
"default": "./dist/vector.mjs"
|
|
26
27
|
},
|
|
27
28
|
"./ngram": {
|
|
28
29
|
"source": "./src/ngram/index.ts",
|
|
29
30
|
"types": "./dist/ngram.d.mts",
|
|
30
|
-
"import": "./dist/ngram.mjs"
|
|
31
|
+
"import": "./dist/ngram.mjs",
|
|
32
|
+
"default": "./dist/ngram.mjs"
|
|
31
33
|
}
|
|
32
34
|
},
|
|
33
35
|
"files": [
|
|
34
36
|
"dist",
|
|
35
|
-
"src"
|
|
37
|
+
"src",
|
|
38
|
+
"!src/**/*.test.ts",
|
|
39
|
+
"!src/**/test-fixtures.ts"
|
|
36
40
|
],
|
|
37
41
|
"dependencies": {
|
|
38
|
-
"@statewalker/db-api": "^0.1.
|
|
39
|
-
"@statewalker/db-sqlite": "^0.1.
|
|
42
|
+
"@statewalker/db-api": "^0.1.6",
|
|
43
|
+
"@statewalker/db-sqlite": "^0.1.2"
|
|
40
44
|
},
|
|
41
45
|
"devDependencies": {
|
|
42
46
|
"@types/node": "^26.6.4",
|
|
47
|
+
"esbuild": "^0.28.2",
|
|
43
48
|
"rimraf": "^6.1.3",
|
|
44
49
|
"tsdown": "^0.23.0",
|
|
45
50
|
"tsx": "^4.23.15",
|
|
46
51
|
"typescript": "^7.0.2",
|
|
47
52
|
"vitest": "^4.1.10 || ^5.0.3",
|
|
48
|
-
"@statewalker/db-sqlite-
|
|
49
|
-
"@statewalker/db-sqlite-
|
|
53
|
+
"@statewalker/db-sqlite-node": "^0.3.2",
|
|
54
|
+
"@statewalker/db-sqlite-do": "^0.1.2",
|
|
55
|
+
"@statewalker/db-tests": "^0.1.1"
|
|
50
56
|
},
|
|
51
57
|
"sideEffects": false,
|
|
52
58
|
"publishConfig": {
|
|
@@ -56,8 +62,10 @@
|
|
|
56
62
|
"build": "tsdown",
|
|
57
63
|
"dev": "tsdown --watch",
|
|
58
64
|
"test": "vitest run",
|
|
65
|
+
"test:do": "vitest run --config vitest.do.config.ts",
|
|
59
66
|
"test:watch": "vitest",
|
|
60
67
|
"typecheck": "tsc --noEmit",
|
|
68
|
+
"typecheck:tests": "tsc --noEmit -p tsconfig.test.json",
|
|
61
69
|
"bench": "tsx bench/vector-bench.ts 50000 384",
|
|
62
70
|
"clean": "rimraf dist",
|
|
63
71
|
"lint": "biome check --write .",
|
package/src/ngram/fold.ts
CHANGED
|
@@ -19,3 +19,47 @@ export function fold(text: string): string {
|
|
|
19
19
|
}
|
|
20
20
|
return out;
|
|
21
21
|
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Highest code point probed by `foldIdentity`. Every cased character is in the
|
|
25
|
+
* BMP or the SMP (Deseret, Osage, Adlam, …); planes 2 and up hold none.
|
|
26
|
+
*/
|
|
27
|
+
const PROBE_END = 0x1ffff;
|
|
28
|
+
const IDENTITY_VERSION = "fold-v1";
|
|
29
|
+
let current: string | undefined;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* A fingerprint of what `foldFn` does to every code point up to U+1FFFF: a
|
|
33
|
+
* 53-bit hash of each code point it changes and what it changes it to. The
|
|
34
|
+
* grams of an n-gram index are built with `fold`, and a contentless FTS5 row is
|
|
35
|
+
* deleted by replaying them, so a runtime whose case mappings differ (a newer
|
|
36
|
+
* Unicode in a newer V8 or ICU) yields a different identity, and the store
|
|
37
|
+
* rebuilds its grams. It needs no runtime API, so it works in Node, browsers
|
|
38
|
+
* and Durable Objects alike. The identity of `fold` is computed once (about
|
|
39
|
+
* 30 ms) and memoized.
|
|
40
|
+
*/
|
|
41
|
+
export function foldIdentity(foldFn: (text: string) => string = fold): string {
|
|
42
|
+
if (foldFn === fold && current !== undefined) return current;
|
|
43
|
+
// cyrb53: two 32-bit FNV-style lanes mixed into 53 bits.
|
|
44
|
+
let h1 = 0xdeadbeef;
|
|
45
|
+
let h2 = 0x41c6ce57;
|
|
46
|
+
const mix = (value: number) => {
|
|
47
|
+
h1 = Math.imul(h1 ^ value, 2654435761);
|
|
48
|
+
h2 = Math.imul(h2 ^ value, 1597334677);
|
|
49
|
+
};
|
|
50
|
+
for (let cp = 0; cp <= PROBE_END; cp++) {
|
|
51
|
+
if (cp >= 0xd800 && cp <= 0xdfff) continue;
|
|
52
|
+
const ch = String.fromCodePoint(cp);
|
|
53
|
+
const folded = foldFn(ch);
|
|
54
|
+
if (folded === ch) continue;
|
|
55
|
+
mix(cp);
|
|
56
|
+
for (let i = 0; i < folded.length; i++) mix(folded.charCodeAt(i));
|
|
57
|
+
mix(0x110000); // separator: the folded form of one code point may be several units
|
|
58
|
+
}
|
|
59
|
+
h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909);
|
|
60
|
+
h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909);
|
|
61
|
+
const hash = 4294967296 * (2097151 & h2) + (h1 >>> 0);
|
|
62
|
+
const identity = `${IDENTITY_VERSION}:${hash.toString(16)}`;
|
|
63
|
+
if (foldFn === fold) current = identity;
|
|
64
|
+
return identity;
|
|
65
|
+
}
|
|
@@ -10,6 +10,8 @@ export type GramBackend = {
|
|
|
10
10
|
add(tx: Db, items: NgramItem[]): Promise<void>;
|
|
11
11
|
/** `items` hold the texts as currently stored, so their grams can be removed exactly. */
|
|
12
12
|
remove(tx: Db, items: NgramItem[]): Promise<void>;
|
|
13
|
+
/** Removes every gram, leaving an empty store (used to rebuild after a fold change). */
|
|
14
|
+
clear(tx: Db): Promise<void>;
|
|
13
15
|
/** Ids whose text contains every literal of at least one branch; ascending, at most `max`. */
|
|
14
16
|
candidates(db: Db, branches: string[][], max: number): Promise<number[]>;
|
|
15
17
|
};
|
|
@@ -41,6 +43,8 @@ export function newFts5Backend(prefix: string): GramBackend {
|
|
|
41
43
|
),
|
|
42
44
|
add: (tx, items) => write(tx, items, "insert"),
|
|
43
45
|
remove: (tx, items) => write(tx, items, "delete"),
|
|
46
|
+
// 'rebuild' needs a content table; a contentless one can only be emptied.
|
|
47
|
+
clear: (tx) => tx.exec(`INSERT INTO ${table}(${table}) VALUES ('delete-all')`),
|
|
44
48
|
async candidates(db, branches, max) {
|
|
45
49
|
// A quoted string is a phrase of consecutive trigrams, i.e. an exact substring.
|
|
46
50
|
const quote = (literal: string) => `"${literal.replaceAll('"', '""')}"`;
|
package/src/ngram/ngram-store.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { Db } from "@statewalker/db-api";
|
|
2
2
|
import { inTransaction } from "@statewalker/db-sqlite";
|
|
3
3
|
import { PREFIX_PATTERN } from "../prefix.js";
|
|
4
|
+
import { foldIdentity } from "./fold.js";
|
|
4
5
|
import { type GramBackend, type NgramItem, newFts5Backend } from "./fts5-backend.js";
|
|
5
6
|
import { chunkJson } from "./json-chunks.js";
|
|
6
7
|
import { planQuery } from "./plan-query.js";
|
|
@@ -31,6 +32,8 @@ export type NgramSearchResult = {
|
|
|
31
32
|
export type NgramStoreOptions = { db: Db; prefix: string; store?: "fts5" | "postings" };
|
|
32
33
|
|
|
33
34
|
const VERIFY_BATCH = 256;
|
|
35
|
+
/** Texts re-indexed per step when the grams are rebuilt after a fold change. */
|
|
36
|
+
const REBUILD_BATCH = 256;
|
|
34
37
|
|
|
35
38
|
export class NgramStore {
|
|
36
39
|
/** `{prefix}texts`: its `id` column holds exactly the stored ids. Callers may join it in SQL. */
|
|
@@ -73,9 +76,43 @@ export class NgramStore {
|
|
|
73
76
|
`CREATE TABLE IF NOT EXISTS ${p}texts(id INTEGER PRIMARY KEY, text TEXT NOT NULL)`,
|
|
74
77
|
);
|
|
75
78
|
await this.#backend.create(tx);
|
|
79
|
+
await this.#checkFold(tx);
|
|
76
80
|
});
|
|
77
81
|
}
|
|
78
82
|
|
|
83
|
+
/**
|
|
84
|
+
* The grams were built with `fold` as the runtime that wrote them defined it, and
|
|
85
|
+
* an fts5 delete replays them; if this runtime folds differently, they are rebuilt
|
|
86
|
+
* from the stored texts before any write. An index without a recorded identity
|
|
87
|
+
* (written before it was recorded) is assumed to match and is not rebuilt.
|
|
88
|
+
*/
|
|
89
|
+
async #checkFold(tx: Db): Promise<void> {
|
|
90
|
+
const p = this.#prefix;
|
|
91
|
+
const identity = foldIdentity();
|
|
92
|
+
const [row] = await tx.query<{ value: string }>(
|
|
93
|
+
`SELECT value FROM ${p}meta WHERE key = 'fold'`,
|
|
94
|
+
);
|
|
95
|
+
if (row?.value === identity) return;
|
|
96
|
+
if (row) await this.#rebuild(tx);
|
|
97
|
+
await tx.query(`INSERT OR REPLACE INTO ${p}meta(key, value) VALUES ('fold', ?)`, [identity]);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
async #rebuild(tx: Db): Promise<void> {
|
|
101
|
+
await this.#backend.clear(tx);
|
|
102
|
+
let after: number | null = null;
|
|
103
|
+
for (;;) {
|
|
104
|
+
const rows: Array<{ id: number; text: string }> = await tx.query(
|
|
105
|
+
`SELECT id, text FROM ${this.#prefix}texts WHERE ? IS NULL OR id > ? ORDER BY id LIMIT ?`,
|
|
106
|
+
[after, after, REBUILD_BATCH],
|
|
107
|
+
);
|
|
108
|
+
const batch = rows.map((r) => ({ id: Number(r.id), text: String(r.text) }));
|
|
109
|
+
const last = batch.at(-1);
|
|
110
|
+
if (!last) return;
|
|
111
|
+
await this.#backend.add(tx, batch);
|
|
112
|
+
after = last.id;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
79
116
|
async upsert(items: NgramItem[]): Promise<void> {
|
|
80
117
|
// The last text per id wins; a repeated id would leave stale grams behind.
|
|
81
118
|
// A lone surrogate would be stored differently from how it was indexed, so a
|
package/src/ngram/plan-query.ts
CHANGED
|
@@ -4,6 +4,8 @@ import { regexLiterals } from "./regex-literals.js";
|
|
|
4
4
|
/** Longer patterns are rejected: a JS regex cannot be interrupted once it runs. */
|
|
5
5
|
export const MAX_PATTERN_LENGTH = 1000;
|
|
6
6
|
|
|
7
|
+
const LONE_SURROGATE = /[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/;
|
|
8
|
+
|
|
7
9
|
export type QueryPlan = {
|
|
8
10
|
/** The pattern as a global regex (`i` added when not case-sensitive); verifies candidates. */
|
|
9
11
|
regex: RegExp;
|
|
@@ -34,7 +36,12 @@ export function planQuery(
|
|
|
34
36
|
const regex = new RegExp(source, opts.caseSensitive === false ? "gi" : "g");
|
|
35
37
|
const branches: string[][] = [];
|
|
36
38
|
for (const literals of regexLiterals(regex)) {
|
|
37
|
-
|
|
39
|
+
// Without the `u` flag a literal may hold a lone surrogate (`ab😀?`, `\uD83D`); indexed
|
|
40
|
+
// text is split by code point, so keep only the well-formed pieces around it.
|
|
41
|
+
const usable = literals
|
|
42
|
+
.flatMap((literal) => literal.split(LONE_SURROGATE))
|
|
43
|
+
.map(fold)
|
|
44
|
+
.filter((literal) => Array.from(literal).length >= 3);
|
|
38
45
|
if (usable.length === 0) return { regex, branches: null };
|
|
39
46
|
branches.push(usable);
|
|
40
47
|
}
|
|
@@ -297,9 +297,12 @@ function parseLiteralClass(src: string, start: number, end: number): string[] |
|
|
|
297
297
|
const c = src[i] ?? "";
|
|
298
298
|
if (c === "\\") {
|
|
299
299
|
const next = src[i + 1];
|
|
300
|
-
|
|
301
|
-
|
|
300
|
+
// Letters are classes or code escapes; digits are octal escapes without the `u` flag.
|
|
301
|
+
if (next === undefined || /[a-zA-Z0-9]/.test(next)) return null;
|
|
302
302
|
i += 2;
|
|
303
|
+
// `[\.-z]`: the escape starts a range, not three literals.
|
|
304
|
+
if (src[i] === "-" && i + 1 < lastInside) return null;
|
|
305
|
+
chars.add(next);
|
|
303
306
|
continue;
|
|
304
307
|
}
|
|
305
308
|
if (i + 2 < lastInside && src[i + 1] === "-") return null;
|
package/src/vector/math.ts
CHANGED
|
@@ -17,7 +17,12 @@ export function prepare(v: ArrayLike<number>, dims: number, metric: Metric): Flo
|
|
|
17
17
|
for (let i = 0; i < f.length; i++) {
|
|
18
18
|
if (!Number.isFinite(f[i])) throw new Error("vector-index: vector contains a non-finite value");
|
|
19
19
|
}
|
|
20
|
-
|
|
20
|
+
if (metric !== "cosine") return f.slice();
|
|
21
|
+
const out = normalize(f);
|
|
22
|
+
if (!out.some((x) => x !== 0)) {
|
|
23
|
+
throw new Error("vector-index: a zero vector has no direction under the cosine metric");
|
|
24
|
+
}
|
|
25
|
+
return out;
|
|
21
26
|
}
|
|
22
27
|
|
|
23
28
|
/** Distance between two float vectors at offsets; lower is closer. */
|
package/src/vector/segment.ts
CHANGED
|
@@ -184,6 +184,9 @@ export interface Candidate {
|
|
|
184
184
|
}
|
|
185
185
|
|
|
186
186
|
/** The in-memory part of a segment: centroids and list sizes. List blocks are fetched by the caller. */
|
|
187
|
+
/** The share of a segment's lists that {@link SegmentIndex.probe} may widen to for `minRows`. */
|
|
188
|
+
export const MAX_WIDEN_FRACTION = 1 / 3;
|
|
189
|
+
|
|
187
190
|
export class SegmentIndex {
|
|
188
191
|
constructor(
|
|
189
192
|
readonly gen: number,
|
|
@@ -197,20 +200,50 @@ export class SegmentIndex {
|
|
|
197
200
|
return this.counts.length;
|
|
198
201
|
}
|
|
199
202
|
|
|
200
|
-
/**
|
|
201
|
-
|
|
203
|
+
/**
|
|
204
|
+
* The non-empty lists closest to the query, in centroid order: at least the
|
|
205
|
+
* `nprobe` nearest, then more until the probed lists hold `minRows` rows (or
|
|
206
|
+
* none are left). `minRows` 0 probes exactly `nprobe` lists. The widening
|
|
207
|
+
* stops at a third of the lists ({@link MAX_WIDEN_FRACTION}), so a large `k`
|
|
208
|
+
* never turns a search into a scan of every list.
|
|
209
|
+
*/
|
|
210
|
+
probe(query: Float32Array, nprobe: number, minRows = 0): number[] {
|
|
211
|
+
const maxLists = Math.max(nprobe, Math.ceil(this.nlist * MAX_WIDEN_FRACTION));
|
|
202
212
|
const m: Metric = this.metric === "l2" ? "l2" : "dot";
|
|
203
|
-
const
|
|
213
|
+
const scored: { d: number; c: number }[] = [];
|
|
204
214
|
for (let c = 0; c < this.nlist; c++) {
|
|
205
215
|
if (this.counts[c] === 0) continue;
|
|
206
|
-
|
|
216
|
+
scored.push({ d: distance(m, query, 0, this.centroids, c * this.dims, this.dims), c });
|
|
207
217
|
}
|
|
208
|
-
|
|
218
|
+
scored.sort((a, b) => a.d - b.d || a.c - b.c);
|
|
219
|
+
const out: number[] = [];
|
|
220
|
+
let rows = 0;
|
|
221
|
+
for (const { c } of scored) {
|
|
222
|
+
if (out.length >= nprobe && (rows >= minRows || out.length >= maxLists)) break;
|
|
223
|
+
out.push(c);
|
|
224
|
+
rows += this.counts[c];
|
|
225
|
+
}
|
|
226
|
+
return out;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Whether stage 1 passes a list of `n` rows whole (in row order, not ranked):
|
|
231
|
+
* a list no longer than `limit`, or a flat segment's only list of at most
|
|
232
|
+
* {@link MIN_TRAIN} rows. Callers must not cut such a list by position.
|
|
233
|
+
*/
|
|
234
|
+
passesWhole(n: number, limit: number): boolean {
|
|
235
|
+
return n <= limit || (this.nlist === 1 && n <= MIN_TRAIN);
|
|
209
236
|
}
|
|
210
237
|
|
|
211
238
|
/**
|
|
212
239
|
* Stage 1: the `limit` rows of a list with the best estimated distance, from
|
|
213
240
|
* residual sign bits plus the stored residual norm and centroid projection.
|
|
241
|
+
*
|
|
242
|
+
* A flat segment (one list of at most {@link MIN_TRAIN} rows) passes every row:
|
|
243
|
+
* its centroid is the mean of all vectors, far from most of them on clustered
|
|
244
|
+
* data, so the 1-bit estimate is weakest exactly there (issue #40), while
|
|
245
|
+
* int8-scoring the whole list costs no more than scanning the tail. A list
|
|
246
|
+
* that does not exceed `limit` also passes whole, without estimating.
|
|
214
247
|
*/
|
|
215
248
|
filterList(
|
|
216
249
|
query: Float32Array,
|
|
@@ -219,6 +252,15 @@ export class SegmentIndex {
|
|
|
219
252
|
limit: number,
|
|
220
253
|
keep?: (id: number) => boolean,
|
|
221
254
|
): Candidate[] {
|
|
255
|
+
const n = data.ids.length;
|
|
256
|
+
if (this.passesWhole(n, limit)) {
|
|
257
|
+
const all: Candidate[] = [];
|
|
258
|
+
for (let r = 0; r < n; r++) {
|
|
259
|
+
const id = data.ids[r];
|
|
260
|
+
if (!keep || keep(id)) all.push({ list: c, row: r, id });
|
|
261
|
+
}
|
|
262
|
+
return all;
|
|
263
|
+
}
|
|
222
264
|
const d = this.dims;
|
|
223
265
|
const w = wordsFor(d);
|
|
224
266
|
const co = c * d;
|
|
@@ -231,7 +273,7 @@ export class SegmentIndex {
|
|
|
231
273
|
const angle = Math.PI / d;
|
|
232
274
|
const l2 = this.metric === "l2";
|
|
233
275
|
const top = new TopK<Candidate>(limit);
|
|
234
|
-
for (let r = 0; r <
|
|
276
|
+
for (let r = 0; r < n; r++) {
|
|
235
277
|
const id = data.ids[r];
|
|
236
278
|
if (keep && !keep(id)) continue;
|
|
237
279
|
const rr = qn * data.rnorm[r] * Math.cos(angle * hamming(qb, data.bits, r * w, w));
|
package/src/vector/types.ts
CHANGED
|
@@ -8,7 +8,10 @@ export interface SearchHit {
|
|
|
8
8
|
|
|
9
9
|
export interface SearchOptions {
|
|
10
10
|
k?: number;
|
|
11
|
-
/**
|
|
11
|
+
/**
|
|
12
|
+
* IVF lists probed per segment, exactly. Default: at least 8, more until the
|
|
13
|
+
* probed lists hold 16 × k × oversample rows (small lists after auto-compaction).
|
|
14
|
+
*/
|
|
12
15
|
nprobe?: number;
|
|
13
16
|
/** Candidates kept per probed list by the 1-bit filter = k * oversample. Default 8. */
|
|
14
17
|
oversample?: number;
|
|
@@ -61,6 +61,13 @@ interface TailEntry {
|
|
|
61
61
|
|
|
62
62
|
/** Ids per lookup (bound as one JSON parameter). */
|
|
63
63
|
const VALIDATE_BATCH = 2000;
|
|
64
|
+
/** Lists probed per segment when the caller passes no `nprobe`. */
|
|
65
|
+
const DEFAULT_NPROBE = 8;
|
|
66
|
+
/**
|
|
67
|
+
* Without an explicit `nprobe`, probe lists until they hold this many times
|
|
68
|
+
* `k × oversample` rows (#67).
|
|
69
|
+
*/
|
|
70
|
+
const PROBE_ROWS_FACTOR = 16;
|
|
64
71
|
/** Upserted rows per transaction. */
|
|
65
72
|
const WRITE_BATCH = 200;
|
|
66
73
|
|
|
@@ -243,14 +250,22 @@ export class VectorIndex {
|
|
|
243
250
|
|
|
244
251
|
async search(query: ArrayLike<number>, opts: SearchOptions = {}): Promise<SearchHit[]> {
|
|
245
252
|
this.#assertReady();
|
|
253
|
+
opts.signal?.throwIfAborted();
|
|
246
254
|
const k = opts.k ?? 10;
|
|
247
|
-
const nprobe = opts.nprobe ?? 8;
|
|
248
255
|
const perList = k * (opts.oversample ?? 8);
|
|
256
|
+
// An explicit nprobe probes exactly that many lists. The default probes at least
|
|
257
|
+
// DEFAULT_NPROBE lists and keeps going while they hold fewer than
|
|
258
|
+
// PROBE_ROWS_FACTOR × perList rows: after auto-compaction lists are small
|
|
259
|
+
// (~65 rows at 4200 vectors), and 8 of them miss neighbours spread over many (#67).
|
|
260
|
+
const nprobe = opts.nprobe ?? DEFAULT_NPROBE;
|
|
261
|
+
const minRows = opts.nprobe === undefined ? PROBE_ROWS_FACTOR * perList : 0;
|
|
249
262
|
const q = prepare(query, this.dims, this.metric);
|
|
250
263
|
const d = this.dims;
|
|
251
264
|
const filter = opts.filter;
|
|
265
|
+
const rerank = opts.exact ? k * 4 : k;
|
|
252
266
|
|
|
253
267
|
return this.#read(async (segments) => {
|
|
268
|
+
opts.signal?.throwIfAborted();
|
|
254
269
|
// Tail: exact, from memory.
|
|
255
270
|
const tailTop = new TopK<number>(k);
|
|
256
271
|
for (const [id, { vector }] of this.#tail) {
|
|
@@ -258,32 +273,63 @@ export class VectorIndex {
|
|
|
258
273
|
tailTop.push(distance(this.metric, q, 0, vector, 0, d), id);
|
|
259
274
|
}
|
|
260
275
|
|
|
261
|
-
//
|
|
262
|
-
type
|
|
263
|
-
const
|
|
264
|
-
// Tail ids are never live in a segment; skip them early.
|
|
265
|
-
const keep = (id: number) => !this.#tail.has(id) && (!filter || filter(id));
|
|
276
|
+
// Probe lists and load their blocks.
|
|
277
|
+
type Probed = { seg: SegmentIndex; list: number; data: ListData };
|
|
278
|
+
const probed: Probed[] = [];
|
|
266
279
|
await Promise.all(
|
|
267
280
|
segments.map(async (seg) => {
|
|
268
|
-
const lists = seg.probe(q, nprobe);
|
|
281
|
+
const lists = seg.probe(q, nprobe, minRows);
|
|
269
282
|
const datas = await this.#store.lists(seg, lists);
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
const data = datas[i] as ListData;
|
|
273
|
-
for (const cand of seg.filterList(q, c, data, perList, keep))
|
|
274
|
-
hits.push({ ...cand, seg, data });
|
|
283
|
+
lists.forEach((list, i) => {
|
|
284
|
+
probed.push({ seg, list, data: datas[i] as ListData });
|
|
275
285
|
});
|
|
276
286
|
}),
|
|
277
287
|
);
|
|
288
|
+
opts.signal?.throwIfAborted();
|
|
278
289
|
|
|
279
|
-
//
|
|
280
|
-
|
|
290
|
+
// Stage 1: keep the best `perList` rows of each list by estimated residual
|
|
291
|
+
// distance. A segment row counts only if its generation holds the live copy
|
|
292
|
+
// of its id. Rows found dead (deleted, or overwritten in another generation)
|
|
293
|
+
// are excluded and the list is filtered again with a doubled limit, so dead
|
|
294
|
+
// rows never take the slots of live ones (#33).
|
|
295
|
+
type Hit = Candidate & { seg: SegmentIndex; data: ListData };
|
|
296
|
+
const gens = new Map<number, number | undefined>(); // id -> live gen; undefined = deleted
|
|
297
|
+
const hits: Hit[] = [];
|
|
298
|
+
let pending = probed.map((p) => ({ ...p, limit: perList }));
|
|
299
|
+
while (pending.length > 0) {
|
|
300
|
+
const rounds = pending.map((p) => {
|
|
301
|
+
// Tail ids are never live in a segment; known-dead rows are skipped too.
|
|
302
|
+
const keep = (id: number) =>
|
|
303
|
+
!this.#tail.has(id) &&
|
|
304
|
+
!(gens.has(id) && gens.get(id) !== p.seg.gen) &&
|
|
305
|
+
(!filter || filter(id));
|
|
306
|
+
return { p, cands: p.seg.filterList(q, p.list, p.data, p.limit, keep) };
|
|
307
|
+
});
|
|
308
|
+
const unknown = rounds
|
|
309
|
+
.flatMap((r) => r.cands.map((c) => c.id))
|
|
310
|
+
.filter((id) => !gens.has(id));
|
|
311
|
+
if (unknown.length > 0) {
|
|
312
|
+
const found = await this.#generations(unknown);
|
|
313
|
+
for (const id of unknown) gens.set(id, found.get(id));
|
|
314
|
+
opts.signal?.throwIfAborted();
|
|
315
|
+
}
|
|
316
|
+
pending = [];
|
|
317
|
+
for (const { p, cands } of rounds) {
|
|
318
|
+
const live = cands.filter((c) => gens.get(c.id) === p.seg.gen);
|
|
319
|
+
// A list passed whole comes in row order: keep all of it, the int8 re-rank ranks it.
|
|
320
|
+
const whole = p.seg.passesWhole(p.data.ids.length, p.limit);
|
|
321
|
+
if (!whole && live.length < perList && cands.length === p.limit) {
|
|
322
|
+
pending.push({ ...p, limit: p.limit * 2 }); // some were dead and more rows remain
|
|
323
|
+
continue;
|
|
324
|
+
}
|
|
325
|
+
for (const cand of whole ? live : live.slice(0, perList))
|
|
326
|
+
hits.push({ ...cand, seg: p.seg, data: p.data });
|
|
327
|
+
}
|
|
328
|
+
}
|
|
281
329
|
|
|
282
330
|
// Stage 2: int8 re-rank.
|
|
283
|
-
const rerank = opts.exact ? k * 4 : k;
|
|
284
331
|
const segTop = new TopK<Hit>(rerank);
|
|
285
332
|
for (const h of hits) {
|
|
286
|
-
if (live.get(h.id) !== h.seg.gen) continue;
|
|
287
333
|
segTop.push(h.seg.scoreInt8(q, h.data, h.row), h);
|
|
288
334
|
}
|
|
289
335
|
let segResults = segTop
|
package/src/ngram/do-smoke.ts
DELETED
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
import type { Db } from "@statewalker/db-api";
|
|
2
|
-
import { NgramStore } from "./ngram-store.js";
|
|
3
|
-
|
|
4
|
-
const check = (ok: boolean, message: string) => {
|
|
5
|
-
if (!ok) throw new Error(`ngram smoke: ${message}`);
|
|
6
|
-
};
|
|
7
|
-
|
|
8
|
-
/** True when the index itself still offers candidates (candidateBudget 0 skips the verifier). */
|
|
9
|
-
const indexed = async (index: NgramStore, pattern: string) =>
|
|
10
|
-
(await index.search(pattern, { candidateBudget: 0 })).truncated;
|
|
11
|
-
|
|
12
|
-
const ids = (result: { hits: { id: number }[] }) => result.hits.map((h) => h.id).join();
|
|
13
|
-
|
|
14
|
-
/** Run inside a real Durable Object by db-sqlite-do's `pnpm test:do`. */
|
|
15
|
-
export const ngramDoScenarios = (["fts5", "postings"] as const).map((store) => ({
|
|
16
|
-
name: `ngram (${store}): upsert, overwrite, lone surrogate, drop inside a transaction`,
|
|
17
|
-
async run(db: Db) {
|
|
18
|
-
const prefix = `smoke_${store}_`;
|
|
19
|
-
const index = new NgramStore({ db, prefix, store });
|
|
20
|
-
await index.init(); // CREATE of the index tables (a contentless trigram table for fts5) in a transaction
|
|
21
|
-
await index.upsert([
|
|
22
|
-
{ id: 1, text: "Hello World" },
|
|
23
|
-
{ id: 2, text: "hello there" },
|
|
24
|
-
{ id: 3, text: "lone \ud800 surrogate" },
|
|
25
|
-
]);
|
|
26
|
-
check(
|
|
27
|
-
ids(await index.search("hello", { caseSensitive: false })) === "1,2",
|
|
28
|
-
"case-insensitive hits",
|
|
29
|
-
);
|
|
30
|
-
check(ids(await index.search("Hello")) === "1", "case-sensitive hits");
|
|
31
|
-
check(ids(await index.search("surrogate")) === "3", "text with a lone surrogate is indexed");
|
|
32
|
-
check((await index.getTexts([3])).get(3) === "lone \ufffd surrogate", "text made well-formed");
|
|
33
|
-
check(await indexed(index, "World"), "index lacks the grams of 'World'");
|
|
34
|
-
check(await indexed(index, "surrogate"), "index lacks the grams of 'surrogate'");
|
|
35
|
-
await index.upsert([{ id: 1, text: "goodbye" }]); // replays the old text to remove its grams
|
|
36
|
-
check(!(await indexed(index, "World")), "stale grams after overwrite");
|
|
37
|
-
await index.upsert([{ id: 3, text: "lone \ud800 changed" }]);
|
|
38
|
-
check(!(await indexed(index, "surrogate")), "stale grams after surrogate overwrite");
|
|
39
|
-
await index.drop(); // DROP of the tables inside a transaction
|
|
40
|
-
const [row] = await db.query<{ n: number }>(
|
|
41
|
-
"SELECT count(*) AS n FROM sqlite_master WHERE name LIKE ?",
|
|
42
|
-
[`${prefix}%`],
|
|
43
|
-
);
|
|
44
|
-
check(Number(row?.n) === 0, `tables left after drop: ${row?.n}`);
|
|
45
|
-
},
|
|
46
|
-
}));
|
package/src/ngram/fold.test.ts
DELETED
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { fold } from "./fold.js";
|
|
3
|
-
|
|
4
|
-
const escapeRegex = (s: string) => s.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&");
|
|
5
|
-
|
|
6
|
-
describe("fold", () => {
|
|
7
|
-
it("lower-cases ASCII and keeps length", () => {
|
|
8
|
-
expect(fold("Hello WORLD")).toBe("hello world");
|
|
9
|
-
});
|
|
10
|
-
|
|
11
|
-
it("maps every case form of sigma to one letter", () => {
|
|
12
|
-
expect(new Set([fold("Σ"), fold("σ"), fold("ς")]).size).toBe(1);
|
|
13
|
-
});
|
|
14
|
-
|
|
15
|
-
it("never separates two BMP characters that a case-insensitive regex treats as equal", () => {
|
|
16
|
-
const broken: string[] = [];
|
|
17
|
-
for (let cp = 0; cp <= 0xffff; cp++) {
|
|
18
|
-
if (cp >= 0xd800 && cp <= 0xdfff) continue;
|
|
19
|
-
const c = String.fromCharCode(cp);
|
|
20
|
-
const re = new RegExp(`^${escapeRegex(c)}$`, "i");
|
|
21
|
-
for (const d of [c.toUpperCase(), c.toLowerCase()]) {
|
|
22
|
-
if (d === c || d.length !== 1 || !re.test(d)) continue;
|
|
23
|
-
if (fold(c) !== fold(d)) broken.push(`U+${cp.toString(16)} ${c}/${d}`);
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
expect(broken).toEqual([]);
|
|
27
|
-
});
|
|
28
|
-
|
|
29
|
-
it("keeps the UTF-16 length of every code point, so offsets stay valid", () => {
|
|
30
|
-
const changed: string[] = [];
|
|
31
|
-
for (let cp = 0; cp <= 0x10ffff; cp++) {
|
|
32
|
-
if (cp >= 0xd800 && cp <= 0xdfff) continue;
|
|
33
|
-
const c = String.fromCodePoint(cp);
|
|
34
|
-
if (fold(c).length !== c.length) changed.push(`U+${cp.toString(16)}`);
|
|
35
|
-
}
|
|
36
|
-
expect(changed).toEqual([]);
|
|
37
|
-
});
|
|
38
|
-
|
|
39
|
-
it("keeps the length of a text with U+0130 and an astral character", () => {
|
|
40
|
-
const folded = fold("İ😀A");
|
|
41
|
-
expect(folded.length).toBe("İ😀A".length);
|
|
42
|
-
expect(folded.endsWith("a")).toBe(true);
|
|
43
|
-
});
|
|
44
|
-
|
|
45
|
-
it("is a homomorphism over concatenation, so substrings stay substrings", () => {
|
|
46
|
-
const a = "ΟΔΟΣ ſtraße";
|
|
47
|
-
const b = "İK";
|
|
48
|
-
expect(fold(a + b)).toBe(fold(a) + fold(b));
|
|
49
|
-
});
|
|
50
|
-
});
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { chunkJson } from "./json-chunks.js";
|
|
3
|
-
|
|
4
|
-
describe("chunkJson", () => {
|
|
5
|
-
const rows = Array.from({ length: 30 }, (_, i) => [i, `t${i}`]);
|
|
6
|
-
|
|
7
|
-
it("splits into chunks of at most maxChars that rebuild the rows in order", () => {
|
|
8
|
-
const chunks = chunkJson(rows, 20);
|
|
9
|
-
expect(chunks.length).toBeGreaterThan(1);
|
|
10
|
-
for (const chunk of chunks) expect(chunk.length).toBeLessThanOrEqual(20);
|
|
11
|
-
expect(chunks.flatMap((chunk) => JSON.parse(chunk))).toEqual(rows);
|
|
12
|
-
});
|
|
13
|
-
|
|
14
|
-
it("keeps a row larger than maxChars as one chunk of its own", () => {
|
|
15
|
-
const big = [1, "x".repeat(50)];
|
|
16
|
-
const chunks = chunkJson([[0, "a"], big, [2, "b"]], 20);
|
|
17
|
-
expect(chunks).toEqual(['[[0,"a"]]', JSON.stringify([big]), '[[2,"b"]]']);
|
|
18
|
-
});
|
|
19
|
-
|
|
20
|
-
it("returns no chunks for no rows", () => {
|
|
21
|
-
expect(chunkJson([], 20)).toEqual([]);
|
|
22
|
-
});
|
|
23
|
-
});
|