@statewalker/db-sqlite-index 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +289 -2
- package/dist/ngram.d.mts +93 -0
- package/dist/ngram.d.mts.map +1 -0
- package/dist/ngram.mjs +615 -0
- package/dist/ngram.mjs.map +1 -0
- package/dist/vector.d.mts +103 -0
- package/dist/vector.d.mts.map +1 -0
- package/dist/vector.mjs +1061 -0
- package/dist/vector.mjs.map +1 -0
- package/package.json +63 -3
- package/src/ngram/do-smoke.ts +46 -0
- package/src/ngram/fold.test.ts +50 -0
- package/src/ngram/fold.ts +21 -0
- package/src/ngram/fts5-backend.ts +55 -0
- package/src/ngram/index.ts +12 -0
- package/src/ngram/json-chunks.test.ts +23 -0
- package/src/ngram/json-chunks.ts +23 -0
- package/src/ngram/ngram-store.test.ts +291 -0
- package/src/ngram/ngram-store.ts +183 -0
- package/src/ngram/plan-query.test.ts +141 -0
- package/src/ngram/plan-query.ts +42 -0
- package/src/ngram/postings-backend.ts +53 -0
- package/src/ngram/regex-literals.ts +359 -0
- package/src/ngram/trigrams.test.ts +28 -0
- package/src/ngram/trigrams.ts +25 -0
- package/src/vector/auto-compact.test.ts +28 -0
- package/src/vector/do-smoke.ts +39 -0
- package/src/vector/index.ts +9 -0
- package/src/vector/kmeans.ts +131 -0
- package/src/vector/math.ts +185 -0
- package/src/vector/segment.test.ts +61 -0
- package/src/vector/segment.ts +256 -0
- package/src/vector/sql-segment-store.ts +218 -0
- package/src/vector/test-fixtures.ts +64 -0
- package/src/vector/types.ts +30 -0
- package/src/vector/vector-index.test.ts +105 -0
- package/src/vector/vector-index.ts +586 -0
- package/src/vector/vector-lifecycle.test.ts +171 -0
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import type { Db } from "@statewalker/db-api";
|
|
2
|
+
import { type BuiltSegment, type ListData, parseList, SegmentIndex } from "./segment.js";
|
|
3
|
+
import { float32Bytes, type Metric, toFloat32 } from "./types.js";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Largest BLOB written in one row. Durable Object SQLite rejects values over
|
|
7
|
+
* ~2 MB ("string or blob too big"); 1 MiB leaves headroom.
|
|
8
|
+
*/
|
|
9
|
+
export const PART_BYTES = 1024 * 1024;
|
|
10
|
+
|
|
11
|
+
export interface SegmentRef {
|
|
12
|
+
gen: number;
|
|
13
|
+
count: number;
|
|
14
|
+
nlist: number;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Segment rows: one centroid row per list, and each list's searchable and
|
|
19
|
+
* float32 blocks split into parts. Reads go through an LRU cache: a
|
|
20
|
+
* generation is immutable, so cached blocks never go stale.
|
|
21
|
+
*/
|
|
22
|
+
export class SqlSegmentStore {
|
|
23
|
+
readonly #db: Db;
|
|
24
|
+
readonly #dims: number;
|
|
25
|
+
readonly #t: { centroids: string; seg: string; f32: string };
|
|
26
|
+
readonly #cache: Lru<ListData | Float32Array>;
|
|
27
|
+
|
|
28
|
+
constructor(db: Db, prefix: string, dims: number, cacheBytes = 64 * 1024 * 1024) {
|
|
29
|
+
this.#db = db;
|
|
30
|
+
this.#dims = dims;
|
|
31
|
+
this.#t = { centroids: `${prefix}centroids`, seg: `${prefix}seg`, f32: `${prefix}f32` };
|
|
32
|
+
this.#cache = new Lru(cacheBytes);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
schema(): string[] {
|
|
36
|
+
const { centroids, seg, f32 } = this.#t;
|
|
37
|
+
return [
|
|
38
|
+
`CREATE TABLE IF NOT EXISTS ${centroids}(gen INTEGER, list INTEGER, count INTEGER NOT NULL,
|
|
39
|
+
centroid BLOB NOT NULL, PRIMARY KEY (gen, list)) WITHOUT ROWID`,
|
|
40
|
+
`CREATE TABLE IF NOT EXISTS ${seg}(gen INTEGER, list INTEGER, part INTEGER,
|
|
41
|
+
bytes BLOB NOT NULL, PRIMARY KEY (gen, list, part)) WITHOUT ROWID`,
|
|
42
|
+
`CREATE TABLE IF NOT EXISTS ${f32}(gen INTEGER, list INTEGER, part INTEGER,
|
|
43
|
+
bytes BLOB NOT NULL, PRIMARY KEY (gen, list, part)) WITHOUT ROWID`,
|
|
44
|
+
];
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Writes every row of a built segment. Call inside `inTransaction`. */
|
|
48
|
+
async write(tx: Db, gen: number, built: BuiltSegment): Promise<void> {
|
|
49
|
+
const d = this.#dims;
|
|
50
|
+
for (let c = 0; c < built.nlist; c++) {
|
|
51
|
+
await tx.query(
|
|
52
|
+
`INSERT INTO ${this.#t.centroids}(gen, list, count, centroid) VALUES (?, ?, ?, ?)`,
|
|
53
|
+
[gen, c, built.counts[c], float32Bytes(built.centroids.subarray(c * d, (c + 1) * d))],
|
|
54
|
+
);
|
|
55
|
+
await writeParts(tx, this.#t.seg, gen, c, built.lists[c] as Uint8Array);
|
|
56
|
+
await writeParts(tx, this.#t.f32, gen, c, built.floats[c] as Uint8Array);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Loads the in-memory part (centroids, list sizes) of a committed segment. */
|
|
61
|
+
async open(ref: SegmentRef, metric: Metric): Promise<SegmentIndex> {
|
|
62
|
+
const d = this.#dims;
|
|
63
|
+
const rows = await this.#db.query<{ list: number; count: number; centroid: Uint8Array }>(
|
|
64
|
+
`SELECT list, count, centroid FROM ${this.#t.centroids} WHERE gen = ? ORDER BY list`,
|
|
65
|
+
[ref.gen],
|
|
66
|
+
);
|
|
67
|
+
if (rows.length !== ref.nlist) {
|
|
68
|
+
throw new Error(
|
|
69
|
+
`vector-index: segment ${ref.gen} has ${rows.length} lists, expected ${ref.nlist}`,
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
const centroids = new Float32Array(ref.nlist * d);
|
|
73
|
+
const counts = new Uint32Array(ref.nlist);
|
|
74
|
+
for (const r of rows) {
|
|
75
|
+
const c = Number(r.list);
|
|
76
|
+
centroids.set(toFloat32(r.centroid), c * d);
|
|
77
|
+
counts[c] = Number(r.count);
|
|
78
|
+
}
|
|
79
|
+
return new SegmentIndex(ref.gen, d, metric, centroids, counts);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Searchable blocks of lists `cs`: cached ones from memory, the rest in one query. */
|
|
83
|
+
async lists(seg: SegmentIndex, cs: number[]): Promise<ListData[]> {
|
|
84
|
+
const out = new Map<number, ListData>();
|
|
85
|
+
const missing: number[] = [];
|
|
86
|
+
for (const c of cs) {
|
|
87
|
+
const hit = this.#cache.get(`s:${seg.gen}:${c}`) as ListData | undefined;
|
|
88
|
+
if (hit) out.set(c, hit);
|
|
89
|
+
else missing.push(c);
|
|
90
|
+
}
|
|
91
|
+
if (missing.length > 0) {
|
|
92
|
+
const blocks = await readParts(this.#db, this.#t.seg, seg.gen, missing);
|
|
93
|
+
for (const c of missing) {
|
|
94
|
+
const bytes = blocks.get(c) ?? new Uint8Array(0);
|
|
95
|
+
const data = parseList(bytes, seg.counts[c] as number, this.#dims);
|
|
96
|
+
this.#cache.set(`s:${seg.gen}:${c}`, data, bytes.byteLength);
|
|
97
|
+
out.set(c, data);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return cs.map((c) => out.get(c) as ListData);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Every float32 vector of list `c` (count × dims). */
|
|
104
|
+
async floats(seg: SegmentIndex, c: number): Promise<Float32Array> {
|
|
105
|
+
const key = `f:${seg.gen}:${c}`;
|
|
106
|
+
const hit = this.#cache.get(key) as Float32Array | undefined;
|
|
107
|
+
if (hit) return hit;
|
|
108
|
+
const blocks = await readParts(this.#db, this.#t.f32, seg.gen, [c]);
|
|
109
|
+
const floats = toFloat32(blocks.get(c) ?? new Uint8Array(0));
|
|
110
|
+
this.#cache.set(key, floats, floats.byteLength);
|
|
111
|
+
return floats;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Deletes the rows of `gens`. Call inside `inTransaction`. */
|
|
115
|
+
async drop(tx: Db, gens: number[]): Promise<void> {
|
|
116
|
+
const json = JSON.stringify(gens);
|
|
117
|
+
for (const table of Object.values(this.#t)) {
|
|
118
|
+
await tx.query(`DELETE FROM ${table} WHERE gen IN (SELECT value FROM json_each(?))`, [json]);
|
|
119
|
+
}
|
|
120
|
+
for (const gen of gens) this.#cache.deletePrefix(`s:${gen}:`, `f:${gen}:`);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Deletes rows of every generation not in `live` (left by a crash before a reap). */
|
|
124
|
+
async sweep(tx: Db, live: number[]): Promise<void> {
|
|
125
|
+
const json = JSON.stringify(live);
|
|
126
|
+
for (const table of Object.values(this.#t)) {
|
|
127
|
+
await tx.query(`DELETE FROM ${table} WHERE gen NOT IN (SELECT value FROM json_each(?))`, [
|
|
128
|
+
json,
|
|
129
|
+
]);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
async function writeParts(tx: Db, table: string, gen: number, list: number, bytes: Uint8Array) {
|
|
135
|
+
for (let part = 0, at = 0; at < bytes.byteLength; part++, at += PART_BYTES) {
|
|
136
|
+
await tx.query(`INSERT INTO ${table}(gen, list, part, bytes) VALUES (?, ?, ?, ?)`, [
|
|
137
|
+
gen,
|
|
138
|
+
list,
|
|
139
|
+
part,
|
|
140
|
+
bytes.slice(at, at + PART_BYTES),
|
|
141
|
+
]);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** The joined parts of each list in `lists`, as fresh arrays starting at offset 0. */
|
|
146
|
+
async function readParts(
|
|
147
|
+
db: Db,
|
|
148
|
+
table: string,
|
|
149
|
+
gen: number,
|
|
150
|
+
lists: number[],
|
|
151
|
+
): Promise<Map<number, Uint8Array>> {
|
|
152
|
+
const rows = await db.query<{ list: number; bytes: Uint8Array }>(
|
|
153
|
+
`SELECT list, bytes FROM ${table}
|
|
154
|
+
WHERE gen = ? AND list IN (SELECT value FROM json_each(?)) ORDER BY list, part`,
|
|
155
|
+
[gen, JSON.stringify(lists)],
|
|
156
|
+
);
|
|
157
|
+
const parts = new Map<number, Uint8Array[]>();
|
|
158
|
+
for (const r of rows) {
|
|
159
|
+
const list = Number(r.list);
|
|
160
|
+
const acc = parts.get(list) ?? [];
|
|
161
|
+
acc.push(r.bytes);
|
|
162
|
+
parts.set(list, acc);
|
|
163
|
+
}
|
|
164
|
+
const out = new Map<number, Uint8Array>();
|
|
165
|
+
for (const [list, ps] of parts) {
|
|
166
|
+
const joined = new Uint8Array(ps.reduce((n, p) => n + p.byteLength, 0));
|
|
167
|
+
let at = 0;
|
|
168
|
+
for (const p of ps) {
|
|
169
|
+
joined.set(p, at);
|
|
170
|
+
at += p.byteLength;
|
|
171
|
+
}
|
|
172
|
+
out.set(list, joined);
|
|
173
|
+
}
|
|
174
|
+
return out;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** Byte-budgeted LRU; Map order is LRU order (re-inserted on every hit). */
|
|
178
|
+
class Lru<V> {
|
|
179
|
+
readonly #max: number;
|
|
180
|
+
readonly #map = new Map<string, { value: V; size: number }>();
|
|
181
|
+
#bytes = 0;
|
|
182
|
+
|
|
183
|
+
constructor(maxBytes: number) {
|
|
184
|
+
this.#max = maxBytes;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
get(key: string): V | undefined {
|
|
188
|
+
const e = this.#map.get(key);
|
|
189
|
+
if (!e) return undefined;
|
|
190
|
+
this.#map.delete(key);
|
|
191
|
+
this.#map.set(key, e);
|
|
192
|
+
return e.value;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
set(key: string, value: V, size: number) {
|
|
196
|
+
const old = this.#map.get(key);
|
|
197
|
+
if (old) {
|
|
198
|
+
this.#map.delete(key);
|
|
199
|
+
this.#bytes -= old.size;
|
|
200
|
+
}
|
|
201
|
+
this.#map.set(key, { value, size });
|
|
202
|
+
this.#bytes += size;
|
|
203
|
+
for (const [k, e] of this.#map) {
|
|
204
|
+
if (this.#bytes <= this.#max) break;
|
|
205
|
+
this.#map.delete(k);
|
|
206
|
+
this.#bytes -= e.size;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
deletePrefix(...prefixes: string[]) {
|
|
211
|
+
for (const [k, e] of this.#map) {
|
|
212
|
+
if (prefixes.some((p) => k.startsWith(p))) {
|
|
213
|
+
this.#map.delete(k);
|
|
214
|
+
this.#bytes -= e.size;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { Db } from "@statewalker/db-api";
|
|
2
|
+
import { newDoSqliteDb } from "@statewalker/db-sqlite-do";
|
|
3
|
+
import { newFakeDoStorage } from "@statewalker/db-sqlite-do/fake";
|
|
4
|
+
import { newNodeSqliteDb } from "@statewalker/db-sqlite-node";
|
|
5
|
+
import { mulberry32 } from "./kmeans.js";
|
|
6
|
+
import { distance, prepare } from "./math.js";
|
|
7
|
+
import type { Metric, SearchOptions } from "./types.js";
|
|
8
|
+
import type { VectorIndex } from "./vector-index.js";
|
|
9
|
+
|
|
10
|
+
/** Every suite runs over each of these. */
|
|
11
|
+
export const BACKENDS: Record<string, () => Promise<Db>> = {
|
|
12
|
+
"node:sqlite": () => newNodeSqliteDb(),
|
|
13
|
+
"durable object (fake storage)": async () => newDoSqliteDb(newFakeDoStorage()),
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
export const D = 64;
|
|
17
|
+
|
|
18
|
+
export interface Item {
|
|
19
|
+
id: number;
|
|
20
|
+
vector: Float32Array;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Clustered data: realistic for IVF (uniform noise makes every ANN look bad). */
|
|
24
|
+
export function dataset(n: number, seed: number, dims = D, clusters = 40): Item[] {
|
|
25
|
+
const r = mulberry32(seed);
|
|
26
|
+
const centers = Array.from({ length: clusters }, () =>
|
|
27
|
+
Float32Array.from({ length: dims }, () => r() * 2 - 1),
|
|
28
|
+
);
|
|
29
|
+
return Array.from({ length: n }, (_, i) => {
|
|
30
|
+
const c = centers[Math.floor(r() * clusters)] as Float32Array;
|
|
31
|
+
return { id: i + 1, vector: Float32Array.from(c, (x) => x + (r() - 0.5) * 0.6) };
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function bruteForce(items: Item[], q: Float32Array, k: number, metric: Metric, dims = D) {
|
|
36
|
+
const qq = prepare(q, dims, metric);
|
|
37
|
+
return items
|
|
38
|
+
.map((it) => ({
|
|
39
|
+
id: it.id,
|
|
40
|
+
d: distance(metric, qq, 0, prepare(it.vector, dims, metric), 0, dims),
|
|
41
|
+
}))
|
|
42
|
+
.sort((a, b) => a.d - b.d)
|
|
43
|
+
.slice(0, k)
|
|
44
|
+
.map((x) => x.id);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export async function recall(
|
|
48
|
+
index: VectorIndex,
|
|
49
|
+
items: Item[],
|
|
50
|
+
metric: Metric,
|
|
51
|
+
opts: SearchOptions = {},
|
|
52
|
+
): Promise<number> {
|
|
53
|
+
const r = mulberry32(7);
|
|
54
|
+
let hit = 0;
|
|
55
|
+
const Q = 30;
|
|
56
|
+
for (let i = 0; i < Q; i++) {
|
|
57
|
+
const base = items[Math.floor(r() * items.length)] as Item;
|
|
58
|
+
const q = base.vector.map((x) => x + (r() - 0.5) * 0.2);
|
|
59
|
+
const truth = new Set(bruteForce(items, q, 10, metric));
|
|
60
|
+
const got = await index.search(q, { k: 10, ...opts });
|
|
61
|
+
hit += got.filter((h) => truth.has(h.id)).length;
|
|
62
|
+
}
|
|
63
|
+
return hit / (Q * 10);
|
|
64
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export type Metric = "cosine" | "dot" | "l2";
|
|
2
|
+
|
|
3
|
+
export interface SearchHit {
|
|
4
|
+
id: number;
|
|
5
|
+
/** Lower is closer: 1 - cos for cosine, -dot for dot, squared L2 for l2. */
|
|
6
|
+
distance: number;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface SearchOptions {
|
|
10
|
+
k?: number;
|
|
11
|
+
/** IVF lists probed per segment. Default 8. */
|
|
12
|
+
nprobe?: number;
|
|
13
|
+
/** Candidates kept per probed list by the 1-bit filter = k * oversample. Default 8. */
|
|
14
|
+
oversample?: number;
|
|
15
|
+
/** Re-rank the int8 winners with the stored float32 vectors. Default false. */
|
|
16
|
+
exact?: boolean;
|
|
17
|
+
/** Keep only ids for which this returns true (applied before re-ranking). */
|
|
18
|
+
filter?: (id: number) => boolean;
|
|
19
|
+
signal?: AbortSignal;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** An aligned Float32Array copy of BLOB bytes (views over arbitrary offsets are unsafe). */
|
|
23
|
+
export function toFloat32(bytes: Uint8Array): Float32Array {
|
|
24
|
+
const copy = bytes.slice();
|
|
25
|
+
return new Float32Array(copy.buffer, 0, copy.byteLength >> 2);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function float32Bytes(v: Float32Array): Uint8Array {
|
|
29
|
+
return new Uint8Array(v.buffer, v.byteOffset, v.byteLength).slice();
|
|
30
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { BACKENDS, D, dataset, recall } from "./test-fixtures.js";
|
|
3
|
+
import type { Metric } from "./types.js";
|
|
4
|
+
import { VectorIndex } from "./vector-index.js";
|
|
5
|
+
|
|
6
|
+
for (const [name, open] of Object.entries(BACKENDS)) {
|
|
7
|
+
describe(`VectorIndex over ${name}`, { timeout: 60_000 }, () => {
|
|
8
|
+
async function setup(metric: Metric = "cosine") {
|
|
9
|
+
const db = await open();
|
|
10
|
+
const index = new VectorIndex({
|
|
11
|
+
db,
|
|
12
|
+
prefix: "vec_",
|
|
13
|
+
dims: D,
|
|
14
|
+
metric,
|
|
15
|
+
autoCompactAt: Infinity,
|
|
16
|
+
});
|
|
17
|
+
await index.init();
|
|
18
|
+
return { db, index };
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
it("tail search is exact", async () => {
|
|
22
|
+
const { index } = await setup();
|
|
23
|
+
const items = dataset(300, 1);
|
|
24
|
+
await index.upsert(items);
|
|
25
|
+
expect(await recall(index, items, "cosine")).toBe(1);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
for (const metric of ["cosine", "dot", "l2"] as const) {
|
|
29
|
+
it(`compacted IVF segment has high recall (${metric})`, async () => {
|
|
30
|
+
const { index } = await setup(metric);
|
|
31
|
+
const items = dataset(5000, 2);
|
|
32
|
+
await index.upsert(items);
|
|
33
|
+
expect(await index.compact()).toBe(5000);
|
|
34
|
+
expect(index.stats.tail).toBe(0);
|
|
35
|
+
const approx = await recall(index, items, metric, { nprobe: 12 });
|
|
36
|
+
const exact = await recall(index, items, metric, { nprobe: 12, exact: true });
|
|
37
|
+
console.log(
|
|
38
|
+
` ${name} ${metric}: recall@10 int8=${approx.toFixed(3)} exact=${exact.toFixed(3)}`,
|
|
39
|
+
);
|
|
40
|
+
expect(approx).toBeGreaterThanOrEqual(0.85);
|
|
41
|
+
expect(exact).toBeGreaterThanOrEqual(approx - 0.01);
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
it("deletes and overwrites hide segment copies; merge drops them", async () => {
|
|
46
|
+
const { index, db } = await setup();
|
|
47
|
+
const items = dataset(2000, 3);
|
|
48
|
+
await index.upsert(items);
|
|
49
|
+
await index.compact();
|
|
50
|
+
|
|
51
|
+
const target = items[10];
|
|
52
|
+
expect((await index.search(target.vector, { k: 1 }))[0]?.id).toBe(target.id);
|
|
53
|
+
|
|
54
|
+
await index.delete([target.id]);
|
|
55
|
+
const after = await index.search(target.vector, { k: 5 });
|
|
56
|
+
expect(after.some((h) => h.id === target.id)).toBe(false);
|
|
57
|
+
|
|
58
|
+
// Overwrite id 20 with the vector of id 30: a search for 30's vector finds both.
|
|
59
|
+
await index.upsert([{ id: 20, vector: items[29].vector }]);
|
|
60
|
+
const both = (await index.search(items[29].vector, { k: 2 }))
|
|
61
|
+
.map((h) => h.id)
|
|
62
|
+
.sort((a, b) => a - b);
|
|
63
|
+
expect(both).toEqual([20, 30]);
|
|
64
|
+
// The old copy of id 20 (still in the segment) must not show up for its old vector.
|
|
65
|
+
const old = await index.search(items[19].vector, { k: 3 });
|
|
66
|
+
expect(old.every((h) => h.id !== 20 || h.distance > 1e-4)).toBe(true);
|
|
67
|
+
|
|
68
|
+
await index.compact(); // a second segment holds the new id 20
|
|
69
|
+
expect(index.stats.segments.length).toBe(2);
|
|
70
|
+
expect(await index.merge()).toBe(1999);
|
|
71
|
+
expect(index.stats.segments.length).toBe(1);
|
|
72
|
+
const both2 = (await index.search(items[29].vector, { k: 2 }))
|
|
73
|
+
.map((h) => h.id)
|
|
74
|
+
.sort((a, b) => a - b);
|
|
75
|
+
expect(both2).toEqual([20, 30]);
|
|
76
|
+
const [row] = await db.query<{ n: number }>("SELECT count(*) AS n FROM vec_ids");
|
|
77
|
+
expect(Number(row?.n)).toBe(1999);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it("state survives a reopen", async () => {
|
|
81
|
+
const { index, db } = await setup();
|
|
82
|
+
const items = dataset(1500, 4);
|
|
83
|
+
await index.upsert(items.slice(0, 1200));
|
|
84
|
+
await index.compact();
|
|
85
|
+
await index.upsert(items.slice(1200));
|
|
86
|
+
const reopened = new VectorIndex({ db, prefix: "vec_", dims: D });
|
|
87
|
+
await reopened.init();
|
|
88
|
+
expect(reopened.stats.tail).toBe(300);
|
|
89
|
+
expect(reopened.stats.segments.length).toBe(1);
|
|
90
|
+
expect(await recall(reopened, items, "cosine", { nprobe: 16 })).toBeGreaterThanOrEqual(0.85);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
it("filters and rejects a mismatched configuration", async () => {
|
|
94
|
+
const { index, db } = await setup();
|
|
95
|
+
const items = dataset(1200, 5);
|
|
96
|
+
await index.upsert(items);
|
|
97
|
+
await index.compact();
|
|
98
|
+
const even = await index.search(items[0].vector, { k: 10, filter: (id) => id % 2 === 0 });
|
|
99
|
+
expect(even.length).toBe(10);
|
|
100
|
+
expect(even.every((h) => h.id % 2 === 0)).toBe(true);
|
|
101
|
+
const wrong = new VectorIndex({ db, prefix: "vec_", dims: D + 1 });
|
|
102
|
+
await expect(wrong.init()).rejects.toThrow(/created with/);
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
}
|