@statewalker/db-sqlite-index 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -4
- package/dist/ngram.d.mts.map +1 -1
- package/dist/ngram.mjs +85 -4
- package/dist/ngram.mjs.map +1 -1
- package/dist/prefix-CTeW1EwH.mjs +7 -0
- package/dist/prefix-CTeW1EwH.mjs.map +1 -0
- package/dist/vector.d.mts +4 -1
- package/dist/vector.d.mts.map +1 -1
- package/dist/vector.mjs +118 -24
- package/dist/vector.mjs.map +1 -1
- package/package.json +17 -9
- package/src/ngram/fold.ts +44 -0
- package/src/ngram/fts5-backend.ts +4 -0
- package/src/ngram/ngram-store.ts +40 -2
- package/src/ngram/plan-query.ts +8 -1
- package/src/ngram/postings-backend.ts +1 -0
- package/src/ngram/regex-literals.ts +5 -2
- package/src/prefix.ts +2 -0
- package/src/vector/math.ts +6 -8
- package/src/vector/segment.ts +48 -6
- package/src/vector/types.ts +4 -1
- package/src/vector/vector-index.ts +65 -19
- package/src/ngram/do-smoke.ts +0 -46
- package/src/ngram/fold.test.ts +0 -50
- package/src/ngram/json-chunks.test.ts +0 -23
- package/src/ngram/ngram-store.test.ts +0 -291
- package/src/ngram/plan-query.test.ts +0 -141
- package/src/ngram/trigrams.test.ts +0 -28
- package/src/vector/auto-compact.test.ts +0 -28
- package/src/vector/do-smoke.ts +0 -39
- package/src/vector/segment.test.ts +0 -61
- package/src/vector/test-fixtures.ts +0 -64
- package/src/vector/vector-index.test.ts +0 -105
- package/src/vector/vector-lifecycle.test.ts +0 -171
|
@@ -1,291 +0,0 @@
|
|
|
1
|
-
import type { Db } from "@statewalker/db-api";
|
|
2
|
-
import { newDoSqliteDb } from "@statewalker/db-sqlite-do";
|
|
3
|
-
import { newFakeDoStorage } from "@statewalker/db-sqlite-do/fake";
|
|
4
|
-
import { newNodeSqliteDb } from "@statewalker/db-sqlite-node";
|
|
5
|
-
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
6
|
-
import { type NgramItem, type NgramSearchOptions, NgramStore } from "./index.js";
|
|
7
|
-
|
|
8
|
-
const DRIVERS: Array<[string, () => Promise<Db>]> = [
|
|
9
|
-
["node:sqlite", () => newNodeSqliteDb()],
|
|
10
|
-
["do (fake storage)", async () => newDoSqliteDb(newFakeDoStorage())],
|
|
11
|
-
];
|
|
12
|
-
const STORES = ["fts5", "postings"] as const;
|
|
13
|
-
|
|
14
|
-
const UNLIMITED: NgramSearchOptions = { limit: 1e9, candidateBudget: 1e9 };
|
|
15
|
-
|
|
16
|
-
/** The oracle: the same pattern over every text, no index. */
|
|
17
|
-
function scan(items: NgramItem[], pattern: string, opts: NgramSearchOptions = {}) {
|
|
18
|
-
const source = opts.regex ? pattern : pattern.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&");
|
|
19
|
-
const re = new RegExp(source, opts.caseSensitive === false ? "gi" : "g");
|
|
20
|
-
return items
|
|
21
|
-
.map(({ id, text }) => ({
|
|
22
|
-
id,
|
|
23
|
-
matches: [...text.matchAll(re)]
|
|
24
|
-
.filter((m) => m[0].length > 0)
|
|
25
|
-
.map((m) => ({ start: m.index, end: m.index + m[0].length })),
|
|
26
|
-
}))
|
|
27
|
-
.filter((hit) => hit.matches.length > 0)
|
|
28
|
-
.sort((a, b) => a.id - b.id);
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
const items = (texts: string[]): NgramItem[] => texts.map((text, i) => ({ id: i + 1, text }));
|
|
32
|
-
|
|
33
|
-
/** Same rows as pg_trgm's contrib/pg_trgm/data/trgm.data. */
|
|
34
|
-
const TRGM = items(
|
|
35
|
-
Array.from({ length: 1000 }, (_, i) => `qwertyu${String(i + 1).padStart(4, "0")}`),
|
|
36
|
-
);
|
|
37
|
-
/** Same rows as pg_trgm's test2 table. */
|
|
38
|
-
const TEST2 = items([
|
|
39
|
-
"abcdef",
|
|
40
|
-
"quark",
|
|
41
|
-
" z foo bar",
|
|
42
|
-
"/123/-45/",
|
|
43
|
-
"line 1",
|
|
44
|
-
"%line 2",
|
|
45
|
-
"line 3%",
|
|
46
|
-
"%line 4%",
|
|
47
|
-
"%li%ne 5%",
|
|
48
|
-
"li_e 6",
|
|
49
|
-
]);
|
|
50
|
-
const UNICODE = items([
|
|
51
|
-
"Straße und STRASSE",
|
|
52
|
-
"ſtraße",
|
|
53
|
-
"ΟΔΟΣ οδός ΟΔΟΣ",
|
|
54
|
-
"😀abc😀 x😀y",
|
|
55
|
-
"東京タワー と 東京駅",
|
|
56
|
-
"İstanbul istanbul",
|
|
57
|
-
"Kelvin K and k",
|
|
58
|
-
"ẞ ß ss",
|
|
59
|
-
]);
|
|
60
|
-
|
|
61
|
-
type Query = [string, NgramItem[], string, NgramSearchOptions];
|
|
62
|
-
const QUERIES: Query[] = [
|
|
63
|
-
["trgm", TRGM, "[qwerty]{2}-?[qwerty]{2}", { regex: true }],
|
|
64
|
-
["trgm", TRGM, "99", {}],
|
|
65
|
-
["trgm", TRGM, "qwerty", {}],
|
|
66
|
-
["trgm", TRGM, "qw", {}],
|
|
67
|
-
["trgm", TRGM, "u0988", {}],
|
|
68
|
-
["trgm", TRGM, "0(1|2)3", { regex: true }],
|
|
69
|
-
["trgm", TRGM, "ty.*99", { regex: true }],
|
|
70
|
-
["test2", TEST2, "BCD", {}],
|
|
71
|
-
["test2", TEST2, "bcd", {}],
|
|
72
|
-
["test2", TEST2, "BCD", { caseSensitive: false }],
|
|
73
|
-
["test2", TEST2, "\\bcd", {}],
|
|
74
|
-
["test2", TEST2, "z foo bar", {}],
|
|
75
|
-
["test2", TEST2, " z foo", {}],
|
|
76
|
-
["test2", TEST2, "%", {}],
|
|
77
|
-
["test2", TEST2, "li_e", {}],
|
|
78
|
-
["test2", TEST2, "%li%", {}],
|
|
79
|
-
["test2", TEST2, "[abc]{3}", { regex: true }],
|
|
80
|
-
["test2", TEST2, "a[bc]+d", { regex: true }],
|
|
81
|
-
["test2", TEST2, "(abc)*$", { regex: true }],
|
|
82
|
-
["test2", TEST2, "DEF", { regex: true, caseSensitive: false }],
|
|
83
|
-
["test2", TEST2, "dEf", { regex: true }],
|
|
84
|
-
["test2", TEST2, "^q", { regex: true, caseSensitive: false }],
|
|
85
|
-
["test2", TEST2, "[abc]{3}[def]{3}", { regex: true, caseSensitive: false }],
|
|
86
|
-
["test2", TEST2, "ab[a-z]{3}", { regex: true, caseSensitive: false }],
|
|
87
|
-
["test2", TEST2, "(^| )qua", { regex: true, caseSensitive: false }],
|
|
88
|
-
["test2", TEST2, "q.*rk$", { regex: true }],
|
|
89
|
-
["test2", TEST2, "q", { regex: true }],
|
|
90
|
-
["test2", TEST2, "[a-z]{3}", { regex: true }],
|
|
91
|
-
["test2", TEST2, "(a{10}|b{10}|c{10}){10}", { regex: true, caseSensitive: false }],
|
|
92
|
-
["test2", TEST2, " z foo bar", { regex: true }],
|
|
93
|
-
["test2", TEST2, "qua(?!foo)", { regex: true }],
|
|
94
|
-
["test2", TEST2, "/\\d+/-\\d", { regex: true }],
|
|
95
|
-
["short", TEST2, "", {}],
|
|
96
|
-
["short", TEST2, "a", {}],
|
|
97
|
-
["short", TEST2, "ab", {}],
|
|
98
|
-
["short", TEST2, "^", { regex: true }],
|
|
99
|
-
["unicode", UNICODE, "straße", {}],
|
|
100
|
-
["unicode", UNICODE, "STRASSE", { caseSensitive: false }],
|
|
101
|
-
["unicode", UNICODE, "ß", {}],
|
|
102
|
-
["unicode", UNICODE, "ẞ", { caseSensitive: false }],
|
|
103
|
-
["unicode", UNICODE, "ΟΔΟΣ", { caseSensitive: false }],
|
|
104
|
-
["unicode", UNICODE, "οδοσ", { caseSensitive: false }],
|
|
105
|
-
["unicode", UNICODE, "😀", {}],
|
|
106
|
-
["unicode", UNICODE, "😀a", {}],
|
|
107
|
-
["unicode", UNICODE, "abc😀", {}],
|
|
108
|
-
["unicode", UNICODE, "東京", {}],
|
|
109
|
-
["unicode", UNICODE, "東京駅", {}],
|
|
110
|
-
["unicode", UNICODE, "istanbul", { caseSensitive: false }],
|
|
111
|
-
["unicode", UNICODE, "kelvin K", { caseSensitive: false }],
|
|
112
|
-
];
|
|
113
|
-
|
|
114
|
-
for (const [driverName, newDb] of DRIVERS) {
|
|
115
|
-
for (const store of STORES) {
|
|
116
|
-
describe(`NgramStore ${store} on ${driverName}`, () => {
|
|
117
|
-
let db: Db;
|
|
118
|
-
let index: NgramStore;
|
|
119
|
-
|
|
120
|
-
beforeEach(async () => {
|
|
121
|
-
db = await newDb();
|
|
122
|
-
index = new NgramStore({ db, prefix: "t_", store });
|
|
123
|
-
await index.init();
|
|
124
|
-
});
|
|
125
|
-
/**
|
|
126
|
-
* Whether the index still yields a candidate for `literal` (3+ code points).
|
|
127
|
-
* With a zero budget, any candidate sets `truncated`; verification cannot hide
|
|
128
|
-
* stale grams here, which is how the write guards are pinned.
|
|
129
|
-
*/
|
|
130
|
-
const indexed = async (literal: string) =>
|
|
131
|
-
(await index.search(literal, { candidateBudget: 0 })).truncated;
|
|
132
|
-
afterEach(async () => {
|
|
133
|
-
await db.close();
|
|
134
|
-
});
|
|
135
|
-
|
|
136
|
-
it.each(QUERIES)("%s: %s %j equals a brute-force scan", async (_c, corpus, pattern, o) => {
|
|
137
|
-
await index.upsert(corpus);
|
|
138
|
-
const result = await index.search(pattern, { ...o, ...UNLIMITED });
|
|
139
|
-
expect(result.hits).toEqual(scan(corpus, pattern, o));
|
|
140
|
-
expect(result.truncated).toBe(false);
|
|
141
|
-
});
|
|
142
|
-
|
|
143
|
-
it("reports UTF-16 offsets in astral text", async () => {
|
|
144
|
-
await index.upsert([{ id: 7, text: "😀abc😀" }]);
|
|
145
|
-
const { hits } = await index.search("abc");
|
|
146
|
-
expect(hits).toEqual([{ id: 7, matches: [{ start: 2, end: 5 }] }]);
|
|
147
|
-
});
|
|
148
|
-
|
|
149
|
-
it("upsert replaces the text: old content stops matching", async () => {
|
|
150
|
-
await index.upsert([{ id: 1, text: "alpha beta" }]);
|
|
151
|
-
await index.upsert([{ id: 1, text: "gamma delta" }]);
|
|
152
|
-
expect((await index.search("alpha")).hits).toEqual([]);
|
|
153
|
-
expect(await indexed("alpha")).toBe(false);
|
|
154
|
-
expect((await index.search("gamma")).hits.map((h) => h.id)).toEqual([1]);
|
|
155
|
-
});
|
|
156
|
-
|
|
157
|
-
it("duplicate ids in one upsert keep the last text", async () => {
|
|
158
|
-
await index.upsert([
|
|
159
|
-
{ id: 1, text: "first version" },
|
|
160
|
-
{ id: 1, text: "second version" },
|
|
161
|
-
]);
|
|
162
|
-
expect((await index.search("first")).hits).toEqual([]);
|
|
163
|
-
expect(await indexed("first")).toBe(false);
|
|
164
|
-
expect((await index.search("version")).hits.map((h) => h.id)).toEqual([1]);
|
|
165
|
-
await index.delete([1]);
|
|
166
|
-
expect((await index.search("version")).hits).toEqual([]);
|
|
167
|
-
expect(await indexed("version")).toBe(false);
|
|
168
|
-
});
|
|
169
|
-
|
|
170
|
-
it("a lone surrogate is stored as U+FFFD, and delete leaves no grams", async () => {
|
|
171
|
-
await index.upsert([{ id: 1, text: "x\uD800y z" }]);
|
|
172
|
-
expect(await index.getTexts([1])).toEqual(new Map([[1, "x\uFFFDy z"]]));
|
|
173
|
-
expect(await indexed("x\uFFFDy")).toBe(true);
|
|
174
|
-
await index.delete([1]);
|
|
175
|
-
expect(await indexed("x\uFFFDy")).toBe(false);
|
|
176
|
-
});
|
|
177
|
-
|
|
178
|
-
it("delete removes items from index and scan", async () => {
|
|
179
|
-
await index.upsert(items(["one two", "two three"]));
|
|
180
|
-
await index.delete([1]);
|
|
181
|
-
expect((await index.search("two")).hits.map((h) => h.id)).toEqual([2]);
|
|
182
|
-
expect(await indexed("one")).toBe(false);
|
|
183
|
-
expect((await index.search("o", {})).hits.map((h) => h.id)).toEqual([2]);
|
|
184
|
-
});
|
|
185
|
-
|
|
186
|
-
it("upserts a batch larger than the 2 MB a DO accepts in one bound value", async () => {
|
|
187
|
-
const big = Array.from({ length: 6000 }, (_, i) => ({
|
|
188
|
-
id: i + 1,
|
|
189
|
-
text: `${"lorem ipsum ".repeat(33)}marker${String(i).padStart(5, "0")}`,
|
|
190
|
-
}));
|
|
191
|
-
await index.upsert(big);
|
|
192
|
-
expect((await index.search("marker05999")).hits.map((h) => h.id)).toEqual([6000]);
|
|
193
|
-
});
|
|
194
|
-
|
|
195
|
-
it("a second store on the same db and prefix sees persisted data", async () => {
|
|
196
|
-
await index.upsert(items(["persisted text"]));
|
|
197
|
-
const reopened = new NgramStore({ db, prefix: "t_", store });
|
|
198
|
-
await reopened.init();
|
|
199
|
-
expect((await reopened.search("persisted")).hits.map((h) => h.id)).toEqual([1]);
|
|
200
|
-
});
|
|
201
|
-
|
|
202
|
-
it("limit 0 returns no hits", async () => {
|
|
203
|
-
await index.upsert(items(["one two", "two three"]));
|
|
204
|
-
expect((await index.search("two", { limit: 0 })).hits).toEqual([]);
|
|
205
|
-
});
|
|
206
|
-
|
|
207
|
-
it("limit stops early and reports truncation", async () => {
|
|
208
|
-
await index.upsert(TRGM);
|
|
209
|
-
const result = await index.search("qwerty", { limit: 5 });
|
|
210
|
-
expect(result.hits.map((h) => h.id)).toEqual([1, 2, 3, 4, 5]);
|
|
211
|
-
expect(result.truncated).toBe(true);
|
|
212
|
-
});
|
|
213
|
-
|
|
214
|
-
it("candidateBudget caps verified candidates and reports truncation", async () => {
|
|
215
|
-
await index.upsert(TRGM);
|
|
216
|
-
const result = await index.search("qwerty", { candidateBudget: 10, limit: 1e9 });
|
|
217
|
-
expect(result.hits).toHaveLength(10);
|
|
218
|
-
expect(result.truncated).toBe(true);
|
|
219
|
-
});
|
|
220
|
-
|
|
221
|
-
it("filter drops candidates before verification", async () => {
|
|
222
|
-
await index.upsert(TRGM);
|
|
223
|
-
const result = await index.search("qwertyu09", { filter: (id) => id % 2 === 0 });
|
|
224
|
-
expect(result.hits.map((h) => h.id)).toEqual(
|
|
225
|
-
scan(TRGM, "qwertyu09")
|
|
226
|
-
.map((h) => h.id)
|
|
227
|
-
.filter((id) => id % 2 === 0),
|
|
228
|
-
);
|
|
229
|
-
});
|
|
230
|
-
|
|
231
|
-
it("an aborted signal rejects with its reason", async () => {
|
|
232
|
-
await index.upsert(TRGM);
|
|
233
|
-
const controller = new AbortController();
|
|
234
|
-
controller.abort(new Error("stop"));
|
|
235
|
-
await expect(index.search("qwerty", { signal: controller.signal })).rejects.toThrow("stop");
|
|
236
|
-
});
|
|
237
|
-
|
|
238
|
-
it("rejects an over-long pattern and an invalid regex", async () => {
|
|
239
|
-
await expect(index.search("x".repeat(1001))).rejects.toThrow(/limit is 1000/);
|
|
240
|
-
await expect(index.search("(x", { regex: true })).rejects.toThrow(SyntaxError);
|
|
241
|
-
});
|
|
242
|
-
|
|
243
|
-
it("drop removes the index; init starts it empty", async () => {
|
|
244
|
-
await index.upsert(items(["dropped text"]));
|
|
245
|
-
await index.drop();
|
|
246
|
-
await index.init();
|
|
247
|
-
expect((await index.search("dropped")).hits).toEqual([]);
|
|
248
|
-
expect((await index.search("d")).hits).toEqual([]);
|
|
249
|
-
});
|
|
250
|
-
|
|
251
|
-
it("getTexts returns stored texts of known ids only", async () => {
|
|
252
|
-
await index.upsert([
|
|
253
|
-
{ id: 3, text: "three" },
|
|
254
|
-
{ id: 7, text: "seven" },
|
|
255
|
-
]);
|
|
256
|
-
await index.delete([7]);
|
|
257
|
-
expect(await index.getTexts([3, 7, 99])).toEqual(new Map([[3, "three"]]));
|
|
258
|
-
const [row] = await db.query<{ n: number }>(`SELECT count(*) AS n FROM ${index.idTable}`);
|
|
259
|
-
expect(Number(row?.n)).toBe(1);
|
|
260
|
-
});
|
|
261
|
-
|
|
262
|
-
it("drop from a default-store handle removes every table of either store", async () => {
|
|
263
|
-
await index.upsert(items(["some text"]));
|
|
264
|
-
await new NgramStore({ db, prefix: "t_" }).drop();
|
|
265
|
-
const left = await db.query<{ name: string }>(
|
|
266
|
-
"SELECT name FROM sqlite_master WHERE name LIKE 't\\_%' ESCAPE '\\'",
|
|
267
|
-
);
|
|
268
|
-
expect(left).toEqual([]);
|
|
269
|
-
});
|
|
270
|
-
});
|
|
271
|
-
}
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
describe("NgramStore options", () => {
|
|
275
|
-
it("rejects a prefix that is not a plain SQL identifier", async () => {
|
|
276
|
-
const db = await newNodeSqliteDb();
|
|
277
|
-
expect(() => new NgramStore({ db, prefix: "x; DROP TABLE y" })).toThrow(/prefix/);
|
|
278
|
-
await db.close();
|
|
279
|
-
});
|
|
280
|
-
});
|
|
281
|
-
|
|
282
|
-
describe("NgramStore store choice", () => {
|
|
283
|
-
it("rejects opening a prefix with the other store", async () => {
|
|
284
|
-
const db = await newNodeSqliteDb();
|
|
285
|
-
await new NgramStore({ db, prefix: "m_", store: "fts5" }).init();
|
|
286
|
-
await expect(new NgramStore({ db, prefix: "m_", store: "postings" }).init()).rejects.toThrow(
|
|
287
|
-
'ngram: index "m_" was created with store "fts5", opened with "postings"',
|
|
288
|
-
);
|
|
289
|
-
await db.close();
|
|
290
|
-
});
|
|
291
|
-
});
|
|
@@ -1,141 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { fold } from "./fold.js";
|
|
3
|
-
import { MAX_PATTERN_LENGTH, planQuery } from "./plan-query.js";
|
|
4
|
-
import { gramsOf, trigrams } from "./trigrams.js";
|
|
5
|
-
|
|
6
|
-
const REGEX_TABLE: Array<[string, string[][] | null]> = [
|
|
7
|
-
["abc", [["abc"]]],
|
|
8
|
-
["abc(def|ghi)", [["abcdef"], ["abcghi"]]],
|
|
9
|
-
["abc.*def", [["abc", "def"]]],
|
|
10
|
-
[
|
|
11
|
-
"abc.*(def|ghi)",
|
|
12
|
-
[
|
|
13
|
-
["abc", "def"],
|
|
14
|
-
["abc", "ghi"],
|
|
15
|
-
],
|
|
16
|
-
],
|
|
17
|
-
["a+hello", [["hello"]]],
|
|
18
|
-
["(abc)(def)", [["abcdef"]]],
|
|
19
|
-
["[ab][cd][ef]", [["ace"], ["acf"], ["ade"], ["adf"], ["bce"], ["bcf"], ["bde"], ["bdf"]]],
|
|
20
|
-
["ab[cd]e", [["abce"], ["abde"]]],
|
|
21
|
-
["a.b", null],
|
|
22
|
-
["[a-z]{3}", null],
|
|
23
|
-
["foo|bar", [["foo"], ["bar"]]],
|
|
24
|
-
["foo|.+", null],
|
|
25
|
-
["(abc)*$", null],
|
|
26
|
-
["abc+de", [["abc"]]],
|
|
27
|
-
["ab{2}c", [["abbc"]]],
|
|
28
|
-
["ab{2,}c", [["abb"]]],
|
|
29
|
-
["a?bcd", [["bcd"]]],
|
|
30
|
-
["qua(?!foo)", [["qua"]]],
|
|
31
|
-
["/\\d+/-\\d", null],
|
|
32
|
-
["x\\.y\\.z", [["x.y.z"]]],
|
|
33
|
-
["\\x41BC", [["abc"]]],
|
|
34
|
-
["(a{10}|b{10}|c{10}){10}", [["aaaaaaaaaa"], ["bbbbbbbbbb"], ["cccccccccc"]]],
|
|
35
|
-
["(?:abc|abd)ef", [["abcef"], ["abdef"]]],
|
|
36
|
-
["(?<n>abc)def", [["abcdef"]]],
|
|
37
|
-
["a(?=bcd)", null],
|
|
38
|
-
["[^abc]def", [["def"]]],
|
|
39
|
-
["\\bword\\b", [["word"]]],
|
|
40
|
-
["[abc]{3}[def]{3}", null],
|
|
41
|
-
["[qwerty]{2}-?[qwerty]{2}", null],
|
|
42
|
-
["[^]|abc]def", null],
|
|
43
|
-
];
|
|
44
|
-
|
|
45
|
-
// A string each pattern matches (and, below, the counterexamples to gluing arms).
|
|
46
|
-
const MATCHES: Array<[string, string]> = [
|
|
47
|
-
["abc", "xabcx"],
|
|
48
|
-
["abc(def|ghi)", "abcghi"],
|
|
49
|
-
["abc.*def", "abc--def"],
|
|
50
|
-
["abc.*(def|ghi)", "abcZghi"],
|
|
51
|
-
["a+hello", "aaahello"],
|
|
52
|
-
["(abc)(def)", "abcdef"],
|
|
53
|
-
["[ab][cd][ef]", "bdf"],
|
|
54
|
-
["ab[cd]e", "abde"],
|
|
55
|
-
["a.b", "aXb"],
|
|
56
|
-
["[a-z]{3}", "xyz"],
|
|
57
|
-
["foo|bar", "bar"],
|
|
58
|
-
["foo|.+", "zzz"],
|
|
59
|
-
["(abc)*$", "abcabc"],
|
|
60
|
-
["abc+de", "abccde"],
|
|
61
|
-
["ab{2}c", "abbc"],
|
|
62
|
-
["ab{2,}c", "abbbc"],
|
|
63
|
-
["a?bcd", "bcd"],
|
|
64
|
-
["qua(?!foo)", "quab"],
|
|
65
|
-
["/\\d+/-\\d", "/12/-3"],
|
|
66
|
-
["x\\.y\\.z", "x.y.z"],
|
|
67
|
-
["\\x41BC", "ABC"],
|
|
68
|
-
["(a{10}|b{10}|c{10}){10}", "c".repeat(100)],
|
|
69
|
-
["(?:abc|abd)ef", "abdef"],
|
|
70
|
-
["(?<n>abc)def", "abcdef"],
|
|
71
|
-
["a(?=bcd)", "abcd"],
|
|
72
|
-
["[^abc]def", "zdef"],
|
|
73
|
-
["\\bword\\b", "a word."],
|
|
74
|
-
["[abc]{3}[def]{3}", "cabfed"],
|
|
75
|
-
["[qwerty]{2}-?[qwerty]{2}", "qw-er"],
|
|
76
|
-
["[^]|abc]def", "x"],
|
|
77
|
-
["x(.abc)", "xZabc"],
|
|
78
|
-
["(abc.)def", "abcXdef"],
|
|
79
|
-
["xy(a*bcd)", "xyabcd"],
|
|
80
|
-
["foo(bar|.baz)", "fooXbaz"],
|
|
81
|
-
["(\\dabc)def", "1abcdef"],
|
|
82
|
-
["x(a+)bcd", "xaabcd"],
|
|
83
|
-
["(?<=abc)def", "abcdef"],
|
|
84
|
-
["\\u{1F600}abc", "u{1F600}abc"],
|
|
85
|
-
["x\\u{3}yz", "xuuuyz"],
|
|
86
|
-
["[😀]abc", "😀abc"],
|
|
87
|
-
];
|
|
88
|
-
|
|
89
|
-
describe("planQuery (regex)", () => {
|
|
90
|
-
it.each(REGEX_TABLE)("%s", (pattern, expected) => {
|
|
91
|
-
expect(planQuery(pattern, { regex: true }).branches).toEqual(expected);
|
|
92
|
-
});
|
|
93
|
-
|
|
94
|
-
it.each(MATCHES)("never rejects a string that %s matches: %s", (pattern, text) => {
|
|
95
|
-
expect(new RegExp(pattern).test(text)).toBe(true);
|
|
96
|
-
const { branches } = planQuery(pattern, { regex: true });
|
|
97
|
-
const folded = fold(text);
|
|
98
|
-
const grams = new Set(trigrams(text));
|
|
99
|
-
const covered = (b: string[]) =>
|
|
100
|
-
b.every((l) => folded.includes(l) && gramsOf(l).every((g) => grams.has(g)));
|
|
101
|
-
expect(branches === null || branches.some(covered)).toBe(true);
|
|
102
|
-
});
|
|
103
|
-
|
|
104
|
-
it("folds literals whatever the case sensitivity", () => {
|
|
105
|
-
expect(planQuery("DEF", { regex: true }).branches).toEqual([["def"]]);
|
|
106
|
-
expect(planQuery("DEF", { regex: true, caseSensitive: false }).branches).toEqual([["def"]]);
|
|
107
|
-
});
|
|
108
|
-
|
|
109
|
-
it("builds a global regex, case-insensitive only on request", () => {
|
|
110
|
-
expect(planQuery("abc").regex.flags).toBe("g");
|
|
111
|
-
expect(planQuery("abc", { caseSensitive: false }).regex.flags).toBe("gi");
|
|
112
|
-
});
|
|
113
|
-
|
|
114
|
-
it("rejects \\p{…} and \\P{…} in regex mode, but not an escaped backslash before p{", () => {
|
|
115
|
-
expect(() => planQuery("abc\\p{Lu}", { regex: true })).toThrow(/not supported in regex mode/);
|
|
116
|
-
expect(() => planQuery("[\\P{L}]abc", { regex: true })).toThrow(SyntaxError);
|
|
117
|
-
expect(() => planQuery("abc\\\\p{2}", { regex: true })).not.toThrow();
|
|
118
|
-
expect(() => planQuery("abc\\p{Lu}", {})).not.toThrow();
|
|
119
|
-
});
|
|
120
|
-
|
|
121
|
-
it("rejects an invalid regex with the engine's SyntaxError", () => {
|
|
122
|
-
expect(() => planQuery("(abc", { regex: true })).toThrow(SyntaxError);
|
|
123
|
-
});
|
|
124
|
-
});
|
|
125
|
-
|
|
126
|
-
describe("planQuery (literal)", () => {
|
|
127
|
-
it("treats regex syntax as text", () => {
|
|
128
|
-
expect(planQuery("a.b*c").branches).toEqual([["a.b*c"]]);
|
|
129
|
-
expect(planQuery("a.b*c").regex.test("xa.b*cx")).toBe(true);
|
|
130
|
-
});
|
|
131
|
-
|
|
132
|
-
it("needs a scan below 3 code points, counting an astral character as one", () => {
|
|
133
|
-
expect(planQuery("ab").branches).toBeNull();
|
|
134
|
-
expect(planQuery("😀a").branches).toBeNull();
|
|
135
|
-
expect(planQuery("😀ab").branches).toEqual([["😀ab"]]);
|
|
136
|
-
});
|
|
137
|
-
|
|
138
|
-
it("rejects patterns over the length cap", () => {
|
|
139
|
-
expect(() => planQuery("a".repeat(MAX_PATTERN_LENGTH + 1))).toThrow(/limit is 1000/);
|
|
140
|
-
});
|
|
141
|
-
});
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { encodeGram, gramsOf, trigrams } from "./trigrams.js";
|
|
3
|
-
|
|
4
|
-
describe("trigrams", () => {
|
|
5
|
-
it("returns distinct folded 3-code-point grams in order of first appearance", () => {
|
|
6
|
-
expect(trigrams("Hello")).toEqual(["hel", "ell", "llo"]);
|
|
7
|
-
expect(trigrams("aaaa")).toEqual(["aaa"]);
|
|
8
|
-
});
|
|
9
|
-
|
|
10
|
-
it("returns nothing for texts shorter than 3 code points", () => {
|
|
11
|
-
expect(trigrams("ab")).toEqual([]);
|
|
12
|
-
expect(trigrams("😀😀")).toEqual([]);
|
|
13
|
-
});
|
|
14
|
-
|
|
15
|
-
it("never splits a surrogate pair", () => {
|
|
16
|
-
expect(trigrams("😀abc")).toEqual(["😀ab", "abc"]);
|
|
17
|
-
});
|
|
18
|
-
});
|
|
19
|
-
|
|
20
|
-
describe("encodeGram", () => {
|
|
21
|
-
it("encodes BMP grams exactly and astral grams above 2^48, all as safe integers", () => {
|
|
22
|
-
const bmp = encodeGram("abc");
|
|
23
|
-
expect(bmp).toBe(0x61 * 2 ** 32 + 0x62 * 2 ** 16 + 0x63);
|
|
24
|
-
const astral = encodeGram(gramsOf("😀ab")[0] ?? "");
|
|
25
|
-
expect(astral).toBeGreaterThanOrEqual(2 ** 48);
|
|
26
|
-
expect(Number.isSafeInteger(astral)).toBe(true);
|
|
27
|
-
});
|
|
28
|
-
});
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { BACKENDS, D, dataset, recall } from "./test-fixtures.js";
|
|
3
|
-
import { VectorIndex } from "./vector-index.js";
|
|
4
|
-
|
|
5
|
-
for (const [name, open] of Object.entries(BACKENDS)) {
|
|
6
|
-
describe(`auto-compaction over ${name}`, { timeout: 60_000 }, () => {
|
|
7
|
-
it("compacts once the tail reaches 4096 vectors by default", async () => {
|
|
8
|
-
const db = await open();
|
|
9
|
-
const index = new VectorIndex({ db, prefix: "vec_", dims: D });
|
|
10
|
-
await index.init();
|
|
11
|
-
const items = dataset(4500, 21);
|
|
12
|
-
await index.upsert(items);
|
|
13
|
-
// Batches of 200: the tail reaches 4200 >= 4096 after 21 batches, then 300 more arrive.
|
|
14
|
-
expect(index.stats.segments.map((s) => s.count)).toEqual([4200]);
|
|
15
|
-
expect(index.stats.tail).toBe(300);
|
|
16
|
-
expect(await recall(index, items, "cosine", { nprobe: 16 })).toBeGreaterThanOrEqual(0.85);
|
|
17
|
-
});
|
|
18
|
-
|
|
19
|
-
it("never compacts on its own when autoCompactAt is Infinity", async () => {
|
|
20
|
-
const db = await open();
|
|
21
|
-
const index = new VectorIndex({ db, prefix: "vec_", dims: D, autoCompactAt: Infinity });
|
|
22
|
-
await index.init();
|
|
23
|
-
await index.upsert(dataset(4500, 22));
|
|
24
|
-
expect(index.stats.segments).toEqual([]);
|
|
25
|
-
expect(index.stats.tail).toBe(4500);
|
|
26
|
-
});
|
|
27
|
-
});
|
|
28
|
-
}
|
package/src/vector/do-smoke.ts
DELETED
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
import type { Db } from "@statewalker/db-api";
|
|
2
|
-
import { VectorIndex } from "./vector-index.js";
|
|
3
|
-
|
|
4
|
-
const check = (ok: boolean, message: string) => {
|
|
5
|
-
if (!ok) throw new Error(`vector smoke: ${message}`);
|
|
6
|
-
};
|
|
7
|
-
|
|
8
|
-
/** Run inside a real Durable Object by db-sqlite-do's `pnpm test:do`. */
|
|
9
|
-
export const vectorDoScenarios = [
|
|
10
|
-
{
|
|
11
|
-
name: "vector: multi-part segment rows, 150 ids in one delete, read-back",
|
|
12
|
-
async run(db: Db) {
|
|
13
|
-
const dims = 256;
|
|
14
|
-
const n = 1100; // flat list; float block 4 × 1100 × 256 ≈ 1.1 MB → 2 parts
|
|
15
|
-
const index = new VectorIndex({
|
|
16
|
-
db,
|
|
17
|
-
prefix: "smoke_",
|
|
18
|
-
dims,
|
|
19
|
-
autoCompactAt: Number.POSITIVE_INFINITY,
|
|
20
|
-
});
|
|
21
|
-
await index.init();
|
|
22
|
-
// Well-spread vectors: a sin(i * 7 + j) phase ramp makes ids ~150 apart near-duplicates.
|
|
23
|
-
const vector = (i: number) =>
|
|
24
|
-
Float32Array.from({ length: dims }, (_, j) => Math.sin((i * 7919 + j * 104729) ** 2));
|
|
25
|
-
await index.upsert(
|
|
26
|
-
Array.from({ length: n }, (_, i) => ({ id: i + 1, vector: vector(i + 1) })),
|
|
27
|
-
);
|
|
28
|
-
check((await index.compact()) === n, "compact sealed every vector");
|
|
29
|
-
const [parts] = await db.query<{ n: number }>("SELECT count(*) AS n FROM smoke_f32");
|
|
30
|
-
check(Number(parts?.n) >= 2, `expected ≥ 2 float parts, got ${parts?.n}`);
|
|
31
|
-
const gone = Array.from({ length: 150 }, (_, i) => i + 1);
|
|
32
|
-
await index.delete(gone);
|
|
33
|
-
const hit = await index.search(vector(500), { k: 1, exact: true });
|
|
34
|
-
check(hit[0]?.id === 500, `nearest of 500 is ${hit[0]?.id}`);
|
|
35
|
-
const back = await index.getVectors([1, 500]);
|
|
36
|
-
check(!back.has(1) && back.has(500), "getVectors returns only live ids");
|
|
37
|
-
},
|
|
38
|
-
},
|
|
39
|
-
];
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { mulberry32 } from "./kmeans.js";
|
|
3
|
-
import { prepare } from "./math.js";
|
|
4
|
-
import { buildSegment, MIN_TRAIN, parseList, SegmentIndex } from "./segment.js";
|
|
5
|
-
|
|
6
|
-
const D = 32;
|
|
7
|
-
|
|
8
|
-
function clustered(n: number, seed: number): { ids: Float64Array; vectors: Float32Array } {
|
|
9
|
-
const r = mulberry32(seed);
|
|
10
|
-
const centers = Array.from({ length: 20 }, () =>
|
|
11
|
-
Float32Array.from({ length: D }, () => r() * 2 - 1),
|
|
12
|
-
);
|
|
13
|
-
const vectors = new Float32Array(n * D);
|
|
14
|
-
for (let i = 0; i < n; i++) {
|
|
15
|
-
const c = centers[Math.floor(r() * centers.length)] as Float32Array;
|
|
16
|
-
vectors.set(
|
|
17
|
-
prepare(
|
|
18
|
-
c.map((x) => x + (r() - 0.5) * 0.6),
|
|
19
|
-
D,
|
|
20
|
-
"cosine",
|
|
21
|
-
),
|
|
22
|
-
i * D,
|
|
23
|
-
);
|
|
24
|
-
}
|
|
25
|
-
return { ids: Float64Array.from({ length: n }, (_, i) => i + 1), vectors };
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
describe("segment codec", () => {
|
|
29
|
-
it("is one flat list below MIN_TRAIN and several lists from MIN_TRAIN on", () => {
|
|
30
|
-
const small = clustered(MIN_TRAIN - 1, 1);
|
|
31
|
-
const large = clustered(MIN_TRAIN, 1);
|
|
32
|
-
expect(buildSegment({ dims: D, metric: "cosine", ...small }).nlist).toBe(1);
|
|
33
|
-
expect(buildSegment({ dims: D, metric: "cosine", ...large }).nlist).toBeGreaterThan(1);
|
|
34
|
-
});
|
|
35
|
-
|
|
36
|
-
it("keeps every row exactly once, and a stored vector ranks itself first", () => {
|
|
37
|
-
const { ids, vectors } = clustered(3000, 2);
|
|
38
|
-
const built = buildSegment({ dims: D, metric: "cosine", ids, vectors });
|
|
39
|
-
const seg = new SegmentIndex(1, D, "cosine", built.centroids, built.counts);
|
|
40
|
-
const lists = built.lists.map((bytes, c) => parseList(bytes, built.counts[c] as number, D));
|
|
41
|
-
|
|
42
|
-
const seen = lists.flatMap((l) => Array.from(l.ids)).sort((a, b) => a - b);
|
|
43
|
-
expect(seen).toEqual(Array.from(ids));
|
|
44
|
-
|
|
45
|
-
const q = vectors.subarray(0, D);
|
|
46
|
-
let best = { id: -1, d: Number.POSITIVE_INFINITY };
|
|
47
|
-
for (const c of seg.probe(q, seg.nlist)) {
|
|
48
|
-
const data = lists[c];
|
|
49
|
-
if (!data) continue;
|
|
50
|
-
for (const cand of seg.filterList(q, c, data, 50)) {
|
|
51
|
-
const d = seg.scoreInt8(q, data, cand.row);
|
|
52
|
-
if (d < best.d) best = { id: cand.id, d };
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
expect(best.id).toBe(1);
|
|
56
|
-
});
|
|
57
|
-
|
|
58
|
-
it("rejects a list block of the wrong size", () => {
|
|
59
|
-
expect(() => parseList(new Uint8Array(10), 3, D)).toThrow(/list block has 10 bytes/);
|
|
60
|
-
});
|
|
61
|
-
});
|
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
import type { Db } from "@statewalker/db-api";
|
|
2
|
-
import { newDoSqliteDb } from "@statewalker/db-sqlite-do";
|
|
3
|
-
import { newFakeDoStorage } from "@statewalker/db-sqlite-do/fake";
|
|
4
|
-
import { newNodeSqliteDb } from "@statewalker/db-sqlite-node";
|
|
5
|
-
import { mulberry32 } from "./kmeans.js";
|
|
6
|
-
import { distance, prepare } from "./math.js";
|
|
7
|
-
import type { Metric, SearchOptions } from "./types.js";
|
|
8
|
-
import type { VectorIndex } from "./vector-index.js";
|
|
9
|
-
|
|
10
|
-
/** Every suite runs over each of these. */
|
|
11
|
-
export const BACKENDS: Record<string, () => Promise<Db>> = {
|
|
12
|
-
"node:sqlite": () => newNodeSqliteDb(),
|
|
13
|
-
"durable object (fake storage)": async () => newDoSqliteDb(newFakeDoStorage()),
|
|
14
|
-
};
|
|
15
|
-
|
|
16
|
-
export const D = 64;
|
|
17
|
-
|
|
18
|
-
export interface Item {
|
|
19
|
-
id: number;
|
|
20
|
-
vector: Float32Array;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
/** Clustered data: realistic for IVF (uniform noise makes every ANN look bad). */
|
|
24
|
-
export function dataset(n: number, seed: number, dims = D, clusters = 40): Item[] {
|
|
25
|
-
const r = mulberry32(seed);
|
|
26
|
-
const centers = Array.from({ length: clusters }, () =>
|
|
27
|
-
Float32Array.from({ length: dims }, () => r() * 2 - 1),
|
|
28
|
-
);
|
|
29
|
-
return Array.from({ length: n }, (_, i) => {
|
|
30
|
-
const c = centers[Math.floor(r() * clusters)] as Float32Array;
|
|
31
|
-
return { id: i + 1, vector: Float32Array.from(c, (x) => x + (r() - 0.5) * 0.6) };
|
|
32
|
-
});
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
export function bruteForce(items: Item[], q: Float32Array, k: number, metric: Metric, dims = D) {
|
|
36
|
-
const qq = prepare(q, dims, metric);
|
|
37
|
-
return items
|
|
38
|
-
.map((it) => ({
|
|
39
|
-
id: it.id,
|
|
40
|
-
d: distance(metric, qq, 0, prepare(it.vector, dims, metric), 0, dims),
|
|
41
|
-
}))
|
|
42
|
-
.sort((a, b) => a.d - b.d)
|
|
43
|
-
.slice(0, k)
|
|
44
|
-
.map((x) => x.id);
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
export async function recall(
|
|
48
|
-
index: VectorIndex,
|
|
49
|
-
items: Item[],
|
|
50
|
-
metric: Metric,
|
|
51
|
-
opts: SearchOptions = {},
|
|
52
|
-
): Promise<number> {
|
|
53
|
-
const r = mulberry32(7);
|
|
54
|
-
let hit = 0;
|
|
55
|
-
const Q = 30;
|
|
56
|
-
for (let i = 0; i < Q; i++) {
|
|
57
|
-
const base = items[Math.floor(r() * items.length)] as Item;
|
|
58
|
-
const q = base.vector.map((x) => x + (r() - 0.5) * 0.2);
|
|
59
|
-
const truth = new Set(bruteForce(items, q, 10, metric));
|
|
60
|
-
const got = await index.search(q, { k: 10, ...opts });
|
|
61
|
-
hit += got.filter((h) => truth.has(h.id)).length;
|
|
62
|
-
}
|
|
63
|
-
return hit / (Q * 10);
|
|
64
|
-
}
|