@statewalker/db-sqlite-index 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +289 -2
- package/dist/ngram.d.mts +93 -0
- package/dist/ngram.d.mts.map +1 -0
- package/dist/ngram.mjs +616 -0
- package/dist/ngram.mjs.map +1 -0
- package/dist/prefix-CTeW1EwH.mjs +7 -0
- package/dist/prefix-CTeW1EwH.mjs.map +1 -0
- package/dist/vector.d.mts +103 -0
- package/dist/vector.d.mts.map +1 -0
- package/dist/vector.mjs +1062 -0
- package/dist/vector.mjs.map +1 -0
- package/package.json +63 -3
- package/src/ngram/do-smoke.ts +46 -0
- package/src/ngram/fold.test.ts +50 -0
- package/src/ngram/fold.ts +21 -0
- package/src/ngram/fts5-backend.ts +55 -0
- package/src/ngram/index.ts +12 -0
- package/src/ngram/json-chunks.test.ts +23 -0
- package/src/ngram/json-chunks.ts +23 -0
- package/src/ngram/ngram-store.test.ts +291 -0
- package/src/ngram/ngram-store.ts +184 -0
- package/src/ngram/plan-query.test.ts +141 -0
- package/src/ngram/plan-query.ts +42 -0
- package/src/ngram/postings-backend.ts +53 -0
- package/src/ngram/regex-literals.ts +359 -0
- package/src/ngram/trigrams.test.ts +28 -0
- package/src/ngram/trigrams.ts +25 -0
- package/src/prefix.ts +2 -0
- package/src/vector/auto-compact.test.ts +28 -0
- package/src/vector/do-smoke.ts +39 -0
- package/src/vector/index.ts +9 -0
- package/src/vector/kmeans.ts +131 -0
- package/src/vector/math.ts +178 -0
- package/src/vector/segment.test.ts +61 -0
- package/src/vector/segment.ts +256 -0
- package/src/vector/sql-segment-store.ts +218 -0
- package/src/vector/test-fixtures.ts +64 -0
- package/src/vector/types.ts +30 -0
- package/src/vector/vector-index.test.ts +105 -0
- package/src/vector/vector-index.ts +586 -0
- package/src/vector/vector-lifecycle.test.ts +171 -0
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
import type { Db } from "@statewalker/db-api";
|
|
2
|
+
import { newDoSqliteDb } from "@statewalker/db-sqlite-do";
|
|
3
|
+
import { newFakeDoStorage } from "@statewalker/db-sqlite-do/fake";
|
|
4
|
+
import { newNodeSqliteDb } from "@statewalker/db-sqlite-node";
|
|
5
|
+
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
6
|
+
import { type NgramItem, type NgramSearchOptions, NgramStore } from "./index.js";
|
|
7
|
+
|
|
8
|
+
const DRIVERS: Array<[string, () => Promise<Db>]> = [
|
|
9
|
+
["node:sqlite", () => newNodeSqliteDb()],
|
|
10
|
+
["do (fake storage)", async () => newDoSqliteDb(newFakeDoStorage())],
|
|
11
|
+
];
|
|
12
|
+
const STORES = ["fts5", "postings"] as const;
|
|
13
|
+
|
|
14
|
+
const UNLIMITED: NgramSearchOptions = { limit: 1e9, candidateBudget: 1e9 };
|
|
15
|
+
|
|
16
|
+
/** The oracle: the same pattern over every text, no index. */
|
|
17
|
+
function scan(items: NgramItem[], pattern: string, opts: NgramSearchOptions = {}) {
|
|
18
|
+
const source = opts.regex ? pattern : pattern.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&");
|
|
19
|
+
const re = new RegExp(source, opts.caseSensitive === false ? "gi" : "g");
|
|
20
|
+
return items
|
|
21
|
+
.map(({ id, text }) => ({
|
|
22
|
+
id,
|
|
23
|
+
matches: [...text.matchAll(re)]
|
|
24
|
+
.filter((m) => m[0].length > 0)
|
|
25
|
+
.map((m) => ({ start: m.index, end: m.index + m[0].length })),
|
|
26
|
+
}))
|
|
27
|
+
.filter((hit) => hit.matches.length > 0)
|
|
28
|
+
.sort((a, b) => a.id - b.id);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const items = (texts: string[]): NgramItem[] => texts.map((text, i) => ({ id: i + 1, text }));
|
|
32
|
+
|
|
33
|
+
/** Same rows as pg_trgm's contrib/pg_trgm/data/trgm.data. */
|
|
34
|
+
const TRGM = items(
|
|
35
|
+
Array.from({ length: 1000 }, (_, i) => `qwertyu${String(i + 1).padStart(4, "0")}`),
|
|
36
|
+
);
|
|
37
|
+
/** Same rows as pg_trgm's test2 table. */
|
|
38
|
+
const TEST2 = items([
|
|
39
|
+
"abcdef",
|
|
40
|
+
"quark",
|
|
41
|
+
" z foo bar",
|
|
42
|
+
"/123/-45/",
|
|
43
|
+
"line 1",
|
|
44
|
+
"%line 2",
|
|
45
|
+
"line 3%",
|
|
46
|
+
"%line 4%",
|
|
47
|
+
"%li%ne 5%",
|
|
48
|
+
"li_e 6",
|
|
49
|
+
]);
|
|
50
|
+
const UNICODE = items([
|
|
51
|
+
"Straße und STRASSE",
|
|
52
|
+
"ſtraße",
|
|
53
|
+
"ΟΔΟΣ οδός ΟΔΟΣ",
|
|
54
|
+
"😀abc😀 x😀y",
|
|
55
|
+
"東京タワー と 東京駅",
|
|
56
|
+
"İstanbul istanbul",
|
|
57
|
+
"Kelvin K and k",
|
|
58
|
+
"ẞ ß ss",
|
|
59
|
+
]);
|
|
60
|
+
|
|
61
|
+
type Query = [string, NgramItem[], string, NgramSearchOptions];
|
|
62
|
+
const QUERIES: Query[] = [
|
|
63
|
+
["trgm", TRGM, "[qwerty]{2}-?[qwerty]{2}", { regex: true }],
|
|
64
|
+
["trgm", TRGM, "99", {}],
|
|
65
|
+
["trgm", TRGM, "qwerty", {}],
|
|
66
|
+
["trgm", TRGM, "qw", {}],
|
|
67
|
+
["trgm", TRGM, "u0988", {}],
|
|
68
|
+
["trgm", TRGM, "0(1|2)3", { regex: true }],
|
|
69
|
+
["trgm", TRGM, "ty.*99", { regex: true }],
|
|
70
|
+
["test2", TEST2, "BCD", {}],
|
|
71
|
+
["test2", TEST2, "bcd", {}],
|
|
72
|
+
["test2", TEST2, "BCD", { caseSensitive: false }],
|
|
73
|
+
["test2", TEST2, "\\bcd", {}],
|
|
74
|
+
["test2", TEST2, "z foo bar", {}],
|
|
75
|
+
["test2", TEST2, " z foo", {}],
|
|
76
|
+
["test2", TEST2, "%", {}],
|
|
77
|
+
["test2", TEST2, "li_e", {}],
|
|
78
|
+
["test2", TEST2, "%li%", {}],
|
|
79
|
+
["test2", TEST2, "[abc]{3}", { regex: true }],
|
|
80
|
+
["test2", TEST2, "a[bc]+d", { regex: true }],
|
|
81
|
+
["test2", TEST2, "(abc)*$", { regex: true }],
|
|
82
|
+
["test2", TEST2, "DEF", { regex: true, caseSensitive: false }],
|
|
83
|
+
["test2", TEST2, "dEf", { regex: true }],
|
|
84
|
+
["test2", TEST2, "^q", { regex: true, caseSensitive: false }],
|
|
85
|
+
["test2", TEST2, "[abc]{3}[def]{3}", { regex: true, caseSensitive: false }],
|
|
86
|
+
["test2", TEST2, "ab[a-z]{3}", { regex: true, caseSensitive: false }],
|
|
87
|
+
["test2", TEST2, "(^| )qua", { regex: true, caseSensitive: false }],
|
|
88
|
+
["test2", TEST2, "q.*rk$", { regex: true }],
|
|
89
|
+
["test2", TEST2, "q", { regex: true }],
|
|
90
|
+
["test2", TEST2, "[a-z]{3}", { regex: true }],
|
|
91
|
+
["test2", TEST2, "(a{10}|b{10}|c{10}){10}", { regex: true, caseSensitive: false }],
|
|
92
|
+
["test2", TEST2, " z foo bar", { regex: true }],
|
|
93
|
+
["test2", TEST2, "qua(?!foo)", { regex: true }],
|
|
94
|
+
["test2", TEST2, "/\\d+/-\\d", { regex: true }],
|
|
95
|
+
["short", TEST2, "", {}],
|
|
96
|
+
["short", TEST2, "a", {}],
|
|
97
|
+
["short", TEST2, "ab", {}],
|
|
98
|
+
["short", TEST2, "^", { regex: true }],
|
|
99
|
+
["unicode", UNICODE, "straße", {}],
|
|
100
|
+
["unicode", UNICODE, "STRASSE", { caseSensitive: false }],
|
|
101
|
+
["unicode", UNICODE, "ß", {}],
|
|
102
|
+
["unicode", UNICODE, "ẞ", { caseSensitive: false }],
|
|
103
|
+
["unicode", UNICODE, "ΟΔΟΣ", { caseSensitive: false }],
|
|
104
|
+
["unicode", UNICODE, "οδοσ", { caseSensitive: false }],
|
|
105
|
+
["unicode", UNICODE, "😀", {}],
|
|
106
|
+
["unicode", UNICODE, "😀a", {}],
|
|
107
|
+
["unicode", UNICODE, "abc😀", {}],
|
|
108
|
+
["unicode", UNICODE, "東京", {}],
|
|
109
|
+
["unicode", UNICODE, "東京駅", {}],
|
|
110
|
+
["unicode", UNICODE, "istanbul", { caseSensitive: false }],
|
|
111
|
+
["unicode", UNICODE, "kelvin K", { caseSensitive: false }],
|
|
112
|
+
];
|
|
113
|
+
|
|
114
|
+
for (const [driverName, newDb] of DRIVERS) {
|
|
115
|
+
for (const store of STORES) {
|
|
116
|
+
describe(`NgramStore ${store} on ${driverName}`, () => {
|
|
117
|
+
let db: Db;
|
|
118
|
+
let index: NgramStore;
|
|
119
|
+
|
|
120
|
+
beforeEach(async () => {
|
|
121
|
+
db = await newDb();
|
|
122
|
+
index = new NgramStore({ db, prefix: "t_", store });
|
|
123
|
+
await index.init();
|
|
124
|
+
});
|
|
125
|
+
/**
|
|
126
|
+
* Whether the index still yields a candidate for `literal` (3+ code points).
|
|
127
|
+
* With a zero budget, any candidate sets `truncated`; verification cannot hide
|
|
128
|
+
* stale grams here, which is how the write guards are pinned.
|
|
129
|
+
*/
|
|
130
|
+
const indexed = async (literal: string) =>
|
|
131
|
+
(await index.search(literal, { candidateBudget: 0 })).truncated;
|
|
132
|
+
afterEach(async () => {
|
|
133
|
+
await db.close();
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it.each(QUERIES)("%s: %s %j equals a brute-force scan", async (_c, corpus, pattern, o) => {
|
|
137
|
+
await index.upsert(corpus);
|
|
138
|
+
const result = await index.search(pattern, { ...o, ...UNLIMITED });
|
|
139
|
+
expect(result.hits).toEqual(scan(corpus, pattern, o));
|
|
140
|
+
expect(result.truncated).toBe(false);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
it("reports UTF-16 offsets in astral text", async () => {
|
|
144
|
+
await index.upsert([{ id: 7, text: "😀abc😀" }]);
|
|
145
|
+
const { hits } = await index.search("abc");
|
|
146
|
+
expect(hits).toEqual([{ id: 7, matches: [{ start: 2, end: 5 }] }]);
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it("upsert replaces the text: old content stops matching", async () => {
|
|
150
|
+
await index.upsert([{ id: 1, text: "alpha beta" }]);
|
|
151
|
+
await index.upsert([{ id: 1, text: "gamma delta" }]);
|
|
152
|
+
expect((await index.search("alpha")).hits).toEqual([]);
|
|
153
|
+
expect(await indexed("alpha")).toBe(false);
|
|
154
|
+
expect((await index.search("gamma")).hits.map((h) => h.id)).toEqual([1]);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
it("duplicate ids in one upsert keep the last text", async () => {
|
|
158
|
+
await index.upsert([
|
|
159
|
+
{ id: 1, text: "first version" },
|
|
160
|
+
{ id: 1, text: "second version" },
|
|
161
|
+
]);
|
|
162
|
+
expect((await index.search("first")).hits).toEqual([]);
|
|
163
|
+
expect(await indexed("first")).toBe(false);
|
|
164
|
+
expect((await index.search("version")).hits.map((h) => h.id)).toEqual([1]);
|
|
165
|
+
await index.delete([1]);
|
|
166
|
+
expect((await index.search("version")).hits).toEqual([]);
|
|
167
|
+
expect(await indexed("version")).toBe(false);
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
it("a lone surrogate is stored as U+FFFD, and delete leaves no grams", async () => {
|
|
171
|
+
await index.upsert([{ id: 1, text: "x\uD800y z" }]);
|
|
172
|
+
expect(await index.getTexts([1])).toEqual(new Map([[1, "x\uFFFDy z"]]));
|
|
173
|
+
expect(await indexed("x\uFFFDy")).toBe(true);
|
|
174
|
+
await index.delete([1]);
|
|
175
|
+
expect(await indexed("x\uFFFDy")).toBe(false);
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
it("delete removes items from index and scan", async () => {
|
|
179
|
+
await index.upsert(items(["one two", "two three"]));
|
|
180
|
+
await index.delete([1]);
|
|
181
|
+
expect((await index.search("two")).hits.map((h) => h.id)).toEqual([2]);
|
|
182
|
+
expect(await indexed("one")).toBe(false);
|
|
183
|
+
expect((await index.search("o", {})).hits.map((h) => h.id)).toEqual([2]);
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
it("upserts a batch larger than the 2 MB a DO accepts in one bound value", async () => {
|
|
187
|
+
const big = Array.from({ length: 6000 }, (_, i) => ({
|
|
188
|
+
id: i + 1,
|
|
189
|
+
text: `${"lorem ipsum ".repeat(33)}marker${String(i).padStart(5, "0")}`,
|
|
190
|
+
}));
|
|
191
|
+
await index.upsert(big);
|
|
192
|
+
expect((await index.search("marker05999")).hits.map((h) => h.id)).toEqual([6000]);
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
it("a second store on the same db and prefix sees persisted data", async () => {
|
|
196
|
+
await index.upsert(items(["persisted text"]));
|
|
197
|
+
const reopened = new NgramStore({ db, prefix: "t_", store });
|
|
198
|
+
await reopened.init();
|
|
199
|
+
expect((await reopened.search("persisted")).hits.map((h) => h.id)).toEqual([1]);
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
it("limit 0 returns no hits", async () => {
|
|
203
|
+
await index.upsert(items(["one two", "two three"]));
|
|
204
|
+
expect((await index.search("two", { limit: 0 })).hits).toEqual([]);
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
it("limit stops early and reports truncation", async () => {
|
|
208
|
+
await index.upsert(TRGM);
|
|
209
|
+
const result = await index.search("qwerty", { limit: 5 });
|
|
210
|
+
expect(result.hits.map((h) => h.id)).toEqual([1, 2, 3, 4, 5]);
|
|
211
|
+
expect(result.truncated).toBe(true);
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
it("candidateBudget caps verified candidates and reports truncation", async () => {
|
|
215
|
+
await index.upsert(TRGM);
|
|
216
|
+
const result = await index.search("qwerty", { candidateBudget: 10, limit: 1e9 });
|
|
217
|
+
expect(result.hits).toHaveLength(10);
|
|
218
|
+
expect(result.truncated).toBe(true);
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
it("filter drops candidates before verification", async () => {
|
|
222
|
+
await index.upsert(TRGM);
|
|
223
|
+
const result = await index.search("qwertyu09", { filter: (id) => id % 2 === 0 });
|
|
224
|
+
expect(result.hits.map((h) => h.id)).toEqual(
|
|
225
|
+
scan(TRGM, "qwertyu09")
|
|
226
|
+
.map((h) => h.id)
|
|
227
|
+
.filter((id) => id % 2 === 0),
|
|
228
|
+
);
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
it("an aborted signal rejects with its reason", async () => {
|
|
232
|
+
await index.upsert(TRGM);
|
|
233
|
+
const controller = new AbortController();
|
|
234
|
+
controller.abort(new Error("stop"));
|
|
235
|
+
await expect(index.search("qwerty", { signal: controller.signal })).rejects.toThrow("stop");
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
it("rejects an over-long pattern and an invalid regex", async () => {
|
|
239
|
+
await expect(index.search("x".repeat(1001))).rejects.toThrow(/limit is 1000/);
|
|
240
|
+
await expect(index.search("(x", { regex: true })).rejects.toThrow(SyntaxError);
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
it("drop removes the index; init starts it empty", async () => {
|
|
244
|
+
await index.upsert(items(["dropped text"]));
|
|
245
|
+
await index.drop();
|
|
246
|
+
await index.init();
|
|
247
|
+
expect((await index.search("dropped")).hits).toEqual([]);
|
|
248
|
+
expect((await index.search("d")).hits).toEqual([]);
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
it("getTexts returns stored texts of known ids only", async () => {
|
|
252
|
+
await index.upsert([
|
|
253
|
+
{ id: 3, text: "three" },
|
|
254
|
+
{ id: 7, text: "seven" },
|
|
255
|
+
]);
|
|
256
|
+
await index.delete([7]);
|
|
257
|
+
expect(await index.getTexts([3, 7, 99])).toEqual(new Map([[3, "three"]]));
|
|
258
|
+
const [row] = await db.query<{ n: number }>(`SELECT count(*) AS n FROM ${index.idTable}`);
|
|
259
|
+
expect(Number(row?.n)).toBe(1);
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
it("drop from a default-store handle removes every table of either store", async () => {
|
|
263
|
+
await index.upsert(items(["some text"]));
|
|
264
|
+
await new NgramStore({ db, prefix: "t_" }).drop();
|
|
265
|
+
const left = await db.query<{ name: string }>(
|
|
266
|
+
"SELECT name FROM sqlite_master WHERE name LIKE 't\\_%' ESCAPE '\\'",
|
|
267
|
+
);
|
|
268
|
+
expect(left).toEqual([]);
|
|
269
|
+
});
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
describe("NgramStore options", () => {
|
|
275
|
+
it("rejects a prefix that is not a plain SQL identifier", async () => {
|
|
276
|
+
const db = await newNodeSqliteDb();
|
|
277
|
+
expect(() => new NgramStore({ db, prefix: "x; DROP TABLE y" })).toThrow(/prefix/);
|
|
278
|
+
await db.close();
|
|
279
|
+
});
|
|
280
|
+
});
|
|
281
|
+
|
|
282
|
+
describe("NgramStore store choice", () => {
|
|
283
|
+
it("rejects opening a prefix with the other store", async () => {
|
|
284
|
+
const db = await newNodeSqliteDb();
|
|
285
|
+
await new NgramStore({ db, prefix: "m_", store: "fts5" }).init();
|
|
286
|
+
await expect(new NgramStore({ db, prefix: "m_", store: "postings" }).init()).rejects.toThrow(
|
|
287
|
+
'ngram: index "m_" was created with store "fts5", opened with "postings"',
|
|
288
|
+
);
|
|
289
|
+
await db.close();
|
|
290
|
+
});
|
|
291
|
+
});
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import type { Db } from "@statewalker/db-api";
|
|
2
|
+
import { inTransaction } from "@statewalker/db-sqlite";
|
|
3
|
+
import { PREFIX_PATTERN } from "../prefix.js";
|
|
4
|
+
import { type GramBackend, type NgramItem, newFts5Backend } from "./fts5-backend.js";
|
|
5
|
+
import { chunkJson } from "./json-chunks.js";
|
|
6
|
+
import { planQuery } from "./plan-query.js";
|
|
7
|
+
import { newPostingsBackend } from "./postings-backend.js";
|
|
8
|
+
|
|
9
|
+
export type { NgramItem };
|
|
10
|
+
export type NgramMatch = { start: number; end: number };
|
|
11
|
+
export type NgramHit = { id: number; matches: NgramMatch[] };
|
|
12
|
+
export type NgramSearchOptions = {
|
|
13
|
+
/** Treat `pattern` as a JS regex source; otherwise it is literal text. */
|
|
14
|
+
regex?: boolean;
|
|
15
|
+
/** Default true (grep semantics). */
|
|
16
|
+
caseSensitive?: boolean;
|
|
17
|
+
/** Maximum hits. Default 100. */
|
|
18
|
+
limit?: number;
|
|
19
|
+
/** Maximum candidates verified. Default 10,000. */
|
|
20
|
+
candidateBudget?: number;
|
|
21
|
+
/** Keep only ids for which this returns true (applied before verification). */
|
|
22
|
+
filter?: (id: number) => boolean;
|
|
23
|
+
signal?: AbortSignal;
|
|
24
|
+
};
|
|
25
|
+
export type NgramSearchResult = {
|
|
26
|
+
/** Ascending id order; every non-empty match, offsets in UTF-16 code units. */
|
|
27
|
+
hits: NgramHit[];
|
|
28
|
+
/** True when the search stopped before checking every candidate (budget or limit). */
|
|
29
|
+
truncated: boolean;
|
|
30
|
+
};
|
|
31
|
+
export type NgramStoreOptions = { db: Db; prefix: string; store?: "fts5" | "postings" };
|
|
32
|
+
|
|
33
|
+
const VERIFY_BATCH = 256;
|
|
34
|
+
|
|
35
|
+
export class NgramStore {
|
|
36
|
+
/** `{prefix}texts`: its `id` column holds exactly the stored ids. Callers may join it in SQL. */
|
|
37
|
+
readonly idTable: string;
|
|
38
|
+
readonly #db: Db;
|
|
39
|
+
readonly #prefix: string;
|
|
40
|
+
readonly #store: "fts5" | "postings";
|
|
41
|
+
readonly #backend: GramBackend;
|
|
42
|
+
|
|
43
|
+
constructor({ db, prefix, store = "fts5" }: NgramStoreOptions) {
|
|
44
|
+
if (!PREFIX_PATTERN.test(prefix)) {
|
|
45
|
+
throw new Error(`ngram: prefix "${prefix}" must match ${PREFIX_PATTERN}`);
|
|
46
|
+
}
|
|
47
|
+
if (store !== "fts5" && store !== "postings")
|
|
48
|
+
throw new Error(`ngram: unknown store "${store}"`);
|
|
49
|
+
this.#db = db;
|
|
50
|
+
this.#prefix = prefix;
|
|
51
|
+
this.idTable = `${prefix}texts`;
|
|
52
|
+
this.#store = store;
|
|
53
|
+
this.#backend = store === "fts5" ? newFts5Backend(prefix) : newPostingsBackend(prefix);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async init(): Promise<void> {
|
|
57
|
+
const p = this.#prefix;
|
|
58
|
+
await inTransaction(this.#db, async (tx) => {
|
|
59
|
+
await tx.exec(
|
|
60
|
+
`CREATE TABLE IF NOT EXISTS ${p}meta(key TEXT PRIMARY KEY, value TEXT NOT NULL)`,
|
|
61
|
+
);
|
|
62
|
+
const [row] = await tx.query<{ value: string }>(
|
|
63
|
+
`SELECT value FROM ${p}meta WHERE key = 'store'`,
|
|
64
|
+
);
|
|
65
|
+
if (row && row.value !== this.#store) {
|
|
66
|
+
throw new Error(
|
|
67
|
+
`ngram: index "${p}" was created with store "${row.value}", opened with "${this.#store}"`,
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
if (!row)
|
|
71
|
+
await tx.query(`INSERT INTO ${p}meta(key, value) VALUES ('store', ?)`, [this.#store]);
|
|
72
|
+
await tx.exec(
|
|
73
|
+
`CREATE TABLE IF NOT EXISTS ${p}texts(id INTEGER PRIMARY KEY, text TEXT NOT NULL)`,
|
|
74
|
+
);
|
|
75
|
+
await this.#backend.create(tx);
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async upsert(items: NgramItem[]): Promise<void> {
|
|
80
|
+
// The last text per id wins; a repeated id would leave stale grams behind.
|
|
81
|
+
// A lone surrogate would be stored differently from how it was indexed, so a
|
|
82
|
+
// later delete could not replay it: texts are made well-formed first.
|
|
83
|
+
const latest = [
|
|
84
|
+
...new Map(items.map(({ id, text }) => [id, { id, text: text.toWellFormed() }])).values(),
|
|
85
|
+
];
|
|
86
|
+
if (latest.length === 0) return;
|
|
87
|
+
await inTransaction(this.#db, async (tx) => {
|
|
88
|
+
await this.#backend.remove(
|
|
89
|
+
tx,
|
|
90
|
+
await this.#load(
|
|
91
|
+
tx,
|
|
92
|
+
latest.map((item) => item.id),
|
|
93
|
+
),
|
|
94
|
+
);
|
|
95
|
+
for (const chunk of chunkJson(latest.map((item) => [item.id, item.text]))) {
|
|
96
|
+
await tx.query(
|
|
97
|
+
`INSERT OR REPLACE INTO ${this.#prefix}texts(id, text) SELECT json_extract(value, '$[0]'), json_extract(value, '$[1]') FROM json_each(?)`,
|
|
98
|
+
[chunk],
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
await this.#backend.add(tx, latest);
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async delete(ids: number[]): Promise<void> {
|
|
106
|
+
if (ids.length === 0) return;
|
|
107
|
+
await inTransaction(this.#db, async (tx) => {
|
|
108
|
+
await this.#backend.remove(tx, await this.#load(tx, ids));
|
|
109
|
+
for (const chunk of chunkJson(ids)) {
|
|
110
|
+
await tx.query(
|
|
111
|
+
`DELETE FROM ${this.#prefix}texts WHERE id IN (SELECT value FROM json_each(?))`,
|
|
112
|
+
[chunk],
|
|
113
|
+
);
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async search(pattern: string, opts: NgramSearchOptions = {}): Promise<NgramSearchResult> {
|
|
119
|
+
const { limit = 100, candidateBudget = 10_000, filter, signal } = opts;
|
|
120
|
+
const plan = planQuery(pattern, {
|
|
121
|
+
regex: opts.regex,
|
|
122
|
+
caseSensitive: opts.caseSensitive ?? true,
|
|
123
|
+
});
|
|
124
|
+
signal?.throwIfAborted();
|
|
125
|
+
if (limit <= 0) return { hits: [], truncated: true };
|
|
126
|
+
const found =
|
|
127
|
+
plan.branches === null
|
|
128
|
+
? (
|
|
129
|
+
await this.#db.query<{ id: number }>(
|
|
130
|
+
`SELECT id FROM ${this.#prefix}texts ORDER BY id LIMIT ?`,
|
|
131
|
+
[candidateBudget + 1],
|
|
132
|
+
)
|
|
133
|
+
).map((row) => Number(row.id))
|
|
134
|
+
: await this.#backend.candidates(this.#db, plan.branches, candidateBudget + 1);
|
|
135
|
+
let truncated = found.length > candidateBudget;
|
|
136
|
+
const ids = found.slice(0, candidateBudget).filter((id) => !filter || filter(id));
|
|
137
|
+
|
|
138
|
+
const hits: NgramHit[] = [];
|
|
139
|
+
let last: number | undefined;
|
|
140
|
+
verify: for (let i = 0; i < ids.length; i += VERIFY_BATCH) {
|
|
141
|
+
signal?.throwIfAborted();
|
|
142
|
+
for (const { id, text } of await this.#load(this.#db, ids.slice(i, i + VERIFY_BATCH))) {
|
|
143
|
+
last = id;
|
|
144
|
+
const matches: NgramMatch[] = [];
|
|
145
|
+
for (const m of text.matchAll(plan.regex)) {
|
|
146
|
+
if (m[0].length > 0) matches.push({ start: m.index, end: m.index + m[0].length });
|
|
147
|
+
}
|
|
148
|
+
if (matches.length === 0) continue;
|
|
149
|
+
hits.push({ id, matches });
|
|
150
|
+
if (hits.length >= limit) break verify;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
if (hits.length >= limit && last !== ids.at(-1)) truncated = true;
|
|
154
|
+
return { hits, truncated };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** The stored original texts of `ids`; unknown ids are absent. */
|
|
158
|
+
async getTexts(ids: number[]): Promise<Map<number, string>> {
|
|
159
|
+
const items = await this.#load(this.#db, ids);
|
|
160
|
+
return new Map(items.map((item) => [item.id, item.text]));
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Drops the tables of both stores, so it works without knowing which one created the prefix. */
|
|
164
|
+
async drop(): Promise<void> {
|
|
165
|
+
const p = this.#prefix;
|
|
166
|
+
await inTransaction(this.#db, async (tx) => {
|
|
167
|
+
for (const t of ["ng", "grams", "texts", "meta"])
|
|
168
|
+
await tx.exec(`DROP TABLE IF EXISTS ${p}${t}`);
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** Stored texts for `ids`, ascending by id; missing ids are skipped. */
|
|
173
|
+
async #load(db: Db, ids: number[]): Promise<NgramItem[]> {
|
|
174
|
+
const out: NgramItem[] = [];
|
|
175
|
+
for (const chunk of chunkJson([...ids].sort((a, b) => a - b))) {
|
|
176
|
+
const rows = await db.query<{ id: number; text: string }>(
|
|
177
|
+
`SELECT id, text FROM ${this.#prefix}texts WHERE id IN (SELECT value FROM json_each(?)) ORDER BY id`,
|
|
178
|
+
[chunk],
|
|
179
|
+
);
|
|
180
|
+
for (const row of rows) out.push({ id: Number(row.id), text: String(row.text) });
|
|
181
|
+
}
|
|
182
|
+
return out;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { fold } from "./fold.js";
|
|
3
|
+
import { MAX_PATTERN_LENGTH, planQuery } from "./plan-query.js";
|
|
4
|
+
import { gramsOf, trigrams } from "./trigrams.js";
|
|
5
|
+
|
|
6
|
+
const REGEX_TABLE: Array<[string, string[][] | null]> = [
|
|
7
|
+
["abc", [["abc"]]],
|
|
8
|
+
["abc(def|ghi)", [["abcdef"], ["abcghi"]]],
|
|
9
|
+
["abc.*def", [["abc", "def"]]],
|
|
10
|
+
[
|
|
11
|
+
"abc.*(def|ghi)",
|
|
12
|
+
[
|
|
13
|
+
["abc", "def"],
|
|
14
|
+
["abc", "ghi"],
|
|
15
|
+
],
|
|
16
|
+
],
|
|
17
|
+
["a+hello", [["hello"]]],
|
|
18
|
+
["(abc)(def)", [["abcdef"]]],
|
|
19
|
+
["[ab][cd][ef]", [["ace"], ["acf"], ["ade"], ["adf"], ["bce"], ["bcf"], ["bde"], ["bdf"]]],
|
|
20
|
+
["ab[cd]e", [["abce"], ["abde"]]],
|
|
21
|
+
["a.b", null],
|
|
22
|
+
["[a-z]{3}", null],
|
|
23
|
+
["foo|bar", [["foo"], ["bar"]]],
|
|
24
|
+
["foo|.+", null],
|
|
25
|
+
["(abc)*$", null],
|
|
26
|
+
["abc+de", [["abc"]]],
|
|
27
|
+
["ab{2}c", [["abbc"]]],
|
|
28
|
+
["ab{2,}c", [["abb"]]],
|
|
29
|
+
["a?bcd", [["bcd"]]],
|
|
30
|
+
["qua(?!foo)", [["qua"]]],
|
|
31
|
+
["/\\d+/-\\d", null],
|
|
32
|
+
["x\\.y\\.z", [["x.y.z"]]],
|
|
33
|
+
["\\x41BC", [["abc"]]],
|
|
34
|
+
["(a{10}|b{10}|c{10}){10}", [["aaaaaaaaaa"], ["bbbbbbbbbb"], ["cccccccccc"]]],
|
|
35
|
+
["(?:abc|abd)ef", [["abcef"], ["abdef"]]],
|
|
36
|
+
["(?<n>abc)def", [["abcdef"]]],
|
|
37
|
+
["a(?=bcd)", null],
|
|
38
|
+
["[^abc]def", [["def"]]],
|
|
39
|
+
["\\bword\\b", [["word"]]],
|
|
40
|
+
["[abc]{3}[def]{3}", null],
|
|
41
|
+
["[qwerty]{2}-?[qwerty]{2}", null],
|
|
42
|
+
["[^]|abc]def", null],
|
|
43
|
+
];
|
|
44
|
+
|
|
45
|
+
// A string each pattern matches (and, below, the counterexamples to gluing arms).
|
|
46
|
+
const MATCHES: Array<[string, string]> = [
|
|
47
|
+
["abc", "xabcx"],
|
|
48
|
+
["abc(def|ghi)", "abcghi"],
|
|
49
|
+
["abc.*def", "abc--def"],
|
|
50
|
+
["abc.*(def|ghi)", "abcZghi"],
|
|
51
|
+
["a+hello", "aaahello"],
|
|
52
|
+
["(abc)(def)", "abcdef"],
|
|
53
|
+
["[ab][cd][ef]", "bdf"],
|
|
54
|
+
["ab[cd]e", "abde"],
|
|
55
|
+
["a.b", "aXb"],
|
|
56
|
+
["[a-z]{3}", "xyz"],
|
|
57
|
+
["foo|bar", "bar"],
|
|
58
|
+
["foo|.+", "zzz"],
|
|
59
|
+
["(abc)*$", "abcabc"],
|
|
60
|
+
["abc+de", "abccde"],
|
|
61
|
+
["ab{2}c", "abbc"],
|
|
62
|
+
["ab{2,}c", "abbbc"],
|
|
63
|
+
["a?bcd", "bcd"],
|
|
64
|
+
["qua(?!foo)", "quab"],
|
|
65
|
+
["/\\d+/-\\d", "/12/-3"],
|
|
66
|
+
["x\\.y\\.z", "x.y.z"],
|
|
67
|
+
["\\x41BC", "ABC"],
|
|
68
|
+
["(a{10}|b{10}|c{10}){10}", "c".repeat(100)],
|
|
69
|
+
["(?:abc|abd)ef", "abdef"],
|
|
70
|
+
["(?<n>abc)def", "abcdef"],
|
|
71
|
+
["a(?=bcd)", "abcd"],
|
|
72
|
+
["[^abc]def", "zdef"],
|
|
73
|
+
["\\bword\\b", "a word."],
|
|
74
|
+
["[abc]{3}[def]{3}", "cabfed"],
|
|
75
|
+
["[qwerty]{2}-?[qwerty]{2}", "qw-er"],
|
|
76
|
+
["[^]|abc]def", "x"],
|
|
77
|
+
["x(.abc)", "xZabc"],
|
|
78
|
+
["(abc.)def", "abcXdef"],
|
|
79
|
+
["xy(a*bcd)", "xyabcd"],
|
|
80
|
+
["foo(bar|.baz)", "fooXbaz"],
|
|
81
|
+
["(\\dabc)def", "1abcdef"],
|
|
82
|
+
["x(a+)bcd", "xaabcd"],
|
|
83
|
+
["(?<=abc)def", "abcdef"],
|
|
84
|
+
["\\u{1F600}abc", "u{1F600}abc"],
|
|
85
|
+
["x\\u{3}yz", "xuuuyz"],
|
|
86
|
+
["[😀]abc", "😀abc"],
|
|
87
|
+
];
|
|
88
|
+
|
|
89
|
+
describe("planQuery (regex)", () => {
|
|
90
|
+
it.each(REGEX_TABLE)("%s", (pattern, expected) => {
|
|
91
|
+
expect(planQuery(pattern, { regex: true }).branches).toEqual(expected);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it.each(MATCHES)("never rejects a string that %s matches: %s", (pattern, text) => {
|
|
95
|
+
expect(new RegExp(pattern).test(text)).toBe(true);
|
|
96
|
+
const { branches } = planQuery(pattern, { regex: true });
|
|
97
|
+
const folded = fold(text);
|
|
98
|
+
const grams = new Set(trigrams(text));
|
|
99
|
+
const covered = (b: string[]) =>
|
|
100
|
+
b.every((l) => folded.includes(l) && gramsOf(l).every((g) => grams.has(g)));
|
|
101
|
+
expect(branches === null || branches.some(covered)).toBe(true);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
it("folds literals whatever the case sensitivity", () => {
|
|
105
|
+
expect(planQuery("DEF", { regex: true }).branches).toEqual([["def"]]);
|
|
106
|
+
expect(planQuery("DEF", { regex: true, caseSensitive: false }).branches).toEqual([["def"]]);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it("builds a global regex, case-insensitive only on request", () => {
|
|
110
|
+
expect(planQuery("abc").regex.flags).toBe("g");
|
|
111
|
+
expect(planQuery("abc", { caseSensitive: false }).regex.flags).toBe("gi");
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it("rejects \\p{…} and \\P{…} in regex mode, but not an escaped backslash before p{", () => {
|
|
115
|
+
expect(() => planQuery("abc\\p{Lu}", { regex: true })).toThrow(/not supported in regex mode/);
|
|
116
|
+
expect(() => planQuery("[\\P{L}]abc", { regex: true })).toThrow(SyntaxError);
|
|
117
|
+
expect(() => planQuery("abc\\\\p{2}", { regex: true })).not.toThrow();
|
|
118
|
+
expect(() => planQuery("abc\\p{Lu}", {})).not.toThrow();
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it("rejects an invalid regex with the engine's SyntaxError", () => {
|
|
122
|
+
expect(() => planQuery("(abc", { regex: true })).toThrow(SyntaxError);
|
|
123
|
+
});
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
describe("planQuery (literal)", () => {
|
|
127
|
+
it("treats regex syntax as text", () => {
|
|
128
|
+
expect(planQuery("a.b*c").branches).toEqual([["a.b*c"]]);
|
|
129
|
+
expect(planQuery("a.b*c").regex.test("xa.b*cx")).toBe(true);
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it("needs a scan below 3 code points, counting an astral character as one", () => {
|
|
133
|
+
expect(planQuery("ab").branches).toBeNull();
|
|
134
|
+
expect(planQuery("😀a").branches).toBeNull();
|
|
135
|
+
expect(planQuery("😀ab").branches).toEqual([["😀ab"]]);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
it("rejects patterns over the length cap", () => {
|
|
139
|
+
expect(() => planQuery("a".repeat(MAX_PATTERN_LENGTH + 1))).toThrow(/limit is 1000/);
|
|
140
|
+
});
|
|
141
|
+
});
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { fold } from "./fold.js";
|
|
2
|
+
import { regexLiterals } from "./regex-literals.js";
|
|
3
|
+
|
|
4
|
+
/** Longer patterns are rejected: a JS regex cannot be interrupted once it runs. */
|
|
5
|
+
export const MAX_PATTERN_LENGTH = 1000;
|
|
6
|
+
|
|
7
|
+
export type QueryPlan = {
|
|
8
|
+
/** The pattern as a global regex (`i` added when not case-sensitive); verifies candidates. */
|
|
9
|
+
regex: RegExp;
|
|
10
|
+
/**
|
|
11
|
+
* Per DNF branch, the folded literals of at least 3 code points that a match must
|
|
12
|
+
* contain. `null` when some branch has none: only a scan can answer the pattern.
|
|
13
|
+
*/
|
|
14
|
+
branches: string[][] | null;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
export function planQuery(
|
|
18
|
+
pattern: string,
|
|
19
|
+
opts: { regex?: boolean; caseSensitive?: boolean } = {},
|
|
20
|
+
): QueryPlan {
|
|
21
|
+
if (pattern.length > MAX_PATTERN_LENGTH) {
|
|
22
|
+
throw new RangeError(
|
|
23
|
+
`ngram: pattern is ${pattern.length} characters, the limit is ${MAX_PATTERN_LENGTH}`,
|
|
24
|
+
);
|
|
25
|
+
}
|
|
26
|
+
// The regex has no `u` flag (case folding is unsound under /iu), so `\p{…}` would silently
|
|
27
|
+
// match the literal text "p{…}". An odd run of backslashes before `p` is a real escape.
|
|
28
|
+
if (opts.regex && /(?<!\\)(?:\\\\)*\\[pP]\{/.test(pattern)) {
|
|
29
|
+
throw new SyntaxError(
|
|
30
|
+
"ngram: \\p{…} and \\P{…} are not supported in regex mode (patterns compile without the u flag)",
|
|
31
|
+
);
|
|
32
|
+
}
|
|
33
|
+
const source = opts.regex ? pattern : pattern.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&");
|
|
34
|
+
const regex = new RegExp(source, opts.caseSensitive === false ? "gi" : "g");
|
|
35
|
+
const branches: string[][] = [];
|
|
36
|
+
for (const literals of regexLiterals(regex)) {
|
|
37
|
+
const usable = literals.map(fold).filter((literal) => Array.from(literal).length >= 3);
|
|
38
|
+
if (usable.length === 0) return { regex, branches: null };
|
|
39
|
+
branches.push(usable);
|
|
40
|
+
}
|
|
41
|
+
return { regex, branches };
|
|
42
|
+
}
|