@statewalker/db-sqlite-index 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +289 -2
- package/dist/ngram.d.mts +93 -0
- package/dist/ngram.d.mts.map +1 -0
- package/dist/ngram.mjs +616 -0
- package/dist/ngram.mjs.map +1 -0
- package/dist/prefix-CTeW1EwH.mjs +7 -0
- package/dist/prefix-CTeW1EwH.mjs.map +1 -0
- package/dist/vector.d.mts +103 -0
- package/dist/vector.d.mts.map +1 -0
- package/dist/vector.mjs +1062 -0
- package/dist/vector.mjs.map +1 -0
- package/package.json +63 -3
- package/src/ngram/do-smoke.ts +46 -0
- package/src/ngram/fold.test.ts +50 -0
- package/src/ngram/fold.ts +21 -0
- package/src/ngram/fts5-backend.ts +55 -0
- package/src/ngram/index.ts +12 -0
- package/src/ngram/json-chunks.test.ts +23 -0
- package/src/ngram/json-chunks.ts +23 -0
- package/src/ngram/ngram-store.test.ts +291 -0
- package/src/ngram/ngram-store.ts +184 -0
- package/src/ngram/plan-query.test.ts +141 -0
- package/src/ngram/plan-query.ts +42 -0
- package/src/ngram/postings-backend.ts +53 -0
- package/src/ngram/regex-literals.ts +359 -0
- package/src/ngram/trigrams.test.ts +28 -0
- package/src/ngram/trigrams.ts +25 -0
- package/src/prefix.ts +2 -0
- package/src/vector/auto-compact.test.ts +28 -0
- package/src/vector/do-smoke.ts +39 -0
- package/src/vector/index.ts +9 -0
- package/src/vector/kmeans.ts +131 -0
- package/src/vector/math.ts +178 -0
- package/src/vector/segment.test.ts +61 -0
- package/src/vector/segment.ts +256 -0
- package/src/vector/sql-segment-store.ts +218 -0
- package/src/vector/test-fixtures.ts +64 -0
- package/src/vector/types.ts +30 -0
- package/src/vector/vector-index.test.ts +105 -0
- package/src/vector/vector-index.ts +586 -0
- package/src/vector/vector-lifecycle.test.ts +171 -0
package/dist/ngram.mjs
ADDED
|
@@ -0,0 +1,616 @@
|
|
|
1
|
+
import { t as PREFIX_PATTERN } from "./prefix-CTeW1EwH.mjs";
|
|
2
|
+
import { inTransaction } from "@statewalker/db-sqlite";
|
|
3
|
+
//#region src/ngram/fold.ts
|
|
4
|
+
/**
|
|
5
|
+
* Case fold per code point, applied to indexed text and to query literals alike.
|
|
6
|
+
*
|
|
7
|
+
* Upper-then-lower maps every character a case-insensitive JS regex treats as
|
|
8
|
+
* equal to one form (Σ σ ς → σ, ſ → s). A character whose upper case is several
|
|
9
|
+
* code points (ß → SS) only matches itself under `/i`, so it keeps its own
|
|
10
|
+
* lower case. Folding per code point makes fold(a + b) === fold(a) + fold(b), so
|
|
11
|
+
* a substring of the text stays a substring of the folded text. A character
|
|
12
|
+
* whose folded form has a different UTF-16 length (İ → i + U+0307) is kept as
|
|
13
|
+
* is, so fold preserves the length and offsets into the folded text are
|
|
14
|
+
* offsets into the original.
|
|
15
|
+
*/
|
|
16
|
+
function fold(text) {
|
|
17
|
+
let out = "";
|
|
18
|
+
for (const ch of text) {
|
|
19
|
+
const upper = ch.toUpperCase();
|
|
20
|
+
const lower = (Array.from(upper).length === 1 ? upper : ch).toLowerCase();
|
|
21
|
+
out += lower.length === ch.length ? lower : ch;
|
|
22
|
+
}
|
|
23
|
+
return out;
|
|
24
|
+
}
|
|
25
|
+
//#endregion
|
|
26
|
+
//#region src/ngram/json-chunks.ts
|
|
27
|
+
/**
|
|
28
|
+
* Serialises rows into JSON array parameters for `json_each(?)`, each at most
|
|
29
|
+
* `maxChars` UTF-16 code units. 500k code units stay under a Durable Object's
|
|
30
|
+
* ~2 MB string limit even at 4 bytes per code point; a single row larger than
|
|
31
|
+
* that still fails there with "string or blob too big".
|
|
32
|
+
*/
|
|
33
|
+
function chunkJson(rows, maxChars = 5e5) {
|
|
34
|
+
const chunks = [];
|
|
35
|
+
let current = [];
|
|
36
|
+
let size = 2;
|
|
37
|
+
for (const row of rows) {
|
|
38
|
+
const json = JSON.stringify(row);
|
|
39
|
+
if (current.length > 0 && size + json.length + 1 > maxChars) {
|
|
40
|
+
chunks.push(`[${current.join(",")}]`);
|
|
41
|
+
current = [];
|
|
42
|
+
size = 2;
|
|
43
|
+
}
|
|
44
|
+
current.push(json);
|
|
45
|
+
size += json.length + 1;
|
|
46
|
+
}
|
|
47
|
+
if (current.length > 0) chunks.push(`[${current.join(",")}]`);
|
|
48
|
+
return chunks;
|
|
49
|
+
}
|
|
50
|
+
//#endregion
|
|
51
|
+
//#region src/ngram/fts5-backend.ts
|
|
52
|
+
const ROW = "json_extract(value, '$[0]'), json_extract(value, '$[1]')";
|
|
53
|
+
/**
|
|
54
|
+
* FTS5 trigram over folded text. Contentless (`content=''`), because NgramStore
|
|
55
|
+
* already keeps the text; a contentless row is deleted by replaying its exact
|
|
56
|
+
* indexed value through the 'delete' command, which is why `remove` needs the
|
|
57
|
+
* stored texts. `case_sensitive 1`: folding is done by `fold`, the same function
|
|
58
|
+
* that folds query literals, so the index and the planner can never disagree.
|
|
59
|
+
*/
|
|
60
|
+
function newFts5Backend(prefix) {
|
|
61
|
+
const table = `${prefix}ng`;
|
|
62
|
+
const write = async (tx, items, command) => {
|
|
63
|
+
const sql = command === "insert" ? `INSERT INTO ${table}(rowid, text) SELECT ${ROW} FROM json_each(?)` : `INSERT INTO ${table}(${table}, rowid, text) SELECT 'delete', ${ROW} FROM json_each(?)`;
|
|
64
|
+
for (const chunk of chunkJson(items.map((item) => [item.id, fold(item.text)]))) await tx.query(sql, [chunk]);
|
|
65
|
+
};
|
|
66
|
+
return {
|
|
67
|
+
create: (tx) => tx.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${table} USING fts5(text, content='', tokenize='trigram case_sensitive 1')`),
|
|
68
|
+
add: (tx, items) => write(tx, items, "insert"),
|
|
69
|
+
remove: (tx, items) => write(tx, items, "delete"),
|
|
70
|
+
async candidates(db, branches, max) {
|
|
71
|
+
const quote = (literal) => `"${literal.replaceAll("\"", "\"\"")}"`;
|
|
72
|
+
const match = branches.map((branch) => `(${branch.map(quote).join(" AND ")})`).join(" OR ");
|
|
73
|
+
return (await db.query(`SELECT rowid AS id FROM ${table} WHERE ${table} MATCH ? ORDER BY rowid LIMIT ?`, [match, max])).map((row) => Number(row.id));
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
//#endregion
|
|
78
|
+
//#region src/ngram/regex-literals.ts
|
|
79
|
+
/** Cross-product cap: past it, the construct counts as a wildcard. */
|
|
80
|
+
const MAX_DNF = 32;
|
|
81
|
+
/**
|
|
82
|
+
* DNF of literals that every match must contain: a string can match only if all
|
|
83
|
+
* literals of at least one branch occur in it. An empty branch matches anything.
|
|
84
|
+
* Conservative: a construct it cannot reason about becomes a wildcard, never a literal.
|
|
85
|
+
*
|
|
86
|
+
* /abc(def|ghi)/ → [["abcdef"], ["abcghi"]] /foo|.+/ → [["foo"], []]
|
|
87
|
+
*/
|
|
88
|
+
function regexLiterals(regex) {
|
|
89
|
+
if (regex.flags.includes("v")) return [[]];
|
|
90
|
+
return expandRegion(regex.source, 0, regex.source.length).dnf;
|
|
91
|
+
}
|
|
92
|
+
function expandRegion(src, start, end) {
|
|
93
|
+
const dnf = [];
|
|
94
|
+
let exact = true;
|
|
95
|
+
for (const [bStart, bEnd] of splitTopLevelAlternation(src, start, end)) {
|
|
96
|
+
const arm = expandBranch(src, bStart, bEnd);
|
|
97
|
+
for (const branch of arm.dnf) dnf.push(branch);
|
|
98
|
+
exact = exact && arm.exact;
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
dnf,
|
|
102
|
+
exact
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
function splitTopLevelAlternation(src, start, end) {
|
|
106
|
+
const spans = [];
|
|
107
|
+
let bStart = start;
|
|
108
|
+
let depth = 0;
|
|
109
|
+
let i = start;
|
|
110
|
+
while (i < end) {
|
|
111
|
+
const c = src[i];
|
|
112
|
+
if (c === "\\") {
|
|
113
|
+
i += 2;
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
if (c === "[") {
|
|
117
|
+
i = skipClass(src, i);
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
if (c === "(") depth += 1;
|
|
121
|
+
else if (c === ")") depth -= 1;
|
|
122
|
+
else if (c === "|" && depth === 0) {
|
|
123
|
+
spans.push([bStart, i]);
|
|
124
|
+
bStart = i + 1;
|
|
125
|
+
}
|
|
126
|
+
i += 1;
|
|
127
|
+
}
|
|
128
|
+
spans.push([bStart, end]);
|
|
129
|
+
return spans;
|
|
130
|
+
}
|
|
131
|
+
function expandBranch(src, start, end) {
|
|
132
|
+
let cross = [{
|
|
133
|
+
literals: [],
|
|
134
|
+
run: ""
|
|
135
|
+
}];
|
|
136
|
+
let lossy = false;
|
|
137
|
+
const flushRuns = () => {
|
|
138
|
+
for (const b of cross) if (b.run.length > 0) {
|
|
139
|
+
b.literals.push(b.run);
|
|
140
|
+
b.run = "";
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
const appendRun = (text, q) => {
|
|
144
|
+
if (q.copies === 0) {
|
|
145
|
+
lossy = true;
|
|
146
|
+
flushRuns();
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
for (const b of cross) b.run += text.repeat(q.copies);
|
|
150
|
+
if (q.openEnded) {
|
|
151
|
+
lossy = true;
|
|
152
|
+
flushRuns();
|
|
153
|
+
}
|
|
154
|
+
};
|
|
155
|
+
let i = start;
|
|
156
|
+
while (i < end) {
|
|
157
|
+
const c = src[i] ?? "";
|
|
158
|
+
if (c === "^" || c === "$") {
|
|
159
|
+
i += 1;
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
if (c === "\\") {
|
|
163
|
+
const next = src[i + 1];
|
|
164
|
+
if (next === void 0) {
|
|
165
|
+
i += 1;
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
const escaped = parseEscapedLiteral(src, i);
|
|
169
|
+
if (escaped) {
|
|
170
|
+
const q = peekQuantifier(src, escaped.end);
|
|
171
|
+
i = q.end;
|
|
172
|
+
appendRun(escaped.char, q);
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
if (/[a-zA-Z0-9]/.test(next)) {
|
|
176
|
+
lossy = true;
|
|
177
|
+
flushRuns();
|
|
178
|
+
i = peekQuantifier(src, skipSpecialEscape(src, i)).end;
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
const q = peekQuantifier(src, i + 2);
|
|
182
|
+
i = q.end;
|
|
183
|
+
appendRun(next, q);
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
if (c === "[") {
|
|
187
|
+
const classEnd = skipClass(src, i);
|
|
188
|
+
const chars = parseLiteralClass(src, i, classEnd);
|
|
189
|
+
const q = peekQuantifier(src, classEnd);
|
|
190
|
+
i = q.end;
|
|
191
|
+
if (q.copies === 0 || chars === null || q.copies > 1 || cross.length * chars.length > MAX_DNF) {
|
|
192
|
+
lossy = true;
|
|
193
|
+
flushRuns();
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
cross = cross.flatMap((b) => chars.map((ch) => ({
|
|
197
|
+
literals: b.literals.slice(),
|
|
198
|
+
run: b.run + ch
|
|
199
|
+
})));
|
|
200
|
+
if (q.openEnded) {
|
|
201
|
+
lossy = true;
|
|
202
|
+
flushRuns();
|
|
203
|
+
}
|
|
204
|
+
continue;
|
|
205
|
+
}
|
|
206
|
+
if (c === "(") {
|
|
207
|
+
const groupEnd = skipGroup(src, i);
|
|
208
|
+
const group = parseGroupPrefix(src, i);
|
|
209
|
+
const q = peekQuantifier(src, groupEnd);
|
|
210
|
+
i = q.end;
|
|
211
|
+
if (group.kind !== "consuming" || q.copies === 0) {
|
|
212
|
+
lossy = true;
|
|
213
|
+
flushRuns();
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
const { dnf: inner, exact } = expandRegion(src, group.innerStart, groupEnd - 1);
|
|
217
|
+
const foldable = exact && q.copies === 1 && inner.length > 0 && inner.every((b) => b.length === 1);
|
|
218
|
+
lossy = lossy || q.openEnded || !foldable;
|
|
219
|
+
if (foldable && cross.length * inner.length <= MAX_DNF) {
|
|
220
|
+
cross = cross.flatMap((b) => inner.map((arm) => ({
|
|
221
|
+
literals: b.literals.slice(),
|
|
222
|
+
run: b.run + (arm[0] ?? "")
|
|
223
|
+
})));
|
|
224
|
+
if (q.openEnded) flushRuns();
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
lossy = true;
|
|
228
|
+
flushRuns();
|
|
229
|
+
if (cross.length * inner.length > MAX_DNF) continue;
|
|
230
|
+
cross = cross.flatMap((b) => inner.map((arm) => ({
|
|
231
|
+
literals: b.literals.concat(arm),
|
|
232
|
+
run: ""
|
|
233
|
+
})));
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
if (c === ".") {
|
|
237
|
+
lossy = true;
|
|
238
|
+
flushRuns();
|
|
239
|
+
i = peekQuantifier(src, i + 1).end;
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
const q = peekQuantifier(src, i + 1);
|
|
243
|
+
i = q.end;
|
|
244
|
+
appendRun(c, q);
|
|
245
|
+
}
|
|
246
|
+
flushRuns();
|
|
247
|
+
return {
|
|
248
|
+
dnf: cross.map((b) => b.literals),
|
|
249
|
+
exact: !lossy && cross.every((b) => b.literals.length === 1)
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
/** Where a group's consuming body starts; lookarounds and unknown `(?…)` are not consuming. */
|
|
253
|
+
function parseGroupPrefix(src, start) {
|
|
254
|
+
if (src[start + 1] !== "?") return {
|
|
255
|
+
kind: "consuming",
|
|
256
|
+
innerStart: start + 1
|
|
257
|
+
};
|
|
258
|
+
const marker = src[start + 2];
|
|
259
|
+
if (marker === ":") return {
|
|
260
|
+
kind: "consuming",
|
|
261
|
+
innerStart: start + 3
|
|
262
|
+
};
|
|
263
|
+
if (marker === "=" || marker === "!") return { kind: "assertion" };
|
|
264
|
+
if (marker === "<") {
|
|
265
|
+
const next = src[start + 3];
|
|
266
|
+
if (next === "=" || next === "!") return { kind: "assertion" };
|
|
267
|
+
const nameEnd = src.indexOf(">", start + 3);
|
|
268
|
+
if (nameEnd >= 0) return {
|
|
269
|
+
kind: "consuming",
|
|
270
|
+
innerStart: nameEnd + 1
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
return { kind: "unknown" };
|
|
274
|
+
}
|
|
275
|
+
/**
|
|
276
|
+
* `\xHH`, `\uHHHH`: escapes that stand for one literal character. `\u{…}` is not one:
|
|
277
|
+
* the planner compiles without the `u` flag, where it means "u" repeated or the text "u{…}".
|
|
278
|
+
*/
|
|
279
|
+
function parseEscapedLiteral(src, start) {
|
|
280
|
+
const kind = src[start + 1];
|
|
281
|
+
if (kind === "x") {
|
|
282
|
+
const hex = src.slice(start + 2, start + 4);
|
|
283
|
+
if (!/^[0-9a-fA-F]{2}$/.test(hex)) return void 0;
|
|
284
|
+
return {
|
|
285
|
+
char: String.fromCharCode(Number.parseInt(hex, 16)),
|
|
286
|
+
end: start + 4
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
if (kind === "u") {
|
|
290
|
+
const hex = src.slice(start + 2, start + 6);
|
|
291
|
+
if (!/^[0-9a-fA-F]{4}$/.test(hex)) return void 0;
|
|
292
|
+
return {
|
|
293
|
+
char: String.fromCharCode(Number.parseInt(hex, 16)),
|
|
294
|
+
end: start + 6
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Skip an escape that is not a literal punctuation character, so its payload
|
|
300
|
+
* (`\x41`, `\p{L}`, `\k<name>`, backrefs) never leaks into a literal. Malformed
|
|
301
|
+
* `\x`, `\u`, `\c` payloads are Annex-B literals and are rescanned as atoms.
|
|
302
|
+
*/
|
|
303
|
+
function skipSpecialEscape(src, start) {
|
|
304
|
+
const next = src[start + 1];
|
|
305
|
+
if (next === "x") return /^[0-9a-fA-F]{2}$/.test(src.slice(start + 2, start + 4)) ? start + 4 : start + 2;
|
|
306
|
+
if (next === "u") {
|
|
307
|
+
if (src[start + 2] === "{") {
|
|
308
|
+
const close = src.indexOf("}", start + 3);
|
|
309
|
+
return close >= 0 ? close + 1 : src.length;
|
|
310
|
+
}
|
|
311
|
+
return /^[0-9a-fA-F]{4}$/.test(src.slice(start + 2, start + 6)) ? start + 6 : start + 2;
|
|
312
|
+
}
|
|
313
|
+
if (next === "c") return /^[a-zA-Z]$/.test(src[start + 2] ?? "") ? start + 3 : start + 2;
|
|
314
|
+
if ((next === "p" || next === "P") && src[start + 2] === "{") {
|
|
315
|
+
const close = src.indexOf("}", start + 3);
|
|
316
|
+
return close >= 0 ? close + 1 : src.length;
|
|
317
|
+
}
|
|
318
|
+
if (next === "k" && src[start + 2] === "<") {
|
|
319
|
+
const close = src.indexOf(">", start + 3);
|
|
320
|
+
return close >= 0 ? close + 1 : src.length;
|
|
321
|
+
}
|
|
322
|
+
if (/[0-9]/.test(next ?? "")) {
|
|
323
|
+
let i = start + 2;
|
|
324
|
+
while (/[0-9]/.test(src[i] ?? "")) i += 1;
|
|
325
|
+
return i;
|
|
326
|
+
}
|
|
327
|
+
return Math.min(start + 2, src.length);
|
|
328
|
+
}
|
|
329
|
+
/** A `[...]` class as literal alternatives, or null if negated, ranged, special or empty. */
|
|
330
|
+
function parseLiteralClass(src, start, end) {
|
|
331
|
+
let i = start + 1;
|
|
332
|
+
const lastInside = end - 1;
|
|
333
|
+
if (src[i] === "^") return null;
|
|
334
|
+
const chars = /* @__PURE__ */ new Set();
|
|
335
|
+
while (i < lastInside) {
|
|
336
|
+
const c = src[i] ?? "";
|
|
337
|
+
if (c === "\\") {
|
|
338
|
+
const next = src[i + 1];
|
|
339
|
+
if (next === void 0 || /[a-zA-Z]/.test(next)) return null;
|
|
340
|
+
chars.add(next);
|
|
341
|
+
i += 2;
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
if (i + 2 < lastInside && src[i + 1] === "-") return null;
|
|
345
|
+
chars.add(c);
|
|
346
|
+
i += 1;
|
|
347
|
+
}
|
|
348
|
+
if ([...chars].some((ch) => /[\ud800-\udfff]/.test(ch))) return null;
|
|
349
|
+
return chars.size === 0 ? null : [...chars];
|
|
350
|
+
}
|
|
351
|
+
/** Mandatory copies of the preceding atom, whether more may follow, and where the quantifier ends. */
|
|
352
|
+
function peekQuantifier(src, i) {
|
|
353
|
+
const c = src[i];
|
|
354
|
+
if (c === "?" || c === "*" || c === "+") {
|
|
355
|
+
const end = src[i + 1] === "?" ? i + 2 : i + 1;
|
|
356
|
+
return {
|
|
357
|
+
copies: c === "+" ? 1 : 0,
|
|
358
|
+
openEnded: true,
|
|
359
|
+
end
|
|
360
|
+
};
|
|
361
|
+
}
|
|
362
|
+
if (c === "{") {
|
|
363
|
+
const close = src.indexOf("}", i);
|
|
364
|
+
if (close < 0) return {
|
|
365
|
+
copies: 1,
|
|
366
|
+
openEnded: false,
|
|
367
|
+
end: i
|
|
368
|
+
};
|
|
369
|
+
const m = /^(\d+)(?:,(\d*))?$/.exec(src.slice(i + 1, close));
|
|
370
|
+
if (!m) return {
|
|
371
|
+
copies: 1,
|
|
372
|
+
openEnded: false,
|
|
373
|
+
end: i
|
|
374
|
+
};
|
|
375
|
+
const min = Number(m[1]);
|
|
376
|
+
const max = m[2] === void 0 ? min : m[2] === "" ? Number.POSITIVE_INFINITY : Number(m[2]);
|
|
377
|
+
const end = src[close + 1] === "?" ? close + 2 : close + 1;
|
|
378
|
+
return {
|
|
379
|
+
copies: min,
|
|
380
|
+
openEnded: max > min,
|
|
381
|
+
end
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
return {
|
|
385
|
+
copies: 1,
|
|
386
|
+
openEnded: false,
|
|
387
|
+
end: i
|
|
388
|
+
};
|
|
389
|
+
}
|
|
390
|
+
function skipClass(src, i) {
|
|
391
|
+
let j = i + 1;
|
|
392
|
+
if (src[j] === "^") j += 1;
|
|
393
|
+
while (j < src.length && src[j] !== "]") j += src[j] === "\\" && j + 1 < src.length ? 2 : 1;
|
|
394
|
+
return j < src.length ? j + 1 : j;
|
|
395
|
+
}
|
|
396
|
+
function skipGroup(src, i) {
|
|
397
|
+
let depth = 1;
|
|
398
|
+
let j = i + 1;
|
|
399
|
+
while (j < src.length && depth > 0) {
|
|
400
|
+
const c = src[j];
|
|
401
|
+
if (c === "\\" && j + 1 < src.length) {
|
|
402
|
+
j += 2;
|
|
403
|
+
continue;
|
|
404
|
+
}
|
|
405
|
+
if (c === "[") {
|
|
406
|
+
j = skipClass(src, j);
|
|
407
|
+
continue;
|
|
408
|
+
}
|
|
409
|
+
if (c === "(") depth += 1;
|
|
410
|
+
else if (c === ")") depth -= 1;
|
|
411
|
+
j += 1;
|
|
412
|
+
}
|
|
413
|
+
return j;
|
|
414
|
+
}
|
|
415
|
+
//#endregion
|
|
416
|
+
//#region src/ngram/plan-query.ts
|
|
417
|
+
/** Longer patterns are rejected: a JS regex cannot be interrupted once it runs. */
|
|
418
|
+
const MAX_PATTERN_LENGTH = 1e3;
|
|
419
|
+
function planQuery(pattern, opts = {}) {
|
|
420
|
+
if (pattern.length > 1e3) throw new RangeError(`ngram: pattern is ${pattern.length} characters, the limit is ${MAX_PATTERN_LENGTH}`);
|
|
421
|
+
if (opts.regex && /(?<!\\)(?:\\\\)*\\[pP]\{/.test(pattern)) throw new SyntaxError("ngram: \\p{…} and \\P{…} are not supported in regex mode (patterns compile without the u flag)");
|
|
422
|
+
const source = opts.regex ? pattern : pattern.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&");
|
|
423
|
+
const regex = new RegExp(source, opts.caseSensitive === false ? "gi" : "g");
|
|
424
|
+
const branches = [];
|
|
425
|
+
for (const literals of regexLiterals(regex)) {
|
|
426
|
+
const usable = literals.map(fold).filter((literal) => Array.from(literal).length >= 3);
|
|
427
|
+
if (usable.length === 0) return {
|
|
428
|
+
regex,
|
|
429
|
+
branches: null
|
|
430
|
+
};
|
|
431
|
+
branches.push(usable);
|
|
432
|
+
}
|
|
433
|
+
return {
|
|
434
|
+
regex,
|
|
435
|
+
branches
|
|
436
|
+
};
|
|
437
|
+
}
|
|
438
|
+
//#endregion
|
|
439
|
+
//#region src/ngram/trigrams.ts
|
|
440
|
+
/** Distinct 3-code-point grams of an already folded string, in first-seen order. */
|
|
441
|
+
function gramsOf(folded) {
|
|
442
|
+
const cps = Array.from(folded);
|
|
443
|
+
const out = /* @__PURE__ */ new Set();
|
|
444
|
+
for (let i = 0; i + 3 <= cps.length; i++) out.add(cps.slice(i, i + 3).join(""));
|
|
445
|
+
return [...out];
|
|
446
|
+
}
|
|
447
|
+
/** Distinct trigrams of `text` as the index sees it: case-folded, by code point. */
|
|
448
|
+
function trigrams(text) {
|
|
449
|
+
return gramsOf(fold(text));
|
|
450
|
+
}
|
|
451
|
+
/**
|
|
452
|
+
* One SQLite INTEGER per gram. Three BMP code points pack exactly into 48 bits.
|
|
453
|
+
* A gram with an astral code point keeps the low 16 bits of each code point
|
|
454
|
+
* above 2^48: a collision only adds candidates, which verification removes.
|
|
455
|
+
*/
|
|
456
|
+
function encodeGram(gram) {
|
|
457
|
+
const [a = 0, b = 0, c = 0] = Array.from(gram, (ch) => ch.codePointAt(0) ?? 0);
|
|
458
|
+
if (a < 65536 && b < 65536 && c < 65536) return a * 2 ** 32 + b * 2 ** 16 + c;
|
|
459
|
+
return 2 ** 48 + a % 65536 * 2 ** 32 + b % 65536 * 2 ** 16 + c % 65536;
|
|
460
|
+
}
|
|
461
|
+
//#endregion
|
|
462
|
+
//#region src/ngram/postings-backend.ts
|
|
463
|
+
const PAIR = "json_extract(value, '$[0]'), json_extract(value, '$[1]')";
|
|
464
|
+
/**
|
|
465
|
+
* One row per (gram, id) in a WITHOUT ROWID table: the primary key is the
|
|
466
|
+
* posting list, clustered by gram. A text is a candidate for a branch when it
|
|
467
|
+
* holds every distinct gram of the branch's literals.
|
|
468
|
+
*/
|
|
469
|
+
function newPostingsBackend(prefix) {
|
|
470
|
+
const table = `${prefix}grams`;
|
|
471
|
+
const pairs = (items) => chunkJson(items.flatMap((item) => trigrams(item.text).map((gram) => [encodeGram(gram), item.id])));
|
|
472
|
+
return {
|
|
473
|
+
create: (tx) => tx.exec(`CREATE TABLE IF NOT EXISTS ${table}(gram INTEGER NOT NULL, id INTEGER NOT NULL, PRIMARY KEY (gram, id)) WITHOUT ROWID`),
|
|
474
|
+
async add(tx, items) {
|
|
475
|
+
for (const chunk of pairs(items)) await tx.query(`INSERT OR IGNORE INTO ${table}(gram, id) SELECT ${PAIR} FROM json_each(?)`, [chunk]);
|
|
476
|
+
},
|
|
477
|
+
async remove(tx, items) {
|
|
478
|
+
for (const chunk of pairs(items)) await tx.query(`DELETE FROM ${table} WHERE (gram, id) IN (SELECT ${PAIR} FROM json_each(?))`, [chunk]);
|
|
479
|
+
},
|
|
480
|
+
async candidates(db, branches, max) {
|
|
481
|
+
const ids = /* @__PURE__ */ new Set();
|
|
482
|
+
for (const branch of branches) {
|
|
483
|
+
const grams = [...new Set(branch.flatMap((literal) => gramsOf(literal).map(encodeGram)))];
|
|
484
|
+
const rows = await db.query(`SELECT id FROM ${table} WHERE gram IN (SELECT value FROM json_each(?)) GROUP BY id HAVING count(*) = ? ORDER BY id LIMIT ?`, [
|
|
485
|
+
JSON.stringify(grams),
|
|
486
|
+
grams.length,
|
|
487
|
+
max
|
|
488
|
+
]);
|
|
489
|
+
for (const row of rows) ids.add(Number(row.id));
|
|
490
|
+
}
|
|
491
|
+
return [...ids].sort((a, b) => a - b).slice(0, max);
|
|
492
|
+
}
|
|
493
|
+
};
|
|
494
|
+
}
|
|
495
|
+
//#endregion
|
|
496
|
+
//#region src/ngram/ngram-store.ts
|
|
497
|
+
const VERIFY_BATCH = 256;
|
|
498
|
+
var NgramStore = class {
|
|
499
|
+
/** `{prefix}texts`: its `id` column holds exactly the stored ids. Callers may join it in SQL. */
|
|
500
|
+
idTable;
|
|
501
|
+
#db;
|
|
502
|
+
#prefix;
|
|
503
|
+
#store;
|
|
504
|
+
#backend;
|
|
505
|
+
constructor({ db, prefix, store = "fts5" }) {
|
|
506
|
+
if (!PREFIX_PATTERN.test(prefix)) throw new Error(`ngram: prefix "${prefix}" must match ${PREFIX_PATTERN}`);
|
|
507
|
+
if (store !== "fts5" && store !== "postings") throw new Error(`ngram: unknown store "${store}"`);
|
|
508
|
+
this.#db = db;
|
|
509
|
+
this.#prefix = prefix;
|
|
510
|
+
this.idTable = `${prefix}texts`;
|
|
511
|
+
this.#store = store;
|
|
512
|
+
this.#backend = store === "fts5" ? newFts5Backend(prefix) : newPostingsBackend(prefix);
|
|
513
|
+
}
|
|
514
|
+
async init() {
|
|
515
|
+
const p = this.#prefix;
|
|
516
|
+
await inTransaction(this.#db, async (tx) => {
|
|
517
|
+
await tx.exec(`CREATE TABLE IF NOT EXISTS ${p}meta(key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
|
|
518
|
+
const [row] = await tx.query(`SELECT value FROM ${p}meta WHERE key = 'store'`);
|
|
519
|
+
if (row && row.value !== this.#store) throw new Error(`ngram: index "${p}" was created with store "${row.value}", opened with "${this.#store}"`);
|
|
520
|
+
if (!row) await tx.query(`INSERT INTO ${p}meta(key, value) VALUES ('store', ?)`, [this.#store]);
|
|
521
|
+
await tx.exec(`CREATE TABLE IF NOT EXISTS ${p}texts(id INTEGER PRIMARY KEY, text TEXT NOT NULL)`);
|
|
522
|
+
await this.#backend.create(tx);
|
|
523
|
+
});
|
|
524
|
+
}
|
|
525
|
+
async upsert(items) {
|
|
526
|
+
const latest = [...new Map(items.map(({ id, text }) => [id, {
|
|
527
|
+
id,
|
|
528
|
+
text: text.toWellFormed()
|
|
529
|
+
}])).values()];
|
|
530
|
+
if (latest.length === 0) return;
|
|
531
|
+
await inTransaction(this.#db, async (tx) => {
|
|
532
|
+
await this.#backend.remove(tx, await this.#load(tx, latest.map((item) => item.id)));
|
|
533
|
+
for (const chunk of chunkJson(latest.map((item) => [item.id, item.text]))) await tx.query(`INSERT OR REPLACE INTO ${this.#prefix}texts(id, text) SELECT json_extract(value, '$[0]'), json_extract(value, '$[1]') FROM json_each(?)`, [chunk]);
|
|
534
|
+
await this.#backend.add(tx, latest);
|
|
535
|
+
});
|
|
536
|
+
}
|
|
537
|
+
async delete(ids) {
|
|
538
|
+
if (ids.length === 0) return;
|
|
539
|
+
await inTransaction(this.#db, async (tx) => {
|
|
540
|
+
await this.#backend.remove(tx, await this.#load(tx, ids));
|
|
541
|
+
for (const chunk of chunkJson(ids)) await tx.query(`DELETE FROM ${this.#prefix}texts WHERE id IN (SELECT value FROM json_each(?))`, [chunk]);
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
async search(pattern, opts = {}) {
|
|
545
|
+
const { limit = 100, candidateBudget = 1e4, filter, signal } = opts;
|
|
546
|
+
const plan = planQuery(pattern, {
|
|
547
|
+
regex: opts.regex,
|
|
548
|
+
caseSensitive: opts.caseSensitive ?? true
|
|
549
|
+
});
|
|
550
|
+
signal?.throwIfAborted();
|
|
551
|
+
if (limit <= 0) return {
|
|
552
|
+
hits: [],
|
|
553
|
+
truncated: true
|
|
554
|
+
};
|
|
555
|
+
const found = plan.branches === null ? (await this.#db.query(`SELECT id FROM ${this.#prefix}texts ORDER BY id LIMIT ?`, [candidateBudget + 1])).map((row) => Number(row.id)) : await this.#backend.candidates(this.#db, plan.branches, candidateBudget + 1);
|
|
556
|
+
let truncated = found.length > candidateBudget;
|
|
557
|
+
const ids = found.slice(0, candidateBudget).filter((id) => !filter || filter(id));
|
|
558
|
+
const hits = [];
|
|
559
|
+
let last;
|
|
560
|
+
verify: for (let i = 0; i < ids.length; i += VERIFY_BATCH) {
|
|
561
|
+
signal?.throwIfAborted();
|
|
562
|
+
for (const { id, text } of await this.#load(this.#db, ids.slice(i, i + VERIFY_BATCH))) {
|
|
563
|
+
last = id;
|
|
564
|
+
const matches = [];
|
|
565
|
+
for (const m of text.matchAll(plan.regex)) if (m[0].length > 0) matches.push({
|
|
566
|
+
start: m.index,
|
|
567
|
+
end: m.index + m[0].length
|
|
568
|
+
});
|
|
569
|
+
if (matches.length === 0) continue;
|
|
570
|
+
hits.push({
|
|
571
|
+
id,
|
|
572
|
+
matches
|
|
573
|
+
});
|
|
574
|
+
if (hits.length >= limit) break verify;
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
if (hits.length >= limit && last !== ids.at(-1)) truncated = true;
|
|
578
|
+
return {
|
|
579
|
+
hits,
|
|
580
|
+
truncated
|
|
581
|
+
};
|
|
582
|
+
}
|
|
583
|
+
/** The stored original texts of `ids`; unknown ids are absent. */
|
|
584
|
+
async getTexts(ids) {
|
|
585
|
+
const items = await this.#load(this.#db, ids);
|
|
586
|
+
return new Map(items.map((item) => [item.id, item.text]));
|
|
587
|
+
}
|
|
588
|
+
/** Drops the tables of both stores, so it works without knowing which one created the prefix. */
|
|
589
|
+
async drop() {
|
|
590
|
+
const p = this.#prefix;
|
|
591
|
+
await inTransaction(this.#db, async (tx) => {
|
|
592
|
+
for (const t of [
|
|
593
|
+
"ng",
|
|
594
|
+
"grams",
|
|
595
|
+
"texts",
|
|
596
|
+
"meta"
|
|
597
|
+
]) await tx.exec(`DROP TABLE IF EXISTS ${p}${t}`);
|
|
598
|
+
});
|
|
599
|
+
}
|
|
600
|
+
/** Stored texts for `ids`, ascending by id; missing ids are skipped. */
|
|
601
|
+
async #load(db, ids) {
|
|
602
|
+
const out = [];
|
|
603
|
+
for (const chunk of chunkJson([...ids].sort((a, b) => a - b))) {
|
|
604
|
+
const rows = await db.query(`SELECT id, text FROM ${this.#prefix}texts WHERE id IN (SELECT value FROM json_each(?)) ORDER BY id`, [chunk]);
|
|
605
|
+
for (const row of rows) out.push({
|
|
606
|
+
id: Number(row.id),
|
|
607
|
+
text: String(row.text)
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
return out;
|
|
611
|
+
}
|
|
612
|
+
};
|
|
613
|
+
//#endregion
|
|
614
|
+
export { MAX_PATTERN_LENGTH, NgramStore, fold, planQuery, trigrams };
|
|
615
|
+
|
|
616
|
+
//# sourceMappingURL=ngram.mjs.map
|