@statewalker/db-sqlite-index 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -4
- package/dist/ngram.d.mts.map +1 -1
- package/dist/ngram.mjs +83 -3
- package/dist/ngram.mjs.map +1 -1
- package/dist/vector.d.mts +4 -1
- package/dist/vector.d.mts.map +1 -1
- package/dist/vector.mjs +115 -22
- package/dist/vector.mjs.map +1 -1
- package/package.json +16 -8
- package/src/ngram/fold.ts +44 -0
- package/src/ngram/fts5-backend.ts +4 -0
- package/src/ngram/ngram-store.ts +37 -0
- package/src/ngram/plan-query.ts +8 -1
- package/src/ngram/postings-backend.ts +1 -0
- package/src/ngram/regex-literals.ts +5 -2
- package/src/vector/math.ts +6 -1
- package/src/vector/segment.ts +48 -6
- package/src/vector/types.ts +4 -1
- package/src/vector/vector-index.ts +62 -16
- package/src/ngram/do-smoke.ts +0 -46
- package/src/ngram/fold.test.ts +0 -50
- package/src/ngram/json-chunks.test.ts +0 -23
- package/src/ngram/ngram-store.test.ts +0 -291
- package/src/ngram/plan-query.test.ts +0 -141
- package/src/ngram/trigrams.test.ts +0 -28
- package/src/vector/auto-compact.test.ts +0 -28
- package/src/vector/do-smoke.ts +0 -39
- package/src/vector/segment.test.ts +0 -61
- package/src/vector/test-fixtures.ts +0 -64
- package/src/vector/vector-index.test.ts +0 -105
- package/src/vector/vector-lifecycle.test.ts +0 -171
package/README.md
CHANGED
|
@@ -45,7 +45,12 @@ const hits = await index.search(query, { k: 10 }); // [{ id, distance }], lower
|
|
|
45
45
|
```
|
|
46
46
|
|
|
47
47
|
`metric` is `"cosine"` (default), `"dot"` or `"l2"`. `search` defaults to `k: 10`,
|
|
48
|
-
`
|
|
48
|
+
`oversample: 8`, `exact: false`, and probes at least 8 lists per segment, more when lists are
|
|
49
|
+
small (see [Search narrows candidates three times](#search-narrows-candidates-three-times));
|
|
50
|
+
an explicit `nprobe` probes exactly that many. It also takes an `AbortSignal` as `signal`
|
|
51
|
+
(an already-aborted signal rejects at once). Under `"cosine"`, `upsert` and `search` reject a
|
|
52
|
+
zero vector (`vector-index: a zero vector has no direction under the cosine metric`): it has no
|
|
53
|
+
direction, so it would sit at the same distance from everything.
|
|
49
54
|
`upsert`, `delete`, `search`, `getVectors`, `compact`, `merge` and `optimize` reject with
|
|
50
55
|
`vector-index: call init() first` until `init()` ran.
|
|
51
56
|
|
|
@@ -119,11 +124,24 @@ one and drops dead rows.
|
|
|
119
124
|
|
|
120
125
|
### Search narrows candidates three times
|
|
121
126
|
|
|
122
|
-
1. Probe the
|
|
127
|
+
1. Probe the lists whose centroids are closest to the query. An explicit `nprobe` probes exactly
|
|
128
|
+
that many. Without one, probe at least 8 and keep adding lists, in centroid order, until the
|
|
129
|
+
probed lists hold `16 × k × oversample` rows (1280 at the defaults), or a third of the
|
|
130
|
+
segment's lists are probed, so a large `k` never scans every list. Small
|
|
131
|
+
lists need this: a 4200-vector segment from auto-compaction has about 65 lists of about 65
|
|
132
|
+
rows, and on data with few natural clusters neighbours spread over many of them (recall@10
|
|
133
|
+
was about 0.7 with 8 lists, 0.94 or more with the adaptive count). Segments with large lists
|
|
134
|
+
still probe 8. The widening costs time on data that did not need it: on 5 auto-compacted
|
|
135
|
+
segments of 40-cluster data, a default search took about 14 ms against 6 ms with
|
|
136
|
+
`nprobe: 8` (for k = 10 to 100) at the same recall. Pass `nprobe` to fix the cost.
|
|
123
137
|
2. A 1-bit filter keeps `k × oversample` rows per list (`oversample` default 8). It
|
|
124
138
|
estimates the real metric from the signs of each vector's residual to its centroid, plus the
|
|
125
139
|
residual's norm and its projection on the centroid (RaBitQ-style). This makes
|
|
126
|
-
it accurate for `dot` and `l2` too, not only `cosine`.
|
|
140
|
+
it accurate for `dot` and `l2` too, not only `cosine`. A list with no more rows than that
|
|
141
|
+
passes whole. So does a flat segment (one list, below 2048 vectors): its centroid is the
|
|
142
|
+
mean of all vectors, so the 1-bit estimate is weak on clustered data (recall@10 fell to
|
|
143
|
+
about 0.5 on 1500 vectors in 3 clusters), and scoring every row with int8 codes costs no
|
|
144
|
+
more than scanning the tail.
|
|
127
145
|
3. Re-rank the survivors with int8 codes. With `exact: true`, re-rank the top
|
|
128
146
|
`4k` again with the stored float32 vectors.
|
|
129
147
|
|
|
@@ -166,6 +184,12 @@ never modified.
|
|
|
166
184
|
Within an instance, writes are serialised. Another writer's changes become
|
|
167
185
|
visible after `refresh()`.
|
|
168
186
|
- **`merge()` holds every live segment vector in memory** while it runs.
|
|
187
|
+
- **Heavy deletes or overwrites call for `merge()`.** Dead segment rows stay in
|
|
188
|
+
their lists until `merge()` drops them. Search skips them, filtering a list
|
|
189
|
+
again with a doubled limit when dead rows took candidate slots, so `k` results
|
|
190
|
+
still come back; but the extra passes and liveness lookups cost time, and the
|
|
191
|
+
centroids still reflect the old data. After deleting or overwriting a large
|
|
192
|
+
share of a segment, call `merge()`.
|
|
169
193
|
- **Opening existing tables with different `dims` or `metric` fails** with
|
|
170
194
|
`vector-index: tables were created with {"dims":…,"metric":…}, not …`.
|
|
171
195
|
- **A search or `getVectors` that overlaps a `compact()` or `merge()` commit runs
|
|
@@ -241,6 +265,35 @@ case-insensitive regex never misses a text because of a case form (`Σ σ ς`, `
|
|
|
241
265
|
The store is recorded at `init`; opening the same prefix with the other store fails
|
|
242
266
|
with `ngram: index "<prefix>" was created with store "fts5", opened with "postings"`.
|
|
243
267
|
|
|
268
|
+
## A runtime with different case mappings rebuilds the grams at `init`
|
|
269
|
+
|
|
270
|
+
`fold` uses the runtime's `toUpperCase` and `toLowerCase`, so a Node, browser or V8 upgrade
|
|
271
|
+
that ships a newer Unicode can fold some characters differently from the runtime that wrote
|
|
272
|
+
the index. The fts5 store deletes a row by replaying its folded text, and a replay that does
|
|
273
|
+
not match the inserted tokens corrupts a contentless FTS5 table; the postings store would keep
|
|
274
|
+
stale grams. So `init` compares the runtime's fold identity with the one recorded in
|
|
275
|
+
`{p}meta` (key `fold`):
|
|
276
|
+
|
|
277
|
+
- **Identity**: `fold-v1:<hash>`, a 53-bit hash of every code point up to U+1FFFF that `fold`
|
|
278
|
+
changes and of what it changes it to (no cased character lies above U+1FFFF). It uses no
|
|
279
|
+
runtime API, so it is the same mechanism in Node, browsers and Durable Objects. It is
|
|
280
|
+
computed once per process, in about 30 ms.
|
|
281
|
+
- **New index**: the identity is recorded.
|
|
282
|
+
- **Same identity**: nothing happens.
|
|
283
|
+
- **Different identity**: inside `init`'s transaction, before any write, every gram is
|
|
284
|
+
removed (fts5 `'delete-all'`, since a contentless table cannot `'rebuild'`; postings:
|
|
285
|
+
every row deleted) and re-added from `{p}texts`, 256 texts per step; then the new identity
|
|
286
|
+
is recorded. The rebuild is one transaction, so on a Durable Object a large index can
|
|
287
|
+
exceed the 15 s operation deadline; the transaction then rolls back, the index is left
|
|
288
|
+
as it was, and the next `init` tries again.
|
|
289
|
+
- **No recorded identity** (an index written by a version before 0.1.2): the current
|
|
290
|
+
identity is recorded and nothing is rebuilt. This assumes the index was written by a
|
|
291
|
+
runtime that folds like the current one; if that may not hold, `drop()` the index and
|
|
292
|
+
re-upsert its texts.
|
|
293
|
+
|
|
294
|
+
Rolling back to a version without this check is safe: it never reads the `fold` key and
|
|
295
|
+
ignores it.
|
|
296
|
+
|
|
244
297
|
## What will surprise you in `./ngram`
|
|
245
298
|
|
|
246
299
|
- `search` stops at `limit` hits (default 100) and verifies at most
|
|
@@ -270,7 +323,7 @@ with `ngram: index "<prefix>" was created with store "fts5", opened with "postin
|
|
|
270
323
|
|
|
271
324
|
## Internals of `./ngram`
|
|
272
325
|
|
|
273
|
-
- Tables per prefix `p`: `{p}meta
|
|
326
|
+
- Tables per prefix `p`: `{p}meta` (keys `store` and `fold`), `{p}texts(id, text)`, and `{p}ng` (fts5,
|
|
274
327
|
contentless, `tokenize='trigram case_sensitive 1'`) or `{p}grams(gram, id)`.
|
|
275
328
|
- The FTS5 table is contentless: the text is kept once, in `{p}texts`, and a row is
|
|
276
329
|
deleted by replaying its indexed value — so writes read the previous text first.
|
|
@@ -285,6 +338,15 @@ with `ngram: index "<prefix>" was created with store "fts5", opened with "postin
|
|
|
285
338
|
- Dependencies: `@statewalker/db-api`, `@statewalker/db-sqlite`. The planner, fold
|
|
286
339
|
and trigram code import neither.
|
|
287
340
|
|
|
341
|
+
## How it is tested against a real Durable Object
|
|
342
|
+
|
|
343
|
+
`pnpm test` runs every suite on `node:sqlite` and on `@statewalker/db-sqlite-do`'s
|
|
344
|
+
`node:sqlite` fake. `pnpm test:do` (from the repository root, after a build) starts `celld dev`
|
|
345
|
+
on `packages/db-sqlite-index/test-do/` and runs the smoke scenarios of `test-do/scenarios.ts`
|
|
346
|
+
(multi-part segment rows, a 150-id delete, the n-gram stores' writes and drops inside a
|
|
347
|
+
transaction) inside a real Durable Object. It needs celld 0.6.x on `PATH`. The scenarios live
|
|
348
|
+
outside `src/`, so they are not published.
|
|
349
|
+
|
|
288
350
|
## License
|
|
289
351
|
|
|
290
352
|
MIT
|
package/dist/ngram.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ngram.d.mts","names":[],"sources":["../src/ngram/fold.ts","../src/ngram/fts5-backend.ts","../src/ngram/ngram-store.ts","../src/ngram/plan-query.ts","../src/ngram/trigrams.ts"],"mappings":";;;;;;;;;;;;;;wBAYgB,KAAK;;;KCRT;EAAc;EAAY;;;;
|
|
1
|
+
{"version":3,"file":"ngram.d.mts","names":[],"sources":["../src/ngram/fold.ts","../src/ngram/fts5-backend.ts","../src/ngram/ngram-store.ts","../src/ngram/plan-query.ts","../src/ngram/trigrams.ts"],"mappings":";;;;;;;;;;;;;;wBAYgB,KAAK;;;KCRT;EAAc;EAAY;;;;KCM1B;EAAe;EAAe;;KAC9B;EAAa;EAAY,SAAS;;KAClC;;EAEV;;EAEA;;EAEA;;EAEA;;EAEA,UAAU;EACV,SAAS;;KAEC;;EAEV,MAAM;;EAEN;;KAEU;EAAsB,IAAI;EAAI;EAAgB;;qBAM7C;;;WAEF;EAMT,cAAc,IAAI,QAAQ,SAAkB;EAatC,QAAQ;EAyDR,OAAO,OAAO,cAAc;EA0B5B,OAAO,gBAAgB;EAavB,OAAO,iBAAiB,OAAM,qBAA0B,QAAQ;;EAwChE,SAAS,gBAAgB,QAAQ;;EAMjC,QAAQ;;;;;qBCpMH;KAID;;EAEV,OAAO;;;;;EAKP;;wBAGc,UACd,iBACA;EAAQ;EAAiB;IACxB;;;;wBCVa,SAAS"}
|
package/dist/ngram.mjs
CHANGED
|
@@ -22,6 +22,47 @@ function fold(text) {
|
|
|
22
22
|
}
|
|
23
23
|
return out;
|
|
24
24
|
}
|
|
25
|
+
/**
|
|
26
|
+
* Highest code point probed by `foldIdentity`. Every cased character is in the
|
|
27
|
+
* BMP or the SMP (Deseret, Osage, Adlam, …); planes 2 and up hold none.
|
|
28
|
+
*/
|
|
29
|
+
const PROBE_END = 131071;
|
|
30
|
+
const IDENTITY_VERSION = "fold-v1";
|
|
31
|
+
let current;
|
|
32
|
+
/**
|
|
33
|
+
* A fingerprint of what `foldFn` does to every code point up to U+1FFFF: a
|
|
34
|
+
* 53-bit hash of each code point it changes and what it changes it to. The
|
|
35
|
+
* grams of an n-gram index are built with `fold`, and a contentless FTS5 row is
|
|
36
|
+
* deleted by replaying them, so a runtime whose case mappings differ (a newer
|
|
37
|
+
* Unicode in a newer V8 or ICU) yields a different identity, and the store
|
|
38
|
+
* rebuilds its grams. It needs no runtime API, so it works in Node, browsers
|
|
39
|
+
* and Durable Objects alike. The identity of `fold` is computed once (about
|
|
40
|
+
* 30 ms) and memoized.
|
|
41
|
+
*/
|
|
42
|
+
function foldIdentity(foldFn = fold) {
|
|
43
|
+
if (foldFn === fold && current !== void 0) return current;
|
|
44
|
+
let h1 = 3735928559;
|
|
45
|
+
let h2 = 1103547991;
|
|
46
|
+
const mix = (value) => {
|
|
47
|
+
h1 = Math.imul(h1 ^ value, 2654435761);
|
|
48
|
+
h2 = Math.imul(h2 ^ value, 1597334677);
|
|
49
|
+
};
|
|
50
|
+
for (let cp = 0; cp <= PROBE_END; cp++) {
|
|
51
|
+
if (cp >= 55296 && cp <= 57343) continue;
|
|
52
|
+
const ch = String.fromCodePoint(cp);
|
|
53
|
+
const folded = foldFn(ch);
|
|
54
|
+
if (folded === ch) continue;
|
|
55
|
+
mix(cp);
|
|
56
|
+
for (let i = 0; i < folded.length; i++) mix(folded.charCodeAt(i));
|
|
57
|
+
mix(1114112);
|
|
58
|
+
}
|
|
59
|
+
h1 = Math.imul(h1 ^ h1 >>> 16, 2246822507) ^ Math.imul(h2 ^ h2 >>> 13, 3266489909);
|
|
60
|
+
h2 = Math.imul(h2 ^ h2 >>> 16, 2246822507) ^ Math.imul(h1 ^ h1 >>> 13, 3266489909);
|
|
61
|
+
const hash = 4294967296 * (2097151 & h2) + (h1 >>> 0);
|
|
62
|
+
const identity = `${IDENTITY_VERSION}:${hash.toString(16)}`;
|
|
63
|
+
if (foldFn === fold) current = identity;
|
|
64
|
+
return identity;
|
|
65
|
+
}
|
|
25
66
|
//#endregion
|
|
26
67
|
//#region src/ngram/json-chunks.ts
|
|
27
68
|
/**
|
|
@@ -67,6 +108,7 @@ function newFts5Backend(prefix) {
|
|
|
67
108
|
create: (tx) => tx.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${table} USING fts5(text, content='', tokenize='trigram case_sensitive 1')`),
|
|
68
109
|
add: (tx, items) => write(tx, items, "insert"),
|
|
69
110
|
remove: (tx, items) => write(tx, items, "delete"),
|
|
111
|
+
clear: (tx) => tx.exec(`INSERT INTO ${table}(${table}) VALUES ('delete-all')`),
|
|
70
112
|
async candidates(db, branches, max) {
|
|
71
113
|
const quote = (literal) => `"${literal.replaceAll("\"", "\"\"")}"`;
|
|
72
114
|
const match = branches.map((branch) => `(${branch.map(quote).join(" AND ")})`).join(" OR ");
|
|
@@ -336,9 +378,10 @@ function parseLiteralClass(src, start, end) {
|
|
|
336
378
|
const c = src[i] ?? "";
|
|
337
379
|
if (c === "\\") {
|
|
338
380
|
const next = src[i + 1];
|
|
339
|
-
if (next === void 0 || /[a-zA-
|
|
340
|
-
chars.add(next);
|
|
381
|
+
if (next === void 0 || /[a-zA-Z0-9]/.test(next)) return null;
|
|
341
382
|
i += 2;
|
|
383
|
+
if (src[i] === "-" && i + 1 < lastInside) return null;
|
|
384
|
+
chars.add(next);
|
|
342
385
|
continue;
|
|
343
386
|
}
|
|
344
387
|
if (i + 2 < lastInside && src[i + 1] === "-") return null;
|
|
@@ -416,6 +459,7 @@ function skipGroup(src, i) {
|
|
|
416
459
|
//#region src/ngram/plan-query.ts
|
|
417
460
|
/** Longer patterns are rejected: a JS regex cannot be interrupted once it runs. */
|
|
418
461
|
const MAX_PATTERN_LENGTH = 1e3;
|
|
462
|
+
const LONE_SURROGATE = /[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/;
|
|
419
463
|
function planQuery(pattern, opts = {}) {
|
|
420
464
|
if (pattern.length > 1e3) throw new RangeError(`ngram: pattern is ${pattern.length} characters, the limit is ${MAX_PATTERN_LENGTH}`);
|
|
421
465
|
if (opts.regex && /(?<!\\)(?:\\\\)*\\[pP]\{/.test(pattern)) throw new SyntaxError("ngram: \\p{…} and \\P{…} are not supported in regex mode (patterns compile without the u flag)");
|
|
@@ -423,7 +467,7 @@ function planQuery(pattern, opts = {}) {
|
|
|
423
467
|
const regex = new RegExp(source, opts.caseSensitive === false ? "gi" : "g");
|
|
424
468
|
const branches = [];
|
|
425
469
|
for (const literals of regexLiterals(regex)) {
|
|
426
|
-
const usable = literals.map(fold).filter((literal) => Array.from(literal).length >= 3);
|
|
470
|
+
const usable = literals.flatMap((literal) => literal.split(LONE_SURROGATE)).map(fold).filter((literal) => Array.from(literal).length >= 3);
|
|
427
471
|
if (usable.length === 0) return {
|
|
428
472
|
regex,
|
|
429
473
|
branches: null
|
|
@@ -477,6 +521,7 @@ function newPostingsBackend(prefix) {
|
|
|
477
521
|
async remove(tx, items) {
|
|
478
522
|
for (const chunk of pairs(items)) await tx.query(`DELETE FROM ${table} WHERE (gram, id) IN (SELECT ${PAIR} FROM json_each(?))`, [chunk]);
|
|
479
523
|
},
|
|
524
|
+
clear: (tx) => tx.exec(`DELETE FROM ${table}`),
|
|
480
525
|
async candidates(db, branches, max) {
|
|
481
526
|
const ids = /* @__PURE__ */ new Set();
|
|
482
527
|
for (const branch of branches) {
|
|
@@ -495,6 +540,8 @@ function newPostingsBackend(prefix) {
|
|
|
495
540
|
//#endregion
|
|
496
541
|
//#region src/ngram/ngram-store.ts
|
|
497
542
|
const VERIFY_BATCH = 256;
|
|
543
|
+
/** Texts re-indexed per step when the grams are rebuilt after a fold change. */
|
|
544
|
+
const REBUILD_BATCH = 256;
|
|
498
545
|
var NgramStore = class {
|
|
499
546
|
/** `{prefix}texts`: its `id` column holds exactly the stored ids. Callers may join it in SQL. */
|
|
500
547
|
idTable;
|
|
@@ -520,8 +567,41 @@ var NgramStore = class {
|
|
|
520
567
|
if (!row) await tx.query(`INSERT INTO ${p}meta(key, value) VALUES ('store', ?)`, [this.#store]);
|
|
521
568
|
await tx.exec(`CREATE TABLE IF NOT EXISTS ${p}texts(id INTEGER PRIMARY KEY, text TEXT NOT NULL)`);
|
|
522
569
|
await this.#backend.create(tx);
|
|
570
|
+
await this.#checkFold(tx);
|
|
523
571
|
});
|
|
524
572
|
}
|
|
573
|
+
/**
|
|
574
|
+
* The grams were built with `fold` as the runtime that wrote them defined it, and
|
|
575
|
+
* an fts5 delete replays them; if this runtime folds differently, they are rebuilt
|
|
576
|
+
* from the stored texts before any write. An index without a recorded identity
|
|
577
|
+
* (written before it was recorded) is assumed to match and is not rebuilt.
|
|
578
|
+
*/
|
|
579
|
+
async #checkFold(tx) {
|
|
580
|
+
const p = this.#prefix;
|
|
581
|
+
const identity = foldIdentity();
|
|
582
|
+
const [row] = await tx.query(`SELECT value FROM ${p}meta WHERE key = 'fold'`);
|
|
583
|
+
if (row?.value === identity) return;
|
|
584
|
+
if (row) await this.#rebuild(tx);
|
|
585
|
+
await tx.query(`INSERT OR REPLACE INTO ${p}meta(key, value) VALUES ('fold', ?)`, [identity]);
|
|
586
|
+
}
|
|
587
|
+
async #rebuild(tx) {
|
|
588
|
+
await this.#backend.clear(tx);
|
|
589
|
+
let after = null;
|
|
590
|
+
for (;;) {
|
|
591
|
+
const batch = (await tx.query(`SELECT id, text FROM ${this.#prefix}texts WHERE ? IS NULL OR id > ? ORDER BY id LIMIT ?`, [
|
|
592
|
+
after,
|
|
593
|
+
after,
|
|
594
|
+
REBUILD_BATCH
|
|
595
|
+
])).map((r) => ({
|
|
596
|
+
id: Number(r.id),
|
|
597
|
+
text: String(r.text)
|
|
598
|
+
}));
|
|
599
|
+
const last = batch.at(-1);
|
|
600
|
+
if (!last) return;
|
|
601
|
+
await this.#backend.add(tx, batch);
|
|
602
|
+
after = last.id;
|
|
603
|
+
}
|
|
604
|
+
}
|
|
525
605
|
async upsert(items) {
|
|
526
606
|
const latest = [...new Map(items.map(({ id, text }) => [id, {
|
|
527
607
|
id,
|
package/dist/ngram.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ngram.mjs","names":[],"sources":["../src/ngram/fold.ts","../src/ngram/json-chunks.ts","../src/ngram/fts5-backend.ts","../src/ngram/regex-literals.ts","../src/ngram/plan-query.ts","../src/ngram/trigrams.ts","../src/ngram/postings-backend.ts","../src/ngram/ngram-store.ts"],"sourcesContent":["/**\n * Case fold per code point, applied to indexed text and to query literals alike.\n *\n * Upper-then-lower maps every character a case-insensitive JS regex treats as\n * equal to one form (Σ σ ς → σ, ſ → s). A character whose upper case is several\n * code points (ß → SS) only matches itself under `/i`, so it keeps its own\n * lower case. Folding per code point makes fold(a + b) === fold(a) + fold(b), so\n * a substring of the text stays a substring of the folded text. A character\n * whose folded form has a different UTF-16 length (İ → i + U+0307) is kept as\n * is, so fold preserves the length and offsets into the folded text are\n * offsets into the original.\n */\nexport function fold(text: string): string {\n let out = \"\";\n for (const ch of text) {\n const upper = ch.toUpperCase();\n const lower = (Array.from(upper).length === 1 ? upper : ch).toLowerCase();\n out += lower.length === ch.length ? lower : ch;\n }\n return out;\n}\n","/**\n * Serialises rows into JSON array parameters for `json_each(?)`, each at most\n * `maxChars` UTF-16 code units. 500k code units stay under a Durable Object's\n * ~2 MB string limit even at 4 bytes per code point; a single row larger than\n * that still fails there with \"string or blob too big\".\n */\nexport function chunkJson(rows: unknown[], maxChars = 500_000): string[] {\n const chunks: string[] = [];\n let current: string[] = [];\n let size = 2;\n for (const row of rows) {\n const json = JSON.stringify(row);\n if (current.length > 0 && size + json.length + 1 > maxChars) {\n chunks.push(`[${current.join(\",\")}]`);\n current = [];\n size = 2;\n }\n current.push(json);\n size += json.length + 1;\n }\n if (current.length > 0) chunks.push(`[${current.join(\",\")}]`);\n return chunks;\n}\n","import type { Db } from \"@statewalker/db-api\";\nimport { fold } from \"./fold.js\";\nimport { chunkJson } from \"./json-chunks.js\";\n\nexport type NgramItem = { id: number; text: string };\n\n/** How one store finds candidate ids; texts and verification live in NgramStore. */\nexport type GramBackend = {\n create(tx: Db): Promise<void>;\n add(tx: Db, items: NgramItem[]): Promise<void>;\n /** `items` hold the texts as currently stored, so their grams can be removed exactly. */\n remove(tx: Db, items: NgramItem[]): Promise<void>;\n /** Ids whose text contains every literal of at least one branch; ascending, at most `max`. */\n candidates(db: Db, branches: string[][], max: number): Promise<number[]>;\n};\n\nconst ROW = \"json_extract(value, '$[0]'), json_extract(value, '$[1]')\";\n\n/**\n * FTS5 trigram over folded text. Contentless (`content=''`), because NgramStore\n * already keeps the text; a contentless row is deleted by replaying its exact\n * indexed value through the 'delete' command, which is why `remove` needs the\n * stored texts. `case_sensitive 1`: folding is done by `fold`, the same function\n * that folds query literals, so the index and the planner can never disagree.\n */\nexport function newFts5Backend(prefix: string): GramBackend {\n const table = `${prefix}ng`;\n const write = async (tx: Db, items: NgramItem[], command: \"insert\" | \"delete\") => {\n const sql =\n command === \"insert\"\n ? `INSERT INTO ${table}(rowid, text) SELECT ${ROW} FROM json_each(?)`\n : `INSERT INTO ${table}(${table}, rowid, text) SELECT 'delete', ${ROW} FROM json_each(?)`;\n for (const chunk of chunkJson(items.map((item) => [item.id, fold(item.text)]))) {\n await tx.query(sql, [chunk]);\n }\n };\n return {\n create: (tx) =>\n tx.exec(\n `CREATE VIRTUAL TABLE IF NOT EXISTS ${table} USING fts5(text, content='', tokenize='trigram case_sensitive 1')`,\n ),\n add: (tx, items) => write(tx, items, \"insert\"),\n remove: (tx, items) => write(tx, items, \"delete\"),\n async candidates(db, branches, max) {\n // A quoted string is a phrase of consecutive trigrams, i.e. an exact substring.\n const quote = (literal: string) => `\"${literal.replaceAll('\"', '\"\"')}\"`;\n const match = branches.map((branch) => `(${branch.map(quote).join(\" AND \")})`).join(\" OR \");\n const rows = await db.query<{ id: number }>(\n `SELECT rowid AS id FROM ${table} WHERE ${table} MATCH ? ORDER BY rowid LIMIT ?`,\n [match, max],\n );\n return rows.map((row) => Number(row.id));\n },\n };\n}\n","/*\n * Mandatory-literal extraction from a regex, after Russ Cox's trigram method.\n * Ported from hypgrep 0.5.3 src/regex.js (Hyperparam), MIT License:\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this\n * software and associated documentation files (the \"Software\"), to deal in the Software\n * without restriction, including without limitation the rights to use, copy, modify,\n * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to\n * permit persons to whom the Software is furnished to do so, subject to the following\n * conditions: The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \"AS IS\",\n * WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED.\n */\n\n/** Cross-product cap: past it, the construct counts as a wildcard. */\nconst MAX_DNF = 32;\n\ntype LiveBranch = { literals: string[]; run: string };\ntype Quantifier = { copies: number; openEnded: boolean; end: number };\ntype GroupPrefix = { kind: \"consuming\"; innerStart: number } | { kind: \"assertion\" | \"unknown\" };\n\n/**\n * DNF of literals that every match must contain: a string can match only if all\n * literals of at least one branch occur in it. An empty branch matches anything.\n * Conservative: a construct it cannot reason about becomes a wildcard, never a literal.\n *\n * /abc(def|ghi)/ → [[\"abcdef\"], [\"abcghi\"]] /foo|.+/ → [[\"foo\"], []]\n */\nexport function regexLiterals(regex: RegExp): string[][] {\n // v-mode class syntax desyncs skipClass and would invent literals.\n if (regex.flags.includes(\"v\")) return [[]];\n return expandRegion(regex.source, 0, regex.source.length).dnf;\n}\n\n/** `exact`: every arm is one contiguous literal that is the arm's whole match, with nothing dropped. */\ntype Region = { dnf: string[][]; exact: boolean };\n\nfunction expandRegion(src: string, start: number, end: number): Region {\n const dnf: string[][] = [];\n let exact = true;\n for (const [bStart, bEnd] of splitTopLevelAlternation(src, start, end)) {\n const arm = expandBranch(src, bStart, bEnd);\n for (const branch of arm.dnf) dnf.push(branch);\n exact = exact && arm.exact;\n }\n return { dnf, exact };\n}\n\nfunction splitTopLevelAlternation(\n src: string,\n start: number,\n end: number,\n): Array<[number, number]> {\n const spans: Array<[number, number]> = [];\n let bStart = start;\n let depth = 0;\n let i = start;\n while (i < end) {\n const c = src[i];\n if (c === \"\\\\\") {\n i += 2;\n continue;\n }\n if (c === \"[\") {\n i = skipClass(src, i);\n continue;\n }\n if (c === \"(\") depth += 1;\n else if (c === \")\") depth -= 1;\n else if (c === \"|\" && depth === 0) {\n spans.push([bStart, i]);\n bStart = i + 1;\n }\n i += 1;\n }\n spans.push([bStart, end]);\n return spans;\n}\n\nfunction expandBranch(src: string, start: number, end: number): Region {\n let cross: LiveBranch[] = [{ literals: [], run: \"\" }];\n // Set when part of the match is not captured in a literal (wildcard, optional or repeated\n // atom, assertion): gluing this branch to neighbouring text would be unsound.\n let lossy = false;\n const flushRuns = () => {\n for (const b of cross) {\n if (b.run.length > 0) {\n b.literals.push(b.run);\n b.run = \"\";\n }\n }\n };\n // Append `text` repeated by the quantifier's mandatory copies to every live run.\n const appendRun = (text: string, q: Quantifier) => {\n if (q.copies === 0) {\n lossy = true;\n flushRuns();\n return;\n }\n for (const b of cross) b.run += text.repeat(q.copies);\n if (q.openEnded) {\n lossy = true;\n flushRuns();\n }\n };\n\n let i = start;\n while (i < end) {\n const c = src[i] ?? \"\";\n\n if (c === \"^\" || c === \"$\") {\n i += 1;\n continue;\n }\n\n if (c === \"\\\\\") {\n const next = src[i + 1];\n if (next === undefined) {\n i += 1;\n continue;\n }\n const escaped = parseEscapedLiteral(src, i);\n if (escaped) {\n const q = peekQuantifier(src, escaped.end);\n i = q.end;\n appendRun(escaped.char, q);\n continue;\n }\n // \\d \\w \\s \\b, backrefs, \\p{…}: a wildcard; consume its payload.\n if (/[a-zA-Z0-9]/.test(next)) {\n lossy = true;\n flushRuns();\n i = peekQuantifier(src, skipSpecialEscape(src, i)).end;\n continue;\n }\n // \\. \\/ \\( …: the literal character.\n const q = peekQuantifier(src, i + 2);\n i = q.end;\n appendRun(next, q);\n continue;\n }\n\n if (c === \"[\") {\n const classEnd = skipClass(src, i);\n const chars = parseLiteralClass(src, i, classEnd);\n const q = peekQuantifier(src, classEnd);\n i = q.end;\n if (\n q.copies === 0 ||\n chars === null ||\n q.copies > 1 ||\n cross.length * chars.length > MAX_DNF\n ) {\n lossy = true;\n flushRuns();\n continue;\n }\n cross = cross.flatMap((b) =>\n chars.map((ch) => ({ literals: b.literals.slice(), run: b.run + ch })),\n );\n if (q.openEnded) {\n lossy = true;\n flushRuns();\n }\n continue;\n }\n\n if (c === \"(\") {\n const groupEnd = skipGroup(src, i);\n const group = parseGroupPrefix(src, i);\n const q = peekQuantifier(src, groupEnd);\n i = q.end;\n if (group.kind !== \"consuming\" || q.copies === 0) {\n lossy = true;\n flushRuns();\n continue;\n }\n const { dnf: inner, exact } = expandRegion(src, group.innerStart, groupEnd - 1);\n // Fold into the run only when every arm is one contiguous literal used once and is the\n // arm's whole match: a repeated group (`(a|b){3}`) may pick different arms per copy, and\n // a wildcard at an arm's edge (`x(.abc)`) must not be glued away.\n const foldable =\n exact && q.copies === 1 && inner.length > 0 && inner.every((b) => b.length === 1);\n lossy = lossy || q.openEnded || !foldable;\n if (foldable && cross.length * inner.length <= MAX_DNF) {\n cross = cross.flatMap((b) =>\n inner.map((arm) => ({ literals: b.literals.slice(), run: b.run + (arm[0] ?? \"\") })),\n );\n if (q.openEnded) flushRuns();\n continue;\n }\n lossy = true;\n flushRuns();\n if (cross.length * inner.length > MAX_DNF) continue;\n cross = cross.flatMap((b) =>\n inner.map((arm) => ({ literals: b.literals.concat(arm), run: \"\" })),\n );\n continue;\n }\n\n if (c === \".\") {\n lossy = true;\n flushRuns();\n i = peekQuantifier(src, i + 1).end;\n continue;\n }\n\n const q = peekQuantifier(src, i + 1);\n i = q.end;\n appendRun(c, q);\n }\n flushRuns();\n return {\n dnf: cross.map((b) => b.literals),\n exact: !lossy && cross.every((b) => b.literals.length === 1),\n };\n}\n\n/** Where a group's consuming body starts; lookarounds and unknown `(?…)` are not consuming. */\nfunction parseGroupPrefix(src: string, start: number): GroupPrefix {\n if (src[start + 1] !== \"?\") return { kind: \"consuming\", innerStart: start + 1 };\n const marker = src[start + 2];\n if (marker === \":\") return { kind: \"consuming\", innerStart: start + 3 };\n if (marker === \"=\" || marker === \"!\") return { kind: \"assertion\" };\n if (marker === \"<\") {\n const next = src[start + 3];\n if (next === \"=\" || next === \"!\") return { kind: \"assertion\" };\n const nameEnd = src.indexOf(\">\", start + 3);\n if (nameEnd >= 0) return { kind: \"consuming\", innerStart: nameEnd + 1 };\n }\n return { kind: \"unknown\" };\n}\n\n/**\n * `\\xHH`, `\\uHHHH`: escapes that stand for one literal character. `\\u{…}` is not one:\n * the planner compiles without the `u` flag, where it means \"u\" repeated or the text \"u{…}\".\n */\nfunction parseEscapedLiteral(\n src: string,\n start: number,\n): { char: string; end: number } | undefined {\n const kind = src[start + 1];\n if (kind === \"x\") {\n const hex = src.slice(start + 2, start + 4);\n if (!/^[0-9a-fA-F]{2}$/.test(hex)) return undefined;\n return { char: String.fromCharCode(Number.parseInt(hex, 16)), end: start + 4 };\n }\n if (kind === \"u\") {\n const hex = src.slice(start + 2, start + 6);\n if (!/^[0-9a-fA-F]{4}$/.test(hex)) return undefined;\n return { char: String.fromCharCode(Number.parseInt(hex, 16)), end: start + 6 };\n }\n return undefined;\n}\n\n/**\n * Skip an escape that is not a literal punctuation character, so its payload\n * (`\\x41`, `\\p{L}`, `\\k<name>`, backrefs) never leaks into a literal. Malformed\n * `\\x`, `\\u`, `\\c` payloads are Annex-B literals and are rescanned as atoms.\n */\nfunction skipSpecialEscape(src: string, start: number): number {\n const next = src[start + 1];\n if (next === \"x\") {\n return /^[0-9a-fA-F]{2}$/.test(src.slice(start + 2, start + 4)) ? start + 4 : start + 2;\n }\n if (next === \"u\") {\n if (src[start + 2] === \"{\") {\n const close = src.indexOf(\"}\", start + 3);\n return close >= 0 ? close + 1 : src.length;\n }\n return /^[0-9a-fA-F]{4}$/.test(src.slice(start + 2, start + 6)) ? start + 6 : start + 2;\n }\n if (next === \"c\") return /^[a-zA-Z]$/.test(src[start + 2] ?? \"\") ? start + 3 : start + 2;\n if ((next === \"p\" || next === \"P\") && src[start + 2] === \"{\") {\n const close = src.indexOf(\"}\", start + 3);\n return close >= 0 ? close + 1 : src.length;\n }\n if (next === \"k\" && src[start + 2] === \"<\") {\n const close = src.indexOf(\">\", start + 3);\n return close >= 0 ? close + 1 : src.length;\n }\n if (/[0-9]/.test(next ?? \"\")) {\n let i = start + 2;\n while (/[0-9]/.test(src[i] ?? \"\")) i += 1;\n return i;\n }\n return Math.min(start + 2, src.length);\n}\n\n/** A `[...]` class as literal alternatives, or null if negated, ranged, special or empty. */\nfunction parseLiteralClass(src: string, start: number, end: number): string[] | null {\n let i = start + 1;\n const lastInside = end - 1;\n if (src[i] === \"^\") return null;\n const chars = new Set<string>();\n while (i < lastInside) {\n const c = src[i] ?? \"\";\n if (c === \"\\\\\") {\n const next = src[i + 1];\n if (next === undefined || /[a-zA-Z]/.test(next)) return null;\n chars.add(next);\n i += 2;\n continue;\n }\n if (i + 2 < lastInside && src[i + 1] === \"-\") return null;\n chars.add(c);\n i += 1;\n }\n // A surrogate half would split an astral character the index sees as one code point.\n if ([...chars].some((ch) => /[\\ud800-\\udfff]/.test(ch))) return null;\n return chars.size === 0 ? null : [...chars];\n}\n\n/** Mandatory copies of the preceding atom, whether more may follow, and where the quantifier ends. */\nfunction peekQuantifier(src: string, i: number): Quantifier {\n const c = src[i];\n if (c === \"?\" || c === \"*\" || c === \"+\") {\n const end = src[i + 1] === \"?\" ? i + 2 : i + 1;\n return { copies: c === \"+\" ? 1 : 0, openEnded: true, end };\n }\n if (c === \"{\") {\n const close = src.indexOf(\"}\", i);\n if (close < 0) return { copies: 1, openEnded: false, end: i };\n const m = /^(\\d+)(?:,(\\d*))?$/.exec(src.slice(i + 1, close));\n if (!m) return { copies: 1, openEnded: false, end: i };\n const min = Number(m[1]);\n const max = m[2] === undefined ? min : m[2] === \"\" ? Number.POSITIVE_INFINITY : Number(m[2]);\n const end = src[close + 1] === \"?\" ? close + 2 : close + 1;\n return { copies: min, openEnded: max > min, end };\n }\n return { copies: 1, openEnded: false, end: i };\n}\n\nfunction skipClass(src: string, i: number): number {\n let j = i + 1;\n if (src[j] === \"^\") j += 1;\n while (j < src.length && src[j] !== \"]\") j += src[j] === \"\\\\\" && j + 1 < src.length ? 2 : 1;\n return j < src.length ? j + 1 : j;\n}\n\nfunction skipGroup(src: string, i: number): number {\n let depth = 1;\n let j = i + 1;\n while (j < src.length && depth > 0) {\n const c = src[j];\n if (c === \"\\\\\" && j + 1 < src.length) {\n j += 2;\n continue;\n }\n if (c === \"[\") {\n j = skipClass(src, j);\n continue;\n }\n if (c === \"(\") depth += 1;\n else if (c === \")\") depth -= 1;\n j += 1;\n }\n return j;\n}\n","import { fold } from \"./fold.js\";\nimport { regexLiterals } from \"./regex-literals.js\";\n\n/** Longer patterns are rejected: a JS regex cannot be interrupted once it runs. */\nexport const MAX_PATTERN_LENGTH = 1000;\n\nexport type QueryPlan = {\n /** The pattern as a global regex (`i` added when not case-sensitive); verifies candidates. */\n regex: RegExp;\n /**\n * Per DNF branch, the folded literals of at least 3 code points that a match must\n * contain. `null` when some branch has none: only a scan can answer the pattern.\n */\n branches: string[][] | null;\n};\n\nexport function planQuery(\n pattern: string,\n opts: { regex?: boolean; caseSensitive?: boolean } = {},\n): QueryPlan {\n if (pattern.length > MAX_PATTERN_LENGTH) {\n throw new RangeError(\n `ngram: pattern is ${pattern.length} characters, the limit is ${MAX_PATTERN_LENGTH}`,\n );\n }\n // The regex has no `u` flag (case folding is unsound under /iu), so `\\p{…}` would silently\n // match the literal text \"p{…}\". An odd run of backslashes before `p` is a real escape.\n if (opts.regex && /(?<!\\\\)(?:\\\\\\\\)*\\\\[pP]\\{/.test(pattern)) {\n throw new SyntaxError(\n \"ngram: \\\\p{…} and \\\\P{…} are not supported in regex mode (patterns compile without the u flag)\",\n );\n }\n const source = opts.regex ? pattern : pattern.replace(/[.*+?^${}()|[\\]\\\\/]/g, \"\\\\$&\");\n const regex = new RegExp(source, opts.caseSensitive === false ? \"gi\" : \"g\");\n const branches: string[][] = [];\n for (const literals of regexLiterals(regex)) {\n const usable = literals.map(fold).filter((literal) => Array.from(literal).length >= 3);\n if (usable.length === 0) return { regex, branches: null };\n branches.push(usable);\n }\n return { regex, branches };\n}\n","import { fold } from \"./fold.js\";\n\n/** Distinct 3-code-point grams of an already folded string, in first-seen order. */\nexport function gramsOf(folded: string): string[] {\n const cps = Array.from(folded);\n const out = new Set<string>();\n for (let i = 0; i + 3 <= cps.length; i++) out.add(cps.slice(i, i + 3).join(\"\"));\n return [...out];\n}\n\n/** Distinct trigrams of `text` as the index sees it: case-folded, by code point. */\nexport function trigrams(text: string): string[] {\n return gramsOf(fold(text));\n}\n\n/**\n * One SQLite INTEGER per gram. Three BMP code points pack exactly into 48 bits.\n * A gram with an astral code point keeps the low 16 bits of each code point\n * above 2^48: a collision only adds candidates, which verification removes.\n */\nexport function encodeGram(gram: string): number {\n const [a = 0, b = 0, c = 0] = Array.from(gram, (ch) => ch.codePointAt(0) ?? 0);\n if (a < 0x10000 && b < 0x10000 && c < 0x10000) return a * 2 ** 32 + b * 2 ** 16 + c;\n return 2 ** 48 + (a % 0x10000) * 2 ** 32 + (b % 0x10000) * 2 ** 16 + (c % 0x10000);\n}\n","import type { GramBackend, NgramItem } from \"./fts5-backend.js\";\nimport { chunkJson } from \"./json-chunks.js\";\nimport { encodeGram, gramsOf, trigrams } from \"./trigrams.js\";\n\nconst PAIR = \"json_extract(value, '$[0]'), json_extract(value, '$[1]')\";\n\n/**\n * One row per (gram, id) in a WITHOUT ROWID table: the primary key is the\n * posting list, clustered by gram. A text is a candidate for a branch when it\n * holds every distinct gram of the branch's literals.\n */\nexport function newPostingsBackend(prefix: string): GramBackend {\n const table = `${prefix}grams`;\n const pairs = (items: NgramItem[]) =>\n chunkJson(\n items.flatMap((item) => trigrams(item.text).map((gram) => [encodeGram(gram), item.id])),\n );\n return {\n create: (tx) =>\n tx.exec(\n `CREATE TABLE IF NOT EXISTS ${table}(gram INTEGER NOT NULL, id INTEGER NOT NULL, PRIMARY KEY (gram, id)) WITHOUT ROWID`,\n ),\n async add(tx, items) {\n for (const chunk of pairs(items)) {\n await tx.query(\n `INSERT OR IGNORE INTO ${table}(gram, id) SELECT ${PAIR} FROM json_each(?)`,\n [chunk],\n );\n }\n },\n async remove(tx, items) {\n for (const chunk of pairs(items)) {\n await tx.query(\n `DELETE FROM ${table} WHERE (gram, id) IN (SELECT ${PAIR} FROM json_each(?))`,\n [chunk],\n );\n }\n },\n async candidates(db, branches, max) {\n const ids = new Set<number>();\n for (const branch of branches) {\n // Literals are already folded: split them without folding again.\n const grams = [...new Set(branch.flatMap((literal) => gramsOf(literal).map(encodeGram)))];\n const rows = await db.query<{ id: number }>(\n `SELECT id FROM ${table} WHERE gram IN (SELECT value FROM json_each(?)) GROUP BY id HAVING count(*) = ? ORDER BY id LIMIT ?`,\n [JSON.stringify(grams), grams.length, max],\n );\n for (const row of rows) ids.add(Number(row.id));\n }\n return [...ids].sort((a, b) => a - b).slice(0, max);\n },\n };\n}\n","import type { Db } from \"@statewalker/db-api\";\nimport { inTransaction } from \"@statewalker/db-sqlite\";\nimport { PREFIX_PATTERN } from \"../prefix.js\";\nimport { type GramBackend, type NgramItem, newFts5Backend } from \"./fts5-backend.js\";\nimport { chunkJson } from \"./json-chunks.js\";\nimport { planQuery } from \"./plan-query.js\";\nimport { newPostingsBackend } from \"./postings-backend.js\";\n\nexport type { NgramItem };\nexport type NgramMatch = { start: number; end: number };\nexport type NgramHit = { id: number; matches: NgramMatch[] };\nexport type NgramSearchOptions = {\n /** Treat `pattern` as a JS regex source; otherwise it is literal text. */\n regex?: boolean;\n /** Default true (grep semantics). */\n caseSensitive?: boolean;\n /** Maximum hits. Default 100. */\n limit?: number;\n /** Maximum candidates verified. Default 10,000. */\n candidateBudget?: number;\n /** Keep only ids for which this returns true (applied before verification). */\n filter?: (id: number) => boolean;\n signal?: AbortSignal;\n};\nexport type NgramSearchResult = {\n /** Ascending id order; every non-empty match, offsets in UTF-16 code units. */\n hits: NgramHit[];\n /** True when the search stopped before checking every candidate (budget or limit). */\n truncated: boolean;\n};\nexport type NgramStoreOptions = { db: Db; prefix: string; store?: \"fts5\" | \"postings\" };\n\nconst VERIFY_BATCH = 256;\n\nexport class NgramStore {\n /** `{prefix}texts`: its `id` column holds exactly the stored ids. Callers may join it in SQL. */\n readonly idTable: string;\n readonly #db: Db;\n readonly #prefix: string;\n readonly #store: \"fts5\" | \"postings\";\n readonly #backend: GramBackend;\n\n constructor({ db, prefix, store = \"fts5\" }: NgramStoreOptions) {\n if (!PREFIX_PATTERN.test(prefix)) {\n throw new Error(`ngram: prefix \"${prefix}\" must match ${PREFIX_PATTERN}`);\n }\n if (store !== \"fts5\" && store !== \"postings\")\n throw new Error(`ngram: unknown store \"${store}\"`);\n this.#db = db;\n this.#prefix = prefix;\n this.idTable = `${prefix}texts`;\n this.#store = store;\n this.#backend = store === \"fts5\" ? newFts5Backend(prefix) : newPostingsBackend(prefix);\n }\n\n async init(): Promise<void> {\n const p = this.#prefix;\n await inTransaction(this.#db, async (tx) => {\n await tx.exec(\n `CREATE TABLE IF NOT EXISTS ${p}meta(key TEXT PRIMARY KEY, value TEXT NOT NULL)`,\n );\n const [row] = await tx.query<{ value: string }>(\n `SELECT value FROM ${p}meta WHERE key = 'store'`,\n );\n if (row && row.value !== this.#store) {\n throw new Error(\n `ngram: index \"${p}\" was created with store \"${row.value}\", opened with \"${this.#store}\"`,\n );\n }\n if (!row)\n await tx.query(`INSERT INTO ${p}meta(key, value) VALUES ('store', ?)`, [this.#store]);\n await tx.exec(\n `CREATE TABLE IF NOT EXISTS ${p}texts(id INTEGER PRIMARY KEY, text TEXT NOT NULL)`,\n );\n await this.#backend.create(tx);\n });\n }\n\n async upsert(items: NgramItem[]): Promise<void> {\n // The last text per id wins; a repeated id would leave stale grams behind.\n // A lone surrogate would be stored differently from how it was indexed, so a\n // later delete could not replay it: texts are made well-formed first.\n const latest = [\n ...new Map(items.map(({ id, text }) => [id, { id, text: text.toWellFormed() }])).values(),\n ];\n if (latest.length === 0) return;\n await inTransaction(this.#db, async (tx) => {\n await this.#backend.remove(\n tx,\n await this.#load(\n tx,\n latest.map((item) => item.id),\n ),\n );\n for (const chunk of chunkJson(latest.map((item) => [item.id, item.text]))) {\n await tx.query(\n `INSERT OR REPLACE INTO ${this.#prefix}texts(id, text) SELECT json_extract(value, '$[0]'), json_extract(value, '$[1]') FROM json_each(?)`,\n [chunk],\n );\n }\n await this.#backend.add(tx, latest);\n });\n }\n\n async delete(ids: number[]): Promise<void> {\n if (ids.length === 0) return;\n await inTransaction(this.#db, async (tx) => {\n await this.#backend.remove(tx, await this.#load(tx, ids));\n for (const chunk of chunkJson(ids)) {\n await tx.query(\n `DELETE FROM ${this.#prefix}texts WHERE id IN (SELECT value FROM json_each(?))`,\n [chunk],\n );\n }\n });\n }\n\n async search(pattern: string, opts: NgramSearchOptions = {}): Promise<NgramSearchResult> {\n const { limit = 100, candidateBudget = 10_000, filter, signal } = opts;\n const plan = planQuery(pattern, {\n regex: opts.regex,\n caseSensitive: opts.caseSensitive ?? true,\n });\n signal?.throwIfAborted();\n if (limit <= 0) return { hits: [], truncated: true };\n const found =\n plan.branches === null\n ? (\n await this.#db.query<{ id: number }>(\n `SELECT id FROM ${this.#prefix}texts ORDER BY id LIMIT ?`,\n [candidateBudget + 1],\n )\n ).map((row) => Number(row.id))\n : await this.#backend.candidates(this.#db, plan.branches, candidateBudget + 1);\n let truncated = found.length > candidateBudget;\n const ids = found.slice(0, candidateBudget).filter((id) => !filter || filter(id));\n\n const hits: NgramHit[] = [];\n let last: number | undefined;\n verify: for (let i = 0; i < ids.length; i += VERIFY_BATCH) {\n signal?.throwIfAborted();\n for (const { id, text } of await this.#load(this.#db, ids.slice(i, i + VERIFY_BATCH))) {\n last = id;\n const matches: NgramMatch[] = [];\n for (const m of text.matchAll(plan.regex)) {\n if (m[0].length > 0) matches.push({ start: m.index, end: m.index + m[0].length });\n }\n if (matches.length === 0) continue;\n hits.push({ id, matches });\n if (hits.length >= limit) break verify;\n }\n }\n if (hits.length >= limit && last !== ids.at(-1)) truncated = true;\n return { hits, truncated };\n }\n\n /** The stored original texts of `ids`; unknown ids are absent. */\n async getTexts(ids: number[]): Promise<Map<number, string>> {\n const items = await this.#load(this.#db, ids);\n return new Map(items.map((item) => [item.id, item.text]));\n }\n\n /** Drops the tables of both stores, so it works without knowing which one created the prefix. */\n async drop(): Promise<void> {\n const p = this.#prefix;\n await inTransaction(this.#db, async (tx) => {\n for (const t of [\"ng\", \"grams\", \"texts\", \"meta\"])\n await tx.exec(`DROP TABLE IF EXISTS ${p}${t}`);\n });\n }\n\n /** Stored texts for `ids`, ascending by id; missing ids are skipped. */\n async #load(db: Db, ids: number[]): Promise<NgramItem[]> {\n const out: NgramItem[] = [];\n for (const chunk of chunkJson([...ids].sort((a, b) => a - b))) {\n const rows = await db.query<{ id: number; text: string }>(\n `SELECT id, text FROM ${this.#prefix}texts WHERE id IN (SELECT value FROM json_each(?)) ORDER BY id`,\n [chunk],\n );\n for (const row of rows) out.push({ id: Number(row.id), text: String(row.text) });\n }\n return out;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;AAYA,SAAgB,KAAK,MAAsB;CACzC,IAAI,MAAM;CACV,KAAK,MAAM,MAAM,MAAM;EACrB,MAAM,QAAQ,GAAG,YAAY;EAC7B,MAAM,SAAS,MAAM,KAAK,KAAK,CAAC,CAAC,WAAW,IAAI,QAAQ,GAAA,CAAI,YAAY;EACxE,OAAO,MAAM,WAAW,GAAG,SAAS,QAAQ;CAC9C;CACA,OAAO;AACT;;;;;;;;;ACdA,SAAgB,UAAU,MAAiB,WAAW,KAAmB;CACvE,MAAM,SAAmB,CAAC;CAC1B,IAAI,UAAoB,CAAC;CACzB,IAAI,OAAO;CACX,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,OAAO,KAAK,UAAU,GAAG;EAC/B,IAAI,QAAQ,SAAS,KAAK,OAAO,KAAK,SAAS,IAAI,UAAU;GAC3D,OAAO,KAAK,IAAI,QAAQ,KAAK,GAAG,EAAE,EAAE;GACpC,UAAU,CAAC;GACX,OAAO;EACT;EACA,QAAQ,KAAK,IAAI;EACjB,QAAQ,KAAK,SAAS;CACxB;CACA,IAAI,QAAQ,SAAS,GAAG,OAAO,KAAK,IAAI,QAAQ,KAAK,GAAG,EAAE,EAAE;CAC5D,OAAO;AACT;;;ACNA,MAAM,MAAM;;;;;;;;AASZ,SAAgB,eAAe,QAA6B;CAC1D,MAAM,QAAQ,GAAG,OAAO;CACxB,MAAM,QAAQ,OAAO,IAAQ,OAAoB,YAAiC;EAChF,MAAM,MACJ,YAAY,WACR,eAAe,MAAM,uBAAuB,IAAI,sBAChD,eAAe,MAAM,GAAG,MAAM,kCAAkC,IAAI;EAC1E,KAAK,MAAM,SAAS,UAAU,MAAM,KAAK,SAAS,CAAC,KAAK,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,GAC3E,MAAM,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC;CAE/B;CACA,OAAO;EACL,SAAS,OACP,GAAG,KACD,sCAAsC,MAAM,mEAC9C;EACF,MAAM,IAAI,UAAU,MAAM,IAAI,OAAO,QAAQ;EAC7C,SAAS,IAAI,UAAU,MAAM,IAAI,OAAO,QAAQ;EAChD,MAAM,WAAW,IAAI,UAAU,KAAK;GAElC,MAAM,SAAS,YAAoB,IAAI,QAAQ,WAAW,MAAK,MAAI,EAAE;GACrE,MAAM,QAAQ,SAAS,KAAK,WAAW,IAAI,OAAO,IAAI,KAAK,CAAC,CAAC,KAAK,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM;GAK1F,QAAO,MAJY,GAAG,MACpB,2BAA2B,MAAM,SAAS,MAAM,kCAChD,CAAC,OAAO,GAAG,CACb,EAAA,CACY,KAAK,QAAQ,OAAO,IAAI,EAAE,CAAC;EACzC;CACF;AACF;;;;ACvCA,MAAM,UAAU;;;;;;;;AAahB,SAAgB,cAAc,OAA2B;CAEvD,IAAI,MAAM,MAAM,SAAS,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC;CACzC,OAAO,aAAa,MAAM,QAAQ,GAAG,MAAM,OAAO,MAAM,CAAC,CAAC;AAC5D;AAKA,SAAS,aAAa,KAAa,OAAe,KAAqB;CACrE,MAAM,MAAkB,CAAC;CACzB,IAAI,QAAQ;CACZ,KAAK,MAAM,CAAC,QAAQ,SAAS,yBAAyB,KAAK,OAAO,GAAG,GAAG;EACtE,MAAM,MAAM,aAAa,KAAK,QAAQ,IAAI;EAC1C,KAAK,MAAM,UAAU,IAAI,KAAK,IAAI,KAAK,MAAM;EAC7C,QAAQ,SAAS,IAAI;CACvB;CACA,OAAO;EAAE;EAAK;CAAM;AACtB;AAEA,SAAS,yBACP,KACA,OACA,KACyB;CACzB,MAAM,QAAiC,CAAC;CACxC,IAAI,SAAS;CACb,IAAI,QAAQ;CACZ,IAAI,IAAI;CACR,OAAO,IAAI,KAAK;EACd,MAAM,IAAI,IAAI;EACd,IAAI,MAAM,MAAM;GACd,KAAK;GACL;EACF;EACA,IAAI,MAAM,KAAK;GACb,IAAI,UAAU,KAAK,CAAC;GACpB;EACF;EACA,IAAI,MAAM,KAAK,SAAS;OACnB,IAAI,MAAM,KAAK,SAAS;OACxB,IAAI,MAAM,OAAO,UAAU,GAAG;GACjC,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;GACtB,SAAS,IAAI;EACf;EACA,KAAK;CACP;CACA,MAAM,KAAK,CAAC,QAAQ,GAAG,CAAC;CACxB,OAAO;AACT;AAEA,SAAS,aAAa,KAAa,OAAe,KAAqB;CACrE,IAAI,QAAsB,CAAC;EAAE,UAAU,CAAC;EAAG,KAAK;CAAG,CAAC;CAGpD,IAAI,QAAQ;CACZ,MAAM,kBAAkB;EACtB,KAAK,MAAM,KAAK,OACd,IAAI,EAAE,IAAI,SAAS,GAAG;GACpB,EAAE,SAAS,KAAK,EAAE,GAAG;GACrB,EAAE,MAAM;EACV;CAEJ;CAEA,MAAM,aAAa,MAAc,MAAkB;EACjD,IAAI,EAAE,WAAW,GAAG;GAClB,QAAQ;GACR,UAAU;GACV;EACF;EACA,KAAK,MAAM,KAAK,OAAO,EAAE,OAAO,KAAK,OAAO,EAAE,MAAM;EACpD,IAAI,EAAE,WAAW;GACf,QAAQ;GACR,UAAU;EACZ;CACF;CAEA,IAAI,IAAI;CACR,OAAO,IAAI,KAAK;EACd,MAAM,IAAI,IAAI,MAAM;EAEpB,IAAI,MAAM,OAAO,MAAM,KAAK;GAC1B,KAAK;GACL;EACF;EAEA,IAAI,MAAM,MAAM;GACd,MAAM,OAAO,IAAI,IAAI;GACrB,IAAI,SAAS,KAAA,GAAW;IACtB,KAAK;IACL;GACF;GACA,MAAM,UAAU,oBAAoB,KAAK,CAAC;GAC1C,IAAI,SAAS;IACX,MAAM,IAAI,eAAe,KAAK,QAAQ,GAAG;IACzC,IAAI,EAAE;IACN,UAAU,QAAQ,MAAM,CAAC;IACzB;GACF;GAEA,IAAI,cAAc,KAAK,IAAI,GAAG;IAC5B,QAAQ;IACR,UAAU;IACV,IAAI,eAAe,KAAK,kBAAkB,KAAK,CAAC,CAAC,CAAC,CAAC;IACnD;GACF;GAEA,MAAM,IAAI,eAAe,KAAK,IAAI,CAAC;GACnC,IAAI,EAAE;GACN,UAAU,MAAM,CAAC;GACjB;EACF;EAEA,IAAI,MAAM,KAAK;GACb,MAAM,WAAW,UAAU,KAAK,CAAC;GACjC,MAAM,QAAQ,kBAAkB,KAAK,GAAG,QAAQ;GAChD,MAAM,IAAI,eAAe,KAAK,QAAQ;GACtC,IAAI,EAAE;GACN,IACE,EAAE,WAAW,KACb,UAAU,QACV,EAAE,SAAS,KACX,MAAM,SAAS,MAAM,SAAS,SAC9B;IACA,QAAQ;IACR,UAAU;IACV;GACF;GACA,QAAQ,MAAM,SAAS,MACrB,MAAM,KAAK,QAAQ;IAAE,UAAU,EAAE,SAAS,MAAM;IAAG,KAAK,EAAE,MAAM;GAAG,EAAE,CACvE;GACA,IAAI,EAAE,WAAW;IACf,QAAQ;IACR,UAAU;GACZ;GACA;EACF;EAEA,IAAI,MAAM,KAAK;GACb,MAAM,WAAW,UAAU,KAAK,CAAC;GACjC,MAAM,QAAQ,iBAAiB,KAAK,CAAC;GACrC,MAAM,IAAI,eAAe,KAAK,QAAQ;GACtC,IAAI,EAAE;GACN,IAAI,MAAM,SAAS,eAAe,EAAE,WAAW,GAAG;IAChD,QAAQ;IACR,UAAU;IACV;GACF;GACA,MAAM,EAAE,KAAK,OAAO,UAAU,aAAa,KAAK,MAAM,YAAY,WAAW,CAAC;GAI9E,MAAM,WACJ,SAAS,EAAE,WAAW,KAAK,MAAM,SAAS,KAAK,MAAM,OAAO,MAAM,EAAE,WAAW,CAAC;GAClF,QAAQ,SAAS,EAAE,aAAa,CAAC;GACjC,IAAI,YAAY,MAAM,SAAS,MAAM,UAAU,SAAS;IACtD,QAAQ,MAAM,SAAS,MACrB,MAAM,KAAK,SAAS;KAAE,UAAU,EAAE,SAAS,MAAM;KAAG,KAAK,EAAE,OAAO,IAAI,MAAM;IAAI,EAAE,CACpF;IACA,IAAI,EAAE,WAAW,UAAU;IAC3B;GACF;GACA,QAAQ;GACR,UAAU;GACV,IAAI,MAAM,SAAS,MAAM,SAAS,SAAS;GAC3C,QAAQ,MAAM,SAAS,MACrB,MAAM,KAAK,SAAS;IAAE,UAAU,EAAE,SAAS,OAAO,GAAG;IAAG,KAAK;GAAG,EAAE,CACpE;GACA;EACF;EAEA,IAAI,MAAM,KAAK;GACb,QAAQ;GACR,UAAU;GACV,IAAI,eAAe,KAAK,IAAI,CAAC,CAAC,CAAC;GAC/B;EACF;EAEA,MAAM,IAAI,eAAe,KAAK,IAAI,CAAC;EACnC,IAAI,EAAE;EACN,UAAU,GAAG,CAAC;CAChB;CACA,UAAU;CACV,OAAO;EACL,KAAK,MAAM,KAAK,MAAM,EAAE,QAAQ;EAChC,OAAO,CAAC,SAAS,MAAM,OAAO,MAAM,EAAE,SAAS,WAAW,CAAC;CAC7D;AACF;;AAGA,SAAS,iBAAiB,KAAa,OAA4B;CACjE,IAAI,IAAI,QAAQ,OAAO,KAAK,OAAO;EAAE,MAAM;EAAa,YAAY,QAAQ;CAAE;CAC9E,MAAM,SAAS,IAAI,QAAQ;CAC3B,IAAI,WAAW,KAAK,OAAO;EAAE,MAAM;EAAa,YAAY,QAAQ;CAAE;CACtE,IAAI,WAAW,OAAO,WAAW,KAAK,OAAO,EAAE,MAAM,YAAY;CACjE,IAAI,WAAW,KAAK;EAClB,MAAM,OAAO,IAAI,QAAQ;EACzB,IAAI,SAAS,OAAO,SAAS,KAAK,OAAO,EAAE,MAAM,YAAY;EAC7D,MAAM,UAAU,IAAI,QAAQ,KAAK,QAAQ,CAAC;EAC1C,IAAI,WAAW,GAAG,OAAO;GAAE,MAAM;GAAa,YAAY,UAAU;EAAE;CACxE;CACA,OAAO,EAAE,MAAM,UAAU;AAC3B;;;;;AAMA,SAAS,oBACP,KACA,OAC2C;CAC3C,MAAM,OAAO,IAAI,QAAQ;CACzB,IAAI,SAAS,KAAK;EAChB,MAAM,MAAM,IAAI,MAAM,QAAQ,GAAG,QAAQ,CAAC;EAC1C,IAAI,CAAC,mBAAmB,KAAK,GAAG,GAAG,OAAO,KAAA;EAC1C,OAAO;GAAE,MAAM,OAAO,aAAa,OAAO,SAAS,KAAK,EAAE,CAAC;GAAG,KAAK,QAAQ;EAAE;CAC/E;CACA,IAAI,SAAS,KAAK;EAChB,MAAM,MAAM,IAAI,MAAM,QAAQ,GAAG,QAAQ,CAAC;EAC1C,IAAI,CAAC,mBAAmB,KAAK,GAAG,GAAG,OAAO,KAAA;EAC1C,OAAO;GAAE,MAAM,OAAO,aAAa,OAAO,SAAS,KAAK,EAAE,CAAC;GAAG,KAAK,QAAQ;EAAE;CAC/E;AAEF;;;;;;AAOA,SAAS,kBAAkB,KAAa,OAAuB;CAC7D,MAAM,OAAO,IAAI,QAAQ;CACzB,IAAI,SAAS,KACX,OAAO,mBAAmB,KAAK,IAAI,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,IAAI,QAAQ,IAAI,QAAQ;CAExF,IAAI,SAAS,KAAK;EAChB,IAAI,IAAI,QAAQ,OAAO,KAAK;GAC1B,MAAM,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC;GACxC,OAAO,SAAS,IAAI,QAAQ,IAAI,IAAI;EACtC;EACA,OAAO,mBAAmB,KAAK,IAAI,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,IAAI,QAAQ,IAAI,QAAQ;CACxF;CACA,IAAI,SAAS,KAAK,OAAO,aAAa,KAAK,IAAI,QAAQ,MAAM,EAAE,IAAI,QAAQ,IAAI,QAAQ;CACvF,KAAK,SAAS,OAAO,SAAS,QAAQ,IAAI,QAAQ,OAAO,KAAK;EAC5D,MAAM,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC;EACxC,OAAO,SAAS,IAAI,QAAQ,IAAI,IAAI;CACtC;CACA,IAAI,SAAS,OAAO,IAAI,QAAQ,OAAO,KAAK;EAC1C,MAAM,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC;EACxC,OAAO,SAAS,IAAI,QAAQ,IAAI,IAAI;CACtC;CACA,IAAI,QAAQ,KAAK,QAAQ,EAAE,GAAG;EAC5B,IAAI,IAAI,QAAQ;EAChB,OAAO,QAAQ,KAAK,IAAI,MAAM,EAAE,GAAG,KAAK;EACxC,OAAO;CACT;CACA,OAAO,KAAK,IAAI,QAAQ,GAAG,IAAI,MAAM;AACvC;;AAGA,SAAS,kBAAkB,KAAa,OAAe,KAA8B;CACnF,IAAI,IAAI,QAAQ;CAChB,MAAM,aAAa,MAAM;CACzB,IAAI,IAAI,OAAO,KAAK,OAAO;CAC3B,MAAM,wBAAQ,IAAI,IAAY;CAC9B,OAAO,IAAI,YAAY;EACrB,MAAM,IAAI,IAAI,MAAM;EACpB,IAAI,MAAM,MAAM;GACd,MAAM,OAAO,IAAI,IAAI;GACrB,IAAI,SAAS,KAAA,KAAa,WAAW,KAAK,IAAI,GAAG,OAAO;GACxD,MAAM,IAAI,IAAI;GACd,KAAK;GACL;EACF;EACA,IAAI,IAAI,IAAI,cAAc,IAAI,IAAI,OAAO,KAAK,OAAO;EACrD,MAAM,IAAI,CAAC;EACX,KAAK;CACP;CAEA,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,OAAO,kBAAkB,KAAK,EAAE,CAAC,GAAG,OAAO;CAChE,OAAO,MAAM,SAAS,IAAI,OAAO,CAAC,GAAG,KAAK;AAC5C;;AAGA,SAAS,eAAe,KAAa,GAAuB;CAC1D,MAAM,IAAI,IAAI;CACd,IAAI,MAAM,OAAO,MAAM,OAAO,MAAM,KAAK;EACvC,MAAM,MAAM,IAAI,IAAI,OAAO,MAAM,IAAI,IAAI,IAAI;EAC7C,OAAO;GAAE,QAAQ,MAAM,MAAM,IAAI;GAAG,WAAW;GAAM;EAAI;CAC3D;CACA,IAAI,MAAM,KAAK;EACb,MAAM,QAAQ,IAAI,QAAQ,KAAK,CAAC;EAChC,IAAI,QAAQ,GAAG,OAAO;GAAE,QAAQ;GAAG,WAAW;GAAO,KAAK;EAAE;EAC5D,MAAM,IAAI,qBAAqB,KAAK,IAAI,MAAM,IAAI,GAAG,KAAK,CAAC;EAC3D,IAAI,CAAC,GAAG,OAAO;GAAE,QAAQ;GAAG,WAAW;GAAO,KAAK;EAAE;EACrD,MAAM,MAAM,OAAO,EAAE,EAAE;EACvB,MAAM,MAAM,EAAE,OAAO,KAAA,IAAY,MAAM,EAAE,OAAO,KAAK,OAAO,oBAAoB,OAAO,EAAE,EAAE;EAC3F,MAAM,MAAM,IAAI,QAAQ,OAAO,MAAM,QAAQ,IAAI,QAAQ;EACzD,OAAO;GAAE,QAAQ;GAAK,WAAW,MAAM;GAAK;EAAI;CAClD;CACA,OAAO;EAAE,QAAQ;EAAG,WAAW;EAAO,KAAK;CAAE;AAC/C;AAEA,SAAS,UAAU,KAAa,GAAmB;CACjD,IAAI,IAAI,IAAI;CACZ,IAAI,IAAI,OAAO,KAAK,KAAK;CACzB,OAAO,IAAI,IAAI,UAAU,IAAI,OAAO,KAAK,KAAK,IAAI,OAAO,QAAQ,IAAI,IAAI,IAAI,SAAS,IAAI;CAC1F,OAAO,IAAI,IAAI,SAAS,IAAI,IAAI;AAClC;AAEA,SAAS,UAAU,KAAa,GAAmB;CACjD,IAAI,QAAQ;CACZ,IAAI,IAAI,IAAI;CACZ,OAAO,IAAI,IAAI,UAAU,QAAQ,GAAG;EAClC,MAAM,IAAI,IAAI;EACd,IAAI,MAAM,QAAQ,IAAI,IAAI,IAAI,QAAQ;GACpC,KAAK;GACL;EACF;EACA,IAAI,MAAM,KAAK;GACb,IAAI,UAAU,KAAK,CAAC;GACpB;EACF;EACA,IAAI,MAAM,KAAK,SAAS;OACnB,IAAI,MAAM,KAAK,SAAS;EAC7B,KAAK;CACP;CACA,OAAO;AACT;;;;AClWA,MAAa,qBAAqB;AAYlC,SAAgB,UACd,SACA,OAAqD,CAAC,GAC3C;CACX,IAAI,QAAQ,SAAA,KACV,MAAM,IAAI,WACR,qBAAqB,QAAQ,OAAO,4BAA4B,oBAClE;CAIF,IAAI,KAAK,SAAS,2BAA2B,KAAK,OAAO,GACvD,MAAM,IAAI,YACR,gGACF;CAEF,MAAM,SAAS,KAAK,QAAQ,UAAU,QAAQ,QAAQ,wBAAwB,MAAM;CACpF,MAAM,QAAQ,IAAI,OAAO,QAAQ,KAAK,kBAAkB,QAAQ,OAAO,GAAG;CAC1E,MAAM,WAAuB,CAAC;CAC9B,KAAK,MAAM,YAAY,cAAc,KAAK,GAAG;EAC3C,MAAM,SAAS,SAAS,IAAI,IAAI,CAAC,CAAC,QAAQ,YAAY,MAAM,KAAK,OAAO,CAAC,CAAC,UAAU,CAAC;EACrF,IAAI,OAAO,WAAW,GAAG,OAAO;GAAE;GAAO,UAAU;EAAK;EACxD,SAAS,KAAK,MAAM;CACtB;CACA,OAAO;EAAE;EAAO;CAAS;AAC3B;;;;ACtCA,SAAgB,QAAQ,QAA0B;CAChD,MAAM,MAAM,MAAM,KAAK,MAAM;CAC7B,MAAM,sBAAM,IAAI,IAAY;CAC5B,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,IAAI,QAAQ,KAAK,IAAI,IAAI,IAAI,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;CAC9E,OAAO,CAAC,GAAG,GAAG;AAChB;;AAGA,SAAgB,SAAS,MAAwB;CAC/C,OAAO,QAAQ,KAAK,IAAI,CAAC;AAC3B;;;;;;AAOA,SAAgB,WAAW,MAAsB;CAC/C,MAAM,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,KAAK,MAAM,KAAK,OAAO,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC;CAC7E,IAAI,IAAI,SAAW,IAAI,SAAW,IAAI,OAAS,OAAO,IAAI,KAAK,KAAK,IAAI,KAAK,KAAK;CAClF,OAAO,KAAK,KAAM,IAAI,QAAW,KAAK,KAAM,IAAI,QAAW,KAAK,KAAM,IAAI;AAC5E;;;ACpBA,MAAM,OAAO;;;;;;AAOb,SAAgB,mBAAmB,QAA6B;CAC9D,MAAM,QAAQ,GAAG,OAAO;CACxB,MAAM,SAAS,UACb,UACE,MAAM,SAAS,SAAS,SAAS,KAAK,IAAI,CAAC,CAAC,KAAK,SAAS,CAAC,WAAW,IAAI,GAAG,KAAK,EAAE,CAAC,CAAC,CACxF;CACF,OAAO;EACL,SAAS,OACP,GAAG,KACD,8BAA8B,MAAM,mFACtC;EACF,MAAM,IAAI,IAAI,OAAO;GACnB,KAAK,MAAM,SAAS,MAAM,KAAK,GAC7B,MAAM,GAAG,MACP,yBAAyB,MAAM,oBAAoB,KAAK,qBACxD,CAAC,KAAK,CACR;EAEJ;EACA,MAAM,OAAO,IAAI,OAAO;GACtB,KAAK,MAAM,SAAS,MAAM,KAAK,GAC7B,MAAM,GAAG,MACP,eAAe,MAAM,+BAA+B,KAAK,sBACzD,CAAC,KAAK,CACR;EAEJ;EACA,MAAM,WAAW,IAAI,UAAU,KAAK;GAClC,MAAM,sBAAM,IAAI,IAAY;GAC5B,KAAK,MAAM,UAAU,UAAU;IAE7B,MAAM,QAAQ,CAAC,GAAG,IAAI,IAAI,OAAO,SAAS,YAAY,QAAQ,OAAO,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC;IACxF,MAAM,OAAO,MAAM,GAAG,MACpB,kBAAkB,MAAM,sGACxB;KAAC,KAAK,UAAU,KAAK;KAAG,MAAM;KAAQ;IAAG,CAC3C;IACA,KAAK,MAAM,OAAO,MAAM,IAAI,IAAI,OAAO,IAAI,EAAE,CAAC;GAChD;GACA,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG;EACpD;CACF;AACF;;;ACpBA,MAAM,eAAe;AAErB,IAAa,aAAb,MAAwB;;CAEtB;CACA;CACA;CACA;CACA;CAEA,YAAY,EAAE,IAAI,QAAQ,QAAQ,UAA6B;EAC7D,IAAI,CAAC,eAAe,KAAK,MAAM,GAC7B,MAAM,IAAI,MAAM,kBAAkB,OAAO,eAAe,gBAAgB;EAE1E,IAAI,UAAU,UAAU,UAAU,YAChC,MAAM,IAAI,MAAM,yBAAyB,MAAM,EAAE;EACnD,KAAK,MAAM;EACX,KAAK,UAAU;EACf,KAAK,UAAU,GAAG,OAAO;EACzB,KAAK,SAAS;EACd,KAAK,WAAW,UAAU,SAAS,eAAe,MAAM,IAAI,mBAAmB,MAAM;CACvF;CAEA,MAAM,OAAsB;EAC1B,MAAM,IAAI,KAAK;EACf,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;GAC1C,MAAM,GAAG,KACP,8BAA8B,EAAE,gDAClC;GACA,MAAM,CAAC,OAAO,MAAM,GAAG,MACrB,qBAAqB,EAAE,yBACzB;GACA,IAAI,OAAO,IAAI,UAAU,KAAK,QAC5B,MAAM,IAAI,MACR,iBAAiB,EAAE,4BAA4B,IAAI,MAAM,kBAAkB,KAAK,OAAO,EACzF;GAEF,IAAI,CAAC,KACH,MAAM,GAAG,MAAM,eAAe,EAAE,uCAAuC,CAAC,KAAK,MAAM,CAAC;GACtF,MAAM,GAAG,KACP,8BAA8B,EAAE,kDAClC;GACA,MAAM,KAAK,SAAS,OAAO,EAAE;EAC/B,CAAC;CACH;CAEA,MAAM,OAAO,OAAmC;EAI9C,MAAM,SAAS,CACb,GAAG,IAAI,IAAI,MAAM,KAAK,EAAE,IAAI,WAAW,CAAC,IAAI;GAAE;GAAI,MAAM,KAAK,aAAa;EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAC1F;EACA,IAAI,OAAO,WAAW,GAAG;EACzB,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;GAC1C,MAAM,KAAK,SAAS,OAClB,IACA,MAAM,KAAK,MACT,IACA,OAAO,KAAK,SAAS,KAAK,EAAE,CAC9B,CACF;GACA,KAAK,MAAM,SAAS,UAAU,OAAO,KAAK,SAAS,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,CAAC,GACtE,MAAM,GAAG,MACP,0BAA0B,KAAK,QAAQ,oGACvC,CAAC,KAAK,CACR;GAEF,MAAM,KAAK,SAAS,IAAI,IAAI,MAAM;EACpC,CAAC;CACH;CAEA,MAAM,OAAO,KAA8B;EACzC,IAAI,IAAI,WAAW,GAAG;EACtB,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;GAC1C,MAAM,KAAK,SAAS,OAAO,IAAI,MAAM,KAAK,MAAM,IAAI,GAAG,CAAC;GACxD,KAAK,MAAM,SAAS,UAAU,GAAG,GAC/B,MAAM,GAAG,MACP,eAAe,KAAK,QAAQ,qDAC5B,CAAC,KAAK,CACR;EAEJ,CAAC;CACH;CAEA,MAAM,OAAO,SAAiB,OAA2B,CAAC,GAA+B;EACvF,MAAM,EAAE,QAAQ,KAAK,kBAAkB,KAAQ,QAAQ,WAAW;EAClE,MAAM,OAAO,UAAU,SAAS;GAC9B,OAAO,KAAK;GACZ,eAAe,KAAK,iBAAiB;EACvC,CAAC;EACD,QAAQ,eAAe;EACvB,IAAI,SAAS,GAAG,OAAO;GAAE,MAAM,CAAC;GAAG,WAAW;EAAK;EACnD,MAAM,QACJ,KAAK,aAAa,QAEZ,MAAM,KAAK,IAAI,MACb,kBAAkB,KAAK,QAAQ,4BAC/B,CAAC,kBAAkB,CAAC,CACtB,EAAA,CACA,KAAK,QAAQ,OAAO,IAAI,EAAE,CAAC,IAC7B,MAAM,KAAK,SAAS,WAAW,KAAK,KAAK,KAAK,UAAU,kBAAkB,CAAC;EACjF,IAAI,YAAY,MAAM,SAAS;EAC/B,MAAM,MAAM,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,QAAQ,OAAO,CAAC,UAAU,OAAO,EAAE,CAAC;EAEhF,MAAM,OAAmB,CAAC;EAC1B,IAAI;EACJ,QAAQ,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK,cAAc;GACzD,QAAQ,eAAe;GACvB,KAAK,MAAM,EAAE,IAAI,UAAU,MAAM,KAAK,MAAM,KAAK,KAAK,IAAI,MAAM,GAAG,IAAI,YAAY,CAAC,GAAG;IACrF,OAAO;IACP,MAAM,UAAwB,CAAC;IAC/B,KAAK,MAAM,KAAK,KAAK,SAAS,KAAK,KAAK,GACtC,IAAI,EAAE,EAAE,CAAC,SAAS,GAAG,QAAQ,KAAK;KAAE,OAAO,EAAE;KAAO,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC;IAAO,CAAC;IAElF,IAAI,QAAQ,WAAW,GAAG;IAC1B,KAAK,KAAK;KAAE;KAAI;IAAQ,CAAC;IACzB,IAAI,KAAK,UAAU,OAAO,MAAM;GAClC;EACF;EACA,IAAI,KAAK,UAAU,SAAS,SAAS,IAAI,GAAG,EAAE,GAAG,YAAY;EAC7D,OAAO;GAAE;GAAM;EAAU;CAC3B;;CAGA,MAAM,SAAS,KAA6C;EAC1D,MAAM,QAAQ,MAAM,KAAK,MAAM,KAAK,KAAK,GAAG;EAC5C,OAAO,IAAI,IAAI,MAAM,KAAK,SAAS,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,CAAC;CAC1D;;CAGA,MAAM,OAAsB;EAC1B,MAAM,IAAI,KAAK;EACf,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;GAC1C,KAAK,MAAM,KAAK;IAAC;IAAM;IAAS;IAAS;GAAM,GAC7C,MAAM,GAAG,KAAK,wBAAwB,IAAI,GAAG;EACjD,CAAC;CACH;;CAGA,MAAM,MAAM,IAAQ,KAAqC;EACvD,MAAM,MAAmB,CAAC;EAC1B,KAAK,MAAM,SAAS,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,GAAG;GAC7D,MAAM,OAAO,MAAM,GAAG,MACpB,wBAAwB,KAAK,QAAQ,iEACrC,CAAC,KAAK,CACR;GACA,KAAK,MAAM,OAAO,MAAM,IAAI,KAAK;IAAE,IAAI,OAAO,IAAI,EAAE;IAAG,MAAM,OAAO,IAAI,IAAI;GAAE,CAAC;EACjF;EACA,OAAO;CACT;AACF"}
|
|
1
|
+
{"version":3,"file":"ngram.mjs","names":[],"sources":["../src/ngram/fold.ts","../src/ngram/json-chunks.ts","../src/ngram/fts5-backend.ts","../src/ngram/regex-literals.ts","../src/ngram/plan-query.ts","../src/ngram/trigrams.ts","../src/ngram/postings-backend.ts","../src/ngram/ngram-store.ts"],"sourcesContent":["/**\n * Case fold per code point, applied to indexed text and to query literals alike.\n *\n * Upper-then-lower maps every character a case-insensitive JS regex treats as\n * equal to one form (Σ σ ς → σ, ſ → s). A character whose upper case is several\n * code points (ß → SS) only matches itself under `/i`, so it keeps its own\n * lower case. Folding per code point makes fold(a + b) === fold(a) + fold(b), so\n * a substring of the text stays a substring of the folded text. A character\n * whose folded form has a different UTF-16 length (İ → i + U+0307) is kept as\n * is, so fold preserves the length and offsets into the folded text are\n * offsets into the original.\n */\nexport function fold(text: string): string {\n let out = \"\";\n for (const ch of text) {\n const upper = ch.toUpperCase();\n const lower = (Array.from(upper).length === 1 ? upper : ch).toLowerCase();\n out += lower.length === ch.length ? lower : ch;\n }\n return out;\n}\n\n/**\n * Highest code point probed by `foldIdentity`. Every cased character is in the\n * BMP or the SMP (Deseret, Osage, Adlam, …); planes 2 and up hold none.\n */\nconst PROBE_END = 0x1ffff;\nconst IDENTITY_VERSION = \"fold-v1\";\nlet current: string | undefined;\n\n/**\n * A fingerprint of what `foldFn` does to every code point up to U+1FFFF: a\n * 53-bit hash of each code point it changes and what it changes it to. The\n * grams of an n-gram index are built with `fold`, and a contentless FTS5 row is\n * deleted by replaying them, so a runtime whose case mappings differ (a newer\n * Unicode in a newer V8 or ICU) yields a different identity, and the store\n * rebuilds its grams. It needs no runtime API, so it works in Node, browsers\n * and Durable Objects alike. The identity of `fold` is computed once (about\n * 30 ms) and memoized.\n */\nexport function foldIdentity(foldFn: (text: string) => string = fold): string {\n if (foldFn === fold && current !== undefined) return current;\n // cyrb53: two 32-bit FNV-style lanes mixed into 53 bits.\n let h1 = 0xdeadbeef;\n let h2 = 0x41c6ce57;\n const mix = (value: number) => {\n h1 = Math.imul(h1 ^ value, 2654435761);\n h2 = Math.imul(h2 ^ value, 1597334677);\n };\n for (let cp = 0; cp <= PROBE_END; cp++) {\n if (cp >= 0xd800 && cp <= 0xdfff) continue;\n const ch = String.fromCodePoint(cp);\n const folded = foldFn(ch);\n if (folded === ch) continue;\n mix(cp);\n for (let i = 0; i < folded.length; i++) mix(folded.charCodeAt(i));\n mix(0x110000); // separator: the folded form of one code point may be several units\n }\n h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909);\n h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909);\n const hash = 4294967296 * (2097151 & h2) + (h1 >>> 0);\n const identity = `${IDENTITY_VERSION}:${hash.toString(16)}`;\n if (foldFn === fold) current = identity;\n return identity;\n}\n","/**\n * Serialises rows into JSON array parameters for `json_each(?)`, each at most\n * `maxChars` UTF-16 code units. 500k code units stay under a Durable Object's\n * ~2 MB string limit even at 4 bytes per code point; a single row larger than\n * that still fails there with \"string or blob too big\".\n */\nexport function chunkJson(rows: unknown[], maxChars = 500_000): string[] {\n const chunks: string[] = [];\n let current: string[] = [];\n let size = 2;\n for (const row of rows) {\n const json = JSON.stringify(row);\n if (current.length > 0 && size + json.length + 1 > maxChars) {\n chunks.push(`[${current.join(\",\")}]`);\n current = [];\n size = 2;\n }\n current.push(json);\n size += json.length + 1;\n }\n if (current.length > 0) chunks.push(`[${current.join(\",\")}]`);\n return chunks;\n}\n","import type { Db } from \"@statewalker/db-api\";\nimport { fold } from \"./fold.js\";\nimport { chunkJson } from \"./json-chunks.js\";\n\nexport type NgramItem = { id: number; text: string };\n\n/** How one store finds candidate ids; texts and verification live in NgramStore. */\nexport type GramBackend = {\n create(tx: Db): Promise<void>;\n add(tx: Db, items: NgramItem[]): Promise<void>;\n /** `items` hold the texts as currently stored, so their grams can be removed exactly. */\n remove(tx: Db, items: NgramItem[]): Promise<void>;\n /** Removes every gram, leaving an empty store (used to rebuild after a fold change). */\n clear(tx: Db): Promise<void>;\n /** Ids whose text contains every literal of at least one branch; ascending, at most `max`. */\n candidates(db: Db, branches: string[][], max: number): Promise<number[]>;\n};\n\nconst ROW = \"json_extract(value, '$[0]'), json_extract(value, '$[1]')\";\n\n/**\n * FTS5 trigram over folded text. Contentless (`content=''`), because NgramStore\n * already keeps the text; a contentless row is deleted by replaying its exact\n * indexed value through the 'delete' command, which is why `remove` needs the\n * stored texts. `case_sensitive 1`: folding is done by `fold`, the same function\n * that folds query literals, so the index and the planner can never disagree.\n */\nexport function newFts5Backend(prefix: string): GramBackend {\n const table = `${prefix}ng`;\n const write = async (tx: Db, items: NgramItem[], command: \"insert\" | \"delete\") => {\n const sql =\n command === \"insert\"\n ? `INSERT INTO ${table}(rowid, text) SELECT ${ROW} FROM json_each(?)`\n : `INSERT INTO ${table}(${table}, rowid, text) SELECT 'delete', ${ROW} FROM json_each(?)`;\n for (const chunk of chunkJson(items.map((item) => [item.id, fold(item.text)]))) {\n await tx.query(sql, [chunk]);\n }\n };\n return {\n create: (tx) =>\n tx.exec(\n `CREATE VIRTUAL TABLE IF NOT EXISTS ${table} USING fts5(text, content='', tokenize='trigram case_sensitive 1')`,\n ),\n add: (tx, items) => write(tx, items, \"insert\"),\n remove: (tx, items) => write(tx, items, \"delete\"),\n // 'rebuild' needs a content table; a contentless one can only be emptied.\n clear: (tx) => tx.exec(`INSERT INTO ${table}(${table}) VALUES ('delete-all')`),\n async candidates(db, branches, max) {\n // A quoted string is a phrase of consecutive trigrams, i.e. an exact substring.\n const quote = (literal: string) => `\"${literal.replaceAll('\"', '\"\"')}\"`;\n const match = branches.map((branch) => `(${branch.map(quote).join(\" AND \")})`).join(\" OR \");\n const rows = await db.query<{ id: number }>(\n `SELECT rowid AS id FROM ${table} WHERE ${table} MATCH ? ORDER BY rowid LIMIT ?`,\n [match, max],\n );\n return rows.map((row) => Number(row.id));\n },\n };\n}\n","/*\n * Mandatory-literal extraction from a regex, after Russ Cox's trigram method.\n * Ported from hypgrep 0.5.3 src/regex.js (Hyperparam), MIT License:\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this\n * software and associated documentation files (the \"Software\"), to deal in the Software\n * without restriction, including without limitation the rights to use, copy, modify,\n * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to\n * permit persons to whom the Software is furnished to do so, subject to the following\n * conditions: The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \"AS IS\",\n * WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED.\n */\n\n/** Cross-product cap: past it, the construct counts as a wildcard. */\nconst MAX_DNF = 32;\n\ntype LiveBranch = { literals: string[]; run: string };\ntype Quantifier = { copies: number; openEnded: boolean; end: number };\ntype GroupPrefix = { kind: \"consuming\"; innerStart: number } | { kind: \"assertion\" | \"unknown\" };\n\n/**\n * DNF of literals that every match must contain: a string can match only if all\n * literals of at least one branch occur in it. An empty branch matches anything.\n * Conservative: a construct it cannot reason about becomes a wildcard, never a literal.\n *\n * /abc(def|ghi)/ → [[\"abcdef\"], [\"abcghi\"]] /foo|.+/ → [[\"foo\"], []]\n */\nexport function regexLiterals(regex: RegExp): string[][] {\n // v-mode class syntax desyncs skipClass and would invent literals.\n if (regex.flags.includes(\"v\")) return [[]];\n return expandRegion(regex.source, 0, regex.source.length).dnf;\n}\n\n/** `exact`: every arm is one contiguous literal that is the arm's whole match, with nothing dropped. */\ntype Region = { dnf: string[][]; exact: boolean };\n\nfunction expandRegion(src: string, start: number, end: number): Region {\n const dnf: string[][] = [];\n let exact = true;\n for (const [bStart, bEnd] of splitTopLevelAlternation(src, start, end)) {\n const arm = expandBranch(src, bStart, bEnd);\n for (const branch of arm.dnf) dnf.push(branch);\n exact = exact && arm.exact;\n }\n return { dnf, exact };\n}\n\nfunction splitTopLevelAlternation(\n src: string,\n start: number,\n end: number,\n): Array<[number, number]> {\n const spans: Array<[number, number]> = [];\n let bStart = start;\n let depth = 0;\n let i = start;\n while (i < end) {\n const c = src[i];\n if (c === \"\\\\\") {\n i += 2;\n continue;\n }\n if (c === \"[\") {\n i = skipClass(src, i);\n continue;\n }\n if (c === \"(\") depth += 1;\n else if (c === \")\") depth -= 1;\n else if (c === \"|\" && depth === 0) {\n spans.push([bStart, i]);\n bStart = i + 1;\n }\n i += 1;\n }\n spans.push([bStart, end]);\n return spans;\n}\n\nfunction expandBranch(src: string, start: number, end: number): Region {\n let cross: LiveBranch[] = [{ literals: [], run: \"\" }];\n // Set when part of the match is not captured in a literal (wildcard, optional or repeated\n // atom, assertion): gluing this branch to neighbouring text would be unsound.\n let lossy = false;\n const flushRuns = () => {\n for (const b of cross) {\n if (b.run.length > 0) {\n b.literals.push(b.run);\n b.run = \"\";\n }\n }\n };\n // Append `text` repeated by the quantifier's mandatory copies to every live run.\n const appendRun = (text: string, q: Quantifier) => {\n if (q.copies === 0) {\n lossy = true;\n flushRuns();\n return;\n }\n for (const b of cross) b.run += text.repeat(q.copies);\n if (q.openEnded) {\n lossy = true;\n flushRuns();\n }\n };\n\n let i = start;\n while (i < end) {\n const c = src[i] ?? \"\";\n\n if (c === \"^\" || c === \"$\") {\n i += 1;\n continue;\n }\n\n if (c === \"\\\\\") {\n const next = src[i + 1];\n if (next === undefined) {\n i += 1;\n continue;\n }\n const escaped = parseEscapedLiteral(src, i);\n if (escaped) {\n const q = peekQuantifier(src, escaped.end);\n i = q.end;\n appendRun(escaped.char, q);\n continue;\n }\n // \\d \\w \\s \\b, backrefs, \\p{…}: a wildcard; consume its payload.\n if (/[a-zA-Z0-9]/.test(next)) {\n lossy = true;\n flushRuns();\n i = peekQuantifier(src, skipSpecialEscape(src, i)).end;\n continue;\n }\n // \\. \\/ \\( …: the literal character.\n const q = peekQuantifier(src, i + 2);\n i = q.end;\n appendRun(next, q);\n continue;\n }\n\n if (c === \"[\") {\n const classEnd = skipClass(src, i);\n const chars = parseLiteralClass(src, i, classEnd);\n const q = peekQuantifier(src, classEnd);\n i = q.end;\n if (\n q.copies === 0 ||\n chars === null ||\n q.copies > 1 ||\n cross.length * chars.length > MAX_DNF\n ) {\n lossy = true;\n flushRuns();\n continue;\n }\n cross = cross.flatMap((b) =>\n chars.map((ch) => ({ literals: b.literals.slice(), run: b.run + ch })),\n );\n if (q.openEnded) {\n lossy = true;\n flushRuns();\n }\n continue;\n }\n\n if (c === \"(\") {\n const groupEnd = skipGroup(src, i);\n const group = parseGroupPrefix(src, i);\n const q = peekQuantifier(src, groupEnd);\n i = q.end;\n if (group.kind !== \"consuming\" || q.copies === 0) {\n lossy = true;\n flushRuns();\n continue;\n }\n const { dnf: inner, exact } = expandRegion(src, group.innerStart, groupEnd - 1);\n // Fold into the run only when every arm is one contiguous literal used once and is the\n // arm's whole match: a repeated group (`(a|b){3}`) may pick different arms per copy, and\n // a wildcard at an arm's edge (`x(.abc)`) must not be glued away.\n const foldable =\n exact && q.copies === 1 && inner.length > 0 && inner.every((b) => b.length === 1);\n lossy = lossy || q.openEnded || !foldable;\n if (foldable && cross.length * inner.length <= MAX_DNF) {\n cross = cross.flatMap((b) =>\n inner.map((arm) => ({ literals: b.literals.slice(), run: b.run + (arm[0] ?? \"\") })),\n );\n if (q.openEnded) flushRuns();\n continue;\n }\n lossy = true;\n flushRuns();\n if (cross.length * inner.length > MAX_DNF) continue;\n cross = cross.flatMap((b) =>\n inner.map((arm) => ({ literals: b.literals.concat(arm), run: \"\" })),\n );\n continue;\n }\n\n if (c === \".\") {\n lossy = true;\n flushRuns();\n i = peekQuantifier(src, i + 1).end;\n continue;\n }\n\n const q = peekQuantifier(src, i + 1);\n i = q.end;\n appendRun(c, q);\n }\n flushRuns();\n return {\n dnf: cross.map((b) => b.literals),\n exact: !lossy && cross.every((b) => b.literals.length === 1),\n };\n}\n\n/** Where a group's consuming body starts; lookarounds and unknown `(?…)` are not consuming. */\nfunction parseGroupPrefix(src: string, start: number): GroupPrefix {\n if (src[start + 1] !== \"?\") return { kind: \"consuming\", innerStart: start + 1 };\n const marker = src[start + 2];\n if (marker === \":\") return { kind: \"consuming\", innerStart: start + 3 };\n if (marker === \"=\" || marker === \"!\") return { kind: \"assertion\" };\n if (marker === \"<\") {\n const next = src[start + 3];\n if (next === \"=\" || next === \"!\") return { kind: \"assertion\" };\n const nameEnd = src.indexOf(\">\", start + 3);\n if (nameEnd >= 0) return { kind: \"consuming\", innerStart: nameEnd + 1 };\n }\n return { kind: \"unknown\" };\n}\n\n/**\n * `\\xHH`, `\\uHHHH`: escapes that stand for one literal character. `\\u{…}` is not one:\n * the planner compiles without the `u` flag, where it means \"u\" repeated or the text \"u{…}\".\n */\nfunction parseEscapedLiteral(\n src: string,\n start: number,\n): { char: string; end: number } | undefined {\n const kind = src[start + 1];\n if (kind === \"x\") {\n const hex = src.slice(start + 2, start + 4);\n if (!/^[0-9a-fA-F]{2}$/.test(hex)) return undefined;\n return { char: String.fromCharCode(Number.parseInt(hex, 16)), end: start + 4 };\n }\n if (kind === \"u\") {\n const hex = src.slice(start + 2, start + 6);\n if (!/^[0-9a-fA-F]{4}$/.test(hex)) return undefined;\n return { char: String.fromCharCode(Number.parseInt(hex, 16)), end: start + 6 };\n }\n return undefined;\n}\n\n/**\n * Skip an escape that is not a literal punctuation character, so its payload\n * (`\\x41`, `\\p{L}`, `\\k<name>`, backrefs) never leaks into a literal. Malformed\n * `\\x`, `\\u`, `\\c` payloads are Annex-B literals and are rescanned as atoms.\n */\nfunction skipSpecialEscape(src: string, start: number): number {\n const next = src[start + 1];\n if (next === \"x\") {\n return /^[0-9a-fA-F]{2}$/.test(src.slice(start + 2, start + 4)) ? start + 4 : start + 2;\n }\n if (next === \"u\") {\n if (src[start + 2] === \"{\") {\n const close = src.indexOf(\"}\", start + 3);\n return close >= 0 ? close + 1 : src.length;\n }\n return /^[0-9a-fA-F]{4}$/.test(src.slice(start + 2, start + 6)) ? start + 6 : start + 2;\n }\n if (next === \"c\") return /^[a-zA-Z]$/.test(src[start + 2] ?? \"\") ? start + 3 : start + 2;\n if ((next === \"p\" || next === \"P\") && src[start + 2] === \"{\") {\n const close = src.indexOf(\"}\", start + 3);\n return close >= 0 ? close + 1 : src.length;\n }\n if (next === \"k\" && src[start + 2] === \"<\") {\n const close = src.indexOf(\">\", start + 3);\n return close >= 0 ? close + 1 : src.length;\n }\n if (/[0-9]/.test(next ?? \"\")) {\n let i = start + 2;\n while (/[0-9]/.test(src[i] ?? \"\")) i += 1;\n return i;\n }\n return Math.min(start + 2, src.length);\n}\n\n/** A `[...]` class as literal alternatives, or null if negated, ranged, special or empty. */\nfunction parseLiteralClass(src: string, start: number, end: number): string[] | null {\n let i = start + 1;\n const lastInside = end - 1;\n if (src[i] === \"^\") return null;\n const chars = new Set<string>();\n while (i < lastInside) {\n const c = src[i] ?? \"\";\n if (c === \"\\\\\") {\n const next = src[i + 1];\n // Letters are classes or code escapes; digits are octal escapes without the `u` flag.\n if (next === undefined || /[a-zA-Z0-9]/.test(next)) return null;\n i += 2;\n // `[\\.-z]`: the escape starts a range, not three literals.\n if (src[i] === \"-\" && i + 1 < lastInside) return null;\n chars.add(next);\n continue;\n }\n if (i + 2 < lastInside && src[i + 1] === \"-\") return null;\n chars.add(c);\n i += 1;\n }\n // A surrogate half would split an astral character the index sees as one code point.\n if ([...chars].some((ch) => /[\\ud800-\\udfff]/.test(ch))) return null;\n return chars.size === 0 ? null : [...chars];\n}\n\n/** Mandatory copies of the preceding atom, whether more may follow, and where the quantifier ends. */\nfunction peekQuantifier(src: string, i: number): Quantifier {\n const c = src[i];\n if (c === \"?\" || c === \"*\" || c === \"+\") {\n const end = src[i + 1] === \"?\" ? i + 2 : i + 1;\n return { copies: c === \"+\" ? 1 : 0, openEnded: true, end };\n }\n if (c === \"{\") {\n const close = src.indexOf(\"}\", i);\n if (close < 0) return { copies: 1, openEnded: false, end: i };\n const m = /^(\\d+)(?:,(\\d*))?$/.exec(src.slice(i + 1, close));\n if (!m) return { copies: 1, openEnded: false, end: i };\n const min = Number(m[1]);\n const max = m[2] === undefined ? min : m[2] === \"\" ? Number.POSITIVE_INFINITY : Number(m[2]);\n const end = src[close + 1] === \"?\" ? close + 2 : close + 1;\n return { copies: min, openEnded: max > min, end };\n }\n return { copies: 1, openEnded: false, end: i };\n}\n\nfunction skipClass(src: string, i: number): number {\n let j = i + 1;\n if (src[j] === \"^\") j += 1;\n while (j < src.length && src[j] !== \"]\") j += src[j] === \"\\\\\" && j + 1 < src.length ? 2 : 1;\n return j < src.length ? j + 1 : j;\n}\n\nfunction skipGroup(src: string, i: number): number {\n let depth = 1;\n let j = i + 1;\n while (j < src.length && depth > 0) {\n const c = src[j];\n if (c === \"\\\\\" && j + 1 < src.length) {\n j += 2;\n continue;\n }\n if (c === \"[\") {\n j = skipClass(src, j);\n continue;\n }\n if (c === \"(\") depth += 1;\n else if (c === \")\") depth -= 1;\n j += 1;\n }\n return j;\n}\n","import { fold } from \"./fold.js\";\nimport { regexLiterals } from \"./regex-literals.js\";\n\n/** Longer patterns are rejected: a JS regex cannot be interrupted once it runs. */\nexport const MAX_PATTERN_LENGTH = 1000;\n\nconst LONE_SURROGATE = /[\\ud800-\\udbff](?![\\udc00-\\udfff])|(?<![\\ud800-\\udbff])[\\udc00-\\udfff]/;\n\nexport type QueryPlan = {\n /** The pattern as a global regex (`i` added when not case-sensitive); verifies candidates. */\n regex: RegExp;\n /**\n * Per DNF branch, the folded literals of at least 3 code points that a match must\n * contain. `null` when some branch has none: only a scan can answer the pattern.\n */\n branches: string[][] | null;\n};\n\nexport function planQuery(\n pattern: string,\n opts: { regex?: boolean; caseSensitive?: boolean } = {},\n): QueryPlan {\n if (pattern.length > MAX_PATTERN_LENGTH) {\n throw new RangeError(\n `ngram: pattern is ${pattern.length} characters, the limit is ${MAX_PATTERN_LENGTH}`,\n );\n }\n // The regex has no `u` flag (case folding is unsound under /iu), so `\\p{…}` would silently\n // match the literal text \"p{…}\". An odd run of backslashes before `p` is a real escape.\n if (opts.regex && /(?<!\\\\)(?:\\\\\\\\)*\\\\[pP]\\{/.test(pattern)) {\n throw new SyntaxError(\n \"ngram: \\\\p{…} and \\\\P{…} are not supported in regex mode (patterns compile without the u flag)\",\n );\n }\n const source = opts.regex ? pattern : pattern.replace(/[.*+?^${}()|[\\]\\\\/]/g, \"\\\\$&\");\n const regex = new RegExp(source, opts.caseSensitive === false ? \"gi\" : \"g\");\n const branches: string[][] = [];\n for (const literals of regexLiterals(regex)) {\n // Without the `u` flag a literal may hold a lone surrogate (`ab😀?`, `\\uD83D`); indexed\n // text is split by code point, so keep only the well-formed pieces around it.\n const usable = literals\n .flatMap((literal) => literal.split(LONE_SURROGATE))\n .map(fold)\n .filter((literal) => Array.from(literal).length >= 3);\n if (usable.length === 0) return { regex, branches: null };\n branches.push(usable);\n }\n return { regex, branches };\n}\n","import { fold } from \"./fold.js\";\n\n/** Distinct 3-code-point grams of an already folded string, in first-seen order. */\nexport function gramsOf(folded: string): string[] {\n const cps = Array.from(folded);\n const out = new Set<string>();\n for (let i = 0; i + 3 <= cps.length; i++) out.add(cps.slice(i, i + 3).join(\"\"));\n return [...out];\n}\n\n/** Distinct trigrams of `text` as the index sees it: case-folded, by code point. */\nexport function trigrams(text: string): string[] {\n return gramsOf(fold(text));\n}\n\n/**\n * One SQLite INTEGER per gram. Three BMP code points pack exactly into 48 bits.\n * A gram with an astral code point keeps the low 16 bits of each code point\n * above 2^48: a collision only adds candidates, which verification removes.\n */\nexport function encodeGram(gram: string): number {\n const [a = 0, b = 0, c = 0] = Array.from(gram, (ch) => ch.codePointAt(0) ?? 0);\n if (a < 0x10000 && b < 0x10000 && c < 0x10000) return a * 2 ** 32 + b * 2 ** 16 + c;\n return 2 ** 48 + (a % 0x10000) * 2 ** 32 + (b % 0x10000) * 2 ** 16 + (c % 0x10000);\n}\n","import type { GramBackend, NgramItem } from \"./fts5-backend.js\";\nimport { chunkJson } from \"./json-chunks.js\";\nimport { encodeGram, gramsOf, trigrams } from \"./trigrams.js\";\n\nconst PAIR = \"json_extract(value, '$[0]'), json_extract(value, '$[1]')\";\n\n/**\n * One row per (gram, id) in a WITHOUT ROWID table: the primary key is the\n * posting list, clustered by gram. A text is a candidate for a branch when it\n * holds every distinct gram of the branch's literals.\n */\nexport function newPostingsBackend(prefix: string): GramBackend {\n const table = `${prefix}grams`;\n const pairs = (items: NgramItem[]) =>\n chunkJson(\n items.flatMap((item) => trigrams(item.text).map((gram) => [encodeGram(gram), item.id])),\n );\n return {\n create: (tx) =>\n tx.exec(\n `CREATE TABLE IF NOT EXISTS ${table}(gram INTEGER NOT NULL, id INTEGER NOT NULL, PRIMARY KEY (gram, id)) WITHOUT ROWID`,\n ),\n async add(tx, items) {\n for (const chunk of pairs(items)) {\n await tx.query(\n `INSERT OR IGNORE INTO ${table}(gram, id) SELECT ${PAIR} FROM json_each(?)`,\n [chunk],\n );\n }\n },\n async remove(tx, items) {\n for (const chunk of pairs(items)) {\n await tx.query(\n `DELETE FROM ${table} WHERE (gram, id) IN (SELECT ${PAIR} FROM json_each(?))`,\n [chunk],\n );\n }\n },\n clear: (tx) => tx.exec(`DELETE FROM ${table}`),\n async candidates(db, branches, max) {\n const ids = new Set<number>();\n for (const branch of branches) {\n // Literals are already folded: split them without folding again.\n const grams = [...new Set(branch.flatMap((literal) => gramsOf(literal).map(encodeGram)))];\n const rows = await db.query<{ id: number }>(\n `SELECT id FROM ${table} WHERE gram IN (SELECT value FROM json_each(?)) GROUP BY id HAVING count(*) = ? ORDER BY id LIMIT ?`,\n [JSON.stringify(grams), grams.length, max],\n );\n for (const row of rows) ids.add(Number(row.id));\n }\n return [...ids].sort((a, b) => a - b).slice(0, max);\n },\n };\n}\n","import type { Db } from \"@statewalker/db-api\";\nimport { inTransaction } from \"@statewalker/db-sqlite\";\nimport { PREFIX_PATTERN } from \"../prefix.js\";\nimport { foldIdentity } from \"./fold.js\";\nimport { type GramBackend, type NgramItem, newFts5Backend } from \"./fts5-backend.js\";\nimport { chunkJson } from \"./json-chunks.js\";\nimport { planQuery } from \"./plan-query.js\";\nimport { newPostingsBackend } from \"./postings-backend.js\";\n\nexport type { NgramItem };\nexport type NgramMatch = { start: number; end: number };\nexport type NgramHit = { id: number; matches: NgramMatch[] };\nexport type NgramSearchOptions = {\n /** Treat `pattern` as a JS regex source; otherwise it is literal text. */\n regex?: boolean;\n /** Default true (grep semantics). */\n caseSensitive?: boolean;\n /** Maximum hits. Default 100. */\n limit?: number;\n /** Maximum candidates verified. Default 10,000. */\n candidateBudget?: number;\n /** Keep only ids for which this returns true (applied before verification). */\n filter?: (id: number) => boolean;\n signal?: AbortSignal;\n};\nexport type NgramSearchResult = {\n /** Ascending id order; every non-empty match, offsets in UTF-16 code units. */\n hits: NgramHit[];\n /** True when the search stopped before checking every candidate (budget or limit). */\n truncated: boolean;\n};\nexport type NgramStoreOptions = { db: Db; prefix: string; store?: \"fts5\" | \"postings\" };\n\nconst VERIFY_BATCH = 256;\n/** Texts re-indexed per step when the grams are rebuilt after a fold change. */\nconst REBUILD_BATCH = 256;\n\nexport class NgramStore {\n /** `{prefix}texts`: its `id` column holds exactly the stored ids. Callers may join it in SQL. */\n readonly idTable: string;\n readonly #db: Db;\n readonly #prefix: string;\n readonly #store: \"fts5\" | \"postings\";\n readonly #backend: GramBackend;\n\n constructor({ db, prefix, store = \"fts5\" }: NgramStoreOptions) {\n if (!PREFIX_PATTERN.test(prefix)) {\n throw new Error(`ngram: prefix \"${prefix}\" must match ${PREFIX_PATTERN}`);\n }\n if (store !== \"fts5\" && store !== \"postings\")\n throw new Error(`ngram: unknown store \"${store}\"`);\n this.#db = db;\n this.#prefix = prefix;\n this.idTable = `${prefix}texts`;\n this.#store = store;\n this.#backend = store === \"fts5\" ? newFts5Backend(prefix) : newPostingsBackend(prefix);\n }\n\n async init(): Promise<void> {\n const p = this.#prefix;\n await inTransaction(this.#db, async (tx) => {\n await tx.exec(\n `CREATE TABLE IF NOT EXISTS ${p}meta(key TEXT PRIMARY KEY, value TEXT NOT NULL)`,\n );\n const [row] = await tx.query<{ value: string }>(\n `SELECT value FROM ${p}meta WHERE key = 'store'`,\n );\n if (row && row.value !== this.#store) {\n throw new Error(\n `ngram: index \"${p}\" was created with store \"${row.value}\", opened with \"${this.#store}\"`,\n );\n }\n if (!row)\n await tx.query(`INSERT INTO ${p}meta(key, value) VALUES ('store', ?)`, [this.#store]);\n await tx.exec(\n `CREATE TABLE IF NOT EXISTS ${p}texts(id INTEGER PRIMARY KEY, text TEXT NOT NULL)`,\n );\n await this.#backend.create(tx);\n await this.#checkFold(tx);\n });\n }\n\n /**\n * The grams were built with `fold` as the runtime that wrote them defined it, and\n * an fts5 delete replays them; if this runtime folds differently, they are rebuilt\n * from the stored texts before any write. An index without a recorded identity\n * (written before it was recorded) is assumed to match and is not rebuilt.\n */\n async #checkFold(tx: Db): Promise<void> {\n const p = this.#prefix;\n const identity = foldIdentity();\n const [row] = await tx.query<{ value: string }>(\n `SELECT value FROM ${p}meta WHERE key = 'fold'`,\n );\n if (row?.value === identity) return;\n if (row) await this.#rebuild(tx);\n await tx.query(`INSERT OR REPLACE INTO ${p}meta(key, value) VALUES ('fold', ?)`, [identity]);\n }\n\n async #rebuild(tx: Db): Promise<void> {\n await this.#backend.clear(tx);\n let after: number | null = null;\n for (;;) {\n const rows: Array<{ id: number; text: string }> = await tx.query(\n `SELECT id, text FROM ${this.#prefix}texts WHERE ? IS NULL OR id > ? ORDER BY id LIMIT ?`,\n [after, after, REBUILD_BATCH],\n );\n const batch = rows.map((r) => ({ id: Number(r.id), text: String(r.text) }));\n const last = batch.at(-1);\n if (!last) return;\n await this.#backend.add(tx, batch);\n after = last.id;\n }\n }\n\n async upsert(items: NgramItem[]): Promise<void> {\n // The last text per id wins; a repeated id would leave stale grams behind.\n // A lone surrogate would be stored differently from how it was indexed, so a\n // later delete could not replay it: texts are made well-formed first.\n const latest = [\n ...new Map(items.map(({ id, text }) => [id, { id, text: text.toWellFormed() }])).values(),\n ];\n if (latest.length === 0) return;\n await inTransaction(this.#db, async (tx) => {\n await this.#backend.remove(\n tx,\n await this.#load(\n tx,\n latest.map((item) => item.id),\n ),\n );\n for (const chunk of chunkJson(latest.map((item) => [item.id, item.text]))) {\n await tx.query(\n `INSERT OR REPLACE INTO ${this.#prefix}texts(id, text) SELECT json_extract(value, '$[0]'), json_extract(value, '$[1]') FROM json_each(?)`,\n [chunk],\n );\n }\n await this.#backend.add(tx, latest);\n });\n }\n\n async delete(ids: number[]): Promise<void> {\n if (ids.length === 0) return;\n await inTransaction(this.#db, async (tx) => {\n await this.#backend.remove(tx, await this.#load(tx, ids));\n for (const chunk of chunkJson(ids)) {\n await tx.query(\n `DELETE FROM ${this.#prefix}texts WHERE id IN (SELECT value FROM json_each(?))`,\n [chunk],\n );\n }\n });\n }\n\n async search(pattern: string, opts: NgramSearchOptions = {}): Promise<NgramSearchResult> {\n const { limit = 100, candidateBudget = 10_000, filter, signal } = opts;\n const plan = planQuery(pattern, {\n regex: opts.regex,\n caseSensitive: opts.caseSensitive ?? true,\n });\n signal?.throwIfAborted();\n if (limit <= 0) return { hits: [], truncated: true };\n const found =\n plan.branches === null\n ? (\n await this.#db.query<{ id: number }>(\n `SELECT id FROM ${this.#prefix}texts ORDER BY id LIMIT ?`,\n [candidateBudget + 1],\n )\n ).map((row) => Number(row.id))\n : await this.#backend.candidates(this.#db, plan.branches, candidateBudget + 1);\n let truncated = found.length > candidateBudget;\n const ids = found.slice(0, candidateBudget).filter((id) => !filter || filter(id));\n\n const hits: NgramHit[] = [];\n let last: number | undefined;\n verify: for (let i = 0; i < ids.length; i += VERIFY_BATCH) {\n signal?.throwIfAborted();\n for (const { id, text } of await this.#load(this.#db, ids.slice(i, i + VERIFY_BATCH))) {\n last = id;\n const matches: NgramMatch[] = [];\n for (const m of text.matchAll(plan.regex)) {\n if (m[0].length > 0) matches.push({ start: m.index, end: m.index + m[0].length });\n }\n if (matches.length === 0) continue;\n hits.push({ id, matches });\n if (hits.length >= limit) break verify;\n }\n }\n if (hits.length >= limit && last !== ids.at(-1)) truncated = true;\n return { hits, truncated };\n }\n\n /** The stored original texts of `ids`; unknown ids are absent. */\n async getTexts(ids: number[]): Promise<Map<number, string>> {\n const items = await this.#load(this.#db, ids);\n return new Map(items.map((item) => [item.id, item.text]));\n }\n\n /** Drops the tables of both stores, so it works without knowing which one created the prefix. */\n async drop(): Promise<void> {\n const p = this.#prefix;\n await inTransaction(this.#db, async (tx) => {\n for (const t of [\"ng\", \"grams\", \"texts\", \"meta\"])\n await tx.exec(`DROP TABLE IF EXISTS ${p}${t}`);\n });\n }\n\n /** Stored texts for `ids`, ascending by id; missing ids are skipped. */\n async #load(db: Db, ids: number[]): Promise<NgramItem[]> {\n const out: NgramItem[] = [];\n for (const chunk of chunkJson([...ids].sort((a, b) => a - b))) {\n const rows = await db.query<{ id: number; text: string }>(\n `SELECT id, text FROM ${this.#prefix}texts WHERE id IN (SELECT value FROM json_each(?)) ORDER BY id`,\n [chunk],\n );\n for (const row of rows) out.push({ id: Number(row.id), text: String(row.text) });\n }\n return out;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;AAYA,SAAgB,KAAK,MAAsB;CACzC,IAAI,MAAM;CACV,KAAK,MAAM,MAAM,MAAM;EACrB,MAAM,QAAQ,GAAG,YAAY;EAC7B,MAAM,SAAS,MAAM,KAAK,KAAK,CAAC,CAAC,WAAW,IAAI,QAAQ,GAAA,CAAI,YAAY;EACxE,OAAO,MAAM,WAAW,GAAG,SAAS,QAAQ;CAC9C;CACA,OAAO;AACT;;;;;AAMA,MAAM,YAAY;AAClB,MAAM,mBAAmB;AACzB,IAAI;;;;;;;;;;;AAYJ,SAAgB,aAAa,SAAmC,MAAc;CAC5E,IAAI,WAAW,QAAQ,YAAY,KAAA,GAAW,OAAO;CAErD,IAAI,KAAK;CACT,IAAI,KAAK;CACT,MAAM,OAAO,UAAkB;EAC7B,KAAK,KAAK,KAAK,KAAK,OAAO,UAAU;EACrC,KAAK,KAAK,KAAK,KAAK,OAAO,UAAU;CACvC;CACA,KAAK,IAAI,KAAK,GAAG,MAAM,WAAW,MAAM;EACtC,IAAI,MAAM,SAAU,MAAM,OAAQ;EAClC,MAAM,KAAK,OAAO,cAAc,EAAE;EAClC,MAAM,SAAS,OAAO,EAAE;EACxB,IAAI,WAAW,IAAI;EACnB,IAAI,EAAE;EACN,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,IAAI,OAAO,WAAW,CAAC,CAAC;EAChE,IAAI,OAAQ;CACd;CACA,KAAK,KAAK,KAAK,KAAM,OAAO,IAAK,UAAU,IAAI,KAAK,KAAK,KAAM,OAAO,IAAK,UAAU;CACrF,KAAK,KAAK,KAAK,KAAM,OAAO,IAAK,UAAU,IAAI,KAAK,KAAK,KAAM,OAAO,IAAK,UAAU;CACrF,MAAM,OAAO,cAAc,UAAU,OAAO,OAAO;CACnD,MAAM,WAAW,GAAG,iBAAiB,GAAG,KAAK,SAAS,EAAE;CACxD,IAAI,WAAW,MAAM,UAAU;CAC/B,OAAO;AACT;;;;;;;;;AC1DA,SAAgB,UAAU,MAAiB,WAAW,KAAmB;CACvE,MAAM,SAAmB,CAAC;CAC1B,IAAI,UAAoB,CAAC;CACzB,IAAI,OAAO;CACX,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,OAAO,KAAK,UAAU,GAAG;EAC/B,IAAI,QAAQ,SAAS,KAAK,OAAO,KAAK,SAAS,IAAI,UAAU;GAC3D,OAAO,KAAK,IAAI,QAAQ,KAAK,GAAG,EAAE,EAAE;GACpC,UAAU,CAAC;GACX,OAAO;EACT;EACA,QAAQ,KAAK,IAAI;EACjB,QAAQ,KAAK,SAAS;CACxB;CACA,IAAI,QAAQ,SAAS,GAAG,OAAO,KAAK,IAAI,QAAQ,KAAK,GAAG,EAAE,EAAE;CAC5D,OAAO;AACT;;;ACJA,MAAM,MAAM;;;;;;;;AASZ,SAAgB,eAAe,QAA6B;CAC1D,MAAM,QAAQ,GAAG,OAAO;CACxB,MAAM,QAAQ,OAAO,IAAQ,OAAoB,YAAiC;EAChF,MAAM,MACJ,YAAY,WACR,eAAe,MAAM,uBAAuB,IAAI,sBAChD,eAAe,MAAM,GAAG,MAAM,kCAAkC,IAAI;EAC1E,KAAK,MAAM,SAAS,UAAU,MAAM,KAAK,SAAS,CAAC,KAAK,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,GAC3E,MAAM,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC;CAE/B;CACA,OAAO;EACL,SAAS,OACP,GAAG,KACD,sCAAsC,MAAM,mEAC9C;EACF,MAAM,IAAI,UAAU,MAAM,IAAI,OAAO,QAAQ;EAC7C,SAAS,IAAI,UAAU,MAAM,IAAI,OAAO,QAAQ;EAEhD,QAAQ,OAAO,GAAG,KAAK,eAAe,MAAM,GAAG,MAAM,wBAAwB;EAC7E,MAAM,WAAW,IAAI,UAAU,KAAK;GAElC,MAAM,SAAS,YAAoB,IAAI,QAAQ,WAAW,MAAK,MAAI,EAAE;GACrE,MAAM,QAAQ,SAAS,KAAK,WAAW,IAAI,OAAO,IAAI,KAAK,CAAC,CAAC,KAAK,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM;GAK1F,QAAO,MAJY,GAAG,MACpB,2BAA2B,MAAM,SAAS,MAAM,kCAChD,CAAC,OAAO,GAAG,CACb,EAAA,CACY,KAAK,QAAQ,OAAO,IAAI,EAAE,CAAC;EACzC;CACF;AACF;;;;AC3CA,MAAM,UAAU;;;;;;;;AAahB,SAAgB,cAAc,OAA2B;CAEvD,IAAI,MAAM,MAAM,SAAS,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC;CACzC,OAAO,aAAa,MAAM,QAAQ,GAAG,MAAM,OAAO,MAAM,CAAC,CAAC;AAC5D;AAKA,SAAS,aAAa,KAAa,OAAe,KAAqB;CACrE,MAAM,MAAkB,CAAC;CACzB,IAAI,QAAQ;CACZ,KAAK,MAAM,CAAC,QAAQ,SAAS,yBAAyB,KAAK,OAAO,GAAG,GAAG;EACtE,MAAM,MAAM,aAAa,KAAK,QAAQ,IAAI;EAC1C,KAAK,MAAM,UAAU,IAAI,KAAK,IAAI,KAAK,MAAM;EAC7C,QAAQ,SAAS,IAAI;CACvB;CACA,OAAO;EAAE;EAAK;CAAM;AACtB;AAEA,SAAS,yBACP,KACA,OACA,KACyB;CACzB,MAAM,QAAiC,CAAC;CACxC,IAAI,SAAS;CACb,IAAI,QAAQ;CACZ,IAAI,IAAI;CACR,OAAO,IAAI,KAAK;EACd,MAAM,IAAI,IAAI;EACd,IAAI,MAAM,MAAM;GACd,KAAK;GACL;EACF;EACA,IAAI,MAAM,KAAK;GACb,IAAI,UAAU,KAAK,CAAC;GACpB;EACF;EACA,IAAI,MAAM,KAAK,SAAS;OACnB,IAAI,MAAM,KAAK,SAAS;OACxB,IAAI,MAAM,OAAO,UAAU,GAAG;GACjC,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;GACtB,SAAS,IAAI;EACf;EACA,KAAK;CACP;CACA,MAAM,KAAK,CAAC,QAAQ,GAAG,CAAC;CACxB,OAAO;AACT;AAEA,SAAS,aAAa,KAAa,OAAe,KAAqB;CACrE,IAAI,QAAsB,CAAC;EAAE,UAAU,CAAC;EAAG,KAAK;CAAG,CAAC;CAGpD,IAAI,QAAQ;CACZ,MAAM,kBAAkB;EACtB,KAAK,MAAM,KAAK,OACd,IAAI,EAAE,IAAI,SAAS,GAAG;GACpB,EAAE,SAAS,KAAK,EAAE,GAAG;GACrB,EAAE,MAAM;EACV;CAEJ;CAEA,MAAM,aAAa,MAAc,MAAkB;EACjD,IAAI,EAAE,WAAW,GAAG;GAClB,QAAQ;GACR,UAAU;GACV;EACF;EACA,KAAK,MAAM,KAAK,OAAO,EAAE,OAAO,KAAK,OAAO,EAAE,MAAM;EACpD,IAAI,EAAE,WAAW;GACf,QAAQ;GACR,UAAU;EACZ;CACF;CAEA,IAAI,IAAI;CACR,OAAO,IAAI,KAAK;EACd,MAAM,IAAI,IAAI,MAAM;EAEpB,IAAI,MAAM,OAAO,MAAM,KAAK;GAC1B,KAAK;GACL;EACF;EAEA,IAAI,MAAM,MAAM;GACd,MAAM,OAAO,IAAI,IAAI;GACrB,IAAI,SAAS,KAAA,GAAW;IACtB,KAAK;IACL;GACF;GACA,MAAM,UAAU,oBAAoB,KAAK,CAAC;GAC1C,IAAI,SAAS;IACX,MAAM,IAAI,eAAe,KAAK,QAAQ,GAAG;IACzC,IAAI,EAAE;IACN,UAAU,QAAQ,MAAM,CAAC;IACzB;GACF;GAEA,IAAI,cAAc,KAAK,IAAI,GAAG;IAC5B,QAAQ;IACR,UAAU;IACV,IAAI,eAAe,KAAK,kBAAkB,KAAK,CAAC,CAAC,CAAC,CAAC;IACnD;GACF;GAEA,MAAM,IAAI,eAAe,KAAK,IAAI,CAAC;GACnC,IAAI,EAAE;GACN,UAAU,MAAM,CAAC;GACjB;EACF;EAEA,IAAI,MAAM,KAAK;GACb,MAAM,WAAW,UAAU,KAAK,CAAC;GACjC,MAAM,QAAQ,kBAAkB,KAAK,GAAG,QAAQ;GAChD,MAAM,IAAI,eAAe,KAAK,QAAQ;GACtC,IAAI,EAAE;GACN,IACE,EAAE,WAAW,KACb,UAAU,QACV,EAAE,SAAS,KACX,MAAM,SAAS,MAAM,SAAS,SAC9B;IACA,QAAQ;IACR,UAAU;IACV;GACF;GACA,QAAQ,MAAM,SAAS,MACrB,MAAM,KAAK,QAAQ;IAAE,UAAU,EAAE,SAAS,MAAM;IAAG,KAAK,EAAE,MAAM;GAAG,EAAE,CACvE;GACA,IAAI,EAAE,WAAW;IACf,QAAQ;IACR,UAAU;GACZ;GACA;EACF;EAEA,IAAI,MAAM,KAAK;GACb,MAAM,WAAW,UAAU,KAAK,CAAC;GACjC,MAAM,QAAQ,iBAAiB,KAAK,CAAC;GACrC,MAAM,IAAI,eAAe,KAAK,QAAQ;GACtC,IAAI,EAAE;GACN,IAAI,MAAM,SAAS,eAAe,EAAE,WAAW,GAAG;IAChD,QAAQ;IACR,UAAU;IACV;GACF;GACA,MAAM,EAAE,KAAK,OAAO,UAAU,aAAa,KAAK,MAAM,YAAY,WAAW,CAAC;GAI9E,MAAM,WACJ,SAAS,EAAE,WAAW,KAAK,MAAM,SAAS,KAAK,MAAM,OAAO,MAAM,EAAE,WAAW,CAAC;GAClF,QAAQ,SAAS,EAAE,aAAa,CAAC;GACjC,IAAI,YAAY,MAAM,SAAS,MAAM,UAAU,SAAS;IACtD,QAAQ,MAAM,SAAS,MACrB,MAAM,KAAK,SAAS;KAAE,UAAU,EAAE,SAAS,MAAM;KAAG,KAAK,EAAE,OAAO,IAAI,MAAM;IAAI,EAAE,CACpF;IACA,IAAI,EAAE,WAAW,UAAU;IAC3B;GACF;GACA,QAAQ;GACR,UAAU;GACV,IAAI,MAAM,SAAS,MAAM,SAAS,SAAS;GAC3C,QAAQ,MAAM,SAAS,MACrB,MAAM,KAAK,SAAS;IAAE,UAAU,EAAE,SAAS,OAAO,GAAG;IAAG,KAAK;GAAG,EAAE,CACpE;GACA;EACF;EAEA,IAAI,MAAM,KAAK;GACb,QAAQ;GACR,UAAU;GACV,IAAI,eAAe,KAAK,IAAI,CAAC,CAAC,CAAC;GAC/B;EACF;EAEA,MAAM,IAAI,eAAe,KAAK,IAAI,CAAC;EACnC,IAAI,EAAE;EACN,UAAU,GAAG,CAAC;CAChB;CACA,UAAU;CACV,OAAO;EACL,KAAK,MAAM,KAAK,MAAM,EAAE,QAAQ;EAChC,OAAO,CAAC,SAAS,MAAM,OAAO,MAAM,EAAE,SAAS,WAAW,CAAC;CAC7D;AACF;;AAGA,SAAS,iBAAiB,KAAa,OAA4B;CACjE,IAAI,IAAI,QAAQ,OAAO,KAAK,OAAO;EAAE,MAAM;EAAa,YAAY,QAAQ;CAAE;CAC9E,MAAM,SAAS,IAAI,QAAQ;CAC3B,IAAI,WAAW,KAAK,OAAO;EAAE,MAAM;EAAa,YAAY,QAAQ;CAAE;CACtE,IAAI,WAAW,OAAO,WAAW,KAAK,OAAO,EAAE,MAAM,YAAY;CACjE,IAAI,WAAW,KAAK;EAClB,MAAM,OAAO,IAAI,QAAQ;EACzB,IAAI,SAAS,OAAO,SAAS,KAAK,OAAO,EAAE,MAAM,YAAY;EAC7D,MAAM,UAAU,IAAI,QAAQ,KAAK,QAAQ,CAAC;EAC1C,IAAI,WAAW,GAAG,OAAO;GAAE,MAAM;GAAa,YAAY,UAAU;EAAE;CACxE;CACA,OAAO,EAAE,MAAM,UAAU;AAC3B;;;;;AAMA,SAAS,oBACP,KACA,OAC2C;CAC3C,MAAM,OAAO,IAAI,QAAQ;CACzB,IAAI,SAAS,KAAK;EAChB,MAAM,MAAM,IAAI,MAAM,QAAQ,GAAG,QAAQ,CAAC;EAC1C,IAAI,CAAC,mBAAmB,KAAK,GAAG,GAAG,OAAO,KAAA;EAC1C,OAAO;GAAE,MAAM,OAAO,aAAa,OAAO,SAAS,KAAK,EAAE,CAAC;GAAG,KAAK,QAAQ;EAAE;CAC/E;CACA,IAAI,SAAS,KAAK;EAChB,MAAM,MAAM,IAAI,MAAM,QAAQ,GAAG,QAAQ,CAAC;EAC1C,IAAI,CAAC,mBAAmB,KAAK,GAAG,GAAG,OAAO,KAAA;EAC1C,OAAO;GAAE,MAAM,OAAO,aAAa,OAAO,SAAS,KAAK,EAAE,CAAC;GAAG,KAAK,QAAQ;EAAE;CAC/E;AAEF;;;;;;AAOA,SAAS,kBAAkB,KAAa,OAAuB;CAC7D,MAAM,OAAO,IAAI,QAAQ;CACzB,IAAI,SAAS,KACX,OAAO,mBAAmB,KAAK,IAAI,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,IAAI,QAAQ,IAAI,QAAQ;CAExF,IAAI,SAAS,KAAK;EAChB,IAAI,IAAI,QAAQ,OAAO,KAAK;GAC1B,MAAM,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC;GACxC,OAAO,SAAS,IAAI,QAAQ,IAAI,IAAI;EACtC;EACA,OAAO,mBAAmB,KAAK,IAAI,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,IAAI,QAAQ,IAAI,QAAQ;CACxF;CACA,IAAI,SAAS,KAAK,OAAO,aAAa,KAAK,IAAI,QAAQ,MAAM,EAAE,IAAI,QAAQ,IAAI,QAAQ;CACvF,KAAK,SAAS,OAAO,SAAS,QAAQ,IAAI,QAAQ,OAAO,KAAK;EAC5D,MAAM,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC;EACxC,OAAO,SAAS,IAAI,QAAQ,IAAI,IAAI;CACtC;CACA,IAAI,SAAS,OAAO,IAAI,QAAQ,OAAO,KAAK;EAC1C,MAAM,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC;EACxC,OAAO,SAAS,IAAI,QAAQ,IAAI,IAAI;CACtC;CACA,IAAI,QAAQ,KAAK,QAAQ,EAAE,GAAG;EAC5B,IAAI,IAAI,QAAQ;EAChB,OAAO,QAAQ,KAAK,IAAI,MAAM,EAAE,GAAG,KAAK;EACxC,OAAO;CACT;CACA,OAAO,KAAK,IAAI,QAAQ,GAAG,IAAI,MAAM;AACvC;;AAGA,SAAS,kBAAkB,KAAa,OAAe,KAA8B;CACnF,IAAI,IAAI,QAAQ;CAChB,MAAM,aAAa,MAAM;CACzB,IAAI,IAAI,OAAO,KAAK,OAAO;CAC3B,MAAM,wBAAQ,IAAI,IAAY;CAC9B,OAAO,IAAI,YAAY;EACrB,MAAM,IAAI,IAAI,MAAM;EACpB,IAAI,MAAM,MAAM;GACd,MAAM,OAAO,IAAI,IAAI;GAErB,IAAI,SAAS,KAAA,KAAa,cAAc,KAAK,IAAI,GAAG,OAAO;GAC3D,KAAK;GAEL,IAAI,IAAI,OAAO,OAAO,IAAI,IAAI,YAAY,OAAO;GACjD,MAAM,IAAI,IAAI;GACd;EACF;EACA,IAAI,IAAI,IAAI,cAAc,IAAI,IAAI,OAAO,KAAK,OAAO;EACrD,MAAM,IAAI,CAAC;EACX,KAAK;CACP;CAEA,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,OAAO,kBAAkB,KAAK,EAAE,CAAC,GAAG,OAAO;CAChE,OAAO,MAAM,SAAS,IAAI,OAAO,CAAC,GAAG,KAAK;AAC5C;;AAGA,SAAS,eAAe,KAAa,GAAuB;CAC1D,MAAM,IAAI,IAAI;CACd,IAAI,MAAM,OAAO,MAAM,OAAO,MAAM,KAAK;EACvC,MAAM,MAAM,IAAI,IAAI,OAAO,MAAM,IAAI,IAAI,IAAI;EAC7C,OAAO;GAAE,QAAQ,MAAM,MAAM,IAAI;GAAG,WAAW;GAAM;EAAI;CAC3D;CACA,IAAI,MAAM,KAAK;EACb,MAAM,QAAQ,IAAI,QAAQ,KAAK,CAAC;EAChC,IAAI,QAAQ,GAAG,OAAO;GAAE,QAAQ;GAAG,WAAW;GAAO,KAAK;EAAE;EAC5D,MAAM,IAAI,qBAAqB,KAAK,IAAI,MAAM,IAAI,GAAG,KAAK,CAAC;EAC3D,IAAI,CAAC,GAAG,OAAO;GAAE,QAAQ;GAAG,WAAW;GAAO,KAAK;EAAE;EACrD,MAAM,MAAM,OAAO,EAAE,EAAE;EACvB,MAAM,MAAM,EAAE,OAAO,KAAA,IAAY,MAAM,EAAE,OAAO,KAAK,OAAO,oBAAoB,OAAO,EAAE,EAAE;EAC3F,MAAM,MAAM,IAAI,QAAQ,OAAO,MAAM,QAAQ,IAAI,QAAQ;EACzD,OAAO;GAAE,QAAQ;GAAK,WAAW,MAAM;GAAK;EAAI;CAClD;CACA,OAAO;EAAE,QAAQ;EAAG,WAAW;EAAO,KAAK;CAAE;AAC/C;AAEA,SAAS,UAAU,KAAa,GAAmB;CACjD,IAAI,IAAI,IAAI;CACZ,IAAI,IAAI,OAAO,KAAK,KAAK;CACzB,OAAO,IAAI,IAAI,UAAU,IAAI,OAAO,KAAK,KAAK,IAAI,OAAO,QAAQ,IAAI,IAAI,IAAI,SAAS,IAAI;CAC1F,OAAO,IAAI,IAAI,SAAS,IAAI,IAAI;AAClC;AAEA,SAAS,UAAU,KAAa,GAAmB;CACjD,IAAI,QAAQ;CACZ,IAAI,IAAI,IAAI;CACZ,OAAO,IAAI,IAAI,UAAU,QAAQ,GAAG;EAClC,MAAM,IAAI,IAAI;EACd,IAAI,MAAM,QAAQ,IAAI,IAAI,IAAI,QAAQ;GACpC,KAAK;GACL;EACF;EACA,IAAI,MAAM,KAAK;GACb,IAAI,UAAU,KAAK,CAAC;GACpB;EACF;EACA,IAAI,MAAM,KAAK,SAAS;OACnB,IAAI,MAAM,KAAK,SAAS;EAC7B,KAAK;CACP;CACA,OAAO;AACT;;;;ACrWA,MAAa,qBAAqB;AAElC,MAAM,iBAAiB;AAYvB,SAAgB,UACd,SACA,OAAqD,CAAC,GAC3C;CACX,IAAI,QAAQ,SAAA,KACV,MAAM,IAAI,WACR,qBAAqB,QAAQ,OAAO,4BAA4B,oBAClE;CAIF,IAAI,KAAK,SAAS,2BAA2B,KAAK,OAAO,GACvD,MAAM,IAAI,YACR,gGACF;CAEF,MAAM,SAAS,KAAK,QAAQ,UAAU,QAAQ,QAAQ,wBAAwB,MAAM;CACpF,MAAM,QAAQ,IAAI,OAAO,QAAQ,KAAK,kBAAkB,QAAQ,OAAO,GAAG;CAC1E,MAAM,WAAuB,CAAC;CAC9B,KAAK,MAAM,YAAY,cAAc,KAAK,GAAG;EAG3C,MAAM,SAAS,SACZ,SAAS,YAAY,QAAQ,MAAM,cAAc,CAAC,CAAC,CACnD,IAAI,IAAI,CAAC,CACT,QAAQ,YAAY,MAAM,KAAK,OAAO,CAAC,CAAC,UAAU,CAAC;EACtD,IAAI,OAAO,WAAW,GAAG,OAAO;GAAE;GAAO,UAAU;EAAK;EACxD,SAAS,KAAK,MAAM;CACtB;CACA,OAAO;EAAE;EAAO;CAAS;AAC3B;;;;AC7CA,SAAgB,QAAQ,QAA0B;CAChD,MAAM,MAAM,MAAM,KAAK,MAAM;CAC7B,MAAM,sBAAM,IAAI,IAAY;CAC5B,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,IAAI,QAAQ,KAAK,IAAI,IAAI,IAAI,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;CAC9E,OAAO,CAAC,GAAG,GAAG;AAChB;;AAGA,SAAgB,SAAS,MAAwB;CAC/C,OAAO,QAAQ,KAAK,IAAI,CAAC;AAC3B;;;;;;AAOA,SAAgB,WAAW,MAAsB;CAC/C,MAAM,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,KAAK,MAAM,KAAK,OAAO,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC;CAC7E,IAAI,IAAI,SAAW,IAAI,SAAW,IAAI,OAAS,OAAO,IAAI,KAAK,KAAK,IAAI,KAAK,KAAK;CAClF,OAAO,KAAK,KAAM,IAAI,QAAW,KAAK,KAAM,IAAI,QAAW,KAAK,KAAM,IAAI;AAC5E;;;ACpBA,MAAM,OAAO;;;;;;AAOb,SAAgB,mBAAmB,QAA6B;CAC9D,MAAM,QAAQ,GAAG,OAAO;CACxB,MAAM,SAAS,UACb,UACE,MAAM,SAAS,SAAS,SAAS,KAAK,IAAI,CAAC,CAAC,KAAK,SAAS,CAAC,WAAW,IAAI,GAAG,KAAK,EAAE,CAAC,CAAC,CACxF;CACF,OAAO;EACL,SAAS,OACP,GAAG,KACD,8BAA8B,MAAM,mFACtC;EACF,MAAM,IAAI,IAAI,OAAO;GACnB,KAAK,MAAM,SAAS,MAAM,KAAK,GAC7B,MAAM,GAAG,MACP,yBAAyB,MAAM,oBAAoB,KAAK,qBACxD,CAAC,KAAK,CACR;EAEJ;EACA,MAAM,OAAO,IAAI,OAAO;GACtB,KAAK,MAAM,SAAS,MAAM,KAAK,GAC7B,MAAM,GAAG,MACP,eAAe,MAAM,+BAA+B,KAAK,sBACzD,CAAC,KAAK,CACR;EAEJ;EACA,QAAQ,OAAO,GAAG,KAAK,eAAe,OAAO;EAC7C,MAAM,WAAW,IAAI,UAAU,KAAK;GAClC,MAAM,sBAAM,IAAI,IAAY;GAC5B,KAAK,MAAM,UAAU,UAAU;IAE7B,MAAM,QAAQ,CAAC,GAAG,IAAI,IAAI,OAAO,SAAS,YAAY,QAAQ,OAAO,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC;IACxF,MAAM,OAAO,MAAM,GAAG,MACpB,kBAAkB,MAAM,sGACxB;KAAC,KAAK,UAAU,KAAK;KAAG,MAAM;KAAQ;IAAG,CAC3C;IACA,KAAK,MAAM,OAAO,MAAM,IAAI,IAAI,OAAO,IAAI,EAAE,CAAC;GAChD;GACA,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG;EACpD;CACF;AACF;;;ACpBA,MAAM,eAAe;;AAErB,MAAM,gBAAgB;AAEtB,IAAa,aAAb,MAAwB;;CAEtB;CACA;CACA;CACA;CACA;CAEA,YAAY,EAAE,IAAI,QAAQ,QAAQ,UAA6B;EAC7D,IAAI,CAAC,eAAe,KAAK,MAAM,GAC7B,MAAM,IAAI,MAAM,kBAAkB,OAAO,eAAe,gBAAgB;EAE1E,IAAI,UAAU,UAAU,UAAU,YAChC,MAAM,IAAI,MAAM,yBAAyB,MAAM,EAAE;EACnD,KAAK,MAAM;EACX,KAAK,UAAU;EACf,KAAK,UAAU,GAAG,OAAO;EACzB,KAAK,SAAS;EACd,KAAK,WAAW,UAAU,SAAS,eAAe,MAAM,IAAI,mBAAmB,MAAM;CACvF;CAEA,MAAM,OAAsB;EAC1B,MAAM,IAAI,KAAK;EACf,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;GAC1C,MAAM,GAAG,KACP,8BAA8B,EAAE,gDAClC;GACA,MAAM,CAAC,OAAO,MAAM,GAAG,MACrB,qBAAqB,EAAE,yBACzB;GACA,IAAI,OAAO,IAAI,UAAU,KAAK,QAC5B,MAAM,IAAI,MACR,iBAAiB,EAAE,4BAA4B,IAAI,MAAM,kBAAkB,KAAK,OAAO,EACzF;GAEF,IAAI,CAAC,KACH,MAAM,GAAG,MAAM,eAAe,EAAE,uCAAuC,CAAC,KAAK,MAAM,CAAC;GACtF,MAAM,GAAG,KACP,8BAA8B,EAAE,kDAClC;GACA,MAAM,KAAK,SAAS,OAAO,EAAE;GAC7B,MAAM,KAAK,WAAW,EAAE;EAC1B,CAAC;CACH;;;;;;;CAQA,MAAM,WAAW,IAAuB;EACtC,MAAM,IAAI,KAAK;EACf,MAAM,WAAW,aAAa;EAC9B,MAAM,CAAC,OAAO,MAAM,GAAG,MACrB,qBAAqB,EAAE,wBACzB;EACA,IAAI,KAAK,UAAU,UAAU;EAC7B,IAAI,KAAK,MAAM,KAAK,SAAS,EAAE;EAC/B,MAAM,GAAG,MAAM,0BAA0B,EAAE,sCAAsC,CAAC,QAAQ,CAAC;CAC7F;CAEA,MAAM,SAAS,IAAuB;EACpC,MAAM,KAAK,SAAS,MAAM,EAAE;EAC5B,IAAI,QAAuB;EAC3B,SAAS;GAKP,MAAM,SAAQ,MAJ0C,GAAG,MACzD,wBAAwB,KAAK,QAAQ,sDACrC;IAAC;IAAO;IAAO;GAAa,CAC9B,EAAA,CACmB,KAAK,OAAO;IAAE,IAAI,OAAO,EAAE,EAAE;IAAG,MAAM,OAAO,EAAE,IAAI;GAAE,EAAE;GAC1E,MAAM,OAAO,MAAM,GAAG,EAAE;GACxB,IAAI,CAAC,MAAM;GACX,MAAM,KAAK,SAAS,IAAI,IAAI,KAAK;GACjC,QAAQ,KAAK;EACf;CACF;CAEA,MAAM,OAAO,OAAmC;EAI9C,MAAM,SAAS,CACb,GAAG,IAAI,IAAI,MAAM,KAAK,EAAE,IAAI,WAAW,CAAC,IAAI;GAAE;GAAI,MAAM,KAAK,aAAa;EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAC1F;EACA,IAAI,OAAO,WAAW,GAAG;EACzB,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;GAC1C,MAAM,KAAK,SAAS,OAClB,IACA,MAAM,KAAK,MACT,IACA,OAAO,KAAK,SAAS,KAAK,EAAE,CAC9B,CACF;GACA,KAAK,MAAM,SAAS,UAAU,OAAO,KAAK,SAAS,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,CAAC,GACtE,MAAM,GAAG,MACP,0BAA0B,KAAK,QAAQ,oGACvC,CAAC,KAAK,CACR;GAEF,MAAM,KAAK,SAAS,IAAI,IAAI,MAAM;EACpC,CAAC;CACH;CAEA,MAAM,OAAO,KAA8B;EACzC,IAAI,IAAI,WAAW,GAAG;EACtB,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;GAC1C,MAAM,KAAK,SAAS,OAAO,IAAI,MAAM,KAAK,MAAM,IAAI,GAAG,CAAC;GACxD,KAAK,MAAM,SAAS,UAAU,GAAG,GAC/B,MAAM,GAAG,MACP,eAAe,KAAK,QAAQ,qDAC5B,CAAC,KAAK,CACR;EAEJ,CAAC;CACH;CAEA,MAAM,OAAO,SAAiB,OAA2B,CAAC,GAA+B;EACvF,MAAM,EAAE,QAAQ,KAAK,kBAAkB,KAAQ,QAAQ,WAAW;EAClE,MAAM,OAAO,UAAU,SAAS;GAC9B,OAAO,KAAK;GACZ,eAAe,KAAK,iBAAiB;EACvC,CAAC;EACD,QAAQ,eAAe;EACvB,IAAI,SAAS,GAAG,OAAO;GAAE,MAAM,CAAC;GAAG,WAAW;EAAK;EACnD,MAAM,QACJ,KAAK,aAAa,QAEZ,MAAM,KAAK,IAAI,MACb,kBAAkB,KAAK,QAAQ,4BAC/B,CAAC,kBAAkB,CAAC,CACtB,EAAA,CACA,KAAK,QAAQ,OAAO,IAAI,EAAE,CAAC,IAC7B,MAAM,KAAK,SAAS,WAAW,KAAK,KAAK,KAAK,UAAU,kBAAkB,CAAC;EACjF,IAAI,YAAY,MAAM,SAAS;EAC/B,MAAM,MAAM,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,QAAQ,OAAO,CAAC,UAAU,OAAO,EAAE,CAAC;EAEhF,MAAM,OAAmB,CAAC;EAC1B,IAAI;EACJ,QAAQ,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK,cAAc;GACzD,QAAQ,eAAe;GACvB,KAAK,MAAM,EAAE,IAAI,UAAU,MAAM,KAAK,MAAM,KAAK,KAAK,IAAI,MAAM,GAAG,IAAI,YAAY,CAAC,GAAG;IACrF,OAAO;IACP,MAAM,UAAwB,CAAC;IAC/B,KAAK,MAAM,KAAK,KAAK,SAAS,KAAK,KAAK,GACtC,IAAI,EAAE,EAAE,CAAC,SAAS,GAAG,QAAQ,KAAK;KAAE,OAAO,EAAE;KAAO,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC;IAAO,CAAC;IAElF,IAAI,QAAQ,WAAW,GAAG;IAC1B,KAAK,KAAK;KAAE;KAAI;IAAQ,CAAC;IACzB,IAAI,KAAK,UAAU,OAAO,MAAM;GAClC;EACF;EACA,IAAI,KAAK,UAAU,SAAS,SAAS,IAAI,GAAG,EAAE,GAAG,YAAY;EAC7D,OAAO;GAAE;GAAM;EAAU;CAC3B;;CAGA,MAAM,SAAS,KAA6C;EAC1D,MAAM,QAAQ,MAAM,KAAK,MAAM,KAAK,KAAK,GAAG;EAC5C,OAAO,IAAI,IAAI,MAAM,KAAK,SAAS,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,CAAC;CAC1D;;CAGA,MAAM,OAAsB;EAC1B,MAAM,IAAI,KAAK;EACf,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;GAC1C,KAAK,MAAM,KAAK;IAAC;IAAM;IAAS;IAAS;GAAM,GAC7C,MAAM,GAAG,KAAK,wBAAwB,IAAI,GAAG;EACjD,CAAC;CACH;;CAGA,MAAM,MAAM,IAAQ,KAAqC;EACvD,MAAM,MAAmB,CAAC;EAC1B,KAAK,MAAM,SAAS,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,GAAG;GAC7D,MAAM,OAAO,MAAM,GAAG,MACpB,wBAAwB,KAAK,QAAQ,iEACrC,CAAC,KAAK,CACR;GACA,KAAK,MAAM,OAAO,MAAM,IAAI,KAAK;IAAE,IAAI,OAAO,IAAI,EAAE;IAAG,MAAM,OAAO,IAAI,IAAI;GAAE,CAAC;EACjF;EACA,OAAO;CACT;AACF"}
|
package/dist/vector.d.mts
CHANGED
|
@@ -8,7 +8,10 @@ interface SearchHit {
|
|
|
8
8
|
}
|
|
9
9
|
interface SearchOptions {
|
|
10
10
|
k?: number;
|
|
11
|
-
/**
|
|
11
|
+
/**
|
|
12
|
+
* IVF lists probed per segment, exactly. Default: at least 8, more until the
|
|
13
|
+
* probed lists hold 16 × k × oversample rows (small lists after auto-compaction).
|
|
14
|
+
*/
|
|
12
15
|
nprobe?: number;
|
|
13
16
|
/** Candidates kept per probed list by the 1-bit filter = k * oversample. Default 8. */
|
|
14
17
|
oversample?: number;
|
package/dist/vector.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector.d.mts","names":[],"sources":["../src/vector/types.ts","../src/vector/sql-segment-store.ts","../src/vector/vector-index.ts"],"mappings":";;KAAY;UAEK;EACf;;EAEA;;UAGe;EACf;;EAEA;;EAEA;;EAEA
|
|
1
|
+
{"version":3,"file":"vector.d.mts","names":[],"sources":["../src/vector/types.ts","../src/vector/sql-segment-store.ts","../src/vector/vector-index.ts"],"mappings":";;KAAY;UAEK;EACf;;EAEA;;UAGe;EACf;;;;;EAKA;;EAEA;;EAEA;;EAEA,UAAU;EACV,SAAS;;;;UCXM;EACf;EACA;EACA;;;;UCOe;EACf,IAAI;;EAEJ;EACA;EACA,SAAS;;EAET,SAAS;;EAET;;;;;;EAMA;;UAGe;;EAEf;EACA,QAAQ;;UAGO;EACf;EACA,UAAU;EACV;;;;;;;;;;;;;;;;;qBAyCW;;WACF;WACA,QAAQ;;WAER;EAkBT,YAAY,MAAM;;EAqBZ,QAAQ;;EA+BR,WAAW;MAqBb,SAAS;EAUP,OAAO,OAAO,eAAe;EAiC7B,OAAO,gBAAgB;EAwBvB,OAAO,OAAO,mBAAmB,OAAM,gBAAqB,QAAQ;;EA2GpE,WAAW,gBAAgB,QAAQ,YAAY;;EAwC/C,WAAW;;;;;EAkDX,SAAS;;EAgET,YAAY;;;wBAiHE,gBAAgB,IAAI,IAAI,iBAAiB"}
|