@statewalker/db-sqlite-index 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +289 -2
- package/dist/ngram.d.mts +93 -0
- package/dist/ngram.d.mts.map +1 -0
- package/dist/ngram.mjs +615 -0
- package/dist/ngram.mjs.map +1 -0
- package/dist/vector.d.mts +103 -0
- package/dist/vector.d.mts.map +1 -0
- package/dist/vector.mjs +1061 -0
- package/dist/vector.mjs.map +1 -0
- package/package.json +63 -3
- package/src/ngram/do-smoke.ts +46 -0
- package/src/ngram/fold.test.ts +50 -0
- package/src/ngram/fold.ts +21 -0
- package/src/ngram/fts5-backend.ts +55 -0
- package/src/ngram/index.ts +12 -0
- package/src/ngram/json-chunks.test.ts +23 -0
- package/src/ngram/json-chunks.ts +23 -0
- package/src/ngram/ngram-store.test.ts +291 -0
- package/src/ngram/ngram-store.ts +183 -0
- package/src/ngram/plan-query.test.ts +141 -0
- package/src/ngram/plan-query.ts +42 -0
- package/src/ngram/postings-backend.ts +53 -0
- package/src/ngram/regex-literals.ts +359 -0
- package/src/ngram/trigrams.test.ts +28 -0
- package/src/ngram/trigrams.ts +25 -0
- package/src/vector/auto-compact.test.ts +28 -0
- package/src/vector/do-smoke.ts +39 -0
- package/src/vector/index.ts +9 -0
- package/src/vector/kmeans.ts +131 -0
- package/src/vector/math.ts +185 -0
- package/src/vector/segment.test.ts +61 -0
- package/src/vector/segment.ts +256 -0
- package/src/vector/sql-segment-store.ts +218 -0
- package/src/vector/test-fixtures.ts +64 -0
- package/src/vector/types.ts +30 -0
- package/src/vector/vector-index.test.ts +105 -0
- package/src/vector/vector-index.ts +586 -0
- package/src/vector/vector-lifecycle.test.ts +171 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mikhail Kotelnikov
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,290 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @statewalker/db-sqlite-index
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
## What it is
|
|
4
|
+
|
|
5
|
+
Indexes that keep all of their data in SQLite tables through `Db` from
|
|
6
|
+
`@statewalker/db-api`, so they run wherever a `Db` runs: Node.js (node:sqlite),
|
|
7
|
+
the browser (sqlite-wasm) and Cloudflare-style Durable Objects. The `./vector`
|
|
8
|
+
subpath is an IVF vector index for approximate nearest-neighbour search over
|
|
9
|
+
integer ids. The `./ngram` subpath finds every stored text that contains a
|
|
10
|
+
literal or matches a JavaScript regex, using a trigram index to pick candidates
|
|
11
|
+
and the real regex to confirm them. Ids are the caller's: typically the rowid of
|
|
12
|
+
its own documents table.
|
|
13
|
+
|
|
14
|
+
## Why it exists
|
|
15
|
+
|
|
16
|
+
Durable Objects allow no native extensions, so sqlite-vec and pgvector are not
|
|
17
|
+
available there, and their SQLite refuses BLOBs and strings over about 2 MB, more than 100
|
|
18
|
+
bound parameters, `PRAGMA` and `BEGIN`. This package is pure TypeScript over plain
|
|
19
|
+
SQL and stays inside those limits, so one index implementation serves every
|
|
20
|
+
runtime. It knows nothing about documents or search ranking. Callers map ids to
|
|
21
|
+
their own records.
|
|
22
|
+
|
|
23
|
+
## How to use
|
|
24
|
+
|
|
25
|
+
```sh
|
|
26
|
+
pnpm add @statewalker/db-sqlite-index @statewalker/db-sqlite
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Create a `Db` with any SQLite driver whose `Db` supports `inTransaction` from
|
|
30
|
+
`@statewalker/db-sqlite` (`@statewalker/db-sqlite-node`,
|
|
31
|
+
`@statewalker/db-sqlite-browser`, `@statewalker/db-sqlite-do`), then:
|
|
32
|
+
|
|
33
|
+
```ts
|
|
34
|
+
import { newNodeSqliteDb } from "@statewalker/db-sqlite-node";
|
|
35
|
+
import { dropVectorIndex, VectorIndex } from "@statewalker/db-sqlite-index/vector";
|
|
36
|
+
|
|
37
|
+
const db = await newNodeSqliteDb({ path: "./vectors.db" });
|
|
38
|
+
const embedding = new Float32Array(384).fill(0.1); // your model's output
|
|
39
|
+
const query = new Float32Array(384).fill(0.2);
|
|
40
|
+
|
|
41
|
+
const index = new VectorIndex({ db, prefix: "vec_", dims: 384, metric: "cosine" });
|
|
42
|
+
await index.init();
|
|
43
|
+
await index.upsert([{ id: 1, vector: embedding }]);
|
|
44
|
+
const hits = await index.search(query, { k: 10 }); // [{ id, distance }], lower is closer
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
`metric` is `"cosine"` (default), `"dot"` or `"l2"`. `search` defaults to `k: 10`,
|
|
48
|
+
`nprobe: 8`, `oversample: 8`, `exact: false`; it also takes an `AbortSignal` as `signal`.
|
|
49
|
+
`upsert`, `delete`, `search`, `getVectors`, `compact`, `merge` and `optimize` reject with
|
|
50
|
+
`vector-index: call init() first` until `init()` ran.
|
|
51
|
+
|
|
52
|
+
## Examples
|
|
53
|
+
|
|
54
|
+
Continuing with `db`, `index` and `query` from above:
|
|
55
|
+
|
|
56
|
+
```ts
|
|
57
|
+
const allowed = new Set([1]);
|
|
58
|
+
|
|
59
|
+
// Filter while searching (applied before re-ranking, so k results still come back).
|
|
60
|
+
const mine = await index.search(query, { k: 10, filter: (id) => allowed.has(id) });
|
|
61
|
+
|
|
62
|
+
// Exact re-rank from stored float32 vectors: slower, closer to brute force.
|
|
63
|
+
const precise = await index.search(query, { k: 10, nprobe: 16, exact: true });
|
|
64
|
+
|
|
65
|
+
// Deletes and overwrites are immediate; segment copies are hidden, not rewritten.
|
|
66
|
+
await index.delete([1, 2, 3]);
|
|
67
|
+
|
|
68
|
+
// Seal the tail into a segment, then rewrite all segments into one.
|
|
69
|
+
await index.compact();
|
|
70
|
+
await index.merge();
|
|
71
|
+
|
|
72
|
+
// compact() then merge(), in one call.
|
|
73
|
+
await index.optimize();
|
|
74
|
+
|
|
75
|
+
// Read stored vectors back (normalised for cosine).
|
|
76
|
+
const vectors = await index.getVectors([1, 2]); // Map<id, Float32Array>
|
|
77
|
+
|
|
78
|
+
// `stats` is { tail, segments, seq }: vectors in the tail, the segment list, the write sequence.
|
|
79
|
+
const { tail, segments } = index.stats;
|
|
80
|
+
|
|
81
|
+
// `idTable` names a table whose `id` column holds exactly the live ids:
|
|
82
|
+
// join it in SQL to count or enumerate what the index contains.
|
|
83
|
+
const [row] = await db.query<{ n: number }>(`SELECT count(*) AS n FROM ${index.idTable}`);
|
|
84
|
+
const count = row?.n ?? 0;
|
|
85
|
+
|
|
86
|
+
// Drop every table of the index; the `index` object is unusable afterwards.
|
|
87
|
+
await dropVectorIndex(db, "vec_");
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Every write method runs its own `inTransaction`. Never call an index method (`init`,
|
|
91
|
+
`refresh`, `upsert`, `delete`, `compact`, `merge`, `optimize`, `search`, `getVectors`) from
|
|
92
|
+
inside an `inTransaction(db, …)` callback: the index holds the outer `Db`, so the inner call
|
|
93
|
+
waits for the outer transaction to finish, which never happens — the callback hangs with no
|
|
94
|
+
error, on every runtime.
|
|
95
|
+
|
|
96
|
+
Each engine needs its own prefix: both create `{p}meta`, with different columns, so a
|
|
97
|
+
vector index and an n-gram index sharing a prefix break each other.
|
|
98
|
+
|
|
99
|
+
How much one call commits:
|
|
100
|
+
|
|
101
|
+
- `upsert()` commits every 200 items and `delete()` every 2,000 ids, each batch as its own
|
|
102
|
+
transaction. A failure part-way leaves the earlier batches committed.
|
|
103
|
+
- `compact()` and `merge()` are one transaction each.
|
|
104
|
+
- An `upsert()` that reaches `autoCompactAt` runs `compact()` after its batch has committed. If
|
|
105
|
+
that compaction fails, `upsert()` rejects although the items are stored. While an
|
|
106
|
+
auto-compact runs, an explicit `compact()` or `merge()` rejects with
|
|
107
|
+
`vector-index: compact or merge already running`: call it again afterwards.
|
|
108
|
+
|
|
109
|
+
## Internals
|
|
110
|
+
|
|
111
|
+
### The index is an LSM: an exact tail, then immutable segments
|
|
112
|
+
|
|
113
|
+
New vectors go to `{p}tail` and are mirrored in memory, where search is exact.
|
|
114
|
+
`compact()` seals the tail into an immutable IVF segment in one transaction.
|
|
115
|
+
`{p}ids(id, gen)` records which generation holds the live copy of each id
|
|
116
|
+
(0 = tail). A segment hit counts only when its generation matches, so deletes
|
|
117
|
+
and overwrites never rewrite segment rows. `merge()` rebuilds all segments into
|
|
118
|
+
one and drops dead rows.
|
|
119
|
+
|
|
120
|
+
### Search narrows candidates three times
|
|
121
|
+
|
|
122
|
+
1. Probe the `nprobe` lists (default 8) whose centroids are closest to the query.
|
|
123
|
+
2. A 1-bit filter keeps `k × oversample` rows per list (`oversample` default 8). It
|
|
124
|
+
estimates the real metric from the signs of each vector's residual to its centroid, plus the
|
|
125
|
+
residual's norm and its projection on the centroid (RaBitQ-style). This makes
|
|
126
|
+
it accurate for `dot` and `l2` too, not only `cosine`.
|
|
127
|
+
3. Re-rank the survivors with int8 codes. With `exact: true`, re-rank the top
|
|
128
|
+
`4k` again with the stored float32 vectors.
|
|
129
|
+
|
|
130
|
+
A segment below 2048 vectors is one flat list: k-means would not pay off. Larger segments
|
|
131
|
+
get `round(√n)` lists, at most 1024, unless `nlist(n)` is passed to the constructor.
|
|
132
|
+
|
|
133
|
+
### upsert() compacts at 4096 vectors, because a Durable Object operation has 15 s
|
|
134
|
+
|
|
135
|
+
Compaction trains k-means and writes the whole segment in one transaction. On
|
|
136
|
+
celld every Durable Object operation has a 15 s deadline, and compacting a much
|
|
137
|
+
larger tail can exceed it. So `upsert()` compacts as soon as the tail holds
|
|
138
|
+
`autoCompactAt` vectors (default 4096). If the deadline is hit anyway (very high
|
|
139
|
+
`dims`, a slow host), the symptom is a timeout and a retried request, not an
|
|
140
|
+
error from this package: the transaction rolls back and the tail is intact.
|
|
141
|
+
Lower `autoCompactAt` in that case. Set it to `Infinity` to compact only when
|
|
142
|
+
you call `compact()`.
|
|
143
|
+
|
|
144
|
+
### Storage stays inside Durable Object limits
|
|
145
|
+
|
|
146
|
+
| Table | Holds |
|
|
147
|
+
|---|---|
|
|
148
|
+
| `{p}meta` | `config` (dims, metric) and `state` (sequence, segment list) |
|
|
149
|
+
| `{p}ids` | live generation per id |
|
|
150
|
+
| `{p}tail` | tail vectors |
|
|
151
|
+
| `{p}centroids` | one row per segment list: centroid and row count |
|
|
152
|
+
| `{p}seg` | each list's searchable block (ids, sign bits, norms, int8 codes), in parts |
|
|
153
|
+
| `{p}f32` | each list's float32 block, in parts |
|
|
154
|
+
|
|
155
|
+
Each list's searchable and float32 blocks are split into parts of at most
|
|
156
|
+
1 MiB (tail and centroid rows hold one vector, `dims × 4` bytes, and are not
|
|
157
|
+
split), and id lists travel as one `json_each(?)` parameter. Durable Object SQLite would otherwise refuse with
|
|
158
|
+
"string or blob too big" or "too many SQL variables". A probe reads the needed
|
|
159
|
+
lists of a segment in one query. Read blocks go into an LRU cache
|
|
160
|
+
(`cacheBytes`, default 64 MiB), which never goes stale because a generation is
|
|
161
|
+
never modified.
|
|
162
|
+
|
|
163
|
+
### Constraints
|
|
164
|
+
|
|
165
|
+
- **One writer per database.** Use one Durable Object, one worker or one tab.
|
|
166
|
+
Within an instance, writes are serialised. Another writer's changes become
|
|
167
|
+
visible after `refresh()`.
|
|
168
|
+
- **`merge()` holds every live segment vector in memory** while it runs.
|
|
169
|
+
- **Opening existing tables with different `dims` or `metric` fails** with
|
|
170
|
+
`vector-index: tables were created with {"dims":…,"metric":…}, not …`.
|
|
171
|
+
- **A search or `getVectors` that overlaps a `compact()` or `merge()` commit runs
|
|
172
|
+
again** on the new segments, so it does not judge old segment rows against the
|
|
173
|
+
new liveness. The third attempt stands even if a third commit overlaps it.
|
|
174
|
+
- **Retired segment rows are deleted once no search reads them.** If the process
|
|
175
|
+
stops before that, the next `init()` sweeps them.
|
|
176
|
+
|
|
177
|
+
### Dependencies
|
|
178
|
+
|
|
179
|
+
`@statewalker/db-api` (the `Db` type) and `@statewalker/db-sqlite`
|
|
180
|
+
(`inTransaction`, which runs every write as one transaction on each runtime).
|
|
181
|
+
The vector math, k-means and segment codec have no dependencies.
|
|
182
|
+
|
|
183
|
+
## Why `./ngram` finds what full-text search cannot
|
|
184
|
+
|
|
185
|
+
Full-text search matches words; it cannot find `idx_le`, `v2.3.1`, a fragment of
|
|
186
|
+
an identifier, or a regex. Scanning every text answers those questions but costs a
|
|
187
|
+
read of the whole store per query. A trigram index answers them by reading only
|
|
188
|
+
texts that contain all the pattern's required three-character pieces, and stays
|
|
189
|
+
exact because each candidate is checked with the regex itself.
|
|
190
|
+
|
|
191
|
+
## How to use (`./ngram`)
|
|
192
|
+
|
|
193
|
+
```ts
|
|
194
|
+
import { newNodeSqliteDb } from "@statewalker/db-sqlite-node";
|
|
195
|
+
import { NgramStore } from "@statewalker/db-sqlite-index/ngram";
|
|
196
|
+
|
|
197
|
+
const db = await newNodeSqliteDb({ path: "index.db" });
|
|
198
|
+
const grams = new NgramStore({ db, prefix: "code_" }); // store: "fts5" (default) | "postings"
|
|
199
|
+
await grams.init();
|
|
200
|
+
await grams.upsert([{ id: 1, text: "const idx_left = 1;" }]);
|
|
201
|
+
|
|
202
|
+
const { hits, truncated } = await grams.search("idx_le");
|
|
203
|
+
// hits: [{ id: 1, matches: [{ start: 6, end: 12 }] }], truncated: false
|
|
204
|
+
await grams.search("idx_(left|right)", { regex: true, caseSensitive: false, limit: 20 });
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
Ids are integers chosen by the caller. Hits come in ascending id order with every
|
|
208
|
+
non-empty match; offsets are UTF-16 code units of the stored text.
|
|
209
|
+
|
|
210
|
+
`search` options: `regex` (default `false`: the pattern is literal text), `caseSensitive`
|
|
211
|
+
(default `true`), `limit` (default 100), `candidateBudget` (default 10,000), `filter(id)`
|
|
212
|
+
(applied before verification) and `signal` (`AbortSignal`). `getTexts(ids)` returns the stored
|
|
213
|
+
texts as a `Map<id, string>`, `drop()` deletes every table of the prefix, and `idTable`
|
|
214
|
+
(`{prefix}texts`) is a table whose `id` column holds exactly the stored ids.
|
|
215
|
+
|
|
216
|
+
`upsert` and `delete` are each one transaction, however many items they get: atomic, but
|
|
217
|
+
unbounded against a Durable Object's 15 s operation deadline, so keep batches small. As in
|
|
218
|
+
the vector section, never call `init`, `upsert`, `delete`, `search`, `getTexts` or `drop` from
|
|
219
|
+
inside an `inTransaction(db, …)` callback on the same `Db`: the call hangs. Use a prefix of its
|
|
220
|
+
own, different from any vector index on the same database.
|
|
221
|
+
|
|
222
|
+
## How a search stays exact
|
|
223
|
+
|
|
224
|
+
1. `planQuery` turns the pattern into a regex and a list of branches, each a set of
|
|
225
|
+
literals that every match must contain (`abc(def|ghi)` → `abcdef` or `abcghi`).
|
|
226
|
+
2. Literals shorter than three code points cannot be looked up; a branch with none
|
|
227
|
+
left makes the search scan every stored text instead.
|
|
228
|
+
3. The store returns ids whose text holds all literals of some branch.
|
|
229
|
+
4. The regex runs on each candidate's original text; only real matches are returned.
|
|
230
|
+
|
|
231
|
+
Text and literals are case-folded the same way (`fold`) before indexing, so a
|
|
232
|
+
case-insensitive regex never misses a text because of a case form (`Σ σ ς`, `ſ`).
|
|
233
|
+
|
|
234
|
+
## Two stores, one behaviour
|
|
235
|
+
|
|
236
|
+
- `fts5` (default): an SQLite FTS5 table with the `trigram` tokenizer. Nothing to
|
|
237
|
+
maintain; uses SQLite's own posting lists.
|
|
238
|
+
- `postings`: one `(gram, id)` row per distinct trigram in a `WITHOUT ROWID` table.
|
|
239
|
+
For SQLite builds without FTS5, and as an independently implemented check.
|
|
240
|
+
|
|
241
|
+
The store is recorded at `init`; opening the same prefix with the other store fails
|
|
242
|
+
with `ngram: index "<prefix>" was created with store "fts5", opened with "postings"`.
|
|
243
|
+
|
|
244
|
+
## What will surprise you in `./ngram`
|
|
245
|
+
|
|
246
|
+
- `search` stops at `limit` hits (default 100) and verifies at most
|
|
247
|
+
`candidateBudget` candidates (default 10,000); either cut sets `truncated: true`.
|
|
248
|
+
A pattern with no three-character literal (`\d+`, `ab`) scans, and the budget
|
|
249
|
+
then counts scanned texts.
|
|
250
|
+
- A pattern that matches only the empty string (`^`, `(abc)*$`) finds nothing.
|
|
251
|
+
- Patterns longer than 1,000 characters fail with
|
|
252
|
+
`ngram: pattern is N characters, the limit is 1000`. A regex with catastrophic
|
|
253
|
+
backtracking still runs to completion; the cap does not prevent that.
|
|
254
|
+
- An invalid regex fails with the engine's `SyntaxError`.
|
|
255
|
+
- **Regex mode compiles without the `u` flag**, because the case folding used for the index
|
|
256
|
+
is unsound under `/iu` (U+0390 and U+1FD3 are equal under `/iu`, but `fold` separates them,
|
|
257
|
+
so candidates would be missed). So `.` matches one UTF-16 code unit and can split a surrogate
|
|
258
|
+
pair (`😀` yields two matches of `.`, and a hit can end inside an emoji), and `\u{1F600}` is
|
|
259
|
+
not a code-point escape. `\p{…}` and `\P{…}` would silently mean the literal text, so they
|
|
260
|
+
are rejected with `ngram: \p{…} and \P{…} are not supported in regex mode (patterns compile
|
|
261
|
+
without the u flag)`. Literal mode is not affected.
|
|
262
|
+
- `search` with `limit: 0` returns no hits and `truncated: true`.
|
|
263
|
+
- A lone surrogate in an upserted text is stored, indexed and returned as U+FFFD
|
|
264
|
+
(`text.toWellFormed()`), so offsets and `getTexts` refer to that replaced text.
|
|
265
|
+
- Methods other than `init` fail with SQLite's `no such table` until `init` ran.
|
|
266
|
+
- On Durable Objects a single text whose JSON-escaped UTF-8 form is over about 2 MB fails with
|
|
267
|
+
`string or blob too big` (on the fake: 1,990,000 ASCII characters pass, 2,000,000 fail;
|
|
268
|
+
with 2-byte characters such as Cyrillic the cut is near 1,000,000 characters). Batches are
|
|
269
|
+
split automatically, single texts are not.
|
|
270
|
+
|
|
271
|
+
## Internals of `./ngram`
|
|
272
|
+
|
|
273
|
+
- Tables per prefix `p`: `{p}meta`, `{p}texts(id, text)`, and `{p}ng` (fts5,
|
|
274
|
+
contentless, `tokenize='trigram case_sensitive 1'`) or `{p}grams(gram, id)`.
|
|
275
|
+
- The FTS5 table is contentless: the text is kept once, in `{p}texts`, and a row is
|
|
276
|
+
deleted by replaying its indexed value — so writes read the previous text first.
|
|
277
|
+
- Trigrams are three code points; a surrogate pair is never split. In `postings` a
|
|
278
|
+
gram of three BMP code points is one exact 48-bit integer; grams with an astral
|
|
279
|
+
code point are hashed above 2^48, where a collision only adds a candidate.
|
|
280
|
+
- Every write runs inside `inTransaction` from `@statewalker/db-sqlite`. Batches
|
|
281
|
+
are sent as JSON through `json_each(?)` in parts of at most 500,000 code units,
|
|
282
|
+
because a Durable Object rejects more than 100 bound parameters and values
|
|
283
|
+
over about 2 MB.
|
|
284
|
+
- The regex-to-literals planner is ported from hypgrep's `regex.js` (MIT).
|
|
285
|
+
- Dependencies: `@statewalker/db-api`, `@statewalker/db-sqlite`. The planner, fold
|
|
286
|
+
and trigram code import neither.
|
|
287
|
+
|
|
288
|
+
## License
|
|
289
|
+
|
|
290
|
+
MIT
|
package/dist/ngram.d.mts
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { Db } from "@statewalker/db-api";
|
|
2
|
+
//#region src/ngram/fold.d.ts
|
|
3
|
+
/**
|
|
4
|
+
* Case fold per code point, applied to indexed text and to query literals alike.
|
|
5
|
+
*
|
|
6
|
+
* Upper-then-lower maps every character a case-insensitive JS regex treats as
|
|
7
|
+
* equal to one form (Σ σ ς → σ, ſ → s). A character whose upper case is several
|
|
8
|
+
* code points (ß → SS) only matches itself under `/i`, so it keeps its own
|
|
9
|
+
* lower case. Folding per code point makes fold(a + b) === fold(a) + fold(b), so
|
|
10
|
+
* a substring of the text stays a substring of the folded text. A character
|
|
11
|
+
* whose folded form has a different UTF-16 length (İ → i + U+0307) is kept as
|
|
12
|
+
* is, so fold preserves the length and offsets into the folded text are
|
|
13
|
+
* offsets into the original.
|
|
14
|
+
*/
|
|
15
|
+
export declare function fold(text: string): string;
|
|
16
|
+
//#endregion
|
|
17
|
+
//#region src/ngram/fts5-backend.d.ts
|
|
18
|
+
type NgramItem = {
|
|
19
|
+
id: number;
|
|
20
|
+
text: string;
|
|
21
|
+
};
|
|
22
|
+
//#endregion
|
|
23
|
+
//#region src/ngram/ngram-store.d.ts
|
|
24
|
+
type NgramMatch = {
|
|
25
|
+
start: number;
|
|
26
|
+
end: number;
|
|
27
|
+
};
|
|
28
|
+
type NgramHit = {
|
|
29
|
+
id: number;
|
|
30
|
+
matches: NgramMatch[];
|
|
31
|
+
};
|
|
32
|
+
type NgramSearchOptions = {
|
|
33
|
+
/** Treat `pattern` as a JS regex source; otherwise it is literal text. */
|
|
34
|
+
regex?: boolean;
|
|
35
|
+
/** Default true (grep semantics). */
|
|
36
|
+
caseSensitive?: boolean;
|
|
37
|
+
/** Maximum hits. Default 100. */
|
|
38
|
+
limit?: number;
|
|
39
|
+
/** Maximum candidates verified. Default 10,000. */
|
|
40
|
+
candidateBudget?: number;
|
|
41
|
+
/** Keep only ids for which this returns true (applied before verification). */
|
|
42
|
+
filter?: (id: number) => boolean;
|
|
43
|
+
signal?: AbortSignal;
|
|
44
|
+
};
|
|
45
|
+
type NgramSearchResult = {
|
|
46
|
+
/** Ascending id order; every non-empty match, offsets in UTF-16 code units. */
|
|
47
|
+
hits: NgramHit[];
|
|
48
|
+
/** True when the search stopped before checking every candidate (budget or limit). */
|
|
49
|
+
truncated: boolean;
|
|
50
|
+
};
|
|
51
|
+
type NgramStoreOptions = {
|
|
52
|
+
db: Db;
|
|
53
|
+
prefix: string;
|
|
54
|
+
store?: "fts5" | "postings";
|
|
55
|
+
};
|
|
56
|
+
export declare class NgramStore {
|
|
57
|
+
#private;
|
|
58
|
+
/** `{prefix}texts`: its `id` column holds exactly the stored ids. Callers may join it in SQL. */
|
|
59
|
+
readonly idTable: string;
|
|
60
|
+
constructor({ db, prefix, store }: NgramStoreOptions);
|
|
61
|
+
init(): Promise<void>;
|
|
62
|
+
upsert(items: NgramItem[]): Promise<void>;
|
|
63
|
+
delete(ids: number[]): Promise<void>;
|
|
64
|
+
search(pattern: string, opts?: NgramSearchOptions): Promise<NgramSearchResult>;
|
|
65
|
+
/** The stored original texts of `ids`; unknown ids are absent. */
|
|
66
|
+
getTexts(ids: number[]): Promise<Map<number, string>>;
|
|
67
|
+
/** Drops the tables of both stores, so it works without knowing which one created the prefix. */
|
|
68
|
+
drop(): Promise<void>;
|
|
69
|
+
}
|
|
70
|
+
//#endregion
|
|
71
|
+
//#region src/ngram/plan-query.d.ts
|
|
72
|
+
/** Longer patterns are rejected: a JS regex cannot be interrupted once it runs. */
|
|
73
|
+
export declare const MAX_PATTERN_LENGTH = 1000;
|
|
74
|
+
type QueryPlan = {
|
|
75
|
+
/** The pattern as a global regex (`i` added when not case-sensitive); verifies candidates. */
|
|
76
|
+
regex: RegExp;
|
|
77
|
+
/**
|
|
78
|
+
* Per DNF branch, the folded literals of at least 3 code points that a match must
|
|
79
|
+
* contain. `null` when some branch has none: only a scan can answer the pattern.
|
|
80
|
+
*/
|
|
81
|
+
branches: string[][] | null;
|
|
82
|
+
};
|
|
83
|
+
export declare function planQuery(pattern: string, opts?: {
|
|
84
|
+
regex?: boolean;
|
|
85
|
+
caseSensitive?: boolean;
|
|
86
|
+
}): QueryPlan;
|
|
87
|
+
//#endregion
|
|
88
|
+
//#region src/ngram/trigrams.d.ts
|
|
89
|
+
/** Distinct trigrams of `text` as the index sees it: case-folded, by code point. */
|
|
90
|
+
export declare function trigrams(text: string): string[];
|
|
91
|
+
//#endregion
|
|
92
|
+
export type { NgramHit, NgramItem, NgramMatch, NgramSearchOptions, NgramSearchResult, NgramStoreOptions, QueryPlan };
|
|
93
|
+
//# sourceMappingURL=ngram.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ngram.d.mts","names":[],"sources":["../src/ngram/fold.ts","../src/ngram/fts5-backend.ts","../src/ngram/ngram-store.ts","../src/ngram/plan-query.ts","../src/ngram/trigrams.ts"],"mappings":";;;;;;;;;;;;;;wBAYgB,KAAK;;;KCRT;EAAc;EAAY;;;;KCI1B;EAAe;EAAe;;KAC9B;EAAa;EAAY,SAAS;;KAClC;;EAEV;;EAEA;;EAEA;;EAEA;;EAEA,UAAU;EACV,SAAS;;KAEC;;EAEV,MAAM;;EAEN;;KAEU;EAAsB,IAAI;EAAI;EAAgB;;qBAI7C;;;WAEF;EAMT,cAAc,IAAI,QAAQ,SAAkB;EAatC,QAAQ;EAuBR,OAAO,OAAO,cAAc;EA0B5B,OAAO,gBAAgB;EAavB,OAAO,iBAAiB,OAAM,qBAA0B,QAAQ;;EAwChE,SAAS,gBAAgB,QAAQ;;EAMjC,QAAQ;;;;;qBC9JH;KAED;;EAEV,OAAO;;;;;EAKP;;wBAGc,UACd,iBACA;EAAQ;EAAiB;IACxB;;;;wBCRa,SAAS"}
|