@statewalker/db-sqlite-index 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@statewalker/db-sqlite-index",
3
- "version": "0.1.1",
3
+ "version": "0.1.2",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "description": "Id-based vector (IVF) and n-gram indexes stored in SQLite through @statewalker/db-api: Node, browser and Durable Objects.",
@@ -22,31 +22,37 @@
22
22
  "./vector": {
23
23
  "source": "./src/vector/index.ts",
24
24
  "types": "./dist/vector.d.mts",
25
- "import": "./dist/vector.mjs"
25
+ "import": "./dist/vector.mjs",
26
+ "default": "./dist/vector.mjs"
26
27
  },
27
28
  "./ngram": {
28
29
  "source": "./src/ngram/index.ts",
29
30
  "types": "./dist/ngram.d.mts",
30
- "import": "./dist/ngram.mjs"
31
+ "import": "./dist/ngram.mjs",
32
+ "default": "./dist/ngram.mjs"
31
33
  }
32
34
  },
33
35
  "files": [
34
36
  "dist",
35
- "src"
37
+ "src",
38
+ "!src/**/*.test.ts",
39
+ "!src/**/test-fixtures.ts"
36
40
  ],
37
41
  "dependencies": {
38
- "@statewalker/db-api": "^0.1.5",
39
- "@statewalker/db-sqlite": "^0.1.1"
42
+ "@statewalker/db-api": "^0.1.6",
43
+ "@statewalker/db-sqlite": "^0.1.2"
40
44
  },
41
45
  "devDependencies": {
42
46
  "@types/node": "^26.6.4",
47
+ "esbuild": "^0.28.2",
43
48
  "rimraf": "^6.1.3",
44
49
  "tsdown": "^0.23.0",
45
50
  "tsx": "^4.23.15",
46
51
  "typescript": "^7.0.2",
47
52
  "vitest": "^4.1.10 || ^5.0.3",
48
- "@statewalker/db-sqlite-do": "^0.1.1",
49
- "@statewalker/db-sqlite-node": "^0.3.1"
53
+ "@statewalker/db-sqlite-node": "^0.3.2",
54
+ "@statewalker/db-sqlite-do": "^0.1.2",
55
+ "@statewalker/db-tests": "^0.1.1"
50
56
  },
51
57
  "sideEffects": false,
52
58
  "publishConfig": {
@@ -56,8 +62,10 @@
56
62
  "build": "tsdown",
57
63
  "dev": "tsdown --watch",
58
64
  "test": "vitest run",
65
+ "test:do": "vitest run --config vitest.do.config.ts",
59
66
  "test:watch": "vitest",
60
67
  "typecheck": "tsc --noEmit",
68
+ "typecheck:tests": "tsc --noEmit -p tsconfig.test.json",
61
69
  "bench": "tsx bench/vector-bench.ts 50000 384",
62
70
  "clean": "rimraf dist",
63
71
  "lint": "biome check --write .",
package/src/ngram/fold.ts CHANGED
@@ -19,3 +19,47 @@ export function fold(text: string): string {
19
19
  }
20
20
  return out;
21
21
  }
22
+
23
+ /**
24
+ * Highest code point probed by `foldIdentity`. Every cased character is in the
25
+ * BMP or the SMP (Deseret, Osage, Adlam, …); planes 2 and up hold none.
26
+ */
27
+ const PROBE_END = 0x1ffff;
28
+ const IDENTITY_VERSION = "fold-v1";
29
+ let current: string | undefined;
30
+
31
+ /**
32
+ * A fingerprint of what `foldFn` does to every code point up to U+1FFFF: a
33
+ * 53-bit hash of each code point it changes and what it changes it to. The
34
+ * grams of an n-gram index are built with `fold`, and a contentless FTS5 row is
35
+ * deleted by replaying them, so a runtime whose case mappings differ (a newer
36
+ * Unicode in a newer V8 or ICU) yields a different identity, and the store
37
+ * rebuilds its grams. It needs no runtime API, so it works in Node, browsers
38
+ * and Durable Objects alike. The identity of `fold` is computed once (about
39
+ * 30 ms) and memoized.
40
+ */
41
+ export function foldIdentity(foldFn: (text: string) => string = fold): string {
42
+ if (foldFn === fold && current !== undefined) return current;
43
+ // cyrb53: two 32-bit FNV-style lanes mixed into 53 bits.
44
+ let h1 = 0xdeadbeef;
45
+ let h2 = 0x41c6ce57;
46
+ const mix = (value: number) => {
47
+ h1 = Math.imul(h1 ^ value, 2654435761);
48
+ h2 = Math.imul(h2 ^ value, 1597334677);
49
+ };
50
+ for (let cp = 0; cp <= PROBE_END; cp++) {
51
+ if (cp >= 0xd800 && cp <= 0xdfff) continue;
52
+ const ch = String.fromCodePoint(cp);
53
+ const folded = foldFn(ch);
54
+ if (folded === ch) continue;
55
+ mix(cp);
56
+ for (let i = 0; i < folded.length; i++) mix(folded.charCodeAt(i));
57
+ mix(0x110000); // separator: the folded form of one code point may be several units
58
+ }
59
+ h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909);
60
+ h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909);
61
+ const hash = 4294967296 * (2097151 & h2) + (h1 >>> 0);
62
+ const identity = `${IDENTITY_VERSION}:${hash.toString(16)}`;
63
+ if (foldFn === fold) current = identity;
64
+ return identity;
65
+ }
@@ -10,6 +10,8 @@ export type GramBackend = {
10
10
  add(tx: Db, items: NgramItem[]): Promise<void>;
11
11
  /** `items` hold the texts as currently stored, so their grams can be removed exactly. */
12
12
  remove(tx: Db, items: NgramItem[]): Promise<void>;
13
+ /** Removes every gram, leaving an empty store (used to rebuild after a fold change). */
14
+ clear(tx: Db): Promise<void>;
13
15
  /** Ids whose text contains every literal of at least one branch; ascending, at most `max`. */
14
16
  candidates(db: Db, branches: string[][], max: number): Promise<number[]>;
15
17
  };
@@ -41,6 +43,8 @@ export function newFts5Backend(prefix: string): GramBackend {
41
43
  ),
42
44
  add: (tx, items) => write(tx, items, "insert"),
43
45
  remove: (tx, items) => write(tx, items, "delete"),
46
+ // 'rebuild' needs a content table; a contentless one can only be emptied.
47
+ clear: (tx) => tx.exec(`INSERT INTO ${table}(${table}) VALUES ('delete-all')`),
44
48
  async candidates(db, branches, max) {
45
49
  // A quoted string is a phrase of consecutive trigrams, i.e. an exact substring.
46
50
  const quote = (literal: string) => `"${literal.replaceAll('"', '""')}"`;
@@ -1,6 +1,7 @@
1
1
  import type { Db } from "@statewalker/db-api";
2
2
  import { inTransaction } from "@statewalker/db-sqlite";
3
3
  import { PREFIX_PATTERN } from "../prefix.js";
4
+ import { foldIdentity } from "./fold.js";
4
5
  import { type GramBackend, type NgramItem, newFts5Backend } from "./fts5-backend.js";
5
6
  import { chunkJson } from "./json-chunks.js";
6
7
  import { planQuery } from "./plan-query.js";
@@ -31,6 +32,8 @@ export type NgramSearchResult = {
31
32
  export type NgramStoreOptions = { db: Db; prefix: string; store?: "fts5" | "postings" };
32
33
 
33
34
  const VERIFY_BATCH = 256;
35
+ /** Texts re-indexed per step when the grams are rebuilt after a fold change. */
36
+ const REBUILD_BATCH = 256;
34
37
 
35
38
  export class NgramStore {
36
39
  /** `{prefix}texts`: its `id` column holds exactly the stored ids. Callers may join it in SQL. */
@@ -73,9 +76,43 @@ export class NgramStore {
73
76
  `CREATE TABLE IF NOT EXISTS ${p}texts(id INTEGER PRIMARY KEY, text TEXT NOT NULL)`,
74
77
  );
75
78
  await this.#backend.create(tx);
79
+ await this.#checkFold(tx);
76
80
  });
77
81
  }
78
82
 
83
+ /**
84
+ * The grams were built with `fold` as the runtime that wrote them defined it, and
85
+ * an fts5 delete replays them; if this runtime folds differently, they are rebuilt
86
+ * from the stored texts before any write. An index without a recorded identity
87
+ * (written before it was recorded) is assumed to match and is not rebuilt.
88
+ */
89
+ async #checkFold(tx: Db): Promise<void> {
90
+ const p = this.#prefix;
91
+ const identity = foldIdentity();
92
+ const [row] = await tx.query<{ value: string }>(
93
+ `SELECT value FROM ${p}meta WHERE key = 'fold'`,
94
+ );
95
+ if (row?.value === identity) return;
96
+ if (row) await this.#rebuild(tx);
97
+ await tx.query(`INSERT OR REPLACE INTO ${p}meta(key, value) VALUES ('fold', ?)`, [identity]);
98
+ }
99
+
100
+ async #rebuild(tx: Db): Promise<void> {
101
+ await this.#backend.clear(tx);
102
+ let after: number | null = null;
103
+ for (;;) {
104
+ const rows: Array<{ id: number; text: string }> = await tx.query(
105
+ `SELECT id, text FROM ${this.#prefix}texts WHERE ? IS NULL OR id > ? ORDER BY id LIMIT ?`,
106
+ [after, after, REBUILD_BATCH],
107
+ );
108
+ const batch = rows.map((r) => ({ id: Number(r.id), text: String(r.text) }));
109
+ const last = batch.at(-1);
110
+ if (!last) return;
111
+ await this.#backend.add(tx, batch);
112
+ after = last.id;
113
+ }
114
+ }
115
+
79
116
  async upsert(items: NgramItem[]): Promise<void> {
80
117
  // The last text per id wins; a repeated id would leave stale grams behind.
81
118
  // A lone surrogate would be stored differently from how it was indexed, so a
@@ -4,6 +4,8 @@ import { regexLiterals } from "./regex-literals.js";
4
4
  /** Longer patterns are rejected: a JS regex cannot be interrupted once it runs. */
5
5
  export const MAX_PATTERN_LENGTH = 1000;
6
6
 
7
+ const LONE_SURROGATE = /[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/;
8
+
7
9
  export type QueryPlan = {
8
10
  /** The pattern as a global regex (`i` added when not case-sensitive); verifies candidates. */
9
11
  regex: RegExp;
@@ -34,7 +36,12 @@ export function planQuery(
34
36
  const regex = new RegExp(source, opts.caseSensitive === false ? "gi" : "g");
35
37
  const branches: string[][] = [];
36
38
  for (const literals of regexLiterals(regex)) {
37
- const usable = literals.map(fold).filter((literal) => Array.from(literal).length >= 3);
39
+ // Without the `u` flag a literal may hold a lone surrogate (`ab😀?`, `\uD83D`); indexed
40
+ // text is split by code point, so keep only the well-formed pieces around it.
41
+ const usable = literals
42
+ .flatMap((literal) => literal.split(LONE_SURROGATE))
43
+ .map(fold)
44
+ .filter((literal) => Array.from(literal).length >= 3);
38
45
  if (usable.length === 0) return { regex, branches: null };
39
46
  branches.push(usable);
40
47
  }
@@ -36,6 +36,7 @@ export function newPostingsBackend(prefix: string): GramBackend {
36
36
  );
37
37
  }
38
38
  },
39
+ clear: (tx) => tx.exec(`DELETE FROM ${table}`),
39
40
  async candidates(db, branches, max) {
40
41
  const ids = new Set<number>();
41
42
  for (const branch of branches) {
@@ -297,9 +297,12 @@ function parseLiteralClass(src: string, start: number, end: number): string[] |
297
297
  const c = src[i] ?? "";
298
298
  if (c === "\\") {
299
299
  const next = src[i + 1];
300
- if (next === undefined || /[a-zA-Z]/.test(next)) return null;
301
- chars.add(next);
300
+ // Letters are classes or code escapes; digits are octal escapes without the `u` flag.
301
+ if (next === undefined || /[a-zA-Z0-9]/.test(next)) return null;
302
302
  i += 2;
303
+ // `[\.-z]`: the escape starts a range, not three literals.
304
+ if (src[i] === "-" && i + 1 < lastInside) return null;
305
+ chars.add(next);
303
306
  continue;
304
307
  }
305
308
  if (i + 2 < lastInside && src[i + 1] === "-") return null;
@@ -17,7 +17,12 @@ export function prepare(v: ArrayLike<number>, dims: number, metric: Metric): Flo
17
17
  for (let i = 0; i < f.length; i++) {
18
18
  if (!Number.isFinite(f[i])) throw new Error("vector-index: vector contains a non-finite value");
19
19
  }
20
- return metric === "cosine" ? normalize(f) : f.slice();
20
+ if (metric !== "cosine") return f.slice();
21
+ const out = normalize(f);
22
+ if (!out.some((x) => x !== 0)) {
23
+ throw new Error("vector-index: a zero vector has no direction under the cosine metric");
24
+ }
25
+ return out;
21
26
  }
22
27
 
23
28
  /** Distance between two float vectors at offsets; lower is closer. */
@@ -184,6 +184,9 @@ export interface Candidate {
184
184
  }
185
185
 
186
186
  /** The in-memory part of a segment: centroids and list sizes. List blocks are fetched by the caller. */
187
+ /** The share of a segment's lists that {@link SegmentIndex.probe} may widen to for `minRows`. */
188
+ export const MAX_WIDEN_FRACTION = 1 / 3;
189
+
187
190
  export class SegmentIndex {
188
191
  constructor(
189
192
  readonly gen: number,
@@ -197,20 +200,50 @@ export class SegmentIndex {
197
200
  return this.counts.length;
198
201
  }
199
202
 
200
- /** The `nprobe` non-empty lists whose centroids are closest to the query. */
201
- probe(query: Float32Array, nprobe: number): number[] {
203
+ /**
204
+ * The non-empty lists closest to the query, in centroid order: at least the
205
+ * `nprobe` nearest, then more until the probed lists hold `minRows` rows (or
206
+ * none are left). `minRows` 0 probes exactly `nprobe` lists. The widening
207
+ * stops at a third of the lists ({@link MAX_WIDEN_FRACTION}), so a large `k`
208
+ * never turns a search into a scan of every list.
209
+ */
210
+ probe(query: Float32Array, nprobe: number, minRows = 0): number[] {
211
+ const maxLists = Math.max(nprobe, Math.ceil(this.nlist * MAX_WIDEN_FRACTION));
202
212
  const m: Metric = this.metric === "l2" ? "l2" : "dot";
203
- const top = new TopK<number>(Math.min(nprobe, this.nlist));
213
+ const scored: { d: number; c: number }[] = [];
204
214
  for (let c = 0; c < this.nlist; c++) {
205
215
  if (this.counts[c] === 0) continue;
206
- top.push(distance(m, query, 0, this.centroids, c * this.dims, this.dims), c);
216
+ scored.push({ d: distance(m, query, 0, this.centroids, c * this.dims, this.dims), c });
207
217
  }
208
- return top.sorted().map((e) => e.item);
218
+ scored.sort((a, b) => a.d - b.d || a.c - b.c);
219
+ const out: number[] = [];
220
+ let rows = 0;
221
+ for (const { c } of scored) {
222
+ if (out.length >= nprobe && (rows >= minRows || out.length >= maxLists)) break;
223
+ out.push(c);
224
+ rows += this.counts[c];
225
+ }
226
+ return out;
227
+ }
228
+
229
+ /**
230
+ * Whether stage 1 passes a list of `n` rows whole (in row order, not ranked):
231
+ * a list no longer than `limit`, or a flat segment's only list of at most
232
+ * {@link MIN_TRAIN} rows. Callers must not cut such a list by position.
233
+ */
234
+ passesWhole(n: number, limit: number): boolean {
235
+ return n <= limit || (this.nlist === 1 && n <= MIN_TRAIN);
209
236
  }
210
237
 
211
238
  /**
212
239
  * Stage 1: the `limit` rows of a list with the best estimated distance, from
213
240
  * residual sign bits plus the stored residual norm and centroid projection.
241
+ *
242
+ * A flat segment (one list of at most {@link MIN_TRAIN} rows) passes every row:
243
+ * its centroid is the mean of all vectors, far from most of them on clustered
244
+ * data, so the 1-bit estimate is weakest exactly there (issue #40), while
245
+ * int8-scoring the whole list costs no more than scanning the tail. A list
246
+ * that does not exceed `limit` also passes whole, without estimating.
214
247
  */
215
248
  filterList(
216
249
  query: Float32Array,
@@ -219,6 +252,15 @@ export class SegmentIndex {
219
252
  limit: number,
220
253
  keep?: (id: number) => boolean,
221
254
  ): Candidate[] {
255
+ const n = data.ids.length;
256
+ if (this.passesWhole(n, limit)) {
257
+ const all: Candidate[] = [];
258
+ for (let r = 0; r < n; r++) {
259
+ const id = data.ids[r];
260
+ if (!keep || keep(id)) all.push({ list: c, row: r, id });
261
+ }
262
+ return all;
263
+ }
222
264
  const d = this.dims;
223
265
  const w = wordsFor(d);
224
266
  const co = c * d;
@@ -231,7 +273,7 @@ export class SegmentIndex {
231
273
  const angle = Math.PI / d;
232
274
  const l2 = this.metric === "l2";
233
275
  const top = new TopK<Candidate>(limit);
234
- for (let r = 0; r < data.ids.length; r++) {
276
+ for (let r = 0; r < n; r++) {
235
277
  const id = data.ids[r];
236
278
  if (keep && !keep(id)) continue;
237
279
  const rr = qn * data.rnorm[r] * Math.cos(angle * hamming(qb, data.bits, r * w, w));
@@ -8,7 +8,10 @@ export interface SearchHit {
8
8
 
9
9
  export interface SearchOptions {
10
10
  k?: number;
11
- /** IVF lists probed per segment. Default 8. */
11
+ /**
12
+ * IVF lists probed per segment, exactly. Default: at least 8, more until the
13
+ * probed lists hold 16 × k × oversample rows (small lists after auto-compaction).
14
+ */
12
15
  nprobe?: number;
13
16
  /** Candidates kept per probed list by the 1-bit filter = k * oversample. Default 8. */
14
17
  oversample?: number;
@@ -61,6 +61,13 @@ interface TailEntry {
61
61
 
62
62
  /** Ids per lookup (bound as one JSON parameter). */
63
63
  const VALIDATE_BATCH = 2000;
64
+ /** Lists probed per segment when the caller passes no `nprobe`. */
65
+ const DEFAULT_NPROBE = 8;
66
+ /**
67
+ * Without an explicit `nprobe`, probe lists until they hold this many times
68
+ * `k × oversample` rows (#67).
69
+ */
70
+ const PROBE_ROWS_FACTOR = 16;
64
71
  /** Upserted rows per transaction. */
65
72
  const WRITE_BATCH = 200;
66
73
 
@@ -243,14 +250,22 @@ export class VectorIndex {
243
250
 
244
251
  async search(query: ArrayLike<number>, opts: SearchOptions = {}): Promise<SearchHit[]> {
245
252
  this.#assertReady();
253
+ opts.signal?.throwIfAborted();
246
254
  const k = opts.k ?? 10;
247
- const nprobe = opts.nprobe ?? 8;
248
255
  const perList = k * (opts.oversample ?? 8);
256
+ // An explicit nprobe probes exactly that many lists. The default probes at least
257
+ // DEFAULT_NPROBE lists and keeps going while they hold fewer than
258
+ // PROBE_ROWS_FACTOR × perList rows: after auto-compaction lists are small
259
+ // (~65 rows at 4200 vectors), and 8 of them miss neighbours spread over many (#67).
260
+ const nprobe = opts.nprobe ?? DEFAULT_NPROBE;
261
+ const minRows = opts.nprobe === undefined ? PROBE_ROWS_FACTOR * perList : 0;
249
262
  const q = prepare(query, this.dims, this.metric);
250
263
  const d = this.dims;
251
264
  const filter = opts.filter;
265
+ const rerank = opts.exact ? k * 4 : k;
252
266
 
253
267
  return this.#read(async (segments) => {
268
+ opts.signal?.throwIfAborted();
254
269
  // Tail: exact, from memory.
255
270
  const tailTop = new TopK<number>(k);
256
271
  for (const [id, { vector }] of this.#tail) {
@@ -258,32 +273,63 @@ export class VectorIndex {
258
273
  tailTop.push(distance(this.metric, q, 0, vector, 0, d), id);
259
274
  }
260
275
 
261
- // Stage 1: probe lists, keep the best rows of each by estimated residual distance.
262
- type Hit = Candidate & { seg: SegmentIndex; data: ListData };
263
- const hits: Hit[] = [];
264
- // Tail ids are never live in a segment; skip them early.
265
- const keep = (id: number) => !this.#tail.has(id) && (!filter || filter(id));
276
+ // Probe lists and load their blocks.
277
+ type Probed = { seg: SegmentIndex; list: number; data: ListData };
278
+ const probed: Probed[] = [];
266
279
  await Promise.all(
267
280
  segments.map(async (seg) => {
268
- const lists = seg.probe(q, nprobe);
281
+ const lists = seg.probe(q, nprobe, minRows);
269
282
  const datas = await this.#store.lists(seg, lists);
270
- opts.signal?.throwIfAborted();
271
- lists.forEach((c, i) => {
272
- const data = datas[i] as ListData;
273
- for (const cand of seg.filterList(q, c, data, perList, keep))
274
- hits.push({ ...cand, seg, data });
283
+ lists.forEach((list, i) => {
284
+ probed.push({ seg, list, data: datas[i] as ListData });
275
285
  });
276
286
  }),
277
287
  );
288
+ opts.signal?.throwIfAborted();
278
289
 
279
- // Liveness: a segment row counts only if its generation holds the live copy.
280
- const live = await this.#generations(hits.map((h) => h.id));
290
+ // Stage 1: keep the best `perList` rows of each list by estimated residual
291
+ // distance. A segment row counts only if its generation holds the live copy
292
+ // of its id. Rows found dead (deleted, or overwritten in another generation)
293
+ // are excluded and the list is filtered again with a doubled limit, so dead
294
+ // rows never take the slots of live ones (#33).
295
+ type Hit = Candidate & { seg: SegmentIndex; data: ListData };
296
+ const gens = new Map<number, number | undefined>(); // id -> live gen; undefined = deleted
297
+ const hits: Hit[] = [];
298
+ let pending = probed.map((p) => ({ ...p, limit: perList }));
299
+ while (pending.length > 0) {
300
+ const rounds = pending.map((p) => {
301
+ // Tail ids are never live in a segment; known-dead rows are skipped too.
302
+ const keep = (id: number) =>
303
+ !this.#tail.has(id) &&
304
+ !(gens.has(id) && gens.get(id) !== p.seg.gen) &&
305
+ (!filter || filter(id));
306
+ return { p, cands: p.seg.filterList(q, p.list, p.data, p.limit, keep) };
307
+ });
308
+ const unknown = rounds
309
+ .flatMap((r) => r.cands.map((c) => c.id))
310
+ .filter((id) => !gens.has(id));
311
+ if (unknown.length > 0) {
312
+ const found = await this.#generations(unknown);
313
+ for (const id of unknown) gens.set(id, found.get(id));
314
+ opts.signal?.throwIfAborted();
315
+ }
316
+ pending = [];
317
+ for (const { p, cands } of rounds) {
318
+ const live = cands.filter((c) => gens.get(c.id) === p.seg.gen);
319
+ // A list passed whole comes in row order: keep all of it, the int8 re-rank ranks it.
320
+ const whole = p.seg.passesWhole(p.data.ids.length, p.limit);
321
+ if (!whole && live.length < perList && cands.length === p.limit) {
322
+ pending.push({ ...p, limit: p.limit * 2 }); // some were dead and more rows remain
323
+ continue;
324
+ }
325
+ for (const cand of whole ? live : live.slice(0, perList))
326
+ hits.push({ ...cand, seg: p.seg, data: p.data });
327
+ }
328
+ }
281
329
 
282
330
  // Stage 2: int8 re-rank.
283
- const rerank = opts.exact ? k * 4 : k;
284
331
  const segTop = new TopK<Hit>(rerank);
285
332
  for (const h of hits) {
286
- if (live.get(h.id) !== h.seg.gen) continue;
287
333
  segTop.push(h.seg.scoreInt8(q, h.data, h.row), h);
288
334
  }
289
335
  let segResults = segTop
@@ -1,46 +0,0 @@
1
- import type { Db } from "@statewalker/db-api";
2
- import { NgramStore } from "./ngram-store.js";
3
-
4
- const check = (ok: boolean, message: string) => {
5
- if (!ok) throw new Error(`ngram smoke: ${message}`);
6
- };
7
-
8
- /** True when the index itself still offers candidates (candidateBudget 0 skips the verifier). */
9
- const indexed = async (index: NgramStore, pattern: string) =>
10
- (await index.search(pattern, { candidateBudget: 0 })).truncated;
11
-
12
- const ids = (result: { hits: { id: number }[] }) => result.hits.map((h) => h.id).join();
13
-
14
- /** Run inside a real Durable Object by db-sqlite-do's `pnpm test:do`. */
15
- export const ngramDoScenarios = (["fts5", "postings"] as const).map((store) => ({
16
- name: `ngram (${store}): upsert, overwrite, lone surrogate, drop inside a transaction`,
17
- async run(db: Db) {
18
- const prefix = `smoke_${store}_`;
19
- const index = new NgramStore({ db, prefix, store });
20
- await index.init(); // CREATE of the index tables (a contentless trigram table for fts5) in a transaction
21
- await index.upsert([
22
- { id: 1, text: "Hello World" },
23
- { id: 2, text: "hello there" },
24
- { id: 3, text: "lone \ud800 surrogate" },
25
- ]);
26
- check(
27
- ids(await index.search("hello", { caseSensitive: false })) === "1,2",
28
- "case-insensitive hits",
29
- );
30
- check(ids(await index.search("Hello")) === "1", "case-sensitive hits");
31
- check(ids(await index.search("surrogate")) === "3", "text with a lone surrogate is indexed");
32
- check((await index.getTexts([3])).get(3) === "lone \ufffd surrogate", "text made well-formed");
33
- check(await indexed(index, "World"), "index lacks the grams of 'World'");
34
- check(await indexed(index, "surrogate"), "index lacks the grams of 'surrogate'");
35
- await index.upsert([{ id: 1, text: "goodbye" }]); // replays the old text to remove its grams
36
- check(!(await indexed(index, "World")), "stale grams after overwrite");
37
- await index.upsert([{ id: 3, text: "lone \ud800 changed" }]);
38
- check(!(await indexed(index, "surrogate")), "stale grams after surrogate overwrite");
39
- await index.drop(); // DROP of the tables inside a transaction
40
- const [row] = await db.query<{ n: number }>(
41
- "SELECT count(*) AS n FROM sqlite_master WHERE name LIKE ?",
42
- [`${prefix}%`],
43
- );
44
- check(Number(row?.n) === 0, `tables left after drop: ${row?.n}`);
45
- },
46
- }));
@@ -1,50 +0,0 @@
1
- import { describe, expect, it } from "vitest";
2
- import { fold } from "./fold.js";
3
-
4
- const escapeRegex = (s: string) => s.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&");
5
-
6
- describe("fold", () => {
7
- it("lower-cases ASCII and keeps length", () => {
8
- expect(fold("Hello WORLD")).toBe("hello world");
9
- });
10
-
11
- it("maps every case form of sigma to one letter", () => {
12
- expect(new Set([fold("Σ"), fold("σ"), fold("ς")]).size).toBe(1);
13
- });
14
-
15
- it("never separates two BMP characters that a case-insensitive regex treats as equal", () => {
16
- const broken: string[] = [];
17
- for (let cp = 0; cp <= 0xffff; cp++) {
18
- if (cp >= 0xd800 && cp <= 0xdfff) continue;
19
- const c = String.fromCharCode(cp);
20
- const re = new RegExp(`^${escapeRegex(c)}$`, "i");
21
- for (const d of [c.toUpperCase(), c.toLowerCase()]) {
22
- if (d === c || d.length !== 1 || !re.test(d)) continue;
23
- if (fold(c) !== fold(d)) broken.push(`U+${cp.toString(16)} ${c}/${d}`);
24
- }
25
- }
26
- expect(broken).toEqual([]);
27
- });
28
-
29
- it("keeps the UTF-16 length of every code point, so offsets stay valid", () => {
30
- const changed: string[] = [];
31
- for (let cp = 0; cp <= 0x10ffff; cp++) {
32
- if (cp >= 0xd800 && cp <= 0xdfff) continue;
33
- const c = String.fromCodePoint(cp);
34
- if (fold(c).length !== c.length) changed.push(`U+${cp.toString(16)}`);
35
- }
36
- expect(changed).toEqual([]);
37
- });
38
-
39
- it("keeps the length of a text with U+0130 and an astral character", () => {
40
- const folded = fold("İ😀A");
41
- expect(folded.length).toBe("İ😀A".length);
42
- expect(folded.endsWith("a")).toBe(true);
43
- });
44
-
45
- it("is a homomorphism over concatenation, so substrings stay substrings", () => {
46
- const a = "ΟΔΟΣ ſtraße";
47
- const b = "İK";
48
- expect(fold(a + b)).toBe(fold(a) + fold(b));
49
- });
50
- });
@@ -1,23 +0,0 @@
1
- import { describe, expect, it } from "vitest";
2
- import { chunkJson } from "./json-chunks.js";
3
-
4
- describe("chunkJson", () => {
5
- const rows = Array.from({ length: 30 }, (_, i) => [i, `t${i}`]);
6
-
7
- it("splits into chunks of at most maxChars that rebuild the rows in order", () => {
8
- const chunks = chunkJson(rows, 20);
9
- expect(chunks.length).toBeGreaterThan(1);
10
- for (const chunk of chunks) expect(chunk.length).toBeLessThanOrEqual(20);
11
- expect(chunks.flatMap((chunk) => JSON.parse(chunk))).toEqual(rows);
12
- });
13
-
14
- it("keeps a row larger than maxChars as one chunk of its own", () => {
15
- const big = [1, "x".repeat(50)];
16
- const chunks = chunkJson([[0, "a"], big, [2, "b"]], 20);
17
- expect(chunks).toEqual(['[[0,"a"]]', JSON.stringify([big]), '[[2,"b"]]']);
18
- });
19
-
20
- it("returns no chunks for no rows", () => {
21
- expect(chunkJson([], 20)).toEqual([]);
22
- });
23
- });