@statewalker/db-sqlite-index 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +289 -2
  3. package/dist/ngram.d.mts +93 -0
  4. package/dist/ngram.d.mts.map +1 -0
  5. package/dist/ngram.mjs +615 -0
  6. package/dist/ngram.mjs.map +1 -0
  7. package/dist/vector.d.mts +103 -0
  8. package/dist/vector.d.mts.map +1 -0
  9. package/dist/vector.mjs +1061 -0
  10. package/dist/vector.mjs.map +1 -0
  11. package/package.json +63 -3
  12. package/src/ngram/do-smoke.ts +46 -0
  13. package/src/ngram/fold.test.ts +50 -0
  14. package/src/ngram/fold.ts +21 -0
  15. package/src/ngram/fts5-backend.ts +55 -0
  16. package/src/ngram/index.ts +12 -0
  17. package/src/ngram/json-chunks.test.ts +23 -0
  18. package/src/ngram/json-chunks.ts +23 -0
  19. package/src/ngram/ngram-store.test.ts +291 -0
  20. package/src/ngram/ngram-store.ts +183 -0
  21. package/src/ngram/plan-query.test.ts +141 -0
  22. package/src/ngram/plan-query.ts +42 -0
  23. package/src/ngram/postings-backend.ts +53 -0
  24. package/src/ngram/regex-literals.ts +359 -0
  25. package/src/ngram/trigrams.test.ts +28 -0
  26. package/src/ngram/trigrams.ts +25 -0
  27. package/src/vector/auto-compact.test.ts +28 -0
  28. package/src/vector/do-smoke.ts +39 -0
  29. package/src/vector/index.ts +9 -0
  30. package/src/vector/kmeans.ts +131 -0
  31. package/src/vector/math.ts +185 -0
  32. package/src/vector/segment.test.ts +61 -0
  33. package/src/vector/segment.ts +256 -0
  34. package/src/vector/sql-segment-store.ts +218 -0
  35. package/src/vector/test-fixtures.ts +64 -0
  36. package/src/vector/types.ts +30 -0
  37. package/src/vector/vector-index.test.ts +105 -0
  38. package/src/vector/vector-index.ts +586 -0
  39. package/src/vector/vector-lifecycle.test.ts +171 -0
@@ -0,0 +1 @@
1
+ {"version":3,"file":"vector.mjs","names":[],"sources":["../src/vector/math.ts","../src/vector/kmeans.ts","../src/vector/segment.ts","../src/vector/types.ts","../src/vector/sql-segment-store.ts","../src/vector/vector-index.ts"],"sourcesContent":["import type { Metric } from \"./types.js\";\n\nexport function normalize(v: Float32Array): Float32Array {\n let n = 0;\n for (let i = 0; i < v.length; i++) n += v[i] * v[i];\n n = Math.sqrt(n);\n const out = new Float32Array(v.length);\n if (n === 0) return out;\n for (let i = 0; i < v.length; i++) out[i] = v[i] / n;\n return out;\n}\n\n/** Vectors are stored pre-normalized for cosine, so cosine reduces to dot. */\nexport function prepare(v: ArrayLike<number>, dims: number, metric: Metric): Float32Array {\n if (v.length !== dims) throw new Error(`vector-index: expected ${dims} dims, got ${v.length}`);\n const f = v instanceof Float32Array ? v : Float32Array.from(v);\n for (let i = 0; i < f.length; i++) {\n if (!Number.isFinite(f[i])) throw new Error(\"vector-index: vector contains a non-finite value\");\n }\n return metric === \"cosine\" ? normalize(f) : f.slice();\n}\n\n/** Distance between two float vectors at offsets; lower is closer. */\nexport function distance(\n metric: Metric,\n a: Float32Array,\n ao: number,\n b: Float32Array,\n bo: number,\n d: number,\n): number {\n if (metric === \"l2\") {\n let s = 0;\n for (let i = 0; i < d; i++) {\n const x = a[ao + i] - b[bo + i];\n s += x * x;\n }\n return s;\n }\n let s = 0;\n for (let i = 0; i < d; i++) s += a[ao + i] * b[bo + i];\n return metric === \"cosine\" ? 1 - s : -s;\n}\n\n// ------------------------------------------------------------ int8\n\n/** Symmetric per-vector int8: x ≈ q * scale / 127. Returns the scale. */\nexport function quantizeInt8(v: Float32Array, vo: number, d: number, out: Int8Array, oo: number) {\n let max = 0;\n for (let i = 0; i < d; i++) max = Math.max(max, Math.abs(v[vo + i]));\n const inv = max === 0 ? 0 : 127 / max;\n for (let i = 0; i < d; i++) out[oo + i] = Math.round(v[vo + i] * inv);\n return max;\n}\n\n/** Distance between a float query and an int8-coded vector. */\nexport function distanceInt8(\n metric: Metric,\n q: Float32Array,\n codes: Int8Array,\n co: number,\n scale: number,\n d: number,\n): number {\n const f = scale / 127;\n if (metric === \"l2\") {\n let s = 0;\n for (let i = 0; i < d; i++) {\n const x = q[i] - codes[co + i] * f;\n s += x * x;\n }\n return s;\n }\n let s = 0;\n for (let i = 0; i < d; i++) s += q[i] * codes[co + i];\n s *= f;\n return metric === \"cosine\" ? 1 - s : -s;\n}\n\nexport function dequantizeInt8(codes: Int8Array, co: number, scale: number, d: number) {\n const out = new Float32Array(d);\n const f = scale / 127;\n for (let i = 0; i < d; i++) out[i] = codes[co + i] * f;\n return out;\n}\n\n// ------------------------------------------------------------ 1-bit\n\nexport const wordsFor = (d: number) => (d + 31) >>> 5;\n\n/** Sign bits of (v - centroid): residual binarization, meaningful for every metric. */\nexport function residualBits(\n v: Float32Array,\n vo: number,\n c: Float32Array,\n co: number,\n d: number,\n out: Uint32Array,\n oo: number,\n) {\n const w = wordsFor(d);\n out.fill(0, oo, oo + w);\n for (let i = 0; i < d; i++) {\n if (v[vo + i] > c[co + i]) out[oo + (i >>> 5)] |= 1 << (i & 31);\n }\n}\n\nexport function hamming(a: Uint32Array, b: Uint32Array, bo: number, w: number): number {\n let n = 0;\n for (let i = 0; i < w; i++) {\n let x = a[i] ^ b[bo + i];\n x -= (x >>> 1) & 0x55555555;\n x = (x & 0x33333333) + ((x >>> 2) & 0x33333333);\n n += Math.imul((x + (x >>> 4)) & 0x0f0f0f0f, 0x01010101) >>> 24;\n }\n return n;\n}\n\n// ------------------------------------------------------------ top-k\n\n/** Keeps the `k` smallest scores. A binary max-heap over parallel arrays. */\nexport class TopK<T> {\n readonly #k: number;\n readonly #scores: number[] = [];\n readonly #items: T[] = [];\n\n constructor(k: number) {\n this.#k = Math.max(0, k);\n }\n\n get size() {\n return this.#scores.length;\n }\n\n /** The score a new item must beat, or Infinity while not full. */\n get bound() {\n return this.#scores.length < this.#k ? Number.POSITIVE_INFINITY : this.#scores[0];\n }\n\n push(score: number, item: T) {\n if (this.#k === 0) return;\n const s = this.#scores;\n const it = this.#items;\n if (s.length < this.#k) {\n s.push(score);\n it.push(item);\n let i = s.length - 1;\n while (i > 0) {\n const p = (i - 1) >> 1;\n if (s[p] >= s[i]) break;\n this.#swap(i, p);\n i = p;\n }\n return;\n }\n if (score >= s[0]) return;\n s[0] = score;\n it[0] = item;\n let i = 0;\n for (;;) {\n const l = 2 * i + 1;\n const r = l + 1;\n let m = i;\n if (l < s.length && s[l] > s[m]) m = l;\n if (r < s.length && s[r] > s[m]) m = r;\n if (m === i) break;\n this.#swap(i, m);\n i = m;\n }\n }\n\n /** Ascending by score. */\n sorted(): { score: number; item: T }[] {\n return this.#scores\n .map((score, i) => ({ score, item: this.#items[i] }))\n .sort((a, b) => a.score - b.score);\n }\n\n #swap(i: number, j: number) {\n const s = this.#scores;\n const it = this.#items;\n [s[i], s[j]] = [s[j], s[i]];\n [it[i], it[j]] = [it[j], it[i]];\n }\n}\n","import { distance, normalize } from \"./math.js\";\nimport type { Metric } from \"./types.js\";\n\nexport interface KMeansOptions {\n iterations?: number;\n /** Training points drawn from the data. Default min(n, 32 * k, 25_000). */\n sampleSize?: number;\n seed?: number;\n}\n\n/** Small deterministic PRNG so segment builds are reproducible. */\nexport function mulberry32(seed: number) {\n let a = seed >>> 0;\n return () => {\n a = (a + 0x6d2b79f5) >>> 0;\n let t = a;\n t = Math.imul(t ^ (t >>> 15), t | 1);\n t ^= t + Math.imul(t ^ (t >>> 7), t | 61);\n return ((t ^ (t >>> 14)) >>> 0) / 4294967296;\n };\n}\n\n/** Index of the closest centroid. */\nexport function nearest(\n metric: Metric,\n data: Float32Array,\n offset: number,\n centroids: Float32Array,\n k: number,\n d: number,\n): number {\n let best = 0;\n let bestD = Number.POSITIVE_INFINITY;\n // Centroids are compared with l2 for l2, and with dot otherwise (spherical k-means).\n const m: Metric = metric === \"l2\" ? \"l2\" : \"dot\";\n for (let c = 0; c < k; c++) {\n const dist = distance(m, data, offset, centroids, c * d, d);\n if (dist < bestD) {\n bestD = dist;\n best = c;\n }\n }\n return best;\n}\n\n/**\n * k-means++ seeding plus Lloyd iterations on a sample. `data` is n × d, flat.\n * For cosine and dot, centroids are re-normalized (spherical k-means).\n */\nexport function kmeans(\n data: Float32Array,\n n: number,\n d: number,\n k: number,\n metric: Metric,\n opts: KMeansOptions = {},\n): Float32Array {\n const rand = mulberry32(opts.seed ?? 42);\n k = Math.max(1, Math.min(k, n));\n const sampleSize = Math.min(n, opts.sampleSize ?? Math.min(32 * k, 25_000));\n\n // Reservoir-free sample: a partial Fisher–Yates over indices.\n const idx = new Uint32Array(n);\n for (let i = 0; i < n; i++) idx[i] = i;\n for (let i = 0; i < sampleSize; i++) {\n const j = i + Math.floor(rand() * (n - i));\n [idx[i], idx[j]] = [idx[j], idx[i]];\n }\n const sample = new Float32Array(sampleSize * d);\n for (let i = 0; i < sampleSize; i++) sample.set(data.subarray(idx[i] * d, idx[i] * d + d), i * d);\n\n const spherical = metric !== \"l2\";\n const centroids = new Float32Array(k * d);\n\n // k-means++ with squared l2 on the sample.\n centroids.set(sample.subarray(0, d), 0);\n const minD = new Float64Array(sampleSize).fill(Number.POSITIVE_INFINITY);\n for (let c = 1; c < k; c++) {\n let total = 0;\n for (let i = 0; i < sampleSize; i++) {\n const dd = distance(\"l2\", sample, i * d, centroids, (c - 1) * d, d);\n if (dd < minD[i]) minD[i] = dd;\n total += minD[i];\n }\n let r = rand() * total;\n let pick = sampleSize - 1;\n for (let i = 0; i < sampleSize; i++) {\n r -= minD[i];\n if (r <= 0) {\n pick = i;\n break;\n }\n }\n centroids.set(sample.subarray(pick * d, pick * d + d), c * d);\n }\n\n const assign = new Uint32Array(sampleSize);\n const counts = new Uint32Array(k);\n const iterations = opts.iterations ?? 10;\n for (let it = 0; it < iterations; it++) {\n let moved = 0;\n for (let i = 0; i < sampleSize; i++) {\n const c = nearest(metric, sample, i * d, centroids, k, d);\n if (c !== assign[i] || it === 0) moved++;\n assign[i] = c;\n }\n const sums = new Float64Array(k * d);\n counts.fill(0);\n for (let i = 0; i < sampleSize; i++) {\n const c = assign[i];\n counts[c]++;\n for (let j = 0; j < d; j++) sums[c * d + j] += sample[i * d + j];\n }\n for (let c = 0; c < k; c++) {\n if (counts[c] === 0) {\n // Re-seed an empty cluster with a random sample point.\n const p = Math.floor(rand() * sampleSize);\n centroids.set(sample.subarray(p * d, p * d + d), c * d);\n continue;\n }\n for (let j = 0; j < d; j++) centroids[c * d + j] = sums[c * d + j] / counts[c];\n }\n if (spherical) {\n for (let c = 0; c < k; c++) {\n centroids.set(normalize(centroids.subarray(c * d, c * d + d)), c * d);\n }\n }\n if (it > 0 && moved === 0) break;\n }\n return centroids;\n}\n","import { kmeans, nearest } from \"./kmeans.js\";\nimport {\n distance,\n distanceInt8,\n hamming,\n quantizeInt8,\n residualBits,\n TopK,\n wordsFor,\n} from \"./math.js\";\nimport type { Metric } from \"./types.js\";\n\n/**\n * An IVF segment as storage-neutral parts. This module does no I/O; a store\n * decides where the parts live.\n *\n * per list: a centroid f32[dims] and a row count\n * per list, the searchable block (little-endian, each section padded to 8 bytes):\n * ids f64[count] | bits u32[count × words] | rnorm f32[count] | cdot f32[count]\n * scales f32[count] | codes i8[count × dims]\n * per list, the float block f32[count × dims] (exact re-rank, lossless merge, read-back)\n *\n * bits = signs of the residual r = x - c (c = the list centroid)\n * rnorm = |r|, cdot = c·r. With them the 1-bit stage estimates the real\n * metric, RaBitQ-style: r_q·r ≈ |r_q| |r| cos(π · hamming / dims).\n */\nconst align8 = (n: number) => (n + 7) & ~7;\n\nexport interface ListLayout {\n count: number;\n idsAt: number;\n bitsAt: number;\n rnormAt: number;\n cdotAt: number;\n scalesAt: number;\n codesAt: number;\n /** Bytes of the searchable block. */\n length: number;\n}\n\nexport function listLayout(count: number, dims: number): ListLayout {\n const w = wordsFor(dims);\n const idsAt = 0;\n const bitsAt = align8(idsAt + 8 * count);\n const rnormAt = align8(bitsAt + 4 * w * count);\n const cdotAt = align8(rnormAt + 4 * count);\n const scalesAt = align8(cdotAt + 4 * count);\n const codesAt = align8(scalesAt + 4 * count);\n const length = align8(codesAt + dims * count);\n return { count, idsAt, bitsAt, rnormAt, cdotAt, scalesAt, codesAt, length };\n}\n\n// ------------------------------------------------------------------ build\n\nexport interface BuildInput {\n dims: number;\n metric: Metric;\n ids: Float64Array;\n /** n × dims, already prepared (normalized for cosine). */\n vectors: Float32Array;\n nlist?: number;\n seed?: number;\n}\n\nexport interface BuiltSegment {\n nlist: number;\n counts: Uint32Array;\n /** nlist × dims */\n centroids: Float32Array;\n /** Searchable block per list. */\n lists: Uint8Array[];\n /** Float32 block per list. */\n floats: Uint8Array[];\n}\n\n/** Below this size a segment is a single flat list: training would not pay off. */\nexport const MIN_TRAIN = 2048;\n\nexport function defaultNlist(n: number) {\n if (n < MIN_TRAIN) return 1;\n return Math.max(1, Math.min(1024, Math.round(Math.sqrt(n))));\n}\n\nexport function buildSegment(input: BuildInput): BuiltSegment {\n const { dims: d, metric, ids, vectors } = input;\n const n = ids.length;\n const nlist = Math.max(1, Math.min(input.nlist ?? defaultNlist(n), Math.max(1, n)));\n\n let centroids: Float32Array;\n if (nlist === 1) {\n centroids = new Float32Array(d);\n for (let i = 0; i < n; i++) for (let j = 0; j < d; j++) centroids[j] += vectors[i * d + j];\n if (n > 0) for (let j = 0; j < d; j++) centroids[j] /= n;\n } else {\n centroids = kmeans(vectors, n, d, nlist, metric, { seed: input.seed });\n }\n\n const assign = new Uint32Array(n);\n const counts = new Uint32Array(nlist);\n for (let i = 0; i < n; i++) {\n const c = nlist === 1 ? 0 : nearest(metric, vectors, i * d, centroids, nlist, d);\n assign[i] = c;\n counts[c]++;\n }\n\n const w = wordsFor(d);\n const layouts = Array.from(counts, (count) => listLayout(count, d));\n const lists = layouts.map((l) => new Uint8Array(l.length));\n const floats = Array.from(counts, (count) => new Uint8Array(4 * count * d));\n const views = layouts.map((l, c) => {\n const buf = lists[c].buffer;\n return {\n ids: new Float64Array(buf, l.idsAt, l.count),\n bits: new Uint32Array(buf, l.bitsAt, l.count * w),\n rnorm: new Float32Array(buf, l.rnormAt, l.count),\n cdot: new Float32Array(buf, l.cdotAt, l.count),\n scales: new Float32Array(buf, l.scalesAt, l.count),\n codes: new Int8Array(buf, l.codesAt, l.count * d),\n floats: new Float32Array(floats[c].buffer, 0, l.count * d),\n };\n });\n\n const fill = new Uint32Array(nlist);\n for (let i = 0; i < n; i++) {\n const c = assign[i];\n const j = fill[c]++;\n const v = views[c];\n v.ids[j] = ids[i];\n residualBits(vectors, i * d, centroids, c * d, d, v.bits, j * w);\n const [rn, cd] = residualStats(vectors, i * d, centroids, c * d, d);\n v.rnorm[j] = rn;\n v.cdot[j] = cd;\n v.scales[j] = quantizeInt8(vectors, i * d, d, v.codes, j * d);\n v.floats.set(vectors.subarray(i * d, i * d + d), j * d);\n }\n return { nlist, counts, centroids, lists, floats };\n}\n\n/** [|x - c|, c·(x - c)] */\nfunction residualStats(x: Float32Array, xo: number, c: Float32Array, co: number, d: number) {\n let nn = 0;\n let cd = 0;\n for (let i = 0; i < d; i++) {\n const r = x[xo + i] - c[co + i];\n nn += r * r;\n cd += c[co + i] * r;\n }\n return [Math.sqrt(nn), cd];\n}\n\n// ------------------------------------------------------------------ read\n\nexport interface ListData {\n ids: Float64Array;\n bits: Uint32Array;\n rnorm: Float32Array;\n cdot: Float32Array;\n scales: Float32Array;\n codes: Int8Array;\n}\n\n/** Typed views over a searchable block (copied only when it does not start at offset 0). */\nexport function parseList(bytes: Uint8Array, count: number, dims: number): ListData {\n const l = listLayout(count, dims);\n if (bytes.byteLength !== l.length) {\n throw new Error(`vector-index: list block has ${bytes.byteLength} bytes, expected ${l.length}`);\n }\n const buf = bytes.byteOffset === 0 ? bytes.buffer : bytes.slice().buffer;\n const w = wordsFor(dims);\n return {\n ids: new Float64Array(buf, l.idsAt, count),\n bits: new Uint32Array(buf, l.bitsAt, count * w),\n rnorm: new Float32Array(buf, l.rnormAt, count),\n cdot: new Float32Array(buf, l.cdotAt, count),\n scales: new Float32Array(buf, l.scalesAt, count),\n codes: new Int8Array(buf, l.codesAt, count * dims),\n };\n}\n\nexport interface Candidate {\n list: number;\n row: number;\n id: number;\n}\n\n/** The in-memory part of a segment: centroids and list sizes. List blocks are fetched by the caller. */\nexport class SegmentIndex {\n constructor(\n readonly gen: number,\n readonly dims: number,\n readonly metric: Metric,\n readonly centroids: Float32Array,\n readonly counts: Uint32Array,\n ) {}\n\n get nlist() {\n return this.counts.length;\n }\n\n /** The `nprobe` non-empty lists whose centroids are closest to the query. */\n probe(query: Float32Array, nprobe: number): number[] {\n const m: Metric = this.metric === \"l2\" ? \"l2\" : \"dot\";\n const top = new TopK<number>(Math.min(nprobe, this.nlist));\n for (let c = 0; c < this.nlist; c++) {\n if (this.counts[c] === 0) continue;\n top.push(distance(m, query, 0, this.centroids, c * this.dims, this.dims), c);\n }\n return top.sorted().map((e) => e.item);\n }\n\n /**\n * Stage 1: the `limit` rows of a list with the best estimated distance, from\n * residual sign bits plus the stored residual norm and centroid projection.\n */\n filterList(\n query: Float32Array,\n c: number,\n data: ListData,\n limit: number,\n keep?: (id: number) => boolean,\n ): Candidate[] {\n const d = this.dims;\n const w = wordsFor(d);\n const co = c * d;\n const qb = new Uint32Array(w);\n residualBits(query, 0, this.centroids, co, d, qb, 0);\n const [qn, qcd] = residualStats(query, 0, this.centroids, co, d);\n let cc = 0;\n for (let i = 0; i < d; i++) cc += this.centroids[co + i] * this.centroids[co + i];\n const qc = cc + qcd; // q·c\n const angle = Math.PI / d;\n const l2 = this.metric === \"l2\";\n const top = new TopK<Candidate>(limit);\n for (let r = 0; r < data.ids.length; r++) {\n const id = data.ids[r];\n if (keep && !keep(id)) continue;\n const rr = qn * data.rnorm[r] * Math.cos(angle * hamming(qb, data.bits, r * w, w));\n // q·x = q·c + c·r_x + r_q·r_x ; |q-x|² = |r_q|² + |r_x|² - 2 r_q·r_x\n const est = l2 ? qn * qn + data.rnorm[r] * data.rnorm[r] - 2 * rr : -(qc + data.cdot[r] + rr);\n if (est < top.bound) top.push(est, { list: c, row: r, id });\n }\n return top.sorted().map((e) => e.item);\n }\n\n /** Stage 2: approximate distance from the int8 codes. */\n scoreInt8(query: Float32Array, data: ListData, row: number): number {\n return distanceInt8(\n this.metric,\n query,\n data.codes,\n row * this.dims,\n data.scales[row],\n this.dims,\n );\n }\n}\n","export type Metric = \"cosine\" | \"dot\" | \"l2\";\n\nexport interface SearchHit {\n id: number;\n /** Lower is closer: 1 - cos for cosine, -dot for dot, squared L2 for l2. */\n distance: number;\n}\n\nexport interface SearchOptions {\n k?: number;\n /** IVF lists probed per segment. Default 8. */\n nprobe?: number;\n /** Candidates kept per probed list by the 1-bit filter = k * oversample. Default 8. */\n oversample?: number;\n /** Re-rank the int8 winners with the stored float32 vectors. Default false. */\n exact?: boolean;\n /** Keep only ids for which this returns true (applied before re-ranking). */\n filter?: (id: number) => boolean;\n signal?: AbortSignal;\n}\n\n/** An aligned Float32Array copy of BLOB bytes (views over arbitrary offsets are unsafe). */\nexport function toFloat32(bytes: Uint8Array): Float32Array {\n const copy = bytes.slice();\n return new Float32Array(copy.buffer, 0, copy.byteLength >> 2);\n}\n\nexport function float32Bytes(v: Float32Array): Uint8Array {\n return new Uint8Array(v.buffer, v.byteOffset, v.byteLength).slice();\n}\n","import type { Db } from \"@statewalker/db-api\";\nimport { type BuiltSegment, type ListData, parseList, SegmentIndex } from \"./segment.js\";\nimport { float32Bytes, type Metric, toFloat32 } from \"./types.js\";\n\n/**\n * Largest BLOB written in one row. Durable Object SQLite rejects values over\n * ~2 MB (\"string or blob too big\"); 1 MiB leaves headroom.\n */\nexport const PART_BYTES = 1024 * 1024;\n\nexport interface SegmentRef {\n gen: number;\n count: number;\n nlist: number;\n}\n\n/**\n * Segment rows: one centroid row per list, and each list's searchable and\n * float32 blocks split into parts. Reads go through an LRU cache: a\n * generation is immutable, so cached blocks never go stale.\n */\nexport class SqlSegmentStore {\n readonly #db: Db;\n readonly #dims: number;\n readonly #t: { centroids: string; seg: string; f32: string };\n readonly #cache: Lru<ListData | Float32Array>;\n\n constructor(db: Db, prefix: string, dims: number, cacheBytes = 64 * 1024 * 1024) {\n this.#db = db;\n this.#dims = dims;\n this.#t = { centroids: `${prefix}centroids`, seg: `${prefix}seg`, f32: `${prefix}f32` };\n this.#cache = new Lru(cacheBytes);\n }\n\n schema(): string[] {\n const { centroids, seg, f32 } = this.#t;\n return [\n `CREATE TABLE IF NOT EXISTS ${centroids}(gen INTEGER, list INTEGER, count INTEGER NOT NULL,\n centroid BLOB NOT NULL, PRIMARY KEY (gen, list)) WITHOUT ROWID`,\n `CREATE TABLE IF NOT EXISTS ${seg}(gen INTEGER, list INTEGER, part INTEGER,\n bytes BLOB NOT NULL, PRIMARY KEY (gen, list, part)) WITHOUT ROWID`,\n `CREATE TABLE IF NOT EXISTS ${f32}(gen INTEGER, list INTEGER, part INTEGER,\n bytes BLOB NOT NULL, PRIMARY KEY (gen, list, part)) WITHOUT ROWID`,\n ];\n }\n\n /** Writes every row of a built segment. Call inside `inTransaction`. */\n async write(tx: Db, gen: number, built: BuiltSegment): Promise<void> {\n const d = this.#dims;\n for (let c = 0; c < built.nlist; c++) {\n await tx.query(\n `INSERT INTO ${this.#t.centroids}(gen, list, count, centroid) VALUES (?, ?, ?, ?)`,\n [gen, c, built.counts[c], float32Bytes(built.centroids.subarray(c * d, (c + 1) * d))],\n );\n await writeParts(tx, this.#t.seg, gen, c, built.lists[c] as Uint8Array);\n await writeParts(tx, this.#t.f32, gen, c, built.floats[c] as Uint8Array);\n }\n }\n\n /** Loads the in-memory part (centroids, list sizes) of a committed segment. */\n async open(ref: SegmentRef, metric: Metric): Promise<SegmentIndex> {\n const d = this.#dims;\n const rows = await this.#db.query<{ list: number; count: number; centroid: Uint8Array }>(\n `SELECT list, count, centroid FROM ${this.#t.centroids} WHERE gen = ? ORDER BY list`,\n [ref.gen],\n );\n if (rows.length !== ref.nlist) {\n throw new Error(\n `vector-index: segment ${ref.gen} has ${rows.length} lists, expected ${ref.nlist}`,\n );\n }\n const centroids = new Float32Array(ref.nlist * d);\n const counts = new Uint32Array(ref.nlist);\n for (const r of rows) {\n const c = Number(r.list);\n centroids.set(toFloat32(r.centroid), c * d);\n counts[c] = Number(r.count);\n }\n return new SegmentIndex(ref.gen, d, metric, centroids, counts);\n }\n\n /** Searchable blocks of lists `cs`: cached ones from memory, the rest in one query. */\n async lists(seg: SegmentIndex, cs: number[]): Promise<ListData[]> {\n const out = new Map<number, ListData>();\n const missing: number[] = [];\n for (const c of cs) {\n const hit = this.#cache.get(`s:${seg.gen}:${c}`) as ListData | undefined;\n if (hit) out.set(c, hit);\n else missing.push(c);\n }\n if (missing.length > 0) {\n const blocks = await readParts(this.#db, this.#t.seg, seg.gen, missing);\n for (const c of missing) {\n const bytes = blocks.get(c) ?? new Uint8Array(0);\n const data = parseList(bytes, seg.counts[c] as number, this.#dims);\n this.#cache.set(`s:${seg.gen}:${c}`, data, bytes.byteLength);\n out.set(c, data);\n }\n }\n return cs.map((c) => out.get(c) as ListData);\n }\n\n /** Every float32 vector of list `c` (count × dims). */\n async floats(seg: SegmentIndex, c: number): Promise<Float32Array> {\n const key = `f:${seg.gen}:${c}`;\n const hit = this.#cache.get(key) as Float32Array | undefined;\n if (hit) return hit;\n const blocks = await readParts(this.#db, this.#t.f32, seg.gen, [c]);\n const floats = toFloat32(blocks.get(c) ?? new Uint8Array(0));\n this.#cache.set(key, floats, floats.byteLength);\n return floats;\n }\n\n /** Deletes the rows of `gens`. Call inside `inTransaction`. */\n async drop(tx: Db, gens: number[]): Promise<void> {\n const json = JSON.stringify(gens);\n for (const table of Object.values(this.#t)) {\n await tx.query(`DELETE FROM ${table} WHERE gen IN (SELECT value FROM json_each(?))`, [json]);\n }\n for (const gen of gens) this.#cache.deletePrefix(`s:${gen}:`, `f:${gen}:`);\n }\n\n /** Deletes rows of every generation not in `live` (left by a crash before a reap). */\n async sweep(tx: Db, live: number[]): Promise<void> {\n const json = JSON.stringify(live);\n for (const table of Object.values(this.#t)) {\n await tx.query(`DELETE FROM ${table} WHERE gen NOT IN (SELECT value FROM json_each(?))`, [\n json,\n ]);\n }\n }\n}\n\nasync function writeParts(tx: Db, table: string, gen: number, list: number, bytes: Uint8Array) {\n for (let part = 0, at = 0; at < bytes.byteLength; part++, at += PART_BYTES) {\n await tx.query(`INSERT INTO ${table}(gen, list, part, bytes) VALUES (?, ?, ?, ?)`, [\n gen,\n list,\n part,\n bytes.slice(at, at + PART_BYTES),\n ]);\n }\n}\n\n/** The joined parts of each list in `lists`, as fresh arrays starting at offset 0. */\nasync function readParts(\n db: Db,\n table: string,\n gen: number,\n lists: number[],\n): Promise<Map<number, Uint8Array>> {\n const rows = await db.query<{ list: number; bytes: Uint8Array }>(\n `SELECT list, bytes FROM ${table}\n WHERE gen = ? AND list IN (SELECT value FROM json_each(?)) ORDER BY list, part`,\n [gen, JSON.stringify(lists)],\n );\n const parts = new Map<number, Uint8Array[]>();\n for (const r of rows) {\n const list = Number(r.list);\n const acc = parts.get(list) ?? [];\n acc.push(r.bytes);\n parts.set(list, acc);\n }\n const out = new Map<number, Uint8Array>();\n for (const [list, ps] of parts) {\n const joined = new Uint8Array(ps.reduce((n, p) => n + p.byteLength, 0));\n let at = 0;\n for (const p of ps) {\n joined.set(p, at);\n at += p.byteLength;\n }\n out.set(list, joined);\n }\n return out;\n}\n\n/** Byte-budgeted LRU; Map order is LRU order (re-inserted on every hit). */\nclass Lru<V> {\n readonly #max: number;\n readonly #map = new Map<string, { value: V; size: number }>();\n #bytes = 0;\n\n constructor(maxBytes: number) {\n this.#max = maxBytes;\n }\n\n get(key: string): V | undefined {\n const e = this.#map.get(key);\n if (!e) return undefined;\n this.#map.delete(key);\n this.#map.set(key, e);\n return e.value;\n }\n\n set(key: string, value: V, size: number) {\n const old = this.#map.get(key);\n if (old) {\n this.#map.delete(key);\n this.#bytes -= old.size;\n }\n this.#map.set(key, { value, size });\n this.#bytes += size;\n for (const [k, e] of this.#map) {\n if (this.#bytes <= this.#max) break;\n this.#map.delete(k);\n this.#bytes -= e.size;\n }\n }\n\n deletePrefix(...prefixes: string[]) {\n for (const [k, e] of this.#map) {\n if (prefixes.some((p) => k.startsWith(p))) {\n this.#map.delete(k);\n this.#bytes -= e.size;\n }\n }\n }\n}\n","import type { Db } from \"@statewalker/db-api\";\nimport { inTransaction } from \"@statewalker/db-sqlite\";\nimport { distance, prepare, TopK } from \"./math.js\";\nimport {\n type BuiltSegment,\n buildSegment,\n type Candidate,\n type ListData,\n SegmentIndex,\n} from \"./segment.js\";\nimport { type SegmentRef, SqlSegmentStore } from \"./sql-segment-store.js\";\nimport {\n float32Bytes,\n type Metric,\n type SearchHit,\n type SearchOptions,\n toFloat32,\n} from \"./types.js\";\n\nexport interface VectorIndexOptions {\n db: Db;\n /** Table name prefix, e.g. \"vec_\". Letters, digits and underscores. */\n prefix: string;\n dims: number;\n metric?: Metric;\n /** IVF lists for a new segment of n vectors. Default ~sqrt(n), flat below 2048. */\n nlist?: (n: number) => number;\n /** Cache budget for list blocks read from SQLite, in bytes. Default 64 MiB. */\n cacheBytes?: number;\n /**\n * upsert() seals the tail into a segment once it holds this many vectors.\n * Default 4096: compacting a larger tail risks the 15 s operation deadline of\n * a Durable Object on celld. Infinity turns it off.\n */\n autoCompactAt?: number;\n}\n\nexport interface VectorItem {\n /** A safe integer: typically the rowid of your documents table. */\n id: number;\n vector: ArrayLike<number>;\n}\n\nexport interface VectorIndexStats {\n tail: number;\n segments: SegmentRef[];\n seq: number;\n}\n\ninterface State {\n seq: number;\n nextGen: number;\n segments: SegmentRef[];\n}\n\ninterface TailEntry {\n seq: number;\n vector: Float32Array;\n}\n\n/** Ids per lookup (bound as one JSON parameter). */\nconst VALIDATE_BATCH = 2000;\n/** Upserted rows per transaction. */\nconst WRITE_BATCH = 200;\n\n/**\n * An LSM-style vector index over a `Db`.\n *\n * - The mutable tail lives in `{p}tail`, mirrored in memory and searched exactly.\n * - `compact()` seals the tail into an immutable IVF segment (1-bit residual\n * filter → int8 re-rank → float32 re-rank on `exact`), written as rows in the\n * same transaction that moves the ids out of the tail.\n * - `{p}ids(id, gen)` says which generation holds the live copy of each id\n * (0 = tail). A segment hit counts only if its generation matches, so deletes\n * and overwrites never touch segment rows. `merge()` rewrites all segments\n * into one, dropping dead rows.\n *\n * One writer per database is assumed (a Durable Object, a worker, one tab);\n * within an instance, writes are serialized.\n */\nexport class VectorIndex {\n readonly dims: number;\n readonly metric: Metric;\n /** `{prefix}ids`: its `id` column holds exactly the live ids. Callers may join it in SQL. */\n readonly idTable: string;\n readonly #db: Db;\n readonly #store: SqlSegmentStore;\n readonly #nlist?: (n: number) => number;\n readonly #autoCompactAt: number;\n readonly #t: { meta: string; ids: string; tail: string };\n\n #state: State = { seq: 0, nextGen: 1, segments: [] };\n #tail = new Map<number, TailEntry>();\n #segments: SegmentIndex[] = [];\n #refs = new Map<SegmentIndex, number>();\n #retired: SegmentIndex[] = [];\n #queue: Promise<unknown> = Promise.resolve();\n #maintaining = false;\n #ready = false;\n /** Bumped before and after each commit that changes the segment list: odd while one runs. */\n #epoch = 0;\n\n constructor(opts: VectorIndexOptions) {\n if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(opts.prefix)) {\n throw new Error(`vector-index: bad prefix ${opts.prefix}`);\n }\n if (!Number.isInteger(opts.dims) || opts.dims < 1) {\n throw new Error(\"vector-index: dims must be a positive integer\");\n }\n this.dims = opts.dims;\n this.metric = opts.metric ?? \"cosine\";\n this.#db = opts.db;\n this.#store = new SqlSegmentStore(opts.db, opts.prefix, opts.dims, opts.cacheBytes);\n this.#nlist = opts.nlist;\n this.#autoCompactAt = opts.autoCompactAt ?? 4096;\n const p = opts.prefix;\n this.#t = { meta: `${p}meta`, ids: `${p}ids`, tail: `${p}tail` };\n this.idTable = this.#t.ids;\n }\n\n // ------------------------------------------------------------ lifecycle\n\n /** Creates tables if missing, loads state and sweeps rows of crashed maintenance. */\n async init(): Promise<void> {\n const { meta, ids, tail } = this.#t;\n for (const s of [\n `CREATE TABLE IF NOT EXISTS ${meta}(k TEXT PRIMARY KEY, v TEXT NOT NULL)`,\n `CREATE TABLE IF NOT EXISTS ${ids}(id INTEGER PRIMARY KEY, gen INTEGER NOT NULL)`,\n `CREATE INDEX IF NOT EXISTS ${ids}_gen ON ${ids}(gen)`,\n `CREATE TABLE IF NOT EXISTS ${tail}(id INTEGER PRIMARY KEY, seq INTEGER NOT NULL, vec BLOB NOT NULL)`,\n `CREATE INDEX IF NOT EXISTS ${tail}_seq ON ${tail}(seq)`,\n ...this.#store.schema(),\n ]) {\n await this.#db.exec(s);\n }\n\n const config = JSON.stringify({ dims: this.dims, metric: this.metric });\n await this.#db.query(\n `INSERT INTO ${meta}(k, v) VALUES ('config', ?) ON CONFLICT(k) DO NOTHING`,\n [config],\n );\n const [stored] = await this.#db.query<{ v: string }>(\n `SELECT v FROM ${meta} WHERE k = 'config'`,\n );\n if (stored?.v !== config) {\n throw new Error(`vector-index: tables were created with ${stored?.v}, not ${config}`);\n }\n await this.refresh();\n const live = this.#state.segments.map((s) => s.gen);\n await inTransaction(this.#db, (tx) => this.#store.sweep(tx, live));\n this.#ready = true;\n }\n\n /** Reloads state, tail and segments from storage (e.g. after another writer changed them). */\n async refresh(): Promise<void> {\n const [row] = await this.#db.query<{ v: string }>(\n `SELECT v FROM ${this.#t.meta} WHERE k = 'state'`,\n );\n const state: State = row ? JSON.parse(row.v) : { seq: 0, nextGen: 1, segments: [] };\n\n const tail = new Map<number, TailEntry>();\n const rows = await this.#db.query<{ id: number; seq: number; vec: Uint8Array }>(\n `SELECT id, seq, vec FROM ${this.#t.tail}`,\n );\n for (const r of rows) tail.set(Number(r.id), { seq: Number(r.seq), vector: toFloat32(r.vec) });\n\n const known = new Map(this.#segments.map((s) => [s.gen, s]));\n const segments = await Promise.all(\n state.segments.map((s) => known.get(s.gen) ?? this.#store.open(s, this.metric)),\n );\n this.#state = state;\n this.#tail = tail;\n this.#segments = segments;\n }\n\n get stats(): VectorIndexStats {\n return {\n tail: this.#tail.size,\n segments: this.#state.segments.map((s) => ({ ...s })),\n seq: this.#state.seq,\n };\n }\n\n // ------------------------------------------------------------ writes\n\n async upsert(items: VectorItem[]): Promise<void> {\n this.#assertReady();\n const prepared = items.map((it) => {\n assertId(it.id);\n return { id: it.id, vector: prepare(it.vector, this.dims, this.metric) };\n });\n const { tail, ids } = this.#t;\n for (let i = 0; i < prepared.length; i += WRITE_BATCH) {\n const batch = prepared.slice(i, i + WRITE_BATCH);\n await this.#exclusive(async () => {\n const seq = this.#state.seq + 1;\n const next = { ...this.#state, seq };\n await inTransaction(this.#db, async (tx) => {\n for (const { id, vector } of batch) {\n await tx.query(\n `INSERT INTO ${tail}(id, seq, vec) VALUES (?, ?, ?)\n ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, vec = excluded.vec`,\n [id, seq, float32Bytes(vector)],\n );\n await tx.query(\n `INSERT INTO ${ids}(id, gen) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET gen = 0`,\n [id],\n );\n }\n await this.#saveState(tx, next);\n });\n this.#state = next;\n for (const { id, vector } of batch) this.#tail.set(id, { seq, vector });\n });\n if (this.#tail.size >= this.#autoCompactAt && !this.#maintaining) await this.compact();\n }\n }\n\n async delete(ids: number[]): Promise<void> {\n this.#assertReady();\n ids.forEach(assertId);\n await this.#exclusive(async () => {\n for (let i = 0; i < ids.length; i += VALIDATE_BATCH) {\n const chunk = ids.slice(i, i + VALIDATE_BATCH);\n const json = JSON.stringify(chunk);\n await inTransaction(this.#db, async (tx) => {\n await tx.query(\n `DELETE FROM ${this.#t.tail} WHERE id IN (SELECT value FROM json_each(?))`,\n [json],\n );\n await tx.query(\n `DELETE FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`,\n [json],\n );\n });\n for (const id of chunk) this.#tail.delete(id);\n }\n });\n }\n\n // ------------------------------------------------------------ search\n\n async search(query: ArrayLike<number>, opts: SearchOptions = {}): Promise<SearchHit[]> {\n this.#assertReady();\n const k = opts.k ?? 10;\n const nprobe = opts.nprobe ?? 8;\n const perList = k * (opts.oversample ?? 8);\n const q = prepare(query, this.dims, this.metric);\n const d = this.dims;\n const filter = opts.filter;\n\n return this.#read(async (segments) => {\n // Tail: exact, from memory.\n const tailTop = new TopK<number>(k);\n for (const [id, { vector }] of this.#tail) {\n if (filter && !filter(id)) continue;\n tailTop.push(distance(this.metric, q, 0, vector, 0, d), id);\n }\n\n // Stage 1: probe lists, keep the best rows of each by estimated residual distance.\n type Hit = Candidate & { seg: SegmentIndex; data: ListData };\n const hits: Hit[] = [];\n // Tail ids are never live in a segment; skip them early.\n const keep = (id: number) => !this.#tail.has(id) && (!filter || filter(id));\n await Promise.all(\n segments.map(async (seg) => {\n const lists = seg.probe(q, nprobe);\n const datas = await this.#store.lists(seg, lists);\n opts.signal?.throwIfAborted();\n lists.forEach((c, i) => {\n const data = datas[i] as ListData;\n for (const cand of seg.filterList(q, c, data, perList, keep))\n hits.push({ ...cand, seg, data });\n });\n }),\n );\n\n // Liveness: a segment row counts only if its generation holds the live copy.\n const live = await this.#generations(hits.map((h) => h.id));\n\n // Stage 2: int8 re-rank.\n const rerank = opts.exact ? k * 4 : k;\n const segTop = new TopK<Hit>(rerank);\n for (const h of hits) {\n if (live.get(h.id) !== h.seg.gen) continue;\n segTop.push(h.seg.scoreInt8(q, h.data, h.row), h);\n }\n let segResults = segTop\n .sorted()\n .map((e) => ({ id: e.item.id, distance: e.score, hit: e.item }));\n\n // Stage 3: exact re-rank from stored floats.\n if (opts.exact) {\n segResults = await Promise.all(\n segResults.map(async (r) => {\n const all = await this.#store.floats(r.hit.seg, r.hit.list);\n const v = all.subarray(r.hit.row * d, (r.hit.row + 1) * d);\n return { ...r, distance: distance(this.metric, q, 0, v, 0, d) };\n }),\n );\n }\n\n const final = new TopK<number>(k);\n for (const r of segResults) final.push(r.distance, r.id);\n for (const e of tailTop.sorted()) final.push(e.score, e.item);\n return final.sorted().map((e) => ({ id: e.item, distance: e.score }));\n });\n }\n\n /** The stored vectors of the live `ids` (normalised for cosine); unknown or deleted ids are absent. */\n async getVectors(ids: number[]): Promise<Map<number, Float32Array>> {\n this.#assertReady();\n ids.forEach(assertId);\n return this.#read(async (segments) => {\n const out = new Map<number, Float32Array>();\n const gens = await this.#generations(ids);\n const wanted = new Map<number, Set<number>>(); // gen -> ids\n for (const [id, gen] of gens) {\n if (gen === 0) {\n const entry = this.#tail.get(id);\n if (entry) out.set(id, entry.vector.slice());\n continue;\n }\n const set = wanted.get(gen) ?? new Set<number>();\n set.add(id);\n wanted.set(gen, set);\n }\n for (const seg of segments) {\n const set = wanted.get(seg.gen);\n if (!set) continue;\n for (let c = 0; c < seg.nlist && set.size > 0; c++) {\n if (seg.counts[c] === 0) continue;\n const [data] = (await this.#store.lists(seg, [c])) as [ListData];\n let floats: Float32Array | null = null;\n for (let r = 0; r < data.ids.length; r++) {\n const id = data.ids[r] as number;\n if (!set.has(id)) continue;\n floats ??= await this.#store.floats(seg, c);\n out.set(id, floats.slice(r * this.dims, (r + 1) * this.dims));\n set.delete(id);\n }\n }\n }\n return out;\n });\n }\n\n // ------------------------------------------------------------ maintenance\n\n /** Seals the current tail into a new segment. Returns the number of vectors sealed. */\n async compact(): Promise<number> {\n this.#assertReady();\n return this.#maintenance(async () => {\n const snapshot = await this.#exclusive(async () => ({\n maxSeq: this.#state.seq,\n entries: [...this.#tail.entries()],\n }));\n if (snapshot.entries.length === 0) return 0;\n\n const n = snapshot.entries.length;\n const ids = new Float64Array(n);\n const vectors = new Float32Array(n * this.dims);\n snapshot.entries.forEach(([id, e], i) => {\n ids[i] = id;\n vectors.set(e.vector, i * this.dims);\n });\n const built = this.#build(ids, vectors);\n\n await this.#exclusive(() =>\n this.#swapSegments(async () => {\n const ref = this.#refOf(built, n);\n const next: State = {\n ...this.#state,\n nextGen: ref.gen + 1,\n segments: [...this.#state.segments, ref],\n };\n const { ids: idsT, tail } = this.#t;\n await inTransaction(this.#db, async (tx) => {\n await this.#store.write(tx, ref.gen, built);\n await tx.query(\n `UPDATE ${idsT} SET gen = ? WHERE gen = 0\n AND id IN (SELECT id FROM ${tail} WHERE seq <= ?)`,\n [ref.gen, snapshot.maxSeq],\n );\n await tx.query(`DELETE FROM ${tail} WHERE seq <= ?`, [snapshot.maxSeq]);\n await this.#saveState(tx, next);\n });\n this.#state = next;\n for (const [id, e] of this.#tail) if (e.seq <= snapshot.maxSeq) this.#tail.delete(id);\n this.#segments = [...this.#segments, this.#segmentOf(ref, built)];\n }),\n );\n return n;\n });\n }\n\n /**\n * Rewrites all segments into one (retraining the IVF lists) and drops dead\n * rows. Holds all live segment vectors in memory while it runs.\n */\n async merge(): Promise<number> {\n this.#assertReady();\n return this.#maintenance(async () => {\n const old = this.#segments;\n if (old.length === 0) return 0;\n const oldGens = old.map((s) => s.gen);\n\n const idList: number[] = [];\n const chunks: Float32Array[] = [];\n for (const seg of old) {\n for (let c = 0; c < seg.nlist; c++) {\n if (seg.counts[c] === 0) continue;\n const [data] = (await this.#store.lists(seg, [c])) as [ListData];\n const floats = await this.#store.floats(seg, c);\n const live = await this.#generations(Array.from(data.ids));\n for (let r = 0; r < data.ids.length; r++) {\n const id = data.ids[r];\n if (live.get(id) !== seg.gen) continue;\n idList.push(id);\n chunks.push(floats.subarray(r * this.dims, (r + 1) * this.dims));\n }\n }\n }\n\n const n = idList.length;\n const vectors = new Float32Array(n * this.dims);\n chunks.forEach((v, i) => {\n vectors.set(v, i * this.dims);\n });\n const built = n > 0 ? this.#build(Float64Array.from(idList), vectors) : null;\n\n await this.#exclusive(() =>\n this.#swapSegments(async () => {\n const ref = built ? this.#refOf(built, n) : null;\n const kept = this.#state.segments.filter((s) => !oldGens.includes(s.gen));\n const next: State = {\n ...this.#state,\n nextGen: ref ? ref.gen + 1 : this.#state.nextGen,\n segments: ref ? [...kept, ref] : kept,\n };\n await inTransaction(this.#db, async (tx) => {\n if (ref && built) {\n await this.#store.write(tx, ref.gen, built);\n await tx.query(\n `UPDATE ${this.#t.ids} SET gen = ? WHERE gen IN (SELECT value FROM json_each(?))`,\n [ref.gen, JSON.stringify(oldGens)],\n );\n }\n await this.#saveState(tx, next);\n });\n this.#state = next;\n this.#segments = [\n ...this.#segments.filter((s) => !oldGens.includes(s.gen)),\n ...(ref && built ? [this.#segmentOf(ref, built)] : []),\n ];\n this.#retired.push(...old);\n }),\n );\n await this.#reap();\n return n;\n });\n }\n\n /** compact() then merge(). */\n async optimize(): Promise<void> {\n await this.compact();\n await this.merge();\n }\n\n // ------------------------------------------------------------ private\n\n #build(ids: Float64Array, vectors: Float32Array): BuiltSegment {\n return buildSegment({\n dims: this.dims,\n metric: this.metric,\n ids,\n vectors,\n nlist: this.#nlist?.(ids.length),\n });\n }\n\n #refOf(built: BuiltSegment, count: number): SegmentRef {\n return { gen: this.#state.nextGen, count, nlist: built.nlist };\n }\n\n #segmentOf(ref: SegmentRef, built: BuiltSegment): SegmentIndex {\n return new SegmentIndex(ref.gen, this.dims, this.metric, built.centroids, built.counts);\n }\n\n async #generations(ids: number[]): Promise<Map<number, number>> {\n const out = new Map<number, number>();\n const unique = [...new Set(ids)];\n for (let i = 0; i < unique.length; i += VALIDATE_BATCH) {\n const rows = await this.#db.query<{ id: number; gen: number }>(\n `SELECT id, gen FROM ${this.#t.ids} WHERE id IN (SELECT value FROM json_each(?))`,\n [JSON.stringify(unique.slice(i, i + VALIDATE_BATCH))],\n );\n for (const r of rows) out.set(Number(r.id), Number(r.gen));\n }\n return out;\n }\n\n async #saveState(tx: Db, state: State): Promise<void> {\n await tx.query(\n `INSERT INTO ${this.#t.meta}(k, v) VALUES ('state', ?)\n ON CONFLICT(k) DO UPDATE SET v = excluded.v`,\n [JSON.stringify(state)],\n );\n }\n\n /** Deletes the rows of retired segments once no search is reading them. */\n async #reap(): Promise<void> {\n const ready = this.#retired.filter((s) => (this.#refs.get(s) ?? 0) === 0);\n if (ready.length === 0) return;\n this.#retired = this.#retired.filter((s) => !ready.includes(s));\n for (const s of ready) this.#refs.delete(s);\n const gens = ready.map((s) => s.gen);\n await inTransaction(this.#db, (tx) => this.#store.drop(tx, gens));\n }\n\n /**\n * Runs `read` over a snapshot of the segments, held against reaping. A read\n * whose liveness lookup straddles a compact or merge commit may not match its\n * snapshot, so it runs again on the new one; the third attempt stands.\n */\n async #read<T>(read: (segments: SegmentIndex[]) => Promise<T>): Promise<T> {\n for (let attempt = 1; ; attempt++) {\n const epoch = this.#epoch;\n const segments = this.#segments;\n for (const s of segments) this.#refs.set(s, (this.#refs.get(s) ?? 0) + 1);\n try {\n const result = await read(segments);\n if (attempt === 3 || (epoch % 2 === 0 && epoch === this.#epoch)) return result;\n } finally {\n for (const s of segments) this.#refs.set(s, (this.#refs.get(s) ?? 1) - 1);\n void this.#reap().catch(() => {}); // a failed reap leaves rows that init() sweeps\n }\n }\n }\n\n /** Commits a segment-list change; readers that overlap it retry (see `#read`). */\n async #swapSegments(fn: () => Promise<void>): Promise<void> {\n this.#epoch++;\n try {\n await fn();\n } finally {\n this.#epoch++;\n }\n }\n\n #exclusive<T>(fn: () => Promise<T>): Promise<T> {\n const run = this.#queue.then(fn, fn);\n this.#queue = run.catch(() => {});\n return run;\n }\n\n async #maintenance<T>(fn: () => Promise<T>): Promise<T> {\n if (this.#maintaining) throw new Error(\"vector-index: compact or merge already running\");\n this.#maintaining = true;\n try {\n return await fn();\n } finally {\n this.#maintaining = false;\n }\n }\n\n #assertReady() {\n if (!this.#ready) throw new Error(\"vector-index: call init() first\");\n }\n}\n\nfunction assertId(id: number) {\n if (!Number.isSafeInteger(id))\n throw new Error(`vector-index: id must be a safe integer, got ${id}`);\n}\n\n/** Drops every table of the index under `prefix`, in one transaction. Needs no dims or metric. */\nexport async function dropVectorIndex(db: Db, prefix: string): Promise<void> {\n if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(prefix))\n throw new Error(`vector-index: bad prefix ${prefix}`);\n await inTransaction(db, async (tx) => {\n for (const t of [\"meta\", \"ids\", \"tail\", \"centroids\", \"seg\", \"f32\"]) {\n await tx.exec(`DROP TABLE IF EXISTS ${prefix}${t}`);\n }\n });\n}\n"],"mappings":";;AAEA,SAAgB,UAAU,GAA+B;CACvD,IAAI,IAAI;CACR,KAAK,IAAI,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK,KAAK,EAAE,KAAK,EAAE;CACjD,IAAI,KAAK,KAAK,CAAC;CACf,MAAM,MAAM,IAAI,aAAa,EAAE,MAAM;CACrC,IAAI,MAAM,GAAG,OAAO;CACpB,KAAK,IAAI,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK,IAAI,KAAK,EAAE,KAAK;CACnD,OAAO;AACT;;AAGA,SAAgB,QAAQ,GAAsB,MAAc,QAA8B;CACxF,IAAI,EAAE,WAAW,MAAM,MAAM,IAAI,MAAM,0BAA0B,KAAK,aAAa,EAAE,QAAQ;CAC7F,MAAM,IAAI,aAAa,eAAe,IAAI,aAAa,KAAK,CAAC;CAC7D,KAAK,IAAI,IAAI,GAAG,IAAI,EAAE,QAAQ,KAC5B,IAAI,CAAC,OAAO,SAAS,EAAE,EAAE,GAAG,MAAM,IAAI,MAAM,kDAAkD;CAEhG,OAAO,WAAW,WAAW,UAAU,CAAC,IAAI,EAAE,MAAM;AACtD;;AAGA,SAAgB,SACd,QACA,GACA,IACA,GACA,IACA,GACQ;CACR,IAAI,WAAW,MAAM;EACnB,IAAI,IAAI;EACR,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;GAC1B,MAAM,IAAI,EAAE,KAAK,KAAK,EAAE,KAAK;GAC7B,KAAK,IAAI;EACX;EACA,OAAO;CACT;CACA,IAAI,IAAI;CACR,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,KAAK,EAAE,KAAK,KAAK,EAAE,KAAK;CACpD,OAAO,WAAW,WAAW,IAAI,IAAI,CAAC;AACxC;;AAKA,SAAgB,aAAa,GAAiB,IAAY,GAAW,KAAgB,IAAY;CAC/F,IAAI,MAAM;CACV,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,MAAM,KAAK,IAAI,KAAK,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;CACnE,MAAM,MAAM,QAAQ,IAAI,IAAI,MAAM;CAClC,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,IAAI,KAAK,KAAK,KAAK,MAAM,EAAE,KAAK,KAAK,GAAG;CACpE,OAAO;AACT;;AAGA,SAAgB,aACd,QACA,GACA,OACA,IACA,OACA,GACQ;CACR,MAAM,IAAI,QAAQ;CAClB,IAAI,WAAW,MAAM;EACnB,IAAI,IAAI;EACR,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;GAC1B,MAAM,IAAI,EAAE,KAAK,MAAM,KAAK,KAAK;GACjC,KAAK,IAAI;EACX;EACA,OAAO;CACT;CACA,IAAI,IAAI;CACR,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,KAAK,EAAE,KAAK,MAAM,KAAK;CACnD,KAAK;CACL,OAAO,WAAW,WAAW,IAAI,IAAI,CAAC;AACxC;AAWA,MAAa,YAAY,MAAe,IAAI,OAAQ;;AAGpD,SAAgB,aACd,GACA,IACA,GACA,IACA,GACA,KACA,IACA;CACA,MAAM,IAAI,SAAS,CAAC;CACpB,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;CACtB,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KACrB,IAAI,EAAE,KAAK,KAAK,EAAE,KAAK,IAAI,IAAI,MAAM,MAAM,OAAO,MAAM,IAAI;AAEhE;AAEA,SAAgB,QAAQ,GAAgB,GAAgB,IAAY,GAAmB;CACrF,IAAI,IAAI;CACR,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,IAAI,IAAI,EAAE,KAAK,EAAE,KAAK;EACtB,KAAM,MAAM,IAAK;EACjB,KAAK,IAAI,cAAgB,MAAM,IAAK;EACpC,KAAK,KAAK,KAAM,KAAK,MAAM,KAAM,WAAY,QAAU,MAAM;CAC/D;CACA,OAAO;AACT;;AAKA,IAAa,OAAb,MAAqB;CACnB;CACA,UAA6B,CAAC;CAC9B,SAAuB,CAAC;CAExB,YAAY,GAAW;EACrB,KAAK,KAAK,KAAK,IAAI,GAAG,CAAC;CACzB;CAEA,IAAI,OAAO;EACT,OAAO,KAAK,QAAQ;CACtB;;CAGA,IAAI,QAAQ;EACV,OAAO,KAAK,QAAQ,SAAS,KAAK,KAAK,OAAO,oBAAoB,KAAK,QAAQ;CACjF;CAEA,KAAK,OAAe,MAAS;EAC3B,IAAI,KAAK,OAAO,GAAG;EACnB,MAAM,IAAI,KAAK;EACf,MAAM,KAAK,KAAK;EAChB,IAAI,EAAE,SAAS,KAAK,IAAI;GACtB,EAAE,KAAK,KAAK;GACZ,GAAG,KAAK,IAAI;GACZ,IAAI,IAAI,EAAE,SAAS;GACnB,OAAO,IAAI,GAAG;IACZ,MAAM,IAAK,IAAI,KAAM;IACrB,IAAI,EAAE,MAAM,EAAE,IAAI;IAClB,KAAK,MAAM,GAAG,CAAC;IACf,IAAI;GACN;GACA;EACF;EACA,IAAI,SAAS,EAAE,IAAI;EACnB,EAAE,KAAK;EACP,GAAG,KAAK;EACR,IAAI,IAAI;EACR,SAAS;GACP,MAAM,IAAI,IAAI,IAAI;GAClB,MAAM,IAAI,IAAI;GACd,IAAI,IAAI;GACR,IAAI,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,IAAI;GACrC,IAAI,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,IAAI;GACrC,IAAI,MAAM,GAAG;GACb,KAAK,MAAM,GAAG,CAAC;GACf,IAAI;EACN;CACF;;CAGA,SAAuC;EACrC,OAAO,KAAK,QACT,KAAK,OAAO,OAAO;GAAE;GAAO,MAAM,KAAK,OAAO;EAAG,EAAE,CAAC,CACpD,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;CACrC;CAEA,MAAM,GAAW,GAAW;EAC1B,MAAM,IAAI,KAAK;EACf,MAAM,KAAK,KAAK;EAChB,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE;EAC1B,CAAC,GAAG,IAAI,GAAG,MAAM,CAAC,GAAG,IAAI,GAAG,EAAE;CAChC;AACF;;;;AC7KA,SAAgB,WAAW,MAAc;CACvC,IAAI,IAAI,SAAS;CACjB,aAAa;EACX,IAAK,IAAI,eAAgB;EACzB,IAAI,IAAI;EACR,IAAI,KAAK,KAAK,IAAK,MAAM,IAAK,IAAI,CAAC;EACnC,KAAK,IAAI,KAAK,KAAK,IAAK,MAAM,GAAI,IAAI,EAAE;EACxC,SAAS,IAAK,MAAM,QAAS,KAAK;CACpC;AACF;;AAGA,SAAgB,QACd,QACA,MACA,QACA,WACA,GACA,GACQ;CACR,IAAI,OAAO;CACX,IAAI,QAAQ,OAAO;CAEnB,MAAM,IAAY,WAAW,OAAO,OAAO;CAC3C,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,MAAM,OAAO,SAAS,GAAG,MAAM,QAAQ,WAAW,IAAI,GAAG,CAAC;EAC1D,IAAI,OAAO,OAAO;GAChB,QAAQ;GACR,OAAO;EACT;CACF;CACA,OAAO;AACT;;;;;AAMA,SAAgB,OACd,MACA,GACA,GACA,GACA,QACA,OAAsB,CAAC,GACT;CACd,MAAM,OAAO,WAAW,KAAK,QAAQ,EAAE;CACvC,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC;CAC9B,MAAM,aAAa,KAAK,IAAI,GAAG,KAAK,cAAc,KAAK,IAAI,KAAK,GAAG,IAAM,CAAC;CAG1E,MAAM,MAAM,IAAI,YAAY,CAAC;CAC7B,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,IAAI,KAAK;CACrC,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAAK;EACnC,MAAM,IAAI,IAAI,KAAK,MAAM,KAAK,KAAK,IAAI,EAAE;EACzC,CAAC,IAAI,IAAI,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE;CACpC;CACA,MAAM,SAAS,IAAI,aAAa,aAAa,CAAC;CAC9C,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAAK,OAAO,IAAI,KAAK,SAAS,IAAI,KAAK,GAAG,IAAI,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;CAEhG,MAAM,YAAY,WAAW;CAC7B,MAAM,YAAY,IAAI,aAAa,IAAI,CAAC;CAGxC,UAAU,IAAI,OAAO,SAAS,GAAG,CAAC,GAAG,CAAC;CACtC,MAAM,OAAO,IAAI,aAAa,UAAU,CAAC,CAAC,KAAK,OAAO,iBAAiB;CACvE,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,IAAI,QAAQ;EACZ,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAAK;GACnC,MAAM,KAAK,SAAS,MAAM,QAAQ,IAAI,GAAG,YAAY,IAAI,KAAK,GAAG,CAAC;GAClE,IAAI,KAAK,KAAK,IAAI,KAAK,KAAK;GAC5B,SAAS,KAAK;EAChB;EACA,IAAI,IAAI,KAAK,IAAI;EACjB,IAAI,OAAO,aAAa;EACxB,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAAK;GACnC,KAAK,KAAK;GACV,IAAI,KAAK,GAAG;IACV,OAAO;IACP;GACF;EACF;EACA,UAAU,IAAI,OAAO,SAAS,OAAO,GAAG,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC;CAC9D;CAEA,MAAM,SAAS,IAAI,YAAY,UAAU;CACzC,MAAM,SAAS,IAAI,YAAY,CAAC;CAChC,MAAM,aAAa,KAAK,cAAc;CACtC,KAAK,IAAI,KAAK,GAAG,KAAK,YAAY,MAAM;EACtC,IAAI,QAAQ;EACZ,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAAK;GACnC,MAAM,IAAI,QAAQ,QAAQ,QAAQ,IAAI,GAAG,WAAW,GAAG,CAAC;GACxD,IAAI,MAAM,OAAO,MAAM,OAAO,GAAG;GACjC,OAAO,KAAK;EACd;EACA,MAAM,OAAO,IAAI,aAAa,IAAI,CAAC;EACnC,OAAO,KAAK,CAAC;EACb,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAAK;GACnC,MAAM,IAAI,OAAO;GACjB,OAAO,EAAE;GACT,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,KAAK,IAAI,IAAI,MAAM,OAAO,IAAI,IAAI;EAChE;EACA,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;GAC1B,IAAI,OAAO,OAAO,GAAG;IAEnB,MAAM,IAAI,KAAK,MAAM,KAAK,IAAI,UAAU;IACxC,UAAU,IAAI,OAAO,SAAS,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC;IACtD;GACF;GACA,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,UAAU,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,OAAO;EAC9E;EACA,IAAI,WACF,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KACrB,UAAU,IAAI,UAAU,UAAU,SAAS,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;EAGxE,IAAI,KAAK,KAAK,UAAU,GAAG;CAC7B;CACA,OAAO;AACT;;;;;;;;;;;;;;;;;ACxGA,MAAM,UAAU,MAAe,IAAI,IAAK;AAcxC,SAAgB,WAAW,OAAe,MAA0B;CAClE,MAAM,IAAI,SAAS,IAAI;CACvB,MAAM,QAAQ;CACd,MAAM,SAAS,OAAO,QAAQ,IAAI,KAAK;CACvC,MAAM,UAAU,OAAO,SAAS,IAAI,IAAI,KAAK;CAC7C,MAAM,SAAS,OAAO,UAAU,IAAI,KAAK;CACzC,MAAM,WAAW,OAAO,SAAS,IAAI,KAAK;CAC1C,MAAM,UAAU,OAAO,WAAW,IAAI,KAAK;CAE3C,OAAO;EAAE;EAAO;EAAO;EAAQ;EAAS;EAAQ;EAAU;EAAS,QADpD,OAAO,UAAU,OAAO,KACiC;CAAE;AAC5E;AA4BA,SAAgB,aAAa,GAAW;CACtC,IAAI,IAAA,MAAe,OAAO;CAC1B,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,MAAM,KAAK,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC;AAC7D;AAEA,SAAgB,aAAa,OAAiC;CAC5D,MAAM,EAAE,MAAM,GAAG,QAAQ,KAAK,YAAY;CAC1C,MAAM,IAAI,IAAI;CACd,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,IAAI,MAAM,SAAS,aAAa,CAAC,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC;CAElF,IAAI;CACJ,IAAI,UAAU,GAAG;EACf,YAAY,IAAI,aAAa,CAAC;EAC9B,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,UAAU,MAAM,QAAQ,IAAI,IAAI;EACxF,IAAI,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,UAAU,MAAM;CACzD,OACE,YAAY,OAAO,SAAS,GAAG,GAAG,OAAO,QAAQ,EAAE,MAAM,MAAM,KAAK,CAAC;CAGvE,MAAM,SAAS,IAAI,YAAY,CAAC;CAChC,MAAM,SAAS,IAAI,YAAY,KAAK;CACpC,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,MAAM,IAAI,UAAU,IAAI,IAAI,QAAQ,QAAQ,SAAS,IAAI,GAAG,WAAW,OAAO,CAAC;EAC/E,OAAO,KAAK;EACZ,OAAO,EAAE;CACX;CAEA,MAAM,IAAI,SAAS,CAAC;CACpB,MAAM,UAAU,MAAM,KAAK,SAAS,UAAU,WAAW,OAAO,CAAC,CAAC;CAClE,MAAM,QAAQ,QAAQ,KAAK,MAAM,IAAI,WAAW,EAAE,MAAM,CAAC;CACzD,MAAM,SAAS,MAAM,KAAK,SAAS,UAAU,IAAI,WAAW,IAAI,QAAQ,CAAC,CAAC;CAC1E,MAAM,QAAQ,QAAQ,KAAK,GAAG,MAAM;EAClC,MAAM,MAAM,MAAM,EAAE,CAAC;EACrB,OAAO;GACL,KAAK,IAAI,aAAa,KAAK,EAAE,OAAO,EAAE,KAAK;GAC3C,MAAM,IAAI,YAAY,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC;GAChD,OAAO,IAAI,aAAa,KAAK,EAAE,SAAS,EAAE,KAAK;GAC/C,MAAM,IAAI,aAAa,KAAK,EAAE,QAAQ,EAAE,KAAK;GAC7C,QAAQ,IAAI,aAAa,KAAK,EAAE,UAAU,EAAE,KAAK;GACjD,OAAO,IAAI,UAAU,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC;GAChD,QAAQ,IAAI,aAAa,OAAO,EAAE,CAAC,QAAQ,GAAG,EAAE,QAAQ,CAAC;EAC3D;CACF,CAAC;CAED,MAAM,OAAO,IAAI,YAAY,KAAK;CAClC,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,MAAM,IAAI,OAAO;EACjB,MAAM,IAAI,KAAK,EAAE;EACjB,MAAM,IAAI,MAAM;EAChB,EAAE,IAAI,KAAK,IAAI;EACf,aAAa,SAAS,IAAI,GAAG,WAAW,IAAI,GAAG,GAAG,EAAE,MAAM,IAAI,CAAC;EAC/D,MAAM,CAAC,IAAI,MAAM,cAAc,SAAS,IAAI,GAAG,WAAW,IAAI,GAAG,CAAC;EAClE,EAAE,MAAM,KAAK;EACb,EAAE,KAAK,KAAK;EACZ,EAAE,OAAO,KAAK,aAAa,SAAS,IAAI,GAAG,GAAG,EAAE,OAAO,IAAI,CAAC;EAC5D,EAAE,OAAO,IAAI,QAAQ,SAAS,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC;CACxD;CACA,OAAO;EAAE;EAAO;EAAQ;EAAW;EAAO;CAAO;AACnD;;AAGA,SAAS,cAAc,GAAiB,IAAY,GAAiB,IAAY,GAAW;CAC1F,IAAI,KAAK;CACT,IAAI,KAAK;CACT,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,MAAM,IAAI,EAAE,KAAK,KAAK,EAAE,KAAK;EAC7B,MAAM,IAAI;EACV,MAAM,EAAE,KAAK,KAAK;CACpB;CACA,OAAO,CAAC,KAAK,KAAK,EAAE,GAAG,EAAE;AAC3B;;AAcA,SAAgB,UAAU,OAAmB,OAAe,MAAwB;CAClF,MAAM,IAAI,WAAW,OAAO,IAAI;CAChC,IAAI,MAAM,eAAe,EAAE,QACzB,MAAM,IAAI,MAAM,gCAAgC,MAAM,WAAW,mBAAmB,EAAE,QAAQ;CAEhG,MAAM,MAAM,MAAM,eAAe,IAAI,MAAM,SAAS,MAAM,MAAM,CAAC,CAAC;CAClE,MAAM,IAAI,SAAS,IAAI;CACvB,OAAO;EACL,KAAK,IAAI,aAAa,KAAK,EAAE,OAAO,KAAK;EACzC,MAAM,IAAI,YAAY,KAAK,EAAE,QAAQ,QAAQ,CAAC;EAC9C,OAAO,IAAI,aAAa,KAAK,EAAE,SAAS,KAAK;EAC7C,MAAM,IAAI,aAAa,KAAK,EAAE,QAAQ,KAAK;EAC3C,QAAQ,IAAI,aAAa,KAAK,EAAE,UAAU,KAAK;EAC/C,OAAO,IAAI,UAAU,KAAK,EAAE,SAAS,QAAQ,IAAI;CACnD;AACF;;AASA,IAAa,eAAb,MAA0B;CAEb;CACA;CACA;CACA;CACA;CALX,YACE,KACA,MACA,QACA,WACA,QACA;EALS,KAAA,MAAA;EACA,KAAA,OAAA;EACA,KAAA,SAAA;EACA,KAAA,YAAA;EACA,KAAA,SAAA;CACR;CAEH,IAAI,QAAQ;EACV,OAAO,KAAK,OAAO;CACrB;;CAGA,MAAM,OAAqB,QAA0B;EACnD,MAAM,IAAY,KAAK,WAAW,OAAO,OAAO;EAChD,MAAM,MAAM,IAAI,KAAa,KAAK,IAAI,QAAQ,KAAK,KAAK,CAAC;EACzD,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,OAAO,KAAK;GACnC,IAAI,KAAK,OAAO,OAAO,GAAG;GAC1B,IAAI,KAAK,SAAS,GAAG,OAAO,GAAG,KAAK,WAAW,IAAI,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC;EAC7E;EACA,OAAO,IAAI,OAAO,CAAC,CAAC,KAAK,MAAM,EAAE,IAAI;CACvC;;;;;CAMA,WACE,OACA,GACA,MACA,OACA,MACa;EACb,MAAM,IAAI,KAAK;EACf,MAAM,IAAI,SAAS,CAAC;EACpB,MAAM,KAAK,IAAI;EACf,MAAM,KAAK,IAAI,YAAY,CAAC;EAC5B,aAAa,OAAO,GAAG,KAAK,WAAW,IAAI,GAAG,IAAI,CAAC;EACnD,MAAM,CAAC,IAAI,OAAO,cAAc,OAAO,GAAG,KAAK,WAAW,IAAI,CAAC;EAC/D,IAAI,KAAK;EACT,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,MAAM,KAAK,UAAU,KAAK,KAAK,KAAK,UAAU,KAAK;EAC/E,MAAM,KAAK,KAAK;EAChB,MAAM,QAAQ,KAAK,KAAK;EACxB,MAAM,KAAK,KAAK,WAAW;EAC3B,MAAM,MAAM,IAAI,KAAgB,KAAK;EACrC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,IAAI,QAAQ,KAAK;GACxC,MAAM,KAAK,KAAK,IAAI;GACpB,IAAI,QAAQ,CAAC,KAAK,EAAE,GAAG;GACvB,MAAM,KAAK,KAAK,KAAK,MAAM,KAAK,KAAK,IAAI,QAAQ,QAAQ,IAAI,KAAK,MAAM,IAAI,GAAG,CAAC,CAAC;GAEjF,MAAM,MAAM,KAAK,KAAK,KAAK,KAAK,MAAM,KAAK,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,KAAK,KAAK,KAAK,KAAK;GAC1F,IAAI,MAAM,IAAI,OAAO,IAAI,KAAK,KAAK;IAAE,MAAM;IAAG,KAAK;IAAG;GAAG,CAAC;EAC5D;EACA,OAAO,IAAI,OAAO,CAAC,CAAC,KAAK,MAAM,EAAE,IAAI;CACvC;;CAGA,UAAU,OAAqB,MAAgB,KAAqB;EAClE,OAAO,aACL,KAAK,QACL,OACA,KAAK,OACL,MAAM,KAAK,MACX,KAAK,OAAO,MACZ,KAAK,IACP;CACF;AACF;;;;ACzOA,SAAgB,UAAU,OAAiC;CACzD,MAAM,OAAO,MAAM,MAAM;CACzB,OAAO,IAAI,aAAa,KAAK,QAAQ,GAAG,KAAK,cAAc,CAAC;AAC9D;AAEA,SAAgB,aAAa,GAA6B;CACxD,OAAO,IAAI,WAAW,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC,MAAM;AACpE;;;;;;;ACrBA,MAAa,aAAa;;;;;;AAa1B,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA;CAEA,YAAY,IAAQ,QAAgB,MAAc,aAAa,UAAkB;EAC/E,KAAK,MAAM;EACX,KAAK,QAAQ;EACb,KAAK,KAAK;GAAE,WAAW,GAAG,OAAO;GAAY,KAAK,GAAG,OAAO;GAAM,KAAK,GAAG,OAAO;EAAK;EACtF,KAAK,SAAS,IAAI,IAAI,UAAU;CAClC;CAEA,SAAmB;EACjB,MAAM,EAAE,WAAW,KAAK,QAAQ,KAAK;EACrC,OAAO;GACL,8BAA8B,UAAU;;GAExC,8BAA8B,IAAI;;GAElC,8BAA8B,IAAI;;EAEpC;CACF;;CAGA,MAAM,MAAM,IAAQ,KAAa,OAAoC;EACnE,MAAM,IAAI,KAAK;EACf,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,OAAO,KAAK;GACpC,MAAM,GAAG,MACP,eAAe,KAAK,GAAG,UAAU,mDACjC;IAAC;IAAK;IAAG,MAAM,OAAO;IAAI,aAAa,MAAM,UAAU,SAAS,IAAI,IAAI,IAAI,KAAK,CAAC,CAAC;GAAC,CACtF;GACA,MAAM,WAAW,IAAI,KAAK,GAAG,KAAK,KAAK,GAAG,MAAM,MAAM,EAAgB;GACtE,MAAM,WAAW,IAAI,KAAK,GAAG,KAAK,KAAK,GAAG,MAAM,OAAO,EAAgB;EACzE;CACF;;CAGA,MAAM,KAAK,KAAiB,QAAuC;EACjE,MAAM,IAAI,KAAK;EACf,MAAM,OAAO,MAAM,KAAK,IAAI,MAC1B,qCAAqC,KAAK,GAAG,UAAU,+BACvD,CAAC,IAAI,GAAG,CACV;EACA,IAAI,KAAK,WAAW,IAAI,OACtB,MAAM,IAAI,MACR,yBAAyB,IAAI,IAAI,OAAO,KAAK,OAAO,mBAAmB,IAAI,OAC7E;EAEF,MAAM,YAAY,IAAI,aAAa,IAAI,QAAQ,CAAC;EAChD,MAAM,SAAS,IAAI,YAAY,IAAI,KAAK;EACxC,KAAK,MAAM,KAAK,MAAM;GACpB,MAAM,IAAI,OAAO,EAAE,IAAI;GACvB,UAAU,IAAI,UAAU,EAAE,QAAQ,GAAG,IAAI,CAAC;GAC1C,OAAO,KAAK,OAAO,EAAE,KAAK;EAC5B;EACA,OAAO,IAAI,aAAa,IAAI,KAAK,GAAG,QAAQ,WAAW,MAAM;CAC/D;;CAGA,MAAM,MAAM,KAAmB,IAAmC;EAChE,MAAM,sBAAM,IAAI,IAAsB;EACtC,MAAM,UAAoB,CAAC;EAC3B,KAAK,MAAM,KAAK,IAAI;GAClB,MAAM,MAAM,KAAK,OAAO,IAAI,KAAK,IAAI,IAAI,GAAG,GAAG;GAC/C,IAAI,KAAK,IAAI,IAAI,GAAG,GAAG;QAClB,QAAQ,KAAK,CAAC;EACrB;EACA,IAAI,QAAQ,SAAS,GAAG;GACtB,MAAM,SAAS,MAAM,UAAU,KAAK,KAAK,KAAK,GAAG,KAAK,IAAI,KAAK,OAAO;GACtE,KAAK,MAAM,KAAK,SAAS;IACvB,MAAM,QAAQ,OAAO,IAAI,CAAC,qBAAK,IAAI,WAAW,CAAC;IAC/C,MAAM,OAAO,UAAU,OAAO,IAAI,OAAO,IAAc,KAAK,KAAK;IACjE,KAAK,OAAO,IAAI,KAAK,IAAI,IAAI,GAAG,KAAK,MAAM,MAAM,UAAU;IAC3D,IAAI,IAAI,GAAG,IAAI;GACjB;EACF;EACA,OAAO,GAAG,KAAK,MAAM,IAAI,IAAI,CAAC,CAAa;CAC7C;;CAGA,MAAM,OAAO,KAAmB,GAAkC;EAChE,MAAM,MAAM,KAAK,IAAI,IAAI,GAAG;EAC5B,MAAM,MAAM,KAAK,OAAO,IAAI,GAAG;EAC/B,IAAI,KAAK,OAAO;EAEhB,MAAM,SAAS,WAAU,MADJ,UAAU,KAAK,KAAK,KAAK,GAAG,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,EACzC,CAAO,IAAI,CAAC,qBAAK,IAAI,WAAW,CAAC,CAAC;EAC3D,KAAK,OAAO,IAAI,KAAK,QAAQ,OAAO,UAAU;EAC9C,OAAO;CACT;;CAGA,MAAM,KAAK,IAAQ,MAA+B;EAChD,MAAM,OAAO,KAAK,UAAU,IAAI;EAChC,KAAK,MAAM,SAAS,OAAO,OAAO,KAAK,EAAE,GACvC,MAAM,GAAG,MAAM,eAAe,MAAM,iDAAiD,CAAC,IAAI,CAAC;EAE7F,KAAK,MAAM,OAAO,MAAM,KAAK,OAAO,aAAa,KAAK,IAAI,IAAI,KAAK,IAAI,EAAE;CAC3E;;CAGA,MAAM,MAAM,IAAQ,MAA+B;EACjD,MAAM,OAAO,KAAK,UAAU,IAAI;EAChC,KAAK,MAAM,SAAS,OAAO,OAAO,KAAK,EAAE,GACvC,MAAM,GAAG,MAAM,eAAe,MAAM,qDAAqD,CACvF,IACF,CAAC;CAEL;AACF;AAEA,eAAe,WAAW,IAAQ,OAAe,KAAa,MAAc,OAAmB;CAC7F,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,KAAK,MAAM,YAAY,QAAQ,MAAM,YAC9D,MAAM,GAAG,MAAM,eAAe,MAAM,+CAA+C;EACjF;EACA;EACA;EACA,MAAM,MAAM,IAAI,KAAK,UAAU;CACjC,CAAC;AAEL;;AAGA,eAAe,UACb,IACA,OACA,KACA,OACkC;CAClC,MAAM,OAAO,MAAM,GAAG,MACpB,2BAA2B,MAAM;sFAEjC,CAAC,KAAK,KAAK,UAAU,KAAK,CAAC,CAC7B;CACA,MAAM,wBAAQ,IAAI,IAA0B;CAC5C,KAAK,MAAM,KAAK,MAAM;EACpB,MAAM,OAAO,OAAO,EAAE,IAAI;EAC1B,MAAM,MAAM,MAAM,IAAI,IAAI,KAAK,CAAC;EAChC,IAAI,KAAK,EAAE,KAAK;EAChB,MAAM,IAAI,MAAM,GAAG;CACrB;CACA,MAAM,sBAAM,IAAI,IAAwB;CACxC,KAAK,MAAM,CAAC,MAAM,OAAO,OAAO;EAC9B,MAAM,SAAS,IAAI,WAAW,GAAG,QAAQ,GAAG,MAAM,IAAI,EAAE,YAAY,CAAC,CAAC;EACtE,IAAI,KAAK;EACT,KAAK,MAAM,KAAK,IAAI;GAClB,OAAO,IAAI,GAAG,EAAE;GAChB,MAAM,EAAE;EACV;EACA,IAAI,IAAI,MAAM,MAAM;CACtB;CACA,OAAO;AACT;;AAGA,IAAM,MAAN,MAAa;CACX;CACA,uBAAgB,IAAI,IAAwC;CAC5D,SAAS;CAET,YAAY,UAAkB;EAC5B,KAAK,OAAO;CACd;CAEA,IAAI,KAA4B;EAC9B,MAAM,IAAI,KAAK,KAAK,IAAI,GAAG;EAC3B,IAAI,CAAC,GAAG,OAAO,KAAA;EACf,KAAK,KAAK,OAAO,GAAG;EACpB,KAAK,KAAK,IAAI,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,IAAI,KAAa,OAAU,MAAc;EACvC,MAAM,MAAM,KAAK,KAAK,IAAI,GAAG;EAC7B,IAAI,KAAK;GACP,KAAK,KAAK,OAAO,GAAG;GACpB,KAAK,UAAU,IAAI;EACrB;EACA,KAAK,KAAK,IAAI,KAAK;GAAE;GAAO;EAAK,CAAC;EAClC,KAAK,UAAU;EACf,KAAK,MAAM,CAAC,GAAG,MAAM,KAAK,MAAM;GAC9B,IAAI,KAAK,UAAU,KAAK,MAAM;GAC9B,KAAK,KAAK,OAAO,CAAC;GAClB,KAAK,UAAU,EAAE;EACnB;CACF;CAEA,aAAa,GAAG,UAAoB;EAClC,KAAK,MAAM,CAAC,GAAG,MAAM,KAAK,MACxB,IAAI,SAAS,MAAM,MAAM,EAAE,WAAW,CAAC,CAAC,GAAG;GACzC,KAAK,KAAK,OAAO,CAAC;GAClB,KAAK,UAAU,EAAE;EACnB;CAEJ;AACF;;;;AC5JA,MAAM,iBAAiB;;AAEvB,MAAM,cAAc;;;;;;;;;;;;;;;;AAiBpB,IAAa,cAAb,MAAyB;CACvB;CACA;;CAEA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAgB;EAAE,KAAK;EAAG,SAAS;EAAG,UAAU,CAAC;CAAE;CACnD,wBAAQ,IAAI,IAAuB;CACnC,YAA4B,CAAC;CAC7B,wBAAQ,IAAI,IAA0B;CACtC,WAA2B,CAAC;CAC5B,SAA2B,QAAQ,QAAQ;CAC3C,eAAe;CACf,SAAS;;CAET,SAAS;CAET,YAAY,MAA0B;EACpC,IAAI,CAAC,2BAA2B,KAAK,KAAK,MAAM,GAC9C,MAAM,IAAI,MAAM,4BAA4B,KAAK,QAAQ;EAE3D,IAAI,CAAC,OAAO,UAAU,KAAK,IAAI,KAAK,KAAK,OAAO,GAC9C,MAAM,IAAI,MAAM,+CAA+C;EAEjE,KAAK,OAAO,KAAK;EACjB,KAAK,SAAS,KAAK,UAAU;EAC7B,KAAK,MAAM,KAAK;EAChB,KAAK,SAAS,IAAI,gBAAgB,KAAK,IAAI,KAAK,QAAQ,KAAK,MAAM,KAAK,UAAU;EAClF,KAAK,SAAS,KAAK;EACnB,KAAK,iBAAiB,KAAK,iBAAiB;EAC5C,MAAM,IAAI,KAAK;EACf,KAAK,KAAK;GAAE,MAAM,GAAG,EAAE;GAAO,KAAK,GAAG,EAAE;GAAM,MAAM,GAAG,EAAE;EAAM;EAC/D,KAAK,UAAU,KAAK,GAAG;CACzB;;CAKA,MAAM,OAAsB;EAC1B,MAAM,EAAE,MAAM,KAAK,SAAS,KAAK;EACjC,KAAK,MAAM,KAAK;GACd,8BAA8B,KAAK;GACnC,8BAA8B,IAAI;GAClC,8BAA8B,IAAI,UAAU,IAAI;GAChD,8BAA8B,KAAK;GACnC,8BAA8B,KAAK,UAAU,KAAK;GAClD,GAAG,KAAK,OAAO,OAAO;EACxB,GACE,MAAM,KAAK,IAAI,KAAK,CAAC;EAGvB,MAAM,SAAS,KAAK,UAAU;GAAE,MAAM,KAAK;GAAM,QAAQ,KAAK;EAAO,CAAC;EACtE,MAAM,KAAK,IAAI,MACb,eAAe,KAAK,wDACpB,CAAC,MAAM,CACT;EACA,MAAM,CAAC,UAAU,MAAM,KAAK,IAAI,MAC9B,iBAAiB,KAAK,oBACxB;EACA,IAAI,QAAQ,MAAM,QAChB,MAAM,IAAI,MAAM,0CAA0C,QAAQ,EAAE,QAAQ,QAAQ;EAEtF,MAAM,KAAK,QAAQ;EACnB,MAAM,OAAO,KAAK,OAAO,SAAS,KAAK,MAAM,EAAE,GAAG;EAClD,MAAM,cAAc,KAAK,MAAM,OAAO,KAAK,OAAO,MAAM,IAAI,IAAI,CAAC;EACjE,KAAK,SAAS;CAChB;;CAGA,MAAM,UAAyB;EAC7B,MAAM,CAAC,OAAO,MAAM,KAAK,IAAI,MAC3B,iBAAiB,KAAK,GAAG,KAAK,mBAChC;EACA,MAAM,QAAe,MAAM,KAAK,MAAM,IAAI,CAAC,IAAI;GAAE,KAAK;GAAG,SAAS;GAAG,UAAU,CAAC;EAAE;EAElF,MAAM,uBAAO,IAAI,IAAuB;EACxC,MAAM,OAAO,MAAM,KAAK,IAAI,MAC1B,4BAA4B,KAAK,GAAG,MACtC;EACA,KAAK,MAAM,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,EAAE,GAAG;GAAE,KAAK,OAAO,EAAE,GAAG;GAAG,QAAQ,UAAU,EAAE,GAAG;EAAE,CAAC;EAE7F,MAAM,QAAQ,IAAI,IAAI,KAAK,UAAU,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;EAC3D,MAAM,WAAW,MAAM,QAAQ,IAC7B,MAAM,SAAS,KAAK,MAAM,MAAM,IAAI,EAAE,GAAG,KAAK,KAAK,OAAO,KAAK,GAAG,KAAK,MAAM,CAAC,CAChF;EACA,KAAK,SAAS;EACd,KAAK,QAAQ;EACb,KAAK,YAAY;CACnB;CAEA,IAAI,QAA0B;EAC5B,OAAO;GACL,MAAM,KAAK,MAAM;GACjB,UAAU,KAAK,OAAO,SAAS,KAAK,OAAO,EAAE,GAAG,EAAE,EAAE;GACpD,KAAK,KAAK,OAAO;EACnB;CACF;CAIA,MAAM,OAAO,OAAoC;EAC/C,KAAK,aAAa;EAClB,MAAM,WAAW,MAAM,KAAK,OAAO;GACjC,SAAS,GAAG,EAAE;GACd,OAAO;IAAE,IAAI,GAAG;IAAI,QAAQ,QAAQ,GAAG,QAAQ,KAAK,MAAM,KAAK,MAAM;GAAE;EACzE,CAAC;EACD,MAAM,EAAE,MAAM,QAAQ,KAAK;EAC3B,KAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK,aAAa;GACrD,MAAM,QAAQ,SAAS,MAAM,GAAG,IAAI,WAAW;GAC/C,MAAM,KAAK,WAAW,YAAY;IAChC,MAAM,MAAM,KAAK,OAAO,MAAM;IAC9B,MAAM,OAAO;KAAE,GAAG,KAAK;KAAQ;IAAI;IACnC,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;KAC1C,KAAK,MAAM,EAAE,IAAI,YAAY,OAAO;MAClC,MAAM,GAAG,MACP,eAAe,KAAK;sFAEpB;OAAC;OAAI;OAAK,aAAa,MAAM;MAAC,CAChC;MACA,MAAM,GAAG,MACP,eAAe,IAAI,gEACnB,CAAC,EAAE,CACL;KACF;KACA,MAAM,KAAK,WAAW,IAAI,IAAI;IAChC,CAAC;IACD,KAAK,SAAS;IACd,KAAK,MAAM,EAAE,IAAI,YAAY,OAAO,KAAK,MAAM,IAAI,IAAI;KAAE;KAAK;IAAO,CAAC;GACxE,CAAC;GACD,IAAI,KAAK,MAAM,QAAQ,KAAK,kBAAkB,CAAC,KAAK,cAAc,MAAM,KAAK,QAAQ;EACvF;CACF;CAEA,MAAM,OAAO,KAA8B;EACzC,KAAK,aAAa;EAClB,IAAI,QAAQ,QAAQ;EACpB,MAAM,KAAK,WAAW,YAAY;GAChC,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK,gBAAgB;IACnD,MAAM,QAAQ,IAAI,MAAM,GAAG,IAAI,cAAc;IAC7C,MAAM,OAAO,KAAK,UAAU,KAAK;IACjC,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;KAC1C,MAAM,GAAG,MACP,eAAe,KAAK,GAAG,KAAK,gDAC5B,CAAC,IAAI,CACP;KACA,MAAM,GAAG,MACP,eAAe,KAAK,GAAG,IAAI,gDAC3B,CAAC,IAAI,CACP;IACF,CAAC;IACD,KAAK,MAAM,MAAM,OAAO,KAAK,MAAM,OAAO,EAAE;GAC9C;EACF,CAAC;CACH;CAIA,MAAM,OAAO,OAA0B,OAAsB,CAAC,GAAyB;EACrF,KAAK,aAAa;EAClB,MAAM,IAAI,KAAK,KAAK;EACpB,MAAM,SAAS,KAAK,UAAU;EAC9B,MAAM,UAAU,KAAK,KAAK,cAAc;EACxC,MAAM,IAAI,QAAQ,OAAO,KAAK,MAAM,KAAK,MAAM;EAC/C,MAAM,IAAI,KAAK;EACf,MAAM,SAAS,KAAK;EAEpB,OAAO,KAAK,MAAM,OAAO,aAAa;GAEpC,MAAM,UAAU,IAAI,KAAa,CAAC;GAClC,KAAK,MAAM,CAAC,IAAI,EAAE,aAAa,KAAK,OAAO;IACzC,IAAI,UAAU,CAAC,OAAO,EAAE,GAAG;IAC3B,QAAQ,KAAK,SAAS,KAAK,QAAQ,GAAG,GAAG,QAAQ,GAAG,CAAC,GAAG,EAAE;GAC5D;GAIA,MAAM,OAAc,CAAC;GAErB,MAAM,QAAQ,OAAe,CAAC,KAAK,MAAM,IAAI,EAAE,MAAM,CAAC,UAAU,OAAO,EAAE;GACzE,MAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,QAAQ;IAC1B,MAAM,QAAQ,IAAI,MAAM,GAAG,MAAM;IACjC,MAAM,QAAQ,MAAM,KAAK,OAAO,MAAM,KAAK,KAAK;IAChD,KAAK,QAAQ,eAAe;IAC5B,MAAM,SAAS,GAAG,MAAM;KACtB,MAAM,OAAO,MAAM;KACnB,KAAK,MAAM,QAAQ,IAAI,WAAW,GAAG,GAAG,MAAM,SAAS,IAAI,GACzD,KAAK,KAAK;MAAE,GAAG;MAAM;MAAK;KAAK,CAAC;IACpC,CAAC;GACH,CAAC,CACH;GAGA,MAAM,OAAO,MAAM,KAAK,aAAa,KAAK,KAAK,MAAM,EAAE,EAAE,CAAC;GAI1D,MAAM,SAAS,IAAI,KADJ,KAAK,QAAQ,IAAI,IAAI,CACD;GACnC,KAAK,MAAM,KAAK,MAAM;IACpB,IAAI,KAAK,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,KAAK;IAClC,OAAO,KAAK,EAAE,IAAI,UAAU,GAAG,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC;GAClD;GACA,IAAI,aAAa,OACd,OAAO,CAAC,CACR,KAAK,OAAO;IAAE,IAAI,EAAE,KAAK;IAAI,UAAU,EAAE;IAAO,KAAK,EAAE;GAAK,EAAE;GAGjE,IAAI,KAAK,OACP,aAAa,MAAM,QAAQ,IACzB,WAAW,IAAI,OAAO,MAAM;IAE1B,MAAM,KAAI,MADQ,KAAK,OAAO,OAAO,EAAE,IAAI,KAAK,EAAE,IAAI,IAAI,EAAA,CAC5C,SAAS,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,KAAK,CAAC;IACzD,OAAO;KAAE,GAAG;KAAG,UAAU,SAAS,KAAK,QAAQ,GAAG,GAAG,GAAG,GAAG,CAAC;IAAE;GAChE,CAAC,CACH;GAGF,MAAM,QAAQ,IAAI,KAAa,CAAC;GAChC,KAAK,MAAM,KAAK,YAAY,MAAM,KAAK,EAAE,UAAU,EAAE,EAAE;GACvD,KAAK,MAAM,KAAK,QAAQ,OAAO,GAAG,MAAM,KAAK,EAAE,OAAO,EAAE,IAAI;GAC5D,OAAO,MAAM,OAAO,CAAC,CAAC,KAAK,OAAO;IAAE,IAAI,EAAE;IAAM,UAAU,EAAE;GAAM,EAAE;EACtE,CAAC;CACH;;CAGA,MAAM,WAAW,KAAmD;EAClE,KAAK,aAAa;EAClB,IAAI,QAAQ,QAAQ;EACpB,OAAO,KAAK,MAAM,OAAO,aAAa;GACpC,MAAM,sBAAM,IAAI,IAA0B;GAC1C,MAAM,OAAO,MAAM,KAAK,aAAa,GAAG;GACxC,MAAM,yBAAS,IAAI,IAAyB;GAC5C,KAAK,MAAM,CAAC,IAAI,QAAQ,MAAM;IAC5B,IAAI,QAAQ,GAAG;KACb,MAAM,QAAQ,KAAK,MAAM,IAAI,EAAE;KAC/B,IAAI,OAAO,IAAI,IAAI,IAAI,MAAM,OAAO,MAAM,CAAC;KAC3C;IACF;IACA,MAAM,MAAM,OAAO,IAAI,GAAG,qBAAK,IAAI,IAAY;IAC/C,IAAI,IAAI,EAAE;IACV,OAAO,IAAI,KAAK,GAAG;GACrB;GACA,KAAK,MAAM,OAAO,UAAU;IAC1B,MAAM,MAAM,OAAO,IAAI,IAAI,GAAG;IAC9B,IAAI,CAAC,KAAK;IACV,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,SAAS,IAAI,OAAO,GAAG,KAAK;KAClD,IAAI,IAAI,OAAO,OAAO,GAAG;KACzB,MAAM,CAAC,QAAS,MAAM,KAAK,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC;KAChD,IAAI,SAA8B;KAClC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,IAAI,QAAQ,KAAK;MACxC,MAAM,KAAK,KAAK,IAAI;MACpB,IAAI,CAAC,IAAI,IAAI,EAAE,GAAG;MAClB,WAAW,MAAM,KAAK,OAAO,OAAO,KAAK,CAAC;MAC1C,IAAI,IAAI,IAAI,OAAO,MAAM,IAAI,KAAK,OAAO,IAAI,KAAK,KAAK,IAAI,CAAC;MAC5D,IAAI,OAAO,EAAE;KACf;IACF;GACF;GACA,OAAO;EACT,CAAC;CACH;;CAKA,MAAM,UAA2B;EAC/B,KAAK,aAAa;EAClB,OAAO,KAAK,aAAa,YAAY;GACnC,MAAM,WAAW,MAAM,KAAK,WAAW,aAAa;IAClD,QAAQ,KAAK,OAAO;IACpB,SAAS,CAAC,GAAG,KAAK,MAAM,QAAQ,CAAC;GACnC,EAAE;GACF,IAAI,SAAS,QAAQ,WAAW,GAAG,OAAO;GAE1C,MAAM,IAAI,SAAS,QAAQ;GAC3B,MAAM,MAAM,IAAI,aAAa,CAAC;GAC9B,MAAM,UAAU,IAAI,aAAa,IAAI,KAAK,IAAI;GAC9C,SAAS,QAAQ,SAAS,CAAC,IAAI,IAAI,MAAM;IACvC,IAAI,KAAK;IACT,QAAQ,IAAI,EAAE,QAAQ,IAAI,KAAK,IAAI;GACrC,CAAC;GACD,MAAM,QAAQ,KAAK,OAAO,KAAK,OAAO;GAEtC,MAAM,KAAK,iBACT,KAAK,cAAc,YAAY;IAC7B,MAAM,MAAM,KAAK,OAAO,OAAO,CAAC;IAChC,MAAM,OAAc;KAClB,GAAG,KAAK;KACR,SAAS,IAAI,MAAM;KACnB,UAAU,CAAC,GAAG,KAAK,OAAO,UAAU,GAAG;IACzC;IACA,MAAM,EAAE,KAAK,MAAM,SAAS,KAAK;IACjC,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;KAC1C,MAAM,KAAK,OAAO,MAAM,IAAI,IAAI,KAAK,KAAK;KAC1C,MAAM,GAAG,MACP,UAAU,KAAK;yCACY,KAAK,mBAChC,CAAC,IAAI,KAAK,SAAS,MAAM,CAC3B;KACA,MAAM,GAAG,MAAM,eAAe,KAAK,kBAAkB,CAAC,SAAS,MAAM,CAAC;KACtE,MAAM,KAAK,WAAW,IAAI,IAAI;IAChC,CAAC;IACD,KAAK,SAAS;IACd,KAAK,MAAM,CAAC,IAAI,MAAM,KAAK,OAAO,IAAI,EAAE,OAAO,SAAS,QAAQ,KAAK,MAAM,OAAO,EAAE;IACpF,KAAK,YAAY,CAAC,GAAG,KAAK,WAAW,KAAK,WAAW,KAAK,KAAK,CAAC;GAClE,CAAC,CACH;GACA,OAAO;EACT,CAAC;CACH;;;;;CAMA,MAAM,QAAyB;EAC7B,KAAK,aAAa;EAClB,OAAO,KAAK,aAAa,YAAY;GACnC,MAAM,MAAM,KAAK;GACjB,IAAI,IAAI,WAAW,GAAG,OAAO;GAC7B,MAAM,UAAU,IAAI,KAAK,MAAM,EAAE,GAAG;GAEpC,MAAM,SAAmB,CAAC;GAC1B,MAAM,SAAyB,CAAC;GAChC,KAAK,MAAM,OAAO,KAChB,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,OAAO,KAAK;IAClC,IAAI,IAAI,OAAO,OAAO,GAAG;IACzB,MAAM,CAAC,QAAS,MAAM,KAAK,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC;IAChD,MAAM,SAAS,MAAM,KAAK,OAAO,OAAO,KAAK,CAAC;IAC9C,MAAM,OAAO,MAAM,KAAK,aAAa,MAAM,KAAK,KAAK,GAAG,CAAC;IACzD,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,IAAI,QAAQ,KAAK;KACxC,MAAM,KAAK,KAAK,IAAI;KACpB,IAAI,KAAK,IAAI,EAAE,MAAM,IAAI,KAAK;KAC9B,OAAO,KAAK,EAAE;KACd,OAAO,KAAK,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,KAAK,KAAK,IAAI,CAAC;IACjE;GACF;GAGF,MAAM,IAAI,OAAO;GACjB,MAAM,UAAU,IAAI,aAAa,IAAI,KAAK,IAAI;GAC9C,OAAO,SAAS,GAAG,MAAM;IACvB,QAAQ,IAAI,GAAG,IAAI,KAAK,IAAI;GAC9B,CAAC;GACD,MAAM,QAAQ,IAAI,IAAI,KAAK,OAAO,aAAa,KAAK,MAAM,GAAG,OAAO,IAAI;GAExE,MAAM,KAAK,iBACT,KAAK,cAAc,YAAY;IAC7B,MAAM,MAAM,QAAQ,KAAK,OAAO,OAAO,CAAC,IAAI;IAC5C,MAAM,OAAO,KAAK,OAAO,SAAS,QAAQ,MAAM,CAAC,QAAQ,SAAS,EAAE,GAAG,CAAC;IACxE,MAAM,OAAc;KAClB,GAAG,KAAK;KACR,SAAS,MAAM,IAAI,MAAM,IAAI,KAAK,OAAO;KACzC,UAAU,MAAM,CAAC,GAAG,MAAM,GAAG,IAAI;IACnC;IACA,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;KAC1C,IAAI,OAAO,OAAO;MAChB,MAAM,KAAK,OAAO,MAAM,IAAI,IAAI,KAAK,KAAK;MAC1C,MAAM,GAAG,MACP,UAAU,KAAK,GAAG,IAAI,6DACtB,CAAC,IAAI,KAAK,KAAK,UAAU,OAAO,CAAC,CACnC;KACF;KACA,MAAM,KAAK,WAAW,IAAI,IAAI;IAChC,CAAC;IACD,KAAK,SAAS;IACd,KAAK,YAAY,CACf,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,QAAQ,SAAS,EAAE,GAAG,CAAC,GACxD,GAAI,OAAO,QAAQ,CAAC,KAAK,WAAW,KAAK,KAAK,CAAC,IAAI,CAAC,CACtD;IACA,KAAK,SAAS,KAAK,GAAG,GAAG;GAC3B,CAAC,CACH;GACA,MAAM,KAAK,MAAM;GACjB,OAAO;EACT,CAAC;CACH;;CAGA,MAAM,WAA0B;EAC9B,MAAM,KAAK,QAAQ;EACnB,MAAM,KAAK,MAAM;CACnB;CAIA,OAAO,KAAmB,SAAqC;EAC7D,OAAO,aAAa;GAClB,MAAM,KAAK;GACX,QAAQ,KAAK;GACb;GACA;GACA,OAAO,KAAK,SAAS,IAAI,MAAM;EACjC,CAAC;CACH;CAEA,OAAO,OAAqB,OAA2B;EACrD,OAAO;GAAE,KAAK,KAAK,OAAO;GAAS;GAAO,OAAO,MAAM;EAAM;CAC/D;CAEA,WAAW,KAAiB,OAAmC;EAC7D,OAAO,IAAI,aAAa,IAAI,KAAK,KAAK,MAAM,KAAK,QAAQ,MAAM,WAAW,MAAM,MAAM;CACxF;CAEA,MAAM,aAAa,KAA6C;EAC9D,MAAM,sBAAM,IAAI,IAAoB;EACpC,MAAM,SAAS,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;EAC/B,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,gBAAgB;GACtD,MAAM,OAAO,MAAM,KAAK,IAAI,MAC1B,uBAAuB,KAAK,GAAG,IAAI,gDACnC,CAAC,KAAK,UAAU,OAAO,MAAM,GAAG,IAAI,cAAc,CAAC,CAAC,CACtD;GACA,KAAK,MAAM,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,GAAG,CAAC;EAC3D;EACA,OAAO;CACT;CAEA,MAAM,WAAW,IAAQ,OAA6B;EACpD,MAAM,GAAG,MACP,eAAe,KAAK,GAAG,KAAK;qDAE5B,CAAC,KAAK,UAAU,KAAK,CAAC,CACxB;CACF;;CAGA,MAAM,QAAuB;EAC3B,MAAM,QAAQ,KAAK,SAAS,QAAQ,OAAO,KAAK,MAAM,IAAI,CAAC,KAAK,OAAO,CAAC;EACxE,IAAI,MAAM,WAAW,GAAG;EACxB,KAAK,WAAW,KAAK,SAAS,QAAQ,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;EAC9D,KAAK,MAAM,KAAK,OAAO,KAAK,MAAM,OAAO,CAAC;EAC1C,MAAM,OAAO,MAAM,KAAK,MAAM,EAAE,GAAG;EACnC,MAAM,cAAc,KAAK,MAAM,OAAO,KAAK,OAAO,KAAK,IAAI,IAAI,CAAC;CAClE;;;;;;CAOA,MAAM,MAAS,MAA4D;EACzE,KAAK,IAAI,UAAU,IAAK,WAAW;GACjC,MAAM,QAAQ,KAAK;GACnB,MAAM,WAAW,KAAK;GACtB,KAAK,MAAM,KAAK,UAAU,KAAK,MAAM,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,CAAC;GACxE,IAAI;IACF,MAAM,SAAS,MAAM,KAAK,QAAQ;IAClC,IAAI,YAAY,KAAM,QAAQ,MAAM,KAAK,UAAU,KAAK,QAAS,OAAO;GAC1E,UAAU;IACR,KAAK,MAAM,KAAK,UAAU,KAAK,MAAM,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,CAAC;IACxE,KAAU,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;GAClC;EACF;CACF;;CAGA,MAAM,cAAc,IAAwC;EAC1D,KAAK;EACL,IAAI;GACF,MAAM,GAAG;EACX,UAAU;GACR,KAAK;EACP;CACF;CAEA,WAAc,IAAkC;EAC9C,MAAM,MAAM,KAAK,OAAO,KAAK,IAAI,EAAE;EACnC,KAAK,SAAS,IAAI,YAAY,CAAC,CAAC;EAChC,OAAO;CACT;CAEA,MAAM,aAAgB,IAAkC;EACtD,IAAI,KAAK,cAAc,MAAM,IAAI,MAAM,gDAAgD;EACvF,KAAK,eAAe;EACpB,IAAI;GACF,OAAO,MAAM,GAAG;EAClB,UAAU;GACR,KAAK,eAAe;EACtB;CACF;CAEA,eAAe;EACb,IAAI,CAAC,KAAK,QAAQ,MAAM,IAAI,MAAM,iCAAiC;CACrE;AACF;AAEA,SAAS,SAAS,IAAY;CAC5B,IAAI,CAAC,OAAO,cAAc,EAAE,GAC1B,MAAM,IAAI,MAAM,gDAAgD,IAAI;AACxE;;AAGA,eAAsB,gBAAgB,IAAQ,QAA+B;CAC3E,IAAI,CAAC,2BAA2B,KAAK,MAAM,GACzC,MAAM,IAAI,MAAM,4BAA4B,QAAQ;CACtD,MAAM,cAAc,IAAI,OAAO,OAAO;EACpC,KAAK,MAAM,KAAK;GAAC;GAAQ;GAAO;GAAQ;GAAa;GAAO;EAAK,GAC/D,MAAM,GAAG,KAAK,wBAAwB,SAAS,GAAG;CAEtD,CAAC;AACH"}
package/package.json CHANGED
@@ -1,6 +1,66 @@
1
1
  {
2
2
  "name": "@statewalker/db-sqlite-index",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
3
+ "version": "0.1.0",
4
+ "private": false,
5
+ "type": "module",
6
+ "description": "Id-based vector (IVF) and n-gram indexes stored in SQLite through @statewalker/db-api: Node, browser and Durable Objects.",
7
+ "bugs": {
8
+ "url": "https://github.com/statewalker/statewalker-db/issues"
9
+ },
10
+ "homepage": "https://github.com/statewalker/statewalker-db/tree/main/packages/db-sqlite-index#readme",
11
+ "author": {
12
+ "name": "Mikhail Kotelnikov",
13
+ "email": "mikhail.kotelnikov@gmail.com"
14
+ },
15
+ "license": "MIT",
16
+ "repository": {
17
+ "type": "git",
18
+ "url": "git+https://github.com/statewalker/statewalker-db.git",
19
+ "directory": "packages/db-sqlite-index"
20
+ },
21
+ "exports": {
22
+ "./vector": {
23
+ "source": "./src/vector/index.ts",
24
+ "types": "./dist/vector.d.mts",
25
+ "import": "./dist/vector.mjs"
26
+ },
27
+ "./ngram": {
28
+ "source": "./src/ngram/index.ts",
29
+ "types": "./dist/ngram.d.mts",
30
+ "import": "./dist/ngram.mjs"
31
+ }
32
+ },
33
+ "files": [
34
+ "dist",
35
+ "src"
36
+ ],
37
+ "dependencies": {
38
+ "@statewalker/db-api": "^0.1.4",
39
+ "@statewalker/db-sqlite": "^0.1.0"
40
+ },
41
+ "devDependencies": {
42
+ "@types/node": "^26.6.4",
43
+ "rimraf": "^6.1.3",
44
+ "tsdown": "^0.23.0",
45
+ "tsx": "^4.23.15",
46
+ "typescript": "^7.0.2",
47
+ "vitest": "^5.0.3",
48
+ "@statewalker/db-sqlite-do": "^0.1.0",
49
+ "@statewalker/db-sqlite-node": "^0.3.0"
50
+ },
51
+ "sideEffects": false,
52
+ "publishConfig": {
53
+ "access": "public"
54
+ },
55
+ "scripts": {
56
+ "build": "tsdown",
57
+ "dev": "tsdown --watch",
58
+ "test": "vitest run",
59
+ "test:watch": "vitest",
60
+ "typecheck": "tsc --noEmit",
61
+ "bench": "tsx bench/vector-bench.ts 50000 384",
62
+ "clean": "rimraf dist",
63
+ "lint": "biome check --write .",
64
+ "format": "biome format --write ."
65
+ }
6
66
  }
@@ -0,0 +1,46 @@
1
+ import type { Db } from "@statewalker/db-api";
2
+ import { NgramStore } from "./ngram-store.js";
3
+
4
+ const check = (ok: boolean, message: string) => {
5
+ if (!ok) throw new Error(`ngram smoke: ${message}`);
6
+ };
7
+
8
+ /** True when the index itself still offers candidates (candidateBudget 0 skips the verifier). */
9
+ const indexed = async (index: NgramStore, pattern: string) =>
10
+ (await index.search(pattern, { candidateBudget: 0 })).truncated;
11
+
12
+ const ids = (result: { hits: { id: number }[] }) => result.hits.map((h) => h.id).join();
13
+
14
+ /** Run inside a real Durable Object by db-sqlite-do's `pnpm test:do`. */
15
+ export const ngramDoScenarios = (["fts5", "postings"] as const).map((store) => ({
16
+ name: `ngram (${store}): upsert, overwrite, lone surrogate, drop inside a transaction`,
17
+ async run(db: Db) {
18
+ const prefix = `smoke_${store}_`;
19
+ const index = new NgramStore({ db, prefix, store });
20
+ await index.init(); // CREATE of the index tables (a contentless trigram table for fts5) in a transaction
21
+ await index.upsert([
22
+ { id: 1, text: "Hello World" },
23
+ { id: 2, text: "hello there" },
24
+ { id: 3, text: "lone \ud800 surrogate" },
25
+ ]);
26
+ check(
27
+ ids(await index.search("hello", { caseSensitive: false })) === "1,2",
28
+ "case-insensitive hits",
29
+ );
30
+ check(ids(await index.search("Hello")) === "1", "case-sensitive hits");
31
+ check(ids(await index.search("surrogate")) === "3", "text with a lone surrogate is indexed");
32
+ check((await index.getTexts([3])).get(3) === "lone \ufffd surrogate", "text made well-formed");
33
+ check(await indexed(index, "World"), "index lacks the grams of 'World'");
34
+ check(await indexed(index, "surrogate"), "index lacks the grams of 'surrogate'");
35
+ await index.upsert([{ id: 1, text: "goodbye" }]); // replays the old text to remove its grams
36
+ check(!(await indexed(index, "World")), "stale grams after overwrite");
37
+ await index.upsert([{ id: 3, text: "lone \ud800 changed" }]);
38
+ check(!(await indexed(index, "surrogate")), "stale grams after surrogate overwrite");
39
+ await index.drop(); // DROP of the tables inside a transaction
40
+ const [row] = await db.query<{ n: number }>(
41
+ "SELECT count(*) AS n FROM sqlite_master WHERE name LIKE ?",
42
+ [`${prefix}%`],
43
+ );
44
+ check(Number(row?.n) === 0, `tables left after drop: ${row?.n}`);
45
+ },
46
+ }));
@@ -0,0 +1,50 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import { fold } from "./fold.js";
3
+
4
+ const escapeRegex = (s: string) => s.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&");
5
+
6
+ describe("fold", () => {
7
+ it("lower-cases ASCII and keeps length", () => {
8
+ expect(fold("Hello WORLD")).toBe("hello world");
9
+ });
10
+
11
+ it("maps every case form of sigma to one letter", () => {
12
+ expect(new Set([fold("Σ"), fold("σ"), fold("ς")]).size).toBe(1);
13
+ });
14
+
15
+ it("never separates two BMP characters that a case-insensitive regex treats as equal", () => {
16
+ const broken: string[] = [];
17
+ for (let cp = 0; cp <= 0xffff; cp++) {
18
+ if (cp >= 0xd800 && cp <= 0xdfff) continue;
19
+ const c = String.fromCharCode(cp);
20
+ const re = new RegExp(`^${escapeRegex(c)}$`, "i");
21
+ for (const d of [c.toUpperCase(), c.toLowerCase()]) {
22
+ if (d === c || d.length !== 1 || !re.test(d)) continue;
23
+ if (fold(c) !== fold(d)) broken.push(`U+${cp.toString(16)} ${c}/${d}`);
24
+ }
25
+ }
26
+ expect(broken).toEqual([]);
27
+ });
28
+
29
+ it("keeps the UTF-16 length of every code point, so offsets stay valid", () => {
30
+ const changed: string[] = [];
31
+ for (let cp = 0; cp <= 0x10ffff; cp++) {
32
+ if (cp >= 0xd800 && cp <= 0xdfff) continue;
33
+ const c = String.fromCodePoint(cp);
34
+ if (fold(c).length !== c.length) changed.push(`U+${cp.toString(16)}`);
35
+ }
36
+ expect(changed).toEqual([]);
37
+ });
38
+
39
+ it("keeps the length of a text with U+0130 and an astral character", () => {
40
+ const folded = fold("İ😀A");
41
+ expect(folded.length).toBe("İ😀A".length);
42
+ expect(folded.endsWith("a")).toBe(true);
43
+ });
44
+
45
+ it("is a homomorphism over concatenation, so substrings stay substrings", () => {
46
+ const a = "ΟΔΟΣ ſtraße";
47
+ const b = "İK";
48
+ expect(fold(a + b)).toBe(fold(a) + fold(b));
49
+ });
50
+ });
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Case fold per code point, applied to indexed text and to query literals alike.
3
+ *
4
+ * Upper-then-lower maps every character a case-insensitive JS regex treats as
5
+ * equal to one form (Σ σ ς → σ, ſ → s). A character whose upper case is several
6
+ * code points (ß → SS) only matches itself under `/i`, so it keeps its own
7
+ * lower case. Folding per code point makes fold(a + b) === fold(a) + fold(b), so
8
+ * a substring of the text stays a substring of the folded text. A character
9
+ * whose folded form has a different UTF-16 length (İ → i + U+0307) is kept as
10
+ * is, so fold preserves the length and offsets into the folded text are
11
+ * offsets into the original.
12
+ */
13
+ export function fold(text: string): string {
14
+ let out = "";
15
+ for (const ch of text) {
16
+ const upper = ch.toUpperCase();
17
+ const lower = (Array.from(upper).length === 1 ? upper : ch).toLowerCase();
18
+ out += lower.length === ch.length ? lower : ch;
19
+ }
20
+ return out;
21
+ }
@@ -0,0 +1,55 @@
1
+ import type { Db } from "@statewalker/db-api";
2
+ import { fold } from "./fold.js";
3
+ import { chunkJson } from "./json-chunks.js";
4
+
5
+ export type NgramItem = { id: number; text: string };
6
+
7
+ /** How one store finds candidate ids; texts and verification live in NgramStore. */
8
+ export type GramBackend = {
9
+ create(tx: Db): Promise<void>;
10
+ add(tx: Db, items: NgramItem[]): Promise<void>;
11
+ /** `items` hold the texts as currently stored, so their grams can be removed exactly. */
12
+ remove(tx: Db, items: NgramItem[]): Promise<void>;
13
+ /** Ids whose text contains every literal of at least one branch; ascending, at most `max`. */
14
+ candidates(db: Db, branches: string[][], max: number): Promise<number[]>;
15
+ };
16
+
17
+ const ROW = "json_extract(value, '$[0]'), json_extract(value, '$[1]')";
18
+
19
+ /**
20
+ * FTS5 trigram over folded text. Contentless (`content=''`), because NgramStore
21
+ * already keeps the text; a contentless row is deleted by replaying its exact
22
+ * indexed value through the 'delete' command, which is why `remove` needs the
23
+ * stored texts. `case_sensitive 1`: folding is done by `fold`, the same function
24
+ * that folds query literals, so the index and the planner can never disagree.
25
+ */
26
+ export function newFts5Backend(prefix: string): GramBackend {
27
+ const table = `${prefix}ng`;
28
+ const write = async (tx: Db, items: NgramItem[], command: "insert" | "delete") => {
29
+ const sql =
30
+ command === "insert"
31
+ ? `INSERT INTO ${table}(rowid, text) SELECT ${ROW} FROM json_each(?)`
32
+ : `INSERT INTO ${table}(${table}, rowid, text) SELECT 'delete', ${ROW} FROM json_each(?)`;
33
+ for (const chunk of chunkJson(items.map((item) => [item.id, fold(item.text)]))) {
34
+ await tx.query(sql, [chunk]);
35
+ }
36
+ };
37
+ return {
38
+ create: (tx) =>
39
+ tx.exec(
40
+ `CREATE VIRTUAL TABLE IF NOT EXISTS ${table} USING fts5(text, content='', tokenize='trigram case_sensitive 1')`,
41
+ ),
42
+ add: (tx, items) => write(tx, items, "insert"),
43
+ remove: (tx, items) => write(tx, items, "delete"),
44
+ async candidates(db, branches, max) {
45
+ // A quoted string is a phrase of consecutive trigrams, i.e. an exact substring.
46
+ const quote = (literal: string) => `"${literal.replaceAll('"', '""')}"`;
47
+ const match = branches.map((branch) => `(${branch.map(quote).join(" AND ")})`).join(" OR ");
48
+ const rows = await db.query<{ id: number }>(
49
+ `SELECT rowid AS id FROM ${table} WHERE ${table} MATCH ? ORDER BY rowid LIMIT ?`,
50
+ [match, max],
51
+ );
52
+ return rows.map((row) => Number(row.id));
53
+ },
54
+ };
55
+ }
@@ -0,0 +1,12 @@
1
+ export { fold } from "./fold.js";
2
+ export {
3
+ type NgramHit,
4
+ type NgramItem,
5
+ type NgramMatch,
6
+ type NgramSearchOptions,
7
+ type NgramSearchResult,
8
+ NgramStore,
9
+ type NgramStoreOptions,
10
+ } from "./ngram-store.js";
11
+ export { MAX_PATTERN_LENGTH, planQuery, type QueryPlan } from "./plan-query.js";
12
+ export { trigrams } from "./trigrams.js";
@@ -0,0 +1,23 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import { chunkJson } from "./json-chunks.js";
3
+
4
+ describe("chunkJson", () => {
5
+ const rows = Array.from({ length: 30 }, (_, i) => [i, `t${i}`]);
6
+
7
+ it("splits into chunks of at most maxChars that rebuild the rows in order", () => {
8
+ const chunks = chunkJson(rows, 20);
9
+ expect(chunks.length).toBeGreaterThan(1);
10
+ for (const chunk of chunks) expect(chunk.length).toBeLessThanOrEqual(20);
11
+ expect(chunks.flatMap((chunk) => JSON.parse(chunk))).toEqual(rows);
12
+ });
13
+
14
+ it("keeps a row larger than maxChars as one chunk of its own", () => {
15
+ const big = [1, "x".repeat(50)];
16
+ const chunks = chunkJson([[0, "a"], big, [2, "b"]], 20);
17
+ expect(chunks).toEqual(['[[0,"a"]]', JSON.stringify([big]), '[[2,"b"]]']);
18
+ });
19
+
20
+ it("returns no chunks for no rows", () => {
21
+ expect(chunkJson([], 20)).toEqual([]);
22
+ });
23
+ });
@@ -0,0 +1,23 @@
1
+ /**
2
+ * Serialises rows into JSON array parameters for `json_each(?)`, each at most
3
+ * `maxChars` UTF-16 code units. 500k code units stay under a Durable Object's
4
+ * ~2 MB string limit even at 4 bytes per code point; a single row larger than
5
+ * that still fails there with "string or blob too big".
6
+ */
7
+ export function chunkJson(rows: unknown[], maxChars = 500_000): string[] {
8
+ const chunks: string[] = [];
9
+ let current: string[] = [];
10
+ let size = 2;
11
+ for (const row of rows) {
12
+ const json = JSON.stringify(row);
13
+ if (current.length > 0 && size + json.length + 1 > maxChars) {
14
+ chunks.push(`[${current.join(",")}]`);
15
+ current = [];
16
+ size = 2;
17
+ }
18
+ current.push(json);
19
+ size += json.length + 1;
20
+ }
21
+ if (current.length > 0) chunks.push(`[${current.join(",")}]`);
22
+ return chunks;
23
+ }