@statewalker/db-sqlite-index 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +289 -2
  3. package/dist/ngram.d.mts +93 -0
  4. package/dist/ngram.d.mts.map +1 -0
  5. package/dist/ngram.mjs +615 -0
  6. package/dist/ngram.mjs.map +1 -0
  7. package/dist/vector.d.mts +103 -0
  8. package/dist/vector.d.mts.map +1 -0
  9. package/dist/vector.mjs +1061 -0
  10. package/dist/vector.mjs.map +1 -0
  11. package/package.json +63 -3
  12. package/src/ngram/do-smoke.ts +46 -0
  13. package/src/ngram/fold.test.ts +50 -0
  14. package/src/ngram/fold.ts +21 -0
  15. package/src/ngram/fts5-backend.ts +55 -0
  16. package/src/ngram/index.ts +12 -0
  17. package/src/ngram/json-chunks.test.ts +23 -0
  18. package/src/ngram/json-chunks.ts +23 -0
  19. package/src/ngram/ngram-store.test.ts +291 -0
  20. package/src/ngram/ngram-store.ts +183 -0
  21. package/src/ngram/plan-query.test.ts +141 -0
  22. package/src/ngram/plan-query.ts +42 -0
  23. package/src/ngram/postings-backend.ts +53 -0
  24. package/src/ngram/regex-literals.ts +359 -0
  25. package/src/ngram/trigrams.test.ts +28 -0
  26. package/src/ngram/trigrams.ts +25 -0
  27. package/src/vector/auto-compact.test.ts +28 -0
  28. package/src/vector/do-smoke.ts +39 -0
  29. package/src/vector/index.ts +9 -0
  30. package/src/vector/kmeans.ts +131 -0
  31. package/src/vector/math.ts +185 -0
  32. package/src/vector/segment.test.ts +61 -0
  33. package/src/vector/segment.ts +256 -0
  34. package/src/vector/sql-segment-store.ts +218 -0
  35. package/src/vector/test-fixtures.ts +64 -0
  36. package/src/vector/types.ts +30 -0
  37. package/src/vector/vector-index.test.ts +105 -0
  38. package/src/vector/vector-index.ts +586 -0
  39. package/src/vector/vector-lifecycle.test.ts +171 -0
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ngram.mjs","names":[],"sources":["../src/ngram/fold.ts","../src/ngram/json-chunks.ts","../src/ngram/fts5-backend.ts","../src/ngram/regex-literals.ts","../src/ngram/plan-query.ts","../src/ngram/trigrams.ts","../src/ngram/postings-backend.ts","../src/ngram/ngram-store.ts"],"sourcesContent":["/**\n * Case fold per code point, applied to indexed text and to query literals alike.\n *\n * Upper-then-lower maps every character a case-insensitive JS regex treats as\n * equal to one form (Σ σ ς → σ, ſ → s). A character whose upper case is several\n * code points (ß → SS) only matches itself under `/i`, so it keeps its own\n * lower case. Folding per code point makes fold(a + b) === fold(a) + fold(b), so\n * a substring of the text stays a substring of the folded text. A character\n * whose folded form has a different UTF-16 length (İ → i + U+0307) is kept as\n * is, so fold preserves the length and offsets into the folded text are\n * offsets into the original.\n */\nexport function fold(text: string): string {\n let out = \"\";\n for (const ch of text) {\n const upper = ch.toUpperCase();\n const lower = (Array.from(upper).length === 1 ? upper : ch).toLowerCase();\n out += lower.length === ch.length ? lower : ch;\n }\n return out;\n}\n","/**\n * Serialises rows into JSON array parameters for `json_each(?)`, each at most\n * `maxChars` UTF-16 code units. 500k code units stay under a Durable Object's\n * ~2 MB string limit even at 4 bytes per code point; a single row larger than\n * that still fails there with \"string or blob too big\".\n */\nexport function chunkJson(rows: unknown[], maxChars = 500_000): string[] {\n const chunks: string[] = [];\n let current: string[] = [];\n let size = 2;\n for (const row of rows) {\n const json = JSON.stringify(row);\n if (current.length > 0 && size + json.length + 1 > maxChars) {\n chunks.push(`[${current.join(\",\")}]`);\n current = [];\n size = 2;\n }\n current.push(json);\n size += json.length + 1;\n }\n if (current.length > 0) chunks.push(`[${current.join(\",\")}]`);\n return chunks;\n}\n","import type { Db } from \"@statewalker/db-api\";\nimport { fold } from \"./fold.js\";\nimport { chunkJson } from \"./json-chunks.js\";\n\nexport type NgramItem = { id: number; text: string };\n\n/** How one store finds candidate ids; texts and verification live in NgramStore. */\nexport type GramBackend = {\n create(tx: Db): Promise<void>;\n add(tx: Db, items: NgramItem[]): Promise<void>;\n /** `items` hold the texts as currently stored, so their grams can be removed exactly. */\n remove(tx: Db, items: NgramItem[]): Promise<void>;\n /** Ids whose text contains every literal of at least one branch; ascending, at most `max`. */\n candidates(db: Db, branches: string[][], max: number): Promise<number[]>;\n};\n\nconst ROW = \"json_extract(value, '$[0]'), json_extract(value, '$[1]')\";\n\n/**\n * FTS5 trigram over folded text. Contentless (`content=''`), because NgramStore\n * already keeps the text; a contentless row is deleted by replaying its exact\n * indexed value through the 'delete' command, which is why `remove` needs the\n * stored texts. `case_sensitive 1`: folding is done by `fold`, the same function\n * that folds query literals, so the index and the planner can never disagree.\n */\nexport function newFts5Backend(prefix: string): GramBackend {\n const table = `${prefix}ng`;\n const write = async (tx: Db, items: NgramItem[], command: \"insert\" | \"delete\") => {\n const sql =\n command === \"insert\"\n ? `INSERT INTO ${table}(rowid, text) SELECT ${ROW} FROM json_each(?)`\n : `INSERT INTO ${table}(${table}, rowid, text) SELECT 'delete', ${ROW} FROM json_each(?)`;\n for (const chunk of chunkJson(items.map((item) => [item.id, fold(item.text)]))) {\n await tx.query(sql, [chunk]);\n }\n };\n return {\n create: (tx) =>\n tx.exec(\n `CREATE VIRTUAL TABLE IF NOT EXISTS ${table} USING fts5(text, content='', tokenize='trigram case_sensitive 1')`,\n ),\n add: (tx, items) => write(tx, items, \"insert\"),\n remove: (tx, items) => write(tx, items, \"delete\"),\n async candidates(db, branches, max) {\n // A quoted string is a phrase of consecutive trigrams, i.e. an exact substring.\n const quote = (literal: string) => `\"${literal.replaceAll('\"', '\"\"')}\"`;\n const match = branches.map((branch) => `(${branch.map(quote).join(\" AND \")})`).join(\" OR \");\n const rows = await db.query<{ id: number }>(\n `SELECT rowid AS id FROM ${table} WHERE ${table} MATCH ? ORDER BY rowid LIMIT ?`,\n [match, max],\n );\n return rows.map((row) => Number(row.id));\n },\n };\n}\n","/*\n * Mandatory-literal extraction from a regex, after Russ Cox's trigram method.\n * Ported from hypgrep 0.5.3 src/regex.js (Hyperparam), MIT License:\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this\n * software and associated documentation files (the \"Software\"), to deal in the Software\n * without restriction, including without limitation the rights to use, copy, modify,\n * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to\n * permit persons to whom the Software is furnished to do so, subject to the following\n * conditions: The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \"AS IS\",\n * WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED.\n */\n\n/** Cross-product cap: past it, the construct counts as a wildcard. */\nconst MAX_DNF = 32;\n\ntype LiveBranch = { literals: string[]; run: string };\ntype Quantifier = { copies: number; openEnded: boolean; end: number };\ntype GroupPrefix = { kind: \"consuming\"; innerStart: number } | { kind: \"assertion\" | \"unknown\" };\n\n/**\n * DNF of literals that every match must contain: a string can match only if all\n * literals of at least one branch occur in it. An empty branch matches anything.\n * Conservative: a construct it cannot reason about becomes a wildcard, never a literal.\n *\n * /abc(def|ghi)/ → [[\"abcdef\"], [\"abcghi\"]] /foo|.+/ → [[\"foo\"], []]\n */\nexport function regexLiterals(regex: RegExp): string[][] {\n // v-mode class syntax desyncs skipClass and would invent literals.\n if (regex.flags.includes(\"v\")) return [[]];\n return expandRegion(regex.source, 0, regex.source.length).dnf;\n}\n\n/** `exact`: every arm is one contiguous literal that is the arm's whole match, with nothing dropped. */\ntype Region = { dnf: string[][]; exact: boolean };\n\nfunction expandRegion(src: string, start: number, end: number): Region {\n const dnf: string[][] = [];\n let exact = true;\n for (const [bStart, bEnd] of splitTopLevelAlternation(src, start, end)) {\n const arm = expandBranch(src, bStart, bEnd);\n for (const branch of arm.dnf) dnf.push(branch);\n exact = exact && arm.exact;\n }\n return { dnf, exact };\n}\n\nfunction splitTopLevelAlternation(\n src: string,\n start: number,\n end: number,\n): Array<[number, number]> {\n const spans: Array<[number, number]> = [];\n let bStart = start;\n let depth = 0;\n let i = start;\n while (i < end) {\n const c = src[i];\n if (c === \"\\\\\") {\n i += 2;\n continue;\n }\n if (c === \"[\") {\n i = skipClass(src, i);\n continue;\n }\n if (c === \"(\") depth += 1;\n else if (c === \")\") depth -= 1;\n else if (c === \"|\" && depth === 0) {\n spans.push([bStart, i]);\n bStart = i + 1;\n }\n i += 1;\n }\n spans.push([bStart, end]);\n return spans;\n}\n\nfunction expandBranch(src: string, start: number, end: number): Region {\n let cross: LiveBranch[] = [{ literals: [], run: \"\" }];\n // Set when part of the match is not captured in a literal (wildcard, optional or repeated\n // atom, assertion): gluing this branch to neighbouring text would be unsound.\n let lossy = false;\n const flushRuns = () => {\n for (const b of cross) {\n if (b.run.length > 0) {\n b.literals.push(b.run);\n b.run = \"\";\n }\n }\n };\n // Append `text` repeated by the quantifier's mandatory copies to every live run.\n const appendRun = (text: string, q: Quantifier) => {\n if (q.copies === 0) {\n lossy = true;\n flushRuns();\n return;\n }\n for (const b of cross) b.run += text.repeat(q.copies);\n if (q.openEnded) {\n lossy = true;\n flushRuns();\n }\n };\n\n let i = start;\n while (i < end) {\n const c = src[i] ?? \"\";\n\n if (c === \"^\" || c === \"$\") {\n i += 1;\n continue;\n }\n\n if (c === \"\\\\\") {\n const next = src[i + 1];\n if (next === undefined) {\n i += 1;\n continue;\n }\n const escaped = parseEscapedLiteral(src, i);\n if (escaped) {\n const q = peekQuantifier(src, escaped.end);\n i = q.end;\n appendRun(escaped.char, q);\n continue;\n }\n // \\d \\w \\s \\b, backrefs, \\p{…}: a wildcard; consume its payload.\n if (/[a-zA-Z0-9]/.test(next)) {\n lossy = true;\n flushRuns();\n i = peekQuantifier(src, skipSpecialEscape(src, i)).end;\n continue;\n }\n // \\. \\/ \\( …: the literal character.\n const q = peekQuantifier(src, i + 2);\n i = q.end;\n appendRun(next, q);\n continue;\n }\n\n if (c === \"[\") {\n const classEnd = skipClass(src, i);\n const chars = parseLiteralClass(src, i, classEnd);\n const q = peekQuantifier(src, classEnd);\n i = q.end;\n if (\n q.copies === 0 ||\n chars === null ||\n q.copies > 1 ||\n cross.length * chars.length > MAX_DNF\n ) {\n lossy = true;\n flushRuns();\n continue;\n }\n cross = cross.flatMap((b) =>\n chars.map((ch) => ({ literals: b.literals.slice(), run: b.run + ch })),\n );\n if (q.openEnded) {\n lossy = true;\n flushRuns();\n }\n continue;\n }\n\n if (c === \"(\") {\n const groupEnd = skipGroup(src, i);\n const group = parseGroupPrefix(src, i);\n const q = peekQuantifier(src, groupEnd);\n i = q.end;\n if (group.kind !== \"consuming\" || q.copies === 0) {\n lossy = true;\n flushRuns();\n continue;\n }\n const { dnf: inner, exact } = expandRegion(src, group.innerStart, groupEnd - 1);\n // Fold into the run only when every arm is one contiguous literal used once and is the\n // arm's whole match: a repeated group (`(a|b){3}`) may pick different arms per copy, and\n // a wildcard at an arm's edge (`x(.abc)`) must not be glued away.\n const foldable =\n exact && q.copies === 1 && inner.length > 0 && inner.every((b) => b.length === 1);\n lossy = lossy || q.openEnded || !foldable;\n if (foldable && cross.length * inner.length <= MAX_DNF) {\n cross = cross.flatMap((b) =>\n inner.map((arm) => ({ literals: b.literals.slice(), run: b.run + (arm[0] ?? \"\") })),\n );\n if (q.openEnded) flushRuns();\n continue;\n }\n lossy = true;\n flushRuns();\n if (cross.length * inner.length > MAX_DNF) continue;\n cross = cross.flatMap((b) =>\n inner.map((arm) => ({ literals: b.literals.concat(arm), run: \"\" })),\n );\n continue;\n }\n\n if (c === \".\") {\n lossy = true;\n flushRuns();\n i = peekQuantifier(src, i + 1).end;\n continue;\n }\n\n const q = peekQuantifier(src, i + 1);\n i = q.end;\n appendRun(c, q);\n }\n flushRuns();\n return {\n dnf: cross.map((b) => b.literals),\n exact: !lossy && cross.every((b) => b.literals.length === 1),\n };\n}\n\n/** Where a group's consuming body starts; lookarounds and unknown `(?…)` are not consuming. */\nfunction parseGroupPrefix(src: string, start: number): GroupPrefix {\n if (src[start + 1] !== \"?\") return { kind: \"consuming\", innerStart: start + 1 };\n const marker = src[start + 2];\n if (marker === \":\") return { kind: \"consuming\", innerStart: start + 3 };\n if (marker === \"=\" || marker === \"!\") return { kind: \"assertion\" };\n if (marker === \"<\") {\n const next = src[start + 3];\n if (next === \"=\" || next === \"!\") return { kind: \"assertion\" };\n const nameEnd = src.indexOf(\">\", start + 3);\n if (nameEnd >= 0) return { kind: \"consuming\", innerStart: nameEnd + 1 };\n }\n return { kind: \"unknown\" };\n}\n\n/**\n * `\\xHH`, `\\uHHHH`: escapes that stand for one literal character. `\\u{…}` is not one:\n * the planner compiles without the `u` flag, where it means \"u\" repeated or the text \"u{…}\".\n */\nfunction parseEscapedLiteral(\n src: string,\n start: number,\n): { char: string; end: number } | undefined {\n const kind = src[start + 1];\n if (kind === \"x\") {\n const hex = src.slice(start + 2, start + 4);\n if (!/^[0-9a-fA-F]{2}$/.test(hex)) return undefined;\n return { char: String.fromCharCode(Number.parseInt(hex, 16)), end: start + 4 };\n }\n if (kind === \"u\") {\n const hex = src.slice(start + 2, start + 6);\n if (!/^[0-9a-fA-F]{4}$/.test(hex)) return undefined;\n return { char: String.fromCharCode(Number.parseInt(hex, 16)), end: start + 6 };\n }\n return undefined;\n}\n\n/**\n * Skip an escape that is not a literal punctuation character, so its payload\n * (`\\x41`, `\\p{L}`, `\\k<name>`, backrefs) never leaks into a literal. Malformed\n * `\\x`, `\\u`, `\\c` payloads are Annex-B literals and are rescanned as atoms.\n */\nfunction skipSpecialEscape(src: string, start: number): number {\n const next = src[start + 1];\n if (next === \"x\") {\n return /^[0-9a-fA-F]{2}$/.test(src.slice(start + 2, start + 4)) ? start + 4 : start + 2;\n }\n if (next === \"u\") {\n if (src[start + 2] === \"{\") {\n const close = src.indexOf(\"}\", start + 3);\n return close >= 0 ? close + 1 : src.length;\n }\n return /^[0-9a-fA-F]{4}$/.test(src.slice(start + 2, start + 6)) ? start + 6 : start + 2;\n }\n if (next === \"c\") return /^[a-zA-Z]$/.test(src[start + 2] ?? \"\") ? start + 3 : start + 2;\n if ((next === \"p\" || next === \"P\") && src[start + 2] === \"{\") {\n const close = src.indexOf(\"}\", start + 3);\n return close >= 0 ? close + 1 : src.length;\n }\n if (next === \"k\" && src[start + 2] === \"<\") {\n const close = src.indexOf(\">\", start + 3);\n return close >= 0 ? close + 1 : src.length;\n }\n if (/[0-9]/.test(next ?? \"\")) {\n let i = start + 2;\n while (/[0-9]/.test(src[i] ?? \"\")) i += 1;\n return i;\n }\n return Math.min(start + 2, src.length);\n}\n\n/** A `[...]` class as literal alternatives, or null if negated, ranged, special or empty. */\nfunction parseLiteralClass(src: string, start: number, end: number): string[] | null {\n let i = start + 1;\n const lastInside = end - 1;\n if (src[i] === \"^\") return null;\n const chars = new Set<string>();\n while (i < lastInside) {\n const c = src[i] ?? \"\";\n if (c === \"\\\\\") {\n const next = src[i + 1];\n if (next === undefined || /[a-zA-Z]/.test(next)) return null;\n chars.add(next);\n i += 2;\n continue;\n }\n if (i + 2 < lastInside && src[i + 1] === \"-\") return null;\n chars.add(c);\n i += 1;\n }\n // A surrogate half would split an astral character the index sees as one code point.\n if ([...chars].some((ch) => /[\\ud800-\\udfff]/.test(ch))) return null;\n return chars.size === 0 ? null : [...chars];\n}\n\n/** Mandatory copies of the preceding atom, whether more may follow, and where the quantifier ends. */\nfunction peekQuantifier(src: string, i: number): Quantifier {\n const c = src[i];\n if (c === \"?\" || c === \"*\" || c === \"+\") {\n const end = src[i + 1] === \"?\" ? i + 2 : i + 1;\n return { copies: c === \"+\" ? 1 : 0, openEnded: true, end };\n }\n if (c === \"{\") {\n const close = src.indexOf(\"}\", i);\n if (close < 0) return { copies: 1, openEnded: false, end: i };\n const m = /^(\\d+)(?:,(\\d*))?$/.exec(src.slice(i + 1, close));\n if (!m) return { copies: 1, openEnded: false, end: i };\n const min = Number(m[1]);\n const max = m[2] === undefined ? min : m[2] === \"\" ? Number.POSITIVE_INFINITY : Number(m[2]);\n const end = src[close + 1] === \"?\" ? close + 2 : close + 1;\n return { copies: min, openEnded: max > min, end };\n }\n return { copies: 1, openEnded: false, end: i };\n}\n\nfunction skipClass(src: string, i: number): number {\n let j = i + 1;\n if (src[j] === \"^\") j += 1;\n while (j < src.length && src[j] !== \"]\") j += src[j] === \"\\\\\" && j + 1 < src.length ? 2 : 1;\n return j < src.length ? j + 1 : j;\n}\n\nfunction skipGroup(src: string, i: number): number {\n let depth = 1;\n let j = i + 1;\n while (j < src.length && depth > 0) {\n const c = src[j];\n if (c === \"\\\\\" && j + 1 < src.length) {\n j += 2;\n continue;\n }\n if (c === \"[\") {\n j = skipClass(src, j);\n continue;\n }\n if (c === \"(\") depth += 1;\n else if (c === \")\") depth -= 1;\n j += 1;\n }\n return j;\n}\n","import { fold } from \"./fold.js\";\nimport { regexLiterals } from \"./regex-literals.js\";\n\n/** Longer patterns are rejected: a JS regex cannot be interrupted once it runs. */\nexport const MAX_PATTERN_LENGTH = 1000;\n\nexport type QueryPlan = {\n /** The pattern as a global regex (`i` added when not case-sensitive); verifies candidates. */\n regex: RegExp;\n /**\n * Per DNF branch, the folded literals of at least 3 code points that a match must\n * contain. `null` when some branch has none: only a scan can answer the pattern.\n */\n branches: string[][] | null;\n};\n\nexport function planQuery(\n pattern: string,\n opts: { regex?: boolean; caseSensitive?: boolean } = {},\n): QueryPlan {\n if (pattern.length > MAX_PATTERN_LENGTH) {\n throw new RangeError(\n `ngram: pattern is ${pattern.length} characters, the limit is ${MAX_PATTERN_LENGTH}`,\n );\n }\n // The regex has no `u` flag (case folding is unsound under /iu), so `\\p{…}` would silently\n // match the literal text \"p{…}\". An odd run of backslashes before `p` is a real escape.\n if (opts.regex && /(?<!\\\\)(?:\\\\\\\\)*\\\\[pP]\\{/.test(pattern)) {\n throw new SyntaxError(\n \"ngram: \\\\p{…} and \\\\P{…} are not supported in regex mode (patterns compile without the u flag)\",\n );\n }\n const source = opts.regex ? pattern : pattern.replace(/[.*+?^${}()|[\\]\\\\/]/g, \"\\\\$&\");\n const regex = new RegExp(source, opts.caseSensitive === false ? \"gi\" : \"g\");\n const branches: string[][] = [];\n for (const literals of regexLiterals(regex)) {\n const usable = literals.map(fold).filter((literal) => Array.from(literal).length >= 3);\n if (usable.length === 0) return { regex, branches: null };\n branches.push(usable);\n }\n return { regex, branches };\n}\n","import { fold } from \"./fold.js\";\n\n/** Distinct 3-code-point grams of an already folded string, in first-seen order. */\nexport function gramsOf(folded: string): string[] {\n const cps = Array.from(folded);\n const out = new Set<string>();\n for (let i = 0; i + 3 <= cps.length; i++) out.add(cps.slice(i, i + 3).join(\"\"));\n return [...out];\n}\n\n/** Distinct trigrams of `text` as the index sees it: case-folded, by code point. */\nexport function trigrams(text: string): string[] {\n return gramsOf(fold(text));\n}\n\n/**\n * One SQLite INTEGER per gram. Three BMP code points pack exactly into 48 bits.\n * A gram with an astral code point keeps the low 16 bits of each code point\n * above 2^48: a collision only adds candidates, which verification removes.\n */\nexport function encodeGram(gram: string): number {\n const [a = 0, b = 0, c = 0] = Array.from(gram, (ch) => ch.codePointAt(0) ?? 0);\n if (a < 0x10000 && b < 0x10000 && c < 0x10000) return a * 2 ** 32 + b * 2 ** 16 + c;\n return 2 ** 48 + (a % 0x10000) * 2 ** 32 + (b % 0x10000) * 2 ** 16 + (c % 0x10000);\n}\n","import type { GramBackend, NgramItem } from \"./fts5-backend.js\";\nimport { chunkJson } from \"./json-chunks.js\";\nimport { encodeGram, gramsOf, trigrams } from \"./trigrams.js\";\n\nconst PAIR = \"json_extract(value, '$[0]'), json_extract(value, '$[1]')\";\n\n/**\n * One row per (gram, id) in a WITHOUT ROWID table: the primary key is the\n * posting list, clustered by gram. A text is a candidate for a branch when it\n * holds every distinct gram of the branch's literals.\n */\nexport function newPostingsBackend(prefix: string): GramBackend {\n const table = `${prefix}grams`;\n const pairs = (items: NgramItem[]) =>\n chunkJson(\n items.flatMap((item) => trigrams(item.text).map((gram) => [encodeGram(gram), item.id])),\n );\n return {\n create: (tx) =>\n tx.exec(\n `CREATE TABLE IF NOT EXISTS ${table}(gram INTEGER NOT NULL, id INTEGER NOT NULL, PRIMARY KEY (gram, id)) WITHOUT ROWID`,\n ),\n async add(tx, items) {\n for (const chunk of pairs(items)) {\n await tx.query(\n `INSERT OR IGNORE INTO ${table}(gram, id) SELECT ${PAIR} FROM json_each(?)`,\n [chunk],\n );\n }\n },\n async remove(tx, items) {\n for (const chunk of pairs(items)) {\n await tx.query(\n `DELETE FROM ${table} WHERE (gram, id) IN (SELECT ${PAIR} FROM json_each(?))`,\n [chunk],\n );\n }\n },\n async candidates(db, branches, max) {\n const ids = new Set<number>();\n for (const branch of branches) {\n // Literals are already folded: split them without folding again.\n const grams = [...new Set(branch.flatMap((literal) => gramsOf(literal).map(encodeGram)))];\n const rows = await db.query<{ id: number }>(\n `SELECT id FROM ${table} WHERE gram IN (SELECT value FROM json_each(?)) GROUP BY id HAVING count(*) = ? ORDER BY id LIMIT ?`,\n [JSON.stringify(grams), grams.length, max],\n );\n for (const row of rows) ids.add(Number(row.id));\n }\n return [...ids].sort((a, b) => a - b).slice(0, max);\n },\n };\n}\n","import type { Db } from \"@statewalker/db-api\";\nimport { inTransaction } from \"@statewalker/db-sqlite\";\nimport { type GramBackend, type NgramItem, newFts5Backend } from \"./fts5-backend.js\";\nimport { chunkJson } from \"./json-chunks.js\";\nimport { planQuery } from \"./plan-query.js\";\nimport { newPostingsBackend } from \"./postings-backend.js\";\n\nexport type { NgramItem };\nexport type NgramMatch = { start: number; end: number };\nexport type NgramHit = { id: number; matches: NgramMatch[] };\nexport type NgramSearchOptions = {\n /** Treat `pattern` as a JS regex source; otherwise it is literal text. */\n regex?: boolean;\n /** Default true (grep semantics). */\n caseSensitive?: boolean;\n /** Maximum hits. Default 100. */\n limit?: number;\n /** Maximum candidates verified. Default 10,000. */\n candidateBudget?: number;\n /** Keep only ids for which this returns true (applied before verification). */\n filter?: (id: number) => boolean;\n signal?: AbortSignal;\n};\nexport type NgramSearchResult = {\n /** Ascending id order; every non-empty match, offsets in UTF-16 code units. */\n hits: NgramHit[];\n /** True when the search stopped before checking every candidate (budget or limit). */\n truncated: boolean;\n};\nexport type NgramStoreOptions = { db: Db; prefix: string; store?: \"fts5\" | \"postings\" };\n\nconst VERIFY_BATCH = 256;\n\nexport class NgramStore {\n /** `{prefix}texts`: its `id` column holds exactly the stored ids. Callers may join it in SQL. */\n readonly idTable: string;\n readonly #db: Db;\n readonly #prefix: string;\n readonly #store: \"fts5\" | \"postings\";\n readonly #backend: GramBackend;\n\n constructor({ db, prefix, store = \"fts5\" }: NgramStoreOptions) {\n if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(prefix)) {\n throw new Error(`ngram: prefix \"${prefix}\" must match /^[A-Za-z_][A-Za-z0-9_]*$/`);\n }\n if (store !== \"fts5\" && store !== \"postings\")\n throw new Error(`ngram: unknown store \"${store}\"`);\n this.#db = db;\n this.#prefix = prefix;\n this.idTable = `${prefix}texts`;\n this.#store = store;\n this.#backend = store === \"fts5\" ? newFts5Backend(prefix) : newPostingsBackend(prefix);\n }\n\n async init(): Promise<void> {\n const p = this.#prefix;\n await inTransaction(this.#db, async (tx) => {\n await tx.exec(\n `CREATE TABLE IF NOT EXISTS ${p}meta(key TEXT PRIMARY KEY, value TEXT NOT NULL)`,\n );\n const [row] = await tx.query<{ value: string }>(\n `SELECT value FROM ${p}meta WHERE key = 'store'`,\n );\n if (row && row.value !== this.#store) {\n throw new Error(\n `ngram: index \"${p}\" was created with store \"${row.value}\", opened with \"${this.#store}\"`,\n );\n }\n if (!row)\n await tx.query(`INSERT INTO ${p}meta(key, value) VALUES ('store', ?)`, [this.#store]);\n await tx.exec(\n `CREATE TABLE IF NOT EXISTS ${p}texts(id INTEGER PRIMARY KEY, text TEXT NOT NULL)`,\n );\n await this.#backend.create(tx);\n });\n }\n\n async upsert(items: NgramItem[]): Promise<void> {\n // The last text per id wins; a repeated id would leave stale grams behind.\n // A lone surrogate would be stored differently from how it was indexed, so a\n // later delete could not replay it: texts are made well-formed first.\n const latest = [\n ...new Map(items.map(({ id, text }) => [id, { id, text: text.toWellFormed() }])).values(),\n ];\n if (latest.length === 0) return;\n await inTransaction(this.#db, async (tx) => {\n await this.#backend.remove(\n tx,\n await this.#load(\n tx,\n latest.map((item) => item.id),\n ),\n );\n for (const chunk of chunkJson(latest.map((item) => [item.id, item.text]))) {\n await tx.query(\n `INSERT OR REPLACE INTO ${this.#prefix}texts(id, text) SELECT json_extract(value, '$[0]'), json_extract(value, '$[1]') FROM json_each(?)`,\n [chunk],\n );\n }\n await this.#backend.add(tx, latest);\n });\n }\n\n async delete(ids: number[]): Promise<void> {\n if (ids.length === 0) return;\n await inTransaction(this.#db, async (tx) => {\n await this.#backend.remove(tx, await this.#load(tx, ids));\n for (const chunk of chunkJson(ids)) {\n await tx.query(\n `DELETE FROM ${this.#prefix}texts WHERE id IN (SELECT value FROM json_each(?))`,\n [chunk],\n );\n }\n });\n }\n\n async search(pattern: string, opts: NgramSearchOptions = {}): Promise<NgramSearchResult> {\n const { limit = 100, candidateBudget = 10_000, filter, signal } = opts;\n const plan = planQuery(pattern, {\n regex: opts.regex,\n caseSensitive: opts.caseSensitive ?? true,\n });\n signal?.throwIfAborted();\n if (limit <= 0) return { hits: [], truncated: true };\n const found =\n plan.branches === null\n ? (\n await this.#db.query<{ id: number }>(\n `SELECT id FROM ${this.#prefix}texts ORDER BY id LIMIT ?`,\n [candidateBudget + 1],\n )\n ).map((row) => Number(row.id))\n : await this.#backend.candidates(this.#db, plan.branches, candidateBudget + 1);\n let truncated = found.length > candidateBudget;\n const ids = found.slice(0, candidateBudget).filter((id) => !filter || filter(id));\n\n const hits: NgramHit[] = [];\n let last: number | undefined;\n verify: for (let i = 0; i < ids.length; i += VERIFY_BATCH) {\n signal?.throwIfAborted();\n for (const { id, text } of await this.#load(this.#db, ids.slice(i, i + VERIFY_BATCH))) {\n last = id;\n const matches: NgramMatch[] = [];\n for (const m of text.matchAll(plan.regex)) {\n if (m[0].length > 0) matches.push({ start: m.index, end: m.index + m[0].length });\n }\n if (matches.length === 0) continue;\n hits.push({ id, matches });\n if (hits.length >= limit) break verify;\n }\n }\n if (hits.length >= limit && last !== ids.at(-1)) truncated = true;\n return { hits, truncated };\n }\n\n /** The stored original texts of `ids`; unknown ids are absent. */\n async getTexts(ids: number[]): Promise<Map<number, string>> {\n const items = await this.#load(this.#db, ids);\n return new Map(items.map((item) => [item.id, item.text]));\n }\n\n /** Drops the tables of both stores, so it works without knowing which one created the prefix. */\n async drop(): Promise<void> {\n const p = this.#prefix;\n await inTransaction(this.#db, async (tx) => {\n for (const t of [\"ng\", \"grams\", \"texts\", \"meta\"])\n await tx.exec(`DROP TABLE IF EXISTS ${p}${t}`);\n });\n }\n\n /** Stored texts for `ids`, ascending by id; missing ids are skipped. */\n async #load(db: Db, ids: number[]): Promise<NgramItem[]> {\n const out: NgramItem[] = [];\n for (const chunk of chunkJson([...ids].sort((a, b) => a - b))) {\n const rows = await db.query<{ id: number; text: string }>(\n `SELECT id, text FROM ${this.#prefix}texts WHERE id IN (SELECT value FROM json_each(?)) ORDER BY id`,\n [chunk],\n );\n for (const row of rows) out.push({ id: Number(row.id), text: String(row.text) });\n }\n return out;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;AAYA,SAAgB,KAAK,MAAsB;CACzC,IAAI,MAAM;CACV,KAAK,MAAM,MAAM,MAAM;EACrB,MAAM,QAAQ,GAAG,YAAY;EAC7B,MAAM,SAAS,MAAM,KAAK,KAAK,CAAC,CAAC,WAAW,IAAI,QAAQ,GAAA,CAAI,YAAY;EACxE,OAAO,MAAM,WAAW,GAAG,SAAS,QAAQ;CAC9C;CACA,OAAO;AACT;;;;;;;;;ACdA,SAAgB,UAAU,MAAiB,WAAW,KAAmB;CACvE,MAAM,SAAmB,CAAC;CAC1B,IAAI,UAAoB,CAAC;CACzB,IAAI,OAAO;CACX,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,OAAO,KAAK,UAAU,GAAG;EAC/B,IAAI,QAAQ,SAAS,KAAK,OAAO,KAAK,SAAS,IAAI,UAAU;GAC3D,OAAO,KAAK,IAAI,QAAQ,KAAK,GAAG,EAAE,EAAE;GACpC,UAAU,CAAC;GACX,OAAO;EACT;EACA,QAAQ,KAAK,IAAI;EACjB,QAAQ,KAAK,SAAS;CACxB;CACA,IAAI,QAAQ,SAAS,GAAG,OAAO,KAAK,IAAI,QAAQ,KAAK,GAAG,EAAE,EAAE;CAC5D,OAAO;AACT;;;ACNA,MAAM,MAAM;;;;;;;;AASZ,SAAgB,eAAe,QAA6B;CAC1D,MAAM,QAAQ,GAAG,OAAO;CACxB,MAAM,QAAQ,OAAO,IAAQ,OAAoB,YAAiC;EAChF,MAAM,MACJ,YAAY,WACR,eAAe,MAAM,uBAAuB,IAAI,sBAChD,eAAe,MAAM,GAAG,MAAM,kCAAkC,IAAI;EAC1E,KAAK,MAAM,SAAS,UAAU,MAAM,KAAK,SAAS,CAAC,KAAK,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,GAC3E,MAAM,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC;CAE/B;CACA,OAAO;EACL,SAAS,OACP,GAAG,KACD,sCAAsC,MAAM,mEAC9C;EACF,MAAM,IAAI,UAAU,MAAM,IAAI,OAAO,QAAQ;EAC7C,SAAS,IAAI,UAAU,MAAM,IAAI,OAAO,QAAQ;EAChD,MAAM,WAAW,IAAI,UAAU,KAAK;GAElC,MAAM,SAAS,YAAoB,IAAI,QAAQ,WAAW,MAAK,MAAI,EAAE;GACrE,MAAM,QAAQ,SAAS,KAAK,WAAW,IAAI,OAAO,IAAI,KAAK,CAAC,CAAC,KAAK,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM;GAK1F,QAAO,MAJY,GAAG,MACpB,2BAA2B,MAAM,SAAS,MAAM,kCAChD,CAAC,OAAO,GAAG,CACb,EAAA,CACY,KAAK,QAAQ,OAAO,IAAI,EAAE,CAAC;EACzC;CACF;AACF;;;;ACvCA,MAAM,UAAU;;;;;;;;AAahB,SAAgB,cAAc,OAA2B;CAEvD,IAAI,MAAM,MAAM,SAAS,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC;CACzC,OAAO,aAAa,MAAM,QAAQ,GAAG,MAAM,OAAO,MAAM,CAAC,CAAC;AAC5D;AAKA,SAAS,aAAa,KAAa,OAAe,KAAqB;CACrE,MAAM,MAAkB,CAAC;CACzB,IAAI,QAAQ;CACZ,KAAK,MAAM,CAAC,QAAQ,SAAS,yBAAyB,KAAK,OAAO,GAAG,GAAG;EACtE,MAAM,MAAM,aAAa,KAAK,QAAQ,IAAI;EAC1C,KAAK,MAAM,UAAU,IAAI,KAAK,IAAI,KAAK,MAAM;EAC7C,QAAQ,SAAS,IAAI;CACvB;CACA,OAAO;EAAE;EAAK;CAAM;AACtB;AAEA,SAAS,yBACP,KACA,OACA,KACyB;CACzB,MAAM,QAAiC,CAAC;CACxC,IAAI,SAAS;CACb,IAAI,QAAQ;CACZ,IAAI,IAAI;CACR,OAAO,IAAI,KAAK;EACd,MAAM,IAAI,IAAI;EACd,IAAI,MAAM,MAAM;GACd,KAAK;GACL;EACF;EACA,IAAI,MAAM,KAAK;GACb,IAAI,UAAU,KAAK,CAAC;GACpB;EACF;EACA,IAAI,MAAM,KAAK,SAAS;OACnB,IAAI,MAAM,KAAK,SAAS;OACxB,IAAI,MAAM,OAAO,UAAU,GAAG;GACjC,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;GACtB,SAAS,IAAI;EACf;EACA,KAAK;CACP;CACA,MAAM,KAAK,CAAC,QAAQ,GAAG,CAAC;CACxB,OAAO;AACT;AAEA,SAAS,aAAa,KAAa,OAAe,KAAqB;CACrE,IAAI,QAAsB,CAAC;EAAE,UAAU,CAAC;EAAG,KAAK;CAAG,CAAC;CAGpD,IAAI,QAAQ;CACZ,MAAM,kBAAkB;EACtB,KAAK,MAAM,KAAK,OACd,IAAI,EAAE,IAAI,SAAS,GAAG;GACpB,EAAE,SAAS,KAAK,EAAE,GAAG;GACrB,EAAE,MAAM;EACV;CAEJ;CAEA,MAAM,aAAa,MAAc,MAAkB;EACjD,IAAI,EAAE,WAAW,GAAG;GAClB,QAAQ;GACR,UAAU;GACV;EACF;EACA,KAAK,MAAM,KAAK,OAAO,EAAE,OAAO,KAAK,OAAO,EAAE,MAAM;EACpD,IAAI,EAAE,WAAW;GACf,QAAQ;GACR,UAAU;EACZ;CACF;CAEA,IAAI,IAAI;CACR,OAAO,IAAI,KAAK;EACd,MAAM,IAAI,IAAI,MAAM;EAEpB,IAAI,MAAM,OAAO,MAAM,KAAK;GAC1B,KAAK;GACL;EACF;EAEA,IAAI,MAAM,MAAM;GACd,MAAM,OAAO,IAAI,IAAI;GACrB,IAAI,SAAS,KAAA,GAAW;IACtB,KAAK;IACL;GACF;GACA,MAAM,UAAU,oBAAoB,KAAK,CAAC;GAC1C,IAAI,SAAS;IACX,MAAM,IAAI,eAAe,KAAK,QAAQ,GAAG;IACzC,IAAI,EAAE;IACN,UAAU,QAAQ,MAAM,CAAC;IACzB;GACF;GAEA,IAAI,cAAc,KAAK,IAAI,GAAG;IAC5B,QAAQ;IACR,UAAU;IACV,IAAI,eAAe,KAAK,kBAAkB,KAAK,CAAC,CAAC,CAAC,CAAC;IACnD;GACF;GAEA,MAAM,IAAI,eAAe,KAAK,IAAI,CAAC;GACnC,IAAI,EAAE;GACN,UAAU,MAAM,CAAC;GACjB;EACF;EAEA,IAAI,MAAM,KAAK;GACb,MAAM,WAAW,UAAU,KAAK,CAAC;GACjC,MAAM,QAAQ,kBAAkB,KAAK,GAAG,QAAQ;GAChD,MAAM,IAAI,eAAe,KAAK,QAAQ;GACtC,IAAI,EAAE;GACN,IACE,EAAE,WAAW,KACb,UAAU,QACV,EAAE,SAAS,KACX,MAAM,SAAS,MAAM,SAAS,SAC9B;IACA,QAAQ;IACR,UAAU;IACV;GACF;GACA,QAAQ,MAAM,SAAS,MACrB,MAAM,KAAK,QAAQ;IAAE,UAAU,EAAE,SAAS,MAAM;IAAG,KAAK,EAAE,MAAM;GAAG,EAAE,CACvE;GACA,IAAI,EAAE,WAAW;IACf,QAAQ;IACR,UAAU;GACZ;GACA;EACF;EAEA,IAAI,MAAM,KAAK;GACb,MAAM,WAAW,UAAU,KAAK,CAAC;GACjC,MAAM,QAAQ,iBAAiB,KAAK,CAAC;GACrC,MAAM,IAAI,eAAe,KAAK,QAAQ;GACtC,IAAI,EAAE;GACN,IAAI,MAAM,SAAS,eAAe,EAAE,WAAW,GAAG;IAChD,QAAQ;IACR,UAAU;IACV;GACF;GACA,MAAM,EAAE,KAAK,OAAO,UAAU,aAAa,KAAK,MAAM,YAAY,WAAW,CAAC;GAI9E,MAAM,WACJ,SAAS,EAAE,WAAW,KAAK,MAAM,SAAS,KAAK,MAAM,OAAO,MAAM,EAAE,WAAW,CAAC;GAClF,QAAQ,SAAS,EAAE,aAAa,CAAC;GACjC,IAAI,YAAY,MAAM,SAAS,MAAM,UAAU,SAAS;IACtD,QAAQ,MAAM,SAAS,MACrB,MAAM,KAAK,SAAS;KAAE,UAAU,EAAE,SAAS,MAAM;KAAG,KAAK,EAAE,OAAO,IAAI,MAAM;IAAI,EAAE,CACpF;IACA,IAAI,EAAE,WAAW,UAAU;IAC3B;GACF;GACA,QAAQ;GACR,UAAU;GACV,IAAI,MAAM,SAAS,MAAM,SAAS,SAAS;GAC3C,QAAQ,MAAM,SAAS,MACrB,MAAM,KAAK,SAAS;IAAE,UAAU,EAAE,SAAS,OAAO,GAAG;IAAG,KAAK;GAAG,EAAE,CACpE;GACA;EACF;EAEA,IAAI,MAAM,KAAK;GACb,QAAQ;GACR,UAAU;GACV,IAAI,eAAe,KAAK,IAAI,CAAC,CAAC,CAAC;GAC/B;EACF;EAEA,MAAM,IAAI,eAAe,KAAK,IAAI,CAAC;EACnC,IAAI,EAAE;EACN,UAAU,GAAG,CAAC;CAChB;CACA,UAAU;CACV,OAAO;EACL,KAAK,MAAM,KAAK,MAAM,EAAE,QAAQ;EAChC,OAAO,CAAC,SAAS,MAAM,OAAO,MAAM,EAAE,SAAS,WAAW,CAAC;CAC7D;AACF;;AAGA,SAAS,iBAAiB,KAAa,OAA4B;CACjE,IAAI,IAAI,QAAQ,OAAO,KAAK,OAAO;EAAE,MAAM;EAAa,YAAY,QAAQ;CAAE;CAC9E,MAAM,SAAS,IAAI,QAAQ;CAC3B,IAAI,WAAW,KAAK,OAAO;EAAE,MAAM;EAAa,YAAY,QAAQ;CAAE;CACtE,IAAI,WAAW,OAAO,WAAW,KAAK,OAAO,EAAE,MAAM,YAAY;CACjE,IAAI,WAAW,KAAK;EAClB,MAAM,OAAO,IAAI,QAAQ;EACzB,IAAI,SAAS,OAAO,SAAS,KAAK,OAAO,EAAE,MAAM,YAAY;EAC7D,MAAM,UAAU,IAAI,QAAQ,KAAK,QAAQ,CAAC;EAC1C,IAAI,WAAW,GAAG,OAAO;GAAE,MAAM;GAAa,YAAY,UAAU;EAAE;CACxE;CACA,OAAO,EAAE,MAAM,UAAU;AAC3B;;;;;AAMA,SAAS,oBACP,KACA,OAC2C;CAC3C,MAAM,OAAO,IAAI,QAAQ;CACzB,IAAI,SAAS,KAAK;EAChB,MAAM,MAAM,IAAI,MAAM,QAAQ,GAAG,QAAQ,CAAC;EAC1C,IAAI,CAAC,mBAAmB,KAAK,GAAG,GAAG,OAAO,KAAA;EAC1C,OAAO;GAAE,MAAM,OAAO,aAAa,OAAO,SAAS,KAAK,EAAE,CAAC;GAAG,KAAK,QAAQ;EAAE;CAC/E;CACA,IAAI,SAAS,KAAK;EAChB,MAAM,MAAM,IAAI,MAAM,QAAQ,GAAG,QAAQ,CAAC;EAC1C,IAAI,CAAC,mBAAmB,KAAK,GAAG,GAAG,OAAO,KAAA;EAC1C,OAAO;GAAE,MAAM,OAAO,aAAa,OAAO,SAAS,KAAK,EAAE,CAAC;GAAG,KAAK,QAAQ;EAAE;CAC/E;AAEF;;;;;;AAOA,SAAS,kBAAkB,KAAa,OAAuB;CAC7D,MAAM,OAAO,IAAI,QAAQ;CACzB,IAAI,SAAS,KACX,OAAO,mBAAmB,KAAK,IAAI,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,IAAI,QAAQ,IAAI,QAAQ;CAExF,IAAI,SAAS,KAAK;EAChB,IAAI,IAAI,QAAQ,OAAO,KAAK;GAC1B,MAAM,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC;GACxC,OAAO,SAAS,IAAI,QAAQ,IAAI,IAAI;EACtC;EACA,OAAO,mBAAmB,KAAK,IAAI,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,IAAI,QAAQ,IAAI,QAAQ;CACxF;CACA,IAAI,SAAS,KAAK,OAAO,aAAa,KAAK,IAAI,QAAQ,MAAM,EAAE,IAAI,QAAQ,IAAI,QAAQ;CACvF,KAAK,SAAS,OAAO,SAAS,QAAQ,IAAI,QAAQ,OAAO,KAAK;EAC5D,MAAM,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC;EACxC,OAAO,SAAS,IAAI,QAAQ,IAAI,IAAI;CACtC;CACA,IAAI,SAAS,OAAO,IAAI,QAAQ,OAAO,KAAK;EAC1C,MAAM,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC;EACxC,OAAO,SAAS,IAAI,QAAQ,IAAI,IAAI;CACtC;CACA,IAAI,QAAQ,KAAK,QAAQ,EAAE,GAAG;EAC5B,IAAI,IAAI,QAAQ;EAChB,OAAO,QAAQ,KAAK,IAAI,MAAM,EAAE,GAAG,KAAK;EACxC,OAAO;CACT;CACA,OAAO,KAAK,IAAI,QAAQ,GAAG,IAAI,MAAM;AACvC;;AAGA,SAAS,kBAAkB,KAAa,OAAe,KAA8B;CACnF,IAAI,IAAI,QAAQ;CAChB,MAAM,aAAa,MAAM;CACzB,IAAI,IAAI,OAAO,KAAK,OAAO;CAC3B,MAAM,wBAAQ,IAAI,IAAY;CAC9B,OAAO,IAAI,YAAY;EACrB,MAAM,IAAI,IAAI,MAAM;EACpB,IAAI,MAAM,MAAM;GACd,MAAM,OAAO,IAAI,IAAI;GACrB,IAAI,SAAS,KAAA,KAAa,WAAW,KAAK,IAAI,GAAG,OAAO;GACxD,MAAM,IAAI,IAAI;GACd,KAAK;GACL;EACF;EACA,IAAI,IAAI,IAAI,cAAc,IAAI,IAAI,OAAO,KAAK,OAAO;EACrD,MAAM,IAAI,CAAC;EACX,KAAK;CACP;CAEA,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,OAAO,kBAAkB,KAAK,EAAE,CAAC,GAAG,OAAO;CAChE,OAAO,MAAM,SAAS,IAAI,OAAO,CAAC,GAAG,KAAK;AAC5C;;AAGA,SAAS,eAAe,KAAa,GAAuB;CAC1D,MAAM,IAAI,IAAI;CACd,IAAI,MAAM,OAAO,MAAM,OAAO,MAAM,KAAK;EACvC,MAAM,MAAM,IAAI,IAAI,OAAO,MAAM,IAAI,IAAI,IAAI;EAC7C,OAAO;GAAE,QAAQ,MAAM,MAAM,IAAI;GAAG,WAAW;GAAM;EAAI;CAC3D;CACA,IAAI,MAAM,KAAK;EACb,MAAM,QAAQ,IAAI,QAAQ,KAAK,CAAC;EAChC,IAAI,QAAQ,GAAG,OAAO;GAAE,QAAQ;GAAG,WAAW;GAAO,KAAK;EAAE;EAC5D,MAAM,IAAI,qBAAqB,KAAK,IAAI,MAAM,IAAI,GAAG,KAAK,CAAC;EAC3D,IAAI,CAAC,GAAG,OAAO;GAAE,QAAQ;GAAG,WAAW;GAAO,KAAK;EAAE;EACrD,MAAM,MAAM,OAAO,EAAE,EAAE;EACvB,MAAM,MAAM,EAAE,OAAO,KAAA,IAAY,MAAM,EAAE,OAAO,KAAK,OAAO,oBAAoB,OAAO,EAAE,EAAE;EAC3F,MAAM,MAAM,IAAI,QAAQ,OAAO,MAAM,QAAQ,IAAI,QAAQ;EACzD,OAAO;GAAE,QAAQ;GAAK,WAAW,MAAM;GAAK;EAAI;CAClD;CACA,OAAO;EAAE,QAAQ;EAAG,WAAW;EAAO,KAAK;CAAE;AAC/C;AAEA,SAAS,UAAU,KAAa,GAAmB;CACjD,IAAI,IAAI,IAAI;CACZ,IAAI,IAAI,OAAO,KAAK,KAAK;CACzB,OAAO,IAAI,IAAI,UAAU,IAAI,OAAO,KAAK,KAAK,IAAI,OAAO,QAAQ,IAAI,IAAI,IAAI,SAAS,IAAI;CAC1F,OAAO,IAAI,IAAI,SAAS,IAAI,IAAI;AAClC;AAEA,SAAS,UAAU,KAAa,GAAmB;CACjD,IAAI,QAAQ;CACZ,IAAI,IAAI,IAAI;CACZ,OAAO,IAAI,IAAI,UAAU,QAAQ,GAAG;EAClC,MAAM,IAAI,IAAI;EACd,IAAI,MAAM,QAAQ,IAAI,IAAI,IAAI,QAAQ;GACpC,KAAK;GACL;EACF;EACA,IAAI,MAAM,KAAK;GACb,IAAI,UAAU,KAAK,CAAC;GACpB;EACF;EACA,IAAI,MAAM,KAAK,SAAS;OACnB,IAAI,MAAM,KAAK,SAAS;EAC7B,KAAK;CACP;CACA,OAAO;AACT;;;;AClWA,MAAa,qBAAqB;AAYlC,SAAgB,UACd,SACA,OAAqD,CAAC,GAC3C;CACX,IAAI,QAAQ,SAAA,KACV,MAAM,IAAI,WACR,qBAAqB,QAAQ,OAAO,4BAA4B,oBAClE;CAIF,IAAI,KAAK,SAAS,2BAA2B,KAAK,OAAO,GACvD,MAAM,IAAI,YACR,gGACF;CAEF,MAAM,SAAS,KAAK,QAAQ,UAAU,QAAQ,QAAQ,wBAAwB,MAAM;CACpF,MAAM,QAAQ,IAAI,OAAO,QAAQ,KAAK,kBAAkB,QAAQ,OAAO,GAAG;CAC1E,MAAM,WAAuB,CAAC;CAC9B,KAAK,MAAM,YAAY,cAAc,KAAK,GAAG;EAC3C,MAAM,SAAS,SAAS,IAAI,IAAI,CAAC,CAAC,QAAQ,YAAY,MAAM,KAAK,OAAO,CAAC,CAAC,UAAU,CAAC;EACrF,IAAI,OAAO,WAAW,GAAG,OAAO;GAAE;GAAO,UAAU;EAAK;EACxD,SAAS,KAAK,MAAM;CACtB;CACA,OAAO;EAAE;EAAO;CAAS;AAC3B;;;;ACtCA,SAAgB,QAAQ,QAA0B;CAChD,MAAM,MAAM,MAAM,KAAK,MAAM;CAC7B,MAAM,sBAAM,IAAI,IAAY;CAC5B,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,IAAI,QAAQ,KAAK,IAAI,IAAI,IAAI,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;CAC9E,OAAO,CAAC,GAAG,GAAG;AAChB;;AAGA,SAAgB,SAAS,MAAwB;CAC/C,OAAO,QAAQ,KAAK,IAAI,CAAC;AAC3B;;;;;;AAOA,SAAgB,WAAW,MAAsB;CAC/C,MAAM,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,KAAK,MAAM,KAAK,OAAO,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC;CAC7E,IAAI,IAAI,SAAW,IAAI,SAAW,IAAI,OAAS,OAAO,IAAI,KAAK,KAAK,IAAI,KAAK,KAAK;CAClF,OAAO,KAAK,KAAM,IAAI,QAAW,KAAK,KAAM,IAAI,QAAW,KAAK,KAAM,IAAI;AAC5E;;;ACpBA,MAAM,OAAO;;;;;;AAOb,SAAgB,mBAAmB,QAA6B;CAC9D,MAAM,QAAQ,GAAG,OAAO;CACxB,MAAM,SAAS,UACb,UACE,MAAM,SAAS,SAAS,SAAS,KAAK,IAAI,CAAC,CAAC,KAAK,SAAS,CAAC,WAAW,IAAI,GAAG,KAAK,EAAE,CAAC,CAAC,CACxF;CACF,OAAO;EACL,SAAS,OACP,GAAG,KACD,8BAA8B,MAAM,mFACtC;EACF,MAAM,IAAI,IAAI,OAAO;GACnB,KAAK,MAAM,SAAS,MAAM,KAAK,GAC7B,MAAM,GAAG,MACP,yBAAyB,MAAM,oBAAoB,KAAK,qBACxD,CAAC,KAAK,CACR;EAEJ;EACA,MAAM,OAAO,IAAI,OAAO;GACtB,KAAK,MAAM,SAAS,MAAM,KAAK,GAC7B,MAAM,GAAG,MACP,eAAe,MAAM,+BAA+B,KAAK,sBACzD,CAAC,KAAK,CACR;EAEJ;EACA,MAAM,WAAW,IAAI,UAAU,KAAK;GAClC,MAAM,sBAAM,IAAI,IAAY;GAC5B,KAAK,MAAM,UAAU,UAAU;IAE7B,MAAM,QAAQ,CAAC,GAAG,IAAI,IAAI,OAAO,SAAS,YAAY,QAAQ,OAAO,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC;IACxF,MAAM,OAAO,MAAM,GAAG,MACpB,kBAAkB,MAAM,sGACxB;KAAC,KAAK,UAAU,KAAK;KAAG,MAAM;KAAQ;IAAG,CAC3C;IACA,KAAK,MAAM,OAAO,MAAM,IAAI,IAAI,OAAO,IAAI,EAAE,CAAC;GAChD;GACA,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG;EACpD;CACF;AACF;;;ACrBA,MAAM,eAAe;AAErB,IAAa,aAAb,MAAwB;;CAEtB;CACA;CACA;CACA;CACA;CAEA,YAAY,EAAE,IAAI,QAAQ,QAAQ,UAA6B;EAC7D,IAAI,CAAC,2BAA2B,KAAK,MAAM,GACzC,MAAM,IAAI,MAAM,kBAAkB,OAAO,wCAAwC;EAEnF,IAAI,UAAU,UAAU,UAAU,YAChC,MAAM,IAAI,MAAM,yBAAyB,MAAM,EAAE;EACnD,KAAK,MAAM;EACX,KAAK,UAAU;EACf,KAAK,UAAU,GAAG,OAAO;EACzB,KAAK,SAAS;EACd,KAAK,WAAW,UAAU,SAAS,eAAe,MAAM,IAAI,mBAAmB,MAAM;CACvF;CAEA,MAAM,OAAsB;EAC1B,MAAM,IAAI,KAAK;EACf,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;GAC1C,MAAM,GAAG,KACP,8BAA8B,EAAE,gDAClC;GACA,MAAM,CAAC,OAAO,MAAM,GAAG,MACrB,qBAAqB,EAAE,yBACzB;GACA,IAAI,OAAO,IAAI,UAAU,KAAK,QAC5B,MAAM,IAAI,MACR,iBAAiB,EAAE,4BAA4B,IAAI,MAAM,kBAAkB,KAAK,OAAO,EACzF;GAEF,IAAI,CAAC,KACH,MAAM,GAAG,MAAM,eAAe,EAAE,uCAAuC,CAAC,KAAK,MAAM,CAAC;GACtF,MAAM,GAAG,KACP,8BAA8B,EAAE,kDAClC;GACA,MAAM,KAAK,SAAS,OAAO,EAAE;EAC/B,CAAC;CACH;CAEA,MAAM,OAAO,OAAmC;EAI9C,MAAM,SAAS,CACb,GAAG,IAAI,IAAI,MAAM,KAAK,EAAE,IAAI,WAAW,CAAC,IAAI;GAAE;GAAI,MAAM,KAAK,aAAa;EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAC1F;EACA,IAAI,OAAO,WAAW,GAAG;EACzB,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;GAC1C,MAAM,KAAK,SAAS,OAClB,IACA,MAAM,KAAK,MACT,IACA,OAAO,KAAK,SAAS,KAAK,EAAE,CAC9B,CACF;GACA,KAAK,MAAM,SAAS,UAAU,OAAO,KAAK,SAAS,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,CAAC,GACtE,MAAM,GAAG,MACP,0BAA0B,KAAK,QAAQ,oGACvC,CAAC,KAAK,CACR;GAEF,MAAM,KAAK,SAAS,IAAI,IAAI,MAAM;EACpC,CAAC;CACH;CAEA,MAAM,OAAO,KAA8B;EACzC,IAAI,IAAI,WAAW,GAAG;EACtB,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;GAC1C,MAAM,KAAK,SAAS,OAAO,IAAI,MAAM,KAAK,MAAM,IAAI,GAAG,CAAC;GACxD,KAAK,MAAM,SAAS,UAAU,GAAG,GAC/B,MAAM,GAAG,MACP,eAAe,KAAK,QAAQ,qDAC5B,CAAC,KAAK,CACR;EAEJ,CAAC;CACH;CAEA,MAAM,OAAO,SAAiB,OAA2B,CAAC,GAA+B;EACvF,MAAM,EAAE,QAAQ,KAAK,kBAAkB,KAAQ,QAAQ,WAAW;EAClE,MAAM,OAAO,UAAU,SAAS;GAC9B,OAAO,KAAK;GACZ,eAAe,KAAK,iBAAiB;EACvC,CAAC;EACD,QAAQ,eAAe;EACvB,IAAI,SAAS,GAAG,OAAO;GAAE,MAAM,CAAC;GAAG,WAAW;EAAK;EACnD,MAAM,QACJ,KAAK,aAAa,QAEZ,MAAM,KAAK,IAAI,MACb,kBAAkB,KAAK,QAAQ,4BAC/B,CAAC,kBAAkB,CAAC,CACtB,EAAA,CACA,KAAK,QAAQ,OAAO,IAAI,EAAE,CAAC,IAC7B,MAAM,KAAK,SAAS,WAAW,KAAK,KAAK,KAAK,UAAU,kBAAkB,CAAC;EACjF,IAAI,YAAY,MAAM,SAAS;EAC/B,MAAM,MAAM,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,QAAQ,OAAO,CAAC,UAAU,OAAO,EAAE,CAAC;EAEhF,MAAM,OAAmB,CAAC;EAC1B,IAAI;EACJ,QAAQ,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK,cAAc;GACzD,QAAQ,eAAe;GACvB,KAAK,MAAM,EAAE,IAAI,UAAU,MAAM,KAAK,MAAM,KAAK,KAAK,IAAI,MAAM,GAAG,IAAI,YAAY,CAAC,GAAG;IACrF,OAAO;IACP,MAAM,UAAwB,CAAC;IAC/B,KAAK,MAAM,KAAK,KAAK,SAAS,KAAK,KAAK,GACtC,IAAI,EAAE,EAAE,CAAC,SAAS,GAAG,QAAQ,KAAK;KAAE,OAAO,EAAE;KAAO,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC;IAAO,CAAC;IAElF,IAAI,QAAQ,WAAW,GAAG;IAC1B,KAAK,KAAK;KAAE;KAAI;IAAQ,CAAC;IACzB,IAAI,KAAK,UAAU,OAAO,MAAM;GAClC;EACF;EACA,IAAI,KAAK,UAAU,SAAS,SAAS,IAAI,GAAG,EAAE,GAAG,YAAY;EAC7D,OAAO;GAAE;GAAM;EAAU;CAC3B;;CAGA,MAAM,SAAS,KAA6C;EAC1D,MAAM,QAAQ,MAAM,KAAK,MAAM,KAAK,KAAK,GAAG;EAC5C,OAAO,IAAI,IAAI,MAAM,KAAK,SAAS,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,CAAC;CAC1D;;CAGA,MAAM,OAAsB;EAC1B,MAAM,IAAI,KAAK;EACf,MAAM,cAAc,KAAK,KAAK,OAAO,OAAO;GAC1C,KAAK,MAAM,KAAK;IAAC;IAAM;IAAS;IAAS;GAAM,GAC7C,MAAM,GAAG,KAAK,wBAAwB,IAAI,GAAG;EACjD,CAAC;CACH;;CAGA,MAAM,MAAM,IAAQ,KAAqC;EACvD,MAAM,MAAmB,CAAC;EAC1B,KAAK,MAAM,SAAS,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,GAAG;GAC7D,MAAM,OAAO,MAAM,GAAG,MACpB,wBAAwB,KAAK,QAAQ,iEACrC,CAAC,KAAK,CACR;GACA,KAAK,MAAM,OAAO,MAAM,IAAI,KAAK;IAAE,IAAI,OAAO,IAAI,EAAE;IAAG,MAAM,OAAO,IAAI,IAAI;GAAE,CAAC;EACjF;EACA,OAAO;CACT;AACF"}
@@ -0,0 +1,103 @@
1
+ import { Db } from "@statewalker/db-api";
2
+ //#region src/vector/types.d.ts
3
+ type Metric = "cosine" | "dot" | "l2";
4
+ interface SearchHit {
5
+ id: number;
6
+ /** Lower is closer: 1 - cos for cosine, -dot for dot, squared L2 for l2. */
7
+ distance: number;
8
+ }
9
+ interface SearchOptions {
10
+ k?: number;
11
+ /** IVF lists probed per segment. Default 8. */
12
+ nprobe?: number;
13
+ /** Candidates kept per probed list by the 1-bit filter = k * oversample. Default 8. */
14
+ oversample?: number;
15
+ /** Re-rank the int8 winners with the stored float32 vectors. Default false. */
16
+ exact?: boolean;
17
+ /** Keep only ids for which this returns true (applied before re-ranking). */
18
+ filter?: (id: number) => boolean;
19
+ signal?: AbortSignal;
20
+ }
21
+ //#endregion
22
+ //#region src/vector/sql-segment-store.d.ts
23
+ interface SegmentRef {
24
+ gen: number;
25
+ count: number;
26
+ nlist: number;
27
+ }
28
+ //#endregion
29
+ //#region src/vector/vector-index.d.ts
30
+ interface VectorIndexOptions {
31
+ db: Db;
32
+ /** Table name prefix, e.g. "vec_". Letters, digits and underscores. */
33
+ prefix: string;
34
+ dims: number;
35
+ metric?: Metric;
36
+ /** IVF lists for a new segment of n vectors. Default ~sqrt(n), flat below 2048. */
37
+ nlist?: (n: number) => number;
38
+ /** Cache budget for list blocks read from SQLite, in bytes. Default 64 MiB. */
39
+ cacheBytes?: number;
40
+ /**
41
+ * upsert() seals the tail into a segment once it holds this many vectors.
42
+ * Default 4096: compacting a larger tail risks the 15 s operation deadline of
43
+ * a Durable Object on celld. Infinity turns it off.
44
+ */
45
+ autoCompactAt?: number;
46
+ }
47
+ interface VectorItem {
48
+ /** A safe integer: typically the rowid of your documents table. */
49
+ id: number;
50
+ vector: ArrayLike<number>;
51
+ }
52
+ interface VectorIndexStats {
53
+ tail: number;
54
+ segments: SegmentRef[];
55
+ seq: number;
56
+ }
57
+ /**
58
+ * An LSM-style vector index over a `Db`.
59
+ *
60
+ * - The mutable tail lives in `{p}tail`, mirrored in memory and searched exactly.
61
+ * - `compact()` seals the tail into an immutable IVF segment (1-bit residual
62
+ * filter → int8 re-rank → float32 re-rank on `exact`), written as rows in the
63
+ * same transaction that moves the ids out of the tail.
64
+ * - `{p}ids(id, gen)` says which generation holds the live copy of each id
65
+ * (0 = tail). A segment hit counts only if its generation matches, so deletes
66
+ * and overwrites never touch segment rows. `merge()` rewrites all segments
67
+ * into one, dropping dead rows.
68
+ *
69
+ * One writer per database is assumed (a Durable Object, a worker, one tab);
70
+ * within an instance, writes are serialized.
71
+ */
72
+ export declare class VectorIndex {
73
+ #private;
74
+ readonly dims: number;
75
+ readonly metric: Metric;
76
+ /** `{prefix}ids`: its `id` column holds exactly the live ids. Callers may join it in SQL. */
77
+ readonly idTable: string;
78
+ constructor(opts: VectorIndexOptions);
79
+ /** Creates tables if missing, loads state and sweeps rows of crashed maintenance. */
80
+ init(): Promise<void>;
81
+ /** Reloads state, tail and segments from storage (e.g. after another writer changed them). */
82
+ refresh(): Promise<void>;
83
+ get stats(): VectorIndexStats;
84
+ upsert(items: VectorItem[]): Promise<void>;
85
+ delete(ids: number[]): Promise<void>;
86
+ search(query: ArrayLike<number>, opts?: SearchOptions): Promise<SearchHit[]>;
87
+ /** The stored vectors of the live `ids` (normalised for cosine); unknown or deleted ids are absent. */
88
+ getVectors(ids: number[]): Promise<Map<number, Float32Array>>;
89
+ /** Seals the current tail into a new segment. Returns the number of vectors sealed. */
90
+ compact(): Promise<number>;
91
+ /**
92
+ * Rewrites all segments into one (retraining the IVF lists) and drops dead
93
+ * rows. Holds all live segment vectors in memory while it runs.
94
+ */
95
+ merge(): Promise<number>;
96
+ /** compact() then merge(). */
97
+ optimize(): Promise<void>;
98
+ }
99
+ /** Drops every table of the index under `prefix`, in one transaction. Needs no dims or metric. */
100
+ export declare function dropVectorIndex(db: Db, prefix: string): Promise<void>;
101
+ //#endregion
102
+ export type { Metric, SearchHit, SearchOptions, SegmentRef, VectorIndexOptions, VectorIndexStats, VectorItem };
103
+ //# sourceMappingURL=vector.d.mts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"vector.d.mts","names":[],"sources":["../src/vector/types.ts","../src/vector/sql-segment-store.ts","../src/vector/vector-index.ts"],"mappings":";;KAAY;UAEK;EACf;;EAEA;;UAGe;EACf;;EAEA;;EAEA;;EAEA;;EAEA,UAAU;EACV,SAAS;;;;UCRM;EACf;EACA;EACA;;;;UCMe;EACf,IAAI;;EAEJ;EACA;EACA,SAAS;;EAET,SAAS;;EAET;;;;;;EAMA;;UAGe;;EAEf;EACA,QAAQ;;UAGO;EACf;EACA,UAAU;EACV;;;;;;;;;;;;;;;;;qBAkCW;;WACF;WACA,QAAQ;;WAER;EAkBT,YAAY,MAAM;;EAqBZ,QAAQ;;EA+BR,WAAW;MAqBb,SAAS;EAUP,OAAO,OAAO,eAAe;EAiC7B,OAAO,gBAAgB;EAwBvB,OAAO,OAAO,mBAAmB,OAAM,gBAAqB,QAAQ;;EAoEpE,WAAW,gBAAgB,QAAQ,YAAY;;EAwC/C,WAAW;;;;;EAkDX,SAAS;;EAgET,YAAY;;;wBAiHE,gBAAgB,IAAI,IAAI,iBAAiB"}