@strav/search 0.3.20 → 0.3.22

package/src/drivers/embedded/engine/fts_query_builder.ts

@@ -0,0 +1,184 @@
+/**
+ * Translate a user-facing query string into a sanitized FTS5 MATCH expression.
+ *
+ * Supported syntax (subset of Google-style search):
+ * - `"foo bar"`   — exact phrase
+ * - `-foo`        — exclude documents containing this token
+ * - `+foo`        — required (default for all positive tokens — accepted for symmetry)
+ * - `foo*`        — prefix match
+ *
+ * Everything else is treated as a positive ANDed token.
+ *
+ * Defends against FTS5 syntax injection by stripping or escaping any FTS5
+ * operator characters from raw user tokens. The user never gets to write a
+ * raw MATCH expression.
+ */
+export interface FtsExpression {
+  /** Final MATCH expression, ready to bind into a query. */
+  match: string
+  /** The positive tokens (no quotes, no operators) — useful for typo expansion. */
+  positiveTokens: string[]
+  /** Whether the expression is empty (caller may short-circuit to "match all"). */
+  isEmpty: boolean
+}
+
+interface ParsedToken {
+  text: string
+  negate: boolean
+  phrase: boolean
+  prefix: boolean
+}
+
+const FTS5_RESERVED = /["()*:^]/g
+const PHRASE_RE = /"([^"]*)"/g
+
+export function compileQuery(input: string): FtsExpression {
+  const trimmed = input.trim()
+  if (!trimmed) return { match: '', positiveTokens: [], isEmpty: true }
+
+  const tokens = parseTokens(trimmed)
+  if (tokens.length === 0) return { match: '', positiveTokens: [], isEmpty: true }
+
+  const positives: string[] = []
+  const negatives: string[] = []
+  const positiveTokens: string[] = []
+
+  for (const tok of tokens) {
+    const rendered = renderToken(tok)
+    if (!rendered) continue
+
+    if (tok.negate) {
+      negatives.push(rendered)
+    } else {
+      positives.push(rendered)
+      if (!tok.phrase && !tok.prefix) positiveTokens.push(tok.text.toLowerCase())
+    }
+  }
+
+  if (positives.length === 0 && negatives.length === 0) {
+    return { match: '', positiveTokens: [], isEmpty: true }
+  }
+
+  // Pure-negative queries can't be expressed in FTS5 — fall back to no-match.
+  if (positives.length === 0) {
+    return { match: '', positiveTokens: [], isEmpty: true }
+  }
+
+  let expr = positives.join(' AND ')
+  if (negatives.length > 0) {
+    expr = `${expr} NOT (${negatives.join(' OR ')})`
+  }
+
+  return { match: expr, positiveTokens, isEmpty: false }
+}
+
+/**
+ * Re-render a previously parsed query but with extra OR-candidates injected
+ * for each positive token. Used by the typo expander.
+ */
+export function compileQueryWithExpansions(
+  input: string,
+  expansions: Map<string, string[]>
+): FtsExpression {
+  const trimmed = input.trim()
+  if (!trimmed) return { match: '', positiveTokens: [], isEmpty: true }
+
+  const tokens = parseTokens(trimmed)
+  const positives: string[] = []
+  const negatives: string[] = []
+  const positiveTokens: string[] = []
+
+  for (const tok of tokens) {
+    if (tok.negate) {
+      const r = renderToken(tok)
+      if (r) negatives.push(r)
+      continue
+    }
+
+    if (tok.phrase || tok.prefix) {
+      const r = renderToken(tok)
+      if (r) positives.push(r)
+      continue
+    }
+
+    const sanitized = sanitizeBareToken(tok.text)
+    if (!sanitized) continue
+    positiveTokens.push(sanitized.toLowerCase())
+
+    const cands = expansions.get(sanitized.toLowerCase()) ?? []
+    if (cands.length === 0) {
+      positives.push(sanitized)
+    } else {
+      const all = [sanitized, ...cands].map(t => sanitizeBareToken(t)).filter(Boolean) as string[]
+      const unique = Array.from(new Set(all))
+      positives.push(`(${unique.join(' OR ')})`)
+    }
+  }
+
+  if (positives.length === 0) return { match: '', positiveTokens: [], isEmpty: true }
+
+  let expr = positives.join(' AND ')
+  if (negatives.length > 0) {
+    expr = `${expr} NOT (${negatives.join(' OR ')})`
+  }
+  return { match: expr, positiveTokens, isEmpty: false }
+}
+
+function parseTokens(input: string): ParsedToken[] {
+  const tokens: ParsedToken[] = []
+  let cursor = 0
+  let working = input
+
+  // Pull out phrase tokens first to avoid splitting on inner whitespace.
+  working = working.replace(PHRASE_RE, (_, phrase, offset) => {
+    const negate = offset > 0 && input[offset - 1] === '-'
+    tokens.push({ text: phrase, negate, phrase: true, prefix: false })
+    return ' '.repeat(_.length + (negate ? 1 : 0))
+  })
+
+  for (const raw of working.split(/\s+/)) {
+    if (!raw) continue
+    let text = raw
+    let negate = false
+    let prefix = false
+
+    if (text.startsWith('-')) {
+      negate = true
+      text = text.slice(1)
+    } else if (text.startsWith('+')) {
+      text = text.slice(1)
+    }
+    if (text.endsWith('*')) {
+      prefix = true
+      text = text.slice(0, -1)
+    }
+    if (!text) continue
+
+    tokens.push({ text, negate, phrase: false, prefix })
+  }
+
+  void cursor
+  return tokens
+}
+
+function renderToken(tok: ParsedToken): string | null {
+  if (tok.phrase) {
+    const cleaned = tok.text.replace(/"/g, '').trim()
+    if (!cleaned) return null
+    return `"${cleaned}"`
+  }
+  const sanitized = sanitizeBareToken(tok.text)
+  if (!sanitized) return null
+  return tok.prefix ? `${sanitized}*` : sanitized
+}
+
+function sanitizeBareToken(token: string): string {
+  // Replace any FTS5 operator characters with a space, then collapse to one
+  // word. If only one word survives we use it bare; otherwise wrap in quotes
+  // so FTS5 treats it as a phrase rather than two ANDed tokens.
+  const cleaned = token.replace(FTS5_RESERVED, ' ').trim()
+  if (!cleaned) return ''
+  const parts = cleaned.split(/\s+/).filter(Boolean)
+  if (parts.length === 1) return parts[0]!
+  return `"${parts.join(' ')}"`
+}

package/src/drivers/embedded/engine/query_compiler.ts

@@ -0,0 +1,134 @@
+import type { SearchOptions } from '../../../types.ts'
+import type { FieldRegistry } from './field_registry.ts'
+import type { FtsExpression } from './fts_query_builder.ts'
+import { compileFilter } from '../filters/filter_compiler.ts'
+import { quoteIdent } from './schema.ts'
+import { OPEN_SENTINEL, CLOSE_SENTINEL } from './snippet_formatter.ts'
+
+export interface CompiledSearch {
+  /** Main SELECT returning hits + score + snippets. */
+  sql: string
+  /** Bound parameters for the main SELECT. */
+  params: unknown[]
+  /** COUNT(*) variant for totalHits. */
+  countSql: string
+  /** Bound parameters for the count query (subset of `params`). */
+  countParams: unknown[]
+  /** Names of columns we asked SQLite to return for snippets, in order. */
+  snippetColumns: string[]
+}
+
+const DEFAULT_SNIPPET_TOKENS = 24
+
+export interface QueryCompilerOptions {
+  registry: FieldRegistry
+  expression: FtsExpression
+  search: SearchOptions
+  /** Per-column BM25 weights, matching `registry.searchable` order. Defaults to all-1. */
+  weights?: number[]
+}
+
+export function compileSearch(opts: QueryCompilerOptions): CompiledSearch {
+  const { registry, expression, search, weights } = opts
+  const filterableSet = new Set(registry.filterable)
+  const sortableSet = new Set(registry.sortable)
+
+  const filter = compileFilter(search.filter, filterableSet)
+
+  const whereParts: string[] = []
+  const matchParams: unknown[] = []
+
+  if (!expression.isEmpty) {
+    whereParts.push('fts.fts MATCH ?')
+    matchParams.push(expression.match)
+  }
+  if (filter.sql) whereParts.push(filter.sql)
+
+  const where = whereParts.length > 0 ? `WHERE ${whereParts.join(' AND ')}` : ''
+
+  // BM25 score (negative = better). Defaults to weight 1.0 for every column.
+  const ws = (weights ?? registry.searchable.map(() => 1)).map(w => Number(w) || 1)
+  const bm25Args = ws.length > 0 ? `, ${ws.join(', ')}` : ''
+
+  const orderBy = compileOrder(search.sort, sortableSet, expression.isEmpty, bm25Args)
+
+  // Build snippet expressions for each field the caller wants highlighted.
+  const wantedHighlights = pickHighlightFields(search.attributesToHighlight, registry)
+  const snippetSelect = wantedHighlights
+    .map(field => {
+      const idx = registry.searchable.indexOf(field)
+      return `snippet(fts.fts, ${idx}, '${OPEN_SENTINEL}', '${CLOSE_SENTINEL}', ' … ', ${DEFAULT_SNIPPET_TOKENS}) AS ${quoteIdent(`__snip_${field}`)}`
+    })
+    .join(', ')
+
+  const perPage = Math.max(1, search.perPage ?? 20)
+  const page = Math.max(1, search.page ?? 1)
+  const offset = (page - 1) * perPage
+
+  const selectCols = [
+    'documents.id AS id',
+    'documents.doc AS doc',
+    expression.isEmpty ? '0 AS score' : `bm25(fts.fts${bm25Args}) AS score`,
+  ]
+  if (snippetSelect) selectCols.push(snippetSelect)
+
+  const sql = `
+    SELECT ${selectCols.join(', ')}
+    FROM documents
+    ${expression.isEmpty ? '' : 'JOIN fts ON fts.rowid = documents.rowid'}
+    ${where}
+    ${orderBy}
+    LIMIT ? OFFSET ?
+  `.trim()
+
+  const countSql = `
+    SELECT COUNT(*) AS n
+    FROM documents
+    ${expression.isEmpty ? '' : 'JOIN fts ON fts.rowid = documents.rowid'}
+    ${where}
+  `.trim()
+
+  const allParams = [...matchParams, ...filter.params]
+  const params = [...allParams, perPage, offset]
+
+  return {
+    sql,
+    params,
+    countSql,
+    countParams: allParams,
+    snippetColumns: wantedHighlights,
+  }
+}
+
+function compileOrder(
+  sort: string[] | undefined,
+  sortableSet: ReadonlySet<string>,
+  matchAll: boolean,
+  bm25Args: string
+): string {
+  if (sort && sort.length > 0) {
+    const parts: string[] = []
+    for (const spec of sort) {
+      const [field, dirRaw] = spec.split(':') as [string, string | undefined]
+      if (!field || !sortableSet.has(field)) {
+        throw new Error(
+          `Field "${field}" is not in sortableAttributes. Add it to the index settings before sorting on it.`
+        )
+      }
+      const dir = dirRaw?.toLowerCase() === 'desc' ? 'DESC' : 'ASC'
+      parts.push(`${quoteIdent(field)} ${dir}`)
+    }
+    return `ORDER BY ${parts.join(', ')}`
+  }
+  if (matchAll) return 'ORDER BY documents.rowid ASC'
+  return `ORDER BY bm25(fts.fts${bm25Args}) ASC`
+}
+
+function pickHighlightFields(
+  requested: string[] | undefined,
+  registry: FieldRegistry
+): string[] {
+  if (registry.usesDefaultTextColumn) return []
+  if (!requested || requested.length === 0) return []
+  return requested.filter(f => registry.searchable.includes(f))
+}

package/src/drivers/embedded/engine/schema.ts

@@ -0,0 +1,99 @@
+import type { Database } from 'bun:sqlite'
+import type { FieldRegistry } from './field_registry.ts'
+
+const SCHEMA_VERSION = 1
+
+/** Quote a SQLite identifier (column or table). Throws on identifiers that contain a NUL byte. */
+export function quoteIdent(name: string): string {
+  if (name.includes('\0')) throw new Error(`Invalid identifier: ${name}`)
+  return `"${name.replace(/"/g, '""')}"`
+}
+
+export function applyConnectionPragmas(
+  db: Database,
+  synchronous: 'OFF' | 'NORMAL' | 'FULL'
+): void {
+  db.exec('PRAGMA journal_mode = WAL')
+  db.exec(`PRAGMA synchronous = ${synchronous}`)
+  db.exec('PRAGMA temp_store = MEMORY')
+  db.exec('PRAGMA foreign_keys = ON')
+}
+
+/**
+ * Create the documents + FTS5 + terms_dict tables for a fresh index.
+ * Idempotent: skips creation if `_meta` already exists.
+ */
+export function createSchema(db: Database, registry: FieldRegistry): void {
+  if (schemaExists(db)) return
+
+  db.exec(`
+    CREATE TABLE _meta (
+      key TEXT PRIMARY KEY,
+      value TEXT NOT NULL
+    )
+  `)
+
+  const typedColumns = registry.typedColumns
+    .map(c => `${quoteIdent(c)} BLOB`)
+    .join(', ')
+  const typedColumnsClause = typedColumns ? `, ${typedColumns}` : ''
+
+  db.exec(`
+    CREATE TABLE documents (
+      rowid INTEGER PRIMARY KEY AUTOINCREMENT,
+      id TEXT NOT NULL UNIQUE,
+      doc TEXT NOT NULL${typedColumnsClause}
+    )
+  `)
+  db.exec('CREATE UNIQUE INDEX documents_id_idx ON documents(id)')
+
+  // One index per filterable column so WHERE clauses can use it.
+  for (const col of registry.filterable) {
+    db.exec(`CREATE INDEX ${quoteIdent(`documents_${col}_idx`)} ON documents(${quoteIdent(col)})`)
+  }
+
+  const ftsCols = registry.searchable.map(quoteIdent).join(', ')
+  // FTS5 default mode (no `content` option): the original text is stored in
+  // FTS5 itself so `snippet()` can echo it back highlighted. The Porter
+  // tokenizer applies English stemming; unicode61 normalises and folds
+  // diacritics so accented input matches its ASCII form.
+  db.exec(`
+    CREATE VIRTUAL TABLE fts USING fts5(
+      ${ftsCols},
+      tokenize = 'porter unicode61 remove_diacritics 2'
+    )
+  `)
+
+  // Maintain a terms dictionary for typo expansion.
+  db.exec(`
+    CREATE TABLE terms_dict (
+      term TEXT PRIMARY KEY,
+      doc_freq INTEGER NOT NULL DEFAULT 0
+    )
+  `)
+  db.exec('CREATE INDEX terms_dict_len_idx ON terms_dict(length(term))')
+
+  // Persist registry layout so we can detect mismatches on reopen.
+  const stmt = db.prepare('INSERT INTO _meta (key, value) VALUES (?, ?)')
+  stmt.run('schema_version', String(SCHEMA_VERSION))
+  stmt.run('searchable', JSON.stringify(registry.searchable))
+  stmt.run('filterable', JSON.stringify(registry.filterable))
+  stmt.run('sortable', JSON.stringify(registry.sortable))
+  stmt.run('primary_key', registry.primaryKey)
+}
+
+function schemaExists(db: Database): boolean {
+  const row = db
+    .query<{ name: string }, []>(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name='_meta'"
+    )
+    .get()
+  return row !== null
+}
+
+export function readMeta(db: Database, key: string): string | null {
+  const row = db
+    .query<{ value: string }, [string]>('SELECT value FROM _meta WHERE key = ?')
+    .get(key)
+  return row ? row.value : null
+}

package/src/drivers/embedded/engine/snippet_formatter.ts

@@ -0,0 +1,29 @@
+/**
+ * SQLite's snippet() returns text with the requested marker tokens around hits.
+ * We use sentinel markers instead of `<mark>` directly so we can safely escape
+ * any HTML in the source text first, then swap sentinels for the real tags.
+ */
+export const OPEN_SENTINEL = '
STRAV_OPEN
'
+export const CLOSE_SENTINEL = '
STRAV_CLOSE
'
+
+export const OPEN_TAG = '<mark>'
+export const CLOSE_TAG = '</mark>'
+
+/**
+ * Convert SQLite-snippet output (already wrapped in sentinels) into the
+ * caller-facing string with `<mark>...</mark>` around hits and HTML-escaped
+ * surrounding text.
+ */
+export function formatSnippet(snippet: string | null | undefined): string {
+  if (!snippet) return ''
+  return escapeHtml(snippet).replaceAll(OPEN_SENTINEL, OPEN_TAG).replaceAll(CLOSE_SENTINEL, CLOSE_TAG)
+}
+
+function escapeHtml(input: string): string {
+  return input
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&#39;')
+}

package/src/drivers/embedded/engine/sqlite_engine.ts

@@ -0,0 +1,255 @@
+import { Database } from 'bun:sqlite'
+import type {
+  SearchDocument,
+  SearchOptions,
+  SearchResult,
+  SearchHit,
+  IndexSettings,
+} from '../../../types.ts'
+import type { ResolvedTypoTolerance } from '../types.ts'
+import { FieldRegistry } from './field_registry.ts'
+import { applyConnectionPragmas, createSchema, quoteIdent } from './schema.ts'
+import { compileQuery, compileQueryWithExpansions } from './fts_query_builder.ts'
+import { compileSearch } from './query_compiler.ts'
+import { formatSnippet } from './snippet_formatter.ts'
+import { recordTerms, unrecordTerms, expandTokens } from './typo_expander.ts'
+
+export interface SqliteEngineOptions {
+  path: string
+  synchronous: 'OFF' | 'NORMAL' | 'FULL'
+  typoTolerance: ResolvedTypoTolerance
+  indexName: string
+  settings?: IndexSettings
+}
+
+/**
+ * One SqliteEngine wraps a single index (a single SQLite file). The driver
+ * holds a Map<indexName, SqliteEngine> and lazily instantiates per index.
+ */
+export class SqliteEngine {
+  readonly db: Database
+  readonly registry: FieldRegistry
+  private readonly typo: ResolvedTypoTolerance
+  private readonly indexName: string
+
+  constructor(opts: SqliteEngineOptions) {
+    this.db = new Database(opts.path)
+    applyConnectionPragmas(this.db, opts.synchronous)
+    this.registry = new FieldRegistry(opts.settings)
+    createSchema(this.db, this.registry)
+    this.typo = opts.typoTolerance
+    this.indexName = opts.indexName
+  }
+
+  // ── Writes ──────────────────────────────────────────────────────────────
+
+  upsert(id: string | number, document: Record<string, unknown>): void {
+    this.runUpsertBatch([{ id, document }])
+  }
+
+  upsertMany(documents: SearchDocument[]): void {
+    if (documents.length === 0) return
+    const batch = documents.map(d => {
+      const { id, ...rest } = d
+      return { id, document: rest as Record<string, unknown> }
+    })
+    this.runUpsertBatch(batch)
+  }
+
+  delete(id: string | number): void {
+    this.runDeleteBatch([id])
+  }
+
+  deleteMany(ids: Array<string | number>): void {
+    if (ids.length === 0) return
+    this.runDeleteBatch(ids)
+  }
+
+  /** Remove all documents from the index, leaving the schema in place. */
+  flush(): void {
+    const tx = this.db.transaction(() => {
+      this.db.exec('DELETE FROM fts')
+      this.db.exec('DELETE FROM documents')
+      this.db.exec('DELETE FROM terms_dict')
+    })
+    tx()
+  }
+
+  /** Force-merge FTS5 segments into one. Run periodically (e.g. nightly via CLI). */
+  optimize(): void {
+    this.db.exec("INSERT INTO fts(fts) VALUES('optimize')")
+  }
+
+  close(): void {
+    try {
+      this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)')
+    } catch {
+      // Ignore — closing should never throw on a checkpoint failure
+    }
+    this.db.close()
+  }
+
+  // ── Reads ───────────────────────────────────────────────────────────────
+
+  search(query: string, options?: SearchOptions): SearchResult {
+    const start = performance.now()
+    const opts = options ?? {}
+    const expression = this.buildExpression(query)
+
+    const compiled = compileSearch({
+      registry: this.registry,
+      expression,
+      search: opts,
+    })
+
+    const rows = this.db
+      .prepare<RawHitRow, any[]>(compiled.sql)
+      .all(...(compiled.params as any[]))
+    const totalRow = this.db
+      .prepare<{ n: number }, any[]>(compiled.countSql)
+      .get(...(compiled.countParams as any[]))
+
+    const projection = opts.attributesToRetrieve
+    const hits: SearchHit[] = rows.map(row => projectHit(row, compiled.snippetColumns, projection))
+
+    return {
+      hits,
+      totalHits: totalRow?.n ?? hits.length,
+      page: Math.max(1, opts.page ?? 1),
+      perPage: Math.max(1, opts.perPage ?? 20),
+      processingTimeMs: Math.round(performance.now() - start),
+    }
+  }
+
+  // ── Internals ───────────────────────────────────────────────────────────
+
+  private buildExpression(query: string) {
+    const base = compileQuery(query)
+    if (!this.typo.enabled || base.isEmpty || base.positiveTokens.length === 0) return base
+
+    const expansions = expandTokens(this.db, base.positiveTokens, this.typo)
+    if (expansions.size === 0) return base
+    return compileQueryWithExpansions(query, expansions)
+  }
+
+  private runUpsertBatch(items: Array<{ id: string | number; document: Record<string, unknown> }>) {
+    const insertDoc = this.prepareInsertDoc()
+    const fetchExisting = this.db.prepare<
+      { rowid: number; doc: string },
+      [string]
+    >('SELECT rowid, doc FROM documents WHERE id = ?')
+    const insertFts = this.prepareInsertFts()
+    const deleteFts = this.db.prepare('DELETE FROM fts WHERE rowid = ?')
+    const indexName = this.indexName
+
+    const tx = this.db.transaction(() => {
+      for (const { id, document: doc } of items) {
+        const idStr = String(id)
+        const docJson = JSON.stringify({ id, ...doc })
+        const ftsValues = this.registry.projectFtsValues(doc)
+        const typedValues = this.registry.projectTypedValues(doc)
+        const newText = this.registry.concatSearchableText(doc)
+
+        const existing = fetchExisting.get(idStr)
+        if (existing) {
+          // Update path
+          const oldDoc = JSON.parse(existing.doc) as Record<string, unknown>
+          const oldText = this.registry.concatSearchableText(oldDoc)
+          unrecordTerms(this.db, oldText)
+
+          deleteFts.run(existing.rowid)
+          insertFts.run(existing.rowid as any, ...(ftsValues as any[]))
+          this.updateDocumentRow(existing.rowid, docJson, typedValues)
+        } else {
+          const result = insertDoc.run(idStr as any, docJson as any, ...(typedValues as any[]))
+          const rowid = Number(result.lastInsertRowid)
+          insertFts.run(rowid as any, ...(ftsValues as any[]))
+        }
+
+        recordTerms(this.db, newText)
+      }
+    })
+    void indexName
+    tx()
+  }
+
+  private runDeleteBatch(ids: Array<string | number>) {
+    const fetchExisting = this.db.prepare<
+      { rowid: number; doc: string },
+      [string]
+    >('SELECT rowid, doc FROM documents WHERE id = ?')
+    const deleteDoc = this.db.prepare('DELETE FROM documents WHERE id = ?')
+    const deleteFts = this.db.prepare('DELETE FROM fts WHERE rowid = ?')
+
+    const tx = this.db.transaction(() => {
+      for (const id of ids) {
+        const idStr = String(id)
+        const existing = fetchExisting.get(idStr)
+        if (!existing) continue
+        const oldDoc = JSON.parse(existing.doc) as Record<string, unknown>
+        unrecordTerms(this.db, this.registry.concatSearchableText(oldDoc))
+        deleteFts.run(existing.rowid)
+        deleteDoc.run(idStr)
+      }
+    })
+    tx()
+  }
+
+  private prepareInsertDoc() {
+    const cols = ['id', 'doc', ...this.registry.typedColumns.map(quoteIdent)]
+    const placeholders = cols.map(() => '?').join(', ')
+    return this.db.prepare(
+      `INSERT INTO documents (${cols.join(', ')}) VALUES (${placeholders})`
+    )
+  }
+
+  private updateDocumentRow(rowid: number, docJson: string, typedValues: unknown[]) {
+    const sets = ['doc = ?']
+    for (const col of this.registry.typedColumns) sets.push(`${quoteIdent(col)} = ?`)
+    const sql = `UPDATE documents SET ${sets.join(', ')} WHERE rowid = ?`
+    this.db.prepare(sql).run(docJson as any, ...(typedValues as any[]), rowid as any)
+  }
+
+  private prepareInsertFts() {
+    const cols = ['rowid', ...this.registry.searchable.map(quoteIdent)]
+    const placeholders = cols.map(() => '?').join(', ')
+    return this.db.prepare(`INSERT INTO fts (${cols.join(', ')}) VALUES (${placeholders})`)
+  }
+}
+
+interface RawHitRow {
+  id: string
+  doc: string
+  score: number
+  [snippetCol: string]: unknown
+}
+
+function projectHit(
+  row: RawHitRow,
+  snippetCols: string[],
+  attributesToRetrieve: string[] | undefined
+): SearchHit {
+  const document = JSON.parse(row.doc) as Record<string, unknown>
+
+  let projected = document
+  if (attributesToRetrieve && attributesToRetrieve.length > 0) {
+    const out: Record<string, unknown> = {}
+    for (const attr of attributesToRetrieve) {
+      if (attr in document) out[attr] = document[attr]
+    }
+    projected = out
+  }
+
+  const hit: SearchHit = { document: projected }
+
+  if (snippetCols.length > 0) {
+    const highlights: Record<string, string> = {}
+    for (const col of snippetCols) {
+      const raw = row[`__snip_${col}`] as string | null | undefined
+      if (raw) highlights[col] = formatSnippet(raw)
+    }
+    if (Object.keys(highlights).length > 0) hit.highlights = highlights
+  }
+
+  return hit
+}