@strav/search 0.3.21 → 0.3.22

package/README.md

@@ -51,7 +51,8 @@ await search.delete('posts', ['1'])
 
 ## Drivers
 
-- **Embedded** — in-process SQLite FTS5, zero deps, recommended for self-host / SMB
+- **Embedded** — in-process SQLite FTS5, zero deps, recommended for self-host / SMB (~50k–500k docs)
+- **Postgres FTS** — tsvector + GIN + pg_trgm, drop-in upgrade for higher volume (1M–100M docs)
 - **Meilisearch** — fast, typo-tolerant, self-hosted
 - **Typesense** — open-source, instant search
 - **Algolia** — hosted search-as-a-service
@@ -91,6 +92,52 @@ embedded: {
 
 Select it as the default with `SEARCH_DRIVER=embedded`.
 
+### Postgres FTS driver
+
+Higher-volume tier (1M–100M docs per index) backed by your existing Postgres. Same `SearchEngine` interface as the embedded driver — drop-in swap by changing one config line.
+
+Features:
+
+- BM25-shaped ranking via `ts_rank_cd(fts, q, 1 | 32)` with per-field weights (`A`/`B`/`C`/`D`)
+- `websearch_to_tsquery` Google-style queries plus prefix (`type*`)
+- Multi-language stemming via Postgres text-search configurations (`english`, `french`, ...) — set per index
+- Levenshtein-near typo tolerance via `pg_trgm` + optional `fuzzystrmatch`
+- `<mark>`-highlighted snippets via `ts_headline`, computed only on the top-K to keep latency bounded
+- Object-form filters with `eq`/`neq`/`gt`/`gte`/`lt`/`lte`/`in`/`nin` against generated typed columns
+- One table per index in a dedicated `strav_search` schema (auto-created)
+
+Requirements:
+
+- Postgres ≥ 15
+- `pg_trgm` extension (auto-`CREATE EXTENSION IF NOT EXISTS` on first use; superuser or owner privilege)
+- `fuzzystrmatch` is optional — if present, typo expansion re-ranks trigram candidates with bounded Levenshtein for higher precision
+
+Configuration:
+
+```ts
+postgres: {
+  driver: 'postgres-fts',
+  // Optional: pass a Bun SQL handle. Falls back to @strav/database's Database.raw.
+  // connection: db.sql,
+  schema: env('SEARCH_PG_SCHEMA', 'strav_search'),
+  language: env('SEARCH_PG_LANGUAGE', 'english'),
+  typoTolerance: env('SEARCH_TYPO_TOLERANCE', 'auto'),
+  workMem: env('SEARCH_PG_WORK_MEM', '64MB'),
+  gin: { fastupdate: false },  // better tail latency
+}
+```
+
+Select it with `SEARCH_DRIVER=postgres`.
+
+Limitations for v1:
+
+- Settings change (e.g. add a new searchable attribute) requires `bun strav search:rebuild <model>`. Tier picked by row count: in-place UPDATE under 100k, batched UPDATE up to 10M, dual-table swap deferred to v1.1 with a clear error above 10M.
+- Adding a new `filterableAttribute` on an existing large table currently rewrites the whole heap (`ALTER TABLE ADD COLUMN ... GENERATED ... STORED`). Plan an offline window for big tables in v1.
+- One language per index — mixed-locale indexes deferred.
+- Object-form filters only; raw SQL filter strings rejected.
+
+Ranking note: `ts_rank_cd` is BM25-*shaped* (length normalisation + bounded mapping), not strict BM25. For the size and shape of corpora the driver targets, the difference is small in practice; the embedded driver remains the answer when strict BM25 matters and the corpus fits.
+
 Model example with per-field weights (column order determines BM25 weight — title first = highest):
 
 ```ts
@@ -132,6 +179,7 @@ You run `bun strav search:import Ticket` once to populate the index, then model
 bun strav search:import <model>      # Import all records for a model
 bun strav search:flush <model>       # Flush all documents from an index
 bun strav search:optimize <model>    # (embedded) Merge FTS5 segments; run periodically
+bun strav search:rebuild <model>     # (postgres) Recompute fts after settings change
 ```
 
 ## Documentation

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@strav/search",
-  "version": "0.3.21",
+  "version": "0.3.22",
   "type": "module",
   "description": "Full-text search for the Strav framework",
   "license": "MIT",
@@ -18,9 +18,9 @@
     "tsconfig.json"
   ],
   "peerDependencies": {
-    "@strav/kernel": "0.3.21",
-    "@strav/database": "0.3.21",
-    "@strav/cli": "0.3.21"
+    "@strav/kernel": "0.3.22",
+    "@strav/database": "0.3.22",
+    "@strav/cli": "0.3.22"
   },
   "scripts": {
     "test": "bun test tests/",

package/src/commands/search_rebuild.ts

@@ -0,0 +1,73 @@
+import type { Command } from 'commander'
+import chalk from 'chalk'
+import { bootstrap, shutdown } from '@strav/cli'
+import { BaseModel } from '@strav/database'
+import SearchManager from '../search_manager.ts'
+import { PostgresFtsDriver } from '../drivers/postgres/index.ts'
+
+export function register(program: Command): void {
+  program
+    .command('search:rebuild <model>')
+    .description("Recompute a model's fts column in place (postgres-fts driver only)")
+    .option('--no-reindex', "Skip the GIN REINDEX after the rebuild")
+    .option('--pause <ms>', 'Pause between batches in tier-2 mode (default 50)', '50')
+    .action(async (modelPath: string, options: { reindex: boolean; pause: string }) => {
+      let db
+      try {
+        const { db: database, config } = await bootstrap()
+        db = database
+
+        new BaseModel(db)
+        new SearchManager(config)
+
+        const resolved = require.resolve(`${process.cwd()}/${modelPath}`)
+        const module = await import(resolved)
+        const ModelClass = module.default ?? (Object.values(module)[0] as any)
+
+        if (typeof ModelClass?.searchableAs !== 'function') {
+          console.error(chalk.red(`Model "${modelPath}" does not use the searchable() mixin.`))
+          process.exit(1)
+        }
+
+        const indexName = SearchManager.indexName(ModelClass.searchableAs())
+        const engine = SearchManager.engine()
+
+        if (!(engine instanceof PostgresFtsDriver)) {
+          console.error(
+            chalk.red(
+              `search:rebuild is only meaningful for the postgres-fts driver (current: ${engine.name}).`
+            )
+          )
+          process.exit(1)
+        }
+
+        // Make sure the engine knows about the model's settings (so rebuild
+        // computes fts with the right weights/language).
+        const settings = (ModelClass.searchableSettings?.() ?? undefined) as any
+        if (settings) await engine.createIndex(indexName, settings)
+
+        console.log(chalk.dim(`Rebuilding "${indexName}"...`))
+        const result = await engine.rebuild(indexName, {
+          reindex: options.reindex !== false,
+          pauseMs: Number(options.pause),
+          onProgress: (done, total) => {
+            const pct = total > 0 ? Math.round((done / total) * 100) : 100
+            process.stdout.write(`\r  ${done}/${total} rows (${pct}%) `)
+          },
+        })
+        if (result.tier === 2) process.stdout.write('\n')
+
+        console.log(
+          chalk.green(
+            `Rebuilt ${result.rows} row(s) in "${indexName}" using tier-${result.tier} ` +
+              `strategy (${result.elapsedMs}ms).`
+          )
+        )
+      } catch (err) {
+        console.error(chalk.red(`Error: ${err instanceof Error ? err.message : err}`))
+        process.exit(1)
+      } finally {
+        if (db) await shutdown(db)
+      }
+    })
+}

package/src/drivers/postgres/engine/field_registry.ts

@@ -0,0 +1,116 @@
+import type { PgIndexSettings } from '../types.ts'
+
+/** Default searchable column when no `searchableAttributes` are configured. */
+export const DEFAULT_TEXT_COLUMN = '_text'
+
+/** FTS5 weight tiers in declaration order. */
+const WEIGHT_TIERS = ['A', 'B', 'C', 'D'] as const
+type WeightTier = (typeof WEIGHT_TIERS)[number]
+
+/** Postgres column type derived from a sample value, or `text` as the conservative default. */
+type PgType = 'text' | 'integer' | 'bigint' | 'double precision' | 'boolean' | 'timestamptz'
+
+export interface TypedColumnSpec {
+  name: string
+  pgType: PgType
+  /** JSONB extraction expression: `(doc->>'name')::pgType` (cast suppressed for text). */
+  expression: string
+}
+
+/**
+ * The schema layout for one index: which document attributes feed which
+ * tsvector segment + weight, and which typed columns exist for filter/sort.
+ *
+ * Mirrors `embedded/engine/field_registry.ts` so the two drivers project
+ * documents identically. Differences:
+ * - Per-attribute weight tier (A/B/C/D) is explicit.
+ * - Typed columns are emitted as `GENERATED ALWAYS AS (...) STORED` SQL.
+ */
+export class FieldRegistry {
+  readonly searchable: string[]
+  readonly weights: Map<string, WeightTier>
+  readonly filterable: string[]
+  readonly sortable: string[]
+  readonly typedColumns: TypedColumnSpec[]
+  readonly primaryKey: string
+  readonly language: string
+
+  constructor(settings?: PgIndexSettings, language = 'english') {
+    this.primaryKey = settings?.primaryKey ?? 'id'
+    this.language = settings?.language ?? language
+    this.searchable =
+      settings?.searchableAttributes && settings.searchableAttributes.length > 0
+        ? [...settings.searchableAttributes]
+        : [DEFAULT_TEXT_COLUMN]
+
+    this.weights = new Map()
+    for (let i = 0; i < this.searchable.length; i++) {
+      const attr = this.searchable[i]!
+      const tier = (settings?.weights?.[attr] ?? WEIGHT_TIERS[Math.min(i, 3)]) as WeightTier
+      this.weights.set(attr, tier)
+    }
+
+    this.filterable = settings?.filterableAttributes ?? []
+    this.sortable = settings?.sortableAttributes ?? []
+
+    const seen = new Set<string>()
+    const typed: TypedColumnSpec[] = []
+    for (const attr of [...this.filterable, ...this.sortable]) {
+      if (seen.has(attr)) continue
+      seen.add(attr)
+      typed.push({ name: attr, pgType: 'text', expression: `(doc->>${literal(attr)})` })
+    }
+    this.typedColumns = typed
+  }
+
+  get usesDefaultTextColumn(): boolean {
+    return this.searchable.length === 1 && this.searchable[0] === DEFAULT_TEXT_COLUMN
+  }
+
+  /**
+   * Project a document into [text, tier] pairs for tsvector construction.
+   * Default mode collapses every string into one A-weighted blob.
+   */
+  projectFtsSegments(document: Record<string, unknown>): Array<{ text: string; tier: WeightTier }> {
+    if (this.usesDefaultTextColumn) {
+      return [{ text: collectStrings(document), tier: 'A' }]
+    }
+    return this.searchable.map(attr => ({
+      text: coerceText(document[attr]),
+      tier: this.weights.get(attr)!,
+    }))
+  }
+
+  /** Single string spanning all searchable text (for terms-dict tokenization). */
+  concatSearchableText(document: Record<string, unknown>): string {
+    return this.projectFtsSegments(document)
+      .map(s => s.text)
+      .filter(Boolean)
+      .join(' ')
+  }
+}
+
+function literal(value: string): string {
+  return `'${value.replace(/'/g, "''")}'`
+}
+
+function coerceText(value: unknown): string {
+  if (value === null || value === undefined) return ''
+  if (typeof value === 'string') return value
+  if (Array.isArray(value)) return value.map(v => coerceText(v)).filter(Boolean).join(' ')
+  if (typeof value === 'number' || typeof value === 'boolean') return String(value)
+  return ''
+}
+
+function collectStrings(document: Record<string, unknown>): string {
+  const parts: string[] = []
+  for (const value of Object.values(document)) {
+    if (typeof value === 'string' && value.length > 0) parts.push(value)
+    else if (Array.isArray(value)) {
+      for (const item of value) {
+        if (typeof item === 'string' && item.length > 0) parts.push(item)
+      }
+    }
+  }
+  return parts.join(' ')
+}

package/src/drivers/postgres/engine/fts_query_builder.ts

@@ -0,0 +1,105 @@
+import { quoteLiteral } from '../storage/identifiers.ts'
+
+/**
+ * Translate a user-facing query string into one that's safe for
+ * `websearch_to_tsquery`, plus extract positive tokens for typo expansion.
+ *
+ * websearch_to_tsquery already accepts Google-style syntax:
+ * - `"foo bar"`  — phrase
+ * - `-foo`       — exclude
+ * - `OR`/`AND`   — boolean (case-insensitive)
+ *
+ * It does NOT support prefix matching (`foo*`); we recognise that ourselves
+ * and emit a separate `to_tsquery('foo:*')` ORed onto the result.
+ */
+export interface ParsedQuery {
+  /** The raw query, ready to pass to `websearch_to_tsquery`. */
+  websearch: string
+  /** Positive bare tokens (no quotes/operators) — used for typo expansion. */
+  positiveTokens: string[]
+  /** Prefix tokens from `foo*` syntax — emitted separately to `to_tsquery`. */
+  prefixTokens: string[]
+  /** Whether the input was effectively empty. */
+  isEmpty: boolean
+}
+
+const PHRASE_RE = /"([^"]*)"/g
+
+export function parseQuery(input: string): ParsedQuery {
+  const trimmed = input.trim()
+  if (!trimmed) {
+    return { websearch: '', positiveTokens: [], prefixTokens: [], isEmpty: true }
+  }
+
+  const positiveTokens: string[] = []
+  const prefixTokens: string[] = []
+
+  // Strip phrases first so we don't tokenize their inner whitespace.
+  const scratch = trimmed.replace(PHRASE_RE, ' ')
+  for (const raw of scratch.split(/\s+/)) {
+    if (!raw) continue
+    let text = raw
+    if (text.startsWith('-') || text.startsWith('+')) text = text.slice(1)
+    if (text.endsWith('*')) {
+      const stem = text.slice(0, -1).toLowerCase().replace(/[^\p{L}\p{N}_-]/gu, '')
+      if (stem) prefixTokens.push(stem)
+      continue
+    }
+    if (text.toUpperCase() === 'AND' || text.toUpperCase() === 'OR') continue
+    const norm = text.toLowerCase().replace(/[^\p{L}\p{N}_-]/gu, '')
+    if (norm.length >= 2) positiveTokens.push(norm)
+  }
+
+  return { websearch: trimmed, positiveTokens, prefixTokens, isEmpty: false }
+}
+
+/**
+ * Build a tsquery SQL expression that ORs together the user's websearch query,
+ * any prefix tokens, and any typo-expanded alternatives. Returns the
+ * expression + the user-text bindings (the language is embedded as a literal
+ * since it's a per-index server-controlled value, not user input).
+ *
+ * `startAt` is the placeholder counter the caller has already used. Returned
+ * `paramCount` lets the caller continue numbering for filter/limit/offset.
+ */
+export function buildTsqueryExpression(
+  parsed: ParsedQuery,
+  expansions: Map<string, string[]>,
+  language: string,
+  startAt = 0
+): { sql: string; params: string[]; paramCount: number } {
+  const params: string[] = []
+  const fragments: string[] = []
+  const lang = `${quoteLiteral(language)}::regconfig`
+  let cursor = startAt
+  const ph = () => `$${++cursor}`
+
+  if (parsed.websearch) {
+    params.push(parsed.websearch)
+    fragments.push(`websearch_to_tsquery(${lang}, ${ph()})`)
+  }
+
+  for (const stem of parsed.prefixTokens) {
+    params.push(`${stem}:*`)
+    fragments.push(`to_tsquery(${lang}, ${ph()})`)
+  }
+
+  for (const token of parsed.positiveTokens) {
+    const cands = expansions.get(token)
+    if (!cands || cands.length === 0) continue
+    const expr = cands.map(sanitiseTsTerm).filter(Boolean).join(' | ')
+    if (!expr) continue
+    params.push(expr)
+    fragments.push(`to_tsquery(${lang}, ${ph()})`)
+  }
+
+  if (fragments.length === 0) {
+    return { sql: '', params: [], paramCount: 0 }
+  }
+  return { sql: fragments.join(' || '), params, paramCount: cursor - startAt }
+}
+
+/** Sanitise a single term for inclusion in a manually built tsquery. */
+function sanitiseTsTerm(term: string): string {
+  return term.toLowerCase().replace(/[^\p{L}\p{N}_-]/gu, '')
+}

package/src/drivers/postgres/engine/pg_engine.ts

@@ -0,0 +1,300 @@
+import type { SQL } from 'bun'
+import type {
+  SearchDocument,
+  SearchOptions,
+  SearchResult,
+  SearchHit,
+} from '../../../types.ts'
+import type { PgIndexSettings, ResolvedTypoTolerance } from '../types.ts'
+import { FieldRegistry } from './field_registry.ts'
+import { ensureIndexTable, dropIndex as dropIndexSchema } from './schema.ts'
+import { parseQuery, buildTsqueryExpression } from './fts_query_builder.ts'
+import { compileSearch } from './query_compiler.ts'
+import { formatSnippet } from './snippet_formatter.ts'
+import {
+  expandTokens,
+  hasFuzzystrmatch,
+  recordTerms,
+  unrecordTerms,
+} from './typo_expander.ts'
+import {
+  indexTableName,
+  termsTableName,
+  quoteIdent,
+  quoteLiteral,
+} from '../storage/identifiers.ts'
+import { rebuildInPlace, type RebuildOptions } from '../rebuild/rebuild_inplace.ts'
+
+export interface PgEngineOptions {
+  sql: SQL
+  schema: string
+  index: string
+  language: string
+  typoTolerance: ResolvedTypoTolerance
+  ginFastUpdate: boolean
+  workMem: string | null
+  settings?: PgIndexSettings
+}
+
+/** Postgres tsvector silently truncates at ~1MB lexemes. Truncate inputs to be safe. */
+const MAX_TEXT_BYTES = 900_000
+
+/** One PgEngine wraps a single index. */
+export class PgEngine {
+  readonly registry: FieldRegistry
+  private readonly sql: SQL
+  private readonly schema: string
+  private readonly index: string
+  private readonly typo: ResolvedTypoTolerance
+  private readonly ginFastUpdate: boolean
+  private readonly workMem: string | null
+  private readonly tableName: string
+  private fuzzyAvailable: boolean | null = null
+  private ensured = false
+
+  constructor(opts: PgEngineOptions) {
+    this.sql = opts.sql
+    this.schema = opts.schema
+    this.index = opts.index
+    this.typo = opts.typoTolerance
+    this.ginFastUpdate = opts.ginFastUpdate
+    this.workMem = opts.workMem
+    this.registry = new FieldRegistry(opts.settings, opts.language)
+    this.tableName = indexTableName(opts.schema, opts.index)
+  }
+
+  /** Lazy: ensure the table + indexes + trigger exist. Idempotent. */
+  async ensure(): Promise<void> {
+    if (this.ensured) return
+    await ensureIndexTable(this.sql, this.schema, this.index, this.registry, this.ginFastUpdate)
+    if (this.typo.enabled && this.fuzzyAvailable === null) {
+      this.fuzzyAvailable = await hasFuzzystrmatch(this.sql)
+    }
+    this.ensured = true
+  }
+
+  // ── Writes ──────────────────────────────────────────────────────────────
+
+  async upsert(id: string | number, document: Record<string, unknown>): Promise<void> {
+    await this.upsertMany([{ id, ...document }])
+  }
+
+  async upsertMany(documents: SearchDocument[]): Promise<void> {
+    if (documents.length === 0) return
+    await this.ensure()
+
+    await this.sql.begin(async (tx: SQL) => {
+      for (const raw of documents) {
+        const { id, ...rest } = raw
+        const idStr = String(id)
+        // Bun's SQL treats stringified JSON as a JSONB string value (double-
+        // encoding the JSON). Passing the object directly lets it generate
+        // proper JSONB so `doc->>'field'` works for the typed generated cols.
+        const doc = { id, ...(rest as Record<string, unknown>) }
+        const newText = truncate(this.registry.concatSearchableText(rest as Record<string, unknown>))
+
+        const oldRows = (await tx.unsafe(
+          `SELECT doc FROM ${this.tableName} WHERE id = $1`,
+          [idStr]
+        )) as Array<{ doc: Record<string, unknown> | string }>
+        if (oldRows.length > 0) {
+          const oldDoc = parseDoc(oldRows[0]!.doc)
+          const oldText = this.registry.concatSearchableText(oldDoc)
+          if (this.typo.enabled) await unrecordTerms(tx, this.schema, this.index, oldText)
+        }
+
+        const ftsExpr = this.buildFtsExpression(rest as Record<string, unknown>)
+        const sqlStr =
+          `INSERT INTO ${this.tableName} (id, doc, fts) VALUES ($1, $2, ${ftsExpr.sql}) ` +
+          `ON CONFLICT (id) DO UPDATE SET doc = EXCLUDED.doc, fts = EXCLUDED.fts`
+        await tx.unsafe(sqlStr, [idStr, doc as any, ...ftsExpr.params])
+
+        if (this.typo.enabled) await recordTerms(tx, this.schema, this.index, newText)
+      }
+    })
+  }
+
+  async delete(id: string | number): Promise<void> {
+    await this.deleteMany([id])
+  }
+
+  async deleteMany(ids: Array<string | number>): Promise<void> {
+    if (ids.length === 0) return
+    await this.ensure()
+
+    await this.sql.begin(async (tx: SQL) => {
+      const idStrs = ids.map(String)
+      const placeholders = idStrs.map((_, i) => `$${i + 1}`).join(', ')
+
+      if (this.typo.enabled) {
+        const rows = (await tx.unsafe(
+          `SELECT doc FROM ${this.tableName} WHERE id IN (${placeholders})`,
+          idStrs
+        )) as Array<{ doc: Record<string, unknown> | string }>
+        for (const r of rows) {
+          const oldDoc = parseDoc(r.doc)
+          await unrecordTerms(tx, this.schema, this.index, this.registry.concatSearchableText(oldDoc))
+        }
+      }
+
+      await tx.unsafe(
+        `DELETE FROM ${this.tableName} WHERE id IN (${placeholders})`,
+        idStrs
+      )
+    })
+  }
+
+  async flush(): Promise<void> {
+    await this.ensure()
+    await this.sql.begin(async (tx: SQL) => {
+      await tx.unsafe(`TRUNCATE ${this.tableName}`)
+      if (this.typo.enabled) {
+        await tx.unsafe(`TRUNCATE ${termsTableName(this.schema, this.index)}`)
+      }
+    })
+  }
+
+  async drop(): Promise<void> {
+    await dropIndexSchema(this.sql, this.schema, this.index)
+    this.ensured = false
+  }
+
+  // ── Reads ───────────────────────────────────────────────────────────────
+
+  async search(query: string, options?: SearchOptions): Promise<SearchResult> {
+    await this.ensure()
+    const start = performance.now()
+    const opts = options ?? {}
+    const parsed = parseQuery(query)
+
+    const expansions = await this.maybeExpand(parsed.positiveTokens)
+    const tsquery = buildTsqueryExpression(parsed, expansions, this.registry.language)
+
+    const compiled = compileSearch({
+      registry: this.registry,
+      schema: this.schema,
+      index: this.index,
+      tsquery: { sql: tsquery.sql, params: tsquery.params },
+      search: opts,
+    })
+
+    const result = await this.sql.begin(async (tx: SQL) => {
+      if (this.workMem) {
+        await tx.unsafe(`SET LOCAL work_mem = ${quoteLiteral(this.workMem)}`)
+      }
+      const rows = (await tx.unsafe(compiled.sql, compiled.params)) as RawHitRow[]
+      const totalRows = (await tx.unsafe(compiled.countSql, compiled.countParams)) as Array<{
+        n: number
+      }>
+      return { rows, total: totalRows[0]?.n ?? rows.length }
+    })
+
+    const projection = opts.attributesToRetrieve
+    const hits: SearchHit[] = result.rows.map(row =>
+      projectHit(row, compiled.snippetColumns, projection)
+    )
+
+    return {
+      hits,
+      totalHits: result.total,
+      page: Math.max(1, opts.page ?? 1),
+      perPage: Math.max(1, opts.perPage ?? 20),
+      processingTimeMs: Math.round(performance.now() - start),
+    }
+  }
+
+  /** REINDEX the GIN index. Periodic maintenance for write-heavy indexes. */
+  async optimize(): Promise<void> {
+    await this.ensure()
+    const ginName = `${quoteIdent(this.schema)}.${quoteIdent(`search_${this.index}_fts_gin`)}`
+    await this.sql.unsafe(`REINDEX INDEX ${ginName}`)
+  }
+
+  /**
+   * Recompute every row's `fts` using the current registry's language + weight
+   * scheme. Auto-picks tier (in-place vs batched) by row count; throws on
+   * tables larger than the supported tier-2 ceiling.
+   */
+  async rebuild(options?: RebuildOptions) {
+    await this.ensure()
+    return rebuildInPlace(this.sql, this.schema, this.index, this.registry, options)
+  }
+
+  // ── Internals ───────────────────────────────────────────────────────────
+
+  private buildFtsExpression(document: Record<string, unknown>): {
+    sql: string
+    params: string[]
+  } {
+    const segments = this.registry.projectFtsSegments(document)
+    const lang = `${quoteLiteral(this.registry.language)}::regconfig`
+    const params: string[] = []
+    const fragments = segments.map(seg => {
+      params.push(truncate(seg.text))
+      return `setweight(to_tsvector(${lang}, $${params.length + 2}), '${seg.tier}')`
+    })
+    // The `+2` above accounts for the leading id ($1) and doc ($2) bindings
+    // that callers prepend. Caller MUST keep those positions stable.
+    return { sql: fragments.join(' || '), params }
+  }
+
+  private async maybeExpand(tokens: string[]): Promise<Map<string, string[]>> {
+    if (!this.typo.enabled || tokens.length === 0) return new Map()
+    return expandTokens(
+      this.sql,
+      this.schema,
+      this.index,
+      tokens,
+      this.typo,
+      this.fuzzyAvailable === true
+    )
+  }
+}
+
+interface RawHitRow {
+  id: string
+  doc: Record<string, unknown> | string
+  score: number
+  [snippetCol: string]: unknown
+}
+
+function projectHit(
+  row: RawHitRow,
+  snippetCols: string[],
+  attributesToRetrieve: string[] | undefined
+): SearchHit {
+  const document = parseDoc(row.doc)
+
+  let projected = document
+  if (attributesToRetrieve && attributesToRetrieve.length > 0) {
+    const out: Record<string, unknown> = {}
+    for (const attr of attributesToRetrieve) {
+      if (attr in document) out[attr] = document[attr]
+    }
+    projected = out
+  }
+
+  const hit: SearchHit = { document: projected }
+
+  if (snippetCols.length > 0) {
+    const highlights: Record<string, string> = {}
+    for (const col of snippetCols) {
+      const raw = row[`__snip_${col}`] as string | null | undefined
+      if (raw) highlights[col] = formatSnippet(raw)
+    }
+    if (Object.keys(highlights).length > 0) hit.highlights = highlights
+  }
+
+  return hit
+}
+
+function parseDoc(doc: Record<string, unknown> | string): Record<string, unknown> {
+  if (typeof doc === 'string') return JSON.parse(doc) as Record<string, unknown>
+  return doc
+}
+
+function truncate(text: string): string {
+  if (Buffer.byteLength(text, 'utf8') <= MAX_TEXT_BYTES) return text
+  // Truncate by char count; over-conservative is fine.
+  return text.slice(0, MAX_TEXT_BYTES)
+}