@strav/search 0.3.21 → 0.3.23

package/src/drivers/postgres/index.ts

@@ -0,0 +1,14 @@
+export { PostgresFtsDriver } from './postgres_fts_driver.ts'
+export type {
+  PostgresFtsConfig,
+  TypoToleranceMode,
+  TypoToleranceSettings,
+  PgIndexSettings,
+} from './types.ts'
+export {
+  PostgresFtsError,
+  MissingExtensionError,
+  RebuildRequiredError,
+  UnsupportedFilterError,
+  MissingConnectionError,
+} from './errors.ts'

package/src/drivers/postgres/postgres_fts_driver.ts

@@ -0,0 +1,184 @@
+import type { SQL } from 'bun'
+import type { SearchEngine } from '../../search_engine.ts'
+import type {
+  SearchDocument,
+  SearchOptions,
+  SearchResult,
+  IndexSettings,
+  DriverConfig,
+} from '../../types.ts'
+import { PgEngine } from './engine/pg_engine.ts'
+import { ensureSchemaAndExtensions } from './engine/schema.ts'
+import { resolveTypoTolerance } from './engine/typo_expander.ts'
+import type { PostgresFtsConfig, PgIndexSettings, ResolvedTypoTolerance } from './types.ts'
+import { MissingConnectionError } from './errors.ts'
+
+const DEFAULT_SCHEMA = 'strav_search'
+const DEFAULT_LANGUAGE = 'english'
+const DEFAULT_WORK_MEM = '64MB'
+
+/**
+ * Postgres-backed full-text search driver. Implements the same `SearchEngine`
+ * interface as the embedded SQLite driver — drop-in swap by config.
+ *
+ * Sized for higher-volume workloads (1M-100M docs per index) using `tsvector`
+ * + GIN + `pg_trgm` for typo tolerance + `ts_headline` for snippets.
+ *
+ * Connection: pass `connection` (a Bun `SQL` instance) in the driver config,
+ * or rely on `Database.raw` from `@strav/database` (must be bootstrapped).
+ */
+export class PostgresFtsDriver implements SearchEngine {
+  readonly name = 'postgres-fts'
+
+  private readonly config: PostgresFtsConfig
+  private readonly schemaName: string
+  private readonly defaultLanguage: string
+  private readonly typo: ResolvedTypoTolerance
+  private readonly ginFastUpdate: boolean
+  private readonly workMem: string | null
+  private readonly engines = new Map<string, PgEngine>()
+  private readonly pendingSettings = new Map<string, PgIndexSettings>()
+  private bootstrapped: Promise<void> | null = null
+  private resolvedSql: SQL | null = null
+
+  constructor(config: DriverConfig) {
+    this.config = (config ?? {}) as PostgresFtsConfig
+    this.schemaName = this.config.schema ?? DEFAULT_SCHEMA
+    this.defaultLanguage = this.config.language ?? DEFAULT_LANGUAGE
+    this.typo = resolveTypoTolerance(this.config.typoTolerance)
+    this.ginFastUpdate = this.config.gin?.fastupdate ?? false
+    this.workMem =
+      this.config.workMem === null
+        ? null
+        : (this.config.workMem ?? DEFAULT_WORK_MEM)
+  }
+
+  // ── Document operations ──────────────────────────────────────────────────
+
+  async upsert(
+    index: string,
+    id: string | number,
+    document: Record<string, unknown>
+  ): Promise<void> {
+    await (await this.engineFor(index)).upsert(id, document)
+  }
+
+  async upsertMany(index: string, documents: SearchDocument[]): Promise<void> {
+    await (await this.engineFor(index)).upsertMany(documents)
+  }
+
+  async delete(index: string, id: string | number): Promise<void> {
+    await (await this.engineFor(index)).delete(id)
+  }
+
+  async deleteMany(index: string, ids: Array<string | number>): Promise<void> {
+    await (await this.engineFor(index)).deleteMany(ids)
+  }
+
+  // ── Index operations ─────────────────────────────────────────────────────
+
+  async flush(index: string): Promise<void> {
+    await (await this.engineFor(index)).flush()
+  }
+
+  async deleteIndex(index: string): Promise<void> {
+    const engine = this.engines.get(index)
+    if (engine) {
+      await engine.drop()
+      this.engines.delete(index)
+    } else {
+      // Drop directly without instantiating an engine.
+      const sql = this.resolveSql()
+      const { dropIndex } = await import('./engine/schema.ts')
+      await dropIndex(sql, this.schemaName, index)
+    }
+    this.pendingSettings.delete(index)
+  }
+
+  async createIndex(index: string, options?: IndexSettings): Promise<void> {
+    if (options) this.pendingSettings.set(index, options as PgIndexSettings)
+    const engine = await this.engineFor(index)
+    await engine.ensure()
+  }
+
+  // ── Search ───────────────────────────────────────────────────────────────
+
+  async search(index: string, query: string, options?: SearchOptions): Promise<SearchResult> {
+    return (await this.engineFor(index)).search(query, options)
+  }
+
+  // ── Lifecycle ────────────────────────────────────────────────────────────
+
+  /** Run REINDEX on every open index, or just one if specified. */
+  async optimize(index?: string): Promise<void> {
+    if (index) {
+      await (await this.engineFor(index)).optimize()
+      return
+    }
+    for (const engine of this.engines.values()) await engine.optimize()
+  }
+
+  /**
+   * Rebuild a single index's `fts` column in place. Use after changing
+   * `searchableAttributes` or weights — without it, existing rows keep the
+   * old fts values.
+   */
+  async rebuild(
+    index: string,
+    options?: { reindex?: boolean; pauseMs?: number; onProgress?: (done: number, total: number) => void }
+  ): Promise<{ tier: 1 | 2; rows: number; elapsedMs: number }> {
+    return (await this.engineFor(index)).rebuild(options)
+  }
+
+  // ── Internals ────────────────────────────────────────────────────────────
+
+  private async engineFor(index: string): Promise<PgEngine> {
+    let engine = this.engines.get(index)
+    if (engine) return engine
+
+    await this.bootstrap()
+    const settings = this.pendingSettings.get(index)
+    engine = new PgEngine({
+      sql: this.resolveSql(),
+      schema: this.schemaName,
+      index,
+      language: settings?.language ?? this.defaultLanguage,
+      typoTolerance: this.typo,
+      ginFastUpdate: this.ginFastUpdate,
+      workMem: this.workMem,
+      settings,
+    })
+    this.engines.set(index, engine)
+    this.pendingSettings.delete(index)
+    return engine
+  }
+
+  /** Resolve the SQL connection (config.connection or Database.raw fallback). */
+  private resolveSql(): SQL {
+    if (this.resolvedSql) return this.resolvedSql
+    if (this.config.connection) {
+      this.resolvedSql = this.config.connection
+      return this.resolvedSql
+    }
+    try {
+      // Lazy require to avoid a hard dep at import time.
+      const databaseModule = require('@strav/database')
+      const Database = databaseModule.default ?? databaseModule.Database
+      this.resolvedSql = Database.raw as SQL
+      return this.resolvedSql
+    } catch {
+      throw new MissingConnectionError()
+    }
+  }
+
+  /** Idempotent: ensure schema + extensions exist, once per driver. */
+  private bootstrap(): Promise<void> {
+    if (this.bootstrapped) return this.bootstrapped
+    this.bootstrapped = ensureSchemaAndExtensions(
+      this.resolveSql(),
+      this.schemaName,
+      this.typo
+    )
+    return this.bootstrapped
+  }
+}

package/src/drivers/postgres/rebuild/rebuild_inplace.ts

@@ -0,0 +1,113 @@
+import type { SQL } from 'bun'
+import type { FieldRegistry } from '../engine/field_registry.ts'
+import { indexTableName, quoteLiteral, quoteIdent } from '../storage/identifiers.ts'
+import { RebuildRequiredError } from '../errors.ts'
+
+/** Tier boundaries for rebuild strategy selection. */
+const TIER1_MAX = 100_000
+const TIER2_MAX = 10_000_000
+
+/** Batch size for tier-2 batched UPDATE. */
+const BATCH_SIZE = 5_000
+
+export interface RebuildOptions {
+  /** If true, run REINDEX on the GIN index after the rebuild. Default true. */
+  reindex?: boolean
+  /** Per-batch sleep in milliseconds (tier 2 only). Default 50. */
+  pauseMs?: number
+  /** Optional progress callback fired after each batch. */
+  onProgress?: (done: number, total: number) => void
+}
+
+/**
+ * Rebuild an index's `fts` column in place using the current registry's
+ * language + weight scheme. Picks tier by row count:
+ * - < 100k → single UPDATE
+ * - 100k-10M → batched UPDATE with pauses
+ * - > 10M → RebuildRequiredError (defer to v1.1 swap strategy)
+ */
+export async function rebuildInPlace(
+  sql: SQL,
+  schema: string,
+  index: string,
+  registry: FieldRegistry,
+  options: RebuildOptions = {}
+): Promise<{ tier: 1 | 2; rows: number; elapsedMs: number }> {
+  const reindex = options.reindex ?? true
+  const pauseMs = options.pauseMs ?? 50
+  const table = indexTableName(schema, index)
+  const start = performance.now()
+
+  const countRows = (await sql.unsafe(
+    `SELECT COUNT(*)::bigint AS n FROM ${table}`
+  )) as Array<{ n: string | number }>
+  const total = Number(countRows[0]?.n ?? 0)
+
+  if (total > TIER2_MAX) {
+    throw new RebuildRequiredError(
+      `Index "${index}" has ${total} rows (>${TIER2_MAX}). ` +
+        `In-place / batched rebuild is unsafe at this scale. ` +
+        `Use the v1.1 dual-table swap strategy (not yet shipped).`
+    )
+  }
+
+  const ftsExpr = buildSetFtsExpression(registry)
+
+  if (total <= TIER1_MAX) {
+    await sql.unsafe(`UPDATE ${table} SET fts = ${ftsExpr}`)
+    if (reindex) await reindexGin(sql, schema, index)
+    return { tier: 1, rows: total, elapsedMs: Math.round(performance.now() - start) }
+  }
+
+  // Tier 2: batched update keyed by id, with pauses for autovacuum.
+  let cursor: string | null = null
+  let done = 0
+
+  while (true) {
+    const where = cursor === null ? '' : `WHERE id > $1`
+    const params = cursor === null ? [] : [cursor]
+    const batch = (await sql.unsafe(
+      `SELECT id FROM ${table} ${where} ORDER BY id LIMIT ${BATCH_SIZE}`,
+      params
+    )) as Array<{ id: string }>
+    if (batch.length === 0) break
+
+    const ids = batch.map(r => r.id)
+    const placeholders = ids.map((_, i) => `$${i + 1}`).join(', ')
+    await sql.unsafe(
+      `UPDATE ${table} SET fts = ${ftsExpr} WHERE id IN (${placeholders})`,
+      ids
+    )
+
+    done += batch.length
+    cursor = ids[ids.length - 1]!
+    options.onProgress?.(done, total)
+    if (pauseMs > 0) await new Promise(r => setTimeout(r, pauseMs))
+  }
+
+  if (reindex) await reindexGin(sql, schema, index)
+  return { tier: 2, rows: total, elapsedMs: Math.round(performance.now() - start) }
+}
+
+function buildSetFtsExpression(registry: FieldRegistry): string {
+  const lang = `${quoteLiteral(registry.language)}::regconfig`
+  if (registry.usesDefaultTextColumn) {
+    return (
+      `setweight(to_tsvector(${lang}, ` +
+      `(SELECT coalesce(string_agg(value, ' '), '') FROM jsonb_each_text(doc))), 'A')`
+    )
+  }
+  return registry.searchable
+    .map(attr => {
+      const weight = registry.weights.get(attr)!
+      return (
+        `setweight(to_tsvector(${lang}, coalesce(doc->>${quoteLiteral(attr)}, '')), '${weight}')`
+      )
+    })
+    .join(' || ')
+}
+
+async function reindexGin(sql: SQL, schema: string, index: string): Promise<void> {
+  const ginName = `${quoteIdent(schema)}.${quoteIdent(`search_${index}_fts_gin`)}`
+  await sql.unsafe(`REINDEX INDEX ${ginName}`)
+}

package/src/drivers/postgres/storage/identifiers.ts

@@ -0,0 +1,46 @@
+import { PostgresFtsError } from '../errors.ts'
+
+const PG_IDENT_MAX = 63
+
+/**
+ * Quote a Postgres identifier (schema, table, column). Throws on identifiers
+ * containing NUL or exceeding the 63-byte name limit.
+ */
+export function quoteIdent(name: string): string {
+  if (name.includes('\0')) throw new PostgresFtsError(`Invalid identifier: contains NUL byte.`)
+  if (Buffer.byteLength(name, 'utf8') > PG_IDENT_MAX) {
+    throw new PostgresFtsError(
+      `Identifier "${name}" exceeds Postgres' ${PG_IDENT_MAX}-byte limit.`
+    )
+  }
+  return `"${name.replace(/"/g, '""')}"`
+}
+
+/** Quote a single-quoted SQL string literal (used inside DDL options). */
+export function quoteLiteral(value: string): string {
+  return `'${value.replace(/'/g, "''")}'`
+}
+
+/** Build the schema-qualified table name for a search index. */
+export function indexTableName(schema: string, index: string): string {
+  return `${quoteIdent(schema)}.${quoteIdent(`search_${index}`)}`
+}
+
+/** Terms-dictionary table name for a given index. */
+export function termsTableName(schema: string, index: string): string {
+  return `${quoteIdent(schema)}.${quoteIdent(`search_${index}_terms`)}`
+}
+
+/** Meta table — single shared table; rows keyed by (index_name, key). */
+export function metaTableName(schema: string): string {
+  return `${quoteIdent(schema)}.${quoteIdent('_meta')}`
+}
+
+/** Bare (unquoted) tablename — useful for pg_class lookups. */
+export function bareIndexTable(index: string): string {
+  return `search_${index}`
+}
+
+export function bareTermsTable(index: string): string {
+  return `search_${index}_terms`
+}

package/src/drivers/postgres/types.ts

@@ -0,0 +1,53 @@
+import type { SQL } from 'bun'
+import type { DriverConfig, IndexSettings } from '../../types.ts'
+
+export type TypoToleranceMode = 'off' | 'auto'
+
+export interface TypoToleranceSettings {
+  /** Minimum token length to consider for fuzzy expansion (default 4). */
+  minTokenLength?: number
+  /** Maximum Levenshtein distance to tolerate (default 1; 2 is supported but slower). */
+  maxDistance?: number
+  /** pg_trgm similarity threshold (default 0.4). Higher = stricter. */
+  similarity?: number
+}
+
+export interface PostgresFtsConfig extends DriverConfig {
+  driver: string
+  /**
+   * Bun SQL connection. If omitted, the driver falls back to
+   * `Database.raw` from `@strav/database` (must be bootstrapped first).
+   */
+  connection?: SQL
+  /** Postgres schema for index tables. Default 'strav_search'. */
+  schema?: string
+  /** Default text-search configuration ('english', 'french', ...). */
+  language?: string
+  /** Typo tolerance: 'off' disables; 'auto' uses defaults; object for fine-grained control. */
+  typoTolerance?: TypoToleranceMode | TypoToleranceSettings
+  /** GIN index tuning. */
+  gin?: {
+    /** Default false — better tail latency for read-heavy search. */
+    fastupdate?: boolean
+  }
+  /** Per-search-transaction work_mem hint, e.g. '64MB'. Set to null/empty to skip. */
+  workMem?: string | null
+}
+
+/** Resolved typo tolerance settings (after defaults applied). */
+export interface ResolvedTypoTolerance {
+  enabled: boolean
+  minTokenLength: number
+  maxDistance: number
+  similarity: number
+}
+
+/** Per-index extra settings stored in `_meta`. */
+export interface PgIndexSettings extends IndexSettings {
+  language?: string
+  /**
+   * Per-attribute weight tier override. Keys must appear in `searchableAttributes`.
+   * Values: 'A' | 'B' | 'C' | 'D'. Default = positional (1st=A, 2nd=B, ...).
+   */
+  weights?: Record<string, 'A' | 'B' | 'C' | 'D'>
+}

package/src/index.ts

@@ -18,6 +18,11 @@ export type {
   TypoToleranceMode,
   TypoToleranceSettings,
 } from './drivers/embedded/index.ts'
+export { PostgresFtsDriver } from './drivers/postgres/index.ts'
+export type {
+  PostgresFtsConfig,
+  PgIndexSettings,
+} from './drivers/postgres/index.ts'
 
 // Mixin
 export { searchable } from './searchable.ts'

package/src/search_manager.ts

@@ -6,6 +6,7 @@ import { TypesenseDriver } from './drivers/typesense_driver.ts'
 import { AlgoliaDriver } from './drivers/algolia_driver.ts'
 import { NullDriver } from './drivers/null_driver.ts'
 import { EmbeddedDriver } from './drivers/embedded/index.ts'
+import { PostgresFtsDriver } from './drivers/postgres/index.ts'
 
 @inject
 export default class SearchManager {
@@ -89,6 +90,9 @@ export default class SearchManager {
         return new AlgoliaDriver(config)
       case 'embedded':
         return new EmbeddedDriver(config)
+      case 'postgres-fts':
+      case 'postgres':
+        return new PostgresFtsDriver(config)
       case 'null':
         return new NullDriver()
       default:

package/stubs/config/search.ts

@@ -38,5 +38,20 @@ export default {
       /** Typo tolerance: 'off' to disable, 'auto' for defaults, or { minTokenLength, maxDistance }. */
       typoTolerance: env('SEARCH_TYPO_TOLERANCE', 'auto'),
     },
+
+    postgres: {
+      driver: 'postgres-fts',
+      /** Postgres schema for index tables. */
+      schema: env('SEARCH_PG_SCHEMA', 'strav_search'),
+      /** Default text-search configuration ('english', 'french', ...). */
+      language: env('SEARCH_PG_LANGUAGE', 'english'),
+      /** Typo tolerance: 'off' to disable, 'auto' for defaults, or { minTokenLength, maxDistance, similarity }. */
+      typoTolerance: env('SEARCH_TYPO_TOLERANCE', 'auto'),
+      /** Per-search work_mem hint. Set to null/empty to skip. */
+      workMem: env('SEARCH_PG_WORK_MEM', '64MB'),
+      /** GIN index tuning — fastupdate=off improves read tail latency. */
+      gin: { fastupdate: false },
+      // `connection` (Bun SQL instance) is resolved from @strav/database at runtime.
+    },
   },
 }