@strav/search 0.3.20 → 0.3.21

package/README.md

@@ -1,6 +1,6 @@
 # @strav/search
 
-Full-text search for the [Strav](https://www.npmjs.com/package/@strav/core) framework. Unified API for Meilisearch, Typesense, and Algolia with automatic indexing via model events.
+Full-text search for the [Strav](https://www.npmjs.com/package/@strav/core) framework. Unified API across several engines — including a built-in `embedded` driver that runs in-process with no external service to deploy.
 
 ## Install
 
@@ -51,16 +51,87 @@ await search.delete('posts', ['1'])
 
 ## Drivers
 
+- **Embedded** — in-process SQLite FTS5, zero deps, recommended for self-host / SMB
 - **Meilisearch** — fast, typo-tolerant, self-hosted
 - **Typesense** — open-source, instant search
 - **Algolia** — hosted search-as-a-service
 - **Null** — no-op driver for testing
 
+### Embedded driver
+
+Runs entirely inside your app process using `bun:sqlite`'s FTS5 engine — no Meilisearch/Typesense container to run. Each index is a single `.sqlite` file in the configured data directory.
+
+Features:
+
+- BM25 ranking with per-field weights (via `searchableAttributes` ordering)
+- Prefix (`type*`), phrase (`"quick brown fox"`), negation (`-foo`), required (`+foo`)
+- Porter stemmer for English morphology
+- Typo tolerance (Levenshtein-1) on the fly, configurable
+- Highlighted snippets with `<mark>` tags
+- Object-form filters with equality, `in`, and comparison operators
+
+Limitations for v1:
+
+- English stemming only (other languages are tokenised but not stemmed)
+- One writer at a time per index file (SQLite WAL — concurrent reads are fine)
+- Object-form filters only; raw SQL filter strings are rejected
+- Index settings changes require recreating the index
+
+Configuration:
+
+```ts
+// config/search.ts
+embedded: {
+  driver: 'embedded',
+  path: env('SEARCH_PATH', './storage/search'),   // directory of .sqlite files
+  synchronous: 'NORMAL',                            // 'OFF' | 'NORMAL' | 'FULL'
+  typoTolerance: 'auto',                            // 'off' | 'auto' | { minTokenLength, maxDistance }
+}
+```
+
+Select it as the default with `SEARCH_DRIVER=embedded`.
+
+Model example with per-field weights (column order determines BM25 weight — title first = highest):
+
+```ts
+class Ticket extends searchable(BaseModel) {
+  static searchableSettings() {
+    return {
+      searchableAttributes: ['subject', 'body'],
+      filterableAttributes: ['status', 'priority'],
+      sortableAttributes: ['priority', 'created_at'],
+    }
+  }
+}
+```
+
+#### Replacing Postgres `tsvector`
+
+If you've been using raw `tsvector` columns, the embedded driver gives you better ranking, typo tolerance, and highlighted snippets without adding a network service. The migration is roughly:
+
+```ts
+// Before: hand-rolled tsvector query
+const rows = await db.sql`
+  SELECT id, subject, ts_rank_cd(fts, q) AS rank
+  FROM tickets, websearch_to_tsquery('english', ${q}) q
+  WHERE fts @@ q ORDER BY rank DESC LIMIT 20
+`
+
+// After: searchable() + embedded driver
+const results = await Ticket.search(q, {
+  perPage: 20,
+  attributesToHighlight: ['subject', 'body'],
+})
+```
+
+You run `bun strav search:import Ticket` once to populate the index, then model events keep it up to date.
+
 ## CLI
 
 ```bash
-bun strav search:import    # Import all searchable models
-bun strav search:flush     # Flush all indexes
+bun strav search:import <model>      # Import all records for a model
+bun strav search:flush <model>       # Flush all documents from an index
+bun strav search:optimize <model>    # (embedded) Merge FTS5 segments; run periodically
 ```
 
 ## Documentation

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@strav/search",
-  "version": "0.3.20",
+  "version": "0.3.21",
   "type": "module",
   "description": "Full-text search for the Strav framework",
   "license": "MIT",
@@ -18,9 +18,9 @@
     "tsconfig.json"
   ],
   "peerDependencies": {
-    "@strav/kernel": "0.3.20",
-    "@strav/database": "0.3.20",
-    "@strav/cli": "0.3.20"
+    "@strav/kernel": "0.3.21",
+    "@strav/database": "0.3.21",
+    "@strav/cli": "0.3.21"
   },
   "scripts": {
     "test": "bun test tests/",

package/src/commands/search_optimize.ts

@@ -0,0 +1,52 @@
+import type { Command } from 'commander'
+import chalk from 'chalk'
+import { bootstrap, shutdown } from '@strav/cli'
+import { BaseModel } from '@strav/database'
+import SearchManager from '../search_manager.ts'
+import { EmbeddedDriver } from '../drivers/embedded/index.ts'
+
+export function register(program: Command): void {
+  program
+    .command('search:optimize <model>')
+    .description("Merge FTS5 segments for a model's index (embedded driver only)")
+    .action(async (modelPath: string) => {
+      let db
+      try {
+        const { db: database, config } = await bootstrap()
+        db = database
+
+        new BaseModel(db)
+        new SearchManager(config)
+
+        const resolved = require.resolve(`${process.cwd()}/${modelPath}`)
+        const module = await import(resolved)
+        const ModelClass = module.default ?? (Object.values(module)[0] as any)
+
+        if (typeof ModelClass?.searchableAs !== 'function') {
+          console.error(chalk.red(`Model "${modelPath}" does not use the searchable() mixin.`))
+          process.exit(1)
+        }
+
+        const indexName = SearchManager.indexName(ModelClass.searchableAs())
+        const engine = SearchManager.engine()
+
+        if (!(engine instanceof EmbeddedDriver)) {
+          console.error(
+            chalk.red(
+              `search:optimize is only meaningful for the embedded driver (current: ${engine.name}).`
+            )
+          )
+          process.exit(1)
+        }
+
+        console.log(chalk.dim(`Optimizing "${indexName}"...`))
+        engine.optimize(indexName)
+        console.log(chalk.green(`Optimized "${indexName}".`))
+      } catch (err) {
+        console.error(chalk.red(`Error: ${err instanceof Error ? err.message : err}`))
+        process.exit(1)
+      } finally {
+        if (db) await shutdown(db)
+      }
+    })
+}

package/src/drivers/embedded/embedded_driver.ts

@@ -0,0 +1,136 @@
+import type { SearchEngine } from '../../search_engine.ts'
+import type {
+  SearchDocument,
+  SearchOptions,
+  SearchResult,
+  IndexSettings,
+  DriverConfig,
+} from '../../types.ts'
+import { SqliteEngine } from './engine/sqlite_engine.ts'
+import { resolveTypoTolerance } from './engine/typo_expander.ts'
+import { resolveIndexPath, MEMORY_PATH } from './storage/paths.ts'
+import type { EmbeddedConfig, ResolvedTypoTolerance } from './types.ts'
+
+/**
+ * In-process full-text search driver backed by SQLite FTS5.
+ *
+ * Each index lives in its own SQLite file (or `:memory:` for tests). The
+ * driver maintains a `Map<indexName, SqliteEngine>` and creates engines
+ * lazily on first reference. This means a fresh `upsert()` against a
+ * never-created index will auto-create a default schema (single `_text`
+ * column). Callers that want per-field weights call `createIndex()` first
+ * with their `IndexSettings`.
+ */
+export class EmbeddedDriver implements SearchEngine {
+  readonly name = 'embedded'
+
+  private readonly config: EmbeddedConfig
+  private readonly synchronous: 'OFF' | 'NORMAL' | 'FULL'
+  private readonly typo: ResolvedTypoTolerance
+  private readonly engines = new Map<string, SqliteEngine>()
+  /** Pending settings for indexes that haven't been opened yet. */
+  private readonly pendingSettings = new Map<string, IndexSettings>()
+
+  constructor(config: DriverConfig) {
+    this.config = (config ?? {}) as EmbeddedConfig
+    this.synchronous = this.config.synchronous ?? 'NORMAL'
+    this.typo = resolveTypoTolerance(this.config.typoTolerance)
+  }
+
+  // ── Document operations ──────────────────────────────────────────────────
+
+  async upsert(
+    index: string,
+    id: string | number,
+    document: Record<string, unknown>
+  ): Promise<void> {
+    this.engineFor(index).upsert(id, document)
+  }
+
+  async upsertMany(index: string, documents: SearchDocument[]): Promise<void> {
+    this.engineFor(index).upsertMany(documents)
+  }
+
+  async delete(index: string, id: string | number): Promise<void> {
+    this.engineFor(index).delete(id)
+  }
+
+  async deleteMany(index: string, ids: Array<string | number>): Promise<void> {
+    this.engineFor(index).deleteMany(ids)
+  }
+
+  // ── Index operations ─────────────────────────────────────────────────────
+
+  async flush(index: string): Promise<void> {
+    this.engineFor(index).flush()
+  }
+
+  async deleteIndex(index: string): Promise<void> {
+    const engine = this.engines.get(index)
+    if (engine) {
+      engine.close()
+      this.engines.delete(index)
+    }
+    this.pendingSettings.delete(index)
+
+    const path = resolveIndexPath(this.config, index)
+    if (path === MEMORY_PATH) return
+
+    const fs = await import('node:fs/promises')
+    for (const suffix of ['', '-wal', '-shm']) {
+      try {
+        await fs.unlink(`${path}${suffix}`)
+      } catch (err: any) {
+        if (err?.code !== 'ENOENT') throw err
+      }
+    }
+  }
+
+  async createIndex(index: string, options?: IndexSettings): Promise<void> {
+    if (options) this.pendingSettings.set(index, options)
+    // Force engine instantiation so the schema exists on disk.
+    this.engineFor(index)
+  }
+
+  // ── Search ───────────────────────────────────────────────────────────────
+
+  async search(index: string, query: string, options?: SearchOptions): Promise<SearchResult> {
+    return this.engineFor(index).search(query, options)
+  }
+
+  // ── Lifecycle ────────────────────────────────────────────────────────────
+
+  /** Close all open engines. Call from app shutdown. */
+  close(): void {
+    for (const engine of this.engines.values()) engine.close()
+    this.engines.clear()
+  }
+
+  /** Run FTS5 segment merge on every open index. Use from CLI for periodic ops. */
+  optimize(index?: string): void {
+    if (index) {
+      this.engineFor(index).optimize()
+      return
+    }
+    for (const engine of this.engines.values()) engine.optimize()
+  }
+
+  // ── Internals ────────────────────────────────────────────────────────────
+
+  private engineFor(index: string): SqliteEngine {
+    let engine = this.engines.get(index)
+    if (engine) return engine
+
+    const settings = this.pendingSettings.get(index)
+    engine = new SqliteEngine({
+      path: resolveIndexPath(this.config, index),
+      synchronous: this.synchronous,
+      typoTolerance: this.typo,
+      indexName: index,
+      settings,
+    })
+    this.engines.set(index, engine)
+    this.pendingSettings.delete(index)
+    return engine
+  }
+}

package/src/drivers/embedded/engine/field_registry.ts

@@ -0,0 +1,97 @@
+import type { IndexSettings } from '../../../types.ts'
+
+/** The default searchable column name used when no `searchableAttributes` are configured. */
+export const DEFAULT_TEXT_COLUMN = '_text'
+
+/**
+ * The schema layout for one index: which document attributes feed which FTS5
+ * column and which typed `documents` columns exist for filtering / sorting.
+ *
+ * When a caller doesn't declare `searchableAttributes`, we fall back to a
+ * single `_text` column that concatenates every string-valued field at
+ * indexing time. Users who want per-field weights opt in by passing
+ * `IndexSettings`.
+ */
+export class FieldRegistry {
+  /** FTS5 columns in declaration order — also the order BM25 weights apply in. */
+  readonly searchable: string[]
+  /** Filterable attributes — materialized as typed columns on `documents`. */
+  readonly filterable: string[]
+  /** Sortable attributes — materialized as typed columns on `documents`. */
+  readonly sortable: string[]
+  /** Union of filterable + sortable, deduplicated. */
+  readonly typedColumns: string[]
+  /** Primary key field name — defaults to 'id'. */
+  readonly primaryKey: string
+
+  constructor(settings?: IndexSettings) {
+    this.primaryKey = settings?.primaryKey ?? 'id'
+    this.searchable =
+      settings?.searchableAttributes && settings.searchableAttributes.length > 0
+        ? [...settings.searchableAttributes]
+        : [DEFAULT_TEXT_COLUMN]
+    this.filterable = settings?.filterableAttributes ?? []
+    this.sortable = settings?.sortableAttributes ?? []
+    this.typedColumns = Array.from(new Set([...this.filterable, ...this.sortable]))
+  }
+
+  /** Whether this registry uses the synthesised `_text` column. */
+  get usesDefaultTextColumn(): boolean {
+    return this.searchable.length === 1 && this.searchable[0] === DEFAULT_TEXT_COLUMN
+  }
+
+  /**
+   * Project a document into the values that go into the FTS5 row.
+   * For default mode, concatenate every string-valued field.
+   * For declared mode, pick each named attribute (coerced to string).
+   */
+  projectFtsValues(document: Record<string, unknown>): string[] {
+    if (this.usesDefaultTextColumn) {
+      const parts: string[] = []
+      for (const value of Object.values(document)) {
+        if (typeof value === 'string' && value.length > 0) parts.push(value)
+        else if (Array.isArray(value)) {
+          for (const item of value) {
+            if (typeof item === 'string' && item.length > 0) parts.push(item)
+          }
+        }
+      }
+      return [parts.join(' ')]
+    }
+
+    return this.searchable.map(attr => coerceText(document[attr]))
+  }
+
+  /**
+   * Project a document into the typed-column values stored on `documents`.
+   * Returned in the same order as `typedColumns`.
+   */
+  projectTypedValues(document: Record<string, unknown>): unknown[] {
+    return this.typedColumns.map(attr => coerceTyped(document[attr]))
+  }
+
+  /**
+   * Concatenate every searchable attribute into one long string suitable for
+   * tokenization (used for terms-dictionary maintenance).
+   */
+  concatSearchableText(document: Record<string, unknown>): string {
+    return this.projectFtsValues(document).join(' ')
+  }
+}
+
+function coerceText(value: unknown): string {
+  if (value === null || value === undefined) return ''
+  if (typeof value === 'string') return value
+  if (Array.isArray(value)) {
+    return value.map(v => coerceText(v)).filter(Boolean).join(' ')
+  }
+  if (typeof value === 'number' || typeof value === 'boolean') return String(value)
+  return ''
+}
+
+function coerceTyped(value: unknown): unknown {
+  if (value === null || value === undefined) return null
+  if (Array.isArray(value)) return JSON.stringify(value)
+  if (typeof value === 'object') return JSON.stringify(value)
+  return value
+}

package/src/drivers/embedded/engine/fts_query_builder.ts

@@ -0,0 +1,184 @@
+/**
+ * Translate a user-facing query string into a sanitized FTS5 MATCH expression.
+ *
+ * Supported syntax (subset of Google-style search):
+ * - `"foo bar"`   — exact phrase
+ * - `-foo`        — exclude documents containing this token
+ * - `+foo`        — required (default for all positive tokens — accepted for symmetry)
+ * - `foo*`        — prefix match
+ *
+ * Everything else is treated as a positive ANDed token.
+ *
+ * Defends against FTS5 syntax injection by stripping or escaping any FTS5
+ * operator characters from raw user tokens. The user never gets to write a
+ * raw MATCH expression.
+ */
+export interface FtsExpression {
+  /** Final MATCH expression, ready to bind into a query. */
+  match: string
+  /** The positive tokens (no quotes, no operators) — useful for typo expansion. */
+  positiveTokens: string[]
+  /** Whether the expression is empty (caller may short-circuit to "match all"). */
+  isEmpty: boolean
+}
+
+interface ParsedToken {
+  text: string
+  negate: boolean
+  phrase: boolean
+  prefix: boolean
+}
+
+const FTS5_RESERVED = /["()*:^]/g
+const PHRASE_RE = /"([^"]*)"/g
+
+export function compileQuery(input: string): FtsExpression {
+  const trimmed = input.trim()
+  if (!trimmed) return { match: '', positiveTokens: [], isEmpty: true }
+
+  const tokens = parseTokens(trimmed)
+  if (tokens.length === 0) return { match: '', positiveTokens: [], isEmpty: true }
+
+  const positives: string[] = []
+  const negatives: string[] = []
+  const positiveTokens: string[] = []
+
+  for (const tok of tokens) {
+    const rendered = renderToken(tok)
+    if (!rendered) continue
+
+    if (tok.negate) {
+      negatives.push(rendered)
+    } else {
+      positives.push(rendered)
+      if (!tok.phrase && !tok.prefix) positiveTokens.push(tok.text.toLowerCase())
+    }
+  }
+
+  if (positives.length === 0 && negatives.length === 0) {
+    return { match: '', positiveTokens: [], isEmpty: true }
+  }
+
+  // Pure-negative queries can't be expressed in FTS5 — fall back to no-match.
+  if (positives.length === 0) {
+    return { match: '', positiveTokens: [], isEmpty: true }
+  }
+
+  let expr = positives.join(' AND ')
+  if (negatives.length > 0) {
+    expr = `${expr} NOT (${negatives.join(' OR ')})`
+  }
+
+  return { match: expr, positiveTokens, isEmpty: false }
+}
+
+/**
+ * Re-render a previously parsed query but with extra OR-candidates injected
+ * for each positive token. Used by the typo expander.
+ */
+export function compileQueryWithExpansions(
+  input: string,
+  expansions: Map<string, string[]>
+): FtsExpression {
+  const trimmed = input.trim()
+  if (!trimmed) return { match: '', positiveTokens: [], isEmpty: true }
+
+  const tokens = parseTokens(trimmed)
+  const positives: string[] = []
+  const negatives: string[] = []
+  const positiveTokens: string[] = []
+
+  for (const tok of tokens) {
+    if (tok.negate) {
+      const r = renderToken(tok)
+      if (r) negatives.push(r)
+      continue
+    }
+
+    if (tok.phrase || tok.prefix) {
+      const r = renderToken(tok)
+      if (r) positives.push(r)
+      continue
+    }
+
+    const sanitized = sanitizeBareToken(tok.text)
+    if (!sanitized) continue
+    positiveTokens.push(sanitized.toLowerCase())
+
+    const cands = expansions.get(sanitized.toLowerCase()) ?? []
+    if (cands.length === 0) {
+      positives.push(sanitized)
+    } else {
+      const all = [sanitized, ...cands].map(t => sanitizeBareToken(t)).filter(Boolean) as string[]
+      const unique = Array.from(new Set(all))
+      positives.push(`(${unique.join(' OR ')})`)
+    }
+  }
+
+  if (positives.length === 0) return { match: '', positiveTokens: [], isEmpty: true }
+
+  let expr = positives.join(' AND ')
+  if (negatives.length > 0) {
+    expr = `${expr} NOT (${negatives.join(' OR ')})`
+  }
+  return { match: expr, positiveTokens, isEmpty: false }
+}
+
+function parseTokens(input: string): ParsedToken[] {
+  const tokens: ParsedToken[] = []
+  let cursor = 0
+  let working = input
+
+  // Pull out phrase tokens first to avoid splitting on inner whitespace.
+  working = working.replace(PHRASE_RE, (_, phrase, offset) => {
+    const negate = offset > 0 && input[offset - 1] === '-'
+    tokens.push({ text: phrase, negate, phrase: true, prefix: false })
+    return ' '.repeat(_.length + (negate ? 1 : 0))
+  })
+
+  for (const raw of working.split(/\s+/)) {
+    if (!raw) continue
+    let text = raw
+    let negate = false
+    let prefix = false
+
+    if (text.startsWith('-')) {
+      negate = true
+      text = text.slice(1)
+    } else if (text.startsWith('+')) {
+      text = text.slice(1)
+    }
+    if (text.endsWith('*')) {
+      prefix = true
+      text = text.slice(0, -1)
+    }
+    if (!text) continue
+
+    tokens.push({ text, negate, phrase: false, prefix })
+  }
+
+  void cursor
+  return tokens
+}
+
+function renderToken(tok: ParsedToken): string | null {
+  if (tok.phrase) {
+    const cleaned = tok.text.replace(/"/g, '').trim()
+    if (!cleaned) return null
+    return `"${cleaned}"`
+  }
+  const sanitized = sanitizeBareToken(tok.text)
+  if (!sanitized) return null
+  return tok.prefix ? `${sanitized}*` : sanitized
+}
+
+function sanitizeBareToken(token: string): string {
+  // Replace any FTS5 operator characters with a space, then collapse to one
+  // word. If only one word survives we use it bare; otherwise wrap in quotes
+  // so FTS5 treats it as a phrase rather than two ANDed tokens.
+  const cleaned = token.replace(FTS5_RESERVED, ' ').trim()
+  if (!cleaned) return ''
+  const parts = cleaned.split(/\s+/).filter(Boolean)
+  if (parts.length === 1) return parts[0]!
+  return `"${parts.join(' ')}"`
+}