@strav/search 0.3.20 → 0.3.22

package/src/drivers/embedded/engine/typo_expander.ts

@@ -0,0 +1,138 @@
+import type { Database } from 'bun:sqlite'
+import type { ResolvedTypoTolerance } from '../types.ts'
+
+/**
+ * Plain-text tokeniser for the terms dictionary.
+ *
+ * Lowercases input, splits on non-letter/digit boundaries, drops tokens shorter
+ * than 2 characters. We deliberately do NOT apply Porter stemming here because:
+ *
+ * - Most typos are on rare/proper nouns (e.g. customer names, product SKUs)
+ *   which Porter doesn't transform anyway.
+ * - Mirroring SQLite's stem inside JS would require shipping a Porter
+ *   implementation just for the dictionary, which is a lot of code for the
+ *   marginal gain on common-word typos.
+ *
+ * The candidate term we feed back into FTS5 is then re-stemmed by FTS5 itself,
+ * so the lookup still works.
+ */
+export function tokenize(text: string): string[] {
+  if (!text) return []
+  const tokens: string[] = []
+  for (const raw of text.toLowerCase().split(/[^\p{L}\p{N}]+/u)) {
+    if (raw.length >= 2) tokens.push(raw)
+  }
+  return tokens
+}
+
+/** Add a document's tokens to the terms dictionary, incrementing per unique term. */
+export function recordTerms(db: Database, text: string): void {
+  const unique = new Set(tokenize(text))
+  if (unique.size === 0) return
+
+  const stmt = db.prepare(
+    'INSERT INTO terms_dict (term, doc_freq) VALUES (?, 1) ' +
+      'ON CONFLICT(term) DO UPDATE SET doc_freq = doc_freq + 1'
+  )
+  for (const term of unique) stmt.run(term)
+}
+
+/** Decrement a document's tokens; remove rows that drop to zero. */
+export function unrecordTerms(db: Database, text: string): void {
+  const unique = new Set(tokenize(text))
+  if (unique.size === 0) return
+
+  const dec = db.prepare('UPDATE terms_dict SET doc_freq = doc_freq - 1 WHERE term = ?')
+  const purge = db.prepare('DELETE FROM terms_dict WHERE doc_freq <= 0')
+  for (const term of unique) dec.run(term)
+  purge.run()
+}
+
+/**
+ * For each token, return up to `maxCandidates` near-misses already present in
+ * the dictionary, using Levenshtein distance ≤ settings.maxDistance.
+ */
+export function expandTokens(
+  db: Database,
+  tokens: string[],
+  settings: ResolvedTypoTolerance,
+  maxCandidates = 8
+): Map<string, string[]> {
+  const out = new Map<string, string[]>()
+  if (!settings.enabled) return out
+
+  const stmt = db.prepare<{ term: string }, [number, number]>(
+    'SELECT term FROM terms_dict WHERE length(term) BETWEEN ? AND ?'
+  )
+
+  for (const token of tokens) {
+    if (token.length < settings.minTokenLength) continue
+
+    const minLen = Math.max(1, token.length - settings.maxDistance)
+    const maxLen = token.length + settings.maxDistance
+
+    const candidates: string[] = []
+    for (const row of stmt.all(minLen, maxLen)) {
+      if (row.term === token) continue
+      if (levenshtein(token, row.term, settings.maxDistance) <= settings.maxDistance) {
+        candidates.push(row.term)
+        if (candidates.length >= maxCandidates) break
+      }
+    }
+    if (candidates.length > 0) out.set(token, candidates)
+  }
+
+  return out
+}
+
+/** Resolve user-provided typo tolerance settings into concrete numbers. */
+export function resolveTypoTolerance(
+  setting:
+    | 'off'
+    | 'auto'
+    | { minTokenLength?: number; maxDistance?: number }
+    | undefined
+): ResolvedTypoTolerance {
+  if (setting === 'off') {
+    return { enabled: false, minTokenLength: 4, maxDistance: 1 }
+  }
+  if (setting === undefined || setting === 'auto') {
+    return { enabled: true, minTokenLength: 4, maxDistance: 1 }
+  }
+  return {
+    enabled: true,
+    minTokenLength: setting.minTokenLength ?? 4,
+    maxDistance: setting.maxDistance ?? 1,
+  }
+}
+
+/**
+ * Bounded Levenshtein distance: returns max+1 once it can prove the distance
+ * exceeds `max` so we can short-circuit. Operates on UTF-16 code units, which
+ * is fine for our supported (ASCII-ish) corpora.
+ */
+function levenshtein(a: string, b: string, max: number): number {
+  if (a === b) return 0
+  if (Math.abs(a.length - b.length) > max) return max + 1
+  if (a.length === 0) return b.length
+  if (b.length === 0) return a.length
+
+  const aLen = a.length
+  const bLen = b.length
+  let prev = new Array<number>(bLen + 1).fill(0)
+  let curr = new Array<number>(bLen + 1).fill(0)
+  for (let j = 0; j <= bLen; j++) prev[j] = j
+
+  for (let i = 1; i <= aLen; i++) {
+    curr[0] = i
+    let rowMin = curr[0]!
+    for (let j = 1; j <= bLen; j++) {
+      const cost = a.charCodeAt(i - 1) === b.charCodeAt(j - 1) ? 0 : 1
+      curr[j] = Math.min(prev[j]! + 1, curr[j - 1]! + 1, prev[j - 1]! + cost)
+      if (curr[j]! < rowMin) rowMin = curr[j]!
+    }
+    if (rowMin > max) return max + 1
+    ;[prev, curr] = [curr, prev]
+  }
+  return prev[bLen]!
+}

package/src/drivers/embedded/errors.ts

@@ -0,0 +1,15 @@
+import { SearchError } from '../../errors.ts'
+
+export class EmbeddedSearchError extends SearchError {}
+
+export class IndexCorruptError extends EmbeddedSearchError {
+  constructor(index: string, cause: string) {
+    super(`Embedded search index "${index}" is corrupt: ${cause}`)
+  }
+}
+
+export class UnsupportedFilterError extends EmbeddedSearchError {
+  constructor(message: string) {
+    super(`Embedded driver filter is unsupported: ${message}`)
+  }
+}

package/src/drivers/embedded/filters/filter_compiler.ts

@@ -0,0 +1,136 @@
+import { UnsupportedFilterError } from '../errors.ts'
+import { quoteIdent } from '../engine/schema.ts'
+
+export interface CompiledFilter {
+  /** SQL fragment to splice into a WHERE clause (no leading 'WHERE'). Empty if no filter. */
+  sql: string
+  /** Bound parameters in the order their `?` placeholders appear. */
+  params: unknown[]
+}
+
+/**
+ * Compile a filter object into a parameterized SQL WHERE fragment.
+ *
+ * Supported value shapes for each key:
+ * - primitive (string/number/boolean/null) → `key = ?`
+ * - array of primitives → `key IN (?, ?, ?)`
+ * - operator object: `{ gt, gte, lt, lte, eq, neq, in }` → composed clauses
+ *
+ * Unknown keys are NOT validated against the field registry here — callers
+ * are expected to have configured `filterableAttributes` correctly. SQLite
+ * raises a clean error if the column doesn't exist.
+ */
+export function compileFilter(
+  filter: Record<string, unknown> | string | undefined,
+  filterableAttributes: ReadonlySet<string>
+): CompiledFilter {
+  if (!filter) return { sql: '', params: [] }
+
+  if (typeof filter === 'string') {
+    throw new UnsupportedFilterError(
+      'Raw string filters are not supported by the embedded driver. ' +
+        'Pass an object like `{ status: "published" }` instead.'
+    )
+  }
+
+  const clauses: string[] = []
+  const params: unknown[] = []
+
+  for (const [key, value] of Object.entries(filter)) {
+    if (value === undefined) continue
+
+    if (!filterableAttributes.has(key)) {
+      throw new UnsupportedFilterError(
+        `Field "${key}" is not in filterableAttributes. ` +
+          'Add it to the index settings before filtering on it.'
+      )
+    }
+
+    const col = quoteIdent(key)
+
+    if (value === null) {
+      clauses.push(`${col} IS NULL`)
+      continue
+    }
+
+    if (Array.isArray(value)) {
+      if (value.length === 0) {
+        clauses.push('1 = 0')
+      } else {
+        const placeholders = value.map(() => '?').join(', ')
+        clauses.push(`${col} IN (${placeholders})`)
+        params.push(...value.map(coerce))
+      }
+      continue
+    }
+
+    if (isOperatorObject(value)) {
+      for (const [op, opValue] of Object.entries(value)) {
+        const compiled = compileOperator(col, op, opValue)
+        clauses.push(compiled.sql)
+        params.push(...compiled.params)
+      }
+      continue
+    }
+
+    if (isPrimitive(value)) {
+      clauses.push(`${col} = ?`)
+      params.push(coerce(value))
+      continue
+    }
+
+    throw new UnsupportedFilterError(
+      `Unsupported filter value for key "${key}": ${JSON.stringify(value)}`
+    )
+  }
+
+  return { sql: clauses.join(' AND '), params }
+}
+
+function isOperatorObject(value: unknown): value is Record<string, unknown> {
+  if (value === null || typeof value !== 'object' || Array.isArray(value)) return false
+  return Object.keys(value).every(k => OPERATORS.has(k))
+}
+
+const OPERATORS = new Set(['eq', 'neq', 'gt', 'gte', 'lt', 'lte', 'in', 'nin'])
+
+function compileOperator(col: string, op: string, value: unknown): CompiledFilter {
+  switch (op) {
+    case 'eq':
+      return { sql: `${col} = ?`, params: [coerce(value)] }
+    case 'neq':
+      return { sql: `${col} <> ?`, params: [coerce(value)] }
+    case 'gt':
+      return { sql: `${col} > ?`, params: [coerce(value)] }
+    case 'gte':
+      return { sql: `${col} >= ?`, params: [coerce(value)] }
+    case 'lt':
+      return { sql: `${col} < ?`, params: [coerce(value)] }
+    case 'lte':
+      return { sql: `${col} <= ?`, params: [coerce(value)] }
+    case 'in': {
+      if (!Array.isArray(value) || value.length === 0) return { sql: '1 = 0', params: [] }
+      const ph = value.map(() => '?').join(', ')
+      return { sql: `${col} IN (${ph})`, params: value.map(coerce) }
+    }
+    case 'nin': {
+      if (!Array.isArray(value) || value.length === 0) return { sql: '1 = 1', params: [] }
+      const ph = value.map(() => '?').join(', ')
+      return { sql: `${col} NOT IN (${ph})`, params: value.map(coerce) }
+    }
+    default:
+      throw new UnsupportedFilterError(`Unknown operator "${op}"`)
+  }
+}
+
+function isPrimitive(value: unknown): boolean {
+  return (
+    typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'
+  )
+}
+
+function coerce(value: unknown): unknown {
+  if (value === null || value === undefined) return null
+  if (typeof value === 'boolean') return value ? 1 : 0
+  return value
+}

package/src/drivers/embedded/index.ts

@@ -0,0 +1,3 @@
+export { EmbeddedDriver } from './embedded_driver.ts'
+export type { EmbeddedConfig, TypoToleranceMode, TypoToleranceSettings } from './types.ts'
+export { EmbeddedSearchError, IndexCorruptError, UnsupportedFilterError } from './errors.ts'

package/src/drivers/embedded/storage/paths.ts

@@ -0,0 +1,23 @@
+import { join, isAbsolute, resolve } from 'node:path'
+import { mkdirSync } from 'node:fs'
+import type { EmbeddedConfig } from '../types.ts'
+
+const MEMORY = ':memory:'
+
+/**
+ * Resolve the on-disk path for a given index, creating the parent directory
+ * if necessary. Returns ':memory:' verbatim when the config asks for it.
+ */
+export function resolveIndexPath(config: EmbeddedConfig, index: string): string {
+  const root = config.path ?? './storage/search'
+
+  if (root === MEMORY) return MEMORY
+
+  const dir = isAbsolute(root) ? root : resolve(process.cwd(), root)
+  mkdirSync(dir, { recursive: true })
+
+  const safeName = index.replace(/[^a-zA-Z0-9_.-]/g, '_')
+  return join(dir, `${safeName}.sqlite`)
+}
+
+export const MEMORY_PATH = MEMORY

package/src/drivers/embedded/types.ts

@@ -0,0 +1,34 @@
+import type { DriverConfig } from '../../types.ts'
+
+export type TypoToleranceMode = 'off' | 'auto'
+
+export interface TypoToleranceSettings {
+  /** Minimum token length to consider for fuzzy expansion (default 4). */
+  minTokenLength?: number
+  /** Maximum Levenshtein distance to tolerate (default 1; 2 is supported but slower). */
+  maxDistance?: number
+}
+
+export interface EmbeddedConfig extends DriverConfig {
+  driver: string
+  /** Directory holding the per-index `.sqlite` files. Use `:memory:` for tests. */
+  path?: string
+  /** SQLite synchronous pragma. Default 'NORMAL' (crash-safe, sub-second write loss possible). */
+  synchronous?: 'OFF' | 'NORMAL' | 'FULL'
+  /** Typo tolerance: 'off' disables; 'auto' uses defaults; object for fine-grained control. */
+  typoTolerance?: TypoToleranceMode | TypoToleranceSettings
+}
+
+/** Resolved typo tolerance settings (after defaults applied). */
+export interface ResolvedTypoTolerance {
+  enabled: boolean
+  minTokenLength: number
+  maxDistance: number
+}
+
+/** Internal row shape from the documents table. */
+export interface DocumentRow {
+  rowid: number
+  id: string
+  doc: string
+}

package/src/drivers/postgres/engine/field_registry.ts

@@ -0,0 +1,116 @@
+import type { PgIndexSettings } from '../types.ts'
+
+/** Default searchable column when no `searchableAttributes` are configured. */
+export const DEFAULT_TEXT_COLUMN = '_text'
+
+/** FTS5 weight tiers in declaration order. */
+const WEIGHT_TIERS = ['A', 'B', 'C', 'D'] as const
+type WeightTier = (typeof WEIGHT_TIERS)[number]
+
+/** Postgres column type derived from a sample value, or `text` as the conservative default. */
+type PgType = 'text' | 'integer' | 'bigint' | 'double precision' | 'boolean' | 'timestamptz'
+
+export interface TypedColumnSpec {
+  name: string
+  pgType: PgType
+  /** JSONB extraction expression: `(doc->>'name')::pgType` (cast suppressed for text). */
+  expression: string
+}
+
+/**
+ * The schema layout for one index: which document attributes feed which
+ * tsvector segment + weight, and which typed columns exist for filter/sort.
+ *
+ * Mirrors `embedded/engine/field_registry.ts` so the two drivers project
+ * documents identically. Differences:
+ * - Per-attribute weight tier (A/B/C/D) is explicit.
+ * - Typed columns are emitted as `GENERATED ALWAYS AS (...) STORED` SQL.
+ */
+export class FieldRegistry {
+  readonly searchable: string[]
+  readonly weights: Map<string, WeightTier>
+  readonly filterable: string[]
+  readonly sortable: string[]
+  readonly typedColumns: TypedColumnSpec[]
+  readonly primaryKey: string
+  readonly language: string
+
+  constructor(settings?: PgIndexSettings, language = 'english') {
+    this.primaryKey = settings?.primaryKey ?? 'id'
+    this.language = settings?.language ?? language
+    this.searchable =
+      settings?.searchableAttributes && settings.searchableAttributes.length > 0
+        ? [...settings.searchableAttributes]
+        : [DEFAULT_TEXT_COLUMN]
+
+    this.weights = new Map()
+    for (let i = 0; i < this.searchable.length; i++) {
+      const attr = this.searchable[i]!
+      const tier = (settings?.weights?.[attr] ?? WEIGHT_TIERS[Math.min(i, 3)]) as WeightTier
+      this.weights.set(attr, tier)
+    }
+
+    this.filterable = settings?.filterableAttributes ?? []
+    this.sortable = settings?.sortableAttributes ?? []
+
+    const seen = new Set<string>()
+    const typed: TypedColumnSpec[] = []
+    for (const attr of [...this.filterable, ...this.sortable]) {
+      if (seen.has(attr)) continue
+      seen.add(attr)
+      typed.push({ name: attr, pgType: 'text', expression: `(doc->>${literal(attr)})` })
+    }
+    this.typedColumns = typed
+  }
+
+  get usesDefaultTextColumn(): boolean {
+    return this.searchable.length === 1 && this.searchable[0] === DEFAULT_TEXT_COLUMN
+  }
+
+  /**
+   * Project a document into [text, tier] pairs for tsvector construction.
+   * Default mode collapses every string into one A-weighted blob.
+   */
+  projectFtsSegments(document: Record<string, unknown>): Array<{ text: string; tier: WeightTier }> {
+    if (this.usesDefaultTextColumn) {
+      return [{ text: collectStrings(document), tier: 'A' }]
+    }
+    return this.searchable.map(attr => ({
+      text: coerceText(document[attr]),
+      tier: this.weights.get(attr)!,
+    }))
+  }
+
+  /** Single string spanning all searchable text (for terms-dict tokenization). */
+  concatSearchableText(document: Record<string, unknown>): string {
+    return this.projectFtsSegments(document)
+      .map(s => s.text)
+      .filter(Boolean)
+      .join(' ')
+  }
+}
+
+function literal(value: string): string {
+  return `'${value.replace(/'/g, "''")}'`
+}
+
+function coerceText(value: unknown): string {
+  if (value === null || value === undefined) return ''
+  if (typeof value === 'string') return value
+  if (Array.isArray(value)) return value.map(v => coerceText(v)).filter(Boolean).join(' ')
+  if (typeof value === 'number' || typeof value === 'boolean') return String(value)
+  return ''
+}
+
+function collectStrings(document: Record<string, unknown>): string {
+  const parts: string[] = []
+  for (const value of Object.values(document)) {
+    if (typeof value === 'string' && value.length > 0) parts.push(value)
+    else if (Array.isArray(value)) {
+      for (const item of value) {
+        if (typeof item === 'string' && item.length > 0) parts.push(item)
+      }
+    }
+  }
+  return parts.join(' ')
+}

package/src/drivers/postgres/engine/fts_query_builder.ts

@@ -0,0 +1,105 @@
+import { quoteLiteral } from '../storage/identifiers.ts'
+
+/**
+ * Translate a user-facing query string into one that's safe for
+ * `websearch_to_tsquery`, plus extract positive tokens for typo expansion.
+ *
+ * websearch_to_tsquery already accepts Google-style syntax:
+ * - `"foo bar"`  — phrase
+ * - `-foo`       — exclude
+ * - `OR`/`AND`   — boolean (case-insensitive)
+ *
+ * It does NOT support prefix matching (`foo*`); we recognise that ourselves
+ * and emit a separate `to_tsquery('foo:*')` ORed onto the result.
+ */
+export interface ParsedQuery {
+  /** The raw query, ready to pass to `websearch_to_tsquery`. */
+  websearch: string
+  /** Positive bare tokens (no quotes/operators) — used for typo expansion. */
+  positiveTokens: string[]
+  /** Prefix tokens from `foo*` syntax — emitted separately to `to_tsquery`. */
+  prefixTokens: string[]
+  /** Whether the input was effectively empty. */
+  isEmpty: boolean
+}
+
+const PHRASE_RE = /"([^"]*)"/g
+
+export function parseQuery(input: string): ParsedQuery {
+  const trimmed = input.trim()
+  if (!trimmed) {
+    return { websearch: '', positiveTokens: [], prefixTokens: [], isEmpty: true }
+  }
+
+  const positiveTokens: string[] = []
+  const prefixTokens: string[] = []
+
+  // Strip phrases first so we don't tokenize their inner whitespace.
+  const scratch = trimmed.replace(PHRASE_RE, ' ')
+  for (const raw of scratch.split(/\s+/)) {
+    if (!raw) continue
+    let text = raw
+    if (text.startsWith('-') || text.startsWith('+')) text = text.slice(1)
+    if (text.endsWith('*')) {
+      const stem = text.slice(0, -1).toLowerCase().replace(/[^\p{L}\p{N}_-]/gu, '')
+      if (stem) prefixTokens.push(stem)
+      continue
+    }
+    if (text.toUpperCase() === 'AND' || text.toUpperCase() === 'OR') continue
+    const norm = text.toLowerCase().replace(/[^\p{L}\p{N}_-]/gu, '')
+    if (norm.length >= 2) positiveTokens.push(norm)
+  }
+
+  return { websearch: trimmed, positiveTokens, prefixTokens, isEmpty: false }
+}
+
+/**
+ * Build a tsquery SQL expression that ORs together the user's websearch query,
+ * any prefix tokens, and any typo-expanded alternatives. Returns the
+ * expression + the user-text bindings (the language is embedded as a literal
+ * since it's a per-index server-controlled value, not user input).
+ *
+ * `startAt` is the placeholder counter the caller has already used. Returned
+ * `paramCount` lets the caller continue numbering for filter/limit/offset.
+ */
+export function buildTsqueryExpression(
+  parsed: ParsedQuery,
+  expansions: Map<string, string[]>,
+  language: string,
+  startAt = 0
+): { sql: string; params: string[]; paramCount: number } {
+  const params: string[] = []
+  const fragments: string[] = []
+  const lang = `${quoteLiteral(language)}::regconfig`
+  let cursor = startAt
+  const ph = () => `$${++cursor}`
+
+  if (parsed.websearch) {
+    params.push(parsed.websearch)
+    fragments.push(`websearch_to_tsquery(${lang}, ${ph()})`)
+  }
+
+  for (const stem of parsed.prefixTokens) {
+    params.push(`${stem}:*`)
+    fragments.push(`to_tsquery(${lang}, ${ph()})`)
+  }
+
+  for (const token of parsed.positiveTokens) {
+    const cands = expansions.get(token)
+    if (!cands || cands.length === 0) continue
+    const expr = cands.map(sanitiseTsTerm).filter(Boolean).join(' | ')
+    if (!expr) continue
+    params.push(expr)
+    fragments.push(`to_tsquery(${lang}, ${ph()})`)
+  }
+
+  if (fragments.length === 0) {
+    return { sql: '', params: [], paramCount: 0 }
+  }
+  return { sql: fragments.join(' || '), params, paramCount: cursor - startAt }
+}
+
+/** Sanitise a single term for inclusion in a manually built tsquery. */
+function sanitiseTsTerm(term: string): string {
+  return term.toLowerCase().replace(/[^\p{L}\p{N}_-]/gu, '')
+}