@strav/search 0.4.30 → 1.0.0-alpha.31

package/src/drivers/embedded/engine/sqlite_engine.ts

@@ -1,255 +0,0 @@
-import { Database } from 'bun:sqlite'
-import type {
-  SearchDocument,
-  SearchOptions,
-  SearchResult,
-  SearchHit,
-  IndexSettings,
-} from '../../../types.ts'
-import type { ResolvedTypoTolerance } from '../types.ts'
-import { FieldRegistry } from './field_registry.ts'
-import { applyConnectionPragmas, createSchema, quoteIdent } from './schema.ts'
-import { compileQuery, compileQueryWithExpansions } from './fts_query_builder.ts'
-import { compileSearch } from './query_compiler.ts'
-import { formatSnippet } from './snippet_formatter.ts'
-import { recordTerms, unrecordTerms, expandTokens } from './typo_expander.ts'
-
-export interface SqliteEngineOptions {
-  path: string
-  synchronous: 'OFF' | 'NORMAL' | 'FULL'
-  typoTolerance: ResolvedTypoTolerance
-  indexName: string
-  settings?: IndexSettings
-}
-
-/**
- * One SqliteEngine wraps a single index (a single SQLite file). The driver
- * holds a Map<indexName, SqliteEngine> and lazily instantiates per index.
- */
-export class SqliteEngine {
-  readonly db: Database
-  readonly registry: FieldRegistry
-  private readonly typo: ResolvedTypoTolerance
-  private readonly indexName: string
-
-  constructor(opts: SqliteEngineOptions) {
-    this.db = new Database(opts.path)
-    applyConnectionPragmas(this.db, opts.synchronous)
-    this.registry = new FieldRegistry(opts.settings)
-    createSchema(this.db, this.registry)
-    this.typo = opts.typoTolerance
-    this.indexName = opts.indexName
-  }
-
-  // ── Writes ──────────────────────────────────────────────────────────────
-
-  upsert(id: string | number, document: Record<string, unknown>): void {
-    this.runUpsertBatch([{ id, document }])
-  }
-
-  upsertMany(documents: SearchDocument[]): void {
-    if (documents.length === 0) return
-    const batch = documents.map(d => {
-      const { id, ...rest } = d
-      return { id, document: rest as Record<string, unknown> }
-    })
-    this.runUpsertBatch(batch)
-  }
-
-  delete(id: string | number): void {
-    this.runDeleteBatch([id])
-  }
-
-  deleteMany(ids: Array<string | number>): void {
-    if (ids.length === 0) return
-    this.runDeleteBatch(ids)
-  }
-
-  /** Remove all documents from the index, leaving the schema in place. */
-  flush(): void {
-    const tx = this.db.transaction(() => {
-      this.db.exec('DELETE FROM fts')
-      this.db.exec('DELETE FROM documents')
-      this.db.exec('DELETE FROM terms_dict')
-    })
-    tx()
-  }
-
-  /** Force-merge FTS5 segments into one. Run periodically (e.g. nightly via CLI). */
-  optimize(): void {
-    this.db.exec("INSERT INTO fts(fts) VALUES('optimize')")
-  }
-
-  close(): void {
-    try {
-      this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)')
-    } catch {
-      // Ignore — closing should never throw on a checkpoint failure
-    }
-    this.db.close()
-  }
-
-  // ── Reads ───────────────────────────────────────────────────────────────
-
-  search(query: string, options?: SearchOptions): SearchResult {
-    const start = performance.now()
-    const opts = options ?? {}
-    const expression = this.buildExpression(query)
-
-    const compiled = compileSearch({
-      registry: this.registry,
-      expression,
-      search: opts,
-    })
-
-    const rows = this.db
-      .prepare<RawHitRow, any[]>(compiled.sql)
-      .all(...(compiled.params as any[]))
-    const totalRow = this.db
-      .prepare<{ n: number }, any[]>(compiled.countSql)
-      .get(...(compiled.countParams as any[]))
-
-    const projection = opts.attributesToRetrieve
-    const hits: SearchHit[] = rows.map(row => projectHit(row, compiled.snippetColumns, projection))
-
-    return {
-      hits,
-      totalHits: totalRow?.n ?? hits.length,
-      page: Math.max(1, opts.page ?? 1),
-      perPage: Math.max(1, opts.perPage ?? 20),
-      processingTimeMs: Math.round(performance.now() - start),
-    }
-  }
-
-  // ── Internals ───────────────────────────────────────────────────────────
-
-  private buildExpression(query: string) {
-    const base = compileQuery(query)
-    if (!this.typo.enabled || base.isEmpty || base.positiveTokens.length === 0) return base
-
-    const expansions = expandTokens(this.db, base.positiveTokens, this.typo)
-    if (expansions.size === 0) return base
-    return compileQueryWithExpansions(query, expansions)
-  }
-
-  private runUpsertBatch(items: Array<{ id: string | number; document: Record<string, unknown> }>) {
-    const insertDoc = this.prepareInsertDoc()
-    const fetchExisting = this.db.prepare<
-      { rowid: number; doc: string },
-      [string]
-    >('SELECT rowid, doc FROM documents WHERE id = ?')
-    const insertFts = this.prepareInsertFts()
-    const deleteFts = this.db.prepare('DELETE FROM fts WHERE rowid = ?')
-    const indexName = this.indexName
-
-    const tx = this.db.transaction(() => {
-      for (const { id, document: doc } of items) {
-        const idStr = String(id)
-        const docJson = JSON.stringify({ id, ...doc })
-        const ftsValues = this.registry.projectFtsValues(doc)
-        const typedValues = this.registry.projectTypedValues(doc)
-        const newText = this.registry.concatSearchableText(doc)
-
-        const existing = fetchExisting.get(idStr)
-        if (existing) {
-          // Update path
-          const oldDoc = JSON.parse(existing.doc) as Record<string, unknown>
-          const oldText = this.registry.concatSearchableText(oldDoc)
-          unrecordTerms(this.db, oldText)
-
-          deleteFts.run(existing.rowid)
-          insertFts.run(existing.rowid as any, ...(ftsValues as any[]))
-          this.updateDocumentRow(existing.rowid, docJson, typedValues)
-        } else {
-          const result = insertDoc.run(idStr as any, docJson as any, ...(typedValues as any[]))
-          const rowid = Number(result.lastInsertRowid)
-          insertFts.run(rowid as any, ...(ftsValues as any[]))
-        }
-
-        recordTerms(this.db, newText)
-      }
-    })
-    void indexName
-    tx()
-  }
-
-  private runDeleteBatch(ids: Array<string | number>) {
-    const fetchExisting = this.db.prepare<
-      { rowid: number; doc: string },
-      [string]
-    >('SELECT rowid, doc FROM documents WHERE id = ?')
-    const deleteDoc = this.db.prepare('DELETE FROM documents WHERE id = ?')
-    const deleteFts = this.db.prepare('DELETE FROM fts WHERE rowid = ?')
-
-    const tx = this.db.transaction(() => {
-      for (const id of ids) {
-        const idStr = String(id)
-        const existing = fetchExisting.get(idStr)
-        if (!existing) continue
-        const oldDoc = JSON.parse(existing.doc) as Record<string, unknown>
-        unrecordTerms(this.db, this.registry.concatSearchableText(oldDoc))
-        deleteFts.run(existing.rowid)
-        deleteDoc.run(idStr)
-      }
-    })
-    tx()
-  }
-
-  private prepareInsertDoc() {
-    const cols = ['id', 'doc', ...this.registry.typedColumns.map(quoteIdent)]
-    const placeholders = cols.map(() => '?').join(', ')
-    return this.db.prepare(
-      `INSERT INTO documents (${cols.join(', ')}) VALUES (${placeholders})`
-    )
-  }
-
-  private updateDocumentRow(rowid: number, docJson: string, typedValues: unknown[]) {
-    const sets = ['doc = ?']
-    for (const col of this.registry.typedColumns) sets.push(`${quoteIdent(col)} = ?`)
-    const sql = `UPDATE documents SET ${sets.join(', ')} WHERE rowid = ?`
-    this.db.prepare(sql).run(docJson as any, ...(typedValues as any[]), rowid as any)
-  }
-
-  private prepareInsertFts() {
-    const cols = ['rowid', ...this.registry.searchable.map(quoteIdent)]
-    const placeholders = cols.map(() => '?').join(', ')
-    return this.db.prepare(`INSERT INTO fts (${cols.join(', ')}) VALUES (${placeholders})`)
-  }
-}
-
-interface RawHitRow {
-  id: string
-  doc: string
-  score: number
-  [snippetCol: string]: unknown
-}
-
-function projectHit(
-  row: RawHitRow,
-  snippetCols: string[],
-  attributesToRetrieve: string[] | undefined
-): SearchHit {
-  const document = JSON.parse(row.doc) as Record<string, unknown>
-
-  let projected = document
-  if (attributesToRetrieve && attributesToRetrieve.length > 0) {
-    const out: Record<string, unknown> = {}
-    for (const attr of attributesToRetrieve) {
-      if (attr in document) out[attr] = document[attr]
-    }
-    projected = out
-  }
-
-  const hit: SearchHit = { document: projected }
-
-  if (snippetCols.length > 0) {
-    const highlights: Record<string, string> = {}
-    for (const col of snippetCols) {
-      const raw = row[`__snip_${col}`] as string | null | undefined
-      if (raw) highlights[col] = formatSnippet(raw)
-    }
-    if (Object.keys(highlights).length > 0) hit.highlights = highlights
-  }
-
-  return hit
-}

package/src/drivers/embedded/engine/typo_expander.ts

@@ -1,138 +0,0 @@
-import type { Database } from 'bun:sqlite'
-import type { ResolvedTypoTolerance } from '../types.ts'
-
-/**
- * Plain-text tokeniser for the terms dictionary.
- *
- * Lowercases input, splits on non-letter/digit boundaries, drops tokens shorter
- * than 2 characters. We deliberately do NOT apply Porter stemming here because:
- *
- * - Most typos are on rare/proper nouns (e.g. customer names, product SKUs)
- *   which Porter doesn't transform anyway.
- * - Mirroring SQLite's stem inside JS would require shipping a Porter
- *   implementation just for the dictionary, which is a lot of code for the
- *   marginal gain on common-word typos.
- *
- * The candidate term we feed back into FTS5 is then re-stemmed by FTS5 itself,
- * so the lookup still works.
- */
-export function tokenize(text: string): string[] {
-  if (!text) return []
-  const tokens: string[] = []
-  for (const raw of text.toLowerCase().split(/[^\p{L}\p{N}]+/u)) {
-    if (raw.length >= 2) tokens.push(raw)
-  }
-  return tokens
-}
-
-/** Add a document's tokens to the terms dictionary, incrementing per unique term. */
-export function recordTerms(db: Database, text: string): void {
-  const unique = new Set(tokenize(text))
-  if (unique.size === 0) return
-
-  const stmt = db.prepare(
-    'INSERT INTO terms_dict (term, doc_freq) VALUES (?, 1) ' +
-      'ON CONFLICT(term) DO UPDATE SET doc_freq = doc_freq + 1'
-  )
-  for (const term of unique) stmt.run(term)
-}
-
-/** Decrement a document's tokens; remove rows that drop to zero. */
-export function unrecordTerms(db: Database, text: string): void {
-  const unique = new Set(tokenize(text))
-  if (unique.size === 0) return
-
-  const dec = db.prepare('UPDATE terms_dict SET doc_freq = doc_freq - 1 WHERE term = ?')
-  const purge = db.prepare('DELETE FROM terms_dict WHERE doc_freq <= 0')
-  for (const term of unique) dec.run(term)
-  purge.run()
-}
-
-/**
- * For each token, return up to `maxCandidates` near-misses already present in
- * the dictionary, using Levenshtein distance ≤ settings.maxDistance.
- */
-export function expandTokens(
-  db: Database,
-  tokens: string[],
-  settings: ResolvedTypoTolerance,
-  maxCandidates = 8
-): Map<string, string[]> {
-  const out = new Map<string, string[]>()
-  if (!settings.enabled) return out
-
-  const stmt = db.prepare<{ term: string }, [number, number]>(
-    'SELECT term FROM terms_dict WHERE length(term) BETWEEN ? AND ?'
-  )
-
-  for (const token of tokens) {
-    if (token.length < settings.minTokenLength) continue
-
-    const minLen = Math.max(1, token.length - settings.maxDistance)
-    const maxLen = token.length + settings.maxDistance
-
-    const candidates: string[] = []
-    for (const row of stmt.all(minLen, maxLen)) {
-      if (row.term === token) continue
-      if (levenshtein(token, row.term, settings.maxDistance) <= settings.maxDistance) {
-        candidates.push(row.term)
-        if (candidates.length >= maxCandidates) break
-      }
-    }
-    if (candidates.length > 0) out.set(token, candidates)
-  }
-
-  return out
-}
-
-/** Resolve user-provided typo tolerance settings into concrete numbers. */
-export function resolveTypoTolerance(
-  setting:
-    | 'off'
-    | 'auto'
-    | { minTokenLength?: number; maxDistance?: number }
-    | undefined
-): ResolvedTypoTolerance {
-  if (setting === 'off') {
-    return { enabled: false, minTokenLength: 4, maxDistance: 1 }
-  }
-  if (setting === undefined || setting === 'auto') {
-    return { enabled: true, minTokenLength: 4, maxDistance: 1 }
-  }
-  return {
-    enabled: true,
-    minTokenLength: setting.minTokenLength ?? 4,
-    maxDistance: setting.maxDistance ?? 1,
-  }
-}
-
-/**
- * Bounded Levenshtein distance: returns max+1 once it can prove the distance
- * exceeds `max` so we can short-circuit. Operates on UTF-16 code units, which
- * is fine for our supported (ASCII-ish) corpora.
- */
-function levenshtein(a: string, b: string, max: number): number {
-  if (a === b) return 0
-  if (Math.abs(a.length - b.length) > max) return max + 1
-  if (a.length === 0) return b.length
-  if (b.length === 0) return a.length
-
-  const aLen = a.length
-  const bLen = b.length
-  let prev = new Array<number>(bLen + 1).fill(0)
-  let curr = new Array<number>(bLen + 1).fill(0)
-  for (let j = 0; j <= bLen; j++) prev[j] = j
-
-  for (let i = 1; i <= aLen; i++) {
-    curr[0] = i
-    let rowMin = curr[0]!
-    for (let j = 1; j <= bLen; j++) {
-      const cost = a.charCodeAt(i - 1) === b.charCodeAt(j - 1) ? 0 : 1
-      curr[j] = Math.min(prev[j]! + 1, curr[j - 1]! + 1, prev[j - 1]! + cost)
-      if (curr[j]! < rowMin) rowMin = curr[j]!
-    }
-    if (rowMin > max) return max + 1
-    ;[prev, curr] = [curr, prev]
-  }
-  return prev[bLen]!
-}

package/src/drivers/embedded/errors.ts

@@ -1,15 +0,0 @@
-import { SearchError } from '../../errors.ts'
-
-export class EmbeddedSearchError extends SearchError {}
-
-export class IndexCorruptError extends EmbeddedSearchError {
-  constructor(index: string, cause: string) {
-    super(`Embedded search index "${index}" is corrupt: ${cause}`)
-  }
-}
-
-export class UnsupportedFilterError extends EmbeddedSearchError {
-  constructor(message: string) {
-    super(`Embedded driver filter is unsupported: ${message}`)
-  }
-}

package/src/drivers/embedded/filters/filter_compiler.ts

@@ -1,136 +0,0 @@
-import { UnsupportedFilterError } from '../errors.ts'
-import { quoteIdent } from '../engine/schema.ts'
-
-export interface CompiledFilter {
-  /** SQL fragment to splice into a WHERE clause (no leading 'WHERE'). Empty if no filter. */
-  sql: string
-  /** Bound parameters in the order their `?` placeholders appear. */
-  params: unknown[]
-}
-
-/**
- * Compile a filter object into a parameterized SQL WHERE fragment.
- *
- * Supported value shapes for each key:
- * - primitive (string/number/boolean/null) → `key = ?`
- * - array of primitives → `key IN (?, ?, ?)`
- * - operator object: `{ gt, gte, lt, lte, eq, neq, in }` → composed clauses
- *
- * Unknown keys are NOT validated against the field registry here — callers
- * are expected to have configured `filterableAttributes` correctly. SQLite
- * raises a clean error if the column doesn't exist.
- */
-export function compileFilter(
-  filter: Record<string, unknown> | string | undefined,
-  filterableAttributes: ReadonlySet<string>
-): CompiledFilter {
-  if (!filter) return { sql: '', params: [] }
-
-  if (typeof filter === 'string') {
-    throw new UnsupportedFilterError(
-      'Raw string filters are not supported by the embedded driver. ' +
-        'Pass an object like `{ status: "published" }` instead.'
-    )
-  }
-
-  const clauses: string[] = []
-  const params: unknown[] = []
-
-  for (const [key, value] of Object.entries(filter)) {
-    if (value === undefined) continue
-
-    if (!filterableAttributes.has(key)) {
-      throw new UnsupportedFilterError(
-        `Field "${key}" is not in filterableAttributes. ` +
-          'Add it to the index settings before filtering on it.'
-      )
-    }
-
-    const col = quoteIdent(key)
-
-    if (value === null) {
-      clauses.push(`${col} IS NULL`)
-      continue
-    }
-
-    if (Array.isArray(value)) {
-      if (value.length === 0) {
-        clauses.push('1 = 0')
-      } else {
-        const placeholders = value.map(() => '?').join(', ')
-        clauses.push(`${col} IN (${placeholders})`)
-        params.push(...value.map(coerce))
-      }
-      continue
-    }
-
-    if (isOperatorObject(value)) {
-      for (const [op, opValue] of Object.entries(value)) {
-        const compiled = compileOperator(col, op, opValue)
-        clauses.push(compiled.sql)
-        params.push(...compiled.params)
-      }
-      continue
-    }
-
-    if (isPrimitive(value)) {
-      clauses.push(`${col} = ?`)
-      params.push(coerce(value))
-      continue
-    }
-
-    throw new UnsupportedFilterError(
-      `Unsupported filter value for key "${key}": ${JSON.stringify(value)}`
-    )
-  }
-
-  return { sql: clauses.join(' AND '), params }
-}
-
-function isOperatorObject(value: unknown): value is Record<string, unknown> {
-  if (value === null || typeof value !== 'object' || Array.isArray(value)) return false
-  return Object.keys(value).every(k => OPERATORS.has(k))
-}
-
-const OPERATORS = new Set(['eq', 'neq', 'gt', 'gte', 'lt', 'lte', 'in', 'nin'])
-
-function compileOperator(col: string, op: string, value: unknown): CompiledFilter {
-  switch (op) {
-    case 'eq':
-      return { sql: `${col} = ?`, params: [coerce(value)] }
-    case 'neq':
-      return { sql: `${col} <> ?`, params: [coerce(value)] }
-    case 'gt':
-      return { sql: `${col} > ?`, params: [coerce(value)] }
-    case 'gte':
-      return { sql: `${col} >= ?`, params: [coerce(value)] }
-    case 'lt':
-      return { sql: `${col} < ?`, params: [coerce(value)] }
-    case 'lte':
-      return { sql: `${col} <= ?`, params: [coerce(value)] }
-    case 'in': {
-      if (!Array.isArray(value) || value.length === 0) return { sql: '1 = 0', params: [] }
-      const ph = value.map(() => '?').join(', ')
-      return { sql: `${col} IN (${ph})`, params: value.map(coerce) }
-    }
-    case 'nin': {
-      if (!Array.isArray(value) || value.length === 0) return { sql: '1 = 1', params: [] }
-      const ph = value.map(() => '?').join(', ')
-      return { sql: `${col} NOT IN (${ph})`, params: value.map(coerce) }
-    }
-    default:
-      throw new UnsupportedFilterError(`Unknown operator "${op}"`)
-  }
-}
-
-function isPrimitive(value: unknown): boolean {
-  return (
-    typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'
-  )
-}
-
-function coerce(value: unknown): unknown {
-  if (value === null || value === undefined) return null
-  if (typeof value === 'boolean') return value ? 1 : 0
-  return value
-}

package/src/drivers/embedded/index.ts

@@ -1,3 +0,0 @@
-export { EmbeddedDriver } from './embedded_driver.ts'
-export type { EmbeddedConfig, TypoToleranceMode, TypoToleranceSettings } from './types.ts'
-export { EmbeddedSearchError, IndexCorruptError, UnsupportedFilterError } from './errors.ts'

package/src/drivers/embedded/storage/paths.ts

@@ -1,23 +0,0 @@
-import { join, isAbsolute, resolve } from 'node:path'
-import { mkdirSync } from 'node:fs'
-import type { EmbeddedConfig } from '../types.ts'
-
-const MEMORY = ':memory:'
-
-/**
- * Resolve the on-disk path for a given index, creating the parent directory
- * if necessary. Returns ':memory:' verbatim when the config asks for it.
- */
-export function resolveIndexPath(config: EmbeddedConfig, index: string): string {
-  const root = config.path ?? './storage/search'
-
-  if (root === MEMORY) return MEMORY
-
-  const dir = isAbsolute(root) ? root : resolve(process.cwd(), root)
-  mkdirSync(dir, { recursive: true })
-
-  const safeName = index.replace(/[^a-zA-Z0-9_.-]/g, '_')
-  return join(dir, `${safeName}.sqlite`)
-}
-
-export const MEMORY_PATH = MEMORY

package/src/drivers/embedded/types.ts

@@ -1,34 +0,0 @@
-import type { DriverConfig } from '../../types.ts'
-
-export type TypoToleranceMode = 'off' | 'auto'
-
-export interface TypoToleranceSettings {
-  /** Minimum token length to consider for fuzzy expansion (default 4). */
-  minTokenLength?: number
-  /** Maximum Levenshtein distance to tolerate (default 1; 2 is supported but slower). */
-  maxDistance?: number
-}
-
-export interface EmbeddedConfig extends DriverConfig {
-  driver: string
-  /** Directory holding the per-index `.sqlite` files. Use `:memory:` for tests. */
-  path?: string
-  /** SQLite synchronous pragma. Default 'NORMAL' (crash-safe, sub-second write loss possible). */
-  synchronous?: 'OFF' | 'NORMAL' | 'FULL'
-  /** Typo tolerance: 'off' disables; 'auto' uses defaults; object for fine-grained control. */
-  typoTolerance?: TypoToleranceMode | TypoToleranceSettings
-}
-
-/** Resolved typo tolerance settings (after defaults applied). */
-export interface ResolvedTypoTolerance {
-  enabled: boolean
-  minTokenLength: number
-  maxDistance: number
-}
-
-/** Internal row shape from the documents table. */
-export interface DocumentRow {
-  rowid: number
-  id: string
-  doc: string
-}