claude-brain 0.30.2 → 0.30.3

package/src/memory/fts5-search.ts

@@ -1,633 +1,692 @@
-/**
- * FTS5 Search Service — Phase 26
- * Full-text search over the observations table using SQLite FTS5.
- * Replaces ChromaDB as the primary search backend.
- */
-
-import { randomUUID } from 'crypto'
-import type { Database } from 'bun:sqlite'
-import type { Logger } from 'pino'
-import { expandQuery } from '@/retrieval/query/expander'
-import { embeddingToBuffer, bufferToEmbedding, cosineSimilarity } from './embedding-utils'
-import type { EmbeddingService } from './embeddings'
-
-export type ObservationCategory = 'decision' | 'pattern' | 'correction' | 'insight' | 'preference'
-
-export interface NewObservation {
-  project: string
-  category: ObservationCategory
-  content: string
-  reasoning?: string
-  context?: string
-  confidence?: number
-  source?: string
-  tags?: string[]
-  file_paths?: string[]
-  symbols?: string[]
-}
-
-export interface ObservationResult {
-  id: string
-  project: string
-  category: ObservationCategory
-  content: string
-  reasoning: string | null
-  context: string | null
-  confidence: number
-  source: string
-  tags: string[]
-  file_paths: string[]
-  symbols: string[]
-  access_count: number
-  last_accessed: string | null
-  created_at: string
-  updated_at: string
-  archived: boolean
-}
-
-export interface ScoredResult extends ObservationResult {
-  score: number
-}
-
-export interface DuplicateResult {
-  id: string
-  content: string
-  score: number
-}
-
-export class FTS5Search {
-  private db: Database
-  private logger: Logger
-
-  constructor(db: Database, logger: Logger) {
-    this.db = db
-    this.logger = logger.child({ component: 'fts5-search' })
-  }
-
-  /**
-   * Search observations via FTS5 full-text search.
-   */
-  search(query: string, project?: string, limit: number = 10): ObservationResult[] {
-    if (!query.trim()) return []
-
-    const ftsQuery = this.buildFTSQuery(query)
-
-    try {
-      let sql: string
-      const params: any[] = []
-
-      if (project) {
-        sql = `
-          SELECT o.*, rank
-          FROM observations o
-          JOIN observations_fts fts ON o.rowid = fts.rowid
-          WHERE observations_fts MATCH ? AND o.project = ? AND o.archived = 0
-          ORDER BY rank
-          LIMIT ?
-        `
-        params.push(ftsQuery, project, limit)
-      } else {
-        sql = `
-          SELECT o.*, rank
-          FROM observations o
-          JOIN observations_fts fts ON o.rowid = fts.rowid
-          WHERE observations_fts MATCH ? AND o.archived = 0
-          ORDER BY rank
-          LIMIT ?
-        `
-        params.push(ftsQuery, limit)
-      }
-
-      const rows = this.db.prepare(sql).all(...params) as any[]
-      return rows.map(row => this.rowToResult(row))
-    } catch (error) {
-      this.logger.warn({ error, query, ftsQuery }, 'FTS5 search failed, trying fallback')
-      return this.fallbackSearch(query, project, limit)
-    }
-  }
-
-  /**
-   * Search with BM25 ranking and confidence scoring.
-   * Returns results with a normalized score between 0 and 1.
-   */
-  searchWithConfidence(query: string, project?: string, limit: number = 10): ScoredResult[] {
-    if (!query.trim()) return []
-
-    const ftsQuery = this.buildFTSQuery(query)
-    const queryLower = query.toLowerCase()
-
-    try {
-      let sql: string
-      const params: any[] = []
-
-      if (project) {
-        sql = `
-          SELECT o.*, bm25(observations_fts) as bm25_score
-          FROM observations o
-          JOIN observations_fts fts ON o.rowid = fts.rowid
-          WHERE observations_fts MATCH ? AND o.project = ? AND o.archived = 0
-          ORDER BY bm25_score
-          LIMIT ?
-        `
-        params.push(ftsQuery, project, limit)
-      } else {
-        sql = `
-          SELECT o.*, bm25(observations_fts) as bm25_score
-          FROM observations o
-          JOIN observations_fts fts ON o.rowid = fts.rowid
-          WHERE observations_fts MATCH ? AND o.archived = 0
-          ORDER BY bm25_score
-          LIMIT ?
-        `
-        params.push(ftsQuery, limit)
-      }
-
-      const rows = this.db.prepare(sql).all(...params) as any[]
-
-      return rows.map(row => {
-        const result = this.rowToResult(row)
-        const bm25 = Math.abs(row.bm25_score as number)
-
-        // Compute confidence score
-        let score = this.normalizeBM25(bm25)
-
-        // Boost for exact content match
-        if (result.content.toLowerCase().includes(queryLower)) {
-          score = Math.min(1.0, score + 0.15)
-        }
-
-        // Boost for tag match
-        if (result.tags.some(t => queryLower.includes(t.toLowerCase()))) {
-          score = Math.min(1.0, score + 0.1)
-        }
-
-        // Boost for project match
-        if (project && result.project === project) {
-          score = Math.min(1.0, score + 0.05)
-        }
-
-        return { ...result, score }
-      }).sort((a, b) => b.score - a.score)
-    } catch (error) {
-      this.logger.warn({ error, query, ftsQuery }, 'FTS5 confidence search failed, trying fallback')
-      return this.fallbackSearch(query, project, limit).map(r => ({ ...r, score: 0.5 }))
-    }
-  }
-
-  /**
-   * Store a new observation. Returns the generated ID.
-   */
-  store(observation: NewObservation, providedId?: string): string {
-    const id = providedId || randomUUID()
-    const now = new Date().toISOString()
-
-    const stmt = this.db.prepare(`
-      INSERT INTO observations (id, project, category, content, reasoning, context, confidence, source, tags, file_paths, symbols, access_count, last_accessed, created_at, updated_at, archived)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, NULL, ?, ?, 0)
-    `)
-
-    stmt.run(
-      id,
-      observation.project,
-      observation.category,
-      observation.content,
-      observation.reasoning || null,
-      observation.context || null,
-      observation.confidence ?? 0.8,
-      observation.source || 'explicit',
-      observation.tags ? JSON.stringify(observation.tags) : null,
-      observation.file_paths ? JSON.stringify(observation.file_paths) : null,
-      observation.symbols ? JSON.stringify(observation.symbols) : null,
-      now,
-      now
-    )
-
-    this.logger.debug({ id, category: observation.category, project: observation.project }, 'Observation stored')
-    return id
-  }
-
-  /**
-   * Check for duplicates via FTS5 text similarity.
-   * Returns the best matching duplicate if above threshold, null otherwise.
-   */
-  searchForDuplicates(content: string, project: string, threshold: number = 0.85): DuplicateResult | null {
-    const ftsQuery = this.buildFTSQuery(content)
-    if (!ftsQuery.trim()) return null
-
-    try {
-      const rows = this.db.prepare(`
-        SELECT o.id, o.content, bm25(observations_fts) as bm25_score
-        FROM observations o
-        JOIN observations_fts fts ON o.rowid = fts.rowid
-        WHERE observations_fts MATCH ? AND o.project = ? AND o.archived = 0
-        ORDER BY bm25_score
-        LIMIT 3
-      `).all(ftsQuery, project) as any[]
-
-      if (rows.length === 0) return null
-
-      // Use word overlap as a proxy for semantic similarity
-      for (const row of rows) {
-        const similarity = this.wordOverlap(content, row.content as string)
-        if (similarity >= threshold) {
-          return {
-            id: row.id as string,
-            content: row.content as string,
-            score: similarity
-          }
-        }
-      }
-
-      return null
-    } catch (error) {
-      this.logger.warn({ error }, 'Duplicate check failed')
-      return null
-    }
-  }
-
-  /**
-   * Fetch all observations for a project, optionally filtered by category.
-   */
-  fetchAll(project?: string, category?: ObservationCategory): ObservationResult[] {
-    let sql: string
-    const params: any[] = []
-
-    if (project && category) {
-      sql = `SELECT * FROM observations WHERE project = ? AND category = ? AND archived = 0 ORDER BY created_at DESC`
-      params.push(project, category)
-    } else if (project) {
-      sql = `SELECT * FROM observations WHERE project = ? AND archived = 0 ORDER BY created_at DESC`
-      params.push(project)
-    } else if (category) {
-      sql = `SELECT * FROM observations WHERE category = ? AND archived = 0 ORDER BY created_at DESC`
-      params.push(category)
-    } else {
-      sql = `SELECT * FROM observations WHERE archived = 0 ORDER BY created_at DESC`
-    }
-
-    const rows = this.db.prepare(sql).all(...params) as any[]
-    return rows.map(row => this.rowToResult(row))
-  }
-
-  /**
-   * Update an observation's fields.
-   */
-  update(id: string, updates: Partial<NewObservation>): void {
-    const fields: string[] = []
-    const values: any[] = []
-
-    if (updates.content !== undefined) { fields.push('content = ?'); values.push(updates.content) }
-    if (updates.reasoning !== undefined) { fields.push('reasoning = ?'); values.push(updates.reasoning) }
-    if (updates.context !== undefined) { fields.push('context = ?'); values.push(updates.context) }
-    if (updates.confidence !== undefined) { fields.push('confidence = ?'); values.push(updates.confidence) }
-    if (updates.source !== undefined) { fields.push('source = ?'); values.push(updates.source) }
-    if (updates.tags !== undefined) { fields.push('tags = ?'); values.push(JSON.stringify(updates.tags)) }
-    if (updates.file_paths !== undefined) { fields.push('file_paths = ?'); values.push(JSON.stringify(updates.file_paths)) }
-    if (updates.symbols !== undefined) { fields.push('symbols = ?'); values.push(JSON.stringify(updates.symbols)) }
-
-    if (fields.length === 0) return
-
-    fields.push('updated_at = ?')
-    values.push(new Date().toISOString())
-    values.push(id)
-
-    this.db.prepare(`UPDATE observations SET ${fields.join(', ')} WHERE id = ?`).run(...values)
-  }
-
-  /**
-   * Delete an observation by ID.
-   */
-  delete(id: string): void {
-    this.db.prepare('DELETE FROM observations WHERE id = ?').run(id)
-  }
-
-  /**
-   * Record an access to bump access_count and last_accessed.
-   */
-  recordAccess(id: string): void {
-    this.db.prepare(`
-      UPDATE observations SET access_count = access_count + 1, last_accessed = ? WHERE id = ?
-    `).run(new Date().toISOString(), id)
-  }
-
-  /**
-   * Get a single observation by ID.
-   */
-  getById(id: string): ObservationResult | null {
-    const row = this.db.prepare('SELECT * FROM observations WHERE id = ?').get(id) as any
-    if (!row) return null
-    return this.rowToResult(row)
-  }
-
-  /**
-   * Phase 27: Fetch observations in a date range for a project.
-   * Ordered by created_at descending (most recent first).
-   */
-  fetchByTimeRange(project: string, start: Date, end: Date, limit: number = 50): ObservationResult[] {
-    const startStr = start.toISOString()
-    const endStr = end.toISOString()
-
-    const rows = this.db.prepare(`
-      SELECT * FROM observations
-      WHERE project = ? AND archived = 0
-        AND created_at >= ? AND created_at <= ?
-      ORDER BY created_at DESC
-      LIMIT ?
-    `).all(project, startStr, endStr, limit) as any[]
-
-    return rows.map(row => this.rowToResult(row))
-  }
-
-  /**
-   * Phase 27: Increment access count and update last_accessed timestamp.
-   * Alias for recordAccess for API consistency.
-   */
-  incrementAccess(id: string): void {
-    this.recordAccess(id)
-  }
-
-  /**
-   * BUG-002: Search within a specific category, optionally scoped to a project.
-   * Returns results ordered by creation date (most recent first).
-   */
-  searchByCategory(category: ObservationCategory, project?: string, limit: number = 10): ObservationResult[] {
-    let sql: string
-    const params: any[] = []
-
-    if (project) {
-      sql = `SELECT * FROM observations WHERE category = ? AND project = ? AND archived = 0 ORDER BY created_at DESC LIMIT ?`
-      params.push(category, project, limit)
-    } else {
-      sql = `SELECT * FROM observations WHERE category = ? AND archived = 0 ORDER BY created_at DESC LIMIT ?`
-      params.push(category, limit)
-    }
-
-    const rows = this.db.prepare(sql).all(...params) as any[]
-    return rows.map(row => this.rowToResult(row))
-  }
-
-  // --- Embedding-based semantic search ---
-
-  /**
-   * Store an embedding for an observation.
-   * Called after storing an observation to enable semantic search.
-   */
-  storeEmbedding(observationId: string, embedding: number[]): void {
-    const buffer = embeddingToBuffer(embedding)
-    const now = new Date().toISOString()
-
-    try {
-      this.db.prepare(`
-        INSERT OR REPLACE INTO observation_embeddings (observation_id, embedding, created_at)
-        VALUES (?, ?, ?)
-      `).run(observationId, buffer, now)
-    } catch (error) {
-      this.logger.warn({ error, observationId }, 'Failed to store embedding')
-    }
-  }
-
-  /**
-   * Delete an embedding for an observation.
-   */
-  deleteEmbedding(observationId: string): void {
-    try {
-      this.db.prepare('DELETE FROM observation_embeddings WHERE observation_id = ?').run(observationId)
-    } catch (error) {
-      this.logger.warn({ error, observationId }, 'Failed to delete embedding')
-    }
-  }
-
-  /**
-   * Semantic search using stored embeddings.
-   * Generates a query embedding, then computes cosine similarity against all
-   * stored observation embeddings. Returns top matches above threshold.
-   */
-  async semanticSearch(
-    queryEmbedding: number[],
-    project?: string,
-    limit: number = 10,
-    minSimilarity: number = 0.3
-  ): Promise<ScoredResult[]> {
-    try {
-      // Load all embeddings with their observation IDs
-      let sql: string
-      const params: any[] = []
-
-      if (project) {
-        sql = `
-          SELECT oe.observation_id, oe.embedding, o.*
-          FROM observation_embeddings oe
-          JOIN observations o ON oe.observation_id = o.id
-          WHERE o.archived = 0 AND o.project = ?
-        `
-        params.push(project)
-      } else {
-        sql = `
-          SELECT oe.observation_id, oe.embedding, o.*
-          FROM observation_embeddings oe
-          JOIN observations o ON oe.observation_id = o.id
-          WHERE o.archived = 0
-        `
-      }
-
-      const rows = this.db.prepare(sql).all(...params) as any[]
-
-      if (rows.length === 0) {
-        this.logger.debug('No embeddings found for semantic search')
-        return []
-      }
-
-      // Compute cosine similarity for each stored embedding
-      const scored: { row: any; similarity: number }[] = []
-      for (const row of rows) {
-        try {
-          const storedEmbedding = bufferToEmbedding(row.embedding)
-          const similarity = cosineSimilarity(queryEmbedding, storedEmbedding)
-          if (similarity >= minSimilarity) {
-            scored.push({ row, similarity })
-          }
-        } catch {
-          // Skip invalid embeddings
-        }
-      }
-
-      // Sort by similarity descending and take top results
-      scored.sort((a, b) => b.similarity - a.similarity)
-      const topResults = scored.slice(0, limit)
-
-      return topResults.map(({ row, similarity }) => ({
-        ...this.rowToResult(row),
-        score: similarity
-      }))
-    } catch (error) {
-      this.logger.warn({ error }, 'Semantic search failed')
-      return []
-    }
-  }
-
-  /**
-   * Check if there are any stored embeddings.
-   */
-  hasEmbeddings(): boolean {
-    try {
-      const row = this.db.prepare('SELECT COUNT(*) as count FROM observation_embeddings').get() as any
-      return (row?.count || 0) > 0
-    } catch {
-      return false
-    }
-  }
-
-  /**
-   * Backfill embeddings for observations that don't have them yet.
-   * Used for migrating existing data.
-   */
-  async backfillEmbeddings(embeddingService: EmbeddingService, batchSize: number = 50): Promise<number> {
-    try {
-      const rows = this.db.prepare(`
-        SELECT o.id, o.content, o.reasoning, o.context
-        FROM observations o
-        LEFT JOIN observation_embeddings oe ON o.id = oe.observation_id
-        WHERE oe.observation_id IS NULL AND o.archived = 0
-        LIMIT ?
-      `).all(batchSize) as any[]
-
-      if (rows.length === 0) return 0
-
-      let count = 0
-      for (const row of rows) {
-        try {
-          const text = [row.content, row.reasoning, row.context].filter(Boolean).join(' ')
-          const embedding = await embeddingService.generateEmbedding(text)
-          this.storeEmbedding(row.id, embedding)
-          count++
-        } catch {
-          // Skip failures, continue with next
-        }
-      }
-
-      this.logger.info({ backfilled: count, total: rows.length }, 'Embedding backfill complete')
-      return count
-    } catch (error) {
-      this.logger.warn({ error }, 'Embedding backfill failed')
-      return 0
-    }
-  }
-
-  // --- Private helpers ---
-
-  /**
-   * Build an FTS5 query from a natural language query.
-   * Tokenizes words and joins with OR for broad matching.
-   */
-  private buildFTSQuery(query: string): string {
-    // Remove special FTS5 characters that could cause syntax errors
-    const cleaned = query.replace(/[":*^~(){}[\]]/g, ' ').trim()
-    if (!cleaned) return ''
-
-    // Expand query with synonyms (e.g., "database" → also search "storage", "persistence")
-    const expanded = expandQuery(cleaned, { useSynonyms: true, maxExpansions: 8 })
-    const allWords = expanded.combined.split(/\s+/).filter(w => w.length >= 2)
-    const unique = [...new Set(allWords)]
-
-    if (unique.length === 0) return ''
-    return unique.map(w => `"${w}"`).join(' OR ')
-  }
-
-  /**
-   * Normalize BM25 score to 0-1 range.
-   * BM25 returns negative scores in SQLite (lower = better match).
-   * Typical range: -20 (excellent) to 0 (poor).
-   */
-  private normalizeBM25(score: number): number {
-    // Map BM25 range to 0.2-0.9 (wider range, allows low scores to be filtered)
-    const normalized = Math.min(1, Math.max(0, score / 20))
-    return 0.2 + normalized * 0.7
-  }
-
-  /**
-   * Compute word overlap between two strings as a similarity proxy.
-   * Returns a value between 0 and 1.
-   */
-  private wordOverlap(a: string, b: string): number {
-    const wordsA = new Set(a.toLowerCase().split(/\s+/).filter(w => w.length >= 3))
-    const wordsB = new Set(b.toLowerCase().split(/\s+/).filter(w => w.length >= 3))
-
-    if (wordsA.size === 0 || wordsB.size === 0) return 0
-
-    let overlap = 0
-    for (const word of wordsA) {
-      if (wordsB.has(word)) overlap++
-    }
-
-    // Jaccard similarity
-    const union = new Set([...wordsA, ...wordsB]).size
-    return overlap / union
-  }
-
-  /**
-   * Fallback LIKE-based search when FTS5 query syntax fails.
-   */
-  private fallbackSearch(query: string, project?: string, limit: number = 10): ObservationResult[] {
-    const pattern = `%${query}%`
-
-    let sql: string
-    const params: any[] = []
-
-    if (project) {
-      sql = `
-        SELECT * FROM observations
-        WHERE (content LIKE ? OR reasoning LIKE ? OR context LIKE ? OR tags LIKE ?)
-        AND project = ? AND archived = 0
-        ORDER BY created_at DESC
-        LIMIT ?
-      `
-      params.push(pattern, pattern, pattern, pattern, project, limit)
-    } else {
-      sql = `
-        SELECT * FROM observations
-        WHERE (content LIKE ? OR reasoning LIKE ? OR context LIKE ? OR tags LIKE ?)
-        AND archived = 0
-        ORDER BY created_at DESC
-        LIMIT ?
-      `
-      params.push(pattern, pattern, pattern, pattern, limit)
-    }
-
-    const rows = this.db.prepare(sql).all(...params) as any[]
-    return rows.map(row => this.rowToResult(row))
-  }
-
-  /**
-   * Convert a raw database row to an ObservationResult.
-   */
-  private rowToResult(row: any): ObservationResult {
-    return {
-      id: row.id,
-      project: row.project,
-      category: row.category,
-      content: row.content,
-      reasoning: row.reasoning || null,
-      context: row.context || null,
-      confidence: row.confidence ?? 0.8,
-      source: row.source || 'explicit',
-      tags: row.tags ? this.parseJsonArray(row.tags) : [],
-      file_paths: row.file_paths ? this.parseJsonArray(row.file_paths) : [],
-      symbols: row.symbols ? this.parseJsonArray(row.symbols) : [],
-      access_count: row.access_count ?? 0,
-      last_accessed: row.last_accessed || null,
-      created_at: row.created_at,
-      updated_at: row.updated_at,
-      archived: row.archived === 1
-    }
-  }
-
-  private parseJsonArray(value: string): string[] {
-    try {
-      const parsed = JSON.parse(value)
-      return Array.isArray(parsed) ? parsed : []
-    } catch {
-      // Handle comma-separated strings as fallback
-      return value.split(',').map(s => s.trim()).filter(Boolean)
-    }
-  }
-}
+/**
+ * FTS5 Search Service — Phase 26
+ * Full-text search over the observations table using SQLite FTS5.
+ * Replaces ChromaDB as the primary search backend.
+ */
+
+import { randomUUID } from 'crypto'
+import type { Database } from 'bun:sqlite'
+import type { Logger } from 'pino'
+import { expandQuery } from '@/retrieval/query/expander'
+import { embeddingToBuffer, bufferToEmbedding, cosineSimilarity } from './embedding-utils'
+import type { EmbeddingService } from './embeddings'
+
+export type ObservationCategory = 'decision' | 'pattern' | 'correction' | 'insight' | 'preference'
+
+export interface NewObservation {
+  project: string
+  category: ObservationCategory
+  content: string
+  reasoning?: string
+  context?: string
+  confidence?: number
+  source?: string
+  tags?: string[]
+  file_paths?: string[]
+  symbols?: string[]
+}
+
+export interface ObservationResult {
+  id: string
+  project: string
+  category: ObservationCategory
+  content: string
+  reasoning: string | null
+  context: string | null
+  confidence: number
+  source: string
+  tags: string[]
+  file_paths: string[]
+  symbols: string[]
+  access_count: number
+  last_accessed: string | null
+  created_at: string
+  updated_at: string
+  archived: boolean
+}
+
+export interface ScoredResult extends ObservationResult {
+  score: number
+}
+
+export interface DuplicateResult {
+  id: string
+  content: string
+  score: number
+}
+
+/** Raw database row from the observations table */
+interface ObservationRow {
+  id: string
+  project: string
+  category: string
+  content: string
+  reasoning: string | null
+  context: string | null
+  tags: string | null
+  confidence: number
+  source: string | null
+  file_paths: string | null
+  symbols: string | null
+  archived: number
+  access_count: number
+  last_accessed: string | null
+  created_at: string
+  updated_at: string
+}
+
+/** Observation row with BM25 score */
+interface ObservationWithBM25Row extends ObservationRow {
+  bm25_score: number
+}
+
+/** Observation row joined with embedding data */
+interface ObservationWithEmbeddingRow extends ObservationRow {
+  observation_id: string
+  embedding: Buffer | Uint8Array
+}
+
+/** Duplicate check result row */
+interface DuplicateCheckRow {
+  id: string
+  content: string
+  bm25_score: number
+}
+
+/** Row for backfill query */
+interface BackfillRow {
+  id: string
+  content: string
+  reasoning: string | null
+  context: string | null
+}
+
+/** Count row */
+interface CountRow {
+  count: number
+}
+
+export class FTS5Search {
+  private db: Database
+  private logger: Logger
+
+  constructor(db: Database, logger: Logger) {
+    this.db = db
+    this.logger = logger.child({ component: 'fts5-search' })
+  }
+
+  /**
+   * Search observations via FTS5 full-text search.
+   */
+  search(query: string, project?: string, limit: number = 10): ObservationResult[] {
+    if (!query.trim()) return []
+
+    const ftsQuery = this.buildFTSQuery(query)
+
+    try {
+      let sql: string
+      const params: (string | number)[] = []
+
+      if (project) {
+        sql = `
+          SELECT o.*, rank
+          FROM observations o
+          JOIN observations_fts fts ON o.rowid = fts.rowid
+          WHERE observations_fts MATCH ? AND o.project = ? AND o.archived = 0
+          ORDER BY rank
+          LIMIT ?
+        `
+        params.push(ftsQuery, project, limit)
+      } else {
+        sql = `
+          SELECT o.*, rank
+          FROM observations o
+          JOIN observations_fts fts ON o.rowid = fts.rowid
+          WHERE observations_fts MATCH ? AND o.archived = 0
+          ORDER BY rank
+          LIMIT ?
+        `
+        params.push(ftsQuery, limit)
+      }
+
+      const rows = this.db.prepare(sql).all(...params) as ObservationRow[]
+      return rows.map(row => this.rowToResult(row))
+    } catch (error) {
+      this.logger.warn({ error, query, ftsQuery }, 'FTS5 search failed, trying fallback')
+      return this.fallbackSearch(query, project, limit)
+    }
+  }
+
+  /**
+   * Search with BM25 ranking and confidence scoring.
+   * Returns results with a normalized score between 0 and 1.
+   */
+  searchWithConfidence(query: string, project?: string, limit: number = 10): ScoredResult[] {
+    if (!query.trim()) return []
+
+    const ftsQuery = this.buildFTSQuery(query)
+    const queryLower = query.toLowerCase()
+
+    try {
+      let sql: string
+      const params: (string | number)[] = []
+
+      if (project) {
+        sql = `
+          SELECT o.*, bm25(observations_fts) as bm25_score
+          FROM observations o
+          JOIN observations_fts fts ON o.rowid = fts.rowid
+          WHERE observations_fts MATCH ? AND o.project = ? AND o.archived = 0
+          ORDER BY bm25_score
+          LIMIT ?
+        `
+        params.push(ftsQuery, project, limit)
+      } else {
+        sql = `
+          SELECT o.*, bm25(observations_fts) as bm25_score
+          FROM observations o
+          JOIN observations_fts fts ON o.rowid = fts.rowid
+          WHERE observations_fts MATCH ? AND o.archived = 0
+          ORDER BY bm25_score
+          LIMIT ?
+        `
+        params.push(ftsQuery, limit)
+      }
+
+      const rows = this.db.prepare(sql).all(...params) as ObservationWithBM25Row[]
+
+      return rows.map(row => {
+        const result = this.rowToResult(row)
+        const bm25 = Math.abs(row.bm25_score as number)
+
+        // Compute confidence score
+        let score = this.normalizeBM25(bm25)
+
+        // Boost for exact content match
+        if (result.content.toLowerCase().includes(queryLower)) {
+          score = Math.min(1.0, score + 0.15)
+        }
+
+        // Boost for tag match
+        if (result.tags.some(t => queryLower.includes(t.toLowerCase()))) {
+          score = Math.min(1.0, score + 0.1)
+        }
+
+        // Boost for project match
+        if (project && result.project === project) {
+          score = Math.min(1.0, score + 0.05)
+        }
+
+        return { ...result, score }
+      }).sort((a, b) => b.score - a.score)
+    } catch (error) {
+      this.logger.warn({ error, query, ftsQuery }, 'FTS5 confidence search failed, trying fallback')
+      return this.fallbackSearch(query, project, limit).map(r => ({ ...r, score: 0.5 }))
+    }
+  }
+
+  /**
+   * Store a new observation. Returns the generated ID.
+   */
+  store(observation: NewObservation, providedId?: string): string {
+    const id = providedId || randomUUID()
+    const now = new Date().toISOString()
+
+    const stmt = this.db.prepare(`
+      INSERT INTO observations (id, project, category, content, reasoning, context, confidence, source, tags, file_paths, symbols, access_count, last_accessed, created_at, updated_at, archived)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, NULL, ?, ?, 0)
+    `)
+
+    stmt.run(
+      id,
+      observation.project,
+      observation.category,
+      observation.content,
+      observation.reasoning || null,
+      observation.context || null,
+      observation.confidence ?? 0.8,
+      observation.source || 'explicit',
+      observation.tags ? JSON.stringify(observation.tags) : null,
+      observation.file_paths ? JSON.stringify(observation.file_paths) : null,
+      observation.symbols ? JSON.stringify(observation.symbols) : null,
+      now,
+      now
+    )
+
+    this.logger.debug({ id, category: observation.category, project: observation.project }, 'Observation stored')
+    return id
+  }
+
+  /**
+   * Check for duplicates via FTS5 text similarity.
+   * Returns the best matching duplicate if above threshold, null otherwise.
+   */
+  searchForDuplicates(content: string, project: string, threshold: number = 0.85): DuplicateResult | null {
+    const ftsQuery = this.buildFTSQuery(content)
+    if (!ftsQuery.trim()) return null
+
+    try {
+      const rows = this.db.prepare(`
+        SELECT o.id, o.content, bm25(observations_fts) as bm25_score
+        FROM observations o
+        JOIN observations_fts fts ON o.rowid = fts.rowid
+        WHERE observations_fts MATCH ? AND o.project = ? AND o.archived = 0
+        ORDER BY bm25_score
+        LIMIT 3
+      `).all(ftsQuery, project) as DuplicateCheckRow[]
+
+      if (rows.length === 0) return null
+
+      // Use word overlap as a proxy for semantic similarity
+      for (const row of rows) {
+        const similarity = this.wordOverlap(content, row.content)
+        if (similarity >= threshold) {
+          return {
+            id: row.id,
+            content: row.content,
+            score: similarity
+          }
+        }
+      }
+
+      return null
+    } catch (error) {
+      this.logger.warn({ error }, 'Duplicate check failed')
+      return null
+    }
+  }
+
+  /**
+   * Fetch all observations for a project, optionally filtered by category.
+   */
+  fetchAll(project?: string, category?: ObservationCategory): ObservationResult[] {
+    let sql: string
+    const params: (string | number)[] = []
+
+    if (project && category) {
+      sql = `SELECT * FROM observations WHERE project = ? AND category = ? AND archived = 0 ORDER BY created_at DESC`
+      params.push(project, category)
+    } else if (project) {
+      sql = `SELECT * FROM observations WHERE project = ? AND archived = 0 ORDER BY created_at DESC`
+      params.push(project)
+    } else if (category) {
+      sql = `SELECT * FROM observations WHERE category = ? AND archived = 0 ORDER BY created_at DESC`
+      params.push(category)
+    } else {
+      sql = `SELECT * FROM observations WHERE archived = 0 ORDER BY created_at DESC`
+    }
+
+    const rows = this.db.prepare(sql).all(...params) as ObservationRow[]
+    return rows.map(row => this.rowToResult(row))
+  }
+
+  /** Allowed fields for observation updates */
+  private static readonly UPDATABLE_FIELDS = new Set([
+    'content', 'reasoning', 'context', 'confidence', 'source', 'tags', 'file_paths', 'symbols'
+  ])
+
+  /**
+   * Update an observation's fields.
+   */
+  update(id: string, updates: Partial<NewObservation>): void {
+    const fields: string[] = []
+    const values: (string | number | null)[] = []
+
+    for (const [key, value] of Object.entries(updates)) {
+      if (value === undefined) continue
+      if (!FTS5Search.UPDATABLE_FIELDS.has(key)) {
+        this.logger.warn({ field: key }, 'Attempted to update non-whitelisted field')
+        continue
+      }
+
+      const serialized = Array.isArray(value) ? JSON.stringify(value) : value
+      fields.push(`${key} = ?`)
+      values.push(serialized)
+    }
+
+    if (fields.length === 0) return
+
+    fields.push('updated_at = ?')
+    values.push(new Date().toISOString())
+    values.push(id)
+
+    this.db.prepare(`UPDATE observations SET ${fields.join(', ')} WHERE id = ?`).run(...values)
+  }
+
+  /**
+   * Delete an observation by ID.
+   */
+  delete(id: string): void {
+    this.db.prepare('DELETE FROM observations WHERE id = ?').run(id)
+  }
+
+  /**
+   * Record an access to bump access_count and last_accessed.
+   */
+  recordAccess(id: string): void {
+    this.db.prepare(`
+      UPDATE observations SET access_count = access_count + 1, last_accessed = ? WHERE id = ?
+    `).run(new Date().toISOString(), id)
+  }
+
+  /**
+   * Get a single observation by ID.
+   */
+  getById(id: string): ObservationResult | null {
+    const row = this.db.prepare('SELECT * FROM observations WHERE id = ?').get(id) as ObservationRow | null
+    if (!row) return null
+    return this.rowToResult(row)
+  }
+
+  /**
+   * Phase 27: Fetch observations in a date range for a project.
+   * Ordered by created_at descending (most recent first).
+   */
+  fetchByTimeRange(project: string, start: Date, end: Date, limit: number = 50): ObservationResult[] {
+    const startStr = start.toISOString()
+    const endStr = end.toISOString()
+
+    const rows = this.db.prepare(`
+      SELECT * FROM observations
+      WHERE project = ? AND archived = 0
+        AND created_at >= ? AND created_at <= ?
+      ORDER BY created_at DESC
+      LIMIT ?
+    `).all(project, startStr, endStr, limit) as ObservationRow[]
+
+    return rows.map(row => this.rowToResult(row))
+  }
+
+  /**
+   * Phase 27: Increment access count and update last_accessed timestamp.
+   * Alias for recordAccess for API consistency.
+   */
+  incrementAccess(id: string): void {
+    this.recordAccess(id)
+  }
+
+  /**
+   * BUG-002: Search within a specific category, optionally scoped to a project.
+   * Returns results ordered by creation date (most recent first).
+   */
+  searchByCategory(category: ObservationCategory, project?: string, limit: number = 10): ObservationResult[] {
+    let sql: string
+    const params: (string | number)[] = []
+
+    if (project) {
+      sql = `SELECT * FROM observations WHERE category = ? AND project = ? AND archived = 0 ORDER BY created_at DESC LIMIT ?`
+      params.push(category, project, limit)
+    } else {
+      sql = `SELECT * FROM observations WHERE category = ? AND archived = 0 ORDER BY created_at DESC LIMIT ?`
+      params.push(category, limit)
+    }
+
+    const rows = this.db.prepare(sql).all(...params) as ObservationRow[]
+    return rows.map(row => this.rowToResult(row))
+  }
+
+  // --- Embedding-based semantic search ---
+
+  /**
+   * Store an embedding for an observation.
+   * Called after storing an observation to enable semantic search.
+   */
+  storeEmbedding(observationId: string, embedding: number[]): void {
+    const buffer = embeddingToBuffer(embedding)
+    const now = new Date().toISOString()
+
+    try {
+      this.db.prepare(`
+        INSERT OR REPLACE INTO observation_embeddings (observation_id, embedding, created_at)
+        VALUES (?, ?, ?)
+      `).run(observationId, buffer, now)
+    } catch (error) {
+      this.logger.warn({ error, observationId }, 'Failed to store embedding')
+    }
+  }
+
+  /**
+   * Delete an embedding for an observation.
+   */
+  deleteEmbedding(observationId: string): void {
+    try {
+      this.db.prepare('DELETE FROM observation_embeddings WHERE observation_id = ?').run(observationId)
+    } catch (error) {
+      this.logger.warn({ error, observationId }, 'Failed to delete embedding')
+    }
+  }
+
+  /**
+   * Semantic search using stored embeddings.
+   * Generates a query embedding, then computes cosine similarity against all
+   * stored observation embeddings. Returns top matches above threshold.
+   */
+  async semanticSearch(
+    queryEmbedding: number[],
+    project?: string,
+    limit: number = 10,
+    minSimilarity: number = 0.3
+  ): Promise<ScoredResult[]> {
+    try {
+      // Load all embeddings with their observation IDs
+      let sql: string
+      const params: string[] = []
+
+      if (project) {
+        sql = `
+          SELECT oe.observation_id, oe.embedding, o.*
+          FROM observation_embeddings oe
+          JOIN observations o ON oe.observation_id = o.id
+          WHERE o.archived = 0 AND o.project = ?
+        `
+        params.push(project)
+      } else {
+        sql = `
+          SELECT oe.observation_id, oe.embedding, o.*
+          FROM observation_embeddings oe
+          JOIN observations o ON oe.observation_id = o.id
+          WHERE o.archived = 0
+        `
+      }
+
+      const rows = this.db.prepare(sql).all(...params) as ObservationWithEmbeddingRow[]
+
+      if (rows.length === 0) {
+        this.logger.debug('No embeddings found for semantic search')
+        return []
+      }
+
+      // Compute cosine similarity for each stored embedding
+      const scored: { row: ObservationWithEmbeddingRow; similarity: number }[] = []
+      for (const row of rows) {
+        try {
+          const storedEmbedding = bufferToEmbedding(row.embedding)
+          const similarity = cosineSimilarity(queryEmbedding, storedEmbedding)
+          if (similarity >= minSimilarity) {
+            scored.push({ row, similarity })
+          }
+        } catch {
+          // Skip invalid embeddings
+        }
+      }
+
+      // Sort by similarity descending and take top results
+      scored.sort((a, b) => b.similarity - a.similarity)
+      const topResults = scored.slice(0, limit)
+
+      return topResults.map(({ row, similarity }) => ({
+        ...this.rowToResult(row),
+        score: similarity
+      }))
+    } catch (error) {
+      this.logger.warn({ error }, 'Semantic search failed')
+      return []
+    }
+  }
+
+  /**
+   * Check if there are any stored embeddings.
+   */
+  hasEmbeddings(): boolean {
+    try {
+      const row = this.db.prepare('SELECT COUNT(*) as count FROM observation_embeddings').get() as CountRow | null
+      return (row?.count || 0) > 0
+    } catch {
+      return false
+    }
+  }
+
+  /**
+   * Backfill embeddings for observations that don't have them yet.
+   * Used for migrating existing data.
+   */
+  async backfillEmbeddings(embeddingService: EmbeddingService, batchSize: number = 50): Promise<number> {
+    try {
+      const rows = this.db.prepare(`
+        SELECT o.id, o.content, o.reasoning, o.context
+        FROM observations o
+        LEFT JOIN observation_embeddings oe ON o.id = oe.observation_id
+        WHERE oe.observation_id IS NULL AND o.archived = 0
+        LIMIT ?
+      `).all(batchSize) as BackfillRow[]
+
+      if (rows.length === 0) return 0
+
+      let count = 0
+      for (const row of rows) {
+        try {
+          const text = [row.content, row.reasoning, row.context].filter(Boolean).join(' ')
+          const embedding = await embeddingService.generateEmbedding(text)
+          this.storeEmbedding(row.id, embedding)
+          count++
+        } catch {
+          // Skip failures, continue with next
+        }
+      }
+
+      this.logger.info({ backfilled: count, total: rows.length }, 'Embedding backfill complete')
+      return count
+    } catch (error) {
+      this.logger.warn({ error }, 'Embedding backfill failed')
+      return 0
+    }
+  }
+
+  // --- Private helpers ---
+
+  /**
+   * Build an FTS5 query from a natural language query.
+   * Tokenizes words and joins with OR for broad matching.
+   */
+  private buildFTSQuery(query: string): string {
+    // Remove special FTS5 characters that could cause syntax errors
+    const cleaned = query.replace(/[":*^~(){}[\]]/g, ' ').trim()
+    if (!cleaned) return ''
+
+    // Expand query with synonyms (e.g., "database" → also search "storage", "persistence")
+    const expanded = expandQuery(cleaned, { useSynonyms: true, maxExpansions: 8 })
+    const allWords = expanded.combined.split(/\s+/).filter(w => w.length >= 2)
+    const unique = [...new Set(allWords)]
+
+    if (unique.length === 0) return ''
+    return unique.map(w => `"${w}"`).join(' OR ')
+  }
+
+  /**
+   * Normalize BM25 score to 0-1 range.
+   * BM25 returns negative scores in SQLite (lower = better match).
+   * Typical range: -20 (excellent) to 0 (poor).
+   */
+  private normalizeBM25(score: number): number {
+    // Map BM25 range to 0.2-0.9 (wider range, allows low scores to be filtered)
+    const normalized = Math.min(1, Math.max(0, score / 20))
+    return 0.2 + normalized * 0.7
+  }
+
+  /**
+   * Compute word overlap between two strings as a similarity proxy.
+   * Returns a value between 0 and 1.
+   */
+  private wordOverlap(a: string, b: string): number {
+    const wordsA = new Set(a.toLowerCase().split(/\s+/).filter(w => w.length >= 3))
+    const wordsB = new Set(b.toLowerCase().split(/\s+/).filter(w => w.length >= 3))
+
+    if (wordsA.size === 0 || wordsB.size === 0) return 0
+
+    let overlap = 0
+    for (const word of wordsA) {
+      if (wordsB.has(word)) overlap++
+    }
+
+    // Jaccard similarity
+    const union = new Set([...wordsA, ...wordsB]).size
+    return overlap / union
+  }
+
+  /**
+   * Fallback LIKE-based search when FTS5 query syntax fails.
+   */
+  private fallbackSearch(query: string, project?: string, limit: number = 10): ObservationResult[] {
+    const pattern = `%${query}%`
+
+    let sql: string
+    const params: (string | number)[] = []
+
+    if (project) {
+      sql = `
+        SELECT * FROM observations
+        WHERE (content LIKE ? OR reasoning LIKE ? OR context LIKE ? OR tags LIKE ?)
+        AND project = ? AND archived = 0
+        ORDER BY created_at DESC
+        LIMIT ?
+      `
+      params.push(pattern, pattern, pattern, pattern, project, limit)
+    } else {
+      sql = `
+        SELECT * FROM observations
+        WHERE (content LIKE ? OR reasoning LIKE ? OR context LIKE ? OR tags LIKE ?)
+        AND archived = 0
+        ORDER BY created_at DESC
+        LIMIT ?
+      `
+      params.push(pattern, pattern, pattern, pattern, limit)
+    }
+
+    const rows = this.db.prepare(sql).all(...params) as ObservationRow[]
+    return rows.map(row => this.rowToResult(row))
+  }
+
+  /**
+   * Convert a raw database row to an ObservationResult.
+   */
+  private rowToResult(row: ObservationRow): ObservationResult {
+    return {
+      id: row.id,
+      project: row.project,
+      category: row.category,
+      content: row.content,
+      reasoning: row.reasoning || null,
+      context: row.context || null,
+      confidence: row.confidence ?? 0.8,
+      source: row.source || 'explicit',
+      tags: row.tags ? this.parseJsonArray(row.tags) : [],
+      file_paths: row.file_paths ? this.parseJsonArray(row.file_paths) : [],
+      symbols: row.symbols ? this.parseJsonArray(row.symbols) : [],
+      access_count: row.access_count ?? 0,
+      last_accessed: row.last_accessed || null,
+      created_at: row.created_at,
+      updated_at: row.updated_at,
+      archived: row.archived === 1
+    }
+  }
+
+  private parseJsonArray(value: string): string[] {
+    try {
+      const parsed = JSON.parse(value)
+      return Array.isArray(parsed) ? parsed : []
+    } catch {
+      // Handle comma-separated strings as fallback
+      return value.split(',').map(s => s.trim()).filter(Boolean)
+    }
+  }
+}