claude-brain 0.17.13 → 0.22.0

package/src/memory/fts5-search.ts

@@ -0,0 +1,456 @@
+/**
+ * FTS5 Search Service — Phase 26
+ * Full-text search over the observations table using SQLite FTS5.
+ * Replaces ChromaDB as the primary search backend.
+ */
+
+import { randomUUID } from 'crypto'
+import type { Database } from 'bun:sqlite'
+import type { Logger } from 'pino'
+
+export type ObservationCategory = 'decision' | 'pattern' | 'correction' | 'insight' | 'preference'
+
+export interface NewObservation {
+  project: string
+  category: ObservationCategory
+  content: string
+  reasoning?: string
+  context?: string
+  confidence?: number
+  source?: string
+  tags?: string[]
+  file_paths?: string[]
+  symbols?: string[]
+}
+
+export interface ObservationResult {
+  id: string
+  project: string
+  category: ObservationCategory
+  content: string
+  reasoning: string | null
+  context: string | null
+  confidence: number
+  source: string
+  tags: string[]
+  file_paths: string[]
+  symbols: string[]
+  access_count: number
+  last_accessed: string | null
+  created_at: string
+  updated_at: string
+  archived: boolean
+}
+
+export interface ScoredResult extends ObservationResult {
+  score: number
+}
+
+export interface DuplicateResult {
+  id: string
+  content: string
+  score: number
+}
+
+export class FTS5Search {
+  private db: Database
+  private logger: Logger
+
+  constructor(db: Database, logger: Logger) {
+    this.db = db
+    this.logger = logger.child({ component: 'fts5-search' })
+  }
+
+  /**
+   * Search observations via FTS5 full-text search.
+   */
+  search(query: string, project?: string, limit: number = 10): ObservationResult[] {
+    if (!query.trim()) return []
+
+    const ftsQuery = this.buildFTSQuery(query)
+
+    try {
+      let sql: string
+      const params: any[] = []
+
+      if (project) {
+        sql = `
+          SELECT o.*, rank
+          FROM observations o
+          JOIN observations_fts fts ON o.rowid = fts.rowid
+          WHERE observations_fts MATCH ? AND o.project = ? AND o.archived = 0
+          ORDER BY rank
+          LIMIT ?
+        `
+        params.push(ftsQuery, project, limit)
+      } else {
+        sql = `
+          SELECT o.*, rank
+          FROM observations o
+          JOIN observations_fts fts ON o.rowid = fts.rowid
+          WHERE observations_fts MATCH ? AND o.archived = 0
+          ORDER BY rank
+          LIMIT ?
+        `
+        params.push(ftsQuery, limit)
+      }
+
+      const rows = this.db.prepare(sql).all(...params) as any[]
+      return rows.map(row => this.rowToResult(row))
+    } catch (error) {
+      this.logger.warn({ error, query, ftsQuery }, 'FTS5 search failed, trying fallback')
+      return this.fallbackSearch(query, project, limit)
+    }
+  }
+
+  /**
+   * Search with BM25 ranking and confidence scoring.
+   * Returns results with a normalized score between 0 and 1.
+   */
+  searchWithConfidence(query: string, project?: string, limit: number = 10): ScoredResult[] {
+    if (!query.trim()) return []
+
+    const ftsQuery = this.buildFTSQuery(query)
+    const queryLower = query.toLowerCase()
+
+    try {
+      let sql: string
+      const params: any[] = []
+
+      if (project) {
+        sql = `
+          SELECT o.*, bm25(observations_fts) as bm25_score
+          FROM observations o
+          JOIN observations_fts fts ON o.rowid = fts.rowid
+          WHERE observations_fts MATCH ? AND o.project = ? AND o.archived = 0
+          ORDER BY bm25_score
+          LIMIT ?
+        `
+        params.push(ftsQuery, project, limit)
+      } else {
+        sql = `
+          SELECT o.*, bm25(observations_fts) as bm25_score
+          FROM observations o
+          JOIN observations_fts fts ON o.rowid = fts.rowid
+          WHERE observations_fts MATCH ? AND o.archived = 0
+          ORDER BY bm25_score
+          LIMIT ?
+        `
+        params.push(ftsQuery, limit)
+      }
+
+      const rows = this.db.prepare(sql).all(...params) as any[]
+
+      return rows.map(row => {
+        const result = this.rowToResult(row)
+        const bm25 = Math.abs(row.bm25_score as number)
+
+        // Compute confidence score
+        let score = this.normalizeBM25(bm25)
+
+        // Boost for exact content match
+        if (result.content.toLowerCase().includes(queryLower)) {
+          score = Math.min(1.0, score + 0.15)
+        }
+
+        // Boost for tag match
+        if (result.tags.some(t => queryLower.includes(t.toLowerCase()))) {
+          score = Math.min(1.0, score + 0.1)
+        }
+
+        // Boost for project match
+        if (project && result.project === project) {
+          score = Math.min(1.0, score + 0.05)
+        }
+
+        return { ...result, score }
+      }).sort((a, b) => b.score - a.score)
+    } catch (error) {
+      this.logger.warn({ error, query, ftsQuery }, 'FTS5 confidence search failed, trying fallback')
+      return this.fallbackSearch(query, project, limit).map(r => ({ ...r, score: 0.5 }))
+    }
+  }
+
+  /**
+   * Store a new observation. Returns the generated ID.
+   */
+  store(observation: NewObservation): string {
+    const id = randomUUID()
+    const now = new Date().toISOString()
+
+    const stmt = this.db.prepare(`
+      INSERT INTO observations (id, project, category, content, reasoning, context, confidence, source, tags, file_paths, symbols, access_count, last_accessed, created_at, updated_at, archived)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, NULL, ?, ?, 0)
+    `)
+
+    stmt.run(
+      id,
+      observation.project,
+      observation.category,
+      observation.content,
+      observation.reasoning || null,
+      observation.context || null,
+      observation.confidence ?? 0.8,
+      observation.source || 'explicit',
+      observation.tags ? JSON.stringify(observation.tags) : null,
+      observation.file_paths ? JSON.stringify(observation.file_paths) : null,
+      observation.symbols ? JSON.stringify(observation.symbols) : null,
+      now,
+      now
+    )
+
+    this.logger.debug({ id, category: observation.category, project: observation.project }, 'Observation stored')
+    return id
+  }
+
+  /**
+   * Check for duplicates via FTS5 text similarity.
+   * Returns the best matching duplicate if above threshold, null otherwise.
+   */
+  searchForDuplicates(content: string, project: string, threshold: number = 0.85): DuplicateResult | null {
+    const ftsQuery = this.buildFTSQuery(content)
+    if (!ftsQuery.trim()) return null
+
+    try {
+      const rows = this.db.prepare(`
+        SELECT o.id, o.content, bm25(observations_fts) as bm25_score
+        FROM observations o
+        JOIN observations_fts fts ON o.rowid = fts.rowid
+        WHERE observations_fts MATCH ? AND o.project = ? AND o.archived = 0
+        ORDER BY bm25_score
+        LIMIT 3
+      `).all(ftsQuery, project) as any[]
+
+      if (rows.length === 0) return null
+
+      // Use word overlap as a proxy for semantic similarity
+      for (const row of rows) {
+        const similarity = this.wordOverlap(content, row.content as string)
+        if (similarity >= threshold) {
+          return {
+            id: row.id as string,
+            content: row.content as string,
+            score: similarity
+          }
+        }
+      }
+
+      return null
+    } catch (error) {
+      this.logger.warn({ error }, 'Duplicate check failed')
+      return null
+    }
+  }
+
+  /**
+   * Fetch all observations for a project, optionally filtered by category.
+   */
+  fetchAll(project: string, category?: ObservationCategory): ObservationResult[] {
+    let sql: string
+    const params: any[] = [project]
+
+    if (category) {
+      sql = `SELECT * FROM observations WHERE project = ? AND category = ? AND archived = 0 ORDER BY created_at DESC`
+      params.push(category)
+    } else {
+      sql = `SELECT * FROM observations WHERE project = ? AND archived = 0 ORDER BY created_at DESC`
+    }
+
+    const rows = this.db.prepare(sql).all(...params) as any[]
+    return rows.map(row => this.rowToResult(row))
+  }
+
+  /**
+   * Update an observation's fields.
+   */
+  update(id: string, updates: Partial<NewObservation>): void {
+    const fields: string[] = []
+    const values: any[] = []
+
+    if (updates.content !== undefined) { fields.push('content = ?'); values.push(updates.content) }
+    if (updates.reasoning !== undefined) { fields.push('reasoning = ?'); values.push(updates.reasoning) }
+    if (updates.context !== undefined) { fields.push('context = ?'); values.push(updates.context) }
+    if (updates.confidence !== undefined) { fields.push('confidence = ?'); values.push(updates.confidence) }
+    if (updates.source !== undefined) { fields.push('source = ?'); values.push(updates.source) }
+    if (updates.tags !== undefined) { fields.push('tags = ?'); values.push(JSON.stringify(updates.tags)) }
+    if (updates.file_paths !== undefined) { fields.push('file_paths = ?'); values.push(JSON.stringify(updates.file_paths)) }
+    if (updates.symbols !== undefined) { fields.push('symbols = ?'); values.push(JSON.stringify(updates.symbols)) }
+
+    if (fields.length === 0) return
+
+    fields.push('updated_at = ?')
+    values.push(new Date().toISOString())
+    values.push(id)
+
+    this.db.prepare(`UPDATE observations SET ${fields.join(', ')} WHERE id = ?`).run(...values)
+  }
+
+  /**
+   * Delete an observation by ID.
+   */
+  delete(id: string): void {
+    this.db.prepare('DELETE FROM observations WHERE id = ?').run(id)
+  }
+
+  /**
+   * Record an access to bump access_count and last_accessed.
+   */
+  recordAccess(id: string): void {
+    this.db.prepare(`
+      UPDATE observations SET access_count = access_count + 1, last_accessed = ? WHERE id = ?
+    `).run(new Date().toISOString(), id)
+  }
+
+  /**
+   * Get a single observation by ID.
+   */
+  getById(id: string): ObservationResult | null {
+    const row = this.db.prepare('SELECT * FROM observations WHERE id = ?').get(id) as any
+    if (!row) return null
+    return this.rowToResult(row)
+  }
+
+  /**
+   * Phase 27: Fetch observations in a date range for a project.
+   * Ordered by created_at descending (most recent first).
+   */
+  fetchByTimeRange(project: string, start: Date, end: Date, limit: number = 50): ObservationResult[] {
+    const startStr = start.toISOString()
+    const endStr = end.toISOString()
+
+    const rows = this.db.prepare(`
+      SELECT * FROM observations
+      WHERE project = ? AND archived = 0
+        AND created_at >= ? AND created_at <= ?
+      ORDER BY created_at DESC
+      LIMIT ?
+    `).all(project, startStr, endStr, limit) as any[]
+
+    return rows.map(row => this.rowToResult(row))
+  }
+
+  /**
+   * Phase 27: Increment access count and update last_accessed timestamp.
+   * Alias for recordAccess for API consistency.
+   */
+  incrementAccess(id: string): void {
+    this.recordAccess(id)
+  }
+
+  // --- Private helpers ---
+
+  /**
+   * Build an FTS5 query from a natural language query.
+   * Tokenizes words and joins with OR for broad matching.
+   */
+  private buildFTSQuery(query: string): string {
+    // Remove special FTS5 characters that could cause syntax errors
+    const cleaned = query.replace(/[":*^~(){}[\]]/g, ' ').trim()
+    if (!cleaned) return ''
+
+    // Split into words, filter short/stop words
+    const words = cleaned.split(/\s+/).filter(w => w.length >= 2)
+    if (words.length === 0) return ''
+
+    // Join with OR for broad matching
+    return words.map(w => `"${w}"`).join(' OR ')
+  }
+
+  /**
+   * Normalize BM25 score to 0-1 range.
+   * BM25 returns negative scores in SQLite (lower = better match).
+   * Typical range: -20 (excellent) to 0 (poor).
+   */
+  private normalizeBM25(score: number): number {
+    // Map typical BM25 range to 0.5-0.85
+    const normalized = Math.min(1, Math.max(0, score / 20))
+    return 0.5 + normalized * 0.35
+  }
+
+  /**
+   * Compute word overlap between two strings as a similarity proxy.
+   * Returns a value between 0 and 1.
+   */
+  private wordOverlap(a: string, b: string): number {
+    const wordsA = new Set(a.toLowerCase().split(/\s+/).filter(w => w.length >= 3))
+    const wordsB = new Set(b.toLowerCase().split(/\s+/).filter(w => w.length >= 3))
+
+    if (wordsA.size === 0 || wordsB.size === 0) return 0
+
+    let overlap = 0
+    for (const word of wordsA) {
+      if (wordsB.has(word)) overlap++
+    }
+
+    // Jaccard similarity
+    const union = new Set([...wordsA, ...wordsB]).size
+    return overlap / union
+  }
+
+  /**
+   * Fallback LIKE-based search when FTS5 query syntax fails.
+   */
+  private fallbackSearch(query: string, project?: string, limit: number = 10): ObservationResult[] {
+    const pattern = `%${query}%`
+
+    let sql: string
+    const params: any[] = []
+
+    if (project) {
+      sql = `
+        SELECT * FROM observations
+        WHERE (content LIKE ? OR reasoning LIKE ? OR context LIKE ? OR tags LIKE ?)
+        AND project = ? AND archived = 0
+        ORDER BY created_at DESC
+        LIMIT ?
+      `
+      params.push(pattern, pattern, pattern, pattern, project, limit)
+    } else {
+      sql = `
+        SELECT * FROM observations
+        WHERE (content LIKE ? OR reasoning LIKE ? OR context LIKE ? OR tags LIKE ?)
+        AND archived = 0
+        ORDER BY created_at DESC
+        LIMIT ?
+      `
+      params.push(pattern, pattern, pattern, pattern, limit)
+    }
+
+    const rows = this.db.prepare(sql).all(...params) as any[]
+    return rows.map(row => this.rowToResult(row))
+  }
+
+  /**
+   * Convert a raw database row to an ObservationResult.
+   */
+  private rowToResult(row: any): ObservationResult {
+    return {
+      id: row.id,
+      project: row.project,
+      category: row.category,
+      content: row.content,
+      reasoning: row.reasoning || null,
+      context: row.context || null,
+      confidence: row.confidence ?? 0.8,
+      source: row.source || 'explicit',
+      tags: row.tags ? this.parseJsonArray(row.tags) : [],
+      file_paths: row.file_paths ? this.parseJsonArray(row.file_paths) : [],
+      symbols: row.symbols ? this.parseJsonArray(row.symbols) : [],
+      access_count: row.access_count ?? 0,
+      last_accessed: row.last_accessed || null,
+      created_at: row.created_at,
+      updated_at: row.updated_at,
+      archived: row.archived === 1
+    }
+  }
+
+  private parseJsonArray(value: string): string[] {
+    try {
+      const parsed = JSON.parse(value)
+      return Array.isArray(parsed) ? parsed : []
+    } catch {
+      // Handle comma-separated strings as fallback
+      return value.split(',').map(s => s.trim()).filter(Boolean)
+    }
+  }
+}