claude-brain 0.17.13 → 0.22.0

package/src/memory/index.ts

@@ -13,6 +13,8 @@ import { SemanticSearch } from './search'
 import { MemoryContextBuilder } from './context-builder'
 import type { MemorySystemStats } from './types'
 import { ChromaManager, DEFAULT_CHROMA_CONFIG, getChromaConfigFromEnv, ChromaMigration, type MigrationOptions } from './chroma'
+import { FTS5Search } from './fts5-search'
+import { addFTS5Tables } from './migrations/add-fts5'
 
 // Re-export all types and classes for external use
 export * from './types'
@@ -38,6 +40,10 @@ export { PatternRecognizer, type Pattern } from './patterns'
 export { LearningSystem, type Correction, type Preference, type LearningInsights } from './learning'
 export { KnowledgeExtractor, type ExtractedKnowledge, type ExtractionResult } from './knowledge-extractor'
 
+// Phase 26: FTS5 Search
+export { FTS5Search } from './fts5-search'
+export type { ObservationCategory, NewObservation, ObservationResult, ScoredResult } from './fts5-search'
+
 /**
  * Unified memory system manager
  * Combines database, embeddings, store, search, and context building
@@ -48,6 +54,9 @@ export class MemoryManager {
   readonly contextBuilder: MemoryContextBuilder
   readonly chroma: ChromaManager
 
+  // Phase 26: FTS5 search (always available, no external deps)
+  private _fts5: FTS5Search | null = null
+
   // Store and search are initialized after database is ready
   private _store: MemoryStore | null = null
   private _search: SemanticSearch | null = null
@@ -98,6 +107,15 @@ export class MemoryManager {
       this._store = new MemoryStore(db, this.embeddings, this.logger)
       this._search = new SemanticSearch(db, this.embeddings, this.logger)
 
+      // Phase 26: Always initialize FTS5 (just SQLite, no external deps)
+      try {
+        addFTS5Tables(db)
+        this._fts5 = new FTS5Search(db, this.logger)
+        this.logger.info('FTS5 search initialized')
+      } catch (error) {
+        this.logger.warn({ error }, 'Failed to initialize FTS5, continuing without it')
+      }
+
       if (this.useChromaDB) {
         try {
           await this.chroma.initialize()
@@ -136,6 +154,13 @@ export class MemoryManager {
     return this._search
   }
 
+  /**
+   * Get the FTS5 search engine (null if not initialized)
+   */
+  get fts5(): FTS5Search | null {
+    return this._fts5
+  }
+
   /**
    * Check if memory system is initialized
    */
@@ -219,8 +244,51 @@ export class MemoryManager {
     reasoning: string,
     options?: { alternatives?: string; tags?: string[] }
   ): Promise<string> {
+    // Phase 26: Always store in FTS5 if available
+    let fts5Id: string | undefined
+    if (this._fts5) {
+      try {
+        // Check for duplicates first
+        const dup = this._fts5.searchForDuplicates(decision, project)
+        if (dup) {
+          this.logger.info({ existingId: dup.id, score: dup.score }, 'FTS5 skipping duplicate decision')
+          return dup.id
+        }
+
+        fts5Id = this._fts5.store({
+          project,
+          category: 'decision',
+          content: decision,
+          reasoning,
+          context,
+          tags: options?.tags
+        })
+      } catch (error) {
+        this.logger.warn({ error }, 'FTS5 store failed, continuing with other backends')
+      }
+    }
+
+    // Store in ChromaDB if available
     if (this.useChromaDB) {
-      return this.chroma.store.storeDecision({
+      try {
+        const chromaId = await this.chroma.store.storeDecision({
+          project,
+          context,
+          decision,
+          reasoning,
+          alternatives: options?.alternatives,
+          tags: options?.tags
+        })
+        return fts5Id || chromaId
+      } catch (error) {
+        this.logger.warn({ error }, 'ChromaDB store failed')
+        if (fts5Id) return fts5Id
+      }
+    }
+
+    // Fallback to legacy SQLite store if no FTS5 and no ChromaDB
+    if (!fts5Id) {
+      const id = await this.store.storeDecision({
         project,
         context,
         decision,
@@ -228,40 +296,83 @@ export class MemoryManager {
         alternatives: options?.alternatives,
         tags: options?.tags
       })
+      fts5Id = id
     }
-    const id = await this.store.storeDecision({
-      project,
-      context,
-      decision,
-      reasoning,
-      alternatives: options?.alternatives,
-      tags: options?.tags
-    })
-    // Notify listeners (e.g., knowledge graph builder) for SQLite path
+
+    // Notify listeners (e.g., knowledge graph builder)
     for (const cb of this.onDecisionStoredCallbacks) {
       try {
-        cb({ project, context, decision, reasoning, alternatives: options?.alternatives, tags: options?.tags, id })
+        cb({ project, context, decision, reasoning, alternatives: options?.alternatives, tags: options?.tags, id: fts5Id })
       } catch {}
     }
-    return id
+    return fts5Id!
   }
 
   /**
-   * Get raw search results - routes to ChromaDB when enabled
+   * Get raw search results - uses FTS5 as primary, ChromaDB as enrichment
    * Use this for internal operations that need raw results
    */
   async searchRaw(
     query: string,
     options?: { project?: string; limit?: number; minSimilarity?: number }
   ): Promise<any[]> {
+    const limit = options?.limit || 5
+
+    // Phase 26: Try FTS5 first (always available)
+    if (this._fts5) {
+      const ftsResults = this._fts5.searchWithConfidence(query, options?.project, limit)
+
+      if (ftsResults.length > 0) {
+        // Transform FTS5 results to match expected MemorySearchResult structure
+        const results = ftsResults.map(r => ({
+          id: r.id,
+          content: r.content,
+          memory: {
+            id: r.id,
+            project: r.project,
+            content: r.content,
+            createdAt: new Date(r.created_at),
+            metadata: {
+              project: r.project,
+              category: r.category,
+              context: r.context || '',
+              reasoning: r.reasoning || '',
+              tags: r.tags,
+              created_at: r.created_at
+            }
+          },
+          similarity: r.score,
+          decision: r.category === 'decision' ? {
+            id: r.id,
+            project: r.project,
+            context: r.context || '',
+            decision: r.content,
+            reasoning: r.reasoning || '',
+            alternatives: '',
+            tags: r.tags,
+            createdAt: new Date(r.created_at)
+          } : undefined,
+          metadata: {
+            project: r.project,
+            category: r.category,
+            context: r.context || '',
+            reasoning: r.reasoning || '',
+            tags: r.tags,
+            created_at: r.created_at
+          }
+        }))
+
+        return results
+      }
+    }
+
+    // Fallback: Try ChromaDB if available
     if (this.useChromaDB) {
       const chromaResults = await this.chroma.search.searchDecisions(query, {
         project: options?.project,
-        limit: options?.limit || 5,
+        limit,
         minSimilarity: options?.minSimilarity || 0.5
       })
-      // Transform ChromaDB results to match MemorySearchResult structure
-      // Includes flat `content` field for direct access (Phase 19)
       return chromaResults.map(r => {
         const memoryContent = typeof r.content === 'string' ? r.content : JSON.stringify(r.content)
         const decisionObj = r.metadata.decision ? {
@@ -277,10 +388,8 @@ export class MemoryManager {
         } : undefined
 
         return {
-          // Flat fields for direct access (Phase 19)
           id: r.id,
           content: decisionObj ? decisionObj.decision : memoryContent,
-          // Nested fields for backward compatibility
           memory: {
             id: r.id,
             project: r.metadata.project || options?.project || 'unknown',
@@ -293,13 +402,14 @@ export class MemoryManager {
           metadata: r.metadata
         }
       })
-    } else {
-      return await this.search.search(query, {
-        project: options?.project,
-        limit: options?.limit || 5,
-        minSimilarity: options?.minSimilarity || 0.5
-      })
     }
+
+    // Final fallback: legacy SQLite search
+    return await this.search.search(query, {
+      project: options?.project,
+      limit,
+      minSimilarity: options?.minSimilarity || 0.5
+    })
   }
 
   async recallSimilar(
@@ -335,7 +445,7 @@ export class MemoryManager {
   }
 
   /**
-   * Store a pattern in memory — routes to ChromaDB or SQLite
+   * Store a pattern in memory — dual-writes to FTS5 + ChromaDB/SQLite
    */
   async storePattern(input: {
     project: string
@@ -346,14 +456,41 @@ export class MemoryManager {
     context?: string
     source?: string
   }): Promise<string> {
+    // Phase 26: Dual-write to FTS5
+    let fts5Id: string | undefined
+    if (this._fts5) {
+      try {
+        fts5Id = this._fts5.store({
+          project: input.project,
+          category: 'pattern',
+          content: input.description,
+          context: input.context,
+          confidence: input.confidence,
+          source: input.source
+        })
+      } catch (error) {
+        this.logger.warn({ error }, 'FTS5 pattern store failed')
+      }
+    }
+
     if (this.useChromaDB) {
-      return this.chroma.store.storePattern(input)
+      try {
+        const chromaId = await this.chroma.store.storePattern(input)
+        return fts5Id || chromaId
+      } catch (error) {
+        this.logger.warn({ error }, 'ChromaDB pattern store failed')
+        if (fts5Id) return fts5Id
+      }
+    }
+
+    if (!fts5Id) {
+      return this.store.storePattern(input)
     }
-    return this.store.storePattern(input)
+    return fts5Id
   }
 
   /**
-   * Store a correction/lesson learned — routes to ChromaDB or SQLite
+   * Store a correction/lesson learned — dual-writes to FTS5 + ChromaDB/SQLite
    */
   async storeCorrection(input: {
     project: string
@@ -363,14 +500,41 @@ export class MemoryManager {
     context?: string
     confidence: number
   }): Promise<string> {
+    // Phase 26: Dual-write to FTS5
+    let fts5Id: string | undefined
+    if (this._fts5) {
+      try {
+        fts5Id = this._fts5.store({
+          project: input.project,
+          category: 'correction',
+          content: input.correction,
+          reasoning: input.reasoning,
+          context: input.context,
+          confidence: input.confidence
+        })
+      } catch (error) {
+        this.logger.warn({ error }, 'FTS5 correction store failed')
+      }
+    }
+
     if (this.useChromaDB) {
-      return this.chroma.store.storeCorrection(input)
+      try {
+        const chromaId = await this.chroma.store.storeCorrection(input)
+        return fts5Id || chromaId
+      } catch (error) {
+        this.logger.warn({ error }, 'ChromaDB correction store failed')
+        if (fts5Id) return fts5Id
+      }
     }
-    return this.store.storeCorrection(input)
+
+    if (!fts5Id) {
+      return this.store.storeCorrection(input)
+    }
+    return fts5Id
   }
 
   /**
-   * Get patterns for a project — routes to ChromaDB or SQLite
+   * Get patterns for a project — routes to FTS5, ChromaDB, or legacy SQLite
    */
   async getPatterns(
     project?: string,
@@ -379,6 +543,24 @@ export class MemoryManager {
       limit?: number
     }
   ): Promise<any[]> {
+    // Phase 26: Try FTS5 first
+    if (this._fts5 && project) {
+      const results = this._fts5.fetchAll(project, 'pattern')
+      if (results.length > 0) {
+        return results.map(r => ({
+          id: r.id,
+          description: r.content,
+          metadata: {
+            project: r.project,
+            pattern_type: r.category,
+            confidence: r.confidence,
+            context: r.context || '',
+            created_at: r.created_at
+          }
+        }))
+      }
+    }
+
     if (this.useChromaDB) {
       if (project) {
         return this.chroma.store.getPatternsByProject(project, options)
@@ -392,12 +574,30 @@ export class MemoryManager {
   }
 
   /**
-   * Get corrections for a project — routes to ChromaDB or SQLite
+   * Get corrections for a project — routes to FTS5, ChromaDB, or legacy SQLite
    */
   async getCorrections(
     project?: string,
     options?: { limit?: number }
   ): Promise<any[]> {
+    // Phase 26: Try FTS5 first
+    if (this._fts5 && project) {
+      const results = this._fts5.fetchAll(project, 'correction')
+      if (results.length > 0) {
+        return results.map(r => ({
+          id: r.id,
+          correction: r.content,
+          metadata: {
+            project: r.project,
+            reasoning: r.reasoning || '',
+            context: r.context || '',
+            confidence: r.confidence,
+            created_at: r.created_at
+          }
+        }))
+      }
+    }
+
     if (this.useChromaDB) {
       if (project) {
         return this.chroma.store.getCorrectionsByProject(project, options?.limit || 10)
@@ -411,10 +611,28 @@ export class MemoryManager {
   }
 
   /**
-   * Fetch all decisions with content — routes to ChromaDB or SQLite
+   * Fetch all decisions with content — routes to FTS5, ChromaDB, or legacy SQLite
    * Used by analytical tools that need bulk access to decision data
    */
   async fetchAllDecisions(project?: string): Promise<any[]> {
+    // Phase 26: Try FTS5 first
+    if (this._fts5 && project) {
+      const results = this._fts5.fetchAll(project, 'decision')
+      if (results.length > 0) {
+        return results.map(r => ({
+          id: r.id,
+          content: r.content,
+          date: r.created_at,
+          project: r.project,
+          context: r.context || '',
+          decision: r.content,
+          reasoning: r.reasoning || '',
+          alternatives: '',
+          tags: r.tags
+        }))
+      }
+    }
+
     if (this.useChromaDB) {
       try {
         const collection = await this.chroma.collections.getDecisions()
@@ -443,9 +661,27 @@ export class MemoryManager {
   }
 
   /**
-   * Fetch all patterns with content — routes to ChromaDB or SQLite
+   * Fetch all patterns with content — routes to FTS5, ChromaDB, or legacy SQLite
    */
   async fetchAllPatterns(project?: string): Promise<any[]> {
+    // Phase 26: Try FTS5 first
+    if (this._fts5 && project) {
+      const results = this._fts5.fetchAll(project, 'pattern')
+      if (results.length > 0) {
+        return results.map(r => ({
+          id: r.id,
+          content: r.content,
+          date: r.created_at,
+          project: r.project,
+          pattern_type: '',
+          description: r.content,
+          example: '',
+          confidence: r.confidence,
+          context: r.context || ''
+        }))
+      }
+    }
+
     if (this.useChromaDB) {
       try {
         const collection = await this.chroma.collections.getPatterns()
@@ -474,9 +710,27 @@ export class MemoryManager {
   }
 
   /**
-   * Fetch all corrections with content — routes to ChromaDB or SQLite
+   * Fetch all corrections with content — routes to FTS5, ChromaDB, or legacy SQLite
    */
   async fetchAllCorrections(project?: string): Promise<any[]> {
+    // Phase 26: Try FTS5 first
+    if (this._fts5 && project) {
+      const results = this._fts5.fetchAll(project, 'correction')
+      if (results.length > 0) {
+        return results.map(r => ({
+          id: r.id,
+          content: r.content,
+          date: r.created_at,
+          project: r.project,
+          original: '',
+          correction: r.content,
+          reasoning: r.reasoning || '',
+          context: r.context || '',
+          confidence: r.confidence
+        }))
+      }
+    }
+
     if (this.useChromaDB) {
       try {
         const collection = await this.chroma.collections.getCorrections()
@@ -505,7 +759,7 @@ export class MemoryManager {
   }
 
   /**
-   * Search patterns by query — routes to ChromaDB or SQLite
+   * Search patterns by query — routes to FTS5, ChromaDB, or legacy SQLite
    */
   async searchPatterns(
     query: string,
@@ -516,6 +770,27 @@ export class MemoryManager {
       minSimilarity?: number
     }
   ): Promise<any[]> {
+    // Phase 26: Try FTS5 first
+    if (this._fts5 && query) {
+      const results = this._fts5.searchWithConfidence(query, options?.project, options?.limit || 10)
+      const patterns = results.filter(r => r.category === 'pattern')
+      if (patterns.length > 0) {
+        return patterns.map(r => ({
+          id: r.id,
+          content: r.content,
+          metadata: {
+            project: r.project,
+            pattern_type: '',
+            description: r.content,
+            confidence: r.confidence,
+            context: r.context || '',
+            created_at: r.created_at
+          },
+          similarity: r.score
+        }))
+      }
+    }
+
     if (this.useChromaDB) {
       return this.chroma.store.searchPatterns(query, options)
     }
@@ -523,7 +798,7 @@ export class MemoryManager {
   }
 
   /**
-   * Search corrections by query — routes to ChromaDB or SQLite
+   * Search corrections by query — routes to FTS5, ChromaDB, or legacy SQLite
    */
   async searchCorrections(
     query: string,
@@ -533,6 +808,26 @@ export class MemoryManager {
       minSimilarity?: number
     }
   ): Promise<any[]> {
+    // Phase 26: Try FTS5 first
+    if (this._fts5 && query) {
+      const results = this._fts5.searchWithConfidence(query, options?.project, options?.limit || 10)
+      const corrections = results.filter(r => r.category === 'correction')
+      if (corrections.length > 0) {
+        return corrections.map(r => ({
+          id: r.id,
+          content: r.content,
+          metadata: {
+            project: r.project,
+            reasoning: r.reasoning || '',
+            context: r.context || '',
+            confidence: r.confidence,
+            created_at: r.created_at
+          },
+          similarity: r.score
+        }))
+      }
+    }
+
     if (this.useChromaDB) {
       return this.chroma.store.searchCorrections(query, options)
     }
@@ -540,9 +835,18 @@ export class MemoryManager {
   }
 
   /**
-   * Delete a decision by ID — routes to ChromaDB or SQLite
+   * Delete a decision by ID — removes from FTS5 + ChromaDB/SQLite
    */
   async deleteDecision(id: string): Promise<void> {
+    // Phase 26: Delete from FTS5
+    if (this._fts5) {
+      try {
+        this._fts5.delete(id)
+      } catch (error) {
+        this.logger.warn({ error, id }, 'FTS5 delete failed')
+      }
+    }
+
     if (this.useChromaDB) {
       await this.chroma.store.deleteDecision(id)
     } else {

package/src/memory/migrations/add-fts5.ts

@@ -0,0 +1,98 @@
+/**
+ * FTS5 Migration — Phase 26
+ * Adds observations table with FTS5 virtual table for full-text search.
+ * This replaces ChromaDB as the primary search backend.
+ */
+
+import type { Database } from 'bun:sqlite'
+
+export function addFTS5Tables(db: Database): void {
+  // Observations table (unified storage replacing separate ChromaDB collections)
+  db.run(`
+    CREATE TABLE IF NOT EXISTS observations (
+      id TEXT PRIMARY KEY,
+      project TEXT NOT NULL,
+      category TEXT NOT NULL CHECK(category IN ('decision', 'pattern', 'correction', 'insight', 'preference')),
+      content TEXT NOT NULL,
+      reasoning TEXT,
+      context TEXT,
+      confidence REAL DEFAULT 0.8,
+      source TEXT DEFAULT 'explicit',
+      tags TEXT,
+      file_paths TEXT,
+      symbols TEXT,
+      access_count INTEGER DEFAULT 0,
+      last_accessed TEXT,
+      created_at TEXT NOT NULL,
+      updated_at TEXT NOT NULL,
+      archived INTEGER DEFAULT 0
+    )
+  `)
+
+  // FTS5 virtual table for full-text search
+  db.run(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS observations_fts USING fts5(
+      content, reasoning, context, tags,
+      content='observations', content_rowid='rowid'
+    )
+  `)
+
+  // Keep FTS index in sync with observations table
+  db.run(`
+    CREATE TRIGGER IF NOT EXISTS observations_ai AFTER INSERT ON observations BEGIN
+      INSERT INTO observations_fts(rowid, content, reasoning, context, tags)
+      VALUES (new.rowid, new.content, new.reasoning, new.context, new.tags);
+    END
+  `)
+
+  db.run(`
+    CREATE TRIGGER IF NOT EXISTS observations_ad AFTER DELETE ON observations BEGIN
+      INSERT INTO observations_fts(observations_fts, rowid, content, reasoning, context, tags)
+      VALUES ('delete', old.rowid, old.content, old.reasoning, old.context, old.tags);
+    END
+  `)
+
+  db.run(`
+    CREATE TRIGGER IF NOT EXISTS observations_au AFTER UPDATE ON observations BEGIN
+      INSERT INTO observations_fts(observations_fts, rowid, content, reasoning, context, tags)
+      VALUES ('delete', old.rowid, old.content, old.reasoning, old.context, old.tags);
+      INSERT INTO observations_fts(rowid, content, reasoning, context, tags)
+      VALUES (new.rowid, new.content, new.reasoning, new.context, new.tags);
+    END
+  `)
+
+  // Indexes for common queries
+  db.run(`CREATE INDEX IF NOT EXISTS idx_obs_project ON observations(project)`)
+  db.run(`CREATE INDEX IF NOT EXISTS idx_obs_category ON observations(category)`)
+  db.run(`CREATE INDEX IF NOT EXISTS idx_obs_created ON observations(created_at)`)
+  db.run(`CREATE INDEX IF NOT EXISTS idx_obs_archived ON observations(archived)`)
+
+  // Activity log table
+  db.run(`
+    CREATE TABLE IF NOT EXISTS activity_log (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id TEXT,
+      tool_name TEXT NOT NULL,
+      tool_input TEXT,
+      tool_output TEXT,
+      project TEXT,
+      file_path TEXT,
+      created_at TEXT NOT NULL
+    )
+  `)
+
+  db.run(`CREATE INDEX IF NOT EXISTS idx_activity_session ON activity_log(session_id)`)
+  db.run(`CREATE INDEX IF NOT EXISTS idx_activity_created ON activity_log(created_at)`)
+  db.run(`CREATE INDEX IF NOT EXISTS idx_activity_project ON activity_log(project)`)
+
+  // Observation links (supersedes/contradicts/supports relationships)
+  db.run(`
+    CREATE TABLE IF NOT EXISTS observation_links (
+      source_id TEXT NOT NULL,
+      target_id TEXT NOT NULL,
+      link_type TEXT NOT NULL,
+      created_at TEXT NOT NULL,
+      PRIMARY KEY (source_id, target_id)
+    )
+  `)
+}