claude-brain 0.30.2 → 0.30.3

package/src/memory/dual-write.ts

@@ -0,0 +1,145 @@
+/**
+ * Dual-Write Manager — Task 3.5
+ *
+ * Extracts the repeated FTS5 → ChromaDB → SQLite fallback pattern
+ * from MemoryManager into a single reusable class.
+ *
+ * The fallback chain for every operation:
+ *   1. FTS5 (primary, always-available SQLite-based)
+ *   2. ChromaDB (secondary, vector store)
+ *   3. Legacy SQLite MemoryStore (final fallback)
+ */
+
+import type { Logger } from 'pino'
+import type { FTS5Search } from './fts5-search'
+import type { ChromaManager } from './chroma'
+import type { EmbeddingService } from './embeddings'
+
+export class DualWriteManager {
+  constructor(
+    private logger: Logger,
+    private getFts5: () => FTS5Search | null,
+    private getChroma: () => { manager: ChromaManager; enabled: boolean },
+    private getEmbeddings: () => EmbeddingService
+  ) {}
+
+  /**
+   * Store to FTS5 → ChromaDB → SQLite with fallback chain.
+   * Returns the ID from whichever backend succeeded first.
+   */
+  async store(opts: {
+    sharedId: string
+    fts5Fn: (fts5: FTS5Search, id: string) => string | undefined
+    chromaFn?: (chroma: ChromaManager, id: string) => Promise<string>
+    sqliteFn?: () => Promise<string>
+    embeddingText?: string
+  }): Promise<string> {
+    let fts5Id: string | undefined
+
+    // Step 1: FTS5
+    const fts5 = this.getFts5()
+    if (fts5) {
+      try {
+        fts5Id = opts.fts5Fn(fts5, opts.sharedId)
+
+        // Fire-and-forget embedding
+        if (fts5Id && opts.embeddingText) {
+          this.storeEmbeddingAsync(fts5Id, opts.embeddingText)
+        }
+      } catch (error) {
+        this.logger.warn({ error }, 'FTS5 store failed, continuing with other backends')
+      }
+    }
+
+    // Step 2: ChromaDB
+    const { manager: chroma, enabled } = this.getChroma()
+    if (enabled && opts.chromaFn) {
+      try {
+        const chromaId = await opts.chromaFn(chroma, opts.sharedId)
+        return fts5Id || chromaId
+      } catch (error) {
+        this.logger.warn({ error }, 'ChromaDB store failed')
+        if (fts5Id) return fts5Id
+      }
+    }
+
+    // Step 3: SQLite fallback
+    if (!fts5Id && opts.sqliteFn) {
+      return await opts.sqliteFn()
+    }
+
+    return fts5Id!
+  }
+
+  /**
+   * Fetch from FTS5 → ChromaDB → SQLite with fallback chain.
+   * The fts5Fn/chromaFn/sqliteFn each return mapped results in the
+   * caller's desired shape, keeping transformation logic in MemoryManager.
+   */
+  async fetch<T>(opts: {
+    fts5Fn?: (fts5: FTS5Search) => T[]
+    chromaFn?: (chroma: ChromaManager) => Promise<T[]>
+    sqliteFn: () => T[] | Promise<T[]>
+  }): Promise<T[]> {
+    // Try FTS5
+    const fts5 = this.getFts5()
+    if (fts5 && opts.fts5Fn) {
+      const results = opts.fts5Fn(fts5)
+      if (results.length > 0) return results
+    }
+
+    // Try ChromaDB
+    const { manager: chroma, enabled } = this.getChroma()
+    if (enabled && opts.chromaFn) {
+      try {
+        return await opts.chromaFn(chroma)
+      } catch (error) {
+        this.logger.warn({ error }, 'ChromaDB fetch failed, falling back to SQLite')
+      }
+    }
+
+    // SQLite fallback
+    return await opts.sqliteFn()
+  }
+
+  /**
+   * Delete from all backends: FTS5, then ChromaDB or SQLite.
+   */
+  async delete(id: string, opts: {
+    fts5Fn?: (fts5: FTS5Search) => void
+    chromaFn?: (chroma: ChromaManager) => Promise<void>
+    sqliteFn?: () => void
+  }): Promise<void> {
+    const fts5 = this.getFts5()
+    if (fts5 && opts.fts5Fn) {
+      try {
+        opts.fts5Fn(fts5)
+      } catch (error) {
+        this.logger.warn({ error, id }, 'FTS5 delete failed')
+      }
+    }
+
+    const { manager: chroma, enabled } = this.getChroma()
+    if (enabled && opts.chromaFn) {
+      await opts.chromaFn(chroma)
+    } else if (opts.sqliteFn) {
+      opts.sqliteFn()
+    }
+  }
+
+  /**
+   * Fire-and-forget embedding generation and storage.
+   */
+  private storeEmbeddingAsync(id: string, text: string): void {
+    const embeddings = this.getEmbeddings()
+    if (!embeddings.isReady()) return
+    const fts5 = this.getFts5()
+    if (!fts5) return
+
+    embeddings.generateEmbedding(text).then(embedding => {
+      fts5.storeEmbedding(id, embedding)
+    }).catch(error => {
+      this.logger.debug({ error, id }, 'Failed to store observation embedding')
+    })
+  }
+}

package/src/memory/embeddings.ts

@@ -1,226 +1,226 @@
-/**
- * Embedding Generation Service
- * Phase 3: Memory and Embedding System
- *
- * Uses transformers.js for local embedding generation
- * Model: all-MiniLM-L6-v2 (384 dimensions)
- */
-
-import { pipeline, env } from '@xenova/transformers'
-import { createHash } from 'crypto'
-import type { Logger } from 'pino'
-import type { EmbeddingCacheStats } from './types'
-import { cosineSimilarity } from './embedding-utils'
-import { CircuitBreaker } from '@/utils'
-
-// Configure transformers.js for Bun compatibility
-env.allowLocalModels = true
-env.allowRemoteModels = false
-env.useBrowserCache = false
-// Disable web workers to avoid ONNX blob URL issues in Bun
-env.backends.onnx.wasm.numThreads = 1
-env.backends.onnx.wasm.simd = true
-
-type FeatureExtractionPipeline = Awaited<ReturnType<typeof pipeline>>
-
-export class EmbeddingService {
-  private embeddingPipeline: FeatureExtractionPipeline | null = null
-  private modelName: string
-  private isInitialized: boolean = false
-  private logger: Logger
-  private cache: Map<string, number[]>
-  private maxCacheSize: number
-  private circuitBreaker: CircuitBreaker
-
-  constructor(
-    logger: Logger,
-    modelName: string = 'Xenova/all-MiniLM-L6-v2',
-    maxCacheSize: number = 1000
-  ) {
-    this.modelName = modelName
-    this.logger = logger.child({ component: 'embedding-service' })
-    this.cache = new Map()
-    this.maxCacheSize = maxCacheSize
-    this.circuitBreaker = new CircuitBreaker(
-      'embedding-service',
-      logger,
-      {
-        failureThreshold: 3,
-        timeout: 30000
-      }
-    )
-  }
-
-  /**
-   * Initialize the embedding pipeline
-   * Downloads model on first run, then uses cache
-   */
-  async initialize(): Promise<void> {
-    if (this.isInitialized) {
-      return
-    }
-
-    try {
-      this.logger.info({ model: this.modelName }, 'Loading embedding model...')
-
-      // Load the feature extraction pipeline
-      this.embeddingPipeline = await pipeline('feature-extraction', this.modelName)
-
-      this.isInitialized = true
-      this.logger.info({ model: this.modelName }, 'Embedding model loaded successfully')
-    } catch (error) {
-      this.logger.error({ error, model: this.modelName }, 'Failed to load embedding model')
-      throw error
-    }
-  }
-
-  /**
-   * Generate embedding for a single text
-   */
-  async generateEmbedding(text: string): Promise<number[]> {
-    if (!this.isInitialized) {
-      await this.initialize()
-    }
-
-    const cacheKey = this.getCacheKey(text)
-    const cached = this.cache.get(cacheKey)
-    if (cached) {
-      this.logger.debug({ cacheKey }, 'Embedding cache hit')
-      return cached
-    }
-
-    return this.circuitBreaker.execute(async () => {
-      const startTime = Date.now()
-
-      const preprocessed = this.preprocessText(text)
-
-      const output = await (this.embeddingPipeline as any)(preprocessed, {
-        pooling: 'mean',
-        normalize: true
-      })
-
-      const embedding = Array.from((output as any).data as Float32Array)
-
-      const duration = Date.now() - startTime
-      this.logger.debug(
-        { textLength: text.length, duration, dimensions: embedding.length },
-        'Embedding generated'
-      )
-
-      this.addToCache(cacheKey, embedding)
-
-      return embedding
-    })
-  }
-
-  /**
-   * Generate embeddings for multiple texts (batched)
-   */
-  async generateEmbeddings(texts: string[]): Promise<number[][]> {
-    if (!this.isInitialized) {
-      await this.initialize()
-    }
-
-    // Process in batches to avoid memory issues
-    const batchSize = 10
-    const results: number[][] = []
-
-    for (let i = 0; i < texts.length; i += batchSize) {
-      const batch = texts.slice(i, i + batchSize)
-      const batchResults = await Promise.all(batch.map((text) => this.generateEmbedding(text)))
-      results.push(...batchResults)
-
-      this.logger.debug(
-        { progress: `${Math.min(i + batchSize, texts.length)}/${texts.length}` },
-        'Batch embedding progress'
-      )
-    }
-
-    return results
-  }
-
-  /**
-   * Preprocess text before embedding
-   */
-  private preprocessText(text: string): string {
-    // Remove excessive whitespace
-    let processed = text.replace(/\s+/g, ' ').trim()
-
-    // Truncate to max length (256 tokens ~ 1024 chars)
-    if (processed.length > 1024) {
-      processed = processed.slice(0, 1024)
-      this.logger.debug('Text truncated to 1024 characters')
-    }
-
-    return processed
-  }
-
-  /**
-   * Generate cache key from text
-   * Uses SHA-256 hash to prevent collisions
-   */
-  private getCacheKey(text: string): string {
-    return createHash('sha256').update(text).digest('hex')
-  }
-
-  /**
-   * Add embedding to cache with LRU eviction
-   */
-  private addToCache(key: string, embedding: number[]): void {
-    // Simple LRU: if at max size, delete oldest entry
-    if (this.cache.size >= this.maxCacheSize) {
-      const firstKey = this.cache.keys().next().value
-      if (firstKey) {
-        this.cache.delete(firstKey)
-      }
-    }
-    this.cache.set(key, embedding)
-  }
-
-  /**
-   * Clear embedding cache
-   */
-  clearCache(): void {
-    this.cache.clear()
-    this.logger.debug('Embedding cache cleared')
-  }
-
-  /**
-   * Get cache statistics
-   */
-  getCacheStats(): EmbeddingCacheStats {
-    return {
-      size: this.cache.size,
-      keys: Array.from(this.cache.keys())
-    }
-  }
-
-  /**
-   * Check if service is initialized
-   */
-  isReady(): boolean {
-    return this.isInitialized
-  }
-
-  /**
-   * Calculate cosine similarity between two embeddings
-   * Static method for external use
-   */
-  static cosineSimilarity(a: number[], b: number[]): number {
-    return cosineSimilarity(a, b)
-  }
-
-  /**
-   * Get the model name
-   */
-  getModelName(): string {
-    return this.modelName
-  }
-
-  /**
-   * Get expected embedding dimensions
-   */
-  getEmbeddingDimensions(): number {
-    return 384 // all-MiniLM-L6-v2 outputs 384 dimensions
-  }
-}
+/**
+ * Embedding Generation Service
+ * Phase 3: Memory and Embedding System
+ *
+ * Uses transformers.js for local embedding generation
+ * Model: all-MiniLM-L6-v2 (384 dimensions)
+ */
+
+import { pipeline, env } from '@xenova/transformers'
+import { createHash } from 'crypto'
+import type { Logger } from 'pino'
+import type { EmbeddingCacheStats } from './types'
+import { cosineSimilarity } from './embedding-utils'
+import { CircuitBreaker } from '@/utils'
+
+// Configure transformers.js for Bun compatibility
+env.allowLocalModels = true
+env.allowRemoteModels = false
+env.useBrowserCache = false
+// Disable web workers to avoid ONNX blob URL issues in Bun
+env.backends.onnx.wasm.numThreads = 1
+env.backends.onnx.wasm.simd = true
+
+type FeatureExtractionPipeline = Awaited<ReturnType<typeof pipeline>>
+
+export class EmbeddingService {
+  private embeddingPipeline: FeatureExtractionPipeline | null = null
+  private modelName: string
+  private isInitialized: boolean = false
+  private logger: Logger
+  private cache: Map<string, number[]>
+  private maxCacheSize: number
+  private circuitBreaker: CircuitBreaker
+
+  constructor(
+    logger: Logger,
+    modelName: string = 'Xenova/all-MiniLM-L6-v2',
+    maxCacheSize: number = 1000
+  ) {
+    this.modelName = modelName
+    this.logger = logger.child({ component: 'embedding-service' })
+    this.cache = new Map()
+    this.maxCacheSize = maxCacheSize
+    this.circuitBreaker = new CircuitBreaker(
+      'embedding-service',
+      logger,
+      {
+        failureThreshold: 3,
+        timeout: 30000
+      }
+    )
+  }
+
+  /**
+   * Initialize the embedding pipeline
+   * Downloads model on first run, then uses cache
+   */
+  async initialize(): Promise<void> {
+    if (this.isInitialized) {
+      return
+    }
+
+    try {
+      this.logger.info({ model: this.modelName }, 'Loading embedding model...')
+
+      // Load the feature extraction pipeline
+      this.embeddingPipeline = await pipeline('feature-extraction', this.modelName)
+
+      this.isInitialized = true
+      this.logger.info({ model: this.modelName }, 'Embedding model loaded successfully')
+    } catch (error) {
+      this.logger.error({ error, model: this.modelName }, 'Failed to load embedding model')
+      throw error
+    }
+  }
+
+  /**
+   * Generate embedding for a single text
+   */
+  async generateEmbedding(text: string): Promise<number[]> {
+    if (!this.isInitialized) {
+      await this.initialize()
+    }
+
+    const cacheKey = this.getCacheKey(text)
+    const cached = this.cache.get(cacheKey)
+    if (cached) {
+      this.logger.debug({ cacheKey }, 'Embedding cache hit')
+      return cached
+    }
+
+    return this.circuitBreaker.execute(async () => {
+      const startTime = Date.now()
+
+      const preprocessed = this.preprocessText(text)
+
+      const output = await (this.embeddingPipeline as any)(preprocessed, {
+        pooling: 'mean',
+        normalize: true
+      })
+
+      const embedding = Array.from((output as any).data as Float32Array)
+
+      const duration = Date.now() - startTime
+      this.logger.debug(
+        { textLength: text.length, duration, dimensions: embedding.length },
+        'Embedding generated'
+      )
+
+      this.addToCache(cacheKey, embedding)
+
+      return embedding
+    })
+  }
+
+  /**
+   * Generate embeddings for multiple texts (batched)
+   */
+  async generateEmbeddings(texts: string[]): Promise<number[][]> {
+    if (!this.isInitialized) {
+      await this.initialize()
+    }
+
+    // Process in batches to avoid memory issues
+    const batchSize = 10
+    const results: number[][] = []
+
+    for (let i = 0; i < texts.length; i += batchSize) {
+      const batch = texts.slice(i, i + batchSize)
+      const batchResults = await Promise.all(batch.map((text) => this.generateEmbedding(text)))
+      results.push(...batchResults)
+
+      this.logger.debug(
+        { progress: `${Math.min(i + batchSize, texts.length)}/${texts.length}` },
+        'Batch embedding progress'
+      )
+    }
+
+    return results
+  }
+
+  /**
+   * Preprocess text before embedding
+   */
+  private preprocessText(text: string): string {
+    // Remove excessive whitespace
+    let processed = text.replace(/\s+/g, ' ').trim()
+
+    // Truncate to max length (256 tokens ~ 1024 chars)
+    if (processed.length > 1024) {
+      processed = processed.slice(0, 1024)
+      this.logger.debug('Text truncated to 1024 characters')
+    }
+
+    return processed
+  }
+
+  /**
+   * Generate cache key from text
+   * Uses SHA-256 hash to prevent collisions
+   */
+  private getCacheKey(text: string): string {
+    return createHash('sha256').update(text).digest('hex')
+  }
+
+  /**
+   * Add embedding to cache with LRU eviction
+   */
+  private addToCache(key: string, embedding: number[]): void {
+    // Simple LRU: if at max size, delete oldest entry
+    if (this.cache.size >= this.maxCacheSize) {
+      const firstKey = this.cache.keys().next().value
+      if (firstKey) {
+        this.cache.delete(firstKey)
+      }
+    }
+    this.cache.set(key, embedding)
+  }
+
+  /**
+   * Clear embedding cache
+   */
+  clearCache(): void {
+    this.cache.clear()
+    this.logger.debug('Embedding cache cleared')
+  }
+
+  /**
+   * Get cache statistics
+   */
+  getCacheStats(): EmbeddingCacheStats {
+    return {
+      size: this.cache.size,
+      keys: Array.from(this.cache.keys())
+    }
+  }
+
+  /**
+   * Check if service is initialized
+   */
+  isReady(): boolean {
+    return this.isInitialized
+  }
+
+  /**
+   * Calculate cosine similarity between two embeddings
+   * Static method for external use
+   */
+  static cosineSimilarity(a: number[], b: number[]): number {
+    return cosineSimilarity(a, b)
+  }
+
+  /**
+   * Get the model name
+   */
+  getModelName(): string {
+    return this.modelName
+  }
+
+  /**
+   * Get expected embedding dimensions
+   */
+  getEmbeddingDimensions(): number {
+    return 384 // all-MiniLM-L6-v2 outputs 384 dimensions
+  }
+}