claude-brain 0.15.2 → 0.17.0

package/src/memory/embeddings.ts

@@ -1,226 +1,226 @@
-/**
- * Embedding Generation Service
- * Phase 3: Memory and Embedding System
- *
- * Uses transformers.js for local embedding generation
- * Model: all-MiniLM-L6-v2 (384 dimensions)
- */
-
-import { pipeline, env } from '@xenova/transformers'
-import { createHash } from 'crypto'
-import type { Logger } from 'pino'
-import type { EmbeddingCacheStats } from './types'
-import { cosineSimilarity } from './embedding-utils'
-import { CircuitBreaker } from '@/utils'
-
-// Configure transformers.js for Bun compatibility
-env.allowLocalModels = true
-env.allowRemoteModels = true
-env.useBrowserCache = false
-// Disable web workers to avoid ONNX blob URL issues in Bun
-env.backends.onnx.wasm.numThreads = 1
-env.backends.onnx.wasm.simd = true
-
-type FeatureExtractionPipeline = Awaited<ReturnType<typeof pipeline>>
-
-export class EmbeddingService {
-  private embeddingPipeline: FeatureExtractionPipeline | null = null
-  private modelName: string
-  private isInitialized: boolean = false
-  private logger: Logger
-  private cache: Map<string, number[]>
-  private maxCacheSize: number
-  private circuitBreaker: CircuitBreaker
-
-  constructor(
-    logger: Logger,
-    modelName: string = 'Xenova/all-MiniLM-L6-v2',
-    maxCacheSize: number = 1000
-  ) {
-    this.modelName = modelName
-    this.logger = logger.child({ component: 'embedding-service' })
-    this.cache = new Map()
-    this.maxCacheSize = maxCacheSize
-    this.circuitBreaker = new CircuitBreaker(
-      'embedding-service',
-      logger,
-      {
-        failureThreshold: 3,
-        timeout: 30000
-      }
-    )
-  }
-
-  /**
-   * Initialize the embedding pipeline
-   * Downloads model on first run, then uses cache
-   */
-  async initialize(): Promise<void> {
-    if (this.isInitialized) {
-      return
-    }
-
-    try {
-      this.logger.info({ model: this.modelName }, 'Loading embedding model...')
-
-      // Load the feature extraction pipeline
-      this.embeddingPipeline = await pipeline('feature-extraction', this.modelName)
-
-      this.isInitialized = true
-      this.logger.info({ model: this.modelName }, 'Embedding model loaded successfully')
-    } catch (error) {
-      this.logger.error({ error, model: this.modelName }, 'Failed to load embedding model')
-      throw error
-    }
-  }
-
-  /**
-   * Generate embedding for a single text
-   */
-  async generateEmbedding(text: string): Promise<number[]> {
-    if (!this.isInitialized) {
-      await this.initialize()
-    }
-
-    const cacheKey = this.getCacheKey(text)
-    const cached = this.cache.get(cacheKey)
-    if (cached) {
-      this.logger.debug({ cacheKey }, 'Embedding cache hit')
-      return cached
-    }
-
-    return this.circuitBreaker.execute(async () => {
-      const startTime = Date.now()
-
-      const preprocessed = this.preprocessText(text)
-
-      const output = await (this.embeddingPipeline as any)(preprocessed, {
-        pooling: 'mean',
-        normalize: true
-      })
-
-      const embedding = Array.from((output as any).data as Float32Array)
-
-      const duration = Date.now() - startTime
-      this.logger.debug(
-        { textLength: text.length, duration, dimensions: embedding.length },
-        'Embedding generated'
-      )
-
-      this.addToCache(cacheKey, embedding)
-
-      return embedding
-    })
-  }
-
-  /**
-   * Generate embeddings for multiple texts (batched)
-   */
-  async generateEmbeddings(texts: string[]): Promise<number[][]> {
-    if (!this.isInitialized) {
-      await this.initialize()
-    }
-
-    // Process in batches to avoid memory issues
-    const batchSize = 10
-    const results: number[][] = []
-
-    for (let i = 0; i < texts.length; i += batchSize) {
-      const batch = texts.slice(i, i + batchSize)
-      const batchResults = await Promise.all(batch.map((text) => this.generateEmbedding(text)))
-      results.push(...batchResults)
-
-      this.logger.debug(
-        { progress: `${Math.min(i + batchSize, texts.length)}/${texts.length}` },
-        'Batch embedding progress'
-      )
-    }
-
-    return results
-  }
-
-  /**
-   * Preprocess text before embedding
-   */
-  private preprocessText(text: string): string {
-    // Remove excessive whitespace
-    let processed = text.replace(/\s+/g, ' ').trim()
-
-    // Truncate to max length (256 tokens ~ 1024 chars)
-    if (processed.length > 1024) {
-      processed = processed.slice(0, 1024)
-      this.logger.debug('Text truncated to 1024 characters')
-    }
-
-    return processed
-  }
-
-  /**
-   * Generate cache key from text
-   * Uses SHA-256 hash to prevent collisions
-   */
-  private getCacheKey(text: string): string {
-    return createHash('sha256').update(text).digest('hex')
-  }
-
-  /**
-   * Add embedding to cache with LRU eviction
-   */
-  private addToCache(key: string, embedding: number[]): void {
-    // Simple LRU: if at max size, delete oldest entry
-    if (this.cache.size >= this.maxCacheSize) {
-      const firstKey = this.cache.keys().next().value
-      if (firstKey) {
-        this.cache.delete(firstKey)
-      }
-    }
-    this.cache.set(key, embedding)
-  }
-
-  /**
-   * Clear embedding cache
-   */
-  clearCache(): void {
-    this.cache.clear()
-    this.logger.debug('Embedding cache cleared')
-  }
-
-  /**
-   * Get cache statistics
-   */
-  getCacheStats(): EmbeddingCacheStats {
-    return {
-      size: this.cache.size,
-      keys: Array.from(this.cache.keys())
-    }
-  }
-
-  /**
-   * Check if service is initialized
-   */
-  isReady(): boolean {
-    return this.isInitialized
-  }
-
-  /**
-   * Calculate cosine similarity between two embeddings
-   * Static method for external use
-   */
-  static cosineSimilarity(a: number[], b: number[]): number {
-    return cosineSimilarity(a, b)
-  }
-
-  /**
-   * Get the model name
-   */
-  getModelName(): string {
-    return this.modelName
-  }
-
-  /**
-   * Get expected embedding dimensions
-   */
-  getEmbeddingDimensions(): number {
-    return 384 // all-MiniLM-L6-v2 outputs 384 dimensions
-  }
-}
+/**
+ * Embedding Generation Service
+ * Phase 3: Memory and Embedding System
+ *
+ * Uses transformers.js for local embedding generation
+ * Model: all-MiniLM-L6-v2 (384 dimensions)
+ */
+
+import { pipeline, env } from '@xenova/transformers'
+import { createHash } from 'crypto'
+import type { Logger } from 'pino'
+import type { EmbeddingCacheStats } from './types'
+import { cosineSimilarity } from './embedding-utils'
+import { CircuitBreaker } from '@/utils'
+
+// Configure transformers.js for Bun compatibility
+env.allowLocalModels = true
+env.allowRemoteModels = true
+env.useBrowserCache = false
+// Disable web workers to avoid ONNX blob URL issues in Bun
+env.backends.onnx.wasm.numThreads = 1
+env.backends.onnx.wasm.simd = true
+
+type FeatureExtractionPipeline = Awaited<ReturnType<typeof pipeline>>
+
+export class EmbeddingService {
+  private embeddingPipeline: FeatureExtractionPipeline | null = null
+  private modelName: string
+  private isInitialized: boolean = false
+  private logger: Logger
+  private cache: Map<string, number[]>
+  private maxCacheSize: number
+  private circuitBreaker: CircuitBreaker
+
+  constructor(
+    logger: Logger,
+    modelName: string = 'Xenova/all-MiniLM-L6-v2',
+    maxCacheSize: number = 1000
+  ) {
+    this.modelName = modelName
+    this.logger = logger.child({ component: 'embedding-service' })
+    this.cache = new Map()
+    this.maxCacheSize = maxCacheSize
+    this.circuitBreaker = new CircuitBreaker(
+      'embedding-service',
+      logger,
+      {
+        failureThreshold: 3,
+        timeout: 30000
+      }
+    )
+  }
+
+  /**
+   * Initialize the embedding pipeline
+   * Downloads model on first run, then uses cache
+   */
+  async initialize(): Promise<void> {
+    if (this.isInitialized) {
+      return
+    }
+
+    try {
+      this.logger.info({ model: this.modelName }, 'Loading embedding model...')
+
+      // Load the feature extraction pipeline
+      this.embeddingPipeline = await pipeline('feature-extraction', this.modelName)
+
+      this.isInitialized = true
+      this.logger.info({ model: this.modelName }, 'Embedding model loaded successfully')
+    } catch (error) {
+      this.logger.error({ error, model: this.modelName }, 'Failed to load embedding model')
+      throw error
+    }
+  }
+
+  /**
+   * Generate embedding for a single text
+   */
+  async generateEmbedding(text: string): Promise<number[]> {
+    if (!this.isInitialized) {
+      await this.initialize()
+    }
+
+    const cacheKey = this.getCacheKey(text)
+    const cached = this.cache.get(cacheKey)
+    if (cached) {
+      this.logger.debug({ cacheKey }, 'Embedding cache hit')
+      return cached
+    }
+
+    return this.circuitBreaker.execute(async () => {
+      const startTime = Date.now()
+
+      const preprocessed = this.preprocessText(text)
+
+      const output = await (this.embeddingPipeline as any)(preprocessed, {
+        pooling: 'mean',
+        normalize: true
+      })
+
+      const embedding = Array.from((output as any).data as Float32Array)
+
+      const duration = Date.now() - startTime
+      this.logger.debug(
+        { textLength: text.length, duration, dimensions: embedding.length },
+        'Embedding generated'
+      )
+
+      this.addToCache(cacheKey, embedding)
+
+      return embedding
+    })
+  }
+
+  /**
+   * Generate embeddings for multiple texts (batched)
+   */
+  async generateEmbeddings(texts: string[]): Promise<number[][]> {
+    if (!this.isInitialized) {
+      await this.initialize()
+    }
+
+    // Process in batches to avoid memory issues
+    const batchSize = 10
+    const results: number[][] = []
+
+    for (let i = 0; i < texts.length; i += batchSize) {
+      const batch = texts.slice(i, i + batchSize)
+      const batchResults = await Promise.all(batch.map((text) => this.generateEmbedding(text)))
+      results.push(...batchResults)
+
+      this.logger.debug(
+        { progress: `${Math.min(i + batchSize, texts.length)}/${texts.length}` },
+        'Batch embedding progress'
+      )
+    }
+
+    return results
+  }
+
+  /**
+   * Preprocess text before embedding
+   */
+  private preprocessText(text: string): string {
+    // Remove excessive whitespace
+    let processed = text.replace(/\s+/g, ' ').trim()
+
+    // Truncate to max length (256 tokens ~ 1024 chars)
+    if (processed.length > 1024) {
+      processed = processed.slice(0, 1024)
+      this.logger.debug('Text truncated to 1024 characters')
+    }
+
+    return processed
+  }
+
+  /**
+   * Generate cache key from text
+   * Uses SHA-256 hash to prevent collisions
+   */
+  private getCacheKey(text: string): string {
+    return createHash('sha256').update(text).digest('hex')
+  }
+
+  /**
+   * Add embedding to cache with LRU eviction
+   */
+  private addToCache(key: string, embedding: number[]): void {
+    // Simple LRU: if at max size, delete oldest entry
+    if (this.cache.size >= this.maxCacheSize) {
+      const firstKey = this.cache.keys().next().value
+      if (firstKey) {
+        this.cache.delete(firstKey)
+      }
+    }
+    this.cache.set(key, embedding)
+  }
+
+  /**
+   * Clear embedding cache
+   */
+  clearCache(): void {
+    this.cache.clear()
+    this.logger.debug('Embedding cache cleared')
+  }
+
+  /**
+   * Get cache statistics
+   */
+  getCacheStats(): EmbeddingCacheStats {
+    return {
+      size: this.cache.size,
+      keys: Array.from(this.cache.keys())
+    }
+  }
+
+  /**
+   * Check if service is initialized
+   */
+  isReady(): boolean {
+    return this.isInitialized
+  }
+
+  /**
+   * Calculate cosine similarity between two embeddings
+   * Static method for external use
+   */
+  static cosineSimilarity(a: number[], b: number[]): number {
+    return cosineSimilarity(a, b)
+  }
+
+  /**
+   * Get the model name
+   */
+  getModelName(): string {
+    return this.modelName
+  }
+
+  /**
+   * Get expected embedding dimensions
+   */
+  getEmbeddingDimensions(): number {
+    return 384 // all-MiniLM-L6-v2 outputs 384 dimensions
+  }
+}

package/src/memory/episodic/detector.ts

@@ -1,108 +1,108 @@
-/**
- * Session Detector
- * Classifies messages as new sessions, continuations, or topic shifts
- */
-
-import type { SessionClassification, EpisodeMessage } from './types'
-
-export interface DetectorOptions {
-  sessionGapMinutes?: number
-}
-
-const GREETING_PATTERNS = [
-  /^(?:hi|hello|hey|good\s+(?:morning|afternoon|evening)|greetings)\b/i,
-  /^(?:what's up|howdy|sup)\b/i
-]
-
-const CONTINUATION_PATTERNS = [
-  /(?:continuing|continue|back\s+to|as\s+(?:we\s+)?discussed|picking\s+up|where\s+we\s+left)/i,
-  /(?:regarding|about\s+(?:the|that)|following\s+up|as\s+I\s+(?:said|mentioned))/i,
-  /(?:also|additionally|furthermore|moreover|in\s+addition)/i
-]
-
-export class SessionDetector {
-  private sessionGapMs: number
-
-  constructor(options: DetectorOptions = {}) {
-    this.sessionGapMs = (options.sessionGapMinutes || 30) * 60 * 1000
-  }
-
-  classifyMessage(
-    message: EpisodeMessage,
-    lastActivity?: string,
-    recentMessages?: EpisodeMessage[]
-  ): SessionClassification {
-    const content = message.content
-
-    // Check time gap
-    if (lastActivity) {
-      const gap = new Date(message.timestamp).getTime() - new Date(lastActivity).getTime()
-      if (gap > this.sessionGapMs) {
-        return 'new_session'
-      }
-    } else {
-      // No previous activity → new session
-      return 'new_session'
-    }
-
-    // Check for greeting patterns
-    if (GREETING_PATTERNS.some(p => p.test(content.trim()))) {
-      return 'new_session'
-    }
-
-    // Check for continuation patterns
-    if (CONTINUATION_PATTERNS.some(p => p.test(content))) {
-      return 'continuation'
-    }
-
-    // Check topic coherence with recent messages
-    if (recentMessages && recentMessages.length > 0) {
-      const coherence = this.calculateTopicCoherence(content, recentMessages)
-      if (coherence < 0.1) {
-        return 'topic_shift'
-      }
-    }
-
-    return 'continuation'
-  }
-
-  private calculateTopicCoherence(content: string, recentMessages: EpisodeMessage[]): number {
-    const currentKeywords = this.extractKeywords(content)
-    if (currentKeywords.size === 0) return 0.5
-
-    const recentText = recentMessages.map(m => m.content).join(' ')
-    const recentKeywords = this.extractKeywords(recentText)
-    if (recentKeywords.size === 0) return 0.5
-
-    // Jaccard-like overlap
-    let overlap = 0
-    for (const kw of currentKeywords) {
-      if (recentKeywords.has(kw)) overlap++
-    }
-
-    return overlap / Math.max(currentKeywords.size, 1)
-  }
-
-  private extractKeywords(text: string): Set<string> {
-    const stopWords = new Set([
-      'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
-      'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
-      'should', 'may', 'might', 'can', 'shall', 'to', 'of', 'in', 'for',
-      'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through', 'during',
-      'before', 'after', 'above', 'below', 'between', 'out', 'off', 'over',
-      'under', 'again', 'further', 'then', 'once', 'and', 'but', 'or', 'nor',
-      'not', 'so', 'if', 'that', 'this', 'it', 'its', 'i', 'me', 'my', 'we',
-      'our', 'you', 'your', 'he', 'she', 'they', 'them', 'their', 'what',
-      'which', 'who', 'when', 'where', 'how', 'all', 'each', 'every', 'both',
-      'few', 'more', 'most', 'other', 'some', 'such', 'no', 'only', 'own',
-      'same', 'than', 'too', 'very', 'just', 'about', 'here', 'there'
-    ])
-
-    const words = text.toLowerCase()
-      .replace(/[^a-z0-9\s-]/g, ' ')
-      .split(/\s+/)
-      .filter(w => w.length > 2 && !stopWords.has(w))
-
-    return new Set(words)
-  }
-}
+/**
+ * Session Detector
+ * Classifies messages as new sessions, continuations, or topic shifts
+ */
+
+import type { SessionClassification, EpisodeMessage } from './types'
+
+export interface DetectorOptions {
+  sessionGapMinutes?: number
+}
+
+const GREETING_PATTERNS = [
+  /^(?:hi|hello|hey|good\s+(?:morning|afternoon|evening)|greetings)\b/i,
+  /^(?:what's up|howdy|sup)\b/i
+]
+
+const CONTINUATION_PATTERNS = [
+  /(?:continuing|continue|back\s+to|as\s+(?:we\s+)?discussed|picking\s+up|where\s+we\s+left)/i,
+  /(?:regarding|about\s+(?:the|that)|following\s+up|as\s+I\s+(?:said|mentioned))/i,
+  /(?:also|additionally|furthermore|moreover|in\s+addition)/i
+]
+
+export class SessionDetector {
+  private sessionGapMs: number
+
+  constructor(options: DetectorOptions = {}) {
+    this.sessionGapMs = (options.sessionGapMinutes || 30) * 60 * 1000
+  }
+
+  classifyMessage(
+    message: EpisodeMessage,
+    lastActivity?: string,
+    recentMessages?: EpisodeMessage[]
+  ): SessionClassification {
+    const content = message.content
+
+    // Check time gap
+    if (lastActivity) {
+      const gap = new Date(message.timestamp).getTime() - new Date(lastActivity).getTime()
+      if (gap > this.sessionGapMs) {
+        return 'new_session'
+      }
+    } else {
+      // No previous activity → new session
+      return 'new_session'
+    }
+
+    // Check for greeting patterns
+    if (GREETING_PATTERNS.some(p => p.test(content.trim()))) {
+      return 'new_session'
+    }
+
+    // Check for continuation patterns
+    if (CONTINUATION_PATTERNS.some(p => p.test(content))) {
+      return 'continuation'
+    }
+
+    // Check topic coherence with recent messages
+    if (recentMessages && recentMessages.length > 0) {
+      const coherence = this.calculateTopicCoherence(content, recentMessages)
+      if (coherence < 0.1) {
+        return 'topic_shift'
+      }
+    }
+
+    return 'continuation'
+  }
+
+  private calculateTopicCoherence(content: string, recentMessages: EpisodeMessage[]): number {
+    const currentKeywords = this.extractKeywords(content)
+    if (currentKeywords.size === 0) return 0.5
+
+    const recentText = recentMessages.map(m => m.content).join(' ')
+    const recentKeywords = this.extractKeywords(recentText)
+    if (recentKeywords.size === 0) return 0.5
+
+    // Jaccard-like overlap
+    let overlap = 0
+    for (const kw of currentKeywords) {
+      if (recentKeywords.has(kw)) overlap++
+    }
+
+    return overlap / Math.max(currentKeywords.size, 1)
+  }
+
+  private extractKeywords(text: string): Set<string> {
+    const stopWords = new Set([
+      'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+      'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+      'should', 'may', 'might', 'can', 'shall', 'to', 'of', 'in', 'for',
+      'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through', 'during',
+      'before', 'after', 'above', 'below', 'between', 'out', 'off', 'over',
+      'under', 'again', 'further', 'then', 'once', 'and', 'but', 'or', 'nor',
+      'not', 'so', 'if', 'that', 'this', 'it', 'its', 'i', 'me', 'my', 'we',
+      'our', 'you', 'your', 'he', 'she', 'they', 'them', 'their', 'what',
+      'which', 'who', 'when', 'where', 'how', 'all', 'each', 'every', 'both',
+      'few', 'more', 'most', 'other', 'some', 'such', 'no', 'only', 'own',
+      'same', 'than', 'too', 'very', 'just', 'about', 'here', 'there'
+    ])
+
+    const words = text.toLowerCase()
+      .replace(/[^a-z0-9\s-]/g, ' ')
+      .split(/\s+/)
+      .filter(w => w.length > 2 && !stopWords.has(w))
+
+    return new Set(words)
+  }
+}