claude-brain 0.3.0

package/src/memory/episodic/summarizer.ts

@@ -0,0 +1,179 @@
+/**
+ * Extractive Summarizer
+ * Generates summaries by scoring and selecting key sentences
+ */
+
+import type { Episode, EpisodeSummary } from './types'
+
+interface ScoredSentence {
+  text: string
+  score: number
+  position: number
+}
+
+export class ExtractiveSummarizer {
+  private briefSize: number
+  private detailedSize: number
+
+  constructor(briefSize: number = 2, detailedSize: number = 8) {
+    this.briefSize = briefSize
+    this.detailedSize = detailedSize
+  }
+
+  summarize(episode: Episode): EpisodeSummary {
+    const allText = episode.messages.map(m => m.content).join('\n')
+    const sentences = this.splitSentences(allText)
+
+    if (sentences.length === 0) {
+      return {
+        brief: 'Empty episode',
+        detailed: 'No messages in this episode.',
+        key_topics: [],
+        decisions_made: [],
+        unresolved_questions: [],
+        entity_count: 0
+      }
+    }
+
+    // Calculate word frequencies for TF scoring
+    const wordFreqs = this.calculateWordFrequencies(allText)
+    const totalSentences = sentences.length
+
+    // Score each sentence
+    const scored: ScoredSentence[] = sentences.map((text, position) => ({
+      text,
+      position,
+      score: this.scoreSentence(text, position, totalSentences, wordFreqs)
+    }))
+
+    // Sort by score descending
+    const ranked = [...scored].sort((a, b) => b.score - a.score)
+
+    // Select top sentences for brief and detailed, then re-order by position
+    const briefSentences = ranked
+      .slice(0, this.briefSize)
+      .sort((a, b) => a.position - b.position)
+      .map(s => s.text)
+
+    const detailedSentences = ranked
+      .slice(0, this.detailedSize)
+      .sort((a, b) => a.position - b.position)
+      .map(s => s.text)
+
+    // Extract key topics from high-frequency words
+    const keyTopics = this.extractKeyTopics(wordFreqs)
+
+    // Find decisions (sentences with decision keywords)
+    const decisionKeywords = ['decided', 'chose', 'selected', 'will use', 'going with', 'recommend', 'should use']
+    const decisions = sentences.filter(s =>
+      decisionKeywords.some(k => s.toLowerCase().includes(k))
+    ).slice(0, 5)
+
+    // Find unresolved questions (last messages with ? that aren't answered)
+    const lastMessages = episode.messages.slice(-Math.ceil(episode.messages.length / 2))
+    const questions = lastMessages
+      .filter(m => m.role === 'user' && m.content.includes('?'))
+      .map(m => {
+        const qSentences = this.splitSentences(m.content)
+        return qSentences.filter(s => s.includes('?'))
+      })
+      .flat()
+      .slice(0, 3)
+
+    return {
+      brief: briefSentences.join(' '),
+      detailed: detailedSentences.join(' '),
+      key_topics: keyTopics,
+      decisions_made: decisions,
+      unresolved_questions: questions,
+      entity_count: keyTopics.length
+    }
+  }
+
+  private splitSentences(text: string): string[] {
+    // Split on sentence boundaries
+    const raw = text
+      .replace(/\n+/g, '. ')
+      .split(/(?<=[.!?])\s+/)
+      .map(s => s.trim())
+      .filter(s => s.length > 10)
+
+    return raw
+  }
+
+  private calculateWordFrequencies(text: string): Map<string, number> {
+    const stopWords = new Set([
+      'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+      'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+      'should', 'can', 'may', 'might', 'to', 'of', 'in', 'for', 'on',
+      'with', 'at', 'by', 'from', 'as', 'and', 'but', 'or', 'not', 'so',
+      'if', 'that', 'this', 'it', 'its', 'i', 'me', 'my', 'we', 'our',
+      'you', 'your', 'he', 'she', 'they', 'them', 'their', 'what', 'which',
+      'who', 'when', 'where', 'how', 'all', 'each', 'both', 'few', 'more',
+      'most', 'other', 'some', 'no', 'only', 'own', 'same', 'than', 'too',
+      'very', 'just', 'about', 'up', 'out', 'off'
+    ])
+
+    const freqs = new Map<string, number>()
+    const words = text.toLowerCase()
+      .replace(/[^a-z0-9\s-]/g, ' ')
+      .split(/\s+/)
+      .filter(w => w.length > 2 && !stopWords.has(w))
+
+    for (const word of words) {
+      freqs.set(word, (freqs.get(word) || 0) + 1)
+    }
+
+    return freqs
+  }
+
+  private scoreSentence(
+    sentence: string,
+    position: number,
+    totalSentences: number,
+    wordFreqs: Map<string, number>
+  ): number {
+    let score = 0
+
+    // 1. TF-based keyword score
+    const words = sentence.toLowerCase()
+      .replace(/[^a-z0-9\s-]/g, ' ')
+      .split(/\s+/)
+      .filter(w => w.length > 2)
+
+    if (words.length === 0) return 0
+
+    let keywordScore = 0
+    for (const word of words) {
+      keywordScore += wordFreqs.get(word) || 0
+    }
+    keywordScore /= words.length // normalize by sentence length
+    score += keywordScore * 0.4
+
+    // 2. Position bonus (first and last sentences score higher)
+    const posRatio = position / Math.max(totalSentences - 1, 1)
+    const positionScore = posRatio < 0.15 ? 1.0 : posRatio > 0.85 ? 0.8 : 0.3
+    score += positionScore * 0.25
+
+    // 3. Decision keyword boost
+    const decisionKeywords = ['decided', 'chose', 'selected', 'recommend', 'should', 'must', 'will use', 'going with', 'use', 'implement', 'adopt']
+    const lowerSentence = sentence.toLowerCase()
+    const hasDecision = decisionKeywords.some(k => lowerSentence.includes(k))
+    if (hasDecision) score += 0.2
+
+    // 4. Length penalty for very short or very long sentences
+    const optimalLength = 20 // words
+    const lengthRatio = Math.min(words.length, optimalLength * 2) / optimalLength
+    const lengthScore = lengthRatio <= 1 ? lengthRatio : 2 - lengthRatio
+    score += Math.max(0, lengthScore) * 0.15
+
+    return score
+  }
+
+  private extractKeyTopics(wordFreqs: Map<string, number>): string[] {
+    return Array.from(wordFreqs.entries())
+      .sort(([, a], [, b]) => b - a)
+      .slice(0, 10)
+      .map(([word]) => word)
+  }
+}

package/src/memory/episodic/types.ts

@@ -0,0 +1,52 @@
+/**
+ * Episodic Memory Types
+ * Type definitions for the episodic memory system
+ */
+
+export interface EpisodeMessage {
+  role: 'user' | 'assistant' | 'system'
+  content: string
+  timestamp: string
+  token_estimate?: number
+}
+
+export interface EpisodeSummary {
+  brief: string
+  detailed: string
+  key_topics: string[]
+  decisions_made: string[]
+  unresolved_questions: string[]
+  entity_count: number
+}
+
+export type SessionClassification = 'new_session' | 'continuation' | 'topic_shift'
+
+export interface Episode {
+  id: string
+  project?: string
+  status: 'active' | 'completed'
+  started_at: string
+  ended_at?: string
+  messages: EpisodeMessage[]
+  summary?: EpisodeSummary
+  related_decisions: string[]
+  related_patterns: string[]
+  related_corrections: string[]
+  token_count: number
+  message_count: number
+}
+
+export interface EpisodeMetadata {
+  episode_id: string
+  project: string
+  status: 'active' | 'completed'
+  started_at: string
+  ended_at: string
+  message_count: number
+  token_count: number
+  key_topics: string
+  brief_summary: string
+  related_decisions: string
+  related_patterns: string
+  related_corrections: string
+}

package/src/memory/index.ts

@@ -0,0 +1,395 @@
+/**
+ * Memory System - Main Module
+ * Phase 3: Memory and Embedding System
+ *
+ * Unified memory system manager that combines all components
+ */
+
+import type { Logger } from 'pino'
+import { MemoryDatabase } from './database'
+import { EmbeddingService } from './embeddings'
+import { MemoryStore } from './store'
+import { SemanticSearch } from './search'
+import { MemoryContextBuilder } from './context-builder'
+import type { MemorySystemStats } from './types'
+import { ChromaManager, DEFAULT_CHROMA_CONFIG, getChromaConfigFromEnv, ChromaMigration, type MigrationOptions } from './chroma'
+
+// Re-export all types and classes for external use
+export * from './types'
+export { MemoryDatabase } from './database'
+export { EmbeddingService } from './embeddings'
+export { MemoryStore } from './store'
+export { SemanticSearch } from './search'
+export { MemoryContextBuilder, type ContextOptions } from './context-builder'
+export {
+  embeddingToBuffer,
+  bufferToEmbedding,
+  normalizeEmbedding,
+  euclideanDistance,
+  cosineSimilarity,
+  dotProduct,
+  magnitude,
+  averageEmbeddings,
+  topKSimilar
+} from './embedding-utils'
+
+// Phase 12: Advanced Memory Features
+export { PatternRecognizer, type Pattern } from './patterns'
+export { LearningSystem, type Correction, type Preference, type LearningInsights } from './learning'
+export { KnowledgeExtractor, type ExtractedKnowledge, type ExtractionResult } from './knowledge-extractor'
+
+/**
+ * Unified memory system manager
+ * Combines database, embeddings, store, search, and context building
+ */
+export class MemoryManager {
+  readonly database: MemoryDatabase
+  readonly embeddings: EmbeddingService
+  readonly contextBuilder: MemoryContextBuilder
+  readonly chroma: ChromaManager
+
+  // Store and search are initialized after database is ready
+  private _store: MemoryStore | null = null
+  private _search: SemanticSearch | null = null
+
+  private logger: Logger
+  private initialized: boolean = false
+  private useChromaDB: boolean = true
+
+  constructor(
+    dbPath: string,
+    logger: Logger,
+    useChromaDB: boolean = true,
+    chromaConfig?: any,
+    customEmbeddings?: EmbeddingService
+  ) {
+    this.logger = logger.child({ component: 'memory-manager' })
+    this.useChromaDB = useChromaDB
+
+    this.database = new MemoryDatabase(dbPath, logger)
+    this.embeddings = customEmbeddings || new EmbeddingService(logger)
+    this.contextBuilder = new MemoryContextBuilder(logger)
+
+    const envConfig = getChromaConfigFromEnv()
+    const config = { ...DEFAULT_CHROMA_CONFIG, ...envConfig, ...chromaConfig }
+    this.chroma = new ChromaManager(logger, config)
+  }
+
+  /**
+   * Initialize memory system
+   * Must be called before using store or search
+   */
+  async initialize(): Promise<void> {
+    if (this.initialized) {
+      this.logger.warn('Memory system already initialized')
+      return
+    }
+
+    try {
+      this.logger.info('Initializing memory system...')
+
+      await this.database.initialize()
+
+      await this.embeddings.initialize()
+
+      const db = this.database.getDb()
+      this._store = new MemoryStore(db, this.embeddings, this.logger)
+      this._search = new SemanticSearch(db, this.embeddings, this.logger)
+
+      if (this.useChromaDB) {
+        try {
+          await this.chroma.initialize()
+          this.logger.info('ChromaDB backend initialized successfully')
+        } catch (error) {
+          this.logger.warn({ error }, 'Failed to initialize ChromaDB, falling back to SQLite backend')
+          this.useChromaDB = false
+        }
+      }
+
+      this.initialized = true
+      this.logger.info('Memory system initialized successfully')
+    } catch (error) {
+      this.logger.error({ error }, 'Failed to initialize memory system')
+      throw error
+    }
+  }
+
+  /**
+   * Get the memory store (throws if not initialized)
+   */
+  get store(): MemoryStore {
+    if (!this._store) {
+      throw new Error('Memory system not initialized. Call initialize() first.')
+    }
+    return this._store
+  }
+
+  /**
+   * Get the semantic search engine (throws if not initialized)
+   */
+  get search(): SemanticSearch {
+    if (!this._search) {
+      throw new Error('Memory system not initialized. Call initialize() first.')
+    }
+    return this._search
+  }
+
+  /**
+   * Check if memory system is initialized
+   */
+  isInitialized(): boolean {
+    return this.initialized
+  }
+
+  close(): void {
+    if (this.useChromaDB) {
+      this.chroma.close()
+    }
+    this.database.close()
+    this._store = null
+    this._search = null
+    this.initialized = false
+    this.logger.info('Memory system closed')
+  }
+
+  /**
+   * Get system statistics
+   */
+  getStats(): MemorySystemStats {
+    if (!this.initialized) {
+      throw new Error('Memory system not initialized')
+    }
+
+    return {
+      database: this.database.getStats(),
+      embeddings: this.embeddings.getCacheStats()
+    }
+  }
+
+  /**
+   * Health check
+   */
+  async healthCheck(): Promise<boolean> {
+    if (!this.initialized) {
+      return false
+    }
+    return this.database.healthCheck()
+  }
+
+  async rememberDecision(
+    project: string,
+    context: string,
+    decision: string,
+    reasoning: string,
+    options?: { alternatives?: string; tags?: string[] }
+  ): Promise<string> {
+    if (this.useChromaDB) {
+      return this.chroma.store.storeDecision({
+        project,
+        context,
+        decision,
+        reasoning,
+        alternatives: options?.alternatives,
+        tags: options?.tags
+      })
+    }
+    return this.store.storeDecision({
+      project,
+      context,
+      decision,
+      reasoning,
+      alternatives: options?.alternatives,
+      tags: options?.tags
+    })
+  }
+
+  /**
+   * Get raw search results - routes to ChromaDB when enabled
+   * Use this for internal operations that need raw results
+   */
+  async searchRaw(
+    query: string,
+    options?: { project?: string; limit?: number; minSimilarity?: number }
+  ): Promise<any[]> {
+    if (this.useChromaDB) {
+      const chromaResults = await this.chroma.search.searchDecisions(query, {
+        project: options?.project,
+        limit: options?.limit || 5,
+        minSimilarity: options?.minSimilarity || 0.5
+      })
+      // Transform ChromaDB results to match MemorySearchResult structure
+      return chromaResults.map(r => ({
+        memory: {
+          id: r.id,
+          project: r.metadata.project || options?.project || 'unknown',
+          content: typeof r.content === 'string' ? r.content : JSON.stringify(r.content),
+          createdAt: r.metadata.created_at ? new Date(r.metadata.created_at) : new Date(),
+          metadata: r.metadata
+        },
+        similarity: r.similarity,
+        decision: r.metadata.decision ? {
+          id: r.id,
+          project: r.metadata.project || options?.project || 'unknown',
+          context: r.metadata.context || '',
+          decision: r.metadata.decision || (typeof r.content === 'string' ? r.content : ''),
+          reasoning: r.metadata.reasoning || '',
+          alternatives: r.metadata.alternatives_considered || '',
+          tags: r.metadata.tags || [],
+          outcome: r.metadata.outcome,
+          createdAt: r.metadata.created_at ? new Date(r.metadata.created_at) : new Date()
+        } : undefined
+      }))
+    } else {
+      return await this.search.search(query, {
+        project: options?.project,
+        limit: options?.limit || 5,
+        minSimilarity: options?.minSimilarity || 0.5
+      })
+    }
+  }
+
+  async recallSimilar(
+    query: string,
+    options?: { project?: string; limit?: number; minSimilarity?: number }
+  ): Promise<string> {
+    const results = await this.searchRaw(query, options)
+    return this.contextBuilder.buildDecisionContext(results)
+  }
+
+  async getRecommendations(
+    currentContext: string,
+    project: string,
+    limit: number = 3
+  ): Promise<string> {
+    const results = await this.search.getRecommendations(currentContext, project, limit)
+    return this.contextBuilder.buildRecommendationContext(results)
+  }
+
+  async migrateToChromaDB(options: MigrationOptions = {}): Promise<any> {
+    if (!this.useChromaDB) {
+      throw new Error('ChromaDB is not enabled')
+    }
+
+    const migration = new ChromaMigration(
+      this.logger,
+      this.database.getDb(),
+      this.chroma.store,
+      this.chroma.collections
+    )
+
+    return migration.migrate(options)
+  }
+
+  /**
+   * Store a pattern in memory
+   */
+  async storePattern(input: {
+    project: string
+    pattern_type: 'solution' | 'anti-pattern' | 'best-practice' | 'common-issue'
+    description: string
+    example?: string
+    confidence: number
+    context?: string
+  }): Promise<string> {
+    if (!this.useChromaDB) {
+      throw new Error('Pattern storage requires ChromaDB backend')
+    }
+    return this.chroma.store.storePattern(input)
+  }
+
+  /**
+   * Store a correction/lesson learned in memory
+   */
+  async storeCorrection(input: {
+    project: string
+    original: string
+    correction: string
+    reasoning: string
+    context?: string
+    confidence: number
+  }): Promise<string> {
+    if (!this.useChromaDB) {
+      throw new Error('Correction storage requires ChromaDB backend')
+    }
+    return this.chroma.store.storeCorrection(input)
+  }
+
+  /**
+   * Get patterns for a project
+   */
+  async getPatterns(
+    project?: string,
+    options?: {
+      pattern_type?: 'solution' | 'anti-pattern' | 'best-practice' | 'common-issue'
+      limit?: number
+    }
+  ): Promise<any[]> {
+    if (!this.useChromaDB) {
+      throw new Error('Pattern retrieval requires ChromaDB backend')
+    }
+    if (project) {
+      return this.chroma.store.getPatternsByProject(project, options)
+    }
+    // If no project specified, search all patterns
+    return this.chroma.store.searchPatterns('', { limit: options?.limit || 10 })
+  }
+
+  /**
+   * Get corrections for a project
+   */
+  async getCorrections(
+    project?: string,
+    options?: { limit?: number }
+  ): Promise<any[]> {
+    if (!this.useChromaDB) {
+      throw new Error('Correction retrieval requires ChromaDB backend')
+    }
+    if (project) {
+      return this.chroma.store.getCorrectionsByProject(project, options?.limit || 10)
+    }
+    // If no project specified, search all corrections
+    return this.chroma.store.searchCorrections('', { limit: options?.limit || 10 })
+  }
+
+  /**
+   * Search patterns by query
+   */
+  async searchPatterns(
+    query: string,
+    options?: {
+      project?: string
+      pattern_type?: 'solution' | 'anti-pattern' | 'best-practice' | 'common-issue'
+      limit?: number
+      minSimilarity?: number
+    }
+  ): Promise<any[]> {
+    if (!this.useChromaDB) {
+      throw new Error('Pattern search requires ChromaDB backend')
+    }
+    return this.chroma.store.searchPatterns(query, options)
+  }
+
+  /**
+   * Search corrections by query
+   */
+  async searchCorrections(
+    query: string,
+    options?: {
+      project?: string
+      limit?: number
+      minSimilarity?: number
+    }
+  ): Promise<any[]> {
+    if (!this.useChromaDB) {
+      throw new Error('Correction search requires ChromaDB backend')
+    }
+    return this.chroma.store.searchCorrections(query, options)
+  }
+}
+
+/**
+ * Create a memory manager instance
+ */
+export function createMemoryManager(dbPath: string, logger: Logger): MemoryManager {
+  return new MemoryManager(dbPath, logger)
+}