claude-brain 0.9.2 → 0.10.0

package/src/routing/search-engine.ts

@@ -0,0 +1,455 @@
+/**
+ * Search Engine
+ * Phase 19: Wires all advanced intelligence features into a unified search path.
+ *
+ * This is the core intelligence layer between the router and raw memory.
+ * Provides: hybrid search, semantic caching, temporal filtering, timeouts.
+ */
+
+import type { Logger } from 'pino'
+import type { NormalizedResult } from './types'
+import { normalizeSearchResults, normalizePatternResults, normalizeCorrectionResults } from './types'
+import {
+  getMemoryService,
+  getSemanticCacheService,
+  getRetrievalPipeline,
+  getKnowledgeGraphService,
+  getEpisodeService,
+  isServicesInitialized
+} from '@/server/services'
+
+/**
+ * Wrap a promise with a timeout. Returns fallback if the promise doesn't resolve in time.
+ */
+function withTimeout<T>(promise: Promise<T>, ms: number, fallback: T): Promise<T> {
+  return Promise.race([
+    promise,
+    new Promise<T>(resolve => setTimeout(() => resolve(fallback), ms))
+  ])
+}
+
+const SEARCH_TIMEOUT = 5000 // 5s timeout for all searches
+
+export class SearchEngine {
+  private logger: Logger
+
+  constructor(logger: Logger) {
+    this.logger = logger.child({ component: 'search-engine' })
+  }
+
+  /**
+   * Enhanced search — uses hybrid retrieval pipeline when available,
+   * falls back to plain searchRaw().
+   * Wraps with semantic cache for repeated queries.
+   */
+  async enhancedSearch(
+    query: string,
+    options?: { project?: string; limit?: number; minSimilarity?: number }
+  ): Promise<NormalizedResult[]> {
+    if (!isServicesInitialized()) return []
+
+    const cacheKey = `search:${query}:${options?.project || ''}:${options?.limit || 5}`
+
+    // Check semantic cache first
+    const cache = getSemanticCacheService()
+    if (cache) {
+      try {
+        const cached = await cache.get(cacheKey)
+        if (cached) {
+          this.logger.debug({ query }, 'Search cache hit')
+          return cached as NormalizedResult[]
+        }
+      } catch {
+        // Cache miss or error, continue
+      }
+    }
+
+    // Try hybrid search via RetrievalPipeline
+    const pipeline = getRetrievalPipeline()
+    if (pipeline) {
+      try {
+        const hybridResults = await withTimeout(
+          pipeline.search(query, {
+            project: options?.project,
+            limit: options?.limit || 5,
+            minSimilarity: options?.minSimilarity || 0.3
+          }),
+          SEARCH_TIMEOUT,
+          []
+        )
+
+        if (hybridResults.length > 0) {
+          const normalized = hybridResults.map((r: any) => ({
+            id: r.id || '',
+            content: r.content || r.document || '',
+            score: r.score || r.similarity || 0,
+            source: 'decision' as const,
+            project: r.metadata?.project || options?.project || '',
+            date: r.metadata?.created_at || '',
+            metadata: r.metadata || {}
+          }))
+
+          // Cache results
+          if (cache) {
+            try { await cache.set(cacheKey, normalized) } catch {}
+          }
+          return normalized
+        }
+      } catch (error) {
+        this.logger.debug({ error }, 'Hybrid search failed, falling back to plain search')
+      }
+    }
+
+    // Fallback: plain searchRaw
+    return this.plainSearch(query, options, cache, cacheKey)
+  }
+
+  /**
+   * Plain search using memory.searchRaw() — always available.
+   */
+  async plainSearch(
+    query: string,
+    options?: { project?: string; limit?: number; minSimilarity?: number },
+    cache?: any,
+    cacheKey?: string
+  ): Promise<NormalizedResult[]> {
+    if (!isServicesInitialized()) return []
+
+    const memory = getMemoryService()
+    try {
+      const rawResults = await withTimeout(
+        memory.searchRaw(query, {
+          project: options?.project,
+          limit: options?.limit || 5,
+          minSimilarity: options?.minSimilarity || 0.3
+        }),
+        SEARCH_TIMEOUT,
+        []
+      )
+      const normalized = normalizeSearchResults(rawResults)
+
+      // Cache results
+      if (cache && cacheKey) {
+        try { await cache.set(cacheKey, normalized) } catch {}
+      }
+
+      return normalized
+    } catch {
+      return []
+    }
+  }
+
+  /**
+   * Search patterns with normalization and timeout
+   */
+  async searchPatterns(
+    query: string,
+    options?: { project?: string; limit?: number; minSimilarity?: number }
+  ): Promise<NormalizedResult[]> {
+    if (!isServicesInitialized()) return []
+    const memory = getMemoryService()
+    try {
+      const results = await withTimeout(
+        memory.searchPatterns(query, {
+          project: options?.project,
+          limit: options?.limit || 3,
+          minSimilarity: options?.minSimilarity || 0.3
+        }),
+        SEARCH_TIMEOUT,
+        []
+      )
+      return normalizePatternResults(results)
+    } catch {
+      return []
+    }
+  }
+
+  /**
+   * Search corrections with normalization and timeout
+   */
+  async searchCorrections(
+    query: string,
+    options?: { project?: string; limit?: number; minSimilarity?: number }
+  ): Promise<NormalizedResult[]> {
+    if (!isServicesInitialized()) return []
+    const memory = getMemoryService()
+    try {
+      const results = await withTimeout(
+        memory.searchCorrections(query, {
+          project: options?.project,
+          limit: options?.limit || 3,
+          minSimilarity: options?.minSimilarity || 0.3
+        }),
+        SEARCH_TIMEOUT,
+        []
+      )
+      return normalizeCorrectionResults(results)
+    } catch {
+      return []
+    }
+  }
+
+  /**
+   * Search knowledge graph for related concepts
+   */
+  async searchGraph(query: string, limit: number = 5): Promise<NormalizedResult[]> {
+    try {
+      const kgService = getKnowledgeGraphService()
+      if (!kgService?.search) return []
+
+      const graphResults = kgService.search.search({ query, limit })
+      if (!graphResults?.nodes?.length) return []
+
+      return graphResults.nodes.map((n: any) => ({
+        id: n.id || '',
+        content: `**${n.label || n.name}** (${n.type})${n.metadata?.description ? `\n${n.metadata.description}` : ''}`,
+        score: n.score || 0.5,
+        source: 'graph' as const,
+        project: n.metadata?.project || '',
+        date: '',
+        metadata: n.metadata || {}
+      }))
+    } catch {
+      return []
+    }
+  }
+
+  /**
+   * Temporal search — parse temporal phrases from query, filter results by date range.
+   * Uses TemporalQueryProcessor when available.
+   */
+  async temporalSearch(
+    query: string,
+    options?: { project?: string; limit?: number }
+  ): Promise<{ results: NormalizedResult[]; cleanedQuery: string }> {
+    if (!isServicesInitialized()) return { results: [], cleanedQuery: query }
+
+    try {
+      // Dynamic import to avoid hard dependency
+      const { TemporalQueryProcessor } = await import('@/temporal/query-processor')
+      const processor = new TemporalQueryProcessor(this.logger)
+      const parsed = processor.parse(query)
+
+      if (parsed.intent === 'none') {
+        // No temporal expressions found, use normal search
+        const results = await this.enhancedSearch(query, options)
+        return { results, cleanedQuery: query }
+      }
+
+      // Search with cleaned query (temporal phrases removed)
+      const results = await this.enhancedSearch(parsed.cleaned || query, {
+        ...options,
+        limit: (options?.limit || 5) * 2 // Fetch more since we'll filter by date
+      })
+
+      // Filter results by date range
+      const filtered = results.filter(r => {
+        if (!r.date) return true // Keep results without dates
+        const date = new Date(r.date)
+        if (isNaN(date.getTime())) return true
+        if (parsed.startDate && date < parsed.startDate) return false
+        if (parsed.endDate && date > parsed.endDate) return false
+        return true
+      })
+
+      return {
+        results: filtered.slice(0, options?.limit || 5),
+        cleanedQuery: parsed.cleaned || query
+      }
+    } catch {
+      // TemporalQueryProcessor not available, fall back
+      const results = await this.enhancedSearch(query, options)
+      return { results, cleanedQuery: query }
+    }
+  }
+
+  /**
+   * Build a timeline for a topic/project
+   */
+  async buildTimeline(
+    options?: { project?: string; topic?: string; limit?: number }
+  ): Promise<any | null> {
+    if (!isServicesInitialized()) return null
+
+    try {
+      const memory = getMemoryService()
+      if (!memory.isChromaDBEnabled()) return null
+
+      const { TimelineBuilder } = await import('@/temporal/timeline')
+      const builder = new TimelineBuilder(
+        this.logger,
+        memory.chroma.collections,
+        memory.chroma.embeddings
+      )
+      return await withTimeout(
+        builder.buildTimeline(options),
+        SEARCH_TIMEOUT,
+        null
+      )
+    } catch {
+      return null
+    }
+  }
+
+  /**
+   * Analyze decision evolution for a topic
+   */
+  async analyzeEvolution(
+    topic: string,
+    options?: { project?: string; limit?: number }
+  ): Promise<any | null> {
+    if (!isServicesInitialized()) return null
+
+    try {
+      const memory = getMemoryService()
+      if (!memory.isChromaDBEnabled()) return null
+
+      const { DecisionEvolutionTracker } = await import('@/temporal/evolution')
+      const tracker = new DecisionEvolutionTracker(
+        this.logger,
+        memory.chroma.collections,
+        memory.chroma.embeddings
+      )
+      return await withTimeout(
+        tracker.analyzeEvolution(topic, options),
+        SEARCH_TIMEOUT,
+        null
+      )
+    } catch {
+      return null
+    }
+  }
+
+  /**
+   * Detect trends
+   */
+  async detectTrends(
+    options?: { project?: string; periodDays?: number; limit?: number }
+  ): Promise<any | null> {
+    if (!isServicesInitialized()) return null
+
+    try {
+      const memory = getMemoryService()
+      if (!memory.isChromaDBEnabled()) return null
+
+      const { TrendDetector } = await import('@/temporal/trends')
+      const detector = new TrendDetector(this.logger, memory.chroma.collections)
+      return await withTimeout(
+        detector.detectTrends(options),
+        SEARCH_TIMEOUT,
+        null
+      )
+    } catch {
+      return null
+    }
+  }
+
+  /**
+   * Multi-hop chain retrieval for complex questions
+   */
+  async chainSearch(
+    query: string,
+    options?: { project?: string; maxHops?: number }
+  ): Promise<any | null> {
+    if (!isServicesInitialized()) return null
+
+    try {
+      const memory = getMemoryService()
+      if (!memory.isChromaDBEnabled()) return null
+
+      const { ChainRetrieval } = await import('@/reasoning/chain-retrieval')
+      const chain = new ChainRetrieval(
+        this.logger,
+        memory.chroma.collections,
+        memory.chroma.embeddings
+      )
+      return await withTimeout(
+        chain.retrieve(query, {
+          maxHops: options?.maxHops || 3,
+          project: options?.project
+        }),
+        SEARCH_TIMEOUT,
+        null
+      )
+    } catch {
+      return null
+    }
+  }
+
+  /**
+   * Get recommendations based on context
+   */
+  async getRecommendations(
+    query: string,
+    options?: { project?: string; limit?: number }
+  ): Promise<any | null> {
+    if (!isServicesInitialized()) return null
+
+    try {
+      const memory = getMemoryService()
+      if (!memory.isChromaDBEnabled()) return null
+
+      const { Recommender } = await import('@/prediction/recommender')
+      const recommender = new Recommender(
+        this.logger,
+        memory.chroma.collections,
+        memory.chroma.embeddings
+      )
+      return await withTimeout(
+        recommender.getRecommendations(query, {
+          project: options?.project,
+          limit: options?.limit || 5
+        }),
+        SEARCH_TIMEOUT,
+        null
+      )
+    } catch {
+      return null
+    }
+  }
+
+  /**
+   * Find cross-project patterns
+   */
+  async findCrossProjectPatterns(
+    options?: { query?: string; minProjects?: number; limit?: number }
+  ): Promise<any | null> {
+    if (!isServicesInitialized()) return null
+
+    try {
+      const memory = getMemoryService()
+      if (!memory.isChromaDBEnabled()) return null
+
+      const { PatternGeneralizer } = await import('@/cross-project/generalizer')
+      const generalizer = new PatternGeneralizer(
+        this.logger,
+        memory.chroma.collections,
+        memory.chroma.embeddings
+      )
+      return await withTimeout(
+        generalizer.findCrossProjectPatterns(options),
+        SEARCH_TIMEOUT,
+        null
+      )
+    } catch {
+      return null
+    }
+  }
+
+  /**
+   * Invalidate cache entries for a project after store/update/delete
+   */
+  invalidateCache(project?: string): void {
+    try {
+      const cache = getSemanticCacheService()
+      if (!cache) return
+      if (project) {
+        cache.invalidateProject(project)
+      } else {
+        cache.clear()
+      }
+    } catch {
+      // Cache not available
+    }
+  }
+}

package/src/routing/types.ts

@@ -0,0 +1,84 @@
+/**
+ * Routing Types
+ * Phase 19: Normalized result types for the brain() router
+ *
+ * Eliminates scattered `r.decision?.decision || r.content?.slice(0, 300)`
+ * by normalizing all search results to a flat structure.
+ */
+
+export interface NormalizedResult {
+  id: string
+  content: string
+  score: number
+  source: 'decision' | 'pattern' | 'correction' | 'memory' | 'graph' | 'episode' | 'cross-project'
+  project?: string
+  date?: string
+  metadata?: Record<string, unknown>
+}
+
+/**
+ * Normalize raw searchRaw() results into flat NormalizedResult[]
+ * Handles the nested { memory: { id }, decision: { id } } structure
+ */
+export function normalizeSearchResults(rawResults: any[]): NormalizedResult[] {
+  return rawResults.map(r => {
+    const id = r.memory?.id || r.decision?.id || r.id || ''
+    const project = r.memory?.project || r.decision?.project || r.metadata?.project || ''
+    const date = r.memory?.createdAt || r.decision?.createdAt || r.metadata?.created_at || ''
+
+    // Prefer decision content, fall back to memory content
+    let content: string
+    if (r.decision?.decision) {
+      content = r.decision.decision
+      if (r.decision.reasoning) {
+        content = `**${content}**\n${r.decision.reasoning}`
+      }
+    } else if (r.memory?.content) {
+      content = r.memory.content
+    } else if (r.content) {
+      content = typeof r.content === 'string' ? r.content : JSON.stringify(r.content)
+    } else {
+      content = ''
+    }
+
+    return {
+      id,
+      content,
+      score: r.similarity || r.score || 0,
+      source: r.decision ? 'decision' as const : 'memory' as const,
+      project: typeof project === 'string' ? project : String(project),
+      date: date instanceof Date ? date.toISOString() : typeof date === 'string' ? date : '',
+      metadata: r.metadata || r.decision || r.memory?.metadata || {}
+    }
+  })
+}
+
+/**
+ * Normalize pattern search results
+ */
+export function normalizePatternResults(results: any[]): NormalizedResult[] {
+  return results.map(p => ({
+    id: p.id || '',
+    content: p.metadata?.description || p.description || p.content || '',
+    score: p.similarity || p.score || 0,
+    source: 'pattern' as const,
+    project: p.metadata?.project || p.project || '',
+    date: p.metadata?.created_at || '',
+    metadata: p.metadata || {}
+  }))
+}
+
+/**
+ * Normalize correction search results
+ */
+export function normalizeCorrectionResults(results: any[]): NormalizedResult[] {
+  return results.map(c => ({
+    id: c.id || '',
+    content: `Original: ${c.metadata?.original || c.original || ''}\nFix: ${c.metadata?.correction || c.correction || c.content || ''}`,
+    score: c.similarity || c.score || 0,
+    source: 'correction' as const,
+    project: c.metadata?.project || c.project || '',
+    date: c.metadata?.created_at || '',
+    metadata: c.metadata || {}
+  }))
+}

package/src/server/handlers/call-tool.ts

@@ -31,23 +31,11 @@ import { handleGetCorrections } from './tools/get-corrections'
 import { handleCreateProject } from './tools/create-project'
 import { handleInitProject } from './tools/init-project'
 
-// Phase 13 Retrieval handlers
-import { handleRateMemory } from './tools/rate-memory'
-
-// Phase 14 Knowledge Graph & Episodic Memory handlers
-import { handleSearchKnowledgeGraph } from './tools/search-knowledge-graph'
-import { handleGetEpisode } from './tools/get-episode'
-import { handleListEpisodes } from './tools/list-episodes'
-
-// Phase 15 Advanced Intelligence & Temporal Reasoning handlers
-import { handleGetDecisionTimeline } from './tools/get-decision-timeline'
-import { handleAnalyzeDecisionEvolution } from './tools/analyze-decision-evolution'
-import { handleDetectTrends } from './tools/detect-trends'
-import { handleWhatIfAnalysis } from './tools/what-if-analysis'
-import { handleGetRecommendations } from './tools/get-recommendations'
-import { handleFindCrossProjectPatterns } from './tools/find-cross-project-patterns'
-
 // Phase 16 Unified Brain Tool
+// Phase 19: Removed 9 redundant tool handlers (rate_memory, search_knowledge_graph,
+// get_episode, list_episodes, get_decision_timeline, analyze_decision_evolution,
+// detect_trends, what_if_analysis, get_recommendations, find_cross_project_patterns).
+// Their functionality is now absorbed into brain().
 import { handleBrain } from './tools/brain'
 
 /**
@@ -134,39 +122,6 @@ export async function handleCallTool(
       case 'get_corrections':
         return await handleGetCorrections(args, logger)
 
-      // Phase 13 Retrieval Tools
-      case 'rate_memory':
-        return await handleRateMemory(args, logger)
-
-      // Phase 14 Knowledge Graph & Episodic Memory Tools
-      case 'search_knowledge_graph':
-        return await handleSearchKnowledgeGraph(args, logger)
-
-      case 'get_episode':
-        return await handleGetEpisode(args, logger)
-
-      case 'list_episodes':
-        return await handleListEpisodes(args, logger)
-
-      // Phase 15 Advanced Intelligence & Temporal Reasoning Tools
-      case 'get_decision_timeline':
-        return await handleGetDecisionTimeline(args, logger)
-
-      case 'analyze_decision_evolution':
-        return await handleAnalyzeDecisionEvolution(args, logger)
-
-      case 'detect_trends':
-        return await handleDetectTrends(args, logger)
-
-      case 'what_if_analysis':
-        return await handleWhatIfAnalysis(args, logger)
-
-      case 'get_recommendations':
-        return await handleGetRecommendations(args, logger)
-
-      case 'find_cross_project_patterns':
-        return await handleFindCrossProjectPatterns(args, logger)
-
       // Phase 16 Unified Brain Tool
       case 'brain':
         return await handleBrain(args, logger)

package/src/server/handlers/tools/index.ts

@@ -24,10 +24,8 @@ export { handleGetCorrections } from './get-corrections'
 export { handleCreateProject } from './create-project'
 export { handleInitProject } from './init-project'
 
-// Phase 15 - Advanced Intelligence & Temporal Reasoning
-export { handleGetDecisionTimeline } from './get-decision-timeline'
-export { handleAnalyzeDecisionEvolution } from './analyze-decision-evolution'
-export { handleDetectTrends } from './detect-trends'
-export { handleWhatIfAnalysis } from './what-if-analysis'
-export { handleGetRecommendations } from './get-recommendations'
-export { handleFindCrossProjectPatterns } from './find-cross-project-patterns'
+// Phase 19: Removed 9 redundant tool exports.
+// rate_memory, search_knowledge_graph, get_episode, list_episodes,
+// get_decision_timeline, analyze_decision_evolution, detect_trends,
+// what_if_analysis, get_recommendations, find_cross_project_patterns
+// are now absorbed into brain(). Handler files kept for reference.

package/src/server/services.ts

@@ -11,6 +11,7 @@ import { VaultManager } from '@/vault'
 import { ContextManager } from '@/context'
 import { Phase12Manager } from '@/phase12'
 import { RetrievalService } from '@/retrieval/service'
+import { RetrievalPipeline } from '@/retrieval/pipeline'
 import { InMemoryKnowledgeGraph } from '@/knowledge/graph/memory-graph'
 import { GraphSearchEngine } from '@/knowledge/graph/search'
 import { KnowledgeGraphBuilder } from '@/knowledge/graph/builder'
@@ -36,6 +37,7 @@ export interface Services {
   context: ContextManager
   phase12: Phase12Manager
   retrieval: RetrievalService | null
+  retrievalPipeline: RetrievalPipeline | null
   knowledgeGraph: KnowledgeGraphServiceContainer | null
   episodeManager: EpisodeManager | null
   semanticCache: SemanticCache | null
@@ -127,6 +129,23 @@ export async function initializeServices(config: Config, logger: Logger): Promis
       serviceLogger.warn('Retrieval service requires ChromaDB, skipping initialization')
     }
 
+    // Initialize Retrieval Pipeline (Phase 19) — hybrid search with BM25 + semantic + fusion
+    let retrievalPipeline: RetrievalPipeline | null = null
+    if (config.retrieval?.enabled && memory.isChromaDBEnabled()) {
+      try {
+        retrievalPipeline = new RetrievalPipeline(
+          logger,
+          memory.chroma.collections,
+          memory.chroma.embeddings,
+          config.retrieval
+        )
+        await retrievalPipeline.initialize()
+        serviceLogger.info('Retrieval pipeline initialized (hybrid search)')
+      } catch (error) {
+        serviceLogger.warn({ error }, 'Failed to initialize retrieval pipeline, continuing without hybrid search')
+      }
+    }
+
     // Initialize Knowledge Graph Service (Phase 14)
     let knowledgeGraph: KnowledgeGraphServiceContainer | null = null
     if (config.knowledge?.graph?.enabled !== false) {
@@ -221,6 +240,7 @@ export async function initializeServices(config: Config, logger: Logger): Promis
       context,
       phase12,
       retrieval,
+      retrievalPipeline,
       knowledgeGraph,
       episodeManager,
       semanticCache,
@@ -283,6 +303,14 @@ export function getRetrievalService(): RetrievalService | null {
   return getServices().retrieval
 }
 
+/**
+ * Get Retrieval Pipeline (Phase 19)
+ * Returns null if hybrid search is not enabled
+ */
+export function getRetrievalPipeline(): RetrievalPipeline | null {
+  return getServices().retrievalPipeline
+}
+
 /**
  * Get Knowledge Graph service
  * Returns null if knowledge graph is not enabled