claude-brain 0.15.2 → 0.16.0

package/src/intelligence/reasoning/chain-retrieval.ts

@@ -1,247 +1,247 @@
-/**
- * Chain Retrieval
- * Iterative search with query refinement for multi-hop reasoning
- */
-
-import type { Logger } from 'pino'
-import type { CollectionManager } from '@/memory/chroma/collection-manager'
-import type { EmbeddingProvider } from '@/memory/chroma/embeddings'
-
-export interface ChainStep {
-  query: string
-  results: ChainResult[]
-  refinedQuery?: string
-}
-
-export interface ChainResult {
-  id: string
-  content: string
-  metadata: Record<string, any>
-  similarity: number
-}
-
-export interface ChainRetrievalResult {
-  query: string
-  hops: number
-  steps: ChainStep[]
-  allResults: ChainResult[]
-  synthesizedAnswer: string
-}
-
-export class ChainRetrieval {
-  private logger: Logger
-  private collections: CollectionManager
-  private embeddings?: EmbeddingProvider
-
-  constructor(logger: Logger, collections: CollectionManager, embeddings?: EmbeddingProvider) {
-    this.logger = logger.child({ component: 'chain-retrieval' })
-    this.collections = collections
-    this.embeddings = embeddings
-  }
-
-  /**
-   * Perform iterative multi-hop retrieval
-   * Each hop refines the query based on previous results
-   */
-  async retrieve(query: string, options: {
-    maxHops?: number
-    resultsPerHop?: number
-    minSimilarity?: number
-    project?: string
-  } = {}): Promise<ChainRetrievalResult> {
-    const {
-      maxHops = 3,
-      resultsPerHop = 5,
-      minSimilarity = 0.3,
-      project
-    } = options
-
-    const steps: ChainStep[] = []
-    const seenIds = new Set<string>()
-    let currentQuery = query
-
-    for (let hop = 0; hop < maxHops; hop++) {
-      // Search with current query
-      const results = await this.search(currentQuery, project, resultsPerHop, minSimilarity)
-
-      // Filter out already-seen results
-      const newResults = results.filter(r => !seenIds.has(r.id))
-      newResults.forEach(r => seenIds.add(r.id))
-
-      // If no new results, stop
-      if (newResults.length === 0 && hop > 0) break
-
-      // Refine query based on results
-      const refinedQuery = this.refineQuery(currentQuery, newResults)
-
-      steps.push({
-        query: currentQuery,
-        results: newResults,
-        refinedQuery: refinedQuery !== currentQuery ? refinedQuery : undefined
-      })
-
-      // If query didn't change, we've converged
-      if (refinedQuery === currentQuery && hop > 0) break
-
-      currentQuery = refinedQuery
-    }
-
-    // Collect all unique results
-    const allResults = steps.flatMap(s => s.results)
-
-    return {
-      query,
-      hops: steps.length,
-      steps,
-      allResults,
-      synthesizedAnswer: this.synthesize(query, allResults)
-    }
-  }
-
-  private async search(
-    query: string,
-    project?: string,
-    limit: number = 5,
-    minSimilarity: number = 0.3
-  ): Promise<ChainResult[]> {
-    try {
-      const collection = await this.collections.getMemories()
-
-      const where: any = project ? { project: { $eq: project } } : undefined
-
-      let results: any
-
-      if (this.embeddings) {
-        const embedding = await this.embeddings.generate(query)
-        results = await collection.query({
-          queryEmbeddings: [embedding],
-          nResults: limit,
-          where,
-          include: ['documents', 'metadatas', 'distances']
-        })
-      } else {
-        results = await collection.query({
-          queryTexts: [query],
-          nResults: limit,
-          where,
-          include: ['documents', 'metadatas', 'distances']
-        })
-      }
-
-      if (!results.ids || !results.ids[0]) return []
-
-      const processed: ChainResult[] = []
-      const ids = results.ids[0]
-      const documents = results.documents?.[0] || []
-      const metadatas = results.metadatas?.[0] || []
-      const distances = results.distances?.[0] || []
-
-      for (let i = 0; i < ids.length; i++) {
-        const similarity = 1 - (distances[i] || 0)
-        if (similarity < minSimilarity) continue
-
-        processed.push({
-          id: ids[i],
-          content: documents[i] || '',
-          metadata: metadatas[i] || {},
-          similarity
-        })
-      }
-
-      return processed
-    } catch (error) {
-      this.logger.warn({ error, query }, 'Chain retrieval search failed')
-      return []
-    }
-  }
-
-  /**
-   * Refine query based on retrieved results
-   * Extracts key terms from results to expand the query
-   */
-  private refineQuery(originalQuery: string, results: ChainResult[]): string {
-    if (results.length === 0) return originalQuery
-
-    // Extract key terms from top results
-    const keyTerms = new Set<string>()
-    const queryWords = new Set(originalQuery.toLowerCase().split(/\s+/))
-
-    for (const result of results.slice(0, 3)) {
-      const words = result.content.toLowerCase().split(/\s+/)
-
-      // Extract distinctive words not in the original query
-      for (const word of words) {
-        if (
-          word.length > 4 &&
-          !queryWords.has(word) &&
-          !this.isStopWord(word)
-        ) {
-          keyTerms.add(word)
-        }
-      }
-
-      // Also add relevant metadata terms
-      if (result.metadata.tags) {
-        const tags = String(result.metadata.tags).split(',')
-        for (const tag of tags) {
-          if (tag.trim() && !queryWords.has(tag.trim().toLowerCase())) {
-            keyTerms.add(tag.trim().toLowerCase())
-          }
-        }
-      }
-    }
-
-    // Add top distinctive terms to query
-    const topTerms = Array.from(keyTerms).slice(0, 3)
-    if (topTerms.length === 0) return originalQuery
-
-    return `${originalQuery} ${topTerms.join(' ')}`
-  }
-
-  /**
-   * Synthesize results into a coherent answer
-   */
-  private synthesize(query: string, results: ChainResult[]): string {
-    if (results.length === 0) {
-      return 'No relevant information found across multiple retrieval hops.'
-    }
-
-    // Group by type/source
-    const byType: Record<string, ChainResult[]> = {}
-    for (const r of results) {
-      const type = r.metadata.type || r.metadata.source || 'general'
-      if (!byType[type]) byType[type] = []
-      byType[type].push(r)
-    }
-
-    const parts: string[] = [`Multi-hop retrieval for: "${query}"`, '']
-
-    for (const [type, items] of Object.entries(byType)) {
-      parts.push(`### ${type.charAt(0).toUpperCase() + type.slice(1)} (${items.length})`)
-      for (const item of items.slice(0, 5)) {
-        const similarity = Math.round(item.similarity * 100)
-        parts.push(`- [${similarity}%] ${item.content.slice(0, 150)}${item.content.length > 150 ? '...' : ''}`)
-      }
-      parts.push('')
-    }
-
-    return parts.join('\n')
-  }
-
-  private isStopWord(word: string): boolean {
-    const stopWords = new Set([
-      'the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can',
-      'her', 'was', 'one', 'our', 'out', 'has', 'had', 'been', 'have',
-      'with', 'this', 'that', 'from', 'they', 'will', 'would', 'there',
-      'their', 'what', 'about', 'which', 'when', 'make', 'like', 'time',
-      'just', 'know', 'take', 'people', 'into', 'year', 'your', 'good',
-      'some', 'could', 'them', 'than', 'other', 'then', 'look', 'only',
-      'come', 'over', 'think', 'also', 'back', 'after', 'work', 'first',
-      'well', 'even', 'give', 'most', 'find', 'here', 'thing', 'many',
-      'still', 'should', 'because', 'does', 'each', 'much', 'before',
-      'between', 'must', 'through', 'being', 'using', 'used', 'decision',
-      'decided', 'recommend', 'instead', 'project', 'context', 'reasoning'
-    ])
-    return stopWords.has(word)
-  }
-}
+/**
+ * Chain Retrieval
+ * Iterative search with query refinement for multi-hop reasoning
+ */
+
+import type { Logger } from 'pino'
+import type { CollectionManager } from '@/memory/chroma/collection-manager'
+import type { EmbeddingProvider } from '@/memory/chroma/embeddings'
+
+export interface ChainStep {
+  query: string
+  results: ChainResult[]
+  refinedQuery?: string
+}
+
+export interface ChainResult {
+  id: string
+  content: string
+  metadata: Record<string, any>
+  similarity: number
+}
+
+export interface ChainRetrievalResult {
+  query: string
+  hops: number
+  steps: ChainStep[]
+  allResults: ChainResult[]
+  synthesizedAnswer: string
+}
+
+export class ChainRetrieval {
+  private logger: Logger
+  private collections: CollectionManager
+  private embeddings?: EmbeddingProvider
+
+  constructor(logger: Logger, collections: CollectionManager, embeddings?: EmbeddingProvider) {
+    this.logger = logger.child({ component: 'chain-retrieval' })
+    this.collections = collections
+    this.embeddings = embeddings
+  }
+
+  /**
+   * Perform iterative multi-hop retrieval
+   * Each hop refines the query based on previous results
+   */
+  async retrieve(query: string, options: {
+    maxHops?: number
+    resultsPerHop?: number
+    minSimilarity?: number
+    project?: string
+  } = {}): Promise<ChainRetrievalResult> {
+    const {
+      maxHops = 3,
+      resultsPerHop = 5,
+      minSimilarity = 0.3,
+      project
+    } = options
+
+    const steps: ChainStep[] = []
+    const seenIds = new Set<string>()
+    let currentQuery = query
+
+    for (let hop = 0; hop < maxHops; hop++) {
+      // Search with current query
+      const results = await this.search(currentQuery, project, resultsPerHop, minSimilarity)
+
+      // Filter out already-seen results
+      const newResults = results.filter(r => !seenIds.has(r.id))
+      newResults.forEach(r => seenIds.add(r.id))
+
+      // If no new results, stop
+      if (newResults.length === 0 && hop > 0) break
+
+      // Refine query based on results
+      const refinedQuery = this.refineQuery(currentQuery, newResults)
+
+      steps.push({
+        query: currentQuery,
+        results: newResults,
+        refinedQuery: refinedQuery !== currentQuery ? refinedQuery : undefined
+      })
+
+      // If query didn't change, we've converged
+      if (refinedQuery === currentQuery && hop > 0) break
+
+      currentQuery = refinedQuery
+    }
+
+    // Collect all unique results
+    const allResults = steps.flatMap(s => s.results)
+
+    return {
+      query,
+      hops: steps.length,
+      steps,
+      allResults,
+      synthesizedAnswer: this.synthesize(query, allResults)
+    }
+  }
+
+  private async search(
+    query: string,
+    project?: string,
+    limit: number = 5,
+    minSimilarity: number = 0.3
+  ): Promise<ChainResult[]> {
+    try {
+      const collection = await this.collections.getMemories()
+
+      const where: any = project ? { project: { $eq: project } } : undefined
+
+      let results: any
+
+      if (this.embeddings) {
+        const embedding = await this.embeddings.generate(query)
+        results = await collection.query({
+          queryEmbeddings: [embedding],
+          nResults: limit,
+          where,
+          include: ['documents', 'metadatas', 'distances']
+        })
+      } else {
+        results = await collection.query({
+          queryTexts: [query],
+          nResults: limit,
+          where,
+          include: ['documents', 'metadatas', 'distances']
+        })
+      }
+
+      if (!results.ids || !results.ids[0]) return []
+
+      const processed: ChainResult[] = []
+      const ids = results.ids[0]
+      const documents = results.documents?.[0] || []
+      const metadatas = results.metadatas?.[0] || []
+      const distances = results.distances?.[0] || []
+
+      for (let i = 0; i < ids.length; i++) {
+        const similarity = 1 - (distances[i] || 0)
+        if (similarity < minSimilarity) continue
+
+        processed.push({
+          id: ids[i],
+          content: documents[i] || '',
+          metadata: metadatas[i] || {},
+          similarity
+        })
+      }
+
+      return processed
+    } catch (error) {
+      this.logger.warn({ error, query }, 'Chain retrieval search failed')
+      return []
+    }
+  }
+
+  /**
+   * Refine query based on retrieved results
+   * Extracts key terms from results to expand the query
+   */
+  private refineQuery(originalQuery: string, results: ChainResult[]): string {
+    if (results.length === 0) return originalQuery
+
+    // Extract key terms from top results
+    const keyTerms = new Set<string>()
+    const queryWords = new Set(originalQuery.toLowerCase().split(/\s+/))
+
+    for (const result of results.slice(0, 3)) {
+      const words = result.content.toLowerCase().split(/\s+/)
+
+      // Extract distinctive words not in the original query
+      for (const word of words) {
+        if (
+          word.length > 4 &&
+          !queryWords.has(word) &&
+          !this.isStopWord(word)
+        ) {
+          keyTerms.add(word)
+        }
+      }
+
+      // Also add relevant metadata terms
+      if (result.metadata.tags) {
+        const tags = String(result.metadata.tags).split(',')
+        for (const tag of tags) {
+          if (tag.trim() && !queryWords.has(tag.trim().toLowerCase())) {
+            keyTerms.add(tag.trim().toLowerCase())
+          }
+        }
+      }
+    }
+
+    // Add top distinctive terms to query
+    const topTerms = Array.from(keyTerms).slice(0, 3)
+    if (topTerms.length === 0) return originalQuery
+
+    return `${originalQuery} ${topTerms.join(' ')}`
+  }
+
+  /**
+   * Synthesize results into a coherent answer
+   */
+  private synthesize(query: string, results: ChainResult[]): string {
+    if (results.length === 0) {
+      return 'No relevant information found across multiple retrieval hops.'
+    }
+
+    // Group by type/source
+    const byType: Record<string, ChainResult[]> = {}
+    for (const r of results) {
+      const type = r.metadata.type || r.metadata.source || 'general'
+      if (!byType[type]) byType[type] = []
+      byType[type].push(r)
+    }
+
+    const parts: string[] = [`Multi-hop retrieval for: "${query}"`, '']
+
+    for (const [type, items] of Object.entries(byType)) {
+      parts.push(`### ${type.charAt(0).toUpperCase() + type.slice(1)} (${items.length})`)
+      for (const item of items.slice(0, 5)) {
+        const similarity = Math.round(item.similarity * 100)
+        parts.push(`- [${similarity}%] ${item.content.slice(0, 150)}${item.content.length > 150 ? '...' : ''}`)
+      }
+      parts.push('')
+    }
+
+    return parts.join('\n')
+  }
+
+  private isStopWord(word: string): boolean {
+    const stopWords = new Set([
+      'the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can',
+      'her', 'was', 'one', 'our', 'out', 'has', 'had', 'been', 'have',
+      'with', 'this', 'that', 'from', 'they', 'will', 'would', 'there',
+      'their', 'what', 'about', 'which', 'when', 'make', 'like', 'time',
+      'just', 'know', 'take', 'people', 'into', 'year', 'your', 'good',
+      'some', 'could', 'them', 'than', 'other', 'then', 'look', 'only',
+      'come', 'over', 'think', 'also', 'back', 'after', 'work', 'first',
+      'well', 'even', 'give', 'most', 'find', 'here', 'thing', 'many',
+      'still', 'should', 'because', 'does', 'each', 'much', 'before',
+      'between', 'must', 'through', 'being', 'using', 'used', 'decision',
+      'decided', 'recommend', 'instead', 'project', 'context', 'reasoning'
+    ])
+    return stopWords.has(word)
+  }
+}