claude-brain 0.5.1 → 0.8.0

package/src/routing/response-filter.ts

@@ -0,0 +1,221 @@
+/**
+ * Brain Response Filter
+ * Phase 16: Filters noise, deduplicates, ranks, and synthesizes results
+ * from the unified brain() tool
+ */
+
+export interface FilterableResult {
+  content: string
+  score: number
+  source: string
+  metadata?: Record<string, unknown>
+}
+
+export interface FilteredResult {
+  content: string
+  score: number
+  source: string
+  relevanceNote: string
+  metadata?: Record<string, unknown>
+}
+
+export interface BrainResponse {
+  action: 'retrieved' | 'stored' | 'analyzed' | 'none'
+  summary: string
+  content: string
+  relevantItems: number
+}
+
+export interface TierResults {
+  label: string
+  results: FilterableResult[]
+}
+
+export class ResponseFilter {
+  // Infrastructure noise terms — strip from non-claude-brain projects
+  private readonly INFRA_NOISE = [
+    'chromadb', 'chroma', 'minisearch', 'compromise', 'better-sqlite3',
+    'pino', 'hono', 'bun:test', 'zod', 'mcp-server', 'claude-brain',
+    'model-context-protocol', 'embedding-service', 'vector-database',
+    'mcp tool', 'tool handler', 'phase 12', 'phase 13', 'phase 14', 'phase 15',
+    'semantic cache', 'precompute engine', 'knowledge graph builder'
+  ]
+
+  /**
+   * Filter a set of results to remove noise and improve relevance
+   */
+  filter(results: FilterableResult[], query: string, project?: string): FilteredResult[] {
+    let filtered = results
+
+    // 1. Remove infrastructure noise (if project !== 'claude-brain')
+    if (project !== 'claude-brain') {
+      filtered = filtered.filter(r => !this.isInfrastructureNoise(r.content))
+    }
+
+    // 2. Remove near-duplicates (>85% word overlap, keep higher score)
+    filtered = this.deduplicateResults(filtered)
+
+    // 3. Apply dynamic threshold (at least 70% of median similarity)
+    filtered = this.applyDynamicThreshold(filtered)
+
+    // 4. Sort by score descending
+    filtered.sort((a, b) => b.score - a.score)
+
+    // 5. Limit to 5 results
+    filtered = filtered.slice(0, 5)
+
+    // 6. Add one-line relevance explanation per result
+    return filtered.map(r => ({
+      content: r.content,
+      score: r.score,
+      source: r.source,
+      relevanceNote: this.generateRelevanceNote(r, query),
+      metadata: r.metadata
+    }))
+  }
+
+  /**
+   * Synthesize results from multiple tiers into a unified BrainResponse
+   */
+  synthesize(
+    tiers: TierResults[],
+    message: string,
+    project?: string,
+    action: BrainResponse['action'] = 'retrieved'
+  ): BrainResponse {
+    // Combine all results from all tiers
+    const allResults: FilterableResult[] = []
+    for (const tier of tiers) {
+      allResults.push(...tier.results)
+    }
+
+    if (allResults.length === 0) {
+      return {
+        action: 'none',
+        summary: 'No relevant information found',
+        content: `No results found for: "${message.slice(0, 100)}"`,
+        relevantItems: 0
+      }
+    }
+
+    // Filter the combined results
+    const filtered = this.filter(allResults, message, project)
+
+    if (filtered.length === 0) {
+      return {
+        action: 'none',
+        summary: 'Results filtered out as noise or irrelevant',
+        content: `No relevant results after filtering for: "${message.slice(0, 100)}"`,
+        relevantItems: 0
+      }
+    }
+
+    // Format filtered results
+    const contentParts: string[] = []
+    for (const result of filtered) {
+      const scoreStr = result.score > 0 ? ` [${Math.round(result.score * 100)}%]` : ''
+      contentParts.push(`**${result.source}**${scoreStr}\n${result.content}`)
+      if (result.relevanceNote) {
+        contentParts.push(`_${result.relevanceNote}_`)
+      }
+      contentParts.push('')
+    }
+
+    const summary = filtered.length === 1
+      ? `Found 1 relevant result`
+      : `Found ${filtered.length} relevant results`
+
+    return {
+      action,
+      summary,
+      content: contentParts.join('\n'),
+      relevantItems: filtered.length
+    }
+  }
+
+  /**
+   * Check if content is infrastructure/internal noise
+   */
+  private isInfrastructureNoise(content: string): boolean {
+    const lower = content.toLowerCase()
+    let noiseHits = 0
+
+    for (const term of this.INFRA_NOISE) {
+      if (lower.includes(term)) {
+        noiseHits++
+      }
+    }
+
+    // If more than 2 infrastructure terms appear, it's likely noise
+    return noiseHits >= 2
+  }
+
+  /**
+   * Remove near-duplicate results (>85% word overlap)
+   */
+  private deduplicateResults(results: FilterableResult[]): FilterableResult[] {
+    const kept: FilterableResult[] = []
+
+    for (const result of results) {
+      const isDuplicate = kept.some(existing => {
+        const overlap = this.calculateWordOverlap(existing.content, result.content)
+        return overlap > 0.85
+      })
+
+      if (!isDuplicate) {
+        kept.push(result)
+      }
+    }
+
+    return kept
+  }
+
+  /**
+   * Apply dynamic threshold — results must be at least 70% of median score
+   */
+  private applyDynamicThreshold(results: FilterableResult[]): FilterableResult[] {
+    if (results.length <= 1) return results
+
+    // Calculate median score
+    const scores = results.map(r => r.score).sort((a, b) => a - b)
+    const mid = Math.floor(scores.length / 2)
+    const left = scores[mid - 1] ?? 0
+    const right = scores[mid] ?? 0
+    const median = scores.length % 2 === 0
+      ? (left + right) / 2
+      : right
+
+    const threshold = median * 0.7
+
+    return results.filter(r => r.score >= threshold)
+  }
+
+  /**
+   * Calculate word overlap between two strings (0-1)
+   */
+  private calculateWordOverlap(a: string, b: string): number {
+    const wordsA = new Set(a.toLowerCase().split(/\s+/).filter(w => w.length > 2))
+    const wordsB = new Set(b.toLowerCase().split(/\s+/).filter(w => w.length > 2))
+
+    if (wordsA.size === 0 || wordsB.size === 0) return 0
+
+    let intersection = 0
+    for (const word of wordsA) {
+      if (wordsB.has(word)) intersection++
+    }
+
+    const smaller = Math.min(wordsA.size, wordsB.size)
+    return intersection / smaller
+  }
+
+  /**
+   * Generate a one-line relevance explanation
+   */
+  private generateRelevanceNote(result: FilterableResult, _query: string): string {
+    const score = Math.round(result.score * 100)
+    if (score >= 90) return `Highly relevant match (${score}%)`
+    if (score >= 70) return `Good match (${score}%)`
+    if (score >= 50) return `Partial match (${score}%)`
+    return `Low relevance match (${score}%)`
+  }
+}