claude-brain 0.15.2 → 0.16.0

package/src/routing/response-filter.ts

@@ -1,258 +1,261 @@
-/**
- * Brain Response Filter
- * Phase 16 + Phase 19: Filters noise, deduplicates, ranks, and synthesizes results
- *
- * Phase 19 changes:
- * - D2: Lower word overlap threshold 0.85 → 0.75, add ID-based and prefix-based dedup
- * - D3: Split INFRA_NOISE into strong (1 hit = filter) and weak (2+ hits = filter)
- */
-
-export interface FilterableResult {
-  content: string
-  score: number
-  source: string
-  metadata?: Record<string, unknown>
-}
-
-export interface FilteredResult {
-  content: string
-  score: number
-  source: string
-  relevanceNote: string
-  metadata?: Record<string, unknown>
-}
-
-export interface BrainResponse {
-  action: 'retrieved' | 'stored' | 'analyzed' | 'none'
-  summary: string
-  content: string
-  relevantItems: number
-}
-
-export interface TierResults {
-  label: string
-  results: FilterableResult[]
-}
-
-export class ResponseFilter {
-  // D3: Strong noise — 1 hit is enough to filter (very specific internal terms)
-  private readonly STRONG_NOISE = [
-    'mcp-server', 'model-context-protocol', 'embedding-service', 'vector-database',
-    'mcp tool', 'tool handler', 'precompute engine', 'knowledge graph builder',
-    'semantic cache', 'bun:test'
-  ]
-
-  // D3: Weak noise — need 2+ hits to filter (terms that could appear in legitimate decisions)
-  private readonly WEAK_NOISE = [
-    'chromadb', 'chroma', 'minisearch', 'compromise', 'better-sqlite3',
-    'pino', 'hono', 'zod', 'claude-brain',
-    'phase 12', 'phase 13', 'phase 14', 'phase 15', 'phase 16',
-    'phase 17', 'phase 18', 'phase 19'
-  ]
-
-  /**
-   * Filter a set of results to remove noise and improve relevance
-   */
-  filter(results: FilterableResult[], query: string, project?: string): FilteredResult[] {
-    let filtered = results
-
-    // 1. Remove infrastructure noise (if project !== 'claude-brain')
-    if (project !== 'claude-brain') {
-      filtered = filtered.filter(r => !this.isInfrastructureNoise(r.content))
-    }
-
-    // 2. Remove near-duplicates (Phase 19 D2: improved dedup)
-    filtered = this.deduplicateResults(filtered)
-
-    // 3. Apply dynamic threshold (at least 70% of median similarity)
-    filtered = this.applyDynamicThreshold(filtered)
-
-    // 4. Sort by score descending
-    filtered.sort((a, b) => b.score - a.score)
-
-    // 5. Limit to 5 results
-    filtered = filtered.slice(0, 5)
-
-    // 6. Add one-line relevance explanation per result
-    return filtered.map(r => ({
-      content: r.content,
-      score: r.score,
-      source: r.source,
-      relevanceNote: this.generateRelevanceNote(r, query),
-      metadata: r.metadata
-    }))
-  }
-
-  /**
-   * Synthesize results from multiple tiers into a unified BrainResponse
-   */
-  synthesize(
-    tiers: TierResults[],
-    message: string,
-    project?: string,
-    action: BrainResponse['action'] = 'retrieved'
-  ): BrainResponse {
-    // Combine all results from all tiers
-    const allResults: FilterableResult[] = []
-    for (const tier of tiers) {
-      allResults.push(...tier.results)
-    }
-
-    if (allResults.length === 0) {
-      return {
-        action: 'none',
-        summary: 'No relevant information found',
-        content: `No results found for: "${message.slice(0, 100)}"`,
-        relevantItems: 0
-      }
-    }
-
-    // Filter the combined results
-    const filtered = this.filter(allResults, message, project)
-
-    if (filtered.length === 0) {
-      return {
-        action: 'none',
-        summary: 'Results filtered out as noise or irrelevant',
-        content: `No relevant results after filtering for: "${message.slice(0, 100)}"`,
-        relevantItems: 0
-      }
-    }
-
-    // Format filtered results
-    const contentParts: string[] = []
-    for (const result of filtered) {
-      const scoreStr = result.score > 0 ? ` [${Math.round(result.score * 100)}%]` : ''
-      // Defensive: ensure content is always a string (prevents [object Object])
-      const contentStr = typeof result.content === 'string'
-        ? result.content
-        : JSON.stringify(result.content ?? '')
-      contentParts.push(`**${result.source}**${scoreStr}\n${contentStr}`)
-      if (result.relevanceNote) {
-        contentParts.push(`_${result.relevanceNote}_`)
-      }
-      contentParts.push('')
-    }
-
-    const summary = filtered.length === 1
-      ? `Found 1 relevant result`
-      : `Found ${filtered.length} relevant results`
-
-    return {
-      action,
-      summary,
-      content: contentParts.join('\n'),
-      relevantItems: filtered.length
-    }
-  }
-
-  /**
-   * D3: Check if content is infrastructure/internal noise
-   * Strong noise: 1 hit filters
-   * Weak noise: 2+ hits filters
-   */
-  private isInfrastructureNoise(content: string): boolean {
-    const lower = content.toLowerCase()
-
-    // Strong noise: a single hit is enough
-    for (const term of this.STRONG_NOISE) {
-      if (lower.includes(term)) {
-        return true
-      }
-    }
-
-    // Weak noise: need 2+ hits
-    let weakHits = 0
-    for (const term of this.WEAK_NOISE) {
-      if (lower.includes(term)) {
-        weakHits++
-      }
-    }
-    return weakHits >= 2
-  }
-
-  /**
-   * D2: Improved deduplication
-   * - ID-based dedup (if metadata has an ID)
-   * - Prefix-based dedup (first 100 chars match)
-   * - Word overlap dedup (lowered from 0.85 to 0.75)
-   */
-  private deduplicateResults(results: FilterableResult[]): FilterableResult[] {
-    const kept: FilterableResult[] = []
-    const seenIds = new Set<string>()
-    const seenPrefixes = new Set<string>()
-
-    for (const result of results) {
-      // ID-based dedup
-      const id = (result.metadata as any)?.id || (result.metadata as any)?.decision_id
-      if (id && seenIds.has(id)) continue
-      if (id) seenIds.add(id)
-
-      // Prefix-based dedup (first 100 chars, normalized)
-      const prefix = result.content.slice(0, 100).toLowerCase().replace(/\s+/g, ' ').trim()
-      if (prefix.length > 20 && seenPrefixes.has(prefix)) continue
-      if (prefix.length > 20) seenPrefixes.add(prefix)
-
-      // Word overlap dedup (D2: lowered to 0.75)
-      const isDuplicate = kept.some(existing => {
-        const overlap = this.calculateWordOverlap(existing.content, result.content)
-        return overlap > 0.75
-      })
-
-      if (!isDuplicate) {
-        kept.push(result)
-      }
-    }
-
-    return kept
-  }
-
-  /**
-   * Apply dynamic threshold — results must be at least 70% of median score
-   */
-  private applyDynamicThreshold(results: FilterableResult[]): FilterableResult[] {
-    if (results.length <= 1) return results
-
-    // Calculate median score
-    const scores = results.map(r => r.score).sort((a, b) => a - b)
-    const mid = Math.floor(scores.length / 2)
-    const left = scores[mid - 1] ?? 0
-    const right = scores[mid] ?? 0
-    const median = scores.length % 2 === 0
-      ? (left + right) / 2
-      : right
-
-    const threshold = median * 0.7
-
-    return results.filter(r => r.score >= threshold)
-  }
-
-  /**
-   * Calculate word overlap between two strings (0-1)
-   */
-  private calculateWordOverlap(a: string, b: string): number {
-    const wordsA = new Set(a.toLowerCase().split(/\s+/).filter(w => w.length > 2))
-    const wordsB = new Set(b.toLowerCase().split(/\s+/).filter(w => w.length > 2))
-
-    if (wordsA.size === 0 || wordsB.size === 0) return 0
-
-    let intersection = 0
-    for (const word of wordsA) {
-      if (wordsB.has(word)) intersection++
-    }
-
-    const smaller = Math.min(wordsA.size, wordsB.size)
-    return intersection / smaller
-  }
-
-  /**
-   * Generate a one-line relevance explanation
-   */
-  private generateRelevanceNote(result: FilterableResult, _query: string): string {
-    const score = Math.round(result.score * 100)
-    if (score >= 90) return `Highly relevant match (${score}%)`
-    if (score >= 70) return `Good match (${score}%)`
-    if (score >= 50) return `Partial match (${score}%)`
-    return `Low relevance match (${score}%)`
-  }
-}
+/**
+ * Brain Response Filter
+ * Phase 16 + Phase 19: Filters noise, deduplicates, ranks, and synthesizes results
+ *
+ * Phase 19 changes:
+ * - D2: Lower word overlap threshold 0.85 → 0.75, add ID-based and prefix-based dedup
+ * - D3: Split INFRA_NOISE into strong (1 hit = filter) and weak (2+ hits = filter)
+ */
+
+export interface FilterableResult {
+  content: string
+  score: number
+  source: string
+  metadata?: Record<string, unknown>
+}
+
+export interface FilteredResult {
+  content: string
+  score: number
+  source: string
+  relevanceNote: string
+  metadata?: Record<string, unknown>
+}
+
+export interface BrainResponse {
+  action: 'retrieved' | 'stored' | 'analyzed' | 'none'
+  summary: string
+  content: string
+  relevantItems: number
+}
+
+export interface TierResults {
+  label: string
+  results: FilterableResult[]
+}
+
+export class ResponseFilter {
+  // D3: Strong noise — 1 hit is enough to filter (very specific internal terms)
+  private readonly STRONG_NOISE = [
+    'mcp-server', 'model-context-protocol', 'embedding-service', 'vector-database',
+    'mcp tool', 'tool handler', 'precompute engine', 'knowledge graph builder',
+    'semantic cache', 'bun:test'
+  ]
+
+  // D3: Weak noise — need 2+ hits to filter (terms that could appear in legitimate decisions)
+  private readonly WEAK_NOISE = [
+    'chromadb', 'minisearch', 'compromise', 'better-sqlite3',
+    'pino', 'hono', 'zod', 'claude-brain',
+    'phase 12', 'phase 13', 'phase 14', 'phase 15', 'phase 16',
+    'phase 17', 'phase 18', 'phase 19'
+  ]
+
+  /**
+   * Filter a set of results to remove noise and improve relevance
+   */
+  filter(results: FilterableResult[], query: string, project?: string): FilteredResult[] {
+    let filtered = results
+
+    // 1. Remove infrastructure noise (if project !== 'claude-brain')
+    if (project !== 'claude-brain') {
+      const queryLower = query.toLowerCase()
+      filtered = filtered.filter(r => !this.isInfrastructureNoise(r.content, queryLower))
+    }
+
+    // 2. Remove near-duplicates (Phase 19 D2: improved dedup)
+    filtered = this.deduplicateResults(filtered)
+
+    // 3. Apply dynamic threshold (at least 70% of median similarity)
+    filtered = this.applyDynamicThreshold(filtered)
+
+    // 4. Sort by score descending
+    filtered.sort((a, b) => b.score - a.score)
+
+    // 5. Limit to 5 results
+    filtered = filtered.slice(0, 5)
+
+    // 6. Add one-line relevance explanation per result
+    return filtered.map(r => ({
+      content: r.content,
+      score: r.score,
+      source: r.source,
+      relevanceNote: this.generateRelevanceNote(r, query),
+      metadata: r.metadata
+    }))
+  }
+
+  /**
+   * Synthesize results from multiple tiers into a unified BrainResponse
+   */
+  synthesize(
+    tiers: TierResults[],
+    message: string,
+    project?: string,
+    action: BrainResponse['action'] = 'retrieved'
+  ): BrainResponse {
+    // Combine all results from all tiers
+    const allResults: FilterableResult[] = []
+    for (const tier of tiers) {
+      allResults.push(...tier.results)
+    }
+
+    if (allResults.length === 0) {
+      return {
+        action: 'none',
+        summary: 'No relevant information found',
+        content: `No results found for: "${message.slice(0, 100)}"`,
+        relevantItems: 0
+      }
+    }
+
+    // Filter the combined results
+    const filtered = this.filter(allResults, message, project)
+
+    if (filtered.length === 0) {
+      return {
+        action: 'none',
+        summary: 'Results filtered out as noise or irrelevant',
+        content: `No relevant results after filtering for: "${message.slice(0, 100)}"`,
+        relevantItems: 0
+      }
+    }
+
+    // Format filtered results
+    const contentParts: string[] = []
+    for (const result of filtered) {
+      const scoreStr = result.score > 0 ? ` [${Math.round(result.score * 100)}%]` : ''
+      // Defensive: ensure content is always a string (prevents [object Object])
+      const contentStr = typeof result.content === 'string'
+        ? result.content
+        : JSON.stringify(result.content ?? '')
+      contentParts.push(`**${result.source}**${scoreStr}\n${contentStr}`)
+      if (result.relevanceNote) {
+        contentParts.push(`_${result.relevanceNote}_`)
+      }
+      contentParts.push('')
+    }
+
+    const summary = filtered.length === 1
+      ? `Found 1 relevant result`
+      : `Found ${filtered.length} relevant results`
+
+    return {
+      action,
+      summary,
+      content: contentParts.join('\n'),
+      relevantItems: filtered.length
+    }
+  }
+
+  /**
+   * D3: Check if content is infrastructure/internal noise
+   * Strong noise: 1 hit filters (unless the term appears in the query)
+   * Weak noise: 2+ hits filters (skipping terms that appear in the query)
+   */
+  private isInfrastructureNoise(content: string, queryLower?: string): boolean {
+    const lower = content.toLowerCase()
+
+    // Strong noise: a single hit is enough (but skip terms the user asked about)
+    for (const term of this.STRONG_NOISE) {
+      if (queryLower && queryLower.includes(term)) continue
+      if (lower.includes(term)) {
+        return true
+      }
+    }
+
+    // Weak noise: need 2+ hits (skip terms the user asked about)
+    let weakHits = 0
+    for (const term of this.WEAK_NOISE) {
+      if (queryLower && queryLower.includes(term)) continue
+      if (lower.includes(term)) {
+        weakHits++
+      }
+    }
+    return weakHits >= 2
+  }
+
+  /**
+   * D2: Improved deduplication
+   * - ID-based dedup (if metadata has an ID)
+   * - Prefix-based dedup (first 100 chars match)
+   * - Word overlap dedup (lowered from 0.85 to 0.75)
+   */
+  private deduplicateResults(results: FilterableResult[]): FilterableResult[] {
+    const kept: FilterableResult[] = []
+    const seenIds = new Set<string>()
+    const seenPrefixes = new Set<string>()
+
+    for (const result of results) {
+      // ID-based dedup
+      const id = (result.metadata as any)?.id || (result.metadata as any)?.decision_id
+      if (id && seenIds.has(id)) continue
+      if (id) seenIds.add(id)
+
+      // Prefix-based dedup (first 100 chars, normalized)
+      const prefix = result.content.slice(0, 100).toLowerCase().replace(/\s+/g, ' ').trim()
+      if (prefix.length > 20 && seenPrefixes.has(prefix)) continue
+      if (prefix.length > 20) seenPrefixes.add(prefix)
+
+      // Word overlap dedup (D2: lowered to 0.75)
+      const isDuplicate = kept.some(existing => {
+        const overlap = this.calculateWordOverlap(existing.content, result.content)
+        return overlap > 0.75
+      })
+
+      if (!isDuplicate) {
+        kept.push(result)
+      }
+    }
+
+    return kept
+  }
+
+  /**
+   * Apply dynamic threshold — results must be at least 70% of median score
+   */
+  private applyDynamicThreshold(results: FilterableResult[]): FilterableResult[] {
+    if (results.length <= 1) return results
+
+    // Calculate median score
+    const scores = results.map(r => r.score).sort((a, b) => a - b)
+    const mid = Math.floor(scores.length / 2)
+    const left = scores[mid - 1] ?? 0
+    const right = scores[mid] ?? 0
+    const median = scores.length % 2 === 0
+      ? (left + right) / 2
+      : right
+
+    const threshold = Math.max(median * 0.7, 0.35)
+
+    return results.filter(r => r.score >= threshold)
+  }
+
+  /**
+   * Calculate word overlap between two strings (0-1)
+   */
+  private calculateWordOverlap(a: string, b: string): number {
+    const wordsA = new Set(a.toLowerCase().split(/\s+/).filter(w => w.length > 2))
+    const wordsB = new Set(b.toLowerCase().split(/\s+/).filter(w => w.length > 2))
+
+    if (wordsA.size === 0 || wordsB.size === 0) return 0
+
+    let intersection = 0
+    for (const word of wordsA) {
+      if (wordsB.has(word)) intersection++
+    }
+
+    const smaller = Math.min(wordsA.size, wordsB.size)
+    return intersection / smaller
+  }
+
+  /**
+   * Generate a one-line relevance explanation
+   */
+  private generateRelevanceNote(result: FilterableResult, _query: string): string {
+    const score = Math.round(result.score * 100)
+    if (score >= 90) return `Highly relevant match (${score}%)`
+    if (score >= 70) return `Good match (${score}%)`
+    if (score >= 50) return `Partial match (${score}%)`
+    return `Low relevance match (${score}%)`
+  }
+}