claude-brain 0.9.3 → 0.13.0

package/src/routing/response-filter.ts

@@ -1,7 +1,10 @@
 /**
  * Brain Response Filter
- * Phase 16: Filters noise, deduplicates, ranks, and synthesizes results
- * from the unified brain() tool
+ * Phase 16 + Phase 19: Filters noise, deduplicates, ranks, and synthesizes results
+ *
+ * Phase 19 changes:
+ * - D2: Lower word overlap threshold 0.85 → 0.75, add ID-based and prefix-based dedup
+ * - D3: Split INFRA_NOISE into strong (1 hit = filter) and weak (2+ hits = filter)
  */
 
 export interface FilterableResult {
@@ -32,13 +35,19 @@ export interface TierResults {
 }
 
 export class ResponseFilter {
-  // Infrastructure noise terms — strip from non-claude-brain projects
-  private readonly INFRA_NOISE = [
+  // D3: Strong noise — 1 hit is enough to filter (very specific internal terms)
+  private readonly STRONG_NOISE = [
+    'mcp-server', 'model-context-protocol', 'embedding-service', 'vector-database',
+    'mcp tool', 'tool handler', 'precompute engine', 'knowledge graph builder',
+    'semantic cache', 'bun:test'
+  ]
+
+  // D3: Weak noise — need 2+ hits to filter (terms that could appear in legitimate decisions)
+  private readonly WEAK_NOISE = [
     'chromadb', 'chroma', 'minisearch', 'compromise', 'better-sqlite3',
-    'pino', 'hono', 'bun:test', 'zod', 'mcp-server', 'claude-brain',
-    'model-context-protocol', 'embedding-service', 'vector-database',
-    'mcp tool', 'tool handler', 'phase 12', 'phase 13', 'phase 14', 'phase 15',
-    'semantic cache', 'precompute engine', 'knowledge graph builder'
+    'pino', 'hono', 'zod', 'claude-brain',
+    'phase 12', 'phase 13', 'phase 14', 'phase 15', 'phase 16',
+    'phase 17', 'phase 18', 'phase 19'
   ]
 
   /**
@@ -52,7 +61,7 @@ export class ResponseFilter {
       filtered = filtered.filter(r => !this.isInfrastructureNoise(r.content))
     }
 
-    // 2. Remove near-duplicates (>85% word overlap, keep higher score)
+    // 2. Remove near-duplicates (Phase 19 D2: improved dedup)
     filtered = this.deduplicateResults(filtered)
 
     // 3. Apply dynamic threshold (at least 70% of median similarity)
@@ -134,32 +143,56 @@ export class ResponseFilter {
   }
 
   /**
-   * Check if content is infrastructure/internal noise
+   * D3: Check if content is infrastructure/internal noise
+   * Strong noise: 1 hit filters
+   * Weak noise: 2+ hits filters
    */
   private isInfrastructureNoise(content: string): boolean {
     const lower = content.toLowerCase()
-    let noiseHits = 0
 
-    for (const term of this.INFRA_NOISE) {
+    // Strong noise: a single hit is enough
+    for (const term of this.STRONG_NOISE) {
       if (lower.includes(term)) {
-        noiseHits++
+        return true
       }
     }
 
-    // If more than 2 infrastructure terms appear, it's likely noise
-    return noiseHits >= 2
+    // Weak noise: need 2+ hits
+    let weakHits = 0
+    for (const term of this.WEAK_NOISE) {
+      if (lower.includes(term)) {
+        weakHits++
+      }
+    }
+    return weakHits >= 2
   }
 
   /**
-   * Remove near-duplicate results (>85% word overlap)
+   * D2: Improved deduplication
+   * - ID-based dedup (if metadata has an ID)
+   * - Prefix-based dedup (first 100 chars match)
+   * - Word overlap dedup (lowered from 0.85 to 0.75)
    */
   private deduplicateResults(results: FilterableResult[]): FilterableResult[] {
     const kept: FilterableResult[] = []
+    const seenIds = new Set<string>()
+    const seenPrefixes = new Set<string>()
 
     for (const result of results) {
+      // ID-based dedup
+      const id = (result.metadata as any)?.id || (result.metadata as any)?.decision_id
+      if (id && seenIds.has(id)) continue
+      if (id) seenIds.add(id)
+
+      // Prefix-based dedup (first 100 chars, normalized)
+      const prefix = result.content.slice(0, 100).toLowerCase().replace(/\s+/g, ' ').trim()
+      if (prefix.length > 20 && seenPrefixes.has(prefix)) continue
+      if (prefix.length > 20) seenPrefixes.add(prefix)
+
+      // Word overlap dedup (D2: lowered to 0.75)
       const isDuplicate = kept.some(existing => {
         const overlap = this.calculateWordOverlap(existing.content, result.content)
-        return overlap > 0.85
+        return overlap > 0.75
       })
 
       if (!isDuplicate) {