neuronlayer 0.1.9 → 0.2.0

package/src/core/deja-vu.ts

@@ -1,421 +0,0 @@
-/**
- * Déjà Vu Detection - Similar Problem Recognition
- *
- * Detects when a query or code pattern is similar to something solved before.
- * Surfaces past solutions to prevent reinventing the wheel.
- * "You solved a similar problem 2 weeks ago in auth.ts"
- */
-
-import type Database from 'better-sqlite3';
-import type { EmbeddingGenerator } from '../indexing/embeddings.js';
-import type { Tier2Storage } from '../storage/tier2.js';
-import { createHash } from 'crypto';
-import { formatTimeAgoWithContext } from '../utils/time.js';
-
-export interface DejaVuMatch {
-  type: 'query' | 'solution' | 'fix' | 'pattern';
-  similarity: number;
-  when: Date;
-  file: string;
-  snippet: string;
-  message: string;
-  context?: string;
-}
-
-export interface PastQuery {
-  queryHash: string;
-  queryText: string;
-  resultFiles: string[];
-  timestamp: Date;
-  wasUseful: boolean;
-  usefulness: number;
-}
-
-export interface PastSolution {
-  file: string;
-  snippet: string;
-  query: string;
-  timestamp: Date;
-  similarity: number;
-}
-
-interface QueryPatternRow {
-  id: number;
-  query_hash: string;
-  query_text: string;
-  result_files: string;
-  hit_count: number;
-  avg_usefulness: number;
-  last_used: number;
-}
-
-interface UsageEventRow {
-  id: number;
-  event_type: string;
-  file_path: string | null;
-  query: string | null;
-  context_used: number;
-  timestamp: number;
-}
-
-export class DejaVuDetector {
-  private db: Database.Database;
-  private tier2: Tier2Storage;
-  private embeddingGenerator: EmbeddingGenerator;
-
-  // Thresholds
-  private readonly SIMILARITY_THRESHOLD = 0.7;
-  private readonly MIN_USEFULNESS = 0.3;
-  private readonly MAX_AGE_DAYS = 90;
-
-  constructor(
-    db: Database.Database,
-    tier2: Tier2Storage,
-    embeddingGenerator: EmbeddingGenerator
-  ) {
-    this.db = db;
-    this.tier2 = tier2;
-    this.embeddingGenerator = embeddingGenerator;
-  }
-
-  /**
-   * Find similar past problems, solutions, and fixes
-   */
-  async findSimilar(query: string, limit: number = 3): Promise<DejaVuMatch[]> {
-    const matches: DejaVuMatch[] = [];
-
-    // Run searches in parallel
-    const [pastQueries, pastSolutions, pastFixes] = await Promise.all([
-      this.searchPastQueries(query),
-      this.searchPastSolutions(query),
-      this.searchPastFixes(query),
-    ]);
-
-    matches.push(...pastQueries, ...pastSolutions, ...pastFixes);
-
-    // Sort by similarity and recency
-    return matches
-      .sort((a, b) => {
-        // Weight similarity higher, but give bonus to more recent
-        const recencyA = this.getRecencyScore(a.when);
-        const recencyB = this.getRecencyScore(b.when);
-        const scoreA = a.similarity * 0.7 + recencyA * 0.3;
-        const scoreB = b.similarity * 0.7 + recencyB * 0.3;
-        return scoreB - scoreA;
-      })
-      .slice(0, limit);
-  }
-
-  /**
-   * Search for similar past queries with high usefulness
-   */
-  async searchPastQueries(query: string): Promise<DejaVuMatch[]> {
-    const matches: DejaVuMatch[] = [];
-
-    try {
-      // Get query embedding
-      const embedding = await this.embeddingGenerator.embed(query);
-
-      // Get recent useful queries
-      const stmt = this.db.prepare(`
-        SELECT query_hash, query_text, result_files, avg_usefulness, last_used
-        FROM query_patterns
-        WHERE avg_usefulness >= ?
-          AND last_used > unixepoch() - ? * 86400
-        ORDER BY avg_usefulness DESC, last_used DESC
-        LIMIT 50
-      `);
-
-      const rows = stmt.all(this.MIN_USEFULNESS, this.MAX_AGE_DAYS) as QueryPatternRow[];
-
-      for (const row of rows) {
-        // Calculate text similarity
-        const similarity = this.calculateTextSimilarity(query, row.query_text);
-
-        if (similarity >= this.SIMILARITY_THRESHOLD) {
-          const resultFiles = this.parseJsonArray(row.result_files);
-          const primaryFile = resultFiles[0] || 'unknown';
-
-          matches.push({
-            type: 'query',
-            similarity,
-            when: new Date(row.last_used * 1000),
-            file: primaryFile,
-            snippet: row.query_text.slice(0, 100),
-            message: formatTimeAgoWithContext(new Date(row.last_used * 1000), 'asked a similar question', primaryFile),
-            context: resultFiles.length > 1 ? `Also involved: ${resultFiles.slice(1, 4).join(', ')}` : undefined,
-          });
-        }
-      }
-    } catch (error) {
-      console.error('Error searching past queries:', error);
-    }
-
-    return matches;
-  }
-
-  /**
-   * Search for past solutions to similar problems
-   */
-  async searchPastSolutions(query: string): Promise<DejaVuMatch[]> {
-    const matches: DejaVuMatch[] = [];
-
-    try {
-      // Search codebase for semantically similar content
-      const embedding = await this.embeddingGenerator.embed(query);
-      const searchResults = this.tier2.search(embedding, 10);
-
-      // Find results that were previously part of useful context
-      for (const result of searchResults) {
-        if (result.similarity < this.SIMILARITY_THRESHOLD) continue;
-
-        // Check if this file was used in a useful query before
-        const usageStmt = this.db.prepare(`
-          SELECT ue.query, ue.timestamp, qp.avg_usefulness
-          FROM usage_events ue
-          LEFT JOIN query_patterns qp ON ue.query = qp.query_text
-          WHERE ue.file_path = ?
-            AND ue.event_type = 'context_used'
-            AND ue.timestamp > unixepoch() - ? * 86400
-            AND (qp.avg_usefulness IS NULL OR qp.avg_usefulness >= ?)
-          ORDER BY ue.timestamp DESC
-          LIMIT 1
-        `);
-
-        const usage = usageStmt.get(result.file, this.MAX_AGE_DAYS, this.MIN_USEFULNESS) as {
-          query: string;
-          timestamp: number;
-          avg_usefulness: number | null;
-        } | undefined;
-
-        if (usage) {
-          matches.push({
-            type: 'solution',
-            similarity: result.similarity,
-            when: new Date(usage.timestamp * 1000),
-            file: result.file,
-            snippet: result.preview.slice(0, 150),
-            message: formatTimeAgoWithContext(new Date(usage.timestamp * 1000), 'worked on this', result.file),
-            context: usage.query ? `For: "${usage.query.slice(0, 50)}..."` : undefined,
-          });
-        }
-      }
-    } catch (error) {
-      console.error('Error searching past solutions:', error);
-    }
-
-    return matches;
-  }
-
-  /**
-   * Search for past bug fixes with similar error patterns
-   */
-  async searchPastFixes(query: string): Promise<DejaVuMatch[]> {
-    const matches: DejaVuMatch[] = [];
-
-    // Check if query looks like an error
-    if (!this.looksLikeError(query)) {
-      return matches;
-    }
-
-    try {
-      // Search for similar error queries
-      const stmt = this.db.prepare(`
-        SELECT ue.query, ue.file_path, ue.timestamp, qp.result_files
-        FROM usage_events ue
-        LEFT JOIN query_patterns qp ON ue.query = qp.query_text
-        WHERE ue.event_type = 'query'
-          AND (ue.query LIKE '%error%' OR ue.query LIKE '%fix%' OR ue.query LIKE '%bug%')
-          AND ue.timestamp > unixepoch() - ? * 86400
-        ORDER BY ue.timestamp DESC
-        LIMIT 100
-      `);
-
-      const rows = stmt.all(this.MAX_AGE_DAYS) as Array<{
-        query: string;
-        file_path: string | null;
-        timestamp: number;
-        result_files: string | null;
-      }>;
-
-      for (const row of rows) {
-        if (!row.query) continue;
-
-        const similarity = this.calculateTextSimilarity(query, row.query);
-
-        if (similarity >= this.SIMILARITY_THRESHOLD * 0.8) { // Slightly lower threshold for errors
-          const resultFiles = row.result_files ? this.parseJsonArray(row.result_files) : [];
-          const file = row.file_path || resultFiles[0] || 'unknown';
-
-          matches.push({
-            type: 'fix',
-            similarity,
-            when: new Date(row.timestamp * 1000),
-            file,
-            snippet: row.query.slice(0, 100),
-            message: formatTimeAgoWithContext(new Date(row.timestamp * 1000), 'encountered a similar issue', file),
-          });
-        }
-      }
-    } catch (error) {
-      console.error('Error searching past fixes:', error);
-    }
-
-    return matches;
-  }
-
-  /**
-   * Record that a query was made (for future déjà vu detection)
-   */
-  recordQuery(query: string, files: string[], wasUseful?: boolean): void {
-    try {
-      const queryHash = this.hashQuery(query);
-
-      const stmt = this.db.prepare(`
-        INSERT INTO query_patterns (query_hash, query_text, result_files, hit_count, last_used)
-        VALUES (?, ?, ?, 1, unixepoch())
-        ON CONFLICT(query_hash) DO UPDATE SET
-          hit_count = hit_count + 1,
-          last_used = unixepoch()
-      `);
-
-      stmt.run(queryHash, query, JSON.stringify(files));
-
-      // Update usefulness if provided
-      if (wasUseful !== undefined) {
-        this.updateUsefulness(queryHash, wasUseful);
-      }
-    } catch (error) {
-      console.error('Error recording query:', error);
-    }
-  }
-
-  /**
-   * Update usefulness score for a query
-   */
-  updateUsefulness(queryHashOrText: string, wasUseful: boolean): void {
-    try {
-      const queryHash = queryHashOrText.length === 16
-        ? queryHashOrText
-        : this.hashQuery(queryHashOrText);
-
-      // Exponential moving average
-      const alpha = 0.3;
-      const newScore = wasUseful ? 1.0 : 0.0;
-
-      const stmt = this.db.prepare(`
-        UPDATE query_patterns
-        SET avg_usefulness = avg_usefulness * (1 - ?) + ? * ?
-        WHERE query_hash = ?
-      `);
-
-      stmt.run(alpha, alpha, newScore, queryHash);
-    } catch (error) {
-      console.error('Error updating usefulness:', error);
-    }
-  }
-
-  /**
-   * Get déjà vu statistics
-   */
-  getStats(): { totalQueries: number; usefulQueries: number; avgUsefulness: number } {
-    try {
-      const stmt = this.db.prepare(`
-        SELECT
-          COUNT(*) as total,
-          SUM(CASE WHEN avg_usefulness >= ? THEN 1 ELSE 0 END) as useful,
-          AVG(avg_usefulness) as avg_useful
-        FROM query_patterns
-        WHERE last_used > unixepoch() - ? * 86400
-      `);
-
-      const result = stmt.get(this.MIN_USEFULNESS, this.MAX_AGE_DAYS) as {
-        total: number;
-        useful: number;
-        avg_useful: number;
-      };
-
-      return {
-        totalQueries: result.total || 0,
-        usefulQueries: result.useful || 0,
-        avgUsefulness: result.avg_useful || 0,
-      };
-    } catch {
-      return { totalQueries: 0, usefulQueries: 0, avgUsefulness: 0 };
-    }
-  }
-
-  // ========== Private Methods ==========
-
-  private calculateTextSimilarity(a: string, b: string): number {
-    // Normalize texts
-    const normalizeText = (text: string) =>
-      text.toLowerCase().replace(/[^a-z0-9\s]/g, '').split(/\s+/).filter(w => w.length > 2);
-
-    const wordsA = new Set(normalizeText(a));
-    const wordsB = new Set(normalizeText(b));
-
-    if (wordsA.size === 0 || wordsB.size === 0) {
-      return 0;
-    }
-
-    // Jaccard similarity
-    let intersection = 0;
-    for (const word of wordsA) {
-      if (wordsB.has(word)) {
-        intersection++;
-      }
-    }
-
-    const union = wordsA.size + wordsB.size - intersection;
-    return intersection / union;
-  }
-
-  private looksLikeError(query: string): boolean {
-    const errorPatterns = [
-      /error/i,
-      /exception/i,
-      /failed/i,
-      /failing/i,
-      /broken/i,
-      /fix/i,
-      /bug/i,
-      /issue/i,
-      /not working/i,
-      /doesn't work/i,
-      /crash/i,
-      /undefined/i,
-      /null/i,
-      /NaN/i,
-      /TypeError/i,
-      /ReferenceError/i,
-      /SyntaxError/i,
-    ];
-
-    return errorPatterns.some(pattern => pattern.test(query));
-  }
-
-  private getRecencyScore(date: Date): number {
-    const now = new Date();
-    const diffMs = now.getTime() - date.getTime();
-    const diffDays = diffMs / (1000 * 60 * 60 * 24);
-
-    // Exponential decay: score of 1 for today, approaching 0 at MAX_AGE_DAYS
-    return Math.exp(-diffDays / (this.MAX_AGE_DAYS / 3));
-  }
-
-  private parseJsonArray(json: string | null): string[] {
-    if (!json) return [];
-    try {
-      const parsed = JSON.parse(json);
-      return Array.isArray(parsed) ? parsed : [];
-    } catch {
-      return [];
-    }
-  }
-
-  private hashQuery(query: string): string {
-    const normalized = query.toLowerCase().trim().replace(/\s+/g, ' ');
-    return createHash('md5').update(normalized).digest('hex').slice(0, 16);
-  }
-}