@eucoder/rag 0.2.0

package/src/optimizer.test.ts

@@ -0,0 +1,232 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { ParameterOptimizer } from "./optimizer.js";
+import { InMemoryFeedbackStorage } from "./feedback.js";
+import type { OptimizableParams } from "./optimizer.js";
+
+describe("ParameterOptimizer", () => {
+  let optimizer: ParameterOptimizer;
+  let storage: InMemoryFeedbackStorage;
+
+  beforeEach(() => {
+    storage = new InMemoryFeedbackStorage();
+    optimizer = new ParameterOptimizer(storage);
+  });
+
+  it("should suggest collecting more data when feedback is insufficient", async () => {
+    const params: OptimizableParams = {
+      minScore: 0.5,
+      topK: 5,
+    };
+
+    // Add only 5 feedback (not enough)
+    for (let i = 0; i < 5; i++) {
+      await storage.saveFeedback({
+        id: `test-${i}`,
+        query: `query ${i}`,
+        answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        rating: 4,
+        relevance: 0.8,
+        completeness: 0.7,
+        citationsQuality: 0.9,
+        timestamp: new Date(),
+        strategy: "strategy-a",
+      });
+    }
+
+    const suggestions = await optimizer.analyzeAndSuggest("strategy-a", params);
+    
+    expect(suggestions).toHaveLength(1);
+    expect(suggestions[0]?.type).toBe("strategy");
+    expect(suggestions[0]?.target).toBe("data_collection");
+  });
+
+  it("should suggest increasing minScore when relevance is low", async () => {
+    const params: OptimizableParams = {
+      minScore: 0.3,
+      topK: 10,
+    };
+
+    // Add 20 feedback with low relevance
+    for (let i = 0; i < 20; i++) {
+      await storage.saveFeedback({
+        id: `test-${i}`,
+        query: `query ${i}`,
+        answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        rating: 3,
+        relevance: 0.5,
+        completeness: 0.6,
+        citationsQuality: 0.7,
+        timestamp: new Date(),
+        strategy: "strategy-a",
+      });
+    }
+
+    const suggestions = await optimizer.analyzeAndSuggest("strategy-a", params);
+    
+    const minScoreSuggestion = suggestions.find(s => s.target === "minScore");
+    expect(minScoreSuggestion).toBeDefined();
+    expect(minScoreSuggestion?.suggested).toBeGreaterThan(params.minScore!);
+  });
+
+  it("should suggest enabling self-reflection when completeness is low", async () => {
+    const params: OptimizableParams = {
+      enableSelfReflection: false,
+      maxIterations: 2,
+    };
+
+    // Add 20 feedback with low completeness
+    for (let i = 0; i < 20; i++) {
+      await storage.saveFeedback({
+        id: `test-${i}`,
+        query: `query ${i}`,
+        answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        rating: 3,
+        relevance: 0.7,
+        completeness: 0.5,
+        citationsQuality: 0.8,
+        timestamp: new Date(),
+        strategy: "strategy-a",
+      });
+    }
+
+    const suggestions = await optimizer.analyzeAndSuggest("strategy-a", params);
+    
+    const selfReflectionSuggestion = suggestions.find(s => s.target === "enableSelfReflection");
+    expect(selfReflectionSuggestion).toBeDefined();
+    expect(selfReflectionSuggestion?.suggested).toBe(true);
+  });
+
+  it("should suggest increasing semanticWeight when citations quality is low", async () => {
+    const params: OptimizableParams = {
+      semanticWeight: 0.5,
+      keywordWeight: 0.5,
+    };
+
+    // Add 20 feedback with low citations quality
+    for (let i = 0; i < 20; i++) {
+      await storage.saveFeedback({
+        id: `test-${i}`,
+        query: `query ${i}`,
+        answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        rating: 3,
+        relevance: 0.7,
+        completeness: 0.7,
+        citationsQuality: 0.5,
+        timestamp: new Date(),
+        strategy: "strategy-a",
+      });
+    }
+
+    const suggestions = await optimizer.analyzeAndSuggest("strategy-a", params);
+    
+    const semanticWeightSuggestion = suggestions.find(s => s.target === "semanticWeight");
+    expect(semanticWeightSuggestion).toBeDefined();
+    expect(semanticWeightSuggestion?.suggested).toBeGreaterThan(params.semanticWeight!);
+  });
+
+  it("should apply suggestions with sufficient confidence", () => {
+    const currentParams: OptimizableParams = {
+      minScore: 0.3,
+      topK: 5,
+      enableSelfReflection: false,
+    };
+
+    const suggestions = [
+      {
+        type: "parameter" as const,
+        target: "minScore",
+        current: 0.3,
+        suggested: 0.5,
+        expectedImprovement: 0.15,
+        confidence: 0.8,
+        reasoning: "Test suggestion",
+      },
+      {
+        type: "parameter" as const,
+        target: "topK",
+        current: 5,
+        suggested: 10,
+        expectedImprovement: 0.1,
+        confidence: 0.6, // Below threshold
+        reasoning: "Test suggestion",
+      },
+    ];
+
+    const optimized = optimizer.applySuggestions(currentParams, suggestions, 0.7);
+
+    expect(optimized.minScore).toBe(0.5);
+    expect(optimized.topK).toBe(5); // Not applied due to low confidence
+    expect(optimized.enableSelfReflection).toBe(false);
+  });
+
+  it("should generate report", async () => {
+    const params: OptimizableParams = {
+      minScore: 0.5,
+      topK: 10,
+    };
+
+    // Add 15 feedback
+    for (let i = 0; i < 15; i++) {
+      await storage.saveFeedback({
+        id: `test-${i}`,
+        query: `query ${i}`,
+        answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        rating: 4,
+        relevance: 0.7,
+        completeness: 0.6,
+        citationsQuality: 0.8,
+        timestamp: new Date(),
+        strategy: "strategy-a",
+      });
+    }
+
+    const report = await optimizer.generateReport("strategy-a", params);
+
+    expect(report).toContain("Report Ottimizzazione: strategy-a");
+    expect(report).toContain("Metriche Attuali");
+    expect(report).toContain("Total Queries");
+    expect(report).toContain("15");
+  });
+
+  it("should detect declining trend", async () => {
+    const params: OptimizableParams = {
+      minScore: 0.5,
+    };
+
+    // Add older feedback with high relevance
+    for (let i = 0; i < 10; i++) {
+      await storage.saveFeedback({
+        id: `old-${i}`,
+        query: `query ${i}`,
+        answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        rating: 5,
+        relevance: 0.9,
+        completeness: 0.9,
+        citationsQuality: 0.9,
+        timestamp: new Date(Date.now() - 1000000),
+        strategy: "strategy-a",
+      });
+    }
+
+    // Add recent feedback with low relevance
+    for (let i = 0; i < 10; i++) {
+      await storage.saveFeedback({
+        id: `new-${i}`,
+        query: `query ${i}`,
+        answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        rating: 2,
+        relevance: 0.4,
+        completeness: 0.4,
+        citationsQuality: 0.4,
+        timestamp: new Date(),
+        strategy: "strategy-a",
+      });
+    }
+
+    const suggestions = await optimizer.analyzeAndSuggest("strategy-a", params);
+    
+    const trendSuggestion = suggestions.find(s => s.target === "overall_strategy");
+    expect(trendSuggestion).toBeDefined();
+    expect(trendSuggestion?.reasoning).toContain("declino");
+  });
+});

package/src/optimizer.ts

@@ -0,0 +1,307 @@
+import type { RagFeedback, FeedbackStorage, OptimizationSuggestion } from "./feedback.js";
+import { InMemoryFeedbackStorage } from "./feedback.js";
+import { MetricsCalculator } from "./metrics.js";
+
+/**
+ * Parametri ottimizzabili per strategie RAG
+ */
+export interface OptimizableParams {
+  // Corrective RAG
+  minGoodHits?: number;
+  minScore?: number;
+  maxRewrites?: number;
+  
+  // Hybrid Search
+  semanticWeight?: number;
+  keywordWeight?: number;
+  rrfK?: number;
+  
+  // Agentic RAG
+  maxIterations?: number;
+  enableSelfReflection?: boolean;
+  enableChainOfThought?: boolean;
+  
+  // General
+  topK?: number;
+}
+
+/**
+ * Ottimizzatore automatico per parametri RAG
+ */
+export class ParameterOptimizer {
+  private storage: FeedbackStorage;
+
+  constructor(storage?: FeedbackStorage) {
+    this.storage = storage || new InMemoryFeedbackStorage();
+  }
+
+  /**
+   * Analizza i feedback e suggerisce ottimizzazioni
+   */
+  async analyzeAndSuggest(
+    strategy: string,
+    currentParams: OptimizableParams
+  ): Promise<OptimizationSuggestion[]> {
+    const feedbacks = await this.storage.getFeedback(strategy);
+    
+    if (feedbacks.length < 10) {
+      return [{
+        type: "strategy",
+        target: "data_collection",
+        current: feedbacks.length,
+        suggested: 50,
+        expectedImprovement: 0,
+        confidence: 1,
+        reasoning: `Raccogli almeno 50 feedback per analisi affidabile (attualmente: ${feedbacks.length})`,
+      }];
+    }
+
+    const suggestions: OptimizationSuggestion[] = [];
+
+    // Analizza problemi di relevance
+    const relevanceSuggestions = this.analyzeRelevance(feedbacks, currentParams);
+    suggestions.push(...relevanceSuggestions);
+
+    // Analizza problemi di completeness
+    const completenessSuggestions = this.analyzeCompleteness(feedbacks, currentParams);
+    suggestions.push(...completenessSuggestions);
+
+    // Analizza problemi di citations quality
+    const citationsSuggestions = this.analyzeCitationsQuality(feedbacks, currentParams);
+    suggestions.push(...citationsSuggestions);
+
+    // Analizza trend temporale
+    const trendSuggestions = this.analyzeTrend(feedbacks, currentParams);
+    suggestions.push(...trendSuggestions);
+
+    return suggestions;
+  }
+
+  /**
+   * Analizza problemi di relevance
+   */
+  private analyzeRelevance(
+    feedbacks: RagFeedback[],
+    params: OptimizableParams
+  ): OptimizationSuggestion[] {
+    const suggestions: OptimizationSuggestion[] = [];
+    const avgRelevance = feedbacks.reduce((sum, f) => sum + f.relevance, 0) / feedbacks.length;
+
+    // Se relevance è bassa, suggerisci di aumentare minScore o topK
+    if (avgRelevance < 0.6) {
+      if (params.minScore !== undefined && params.minScore < 0.5) {
+        suggestions.push({
+          type: "parameter",
+          target: "minScore",
+          current: params.minScore,
+          suggested: Math.min(0.7, params.minScore + 0.1),
+          expectedImprovement: 0.15,
+          confidence: 0.8,
+          reasoning: `Relevance media bassa (${avgRelevance.toFixed(2)}). Aumenta minScore per filtrare risultati meno rilevanti.`,
+        });
+      }
+
+      if (params.topK !== undefined && params.topK < 10) {
+        suggestions.push({
+          type: "parameter",
+          target: "topK",
+          current: params.topK,
+          suggested: Math.min(15, params.topK + 3),
+          expectedImprovement: 0.1,
+          confidence: 0.7,
+          reasoning: `Relevance media bassa (${avgRelevance.toFixed(2)}). Aumenta topK per avere più candidati da cui scegliere.`,
+        });
+      }
+    }
+
+    // Se relevance è molto alta, potresti essere troppo conservativo
+    if (avgRelevance > 0.9 && params.minScore !== undefined && params.minScore > 0.7) {
+      suggestions.push({
+        type: "parameter",
+        target: "minScore",
+        current: params.minScore,
+        suggested: Math.max(0.5, params.minScore - 0.1),
+        expectedImprovement: 0.05,
+        confidence: 0.6,
+        reasoning: `Relevance molto alta (${avgRelevance.toFixed(2)}) ma potresti perdere risultati utili. Riduci minScore per aumentare recall.`,
+      });
+    }
+
+    return suggestions;
+  }
+
+  /**
+   * Analizza problemi di completeness
+   */
+  private analyzeCompleteness(
+    feedbacks: RagFeedback[],
+    params: OptimizableParams
+  ): OptimizationSuggestion[] {
+    const suggestions: OptimizationSuggestion[] = [];
+    const avgCompleteness = feedbacks.reduce((sum, f) => sum + f.completeness, 0) / feedbacks.length;
+
+    // Se completeness è bassa, suggerisci di abilitare self-reflection o aumentare iterazioni
+    if (avgCompleteness < 0.6) {
+      if (params.enableSelfReflection === false) {
+        suggestions.push({
+          type: "parameter",
+          target: "enableSelfReflection",
+          current: false,
+          suggested: true,
+          expectedImprovement: 0.2,
+          confidence: 0.85,
+          reasoning: `Completeness bassa (${avgCompleteness.toFixed(2)}). Abilita self-reflection per migliorare qualità risposte.`,
+        });
+      }
+
+      if (params.maxIterations !== undefined && params.maxIterations < 3) {
+        suggestions.push({
+          type: "parameter",
+          target: "maxIterations",
+          current: params.maxIterations,
+          suggested: Math.min(5, params.maxIterations + 1),
+          expectedImprovement: 0.15,
+          confidence: 0.75,
+          reasoning: `Completeness bassa (${avgCompleteness.toFixed(2)}). Aumenta maxIterations per permettere più tentativi di miglioramento.`,
+        });
+      }
+
+      if (params.enableChainOfThought === false) {
+        suggestions.push({
+          type: "parameter",
+          target: "enableChainOfThought",
+          current: false,
+          suggested: true,
+          expectedImprovement: 0.18,
+          confidence: 0.8,
+          reasoning: `Completeness bassa (${avgCompleteness.toFixed(2)}). Abilita chain-of-thought per risposte più strutturate.`,
+        });
+      }
+    }
+
+    return suggestions;
+  }
+
+  /**
+   * Analizza problemi di citations quality
+   */
+  private analyzeCitationsQuality(
+    feedbacks: RagFeedback[],
+    params: OptimizableParams
+  ): OptimizationSuggestion[] {
+    const suggestions: OptimizationSuggestion[] = [];
+    const avgCitations = feedbacks.reduce((sum, f) => sum + f.citationsQuality, 0) / feedbacks.length;
+
+    // Se citations quality è bassa, suggerisci di migliorare hybrid search
+    if (avgCitations < 0.6) {
+      if (params.semanticWeight !== undefined && params.semanticWeight < 0.6) {
+        suggestions.push({
+          type: "parameter",
+          target: "semanticWeight",
+          current: params.semanticWeight,
+          suggested: Math.min(0.7, params.semanticWeight + 0.1),
+          expectedImprovement: 0.12,
+          confidence: 0.7,
+          reasoning: `Citations quality bassa (${avgCitations.toFixed(2)}). Aumenta semanticWeight per risultati più semanticamente rilevanti.`,
+        });
+      }
+
+      if (params.rrfK !== undefined && params.rrfK > 80) {
+        suggestions.push({
+          type: "parameter",
+          target: "rrfK",
+          current: params.rrfK,
+          suggested: Math.max(50, params.rrfK - 20),
+          expectedImprovement: 0.1,
+          confidence: 0.65,
+          reasoning: `Citations quality bassa (${avgCitations.toFixed(2)}). Riduci rrfK per dare più peso ai risultati top-ranked.`,
+        });
+      }
+    }
+
+    return suggestions;
+  }
+
+  /**
+   * Analizza trend temporale
+   */
+  private analyzeTrend(
+    feedbacks: RagFeedback[],
+    _params: OptimizableParams
+  ): OptimizationSuggestion[] {
+    const suggestions: OptimizationSuggestion[] = [];
+    const trend = MetricsCalculator.identifyTrend(feedbacks);
+
+    if (trend === "declining") {
+      suggestions.push({
+        type: "strategy",
+        target: "overall_strategy",
+        current: "current",
+        suggested: "review",
+        expectedImprovement: 0.1,
+        confidence: 0.7,
+        reasoning: `Trend in declino negli ultimi feedback. Considera di rivedere la strategia o raccogliere feedback qualitativi per capire cosa non funziona.`,
+      });
+    }
+
+    return suggestions;
+  }
+
+  /**
+   * Applica automaticamente le ottimizzazioni suggerite
+   */
+  applySuggestions(
+    currentParams: OptimizableParams,
+    suggestions: OptimizationSuggestion[],
+    minConfidence: number = 0.7
+  ): OptimizableParams {
+    const optimized = { ...currentParams };
+
+    for (const suggestion of suggestions) {
+      if (suggestion.confidence >= minConfidence && suggestion.type === "parameter") {
+        (optimized as any)[suggestion.target] = suggestion.suggested;
+      }
+    }
+
+    return optimized;
+  }
+
+  /**
+   * Genera report di ottimizzazione
+   */
+  async generateReport(
+    strategy: string,
+    currentParams: OptimizableParams
+  ): Promise<string> {
+    const feedbacks = await this.storage.getFeedback(strategy);
+    const metrics = MetricsCalculator.calculateMetrics(feedbacks, strategy);
+    const suggestions = await this.analyzeAndSuggest(strategy, currentParams);
+
+    let report = `# Report Ottimizzazione: ${strategy}\n\n`;
+    report += `## Metriche Attuali\n`;
+    report += `- **Total Queries**: ${metrics.totalQueries}\n`;
+    report += `- **Overall Score**: ${metrics.overallScore.toFixed(3)}\n`;
+    report += `- **Average Relevance**: ${metrics.averageRelevance.toFixed(3)}\n`;
+    report += `- **Average Completeness**: ${metrics.averageCompleteness.toFixed(3)}\n`;
+    report += `- **Average Citations Quality**: ${metrics.averageCitationsQuality.toFixed(3)}\n`;
+    report += `- **Confidence**: ${(metrics.confidence * 100).toFixed(0)}%\n\n`;
+
+    report += `## Suggerimenti di Ottimizzazione\n\n`;
+    
+    if (suggestions.length === 0) {
+      report += `Nessun suggerimento. I parametri attuali sembrano ottimali.\n`;
+    } else {
+      for (const suggestion of suggestions) {
+        report += `### ${suggestion.target}\n`;
+        report += `- **Tipo**: ${suggestion.type}\n`;
+        report += `- **Valore attuale**: ${JSON.stringify(suggestion.current)}\n`;
+        report += `- **Valore suggerito**: ${JSON.stringify(suggestion.suggested)}\n`;
+        report += `- **Miglioramento atteso**: +${(suggestion.expectedImprovement * 100).toFixed(1)}%\n`;
+        report += `- **Confidence**: ${(suggestion.confidence * 100).toFixed(0)}%\n`;
+        report += `- **Ragionamento**: ${suggestion.reasoning}\n\n`;
+      }
+    }
+
+    return report;
+  }
+}