@eucoder/rag 0.2.0

package/src/self-improving.test.ts

@@ -0,0 +1,341 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { SelfImprovingRag } from "./self-improving.js";
+import { InMemoryFeedbackStorage } from "./feedback.js";
+import type { RagStrategyConfig } from "./ab-testing.js";
+
+describe("SelfImprovingRag", () => {
+  let selfImproving: SelfImprovingRag;
+  let storage: InMemoryFeedbackStorage;
+
+  beforeEach(() => {
+    storage = new InMemoryFeedbackStorage();
+    selfImproving = new SelfImprovingRag(
+      {
+        minFeedbackForOptimization: 10,
+        minConfidenceForSuggestion: 0.7,
+        minSamplesForABTest: 15,
+        autoApplySuggestions: false,
+      },
+      storage
+    );
+  });
+
+  it("should register strategies with parameters", () => {
+    const config: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    const params = {
+      minScore: 0.5,
+      topK: 10,
+    };
+
+    selfImproving.registerStrategy(config, params);
+    
+    const retrievedParams = selfImproving.getParams("strategy-a");
+    expect(retrievedParams).toEqual(params);
+  });
+
+  it("should select best strategy based on metrics", async () => {
+    const configA: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    const configB: RagStrategyConfig = {
+      name: "strategy-b",
+      type: "hybrid",
+      params: {},
+    };
+
+    selfImproving.registerStrategy(configA, { minScore: 0.5 });
+    selfImproving.registerStrategy(configB, { semanticWeight: 0.6 });
+
+    // Add feedback for strategy A (lower quality)
+    for (let i = 0; i < 20; i++) {
+      await selfImproving.recordFeedback(
+        `query ${i}`,
+        { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        {
+          rating: 3,
+          relevance: 0.6,
+          completeness: 0.5,
+          citationsQuality: 0.7,
+        },
+        "strategy-a"
+      );
+    }
+
+    // Add feedback for strategy B (higher quality)
+    for (let i = 0; i < 20; i++) {
+      await selfImproving.recordFeedback(
+        `query ${i}`,
+        { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        {
+          rating: 5,
+          relevance: 0.9,
+          completeness: 0.85,
+          citationsQuality: 0.95,
+        },
+        "strategy-b"
+      );
+    }
+
+    const best = await selfImproving.selectBestStrategy();
+    expect(best).not.toBeNull();
+    expect(best?.strategy).toBe("strategy-b");
+  });
+
+  it("should record feedback and trigger optimization check", async () => {
+    const config: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    selfImproving.registerStrategy(config, { minScore: 0.3 });
+
+    let optimizationTriggered = false;
+    selfImproving = new SelfImprovingRag(
+      {
+        minFeedbackForOptimization: 10,
+        onOptimization: () => {
+          optimizationTriggered = true;
+        },
+      },
+      storage
+    );
+    selfImproving.registerStrategy(config, { minScore: 0.3 });
+
+    // Add 15 feedback with low relevance to trigger optimization
+    for (let i = 0; i < 15; i++) {
+      await selfImproving.recordFeedback(
+        `query ${i}`,
+        { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        {
+          rating: 3,
+          relevance: 0.5,
+          completeness: 0.6,
+          citationsQuality: 0.7,
+        },
+        "strategy-a"
+      );
+    }
+
+    expect(optimizationTriggered).toBe(true);
+  });
+
+  it("should auto-apply suggestions when configured", async () => {
+    const config: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    selfImproving = new SelfImprovingRag(
+      {
+        minFeedbackForOptimization: 10,
+        autoApplySuggestions: true,
+        minConfidenceForSuggestion: 0.7,
+      },
+      storage
+    );
+    selfImproving.registerStrategy(config, { minScore: 0.3 });
+
+    // Add 15 feedback with low relevance
+    for (let i = 0; i < 15; i++) {
+      await selfImproving.recordFeedback(
+        `query ${i}`,
+        { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        {
+          rating: 3,
+          relevance: 0.5,
+          completeness: 0.6,
+          citationsQuality: 0.7,
+        },
+        "strategy-a"
+      );
+    }
+
+    const params = selfImproving.getParams("strategy-a");
+    // Parameters should be updated if suggestions were applied
+    expect(params).toBeDefined();
+  });
+
+  it("should run A/B test", async () => {
+    const configA: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    const configB: RagStrategyConfig = {
+      name: "strategy-b",
+      type: "hybrid",
+      params: {},
+    };
+
+    selfImproving.registerStrategy(configA, {});
+    selfImproving.registerStrategy(configB, {});
+
+    // Add sufficient samples
+    for (let i = 0; i < 20; i++) {
+      await selfImproving.recordFeedback(
+        `query ${i}`,
+        { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        {
+          rating: 3,
+          relevance: 0.6,
+          completeness: 0.5,
+          citationsQuality: 0.7,
+        },
+        "strategy-a"
+      );
+
+      await selfImproving.recordFeedback(
+        `query ${i}`,
+        { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        {
+          rating: 5,
+          relevance: 0.9,
+          completeness: 0.85,
+          citationsQuality: 0.95,
+        },
+        "strategy-b"
+      );
+    }
+
+    const result = await selfImproving.runABTest("strategy-a", "strategy-b");
+    expect(result).not.toBeNull();
+    expect(result.winner).toBe("B");
+  });
+
+  it("should update parameters manually", () => {
+    const config: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    selfImproving.registerStrategy(config, { minScore: 0.5 });
+
+    selfImproving.updateParams("strategy-a", { minScore: 0.7, topK: 15 });
+
+    const params = selfImproving.getParams("strategy-a");
+    expect(params?.minScore).toBe(0.7);
+    expect(params?.topK).toBe(15);
+  });
+
+  it("should get metrics for strategy", async () => {
+    const config: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    selfImproving.registerStrategy(config, {});
+
+    for (let i = 0; i < 5; i++) {
+      await selfImproving.recordFeedback(
+        `query ${i}`,
+        { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        {
+          rating: 4,
+          relevance: 0.8,
+          completeness: 0.7,
+          citationsQuality: 0.9,
+        },
+        "strategy-a"
+      );
+    }
+
+    const metrics = await selfImproving.getMetrics("strategy-a");
+    expect(metrics).not.toBeNull();
+    expect(metrics?.totalQueries).toBe(5);
+  });
+
+  it("should generate report", async () => {
+    const config: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    selfImproving.registerStrategy(config, { minScore: 0.5 });
+
+    for (let i = 0; i < 10; i++) {
+      await selfImproving.recordFeedback(
+        `query ${i}`,
+        { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        {
+          rating: 4,
+          relevance: 0.7,
+          completeness: 0.6,
+          citationsQuality: 0.8,
+        },
+        "strategy-a"
+      );
+    }
+
+    const report = await selfImproving.generateReport("strategy-a");
+    expect(report).toContain("Report Ottimizzazione: strategy-a");
+    expect(report).toContain("Metriche Attuali");
+  });
+
+  it("should get suggestions", async () => {
+    const config: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    selfImproving.registerStrategy(config, { minScore: 0.3 });
+
+    for (let i = 0; i < 15; i++) {
+      await selfImproving.recordFeedback(
+        `query ${i}`,
+        { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        {
+          rating: 3,
+          relevance: 0.5,
+          completeness: 0.6,
+          citationsQuality: 0.7,
+        },
+        "strategy-a"
+      );
+    }
+
+    const suggestions = await selfImproving.getSuggestions("strategy-a");
+    expect(suggestions.length).toBeGreaterThan(0);
+  });
+
+  it("should reset feedback", async () => {
+    const config: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    selfImproving.registerStrategy(config, {});
+
+    await selfImproving.recordFeedback(
+      "query",
+      { text: "answer", citations: [], steps: [], rewrites: 0 },
+      {
+        rating: 4,
+        relevance: 0.8,
+        completeness: 0.7,
+        citationsQuality: 0.9,
+      },
+      "strategy-a"
+    );
+
+    await selfImproving.reset("strategy-a");
+
+    const metrics = await selfImproving.getMetrics("strategy-a");
+    expect(metrics).toBeNull();
+  });
+});

package/src/self-improving.ts

@@ -0,0 +1,239 @@
+import type { RagAnswer } from "./types.js";
+import type { FeedbackStorage, OptimizationSuggestion } from "./feedback.js";
+import { InMemoryFeedbackStorage } from "./feedback.js";
+import { ABTestFramework, type RagStrategyConfig } from "./ab-testing.js";
+import { ParameterOptimizer, type OptimizableParams } from "./optimizer.js";
+
+/**
+ * Configurazione per il sistema self-improving
+ */
+export interface SelfImprovingConfig {
+  // Soglie per attivazione ottimizzazione
+  minFeedbackForOptimization: number;
+  minConfidenceForSuggestion: number;
+  
+  // Soglie per A/B testing
+  minSamplesForABTest: number;
+  
+  // Auto-apply suggestions
+  autoApplySuggestions: boolean;
+  
+  // Callback per notifiche
+  onOptimization?: (suggestions: OptimizationSuggestion[]) => void;
+  onABTestComplete?: (result: any) => void;
+}
+
+/**
+ * Sistema self-improving per RAG
+ * Integra feedback collection, A/B testing e ottimizzazione automatica
+ */
+export class SelfImprovingRag {
+  private storage: FeedbackStorage;
+  private abFramework: ABTestFramework;
+  private optimizer: ParameterOptimizer;
+  private config: Required<SelfImprovingConfig>;
+  private currentParams: Map<string, OptimizableParams> = new Map();
+
+  constructor(config?: Partial<SelfImprovingConfig>, storage?: FeedbackStorage) {
+    this.storage = storage || new InMemoryFeedbackStorage();
+    this.abFramework = new ABTestFramework(this.storage);
+    this.optimizer = new ParameterOptimizer(this.storage);
+    
+    this.config = {
+      minFeedbackForOptimization: config?.minFeedbackForOptimization ?? 20,
+      minConfidenceForSuggestion: config?.minConfidenceForSuggestion ?? 0.7,
+      minSamplesForABTest: config?.minSamplesForABTest ?? 30,
+      autoApplySuggestions: config?.autoApplySuggestions ?? false,
+      onOptimization: config?.onOptimization ?? (() => {}),
+      onABTestComplete: config?.onABTestComplete ?? (() => {}),
+    };
+  }
+
+  /**
+   * Registra una strategia RAG
+   */
+  registerStrategy(config: RagStrategyConfig, params: OptimizableParams): void {
+    this.abFramework.registerStrategy(config);
+    this.currentParams.set(config.name, params);
+  }
+
+  /**
+   * Seleziona la migliore strategia basata su metriche storiche
+   */
+  async selectBestStrategy(): Promise<{ strategy: string; params: OptimizableParams } | null> {
+    const allMetrics = await this.abFramework.getAllMetrics();
+    
+    if (allMetrics.length === 0) {
+      // Nessuna metrica disponibile, usa prima strategia registrata
+      const strategies = Array.from(this.currentParams.keys());
+      if (strategies.length === 0) return null;
+      
+      const strategy = strategies[0]!;
+      return {
+        strategy,
+        params: this.currentParams.get(strategy)!,
+      };
+    }
+
+    // Seleziona strategia con score più alto e confidence sufficiente
+    const validMetrics = allMetrics.filter(m => m.confidence >= 0.5);
+    if (validMetrics.length === 0) {
+      // Usa quella con più feedback
+      const best = allMetrics[0]!;
+      return {
+        strategy: best.strategy,
+        params: this.currentParams.get(best.strategy) || {},
+      };
+    }
+
+    const best = validMetrics[0]!;
+    return {
+      strategy: best.strategy,
+      params: this.currentParams.get(best.strategy) || {},
+    };
+  }
+
+  /**
+   * Registra feedback per una risposta
+   */
+  async recordFeedback(
+    query: string,
+    answer: RagAnswer,
+    feedback: {
+      rating: number;
+      relevance: number;
+      completeness: number;
+      citationsQuality: number;
+      comments?: string;
+    },
+    strategy: string
+  ): Promise<void> {
+    await this.abFramework.recordFeedback(
+      query,
+      answer,
+      feedback.rating,
+      feedback.relevance,
+      feedback.completeness,
+      feedback.citationsQuality,
+      strategy,
+      feedback.comments
+    );
+
+    // Controlla se è tempo di ottimizzare
+    await this.checkAndOptimize(strategy);
+  }
+
+  /**
+   * Controlla se è tempo di ottimizzare e applica suggerimenti
+   */
+  private async checkAndOptimize(strategy: string): Promise<void> {
+    const feedbacks = await this.storage.getFeedback(strategy);
+    
+    if (feedbacks.length < this.config.minFeedbackForOptimization) {
+      return; // Non abbastanza feedback
+    }
+
+    const currentParams = this.currentParams.get(strategy);
+    if (!currentParams) return;
+
+    const suggestions = await this.optimizer.analyzeAndSuggest(strategy, currentParams);
+    
+    if (suggestions.length > 0) {
+      // Notifica suggerimenti
+      this.config.onOptimization(suggestions);
+
+      // Auto-applica se configurato
+      if (this.config.autoApplySuggestions) {
+        const optimizedParams = this.optimizer.applySuggestions(
+          currentParams,
+          suggestions,
+          this.config.minConfidenceForSuggestion
+        );
+        this.currentParams.set(strategy, optimizedParams);
+      }
+    }
+  }
+
+  /**
+   * Esegue A/B test tra due strategie
+   */
+  async runABTest(strategyA: string, strategyB: string): Promise<any> {
+    try {
+      const result = await this.abFramework.runABTest(
+        strategyA,
+        strategyB,
+        this.config.minSamplesForABTest
+      );
+
+      this.config.onABTestComplete(result);
+
+      // Se c'è un vincitore chiaro, aggiorna parametri
+      if (result.winner !== "tie" && result.statisticalSignificance > 0.8) {
+        const winnerStrategy = result.winner === "A" ? strategyA : strategyB;
+        const winnerMetrics = result.winner === "A" ? result.metricsA : result.metricsB;
+        
+        // Log vittoria
+        console.log(
+          `A/B Test: ${winnerStrategy} vince con score ${winnerMetrics.overallScore.toFixed(3)} ` +
+          `(miglioramento: ${(result.improvement * 100).toFixed(1)}%)`
+        );
+      }
+
+      return result;
+    } catch (error) {
+      console.warn("A/B test non può essere eseguito:", (error as Error).message);
+      return null;
+    }
+  }
+
+  /**
+   * Ottiene parametri correnti per una strategia
+   */
+  getParams(strategy: string): OptimizableParams | undefined {
+    return this.currentParams.get(strategy);
+  }
+
+  /**
+   * Aggiorna parametri per una strategia
+   */
+  updateParams(strategy: string, params: OptimizableParams): void {
+    this.currentParams.set(strategy, params);
+  }
+
+  /**
+   * Ottiene metriche per una strategia
+   */
+  async getMetrics(strategy: string) {
+    return this.abFramework.getStrategyMetrics(strategy);
+  }
+
+  /**
+   * Ottiene tutte le metriche
+   */
+  async getAllMetrics() {
+    return this.abFramework.getAllMetrics();
+  }
+
+  /**
+   * Genera report completo
+   */
+  async generateReport(strategy: string): Promise<string> {
+    const currentParams = this.currentParams.get(strategy) || {};
+    return this.optimizer.generateReport(strategy, currentParams);
+  }
+
+  /**
+   * Ottiene suggerimenti di ottimizzazione
+   */
+  async getSuggestions(strategy: string): Promise<OptimizationSuggestion[]> {
+    const currentParams = this.currentParams.get(strategy) || {};
+    return this.optimizer.analyzeAndSuggest(strategy, currentParams);
+  }
+
+  /**
+   * Resetta tutti i feedback (per testing)
+   */
+  async reset(strategy?: string): Promise<void> {
+    await this.storage.clearFeedback(strategy);
+  }
+}

package/src/types.ts

@@ -0,0 +1,139 @@
+import type { SearchHit } from "@eucode/indexer";
+
+export interface Retriever {
+  search(query: string, topK: number): Promise<SearchHit[]>;
+}
+
+export interface KeywordRetriever {
+  search(query: string, topK: number): Promise<SearchHit[]>;
+}
+
+export interface GraderResult {
+  relevant: boolean;
+  score: number;
+}
+
+export interface Grader {
+  grade(query: string, hit: SearchHit): Promise<GraderResult>;
+}
+
+export interface QueryRewriter {
+  rewrite(query: string, context: string): Promise<string>;
+}
+
+export interface CorrectiveRagOptions {
+  retriever: Retriever;
+  grader: Grader;
+  rewriter: QueryRewriter;
+  maxRewrites?: number;
+  minGoodHits?: number;
+  minScore?: number;
+  webFallback?: (query: string) => Promise<SearchHit[]>;
+}
+
+export interface RagCitation {
+  path: string;
+  startLine: number;
+  endLine: number;
+  text: string;
+}
+
+export interface RagAnswer {
+  text: string;
+  citations: RagCitation[];
+  steps: string[];
+  rewrites: number;
+}
+
+// Hybrid Search Types
+export interface HybridSearchOptions {
+  semanticRetriever: Retriever;
+  keywordRetriever: KeywordRetriever;
+  semanticWeight?: number;
+  keywordWeight?: number;
+  rrfK?: number;
+  topK?: number;
+}
+
+export interface HybridSearchResult {
+  hits: SearchHit[];
+  fusionMethod: "rrf" | "weighted";
+  semanticCount: number;
+  keywordCount: number;
+}
+
+// Knowledge Graph Types
+export interface Entity {
+  id: string;
+  name: string;
+  type: string;
+  properties: Record<string, unknown>;
+}
+
+export interface Relation {
+  id: string;
+  source: string;
+  target: string;
+  type: string;
+  properties: Record<string, unknown>;
+}
+
+export interface KnowledgeGraph {
+  entities: Map<string, Entity>;
+  relations: Relation[];
+  addEntity(entity: Entity): void;
+  addRelation(relation: Relation): void;
+  getEntity(id: string): Entity | undefined;
+  getRelations(entityId: string): Relation[];
+  getConnectedEntities(entityId: string, depth?: number): Entity[];
+}
+
+export interface EntityExtractor {
+  extract(text: string): Promise<Entity[]>;
+}
+
+export interface RelationExtractor {
+  extract(text: string, entities: Entity[]): Promise<Relation[]>;
+}
+
+export interface KnowledgeGraphRagOptions {
+  retriever: Retriever;
+  entityExtractor: EntityExtractor;
+  relationExtractor: RelationExtractor;
+  graph: KnowledgeGraph;
+  topK?: number;
+  maxDepth?: number;
+}
+
+export interface KnowledgeGraphRagResult {
+  answer: string;
+  citations: RagCitation[];
+  entities: Entity[];
+  relations: Relation[];
+  graphContext: string;
+}
+
+// Agentic RAG Types
+export interface ReasoningStep {
+  thought: string;
+  action: string;
+  observation: string;
+}
+
+export interface AgenticRagOptions {
+  retriever: Retriever;
+  grader: Grader;
+  rewriter: QueryRewriter;
+  maxIterations?: number;
+  enableSelfReflection?: boolean;
+  enableChainOfThought?: boolean;
+}
+
+export interface AgenticRagResult {
+  answer: string;
+  citations: RagCitation[];
+  reasoningSteps: ReasoningStep[];
+  selfReflection?: string;
+  iterations: number;
+  confidence: number;
+}