@eucoder/rag 0.2.0

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@eucoder/rag",
+  "version": "0.2.0",
+  "description": "Advanced RAG module with Corrective RAG, Hybrid Search, Knowledge Graph, Agentic RAG, and Self-Improving capabilities",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run --passWithNoTests"
+  },
+  "keywords": [
+    "rag",
+    "retrieval-augmented-generation",
+    "corrective-rag",
+    "hybrid-search",
+    "knowledge-graph",
+    "agentic-rag",
+    "self-improving",
+    "ab-testing",
+    "llm"
+  ],
+  "author": "EUCoder Team",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@eucode/indexer": "workspace:*",
+    "@eucode/llm": "workspace:*"
+  },
+  "devDependencies": {
+    "@types/node": "^22.10.5",
+    "typescript": "^5.7.3",
+    "vitest": "^3.0.5"
+  }
+}

package/src/ab-testing.test.ts

@@ -0,0 +1,239 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { ABTestFramework } from "./ab-testing.js";
+import { InMemoryFeedbackStorage } from "./feedback.js";
+import type { RagStrategyConfig } from "./ab-testing.js";
+
+describe("ABTestFramework", () => {
+  let framework: ABTestFramework;
+  let storage: InMemoryFeedbackStorage;
+
+  beforeEach(() => {
+    storage = new InMemoryFeedbackStorage();
+    framework = new ABTestFramework(storage);
+  });
+
+  it("should register strategies", () => {
+    const config: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: { minGoodHits: 2 },
+    };
+
+    framework.registerStrategy(config);
+    // Strategy registered successfully
+    expect(true).toBe(true);
+  });
+
+  it("should select strategy randomly", () => {
+    const configA: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    const configB: RagStrategyConfig = {
+      name: "strategy-b",
+      type: "hybrid",
+      params: {},
+    };
+
+    framework.registerStrategy(configA);
+    framework.registerStrategy(configB);
+
+    const selected = framework.selectStrategy();
+    expect(["strategy-a", "strategy-b"]).toContain(selected.name);
+  });
+
+  it("should throw error when no strategies registered", () => {
+    expect(() => framework.selectStrategy()).toThrow("No strategies registered");
+  });
+
+  it("should record feedback", async () => {
+    const config: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    framework.registerStrategy(config);
+
+    await framework.recordFeedback(
+      "test query",
+      { text: "test answer", citations: [], steps: [], rewrites: 0 },
+      4,
+      0.8,
+      0.7,
+      0.9,
+      "strategy-a"
+    );
+
+    const feedback = await storage.getFeedback("strategy-a");
+    expect(feedback).toHaveLength(1);
+    expect(feedback[0]?.rating).toBe(4);
+  });
+
+  it("should run A/B test with sufficient samples", async () => {
+    const configA: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    const configB: RagStrategyConfig = {
+      name: "strategy-b",
+      type: "hybrid",
+      params: {},
+    };
+
+    framework.registerStrategy(configA);
+    framework.registerStrategy(configB);
+
+    // Add 30 samples for strategy A (lower quality)
+    for (let i = 0; i < 30; i++) {
+      await framework.recordFeedback(
+        `query ${i}`,
+        { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        3,
+        0.6,
+        0.5,
+        0.7,
+        "strategy-a"
+      );
+    }
+
+    // Add 30 samples for strategy B (higher quality)
+    for (let i = 0; i < 30; i++) {
+      await framework.recordFeedback(
+        `query ${i}`,
+        { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        5,
+        0.9,
+        0.85,
+        0.95,
+        "strategy-b"
+      );
+    }
+
+    const result = await framework.runABTest("strategy-a", "strategy-b", 20);
+
+    expect(result.strategyA).toBe("strategy-a");
+    expect(result.strategyB).toBe("strategy-b");
+    expect(result.winner).toBe("B");
+    expect(result.improvement).toBeGreaterThan(0);
+    expect(result.statisticalSignificance).toBeGreaterThan(0);
+    expect(result.recommendation).toContain("B");
+  });
+
+  it("should throw error with insufficient samples", async () => {
+    const configA: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    const configB: RagStrategyConfig = {
+      name: "strategy-b",
+      type: "hybrid",
+      params: {},
+    };
+
+    framework.registerStrategy(configA);
+    framework.registerStrategy(configB);
+
+    // Add only 10 samples (not enough)
+    for (let i = 0; i < 10; i++) {
+      await framework.recordFeedback(
+        `query ${i}`,
+        { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        4,
+        0.8,
+        0.7,
+        0.9,
+        "strategy-a"
+      );
+    }
+
+    await expect(framework.runABTest("strategy-a", "strategy-b", 20))
+      .rejects.toThrow("Insufficient samples");
+  });
+
+  it("should get strategy metrics", async () => {
+    const config: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    framework.registerStrategy(config);
+
+    for (let i = 0; i < 5; i++) {
+      await framework.recordFeedback(
+        `query ${i}`,
+        { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        4,
+        0.8,
+        0.7,
+        0.9,
+        "strategy-a"
+      );
+    }
+
+    const metrics = await framework.getStrategyMetrics("strategy-a");
+    expect(metrics).not.toBeNull();
+    expect(metrics?.totalQueries).toBe(5);
+    expect(metrics?.averageRating).toBe(4);
+  });
+
+  it("should return null for non-existent strategy metrics", async () => {
+    const metrics = await framework.getStrategyMetrics("non-existent");
+    expect(metrics).toBeNull();
+  });
+
+  it("should get all metrics sorted by score", async () => {
+    const configA: RagStrategyConfig = {
+      name: "strategy-a",
+      type: "corrective",
+      params: {},
+    };
+
+    const configB: RagStrategyConfig = {
+      name: "strategy-b",
+      type: "hybrid",
+      params: {},
+    };
+
+    framework.registerStrategy(configA);
+    framework.registerStrategy(configB);
+
+    // Strategy A: lower quality
+    for (let i = 0; i < 5; i++) {
+      await framework.recordFeedback(
+        `query ${i}`,
+        { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        3,
+        0.6,
+        0.5,
+        0.7,
+        "strategy-a"
+      );
+    }
+
+    // Strategy B: higher quality
+    for (let i = 0; i < 5; i++) {
+      await framework.recordFeedback(
+        `query ${i}`,
+        { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        5,
+        0.9,
+        0.85,
+        0.95,
+        "strategy-b"
+      );
+    }
+
+    const allMetrics = await framework.getAllMetrics();
+    expect(allMetrics).toHaveLength(2);
+    expect(allMetrics[0]?.strategy).toBe("strategy-b");
+    expect(allMetrics[0]?.overallScore).toBeGreaterThan(allMetrics[1]?.overallScore || 0);
+  });
+});

package/src/ab-testing.ts

@@ -0,0 +1,214 @@
+import type { RagAnswer } from "./types.js";
+import type { RagFeedback, RagMetrics, ABTestResult, FeedbackStorage } from "./feedback.js";
+import { InMemoryFeedbackStorage } from "./feedback.js";
+import { MetricsCalculator } from "./metrics.js";
+
+/**
+ * Configurazione per una strategia RAG
+ */
+export interface RagStrategyConfig {
+  name: string;
+  type: "corrective" | "hybrid" | "agentic";
+  params: Record<string, unknown>;
+}
+
+/**
+ * A/B Testing framework per confrontare strategie RAG
+ */
+export class ABTestFramework {
+  private storage: FeedbackStorage;
+  private strategies: Map<string, RagStrategyConfig> = new Map();
+
+  constructor(storage?: FeedbackStorage) {
+    this.storage = storage || new InMemoryFeedbackStorage();
+  }
+
+  /**
+   * Registra una strategia per il testing
+   */
+  registerStrategy(config: RagStrategyConfig): void {
+    this.strategies.set(config.name, config);
+  }
+
+  /**
+   * Seleziona casualmente una strategia per A/B testing
+   * Usa weighted random per bilanciare il traffico
+   */
+  selectStrategy(): RagStrategyConfig {
+    const strategies = Array.from(this.strategies.values());
+    if (strategies.length === 0) {
+      throw new Error("No strategies registered for A/B testing");
+    }
+
+    // Simple random selection (potrebbe essere esteso con weighted selection)
+    const index = Math.floor(Math.random() * strategies.length);
+    return strategies[index]!;
+  }
+
+  /**
+   * Registra feedback per una risposta
+   */
+  async recordFeedback(
+    query: string,
+    answer: RagAnswer,
+    rating: number,
+    relevance: number,
+    completeness: number,
+    citationsQuality: number,
+    strategy: string,
+    comments?: string
+  ): Promise<void> {
+    const feedback: RagFeedback = {
+      id: `feedback-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
+      query,
+      answer,
+      rating,
+      relevance,
+      completeness,
+      citationsQuality,
+      comments,
+      timestamp: new Date(),
+      strategy,
+    };
+
+    await this.storage.saveFeedback(feedback);
+  }
+
+  /**
+   * Esegue A/B test tra due strategie
+   */
+  async runABTest(
+    strategyA: string,
+    strategyB: string,
+    minSamples: number = 20
+  ): Promise<ABTestResult> {
+    const feedbacksA = await this.storage.getFeedback(strategyA);
+    const feedbacksB = await this.storage.getFeedback(strategyB);
+
+    if (feedbacksA.length < minSamples || feedbacksB.length < minSamples) {
+      throw new Error(
+        `Insufficient samples for A/B test. Need at least ${minSamples} per strategy. ` +
+        `Got ${feedbacksA.length} for ${strategyA} and ${feedbacksB.length} for ${strategyB}.`
+      );
+    }
+
+    const metricsA = MetricsCalculator.calculateMetrics(feedbacksA, strategyA);
+    const metricsB = MetricsCalculator.calculateMetrics(feedbacksB, strategyB);
+
+    // Determina vincitore basato su overallScore
+    const diff = metricsB.overallScore - metricsA.overallScore;
+    const improvement = Math.abs(diff) / Math.max(metricsA.overallScore, metricsB.overallScore);
+    
+    // Calcola significatività statistica (test t semplificato)
+    const statisticalSignificance = this.calculateStatisticalSignificance(
+      feedbacksA,
+      feedbacksB
+    );
+
+    let winner: "A" | "B" | "tie";
+    if (Math.abs(diff) < 0.02 || statisticalSignificance < 0.7) {
+      winner = "tie";
+    } else {
+      winner = diff > 0 ? "B" : "A";
+    }
+
+    const recommendation = this.generateRecommendation(
+      winner,
+      metricsA,
+      metricsB,
+      improvement,
+      statisticalSignificance
+    );
+
+    return {
+      strategyA,
+      strategyB,
+      winner,
+      metricsA,
+      metricsB,
+      improvement,
+      statisticalSignificance,
+      recommendation,
+    };
+  }
+
+  /**
+   * Calcola significatività statistica (test t semplificato)
+   */
+  private calculateStatisticalSignificance(
+    feedbacksA: RagFeedback[],
+    feedbacksB: RagFeedback[]
+  ): number {
+    const meanA = feedbacksA.reduce((sum, f) => sum + f.relevance, 0) / feedbacksA.length;
+    const meanB = feedbacksB.reduce((sum, f) => sum + f.relevance, 0) / feedbacksB.length;
+    
+    const stdDevA = MetricsCalculator.calculateStandardDeviation(feedbacksA, "relevance");
+    const stdDevB = MetricsCalculator.calculateStandardDeviation(feedbacksB, "relevance");
+    
+    const nA = feedbacksA.length;
+    const nB = feedbacksB.length;
+    
+    // Standard error
+    const se = Math.sqrt(
+      (stdDevA * stdDevA) / nA + (stdDevB * stdDevB) / nB
+    );
+    
+    if (se === 0) return 1;
+    
+    // t-statistic
+    const t = Math.abs(meanB - meanA) / se;
+    
+    // Approssimazione p-value (semplificata)
+    // t > 2.0 ≈ p < 0.05 (significativo)
+    // t > 2.6 ≈ p < 0.01 (molto significativo)
+    const significance = Math.min(1, t / 3);
+    
+    return significance;
+  }
+
+  /**
+   * Genera raccomandazione basata sui risultati del test
+   */
+  private generateRecommendation(
+    winner: "A" | "B" | "tie",
+    metricsA: RagMetrics,
+    metricsB: RagMetrics,
+    improvement: number,
+    significance: number
+  ): string {
+    if (winner === "tie") {
+      return `Le strategie sono equivalenti (differenza: ${(improvement * 100).toFixed(1)}%, significatività: ${(significance * 100).toFixed(0)}%). ` +
+        `Continua il testing per raccogliere più dati.`;
+    }
+
+    const winnerMetrics = winner === "A" ? metricsA : metricsB;
+    const loserMetrics = winner === "A" ? metricsB : metricsA;
+
+    return `Strategia ${winner} vince con ${(improvement * 100).toFixed(1)}% di miglioramento ` +
+      `(significatività: ${(significance * 100).toFixed(0)}%). ` +
+      `Score: ${winnerMetrics.overallScore.toFixed(3)} vs ${loserMetrics.overallScore.toFixed(3)}. ` +
+      `Raccomandazione: adotta la strategia ${winner} come default.`;
+  }
+
+  /**
+   * Ottiene metriche per una strategia specifica
+   */
+  async getStrategyMetrics(strategy: string): Promise<RagMetrics | null> {
+    const feedbacks = await this.storage.getFeedback(strategy);
+    if (feedbacks.length === 0) return null;
+    
+    return MetricsCalculator.calculateMetrics(feedbacks, strategy);
+  }
+
+  /**
+   * Ottiene tutte le metriche delle strategie registrate
+   */
+  async getAllMetrics(): Promise<RagMetrics[]> {
+    const metrics: RagMetrics[] = [];
+    for (const strategyName of this.strategies.keys()) {
+      const m = await this.getStrategyMetrics(strategyName);
+      if (m) metrics.push(m);
+    }
+    return metrics.sort((a, b) => b.overallScore - a.overallScore);
+  }
+}

package/src/agentic-rag.test.ts

@@ -0,0 +1,201 @@
+import { describe, it, expect } from "vitest";
+import { AgenticRag } from "./agentic-rag.js";
+import type { SearchHit } from "@eucode/indexer";
+import type { Retriever, Grader, QueryRewriter } from "./types.js";
+
+class MockRetriever implements Retriever {
+  private callCount = 0;
+  
+  async search(_query: string, topK: number): Promise<SearchHit[]> {
+    this.callCount++;
+    
+    // First call returns poor results, second call returns good results
+    if (this.callCount === 1) {
+      return [
+        {
+          id: "hit-1",
+          path: "src/unrelated.ts",
+          startLine: 1,
+          endLine: 10,
+          text: "Unrelated content",
+          score: 0.3,
+        },
+      ].slice(0, topK);
+    }
+    
+    return [
+      {
+        id: "hit-2",
+        path: "src/auth.ts",
+        startLine: 10,
+        endLine: 20,
+        text: "Authentication implementation",
+        score: 0.9,
+      },
+      {
+        id: "hit-3",
+        path: "src/login.ts",
+        startLine: 5,
+        endLine: 15,
+        text: "Login logic",
+        score: 0.85,
+      },
+    ].slice(0, topK);
+  }
+}
+
+class MockGrader implements Grader {
+  async grade(_query: string, hit: SearchHit) {
+    return {
+      relevant: hit.score > 0.5,
+      score: hit.score,
+    };
+  }
+}
+
+class MockRewriter implements QueryRewriter {
+  async rewrite(query: string, _context: string): Promise<string> {
+    return `${query} (improved)`;
+  }
+}
+
+describe("AgenticRag", () => {
+  it("should perform iterative search with reasoning", async () => {
+    const agentic = new AgenticRag({
+      retriever: new MockRetriever(),
+      grader: new MockGrader(),
+      rewriter: new MockRewriter(),
+      maxIterations: 3,
+      enableSelfReflection: true,
+      enableChainOfThought: true,
+    });
+
+    const result = await agentic.search("authentication");
+
+    expect(result.iterations).toBeGreaterThan(0);
+    expect(result.reasoningSteps.length).toBeGreaterThan(0);
+    expect(result.answer).toContain("Processo di Ragionamento");
+    expect(result.citations.length).toBeGreaterThan(0);
+  });
+
+  it("should stop early when results are good enough", async () => {
+    const goodRetriever: Retriever = {
+      async search(_query: string, topK: number): Promise<SearchHit[]> {
+        return [
+          {
+            id: "hit-1",
+            path: "src/auth.ts",
+            startLine: 10,
+            endLine: 20,
+            text: "Authentication implementation",
+            score: 0.9,
+          },
+          {
+            id: "hit-2",
+            path: "src/login.ts",
+            startLine: 5,
+            endLine: 15,
+            text: "Login logic",
+            score: 0.85,
+          },
+          {
+            id: "hit-3",
+            path: "src/session.ts",
+            startLine: 1,
+            endLine: 10,
+            text: "Session management",
+            score: 0.8,
+          },
+        ].slice(0, topK);
+      },
+    };
+
+    const agentic = new AgenticRag({
+      retriever: goodRetriever,
+      grader: new MockGrader(),
+      rewriter: new MockRewriter(),
+      maxIterations: 5,
+      enableSelfReflection: true,
+    });
+
+    const result = await agentic.search("authentication");
+
+    // Should stop after 1 iteration because results are good
+    expect(result.iterations).toBe(1);
+    expect(result.confidence).toBeGreaterThan(0.7);
+  });
+
+  it("should rewrite query when results are poor", async () => {
+    const poorRetriever: Retriever = {
+      async search(_query: string, topK: number): Promise<SearchHit[]> {
+        return [
+          {
+            id: "hit-1",
+            path: "src/unrelated.ts",
+            startLine: 1,
+            endLine: 10,
+            text: "Unrelated content",
+            score: 0.2,
+          },
+        ].slice(0, topK);
+      },
+    };
+
+    const agentic = new AgenticRag({
+      retriever: poorRetriever,
+      grader: new MockGrader(),
+      rewriter: new MockRewriter(),
+      maxIterations: 3,
+      enableSelfReflection: true,
+    });
+
+    const result = await agentic.search("authentication");
+
+    // Should iterate multiple times trying to improve
+    expect(result.iterations).toBeGreaterThan(1);
+    expect(result.confidence).toBeLessThan(0.5);
+  });
+
+  it("should include self-reflection when enabled", async () => {
+    const agentic = new AgenticRag({
+      retriever: new MockRetriever(),
+      grader: new MockGrader(),
+      rewriter: new MockRewriter(),
+      maxIterations: 2,
+      enableSelfReflection: true,
+    });
+
+    const result = await agentic.search("authentication");
+
+    expect(result.selfReflection).toBeDefined();
+    expect(result.selfReflection).toContain("Valutazione Finale");
+  });
+
+  it("should skip self-reflection when disabled", async () => {
+    const agentic = new AgenticRag({
+      retriever: new MockRetriever(),
+      grader: new MockGrader(),
+      rewriter: new MockRewriter(),
+      maxIterations: 2,
+      enableSelfReflection: false,
+    });
+
+    const result = await agentic.search("authentication");
+
+    expect(result.selfReflection).toBeUndefined();
+  });
+
+  it("should skip chain of thought when disabled", async () => {
+    const agentic = new AgenticRag({
+      retriever: new MockRetriever(),
+      grader: new MockGrader(),
+      rewriter: new MockRewriter(),
+      maxIterations: 2,
+      enableChainOfThought: false,
+    });
+
+    const result = await agentic.search("authentication");
+
+    expect(result.answer).not.toContain("Processo di Ragionamento");
+  });
+});