@eucoder/rag 0.2.0

package/src/feedback.test.ts

@@ -0,0 +1,227 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { InMemoryFeedbackStorage } from "./feedback.js";
+import { MetricsCalculator } from "./metrics.js";
+import type { RagFeedback } from "./feedback.js";
+
+describe("InMemoryFeedbackStorage", () => {
+  let storage: InMemoryFeedbackStorage;
+
+  beforeEach(() => {
+    storage = new InMemoryFeedbackStorage();
+  });
+
+  it("should save and retrieve feedback", async () => {
+    const feedback: RagFeedback = {
+      id: "test-1",
+      query: "test query",
+      answer: {
+        text: "test answer",
+        citations: [],
+        steps: [],
+        rewrites: 0,
+      },
+      rating: 4,
+      relevance: 0.8,
+      completeness: 0.7,
+      citationsQuality: 0.9,
+      timestamp: new Date(),
+      strategy: "strategy-a",
+    };
+
+    await storage.saveFeedback(feedback);
+    const retrieved = await storage.getFeedback("strategy-a");
+
+    expect(retrieved).toHaveLength(1);
+    expect(retrieved[0]?.id).toBe("test-1");
+  });
+
+  it("should filter feedback by strategy", async () => {
+    const feedback1: RagFeedback = {
+      id: "test-1",
+      query: "query 1",
+      answer: { text: "answer 1", citations: [], steps: [], rewrites: 0 },
+      rating: 4,
+      relevance: 0.8,
+      completeness: 0.7,
+      citationsQuality: 0.9,
+      timestamp: new Date(),
+      strategy: "strategy-a",
+    };
+
+    const feedback2: RagFeedback = {
+      id: "test-2",
+      query: "query 2",
+      answer: { text: "answer 2", citations: [], steps: [], rewrites: 0 },
+      rating: 5,
+      relevance: 0.9,
+      completeness: 0.8,
+      citationsQuality: 0.95,
+      timestamp: new Date(),
+      strategy: "strategy-b",
+    };
+
+    await storage.saveFeedback(feedback1);
+    await storage.saveFeedback(feedback2);
+
+    const retrievedA = await storage.getFeedback("strategy-a");
+    const retrievedB = await storage.getFeedback("strategy-b");
+
+    expect(retrievedA).toHaveLength(1);
+    expect(retrievedB).toHaveLength(1);
+    expect(retrievedA[0]?.strategy).toBe("strategy-a");
+    expect(retrievedB[0]?.strategy).toBe("strategy-b");
+  });
+
+  it("should limit feedback results", async () => {
+    for (let i = 0; i < 10; i++) {
+      await storage.saveFeedback({
+        id: `test-${i}`,
+        query: `query ${i}`,
+        answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        rating: 4,
+        relevance: 0.8,
+        completeness: 0.7,
+        citationsQuality: 0.9,
+        timestamp: new Date(),
+        strategy: "strategy-a",
+      });
+    }
+
+    const retrieved = await storage.getFeedback("strategy-a", 5);
+    expect(retrieved).toHaveLength(5);
+  });
+
+  it("should clear feedback", async () => {
+    await storage.saveFeedback({
+      id: "test-1",
+      query: "query 1",
+      answer: { text: "answer 1", citations: [], steps: [], rewrites: 0 },
+      rating: 4,
+      relevance: 0.8,
+      completeness: 0.7,
+      citationsQuality: 0.9,
+      timestamp: new Date(),
+      strategy: "strategy-a",
+    });
+
+    await storage.clearFeedback("strategy-a");
+    const retrieved = await storage.getFeedback("strategy-a");
+    expect(retrieved).toHaveLength(0);
+  });
+});
+
+describe("MetricsCalculator", () => {
+  it("should calculate metrics from feedback", () => {
+    const feedbacks: RagFeedback[] = [
+      {
+        id: "test-1",
+        query: "query 1",
+        answer: { text: "answer 1", citations: [], steps: [], rewrites: 0 },
+        rating: 4,
+        relevance: 0.8,
+        completeness: 0.7,
+        citationsQuality: 0.9,
+        timestamp: new Date(),
+        strategy: "strategy-a",
+      },
+      {
+        id: "test-2",
+        query: "query 2",
+        answer: { text: "answer 2", citations: [], steps: [], rewrites: 0 },
+        rating: 5,
+        relevance: 0.9,
+        completeness: 0.8,
+        citationsQuality: 0.95,
+        timestamp: new Date(),
+        strategy: "strategy-a",
+      },
+    ];
+
+    const metrics = MetricsCalculator.calculateMetrics(feedbacks, "strategy-a");
+
+    expect(metrics.strategy).toBe("strategy-a");
+    expect(metrics.totalQueries).toBe(2);
+    expect(metrics.averageRating).toBe(4.5);
+    expect(metrics.averageRelevance).toBeCloseTo(0.85, 10);
+    expect(metrics.averageCompleteness).toBeCloseTo(0.75, 10);
+    expect(metrics.averageCitationsQuality).toBeCloseTo(0.925, 10);
+    expect(metrics.overallScore).toBeGreaterThan(0);
+    expect(metrics.confidence).toBeGreaterThan(0);
+  });
+
+  it("should return zero metrics for empty feedback", () => {
+    const metrics = MetricsCalculator.calculateMetrics([], "strategy-a");
+
+    expect(metrics.totalQueries).toBe(0);
+    expect(metrics.averageRating).toBe(0);
+    expect(metrics.overallScore).toBe(0);
+    expect(metrics.confidence).toBe(0);
+  });
+
+  it("should calculate standard deviation", () => {
+    const feedbacks: RagFeedback[] = [
+      {
+        id: "test-1",
+        query: "query 1",
+        answer: { text: "answer 1", citations: [], steps: [], rewrites: 0 },
+        rating: 3,
+        relevance: 0.6,
+        completeness: 0.5,
+        citationsQuality: 0.7,
+        timestamp: new Date(),
+        strategy: "strategy-a",
+      },
+      {
+        id: "test-2",
+        query: "query 2",
+        answer: { text: "answer 2", citations: [], steps: [], rewrites: 0 },
+        rating: 5,
+        relevance: 0.9,
+        completeness: 0.8,
+        citationsQuality: 0.95,
+        timestamp: new Date(),
+        strategy: "strategy-a",
+      },
+    ];
+
+    const stdDev = MetricsCalculator.calculateStandardDeviation(feedbacks, "relevance");
+    expect(stdDev).toBeGreaterThan(0);
+  });
+
+  it("should identify trend", () => {
+    const feedbacks: RagFeedback[] = [];
+    
+    // Older feedback with lower relevance
+    for (let i = 0; i < 10; i++) {
+      feedbacks.push({
+        id: `old-${i}`,
+        query: `query ${i}`,
+        answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        rating: 3,
+        relevance: 0.5,
+        completeness: 0.5,
+        citationsQuality: 0.5,
+        timestamp: new Date(Date.now() - 1000000),
+        strategy: "strategy-a",
+      });
+    }
+
+    // Recent feedback with higher relevance
+    for (let i = 0; i < 10; i++) {
+      feedbacks.push({
+        id: `new-${i}`,
+        query: `query ${i}`,
+        answer: { text: `answer ${i}`, citations: [], steps: [], rewrites: 0 },
+        rating: 5,
+        relevance: 0.9,
+        completeness: 0.9,
+        citationsQuality: 0.9,
+        timestamp: new Date(),
+        strategy: "strategy-a",
+      });
+    }
+
+    const trend = MetricsCalculator.identifyTrend(feedbacks, 10);
+    expect(trend).toBe("improving");
+  });
+});

package/src/feedback.ts

@@ -0,0 +1,118 @@
+import type { RagAnswer } from "./types.js";
+
+/**
+ * Feedback per una risposta RAG
+ */
+export interface RagFeedback {
+  id: string;
+  query: string;
+  answer: RagAnswer;
+  rating: number; // 1-5
+  relevance: number; // 0-1
+  completeness: number; // 0-1
+  citationsQuality: number; // 0-1
+  comments?: string;
+  timestamp: Date;
+  strategy?: string; // Nome della strategia usata (per A/B testing)
+}
+
+/**
+ * Metriche aggregate per una strategia RAG
+ */
+export interface RagMetrics {
+  strategy: string;
+  totalQueries: number;
+  averageRating: number;
+  averageRelevance: number;
+  averageCompleteness: number;
+  averageCitationsQuality: number;
+  overallScore: number; // Media pesata delle metriche
+  confidence: number; // 0-1, basato sul numero di sample
+}
+
+/**
+ * Risultato A/B test
+ */
+export interface ABTestResult {
+  strategyA: string;
+  strategyB: string;
+  winner: "A" | "B" | "tie";
+  metricsA: RagMetrics;
+  metricsB: RagMetrics;
+  improvement: number; // Percentuale di miglioramento
+  statisticalSignificance: number; // 0-1
+  recommendation: string;
+}
+
+/**
+ * Suggerimento di ottimizzazione
+ */
+export interface OptimizationSuggestion {
+  type: "parameter" | "strategy" | "prompt";
+  target: string;
+  current: unknown;
+  suggested: unknown;
+  expectedImprovement: number; // 0-1
+  confidence: number; // 0-1
+  reasoning: string;
+}
+
+/**
+ * Storage per feedback e metriche
+ */
+export interface FeedbackStorage {
+  saveFeedback(feedback: RagFeedback): Promise<void>;
+  getFeedback(strategy?: string, limit?: number): Promise<RagFeedback[]>;
+  getMetrics(strategy: string): Promise<RagMetrics | null>;
+  saveMetrics(metrics: RagMetrics): Promise<void>;
+  clearFeedback(strategy?: string): Promise<void>;
+}
+
+/**
+ * In-memory implementation di FeedbackStorage
+ */
+export class InMemoryFeedbackStorage implements FeedbackStorage {
+  private feedbacks: Map<string, RagFeedback[]> = new Map();
+  private metrics: Map<string, RagMetrics> = new Map();
+
+  async saveFeedback(feedback: RagFeedback): Promise<void> {
+    const strategy = feedback.strategy || "default";
+    if (!this.feedbacks.has(strategy)) {
+      this.feedbacks.set(strategy, []);
+    }
+    this.feedbacks.get(strategy)!.push(feedback);
+  }
+
+  async getFeedback(strategy?: string, limit?: number): Promise<RagFeedback[]> {
+    if (strategy) {
+      const feedbacks = this.feedbacks.get(strategy) || [];
+      return limit ? feedbacks.slice(-limit) : feedbacks;
+    }
+    
+    // Tutti i feedback
+    const all: RagFeedback[] = [];
+    for (const feedbacks of this.feedbacks.values()) {
+      all.push(...feedbacks);
+    }
+    all.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
+    return limit ? all.slice(-limit) : all;
+  }
+
+  async getMetrics(strategy: string): Promise<RagMetrics | null> {
+    return this.metrics.get(strategy) || null;
+  }
+
+  async saveMetrics(metrics: RagMetrics): Promise<void> {
+    this.metrics.set(metrics.strategy, metrics);
+  }
+
+  async clearFeedback(strategy?: string): Promise<void> {
+    if (strategy) {
+      this.feedbacks.delete(strategy);
+      this.metrics.delete(strategy);
+    } else {
+      this.feedbacks.clear();
+      this.metrics.clear();
+    }
+  }
+}

package/src/hybrid-search.test.ts

@@ -0,0 +1,107 @@
+import { describe, it, expect } from "vitest";
+import { HybridSearchRag } from "./hybrid-search.js";
+import type { SearchHit } from "@eucode/indexer";
+import type { Retriever, KeywordRetriever } from "./types.js";
+
+class MockSemanticRetriever implements Retriever {
+  async search(_query: string, topK: number): Promise<SearchHit[]> {
+    return [
+      {
+        id: "hit-1",
+        path: "src/auth.ts",
+        startLine: 10,
+        endLine: 20,
+        text: "Authentication logic",
+        score: 0.9,
+      },
+      {
+        id: "hit-2",
+        path: "src/login.ts",
+        startLine: 5,
+        endLine: 15,
+        text: "Login implementation",
+        score: 0.8,
+      },
+    ].slice(0, topK);
+  }
+}
+
+class MockKeywordRetriever implements KeywordRetriever {
+  async search(_query: string, topK: number): Promise<SearchHit[]> {
+    return [
+      {
+        id: "hit-2",
+        path: "src/login.ts",
+        startLine: 5,
+        endLine: 15,
+        text: "Login implementation",
+        score: 0.85,
+      },
+      {
+        id: "hit-3",
+        path: "src/session.ts",
+        startLine: 1,
+        endLine: 10,
+        text: "Session management",
+        score: 0.7,
+      },
+    ].slice(0, topK);
+  }
+}
+
+describe("HybridSearchRag", () => {
+  it("should combine semantic and keyword results using RRF", async () => {
+    const hybrid = new HybridSearchRag({
+      semanticRetriever: new MockSemanticRetriever(),
+      keywordRetriever: new MockKeywordRetriever(),
+      topK: 5,
+    });
+
+    const result = await hybrid.search("authentication");
+
+    expect(result.hits.length).toBeGreaterThan(0);
+    expect(result.fusionMethod).toBe("rrf");
+    expect(result.semanticCount).toBe(2);
+    expect(result.keywordCount).toBe(2);
+  });
+
+  it("should rank overlapping results higher", async () => {
+    const hybrid = new HybridSearchRag({
+      semanticRetriever: new MockSemanticRetriever(),
+      keywordRetriever: new MockKeywordRetriever(),
+      topK: 5,
+    });
+
+    const result = await hybrid.search("login");
+
+    // src/login.ts appears in both, should be ranked higher
+    const loginHit = result.hits.find((h) => h.path === "src/login.ts");
+    expect(loginHit).toBeDefined();
+    expect(result.hits[0]?.path).toBe("src/login.ts");
+  });
+
+  it("should respect custom weights", async () => {
+    const hybridSemantic = new HybridSearchRag({
+      semanticRetriever: new MockSemanticRetriever(),
+      keywordRetriever: new MockKeywordRetriever(),
+      semanticWeight: 0.9,
+      keywordWeight: 0.1,
+      topK: 5,
+    });
+
+    const resultSemantic = await hybridSemantic.search("auth");
+
+    const hybridKeyword = new HybridSearchRag({
+      semanticRetriever: new MockSemanticRetriever(),
+      keywordRetriever: new MockKeywordRetriever(),
+      semanticWeight: 0.1,
+      keywordWeight: 0.9,
+      topK: 5,
+    });
+
+    const resultKeyword = await hybridKeyword.search("auth");
+
+    // Results should differ based on weights
+    expect(resultSemantic.hits[0]?.score).not.toBe(resultKeyword.hits[0]?.score);
+  });
+});

package/src/hybrid-search.ts

@@ -0,0 +1,86 @@
+import type { SearchHit } from "@eucode/indexer";
+import type { HybridSearchOptions, HybridSearchResult } from "./types.js";
+
+/**
+ * Hybrid Search RAG - Combina semantic search e keyword search usando RRF (Reciprocal Rank Fusion)
+ * 
+ * RRF Formula: score(d) = Σ 1/(k + rank_i(d))
+ * dove k è una costante (tipicamente 60) e rank_i è la posizione del documento nella lista i
+ */
+export class HybridSearchRag {
+  private options: Required<HybridSearchOptions>;
+
+  constructor(options: HybridSearchOptions) {
+    this.options = {
+      semanticWeight: options.semanticWeight ?? 0.5,
+      keywordWeight: options.keywordWeight ?? 0.5,
+      rrfK: options.rrfK ?? 60,
+      topK: options.topK ?? 10,
+      ...options,
+    };
+  }
+
+  async search(query: string, topK?: number): Promise<HybridSearchResult> {
+    const limit = topK ?? this.options.topK;
+
+    // Esegui entrambe le ricerche in parallelo
+    const [semanticHits, keywordHits] = await Promise.all([
+      this.options.semanticRetriever.search(query, limit * 2),
+      this.options.keywordRetriever.search(query, limit * 2),
+    ]);
+
+    // Applica RRF fusion
+    const fusedHits = this.reciprocalRankFusion(semanticHits, keywordHits);
+
+    // Prendi i top K risultati
+    const finalHits = fusedHits.slice(0, limit);
+
+    return {
+      hits: finalHits,
+      fusionMethod: "rrf",
+      semanticCount: semanticHits.length,
+      keywordCount: keywordHits.length,
+    };
+  }
+
+  private reciprocalRankFusion(
+    semanticHits: SearchHit[],
+    keywordHits: SearchHit[]
+  ): SearchHit[] {
+    const scoreMap = new Map<string, { hit: SearchHit; score: number }>();
+    const k = this.options.rrfK;
+
+    // Calcola RRF score per semantic hits
+    semanticHits.forEach((hit, index) => {
+      const key = `${hit.path}:${hit.startLine}`;
+      const rrfScore = 1 / (k + index + 1);
+      
+      const existing = scoreMap.get(key);
+      if (existing) {
+        existing.score += rrfScore * this.options.semanticWeight;
+      } else {
+        scoreMap.set(key, { hit, score: rrfScore * this.options.semanticWeight });
+      }
+    });
+
+    // Calcola RRF score per keyword hits
+    keywordHits.forEach((hit, index) => {
+      const key = `${hit.path}:${hit.startLine}`;
+      const rrfScore = 1 / (k + index + 1);
+      
+      const existing = scoreMap.get(key);
+      if (existing) {
+        existing.score += rrfScore * this.options.keywordWeight;
+      } else {
+        scoreMap.set(key, { hit, score: rrfScore * this.options.keywordWeight });
+      }
+    });
+
+    // Ordina per score decrescente
+    const sorted = Array.from(scoreMap.values())
+      .sort((a, b) => b.score - a.score)
+      .map(({ hit, score }) => ({ ...hit, score }));
+
+    return sorted;
+  }
+}

package/src/index.ts

@@ -0,0 +1,57 @@
+export { CorrectiveRag, createCorrectiveRag } from "./corrective-rag.js";
+export { LlmGrader, LlmQueryRewriter } from "./llm-grader.js";
+export { HybridSearchRag } from "./hybrid-search.js";
+export { KnowledgeGraphRag, InMemoryKnowledgeGraph } from "./knowledge-graph.js";
+export { AgenticRag } from "./agentic-rag.js";
+
+// Fase 4: Self-Improving
+export { InMemoryFeedbackStorage } from "./feedback.js";
+export { MetricsCalculator } from "./metrics.js";
+export { ABTestFramework } from "./ab-testing.js";
+export { ParameterOptimizer } from "./optimizer.js";
+export { SelfImprovingRag } from "./self-improving.js";
+
+export type { LlmProvider } from "./llm-grader.js";
+export type {
+  Retriever,
+  KeywordRetriever,
+  Grader,
+  GraderResult,
+  QueryRewriter,
+  CorrectiveRagOptions,
+  RagAnswer,
+  RagCitation,
+  HybridSearchOptions,
+  HybridSearchResult,
+  Entity,
+  Relation,
+  KnowledgeGraph,
+  EntityExtractor,
+  RelationExtractor,
+  KnowledgeGraphRagOptions,
+  KnowledgeGraphRagResult,
+  ReasoningStep,
+  AgenticRagOptions,
+  AgenticRagResult,
+} from "./types.js";
+
+// Fase 4: Self-Improving Types
+export type {
+  RagFeedback,
+  RagMetrics,
+  ABTestResult,
+  OptimizationSuggestion,
+  FeedbackStorage,
+} from "./feedback.js";
+
+export type {
+  RagStrategyConfig,
+} from "./ab-testing.js";
+
+export type {
+  OptimizableParams,
+} from "./optimizer.js";
+
+export type {
+  SelfImprovingConfig,
+} from "./self-improving.js";