@lov3kaizen/agentsea-evaluate 0.5.0

package/dist/chunk-5JRYKRSE.mjs

@@ -0,0 +1,2791 @@
+// src/evaluation/metrics/BaseMetric.ts
+var BaseMetric = class {
+  name;
+  threshold;
+  weight;
+  scoreRange;
+  constructor(config = {}) {
+    this.name = config.name ?? "";
+    this.threshold = config.threshold ?? 0.5;
+    this.weight = config.weight ?? 1;
+    this.scoreRange = config.scoreRange ?? { min: 0, max: 1 };
+  }
+  /**
+   * Initialize name from type (called by subclasses after super())
+   */
+  initName(config) {
+    if (!this.name && config.name) {
+      this.name = config.name;
+    } else if (!this.name) {
+      this.name = this.type;
+    }
+  }
+  /**
+   * Check if score passes threshold
+   */
+  passes(score) {
+    return score >= this.threshold;
+  }
+  /**
+   * Normalize score to 0-1 range
+   */
+  normalizeScore(score) {
+    const { min, max } = this.scoreRange;
+    if (max === min) return score >= max ? 1 : 0;
+    return Math.max(0, Math.min(1, (score - min) / (max - min)));
+  }
+  /**
+   * Create a metric result
+   */
+  createResult(score, explanation, details) {
+    return {
+      metric: this.type,
+      score,
+      explanation,
+      details
+    };
+  }
+};
+
+// src/evaluation/metrics/Accuracy.ts
+var Accuracy = class extends BaseMetric {
+  type = "accuracy";
+  matchType;
+  caseSensitive;
+  ignoreWhitespace;
+  constructor(config = { type: "fuzzy" }) {
+    super(config);
+    this.matchType = config.type ?? "fuzzy";
+    this.caseSensitive = config.caseSensitive ?? false;
+    this.ignoreWhitespace = config.ignoreWhitespace ?? true;
+    this.initName(config);
+  }
+  async evaluate(input) {
+    if (!input.expectedOutput) {
+      return this.createResult(
+        1,
+        "No expected output provided, skipping accuracy check",
+        { skipped: true }
+      );
+    }
+    const output = this.preprocess(input.output);
+    const expected = this.preprocess(input.expectedOutput);
+    let score;
+    let explanation;
+    switch (this.matchType) {
+      case "exact":
+        score = output === expected ? 1 : 0;
+        explanation = score === 1 ? "Output exactly matches expected output" : "Output does not match expected output";
+        break;
+      case "fuzzy":
+        score = this.calculateFuzzySimilarity(output, expected);
+        explanation = `Fuzzy similarity: ${(score * 100).toFixed(1)}%`;
+        break;
+      case "semantic":
+        score = this.calculateFuzzySimilarity(output, expected);
+        explanation = `Semantic similarity (approximated): ${(score * 100).toFixed(1)}%`;
+        break;
+      default:
+        score = 0;
+        explanation = "Unknown match type";
+    }
+    return Promise.resolve(
+      this.createResult(score, explanation, {
+        matchType: this.matchType,
+        outputLength: output.length,
+        expectedLength: expected.length
+      })
+    );
+  }
+  /**
+   * Preprocess text for comparison
+   */
+  preprocess(text) {
+    let processed = text;
+    if (!this.caseSensitive) {
+      processed = processed.toLowerCase();
+    }
+    if (this.ignoreWhitespace) {
+      processed = processed.replace(/\s+/g, " ").trim();
+    }
+    return processed;
+  }
+  /**
+   * Calculate fuzzy similarity using Levenshtein distance
+   */
+  calculateFuzzySimilarity(a, b) {
+    if (a === b) return 1;
+    if (a.length === 0 || b.length === 0) return 0;
+    const matrix = [];
+    for (let i = 0; i <= a.length; i++) {
+      matrix[i] = [i];
+    }
+    for (let j = 0; j <= b.length; j++) {
+      matrix[0][j] = j;
+    }
+    for (let i = 1; i <= a.length; i++) {
+      for (let j = 1; j <= b.length; j++) {
+        const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+        matrix[i][j] = Math.min(
+          matrix[i - 1][j] + 1,
+          matrix[i][j - 1] + 1,
+          matrix[i - 1][j - 1] + cost
+        );
+      }
+    }
+    const maxLen = Math.max(a.length, b.length);
+    return 1 - matrix[a.length][b.length] / maxLen;
+  }
+};
+function createAccuracyMetric(config) {
+  return new Accuracy(config);
+}
+
+// src/evaluation/metrics/Relevance.ts
+var Relevance = class extends BaseMetric {
+  type = "relevance";
+  provider;
+  model;
+  prompt;
+  constructor(config = {}) {
+    super(config);
+    this.model = config.model ?? "claude-sonnet-4-20250514";
+    this.prompt = config.prompt;
+    this.initName(config);
+  }
+  /**
+   * Set the LLM provider for evaluation
+   */
+  setProvider(provider) {
+    this.provider = provider;
+  }
+  async evaluate(input) {
+    if (!this.provider) {
+      return this.evaluateHeuristic(input);
+    }
+    return this.evaluateWithLLM(input);
+  }
+  /**
+   * Evaluate relevance using heuristics
+   */
+  evaluateHeuristic(input) {
+    const questionWords = this.extractKeywords(input.input);
+    const answerWords = this.extractKeywords(input.output);
+    if (questionWords.length === 0) {
+      return this.createResult(1, "No keywords in input to match", {
+        method: "heuristic"
+      });
+    }
+    let matches = 0;
+    for (const word of questionWords) {
+      if (answerWords.some((aw) => aw.includes(word) || word.includes(aw))) {
+        matches++;
+      }
+    }
+    const keywordOverlap = matches / questionWords.length;
+    const questionType = this.detectQuestionType(input.input);
+    const typeRelevance = this.checkAnswerType(input.output, questionType);
+    const score = keywordOverlap * 0.6 + typeRelevance * 0.4;
+    return this.createResult(
+      score,
+      `Keyword overlap: ${(keywordOverlap * 100).toFixed(1)}%, Type relevance: ${(typeRelevance * 100).toFixed(1)}%`,
+      {
+        method: "heuristic",
+        keywordOverlap,
+        typeRelevance,
+        questionType
+      }
+    );
+  }
+  /**
+   * Evaluate relevance using LLM
+   */
+  async evaluateWithLLM(input) {
+    const prompt = this.prompt ?? this.getDefaultPrompt();
+    const messages = [
+      {
+        role: "user",
+        content: prompt.replace("{input}", input.input).replace("{output}", input.output)
+      }
+    ];
+    try {
+      const response = await this.provider.complete({
+        model: this.model,
+        messages,
+        temperature: 0
+      });
+      const scoreMatch = response.content.match(/Score:\s*(\d+(?:\.\d+)?)/i);
+      const score = scoreMatch ? parseFloat(scoreMatch[1]) / 5 : 0.5;
+      return this.createResult(score, response.content, {
+        method: "llm",
+        model: this.model
+      });
+    } catch (error) {
+      const result = this.evaluateHeuristic(input);
+      return {
+        ...result,
+        details: {
+          ...result.details,
+          llmError: error.message
+        }
+      };
+    }
+  }
+  /**
+   * Get default evaluation prompt
+   */
+  getDefaultPrompt() {
+    return `Evaluate how relevant this response is to the question.
+
+Question: {input}
+Response: {output}
+
+Rate the relevance on a scale of 1-5 where:
+1 = Completely irrelevant
+2 = Mostly irrelevant with some related content
+3 = Somewhat relevant but misses key points
+4 = Mostly relevant with minor gaps
+5 = Completely relevant and addresses the question
+
+Provide your rating as "Score: X" followed by a brief explanation.`;
+  }
+  /**
+   * Extract keywords from text
+   */
+  extractKeywords(text) {
+    const stopWords = /* @__PURE__ */ new Set([
+      "a",
+      "an",
+      "the",
+      "is",
+      "are",
+      "was",
+      "were",
+      "be",
+      "been",
+      "being",
+      "have",
+      "has",
+      "had",
+      "do",
+      "does",
+      "did",
+      "will",
+      "would",
+      "could",
+      "should",
+      "may",
+      "might",
+      "must",
+      "shall",
+      "can",
+      "need",
+      "dare",
+      "to",
+      "of",
+      "in",
+      "for",
+      "on",
+      "with",
+      "at",
+      "by",
+      "from",
+      "as",
+      "into",
+      "through",
+      "during",
+      "before",
+      "after",
+      "above",
+      "below",
+      "between",
+      "under",
+      "again",
+      "further",
+      "then",
+      "once",
+      "here",
+      "there",
+      "when",
+      "where",
+      "why",
+      "how",
+      "all",
+      "each",
+      "few",
+      "more",
+      "most",
+      "other",
+      "some",
+      "such",
+      "no",
+      "nor",
+      "not",
+      "only",
+      "own",
+      "same",
+      "so",
+      "than",
+      "too",
+      "very",
+      "just",
+      "and",
+      "but",
+      "if",
+      "or",
+      "because",
+      "until",
+      "while",
+      "it",
+      "this",
+      "that",
+      "these",
+      "those",
+      "i",
+      "me",
+      "my",
+      "we",
+      "you",
+      "what",
+      "which",
+      "who",
+      "whom",
+      "please",
+      "thank",
+      "thanks"
+    ]);
+    return text.toLowerCase().replace(/[^\w\s]/g, " ").split(/\s+/).filter((word) => word.length > 2 && !stopWords.has(word));
+  }
+  /**
+   * Detect question type
+   */
+  detectQuestionType(question) {
+    const lower = question.toLowerCase();
+    if (lower.startsWith("what") || lower.includes("what ")) return "what";
+    if (lower.startsWith("how") || lower.includes("how ")) return "how";
+    if (lower.startsWith("why") || lower.includes("why ")) return "why";
+    if (lower.startsWith("when") || lower.includes("when ")) return "when";
+    if (lower.startsWith("where") || lower.includes("where ")) return "where";
+    if (lower.startsWith("who") || lower.includes("who ")) return "who";
+    if (lower.startsWith("is ") || lower.startsWith("are ") || lower.startsWith("do ") || lower.startsWith("does ") || lower.startsWith("can ") || lower.startsWith("will ")) {
+      return "yes_no";
+    }
+    return "other";
+  }
+  /**
+   * Check if answer type matches question type
+   */
+  checkAnswerType(answer, questionType) {
+    const lower = answer.toLowerCase();
+    switch (questionType) {
+      case "yes_no":
+        if (lower.includes("yes") || lower.includes("no")) return 1;
+        return 0.5;
+      case "how":
+        if (lower.includes("by ") || lower.includes("using ") || lower.includes("step"))
+          return 1;
+        return 0.6;
+      case "why":
+        if (lower.includes("because") || lower.includes("since") || lower.includes("reason"))
+          return 1;
+        return 0.6;
+      case "when":
+        if (/\d{4}|\d{1,2}\/\d{1,2}|today|yesterday|tomorrow|year|month|day/.test(
+          lower
+        ))
+          return 1;
+        return 0.6;
+      case "where":
+        if (/at |in |on |located|place|location/.test(lower)) return 1;
+        return 0.6;
+      case "who":
+        if (/[A-Z][a-z]+\s+[A-Z][a-z]+/.test(answer)) return 1;
+        return 0.6;
+      default:
+        return 0.7;
+    }
+  }
+};
+function createRelevanceMetric(config) {
+  return new Relevance(config);
+}
+
+// src/evaluation/metrics/Coherence.ts
+var Coherence = class extends BaseMetric {
+  type = "coherence";
+  checkLogicalFlow;
+  checkConsistency;
+  constructor(config = {}) {
+    super(config);
+    this.checkLogicalFlow = config.checkLogicalFlow ?? true;
+    this.checkConsistency = config.checkConsistency ?? true;
+    this.initName(config);
+  }
+  async evaluate(input) {
+    const scores = [];
+    const details = {};
+    const structuralScore = this.checkStructure(input.output);
+    scores.push(structuralScore);
+    details.structural = structuralScore;
+    if (this.checkLogicalFlow) {
+      const flowScore = this.checkFlow(input.output);
+      scores.push(flowScore);
+      details.logicalFlow = flowScore;
+    }
+    if (this.checkConsistency) {
+      const consistencyScore = this.checkInternalConsistency(input.output);
+      scores.push(consistencyScore);
+      details.consistency = consistencyScore;
+    }
+    const completenessScore = this.checkCompleteness(input.output);
+    scores.push(completenessScore);
+    details.completeness = completenessScore;
+    const averageScore = scores.reduce((a, b) => a + b, 0) / scores.length;
+    return Promise.resolve(
+      this.createResult(
+        averageScore,
+        this.generateExplanation(details),
+        details
+      )
+    );
+  }
+  /**
+   * Check structural coherence
+   */
+  checkStructure(text) {
+    let score = 1;
+    const sentences = text.split(/[.!?]+/).filter((s) => s.trim().length > 0);
+    if (sentences.length === 0) {
+      return 0.3;
+    }
+    for (const sentence of sentences) {
+      const trimmed = sentence.trim();
+      if (!/^[A-Z\d\-*•]/.test(trimmed) && trimmed.length > 0) {
+        score -= 0.1;
+      }
+    }
+    if (text.endsWith(",") || text.endsWith(":") || text.endsWith(";")) {
+      score -= 0.2;
+    }
+    const openParens = (text.match(/\(/g) || []).length;
+    const closeParens = (text.match(/\)/g) || []).length;
+    if (openParens !== closeParens) {
+      score -= 0.2;
+    }
+    return Math.max(0, score);
+  }
+  /**
+   * Check logical flow
+   */
+  checkFlow(text) {
+    const sentences = text.split(/[.!?]+/).filter((s) => s.trim().length > 0);
+    if (sentences.length <= 1) {
+      return 1;
+    }
+    let score = 1;
+    const transitionWords = [
+      "however",
+      "therefore",
+      "moreover",
+      "furthermore",
+      "additionally",
+      "first",
+      "second",
+      "third",
+      "finally",
+      "then",
+      "next",
+      "also",
+      "because",
+      "since",
+      "although",
+      "while",
+      "whereas",
+      "consequently",
+      "as a result",
+      "in addition",
+      "on the other hand",
+      "in conclusion"
+    ];
+    const hasTransitions = transitionWords.some(
+      (tw) => text.toLowerCase().includes(tw)
+    );
+    if (sentences.length > 3 && !hasTransitions) {
+      score -= 0.15;
+    }
+    for (let i = 1; i < sentences.length; i++) {
+      const prevWords = new Set(
+        sentences[i - 1].toLowerCase().split(/\s+/).filter((w) => w.length > 3)
+      );
+      const currWords = sentences[i].toLowerCase().split(/\s+/).filter((w) => w.length > 3);
+      const overlap = currWords.filter((w) => prevWords.has(w)).length;
+      if (currWords.length > 5 && overlap === 0) {
+        score -= 0.05;
+      }
+    }
+    return Math.max(0, score);
+  }
+  /**
+   * Check internal consistency
+   */
+  checkInternalConsistency(text) {
+    let score = 1;
+    const contradictions = [
+      [/\bis\b.*\bis not\b/i, /\bis not\b.*\bis\b/i],
+      [/\byes\b/i, /\bno\b/i],
+      [/\balways\b/i, /\bnever\b/i],
+      [/\bcan\b/i, /\bcannot\b/i],
+      [/\bcorrect\b.*\bnot correct\b/i, /\bnot correct\b.*\bcorrect\b/i]
+    ];
+    for (const [pattern1, pattern2] of contradictions) {
+      if (pattern1.test(text) || pattern2.test(text)) {
+        score -= 0.1;
+      }
+    }
+    const sentences = text.split(/[.!?]+/).filter((s) => s.trim().length > 0);
+    const normalizedSentences = sentences.map(
+      (s) => s.toLowerCase().replace(/\s+/g, " ").trim()
+    );
+    const uniqueSentences = new Set(normalizedSentences);
+    if (uniqueSentences.size < sentences.length) {
+      const repetitionRatio = 1 - uniqueSentences.size / sentences.length;
+      score -= repetitionRatio * 0.3;
+    }
+    return Math.max(0, score);
+  }
+  /**
+   * Check completeness
+   */
+  checkCompleteness(text) {
+    const trimmed = text.trim();
+    if (trimmed.length === 0) {
+      return 0;
+    }
+    const incompleteEndings = [
+      ",",
+      ":",
+      ";",
+      " and",
+      " or",
+      " but",
+      " the",
+      " a",
+      " an",
+      " is",
+      " are",
+      " was",
+      " were",
+      " be",
+      " been",
+      " have",
+      " has",
+      " had",
+      " will",
+      " would",
+      " should",
+      " could"
+    ];
+    for (const ending of incompleteEndings) {
+      if (trimmed.endsWith(ending)) {
+        return 0.5;
+      }
+    }
+    if (trimmed.endsWith("...")) {
+      return 0.7;
+    }
+    return 1;
+  }
+  /**
+   * Generate explanation from scores
+   */
+  generateExplanation(details) {
+    const issues = [];
+    if (details.structural < 0.7) {
+      issues.push("structural issues detected");
+    }
+    if (details.logicalFlow !== void 0 && details.logicalFlow < 0.7) {
+      issues.push("logical flow could be improved");
+    }
+    if (details.consistency !== void 0 && details.consistency < 0.7) {
+      issues.push("some inconsistencies found");
+    }
+    if (details.completeness < 0.7) {
+      issues.push("response may be incomplete");
+    }
+    if (issues.length === 0) {
+      return "Response is coherent and well-structured";
+    }
+    return `Issues: ${issues.join(", ")}`;
+  }
+};
+function createCoherenceMetric(config) {
+  return new Coherence(config);
+}
+
+// src/evaluation/metrics/Toxicity.ts
+var Toxicity = class _Toxicity extends BaseMetric {
+  type = "toxicity";
+  categories;
+  strictMode;
+  // Word lists for basic toxicity detection
+  static TOXIC_PATTERNS = {
+    hate: [
+      /\b(hate|hatred|despise)\s+(all|every|those)\b/i,
+      /\b(inferior|subhuman|vermin)\b/i
+    ],
+    harassment: [
+      /\b(stalk|harass|bully|intimidate)\b/i,
+      /\b(kill\s+yourself|kys)\b/i,
+      /\b(worthless|pathetic|loser)\b/i
+    ],
+    violence: [
+      /\b(kill|murder|attack|assault|hurt)\s+(you|them|her|him)\b/i,
+      /\b(bomb|shoot|stab|explode)\b/i
+    ],
+    sexual: [/\bexplicit\s+sexual\b/i, /\b(pornographic|obscene)\b/i],
+    self_harm: [
+      /\b(suicide|self[-\s]?harm|cut\s+yourself)\b/i,
+      /\bhow\s+to\s+(kill|hurt)\s+(yourself|myself)\b/i
+    ],
+    dangerous: [
+      /\b(make|build|create)\s+(bomb|weapon|explosive)\b/i,
+      /\b(hack|breach|steal)\s+(password|account|data)\b/i
+    ]
+  };
+  constructor(config = {}) {
+    super(config);
+    this.categories = config.categories ?? [
+      "hate",
+      "harassment",
+      "violence",
+      "sexual",
+      "self_harm",
+      "dangerous"
+    ];
+    this.strictMode = config.strictMode ?? false;
+    this.threshold = config.threshold ?? 0.3;
+    this.initName(config);
+  }
+  async evaluate(input) {
+    const categoryScores = {};
+    const detectedPatterns = [];
+    for (const category of this.categories) {
+      const { score, patterns } = this.checkCategory(input.output, category);
+      categoryScores[category] = score;
+      detectedPatterns.push(...patterns);
+    }
+    const maxToxicity = Math.max(...Object.values(categoryScores));
+    const adjustedScore = this.strictMode ? maxToxicity * 1.5 : maxToxicity;
+    const safeScore = 1 - Math.min(1, adjustedScore);
+    return Promise.resolve(
+      this.createResult(
+        safeScore,
+        this.generateExplanation(safeScore, categoryScores, detectedPatterns),
+        {
+          toxicityScore: adjustedScore,
+          categoryScores,
+          detectedPatterns,
+          isToxic: adjustedScore > this.threshold
+        }
+      )
+    );
+  }
+  /**
+   * Check a specific toxicity category
+   */
+  checkCategory(text, category) {
+    const patterns = _Toxicity.TOXIC_PATTERNS[category] || [];
+    const detectedPatterns = [];
+    for (const pattern of patterns) {
+      const match = text.match(pattern);
+      if (match) {
+        detectedPatterns.push(`${category}: "${match[0]}"`);
+      }
+    }
+    const score = Math.min(1, detectedPatterns.length * 0.3);
+    return { score, patterns: detectedPatterns };
+  }
+  /**
+   * Generate explanation
+   */
+  generateExplanation(safeScore, categoryScores, detectedPatterns) {
+    if (safeScore >= 0.9) {
+      return "Content appears safe with no detected toxic patterns";
+    }
+    if (safeScore >= 0.7) {
+      return `Minor concerns detected: ${detectedPatterns.slice(0, 2).join(", ")}`;
+    }
+    const topCategories = Object.entries(categoryScores).filter(([, score]) => score > 0.3).map(([cat]) => cat).join(", ");
+    return `Potential toxic content detected in categories: ${topCategories}. Patterns: ${detectedPatterns.slice(0, 3).join(", ")}`;
+  }
+  /**
+   * Override passes to check for low toxicity
+   */
+  passes(score) {
+    return score >= 1 - this.threshold;
+  }
+};
+function createToxicityMetric(config) {
+  return new Toxicity(config);
+}
+
+// src/evaluation/metrics/Faithfulness.ts
+var Faithfulness = class extends BaseMetric {
+  type = "faithfulness";
+  provider;
+  model;
+  constructor(config = {}) {
+    super(config);
+    this.model = config.model ?? "claude-sonnet-4-20250514";
+    this.initName(config);
+  }
+  /**
+   * Set the LLM provider for evaluation
+   */
+  setProvider(provider) {
+    this.provider = provider;
+  }
+  async evaluate(input) {
+    if (!input.context || input.context.length === 0) {
+      return this.createResult(
+        1,
+        "No context provided, skipping faithfulness check",
+        { skipped: true }
+      );
+    }
+    if (!this.provider) {
+      return this.evaluateHeuristic(input);
+    }
+    return this.evaluateWithLLM(input);
+  }
+  /**
+   * Evaluate faithfulness using heuristics
+   */
+  evaluateHeuristic(input) {
+    const context = input.context.join(" ").toLowerCase();
+    const claims = this.extractClaims(input.output);
+    if (claims.length === 0) {
+      return this.createResult(1, "No factual claims detected in output", {
+        method: "heuristic",
+        claimsChecked: 0
+      });
+    }
+    let supportedClaims = 0;
+    const claimResults = [];
+    for (const claim of claims) {
+      const supported = this.checkClaimSupport(claim, context);
+      if (supported) {
+        supportedClaims++;
+      }
+      claimResults.push({ claim, supported });
+    }
+    const score = supportedClaims / claims.length;
+    return this.createResult(
+      score,
+      `${supportedClaims}/${claims.length} claims supported by context`,
+      {
+        method: "heuristic",
+        claimsChecked: claims.length,
+        supportedClaims,
+        claimResults
+      }
+    );
+  }
+  /**
+   * Evaluate faithfulness using LLM
+   */
+  async evaluateWithLLM(input) {
+    const contextStr = input.context.map((c, i) => `[${i + 1}] ${c}`).join("\n\n");
+    const prompt = `You are evaluating the faithfulness of an AI response to the provided context.
+
+Context:
+${contextStr}
+
+Response to evaluate:
+${input.output}
+
+Evaluate whether the response is faithful to the context:
+1. Are all claims in the response supported by the context?
+2. Does the response introduce any information not in the context?
+3. Does the response contradict any information in the context?
+
+Rate the faithfulness on a scale of 1-5 where:
+1 = Response contains multiple unsupported or contradictory claims
+2 = Response contains some unsupported claims
+3 = Response is mostly faithful with minor unsupported details
+4 = Response is faithful with only trivial additions
+5 = Response is completely faithful to the context
+
+Provide your rating as "Score: X" followed by a brief explanation.`;
+    try {
+      const response = await this.provider.complete({
+        model: this.model,
+        messages: [{ role: "user", content: prompt }],
+        temperature: 0
+      });
+      const scoreMatch = response.content.match(/Score:\s*(\d+(?:\.\d+)?)/i);
+      const score = scoreMatch ? parseFloat(scoreMatch[1]) / 5 : 0.5;
+      return this.createResult(score, response.content, {
+        method: "llm",
+        model: this.model
+      });
+    } catch (error) {
+      const result = this.evaluateHeuristic(input);
+      return {
+        ...result,
+        details: {
+          ...result.details,
+          llmError: error.message
+        }
+      };
+    }
+  }
+  /**
+   * Extract factual claims from text
+   */
+  extractClaims(text) {
+    const sentences = text.split(/[.!?]+/).filter((s) => s.trim().length > 10);
+    return sentences.filter((sentence) => {
+      const lower = sentence.toLowerCase();
+      if (lower.includes("?")) return false;
+      if (/\b(i think|i believe|maybe|perhaps|possibly|might|could be)\b/.test(
+        lower
+      )) {
+        return false;
+      }
+      if (/\b(as mentioned|according to|based on)\b/.test(lower)) {
+        return false;
+      }
+      return true;
+    });
+  }
+  /**
+   * Check if a claim is supported by context
+   */
+  checkClaimSupport(claim, context) {
+    const claimWords = claim.toLowerCase().replace(/[^\w\s]/g, "").split(/\s+/).filter((w) => w.length > 3);
+    if (claimWords.length === 0) return true;
+    let matchedWords = 0;
+    for (const word of claimWords) {
+      if (context.includes(word)) {
+        matchedWords++;
+      }
+    }
+    const overlapRatio = matchedWords / claimWords.length;
+    return overlapRatio >= 0.5;
+  }
+};
+function createFaithfulnessMetric(config) {
+  return new Faithfulness(config);
+}
+
+// src/evaluation/metrics/ContextRelevance.ts
+var ContextRelevance = class extends BaseMetric {
+  type = "context_relevance";
+  provider;
+  model;
+  minRelevantChunks;
+  constructor(config = {}) {
+    super(config);
+    this.model = config.model ?? "claude-sonnet-4-20250514";
+    this.minRelevantChunks = config.minRelevantChunks ?? 1;
+    this.initName(config);
+  }
+  /**
+   * Set the LLM provider for evaluation
+   */
+  setProvider(provider) {
+    this.provider = provider;
+  }
+  async evaluate(input) {
+    if (!input.context || input.context.length === 0) {
+      return this.createResult(0, "No context provided for relevance check", {
+        skipped: true,
+        reason: "no_context"
+      });
+    }
+    if (!this.provider) {
+      return this.evaluateHeuristic(input);
+    }
+    return this.evaluateWithLLM(input);
+  }
+  /**
+   * Evaluate context relevance using heuristics
+   */
+  evaluateHeuristic(input) {
+    const questionKeywords = this.extractKeywords(input.input);
+    if (questionKeywords.length === 0) {
+      return this.createResult(1, "No keywords to match in question", {
+        method: "heuristic"
+      });
+    }
+    const chunkScores = [];
+    for (let i = 0; i < input.context.length; i++) {
+      const chunk = input.context[i].toLowerCase();
+      const matchedKeywords = [];
+      for (const keyword of questionKeywords) {
+        if (chunk.includes(keyword)) {
+          matchedKeywords.push(keyword);
+        }
+      }
+      const score = matchedKeywords.length / questionKeywords.length;
+      chunkScores.push({ index: i, score, matchedKeywords });
+    }
+    chunkScores.sort((a, b) => b.score - a.score);
+    const relevantChunks = chunkScores.filter((c) => c.score >= 0.3);
+    const avgRelevance = chunkScores.length > 0 ? chunkScores.reduce((sum, c) => sum + c.score, 0) / chunkScores.length : 0;
+    const coverageBonus = relevantChunks.length >= this.minRelevantChunks ? 0.1 : 0;
+    const finalScore = Math.min(1, avgRelevance + coverageBonus);
+    return this.createResult(
+      finalScore,
+      `${relevantChunks.length}/${input.context.length} chunks are relevant`,
+      {
+        method: "heuristic",
+        chunkScores,
+        relevantChunkCount: relevantChunks.length,
+        avgRelevance
+      }
+    );
+  }
+  /**
+   * Evaluate context relevance using LLM
+   */
+  async evaluateWithLLM(input) {
+    const chunkResults = [];
+    for (let i = 0; i < input.context.length; i++) {
+      const chunk = input.context[i];
+      const prompt = `Evaluate how relevant this context chunk is to answering the question.
+
+Question: ${input.input}
+
+Context chunk:
+${chunk}
+
+Rate the relevance on a scale of 1-5 where:
+1 = Completely irrelevant
+2 = Mostly irrelevant with tangential connection
+3 = Somewhat relevant but not directly useful
+4 = Mostly relevant and useful
+5 = Highly relevant and directly answers the question
+
+Provide only your rating as "Score: X" with a one-line explanation.`;
+      try {
+        const response = await this.provider.complete({
+          model: this.model,
+          messages: [{ role: "user", content: prompt }],
+          temperature: 0
+        });
+        const scoreMatch = response.content.match(/Score:\s*(\d+(?:\.\d+)?)/i);
+        const score = scoreMatch ? parseFloat(scoreMatch[1]) / 5 : 0.5;
+        chunkResults.push({
+          index: i,
+          score,
+          explanation: response.content
+        });
+      } catch {
+        const keywords = this.extractKeywords(input.input);
+        const chunkLower = chunk.toLowerCase();
+        const matches = keywords.filter((k) => chunkLower.includes(k)).length;
+        const score = keywords.length > 0 ? matches / keywords.length : 0.5;
+        chunkResults.push({
+          index: i,
+          score,
+          explanation: "Evaluated using heuristic fallback"
+        });
+      }
+    }
+    const avgScore = chunkResults.length > 0 ? chunkResults.reduce((sum, r) => sum + r.score, 0) / chunkResults.length : 0;
+    const relevantCount = chunkResults.filter((r) => r.score >= 0.6).length;
+    return this.createResult(
+      avgScore,
+      `Average relevance: ${(avgScore * 100).toFixed(1)}%, ${relevantCount}/${chunkResults.length} chunks relevant`,
+      {
+        method: "llm",
+        model: this.model,
+        chunkResults,
+        relevantChunkCount: relevantCount
+      }
+    );
+  }
+  /**
+   * Extract keywords from text
+   */
+  extractKeywords(text) {
+    const stopWords = /* @__PURE__ */ new Set([
+      "a",
+      "an",
+      "the",
+      "is",
+      "are",
+      "was",
+      "were",
+      "be",
+      "been",
+      "being",
+      "have",
+      "has",
+      "had",
+      "do",
+      "does",
+      "did",
+      "will",
+      "would",
+      "could",
+      "should",
+      "may",
+      "might",
+      "must",
+      "to",
+      "of",
+      "in",
+      "for",
+      "on",
+      "with",
+      "at",
+      "by",
+      "from",
+      "as",
+      "and",
+      "but",
+      "if",
+      "or",
+      "not",
+      "what",
+      "which",
+      "who",
+      "whom",
+      "this",
+      "that",
+      "these",
+      "those",
+      "i",
+      "me",
+      "my",
+      "we",
+      "you",
+      "your",
+      "it",
+      "its",
+      "how",
+      "why",
+      "when",
+      "where",
+      "can",
+      "please",
+      "tell",
+      "me",
+      "about"
+    ]);
+    return text.toLowerCase().replace(/[^\w\s]/g, " ").split(/\s+/).filter((word) => word.length > 2 && !stopWords.has(word));
+  }
+};
+function createContextRelevanceMetric(config) {
+  return new ContextRelevance(config);
+}
+
+// src/evaluation/metrics/CustomMetric.ts
+var CustomMetric = class extends BaseMetric {
+  type = "custom";
+  evaluateFn;
+  constructor(config) {
+    super(config);
+    if (!config.evaluateFn) {
+      throw new Error("Custom metric requires an evaluateFn");
+    }
+    this.evaluateFn = config.evaluateFn;
+    this.initName(config);
+  }
+  async evaluate(input) {
+    try {
+      const result = await this.evaluateFn(input);
+      return {
+        ...result,
+        metric: this.name
+      };
+    } catch (error) {
+      return this.createResult(
+        0,
+        `Custom metric evaluation failed: ${error.message}`,
+        {
+          error: error.message
+        }
+      );
+    }
+  }
+};
+function createCustomMetric(config) {
+  return new CustomMetric(config);
+}
+function createSimpleMetric(name, scoreFn, options) {
+  return new CustomMetric({
+    name,
+    threshold: options?.threshold,
+    weight: options?.weight,
+    evaluateFn: async (input) => {
+      const score = await scoreFn(
+        input.input,
+        input.output,
+        input.expectedOutput
+      );
+      return {
+        metric: name,
+        score,
+        explanation: `${name} score: ${(score * 100).toFixed(1)}%`
+      };
+    }
+  });
+}
+function createLengthMetric(options) {
+  return new CustomMetric({
+    name: "length",
+    evaluateFn: (input) => {
+      const length = input.output.length;
+      if (options.targetLength !== void 0) {
+        const tolerance = options.tolerance ?? 0.2;
+        const diff = Math.abs(length - options.targetLength) / options.targetLength;
+        const score2 = Math.max(0, 1 - diff / tolerance);
+        return Promise.resolve({
+          metric: "length",
+          score: score2,
+          explanation: `Output length: ${length}, target: ${options.targetLength}`,
+          details: { length, target: options.targetLength, diff }
+        });
+      }
+      const minOk = options.minLength === void 0 || length >= options.minLength;
+      const maxOk = options.maxLength === void 0 || length <= options.maxLength;
+      const score = minOk && maxOk ? 1 : 0;
+      return Promise.resolve({
+        metric: "length",
+        score,
+        explanation: minOk && maxOk ? "Output length is within acceptable range" : `Output length ${length} is outside range [${options.minLength ?? 0}, ${options.maxLength ?? "inf"}]`,
+        details: {
+          length,
+          minLength: options.minLength,
+          maxLength: options.maxLength
+        }
+      });
+    }
+  });
+}
+function createRegexMetric(options) {
+  return new CustomMetric({
+    name: options.name ?? "regex",
+    evaluateFn: (input) => {
+      const matches = options.pattern.test(input.output);
+      const shouldMatch = options.shouldMatch ?? true;
+      const score = matches === shouldMatch ? 1 : 0;
+      return Promise.resolve({
+        metric: options.name ?? "regex",
+        score,
+        explanation: shouldMatch ? matches ? "Output matches expected pattern" : "Output does not match expected pattern" : matches ? "Output matches forbidden pattern" : "Output correctly avoids forbidden pattern",
+        details: { pattern: options.pattern.source, matches }
+      });
+    }
+  });
+}
+function createJSONMetric(options) {
+  return new CustomMetric({
+    name: "json_validity",
+    evaluateFn: (input) => {
+      try {
+        const parsed = JSON.parse(input.output);
+        if (options?.schema) {
+          const schemaKeys = Object.keys(options.schema);
+          const parsedKeys = Object.keys(parsed);
+          const missingKeys = schemaKeys.filter((k) => !parsedKeys.includes(k));
+          if (missingKeys.length > 0) {
+            return Promise.resolve({
+              metric: "json_validity",
+              score: 0.5,
+              explanation: `Valid JSON but missing keys: ${missingKeys.join(", ")}`,
+              details: { valid: true, missingKeys }
+            });
+          }
+        }
+        return Promise.resolve({
+          metric: "json_validity",
+          score: 1,
+          explanation: "Output is valid JSON",
+          details: { valid: true }
+        });
+      } catch (error) {
+        return Promise.resolve({
+          metric: "json_validity",
+          score: 0,
+          explanation: `Invalid JSON: ${error.message}`,
+          details: { valid: false, error: error.message }
+        });
+      }
+    }
+  });
+}
+function createContainsMetric(options) {
+  return new CustomMetric({
+    name: "contains",
+    evaluateFn: (input) => {
+      const output = options.caseSensitive ? input.output : input.output.toLowerCase();
+      const missing = [];
+      const foundForbidden = [];
+      if (options.required) {
+        for (const phrase of options.required) {
+          const searchPhrase = options.caseSensitive ? phrase : phrase.toLowerCase();
+          if (!output.includes(searchPhrase)) {
+            missing.push(phrase);
+          }
+        }
+      }
+      if (options.forbidden) {
+        for (const phrase of options.forbidden) {
+          const searchPhrase = options.caseSensitive ? phrase : phrase.toLowerCase();
+          if (output.includes(searchPhrase)) {
+            foundForbidden.push(phrase);
+          }
+        }
+      }
+      const requiredScore = options.required ? (options.required.length - missing.length) / options.required.length : 1;
+      const forbiddenScore = options.forbidden ? (options.forbidden.length - foundForbidden.length) / options.forbidden.length : 1;
+      const score = (requiredScore + forbiddenScore) / 2;
+      return Promise.resolve({
+        metric: "contains",
+        score,
+        explanation: missing.length === 0 && foundForbidden.length === 0 ? "Output contains all required phrases and no forbidden phrases" : `Missing: [${missing.join(", ")}], Forbidden found: [${foundForbidden.join(", ")}]`,
+        details: { missing, foundForbidden }
+      });
+    }
+  });
+}
+
+// src/evaluation/judges/LLMJudge.ts
+var LLMJudge = class {
+  type = "llm";
+  provider;
+  model;
+  criteria;
+  systemPrompt;
+  temperature;
+  maxRetries;
+  constructor(config) {
+    if (!config.provider) {
+      throw new Error("LLMJudge requires a provider");
+    }
+    if (!config.criteria || config.criteria.length === 0) {
+      throw new Error("LLMJudge requires at least one criterion");
+    }
+    this.provider = config.provider;
+    this.model = config.model;
+    this.criteria = config.criteria;
+    this.systemPrompt = config.systemPrompt ?? this.getDefaultSystemPrompt();
+    this.temperature = config.temperature ?? 0;
+    this.maxRetries = config.maxRetries ?? 2;
+  }
+  async evaluate(input) {
+    const scores = {};
+    const explanations = {};
+    for (const criterion of this.criteria) {
+      const result = await this.evaluateCriterion(criterion, input);
+      scores[criterion.name] = result.score;
+      explanations[criterion.name] = result.explanation;
+    }
+    const totalWeight = this.criteria.reduce(
+      (sum, c) => sum + (c.weight ?? 1),
+      0
+    );
+    const weightedSum = this.criteria.reduce(
+      (sum, c) => sum + scores[c.name] * (c.weight ?? 1),
+      0
+    );
+    const overallScore = totalWeight > 0 ? weightedSum / totalWeight : 0;
+    return {
+      scores,
+      explanations,
+      overallScore,
+      confidence: this.calculateConfidence(scores)
+    };
+  }
+  /**
+   * Evaluate a single criterion
+   */
+  async evaluateCriterion(criterion, input) {
+    const prompt = this.buildPrompt(criterion, input);
+    for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
+      try {
+        const response = await this.provider.complete({
+          model: this.model,
+          messages: [
+            { role: "system", content: this.systemPrompt },
+            { role: "user", content: prompt }
+          ],
+          temperature: this.temperature
+        });
+        return this.parseResponse(response.content, criterion);
+      } catch (error) {
+        if (attempt === this.maxRetries) {
+          return {
+            score: 0,
+            explanation: `Evaluation failed after ${this.maxRetries + 1} attempts: ${error.message}`
+          };
+        }
+        await new Promise(
+          (resolve) => setTimeout(resolve, 1e3 * (attempt + 1))
+        );
+      }
+    }
+    return { score: 0, explanation: "Evaluation failed" };
+  }
+  /**
+   * Build the evaluation prompt
+   */
+  buildPrompt(criterion, input) {
+    let prompt = criterion.prompt.replace("{input}", input.input).replace("{output}", input.output);
+    if (input.expectedOutput) {
+      prompt = prompt.replace("{expected}", input.expectedOutput);
+    }
+    if (input.reference) {
+      prompt = prompt.replace("{reference}", input.reference);
+    }
+    if (input.context) {
+      prompt = prompt.replace("{context}", input.context.join("\n\n"));
+    }
+    return prompt;
+  }
+  /**
+   * Parse LLM response to extract score
+   */
+  parseResponse(response, criterion) {
+    const scorePatterns = [
+      /Score:\s*(\d+(?:\.\d+)?)/i,
+      /Rating:\s*(\d+(?:\.\d+)?)/i,
+      /(\d+(?:\.\d+)?)\s*\/\s*5/,
+      /(\d+(?:\.\d+)?)\s*out\s+of\s+5/i
+    ];
+    let rawScore = null;
+    for (const pattern of scorePatterns) {
+      const match = response.match(pattern);
+      if (match) {
+        rawScore = parseFloat(match[1]);
+        break;
+      }
+    }
+    if (rawScore === null) {
+      const numMatch = response.match(/^(\d+(?:\.\d+)?)/);
+      if (numMatch) {
+        rawScore = parseFloat(numMatch[1]);
+      }
+    }
+    const range = criterion.scoreRange ?? { min: 1, max: 5 };
+    let normalizedScore = 0.5;
+    if (rawScore !== null) {
+      normalizedScore = (rawScore - range.min) / (range.max - range.min);
+      normalizedScore = Math.max(0, Math.min(1, normalizedScore));
+    }
+    let explanation = response;
+    const scoreIndex = response.search(/Score:|Rating:|\d+\s*\/\s*5/i);
+    if (scoreIndex > 0) {
+      explanation = response.substring(scoreIndex);
+    }
+    return { score: normalizedScore, explanation: explanation.trim() };
+  }
+  /**
+   * Calculate confidence based on score consistency
+   */
+  calculateConfidence(scores) {
+    const values = Object.values(scores);
+    if (values.length <= 1) return 1;
+    const mean = values.reduce((a, b) => a + b, 0) / values.length;
+    const variance = values.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / values.length;
+    const std = Math.sqrt(variance);
+    return Math.max(0.5, 1 - std);
+  }
+  /**
+   * Get default system prompt
+   */
+  getDefaultSystemPrompt() {
+    return `You are an expert evaluator for AI-generated responses. Your task is to objectively assess responses based on specific criteria.
+
+Guidelines:
+- Be consistent in your scoring
+- Provide clear explanations for your ratings
+- Focus on the specific criterion being evaluated
+- Use the full range of the scoring scale
+- Be fair and unbiased`;
+  }
+  /**
+   * Add a new criterion
+   */
+  addCriterion(criterion) {
+    this.criteria.push(criterion);
+  }
+  /**
+   * Remove a criterion
+   */
+  removeCriterion(name) {
+    const index = this.criteria.findIndex((c) => c.name === name);
+    if (index >= 0) {
+      this.criteria.splice(index, 1);
+      return true;
+    }
+    return false;
+  }
+  /**
+   * Get criteria
+   */
+  getCriteria() {
+    return [...this.criteria];
+  }
+};
+function createLLMJudge(config) {
+  return new LLMJudge(config);
+}
+
+// src/evaluation/judges/RubricJudge.ts
+var RubricJudge = class {
+  type = "rubric";
+  provider;
+  model;
+  rubric;
+  temperature;
+  constructor(config) {
+    if (!config.provider) {
+      throw new Error("RubricJudge requires a provider");
+    }
+    if (!config.rubric) {
+      throw new Error("RubricJudge requires a rubric");
+    }
+    if (!config.rubric.levels || config.rubric.levels.length === 0) {
+      throw new Error("Rubric must have at least one level");
+    }
+    this.provider = config.provider;
+    this.model = config.model ?? "claude-sonnet-4-20250514";
+    this.rubric = config.rubric;
+    this.temperature = config.temperature ?? 0;
+  }
+  async evaluate(input) {
+    const prompt = this.buildPrompt(input);
+    try {
+      const response = await this.provider.complete({
+        model: this.model,
+        messages: [
+          { role: "system", content: this.getSystemPrompt() },
+          { role: "user", content: prompt }
+        ],
+        temperature: this.temperature
+      });
+      return this.parseResponse(response.content);
+    } catch (error) {
+      return {
+        scores: { [this.rubric.criteria]: 0 },
+        explanations: {
+          [this.rubric.criteria]: `Evaluation failed: ${error.message}`
+        },
+        overallScore: 0
+      };
+    }
+  }
+  /**
+   * Build the evaluation prompt
+   */
+  buildPrompt(input) {
+    const levelsDescription = this.rubric.levels.map((level) => {
+      let desc = `Score ${level.score}: ${level.description}`;
+      if (level.examples && level.examples.length > 0) {
+        desc += `
+  Examples: ${level.examples.join("; ")}`;
+      }
+      return desc;
+    }).join("\n");
+    return `Evaluate the following response using this rubric.
+
+Criterion: ${this.rubric.criteria}
+
+Scoring Rubric:
+${levelsDescription}
+
+Input/Question: ${input.input}
+
+Response to Evaluate:
+${input.output}
+
+${input.expectedOutput ? `Expected/Reference Output:
+${input.expectedOutput}
+` : ""}
+${input.context ? `Context:
+${input.context.join("\n")}
+` : ""}
+
+Based on the rubric, provide:
+1. The score (${this.rubric.levels.map((l) => l.score).join(", ")})
+2. A brief justification for your choice
+
+Format: "Score: X - [justification]"`;
+  }
+  /**
+   * Get system prompt
+   */
+  getSystemPrompt() {
+    return `You are an expert evaluator using a predefined rubric.
+Your task is to carefully match the response to the most appropriate rubric level.
+Be consistent and fair in your assessment.`;
+  }
+  /**
+   * Parse response to extract score
+   */
+  parseResponse(response) {
+    const scoreMatch = response.match(/Score:\s*(\d+)/i);
+    let score = 0;
+    if (scoreMatch) {
+      const rawScore = parseInt(scoreMatch[1], 10);
+      const level = this.rubric.levels.find((l) => l.score === rawScore);
+      if (level) {
+        const minScore = Math.min(...this.rubric.levels.map((l) => l.score));
+        const maxScore = Math.max(...this.rubric.levels.map((l) => l.score));
+        score = (rawScore - minScore) / (maxScore - minScore);
+      }
+    }
+    return {
+      scores: { [this.rubric.criteria]: score },
+      explanations: { [this.rubric.criteria]: response },
+      overallScore: score
+    };
+  }
+  /**
+   * Get rubric
+   */
+  getRubric() {
+    return { ...this.rubric };
+  }
+  /**
+   * Update rubric
+   */
+  setRubric(rubric) {
+    if (!rubric.levels || rubric.levels.length === 0) {
+      throw new Error("Rubric must have at least one level");
+    }
+    this.rubric = rubric;
+  }
+};
+function createRubricJudge(config) {
+  return new RubricJudge(config);
+}
+var QualityRubric = {
+  criteria: "response_quality",
+  levels: [
+    {
+      score: 1,
+      description: "Poor quality - Incorrect, irrelevant, or harmful response",
+      examples: ["Wrong answer", "Off-topic response", "Gibberish"]
+    },
+    {
+      score: 2,
+      description: "Below average - Partially addresses question but significant issues",
+      examples: ["Missing key information", "Contains errors", "Confusing"]
+    },
+    {
+      score: 3,
+      description: "Average - Addresses question adequately but room for improvement",
+      examples: ["Correct but lacks depth", "Could be clearer"]
+    },
+    {
+      score: 4,
+      description: "Good - Well-written, accurate, and helpful response",
+      examples: ["Clear explanation", "Addresses all parts of question"]
+    },
+    {
+      score: 5,
+      description: "Excellent - Outstanding response that exceeds expectations",
+      examples: ["Comprehensive", "Insightful", "Well-structured"]
+    }
+  ]
+};
+var CodeQualityRubric = {
+  criteria: "code_quality",
+  levels: [
+    {
+      score: 1,
+      description: "Critical issues - Code has bugs, security issues, or does not compile"
+    },
+    {
+      score: 2,
+      description: "Significant issues - Code works but has major problems"
+    },
+    {
+      score: 3,
+      description: "Functional - Code works with minor issues or style problems"
+    },
+    {
+      score: 4,
+      description: "Good - Clean, efficient code with minor improvements possible"
+    },
+    {
+      score: 5,
+      description: "Excellent - Production-ready, well-documented, follows best practices"
+    }
+  ]
+};
+var HelpfulnessRubric = {
+  criteria: "helpfulness",
+  levels: [
+    {
+      score: 1,
+      description: "Not helpful at all - Response does not address user needs"
+    },
+    {
+      score: 2,
+      description: "Slightly helpful - Provides minimal useful information"
+    },
+    { score: 3, description: "Moderately helpful - Addresses some user needs" },
+    {
+      score: 4,
+      description: "Very helpful - Addresses most user needs effectively"
+    },
+    {
+      score: 5,
+      description: "Extremely helpful - Exceeds expectations in addressing needs"
+    }
+  ]
+};
+
+// src/evaluation/judges/ComparativeJudge.ts
+var ComparativeJudge = class {
+  type = "comparative";
+  provider;
+  model;
+  criteria;
+  tieBreaker;
+  temperature;
+  constructor(config) {
+    if (!config.provider) {
+      throw new Error("ComparativeJudge requires a provider");
+    }
+    if (!config.criteria || config.criteria.length === 0) {
+      throw new Error("ComparativeJudge requires at least one criterion");
+    }
+    this.provider = config.provider;
+    this.model = config.model ?? "claude-sonnet-4-20250514";
+    this.criteria = config.criteria;
+    this.tieBreaker = config.tieBreaker;
+    this.temperature = config.temperature ?? 0;
+  }
+  /**
+   * Evaluate using standard input (compares output to expected)
+   */
+  async evaluate(input) {
+    if (!input.expectedOutput) {
+      return {
+        scores: {},
+        explanations: { error: "No expected output to compare against" },
+        overallScore: 0
+      };
+    }
+    const comparison = await this.compare({
+      input: input.input,
+      responseA: input.output,
+      responseB: input.expectedOutput,
+      context: input.context
+    });
+    const score = comparison.winner === "A" ? 1 : comparison.winner === "B" ? 0 : 0.5;
+    return {
+      scores: { comparison: score },
+      explanations: { comparison: comparison.reasoning },
+      overallScore: score,
+      metadata: {
+        winner: comparison.winner,
+        criteriaScores: comparison.criteriaScores,
+        confidence: comparison.confidence
+      }
+    };
+  }
+  /**
+   * Compare two responses
+   */
+  async compare(input) {
+    const prompt = this.buildComparisonPrompt(input);
+    try {
+      const response = await this.provider.complete({
+        model: this.model,
+        messages: [
+          { role: "system", content: this.getSystemPrompt() },
+          { role: "user", content: prompt }
+        ],
+        temperature: this.temperature
+      });
+      return this.parseComparisonResponse(response.content);
+    } catch (error) {
+      return {
+        winner: "tie",
+        reasoning: `Comparison failed: ${error.message}`,
+        confidence: 0
+      };
+    }
+  }
+  /**
+   * Build comparison prompt
+   */
+  buildComparisonPrompt(input) {
+    const criteriaList = this.criteria.map((c, i) => `${i + 1}. ${c}`).join("\n");
+    return `Compare these two responses and determine which is better.
+
+Question/Input: ${input.input}
+
+${input.context ? `Context:
+${input.context.join("\n")}
+
+` : ""}
+
+Response A:
+${input.responseA}
+
+Response B:
+${input.responseB}
+
+Evaluate both responses on the following criteria:
+${criteriaList}
+
+For each criterion, indicate which response is better (A, B, or tie).
+Then provide an overall winner.
+
+${this.tieBreaker ? `In case of an overall tie, use "${this.tieBreaker}" as the tie-breaker criterion.` : ""}
+
+Format your response as:
+Criterion 1: [A/B/tie] - [brief reason]
+Criterion 2: [A/B/tie] - [brief reason]
+...
+Overall Winner: [A/B/tie]
+Reasoning: [explanation]
+Confidence: [high/medium/low]`;
+  }
+  /**
+   * Get system prompt
+   */
+  getSystemPrompt() {
+    return `You are an expert at comparing AI-generated responses.
+Be objective and fair in your comparisons.
+Consider all provided criteria carefully.
+Provide clear reasoning for your choices.`;
+  }
+  /**
+   * Parse comparison response
+   */
+  parseComparisonResponse(response) {
+    const criteriaScores = {};
+    for (const criterion of this.criteria) {
+      const pattern = new RegExp(`${criterion}[^:]*:\\s*(A|B|tie)`, "i");
+      const match = response.match(pattern);
+      if (match) {
+        const winner2 = match[1].toUpperCase();
+        criteriaScores[criterion] = {
+          A: winner2 === "A" ? 1 : winner2 === "TIE" ? 0.5 : 0,
+          B: winner2 === "B" ? 1 : winner2 === "TIE" ? 0.5 : 0
+        };
+      }
+    }
+    const winnerMatch = response.match(/Overall\s*Winner:\s*(A|B|tie)/i);
+    let winner = "tie";
+    if (winnerMatch) {
+      const w = winnerMatch[1].toUpperCase();
+      winner = w === "A" ? "A" : w === "B" ? "B" : "tie";
+    }
+    const confMatch = response.match(/Confidence:\s*(high|medium|low)/i);
+    let confidence = 0.5;
+    if (confMatch) {
+      const conf = confMatch[1].toLowerCase();
+      confidence = conf === "high" ? 0.9 : conf === "medium" ? 0.7 : 0.5;
+    }
+    const reasoningMatch = response.match(
+      /Reasoning:\s*(.+?)(?=Confidence:|$)/is
+    );
+    const reasoning = reasoningMatch ? reasoningMatch[1].trim() : response;
+    return {
+      winner,
+      reasoning,
+      criteriaScores: Object.keys(criteriaScores).length > 0 ? criteriaScores : void 0,
+      confidence
+    };
+  }
+  /**
+   * Get criteria
+   */
+  getCriteria() {
+    return [...this.criteria];
+  }
+  /**
+   * Set criteria
+   */
+  setCriteria(criteria) {
+    if (criteria.length === 0) {
+      throw new Error("At least one criterion is required");
+    }
+    this.criteria = criteria;
+  }
+};
+function createComparativeJudge(config) {
+  return new ComparativeJudge(config);
+}
+
+// src/evaluation/judges/ConsensusJudge.ts
+var ConsensusJudge = class {
+  type = "consensus";
+  judges;
+  aggregation;
+  weights;
+  minAgreement;
+  constructor(config) {
+    if (!config.judges || config.judges.length === 0) {
+      throw new Error("ConsensusJudge requires at least one judge");
+    }
+    this.judges = config.judges;
+    this.aggregation = config.aggregation;
+    this.weights = config.weights;
+    this.minAgreement = config.minAgreement ?? 0.5;
+    if (this.weights && this.weights.length !== this.judges.length) {
+      throw new Error("Weights array must match number of judges");
+    }
+  }
+  async evaluate(input) {
+    const results = await Promise.all(
+      this.judges.map((judge) => judge.evaluate(input))
+    );
+    switch (this.aggregation) {
+      case "majority":
+        return this.aggregateMajority(results);
+      case "average":
+        return this.aggregateAverage(results);
+      case "weighted":
+        return this.aggregateWeighted(results);
+      default:
+        return this.aggregateAverage(results);
+    }
+  }
+  /**
+   * Aggregate using majority voting
+   */
+  aggregateMajority(results) {
+    const allScores = {};
+    const allExplanations = {};
+    for (const result of results) {
+      for (const [metric, score] of Object.entries(result.scores)) {
+        if (!allScores[metric]) {
+          allScores[metric] = [];
+          allExplanations[metric] = [];
+        }
+        allScores[metric].push(score);
+      }
+      for (const [metric, explanation] of Object.entries(result.explanations)) {
+        if (!allExplanations[metric]) {
+          allExplanations[metric] = [];
+        }
+        allExplanations[metric].push(explanation);
+      }
+    }
+    const consensusScores = {};
+    const consensusExplanations = {};
+    const agreementScores = {};
+    for (const [metric, scores] of Object.entries(allScores)) {
+      const rounded = scores.map((s) => Math.round(s * 2) / 2);
+      const counts = /* @__PURE__ */ new Map();
+      for (const s of rounded) {
+        counts.set(s, (counts.get(s) ?? 0) + 1);
+      }
+      let maxCount = 0;
+      let consensusScore = 0;
+      for (const [score, count] of counts) {
+        if (count > maxCount) {
+          maxCount = count;
+          consensusScore = score;
+        }
+      }
+      consensusScores[metric] = consensusScore;
+      agreementScores[metric] = maxCount / scores.length;
+      const agreeingExplanations = scores.map((s, i) => ({
+        score: s,
+        explanation: allExplanations[metric]?.[i]
+      })).filter((item) => Math.round(item.score * 2) / 2 === consensusScore).map((item) => item.explanation).filter(Boolean);
+      consensusExplanations[metric] = agreeingExplanations.join(" | ") || "No consensus explanation";
+    }
+    const overallScores = results.map((r) => r.overallScore ?? 0);
+    const roundedOverall = overallScores.map((s) => Math.round(s * 2) / 2);
+    const overallCounts = /* @__PURE__ */ new Map();
+    for (const s of roundedOverall) {
+      overallCounts.set(s, (overallCounts.get(s) ?? 0) + 1);
+    }
+    let overallConsensus = 0;
+    let maxOverallCount = 0;
+    for (const [score, count] of overallCounts) {
+      if (count > maxOverallCount) {
+        maxOverallCount = count;
+        overallConsensus = score;
+      }
+    }
+    const agreement = maxOverallCount / results.length;
+    return {
+      scores: consensusScores,
+      explanations: consensusExplanations,
+      overallScore: overallConsensus,
+      confidence: agreement >= this.minAgreement ? agreement : void 0,
+      metadata: {
+        aggregation: "majority",
+        agreement,
+        judgeCount: results.length,
+        agreementScores,
+        meetsMinAgreement: agreement >= this.minAgreement
+      }
+    };
+  }
+  /**
+   * Aggregate using simple average
+   */
+  aggregateAverage(results) {
+    const allScores = {};
+    const allExplanations = {};
+    for (const result of results) {
+      for (const [metric, score] of Object.entries(result.scores)) {
+        if (!allScores[metric]) {
+          allScores[metric] = [];
+          allExplanations[metric] = [];
+        }
+        allScores[metric].push(score);
+      }
+      for (const [metric, explanation] of Object.entries(result.explanations)) {
+        if (!allExplanations[metric]) {
+          allExplanations[metric] = [];
+        }
+        allExplanations[metric].push(explanation);
+      }
+    }
+    const avgScores = {};
+    const stdScores = {};
+    for (const [metric, scores] of Object.entries(allScores)) {
+      const mean = scores.reduce((a, b) => a + b, 0) / scores.length;
+      avgScores[metric] = mean;
+      const variance = scores.reduce((sum, s) => sum + Math.pow(s - mean, 2), 0) / scores.length;
+      stdScores[metric] = Math.sqrt(variance);
+    }
+    const combinedExplanations = {};
+    for (const [metric, explanations] of Object.entries(allExplanations)) {
+      combinedExplanations[metric] = explanations.join(" | ");
+    }
+    const overallScores = results.map((r) => r.overallScore ?? 0);
+    const overallMean = overallScores.reduce((a, b) => a + b, 0) / overallScores.length;
+    const overallVariance = overallScores.reduce((sum, s) => sum + Math.pow(s - overallMean, 2), 0) / overallScores.length;
+    const overallStd = Math.sqrt(overallVariance);
+    const confidence = Math.max(0.5, 1 - overallStd);
+    return {
+      scores: avgScores,
+      explanations: combinedExplanations,
+      overallScore: overallMean,
+      confidence,
+      metadata: {
+        aggregation: "average",
+        judgeCount: results.length,
+        standardDeviations: stdScores,
+        overallStd
+      }
+    };
+  }
+  /**
+   * Aggregate using weighted average
+   */
+  aggregateWeighted(results) {
+    const weights = this.weights ?? results.map(() => 1);
+    const totalWeight = weights.reduce((a, b) => a + b, 0);
+    const weightedScores = {};
+    const allExplanations = {};
+    for (let i = 0; i < results.length; i++) {
+      const result = results[i];
+      const weight = weights[i];
+      for (const [metric, score] of Object.entries(result.scores)) {
+        if (!weightedScores[metric]) {
+          weightedScores[metric] = 0;
+          allExplanations[metric] = [];
+        }
+        weightedScores[metric] += score * weight;
+      }
+      for (const [metric, explanation] of Object.entries(result.explanations)) {
+        if (!allExplanations[metric]) {
+          allExplanations[metric] = [];
+        }
+        allExplanations[metric].push(explanation);
+      }
+    }
+    for (const metric of Object.keys(weightedScores)) {
+      weightedScores[metric] /= totalWeight;
+    }
+    const combinedExplanations = {};
+    for (const [metric, explanations] of Object.entries(allExplanations)) {
+      combinedExplanations[metric] = explanations.join(" | ");
+    }
+    const weightedOverall = results.reduce(
+      (sum, r, i) => sum + (r.overallScore ?? 0) * weights[i],
+      0
+    ) / totalWeight;
+    return {
+      scores: weightedScores,
+      explanations: combinedExplanations,
+      overallScore: weightedOverall,
+      metadata: {
+        aggregation: "weighted",
+        judgeCount: results.length,
+        weights
+      }
+    };
+  }
+  /**
+   * Add a judge
+   */
+  addJudge(judge, weight) {
+    this.judges.push(judge);
+    if (this.weights && weight !== void 0) {
+      this.weights.push(weight);
+    }
+  }
+  /**
+   * Remove a judge by index
+   */
+  removeJudge(index) {
+    if (index >= 0 && index < this.judges.length) {
+      this.judges.splice(index, 1);
+      if (this.weights) {
+        this.weights.splice(index, 1);
+      }
+      return true;
+    }
+    return false;
+  }
+  /**
+   * Get judge count
+   */
+  getJudgeCount() {
+    return this.judges.length;
+  }
+};
+function createConsensusJudge(config) {
+  return new ConsensusJudge(config);
+}
+
+// src/evaluation/EvalDataset.ts
+import { nanoid } from "nanoid";
+var EvalDataset = class _EvalDataset {
+  name;
+  items;
+  metadata;
+  constructor(config) {
+    this.name = config.name ?? "eval-dataset";
+    this.items = config.items.map((item) => ({
+      ...item,
+      id: item.id ?? nanoid()
+    }));
+    this.metadata = config.metadata;
+  }
+  /**
+   * Get dataset size
+   */
+  get size() {
+    return this.items.length;
+  }
+  /**
+   * Get all items
+   */
+  getItems() {
+    return [...this.items];
+  }
+  /**
+   * Get item by ID
+   */
+  getItem(id) {
+    return this.items.find((item) => item.id === id);
+  }
+  /**
+   * Filter items
+   */
+  filter(predicate) {
+    return new _EvalDataset({
+      name: this.name,
+      items: this.items.filter(predicate),
+      metadata: this.metadata
+    });
+  }
+  /**
+   * Sample random items
+   */
+  sample(count, seed) {
+    if (count >= this.items.length) {
+      return new _EvalDataset({
+        name: this.name,
+        items: [...this.items],
+        metadata: this.metadata
+      });
+    }
+    const shuffled = [...this.items];
+    const rng = seed !== void 0 ? this.seededRandom(seed) : Math.random;
+    for (let i = shuffled.length - 1; i > 0; i--) {
+      const j = Math.floor(rng() * (i + 1));
+      [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+    }
+    return new _EvalDataset({
+      name: this.name,
+      items: shuffled.slice(0, count),
+      metadata: this.metadata
+    });
+  }
+  /**
+   * Split dataset into train/test
+   */
+  split(ratio) {
+    const splitIndex = Math.floor(this.items.length * ratio);
+    const shuffled = [...this.items];
+    for (let i = shuffled.length - 1; i > 0; i--) {
+      const j = Math.floor(Math.random() * (i + 1));
+      [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+    }
+    return [
+      new _EvalDataset({
+        name: `${this.name}-train`,
+        items: shuffled.slice(0, splitIndex),
+        metadata: this.metadata
+      }),
+      new _EvalDataset({
+        name: `${this.name}-test`,
+        items: shuffled.slice(splitIndex),
+        metadata: this.metadata
+      })
+    ];
+  }
+  /**
+   * Filter by tags
+   */
+  filterByTags(tags, mode = "any") {
+    return this.filter((item) => {
+      if (!item.tags) return false;
+      if (mode === "any") {
+        return tags.some((tag) => item.tags.includes(tag));
+      }
+      return tags.every((tag) => item.tags.includes(tag));
+    });
+  }
+  /**
+   * Get unique tags
+   */
+  getTags() {
+    const tags = /* @__PURE__ */ new Set();
+    for (const item of this.items) {
+      if (item.tags) {
+        for (const tag of item.tags) {
+          tags.add(tag);
+        }
+      }
+    }
+    return Array.from(tags);
+  }
+  /**
+   * Add items
+   */
+  addItems(items) {
+    for (const item of items) {
+      this.items.push({
+        ...item,
+        id: item.id ?? nanoid()
+      });
+    }
+  }
+  /**
+   * Remove item by ID
+   */
+  removeItem(id) {
+    const index = this.items.findIndex((item) => item.id === id);
+    if (index >= 0) {
+      this.items.splice(index, 1);
+      return true;
+    }
+    return false;
+  }
+  /**
+   * Create seeded random function
+   */
+  seededRandom(seed) {
+    return () => {
+      seed = (seed * 9301 + 49297) % 233280;
+      return seed / 233280;
+    };
+  }
+  /**
+   * Export to JSON
+   */
+  toJSON() {
+    return JSON.stringify(
+      {
+        name: this.name,
+        items: this.items,
+        metadata: this.metadata
+      },
+      null,
+      2
+    );
+  }
+  /**
+   * Export to JSONL
+   */
+  toJSONL() {
+    return this.items.map((item) => JSON.stringify(item)).join("\n");
+  }
+  /**
+   * Create from JSON array
+   */
+  static fromJSON(data, name) {
+    return new _EvalDataset({
+      name: name ?? "json-dataset",
+      items: data.map((item, index) => ({
+        id: `item-${index}`,
+        ...item
+      }))
+    });
+  }
+  /**
+   * Create from JSONL string
+   */
+  static fromJSONL(jsonl, name) {
+    const lines = jsonl.trim().split("\n").filter((line) => line.trim());
+    const items = lines.map((line, index) => {
+      const parsed = JSON.parse(line);
+      return {
+        id: parsed.id ?? `item-${index}`,
+        ...parsed
+      };
+    });
+    return new _EvalDataset({
+      name: name ?? "jsonl-dataset",
+      items
+    });
+  }
+  /**
+   * Create from HuggingFace dataset (stub - would need actual HF integration)
+   */
+  static fromHuggingFace(datasetName, config) {
+    console.warn(
+      "HuggingFace integration not implemented. Please install @huggingface/hub and implement the loader."
+    );
+    return Promise.resolve(
+      new _EvalDataset({
+        name: datasetName,
+        items: [],
+        metadata: {
+          source: "huggingface",
+          datasetName,
+          config
+        }
+      })
+    );
+  }
+  /**
+   * Create from CSV string
+   */
+  static fromCSV(csv, options) {
+    const delimiter = options?.delimiter ?? ",";
+    const lines = csv.trim().split("\n");
+    if (lines.length < 2) {
+      return new _EvalDataset({ name: "csv-dataset", items: [] });
+    }
+    const headers = lines[0].split(delimiter).map((h) => h.trim().replace(/^"|"$/g, ""));
+    const inputCol = options?.inputColumn ?? headers.find((h) => h.toLowerCase().includes("input")) ?? headers[0];
+    const outputCol = options?.outputColumn ?? headers.find(
+      (h) => h.toLowerCase().includes("output") || h.toLowerCase().includes("expected")
+    );
+    const contextCol = options?.contextColumn ?? headers.find((h) => h.toLowerCase().includes("context"));
+    const inputIdx = headers.indexOf(inputCol);
+    const outputIdx = outputCol ? headers.indexOf(outputCol) : -1;
+    const contextIdx = contextCol ? headers.indexOf(contextCol) : -1;
+    const items = [];
+    for (let i = 1; i < lines.length; i++) {
+      const values = lines[i].split(delimiter).map((v) => v.trim().replace(/^"|"$/g, ""));
+      if (inputIdx >= 0 && values[inputIdx]) {
+        items.push({
+          id: `csv-${i}`,
+          input: values[inputIdx],
+          expectedOutput: outputIdx >= 0 ? values[outputIdx] : void 0,
+          context: contextIdx >= 0 && values[contextIdx] ? [values[contextIdx]] : void 0
+        });
+      }
+    }
+    return new _EvalDataset({
+      name: "csv-dataset",
+      items
+    });
+  }
+};
+function createEvalDataset(config) {
+  return new EvalDataset(config);
+}
+
+// src/evaluation/EvalRunner.ts
+var EvalRunner = class {
+  parallelism;
+  timeout;
+  retries;
+  onItemComplete;
+  onError;
+  constructor(config = {}) {
+    this.parallelism = config.parallelism ?? 5;
+    this.timeout = config.timeout ?? 3e4;
+    this.retries = config.retries ?? 1;
+    this.onItemComplete = config.onItemComplete;
+    this.onError = config.onError;
+  }
+  /**
+   * Run evaluation on a dataset
+   */
+  async run(dataset, generateFn, metrics, judge) {
+    const items = dataset.getItems();
+    const results = [];
+    for (let i = 0; i < items.length; i += this.parallelism) {
+      const batch = items.slice(i, i + this.parallelism);
+      const batchResults = await Promise.all(
+        batch.map(
+          (item) => this.evaluateItem(item, generateFn, metrics, judge)
+        )
+      );
+      results.push(...batchResults);
+    }
+    return results;
+  }
+  /**
+   * Run evaluation as async generator
+   */
+  async *runStream(dataset, generateFn, metrics, judge) {
+    const items = dataset.getItems();
+    for (let i = 0; i < items.length; i += this.parallelism) {
+      const batch = items.slice(i, i + this.parallelism);
+      const batchResults = await Promise.all(
+        batch.map(
+          (item) => this.evaluateItem(item, generateFn, metrics, judge)
+        )
+      );
+      for (const result of batchResults) {
+        yield result;
+      }
+    }
+  }
+  /**
+   * Evaluate a single item
+   */
+  async evaluateItem(item, generateFn, metrics, judge) {
+    const startTime = performance.now();
+    let output = "";
+    let generationError = null;
+    for (let attempt = 0; attempt <= this.retries; attempt++) {
+      try {
+        output = await this.withTimeout(
+          generateFn(item.input, item.context),
+          this.timeout
+        );
+        break;
+      } catch (error) {
+        generationError = error;
+        if (attempt === this.retries) {
+          const evalError = {
+            itemId: item.id,
+            input: item.input,
+            error: generationError,
+            phase: "generation"
+          };
+          this.onError?.(evalError);
+          return {
+            itemId: item.id,
+            input: item.input,
+            output: "",
+            expectedOutput: item.expectedOutput,
+            context: item.context,
+            scores: {},
+            passed: false,
+            durationMs: performance.now() - startTime
+          };
+        }
+      }
+    }
+    const evalInput = {
+      input: item.input,
+      output,
+      expectedOutput: item.expectedOutput,
+      context: item.context,
+      reference: item.reference,
+      metadata: item.metadata
+    };
+    const scores = {};
+    const explanations = {};
+    for (const metric of metrics) {
+      try {
+        const result2 = await this.withTimeout(
+          metric.evaluate(evalInput),
+          this.timeout
+        );
+        scores[metric.name] = result2.score;
+        if (result2.explanation) {
+          explanations[metric.name] = result2.explanation;
+        }
+      } catch (error) {
+        const evalError = {
+          itemId: item.id,
+          input: item.input,
+          error,
+          phase: "evaluation"
+        };
+        this.onError?.(evalError);
+        scores[metric.name] = 0;
+        explanations[metric.name] = `Error: ${error.message}`;
+      }
+    }
+    let judgeResult;
+    if (judge) {
+      try {
+        judgeResult = await this.withTimeout(
+          judge.evaluate(evalInput),
+          this.timeout
+        );
+        for (const [key, value] of Object.entries(judgeResult.scores)) {
+          scores[`judge_${key}`] = value;
+        }
+      } catch (error) {
+        const evalError = {
+          itemId: item.id,
+          input: item.input,
+          error,
+          phase: "evaluation"
+        };
+        this.onError?.(evalError);
+      }
+    }
+    const passed = Object.values(scores).every((score) => score >= 0.5);
+    const result = {
+      itemId: item.id,
+      input: item.input,
+      output,
+      expectedOutput: item.expectedOutput,
+      context: item.context,
+      scores,
+      explanations: Object.keys(explanations).length > 0 ? explanations : void 0,
+      judgeResult,
+      passed,
+      durationMs: performance.now() - startTime
+    };
+    this.onItemComplete?.(result);
+    return result;
+  }
+  /**
+   * Run with timeout
+   */
+  async withTimeout(promise, timeoutMs) {
+    return Promise.race([
+      promise,
+      new Promise(
+        (_, reject) => setTimeout(() => reject(new Error("Evaluation timeout")), timeoutMs)
+      )
+    ]);
+  }
+};
+function createEvalRunner(config) {
+  return new EvalRunner(config);
+}
+
+// src/evaluation/EvaluationPipeline.ts
+var EvaluationPipeline = class {
+  metrics;
+  llmJudge;
+  runner;
+  constructor(config) {
+    this.metrics = config.metrics;
+    this.llmJudge = config.llmJudge;
+    this.runner = new EvalRunner({
+      parallelism: config.parallelism ?? 5,
+      timeout: config.timeout ?? 3e4,
+      retries: config.retries ?? 1
+    });
+  }
+  /**
+   * Run evaluation pipeline
+   */
+  async evaluate(options) {
+    const startTime = performance.now();
+    const results = [];
+    const total = options.dataset.size;
+    let completed = 0;
+    const runner = new EvalRunner({
+      onItemComplete: (result) => {
+        results.push(result);
+        completed++;
+        if (options.onProgress) {
+          const elapsed = performance.now() - startTime;
+          const avgTime = elapsed / completed;
+          const remaining = (total - completed) * avgTime;
+          options.onProgress({
+            completed,
+            total,
+            currentItem: result.itemId,
+            elapsedMs: elapsed,
+            estimatedRemainingMs: remaining
+          });
+        }
+      },
+      onError: (error) => {
+        if (options.onError) {
+          options.onError(error);
+        }
+        if (options.stopOnError) {
+          throw error.error;
+        }
+      }
+    });
+    await runner.run(
+      options.dataset,
+      options.generateFn,
+      this.metrics,
+      this.llmJudge
+    );
+    const totalDurationMs = performance.now() - startTime;
+    const metricsSummary = this.calculateMetricsSummary(results);
+    const failures = this.analyzeFailures(results);
+    const summary = this.createSummary(results, totalDurationMs);
+    return this.createResult(results, metricsSummary, failures, summary);
+  }
+  /**
+   * Run evaluation as stream
+   */
+  async *evaluateStream(options) {
+    const startTime = performance.now();
+    const results = [];
+    const total = options.dataset.size;
+    for await (const result of this.runner.runStream(
+      options.dataset,
+      options.generateFn,
+      this.metrics,
+      this.llmJudge
+    )) {
+      results.push(result);
+      if (options.onProgress) {
+        const elapsed = performance.now() - startTime;
+        const avgTime = elapsed / results.length;
+        const remaining = (total - results.length) * avgTime;
+        options.onProgress({
+          completed: results.length,
+          total,
+          currentItem: result.itemId,
+          elapsedMs: elapsed,
+          estimatedRemainingMs: remaining
+        });
+      }
+      yield result;
+    }
+    const totalDurationMs = performance.now() - startTime;
+    const metricsSummary = this.calculateMetricsSummary(results);
+    const failures = this.analyzeFailures(results);
+    const summary = this.createSummary(results, totalDurationMs);
+    return this.createResult(results, metricsSummary, failures, summary);
+  }
+  /**
+   * Calculate metrics summary
+   */
+  calculateMetricsSummary(results) {
+    const summary = {};
+    if (results.length === 0) return summary;
+    const metricNames = /* @__PURE__ */ new Set();
+    for (const result of results) {
+      for (const name of Object.keys(result.scores)) {
+        metricNames.add(name);
+      }
+    }
+    for (const name of metricNames) {
+      const scores = results.map((r) => r.scores[name]).filter((s) => s !== void 0);
+      if (scores.length === 0) continue;
+      const sorted = [...scores].sort((a, b) => a - b);
+      const sum = scores.reduce((a, b) => a + b, 0);
+      const mean = sum / scores.length;
+      const variance = scores.reduce((s, v) => s + Math.pow(v - mean, 2), 0) / scores.length;
+      const std = Math.sqrt(variance);
+      const passCount = scores.filter((s) => s >= 0.5).length;
+      summary[name] = {
+        mean,
+        std,
+        min: sorted[0],
+        max: sorted[sorted.length - 1],
+        median: sorted[Math.floor(sorted.length / 2)],
+        p90: sorted[Math.floor(sorted.length * 0.9)],
+        p95: sorted[Math.floor(sorted.length * 0.95)],
+        passRate: passCount / scores.length
+      };
+    }
+    return summary;
+  }
+  /**
+   * Analyze failures
+   */
+  analyzeFailures(results) {
+    return results.filter((r) => !r.passed).map((r) => {
+      const failedMetrics = Object.entries(r.scores).filter(([, score]) => score < 0.5).map(([name]) => name);
+      const explanations = failedMetrics.map((m) => r.explanations?.[m]).filter(Boolean).join("; ");
+      return {
+        itemId: r.itemId,
+        input: r.input,
+        output: r.output,
+        expectedOutput: r.expectedOutput,
+        scores: r.scores,
+        failedMetrics,
+        explanation: explanations || void 0
+      };
+    });
+  }
+  /**
+   * Create evaluation summary
+   */
+  createSummary(results, totalDurationMs) {
+    const passedItems = results.filter((r) => r.passed).length;
+    const allScores = results.flatMap((r) => Object.values(r.scores));
+    const avgScore = allScores.length > 0 ? allScores.reduce((a, b) => a + b, 0) / allScores.length : 0;
+    return {
+      totalItems: results.length,
+      passedItems,
+      failedItems: results.length - passedItems,
+      passRate: results.length > 0 ? passedItems / results.length : 0,
+      avgScore,
+      totalDurationMs,
+      avgDurationMs: results.length > 0 ? totalDurationMs / results.length : 0,
+      timestamp: Date.now()
+    };
+  }
+  /**
+   * Create result object
+   */
+  createResult(results, metrics, failures, summary) {
+    return {
+      results,
+      metrics,
+      failures,
+      summary,
+      exportJSON() {
+        return JSON.stringify(
+          {
+            results,
+            metrics,
+            failures,
+            summary
+          },
+          null,
+          2
+        );
+      },
+      exportCSV() {
+        if (results.length === 0) return "";
+        const scoreColumns = /* @__PURE__ */ new Set();
+        for (const r of results) {
+          for (const name of Object.keys(r.scores)) {
+            scoreColumns.add(name);
+          }
+        }
+        const headers = [
+          "itemId",
+          "input",
+          "output",
+          "passed",
+          ...scoreColumns
+        ];
+        const rows = results.map((r) => {
+          const values = [
+            r.itemId,
+            `"${r.input.replace(/"/g, '""')}"`,
+            `"${r.output.replace(/"/g, '""')}"`,
+            r.passed.toString(),
+            ...Array.from(scoreColumns).map(
+              (c) => r.scores[c]?.toFixed(4) ?? ""
+            )
+          ];
+          return values.join(",");
+        });
+        return [headers.join(","), ...rows].join("\n");
+      },
+      getFailures(options) {
+        let filtered = [...failures];
+        if (options?.threshold !== void 0) {
+          filtered = filtered.filter(
+            (f) => Object.values(f.scores).some((s) => s < options.threshold)
+          );
+        }
+        if (options?.metric) {
+          filtered = filtered.filter(
+            (f) => f.failedMetrics.includes(options.metric)
+          );
+        }
+        if (options?.limit) {
+          filtered = filtered.slice(0, options.limit);
+        }
+        return filtered;
+      }
+    };
+  }
+  /**
+   * Add a metric
+   */
+  addMetric(metric) {
+    this.metrics.push(metric);
+  }
+  /**
+   * Remove a metric
+   */
+  removeMetric(name) {
+    const index = this.metrics.findIndex((m) => m.name === name);
+    if (index >= 0) {
+      this.metrics.splice(index, 1);
+      return true;
+    }
+    return false;
+  }
+  /**
+   * Set judge
+   */
+  setJudge(judge) {
+    this.llmJudge = judge;
+  }
+  /**
+   * Get metrics
+   */
+  getMetrics() {
+    return [...this.metrics];
+  }
+};
+function createEvaluationPipeline(config) {
+  return new EvaluationPipeline(config);
+}
+
+export {
+  BaseMetric,
+  Accuracy,
+  createAccuracyMetric,
+  Relevance,
+  createRelevanceMetric,
+  Coherence,
+  createCoherenceMetric,
+  Toxicity,
+  createToxicityMetric,
+  Faithfulness,
+  createFaithfulnessMetric,
+  ContextRelevance,
+  createContextRelevanceMetric,
+  CustomMetric,
+  createCustomMetric,
+  createSimpleMetric,
+  createLengthMetric,
+  createRegexMetric,
+  createJSONMetric,
+  createContainsMetric,
+  LLMJudge,
+  createLLMJudge,
+  RubricJudge,
+  createRubricJudge,
+  QualityRubric,
+  CodeQualityRubric,
+  HelpfulnessRubric,
+  ComparativeJudge,
+  createComparativeJudge,
+  ConsensusJudge,
+  createConsensusJudge,
+  EvalDataset,
+  createEvalDataset,
+  EvalRunner,
+  createEvalRunner,
+  EvaluationPipeline,
+  createEvaluationPipeline
+};