@massiangelone/rag-eval 0.1.0-alpha.0

package/dist/index.js

@@ -0,0 +1,1043 @@
+#!/usr/bin/env node
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+// node_modules/.pnpm/tsup@8.5.1_tsx@4.21.0_typescript@6.0.3/node_modules/tsup/assets/esm_shims.js
+import path from "path";
+import { fileURLToPath } from "url";
+var init_esm_shims = __esm({
+  "node_modules/.pnpm/tsup@8.5.1_tsx@4.21.0_typescript@6.0.3/node_modules/tsup/assets/esm_shims.js"() {
+    "use strict";
+  }
+});
+
+// src/config/schema.ts
+import { z } from "zod";
+var EvalEntrySchema, EndpointConfigSchema, JudgeConfigSchema, JudgeOutputJSONSchema, ScoringConfigSchema, ConfigSchema;
+var init_schema = __esm({
+  "src/config/schema.ts"() {
+    "use strict";
+    init_esm_shims();
+    EvalEntrySchema = z.object({
+      id: z.string().min(1),
+      question: z.string().min(1),
+      expected_source: z.union([z.string(), z.array(z.string())]),
+      expected_answer: z.string().optional()
+    });
+    EndpointConfigSchema = z.object({
+      url: z.string().url(),
+      method: z.enum(["GET", "POST"]).default("POST"),
+      headers: z.record(z.string(), z.string()).optional(),
+      body: z.record(z.string(), z.unknown()).optional(),
+      responsePaths: z.object({
+        answer: z.string(),
+        sources: z.string(),
+        sourceContents: z.string().optional()
+      }),
+      timeoutMs: z.number().int().positive().default(3e4)
+    });
+    JudgeConfigSchema = z.object({
+      provider: z.enum(["claude", "openai"]).default("claude"),
+      model: z.string().optional()
+    }).transform((data) => {
+      const defaultModel = data.provider === "openai" ? "gpt-4o-mini" : "claude-sonnet-4-6";
+      return {
+        provider: data.provider,
+        model: data.model ?? defaultModel
+      };
+    });
+    JudgeOutputJSONSchema = z.object({
+      faithfulness: z.union([z.number().min(0).max(1), z.null()]),
+      correctness: z.union([z.number().min(0).max(1), z.null()]),
+      rationale: z.string().min(1).max(2e3)
+    });
+    ScoringConfigSchema = z.object({
+      retrievalK: z.number().int().positive().default(5),
+      weights: z.object({
+        retrieval: z.number().min(0).max(1).default(0.4),
+        faithfulness: z.number().min(0).max(1).default(0.3),
+        correctness: z.number().min(0).max(1).default(0.3)
+      }).refine(
+        (w) => Math.abs(w.retrieval + w.faithfulness + w.correctness - 1) < 0.01,
+        { message: "Scoring weights must sum to 1.0" }
+      )
+    });
+    ConfigSchema = z.object({
+      endpoint: EndpointConfigSchema,
+      judge: JudgeConfigSchema.optional(),
+      scoring: ScoringConfigSchema.default({
+        retrievalK: 5,
+        weights: { retrieval: 0.4, faithfulness: 0.3, correctness: 0.3 }
+      })
+    });
+  }
+});
+
+// src/config/loader.ts
+import { readFileSync } from "fs";
+import { resolve } from "path";
+function loadConfig(path2) {
+  const absPath = resolve(process.cwd(), path2);
+  let raw;
+  try {
+    raw = readFileSync(absPath, "utf-8");
+  } catch {
+    throw new ConfigError(`Could not read config file: ${absPath}`, "load");
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (e) {
+    throw new ConfigError(
+      `Config is not valid JSON: ${e.message}`,
+      "parse"
+    );
+  }
+  const result = ConfigSchema.safeParse(parsed);
+  if (!result.success) {
+    const issues = result.error.issues.map((i) => `  \u2022 ${i.path.join(".")}: ${i.message}`).join("\n");
+    throw new ConfigError(`Config validation failed:
+${issues}`, "validate");
+  }
+  return result.data;
+}
+function loadEvalSet(path2) {
+  const absPath = resolve(process.cwd(), path2);
+  let raw;
+  try {
+    raw = readFileSync(absPath, "utf-8");
+  } catch {
+    throw new ConfigError(`Could not read eval-set: ${absPath}`, "load-evalset");
+  }
+  const lines = raw.split("\n").filter((l) => l.trim().length > 0);
+  if (lines.length === 0) {
+    throw new ConfigError(`Eval-set is empty: ${absPath}`, "evalset-empty");
+  }
+  const entries = [];
+  for (let i = 0; i < lines.length; i++) {
+    let parsed;
+    try {
+      parsed = JSON.parse(lines[i]);
+    } catch (e) {
+      throw new ConfigError(
+        `Line ${i + 1} is not valid JSON: ${e.message}`,
+        "evalset-parse"
+      );
+    }
+    const result = EvalEntrySchema.safeParse(parsed);
+    if (!result.success) {
+      const issues = result.error.issues.map((iss) => `${iss.path.join(".")}: ${iss.message}`).join(", ");
+      throw new ConfigError(`Line ${i + 1} invalid: ${issues}`, "evalset-validate");
+    }
+    entries.push(result.data);
+  }
+  return entries;
+}
+var ConfigError;
+var init_loader = __esm({
+  "src/config/loader.ts"() {
+    "use strict";
+    init_esm_shims();
+    init_schema();
+    ConfigError = class extends Error {
+      constructor(message, source) {
+        super(`[config:${source}] ${message}`);
+        this.source = source;
+        this.name = "ConfigError";
+      }
+      source;
+    };
+  }
+});
+
+// src/adapters/endpoint.ts
+function substitutePlaceholders(template, entry) {
+  if (typeof template === "string") {
+    return template.replaceAll("{{question}}", entry.question).replaceAll("{{id}}", entry.id);
+  }
+  if (Array.isArray(template)) {
+    return template.map((item) => substitutePlaceholders(item, entry));
+  }
+  if (template && typeof template === "object") {
+    const out = {};
+    for (const [k, v] of Object.entries(template)) {
+      out[k] = substitutePlaceholders(v, entry);
+    }
+    return out;
+  }
+  return template;
+}
+function getPath(obj, path2) {
+  const parts = path2.split(".");
+  let current = obj;
+  for (let i = 0; i < parts.length; i++) {
+    const part = parts[i];
+    if (current === null || current === void 0) return void 0;
+    const arrayMatch = part.match(/^(.+?)\[\]$/);
+    if (arrayMatch) {
+      const arr = current[arrayMatch[1]];
+      if (!Array.isArray(arr)) return void 0;
+      const remainingPath = parts.slice(i + 1).join(".");
+      if (!remainingPath) return arr;
+      return arr.map((item) => getPath(item, remainingPath));
+    }
+    current = current[part];
+  }
+  return current;
+}
+async function callEndpoint(config, entry) {
+  const start = Date.now();
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), config.timeoutMs);
+  let response;
+  try {
+    const init = {
+      method: config.method,
+      headers: config.headers,
+      signal: controller.signal
+    };
+    if (config.method === "POST" && config.body) {
+      const body = substitutePlaceholders(config.body, entry);
+      init.body = JSON.stringify(body);
+    }
+    response = await fetch(config.url, init);
+  } catch (e) {
+    clearTimeout(timeout);
+    if (e.name === "AbortError") {
+      throw new EndpointError(`Request timed out after ${config.timeoutMs}ms`);
+    }
+    throw new EndpointError(`Network error: ${e.message}`);
+  }
+  clearTimeout(timeout);
+  if (!response.ok) {
+    throw new EndpointError(
+      `Endpoint returned ${response.status}: ${response.statusText}`,
+      response.status
+    );
+  }
+  let raw;
+  try {
+    raw = await response.json();
+  } catch {
+    throw new EndpointError(`Endpoint response is not valid JSON`);
+  }
+  const answerRaw = getPath(raw, config.responsePaths.answer);
+  const sourcesRaw = getPath(raw, config.responsePaths.sources);
+  if (typeof answerRaw !== "string") {
+    throw new EndpointError(
+      `Response path "${config.responsePaths.answer}" did not resolve to a string`
+    );
+  }
+  let sources = [];
+  if (Array.isArray(sourcesRaw)) {
+    sources = sourcesRaw.map((s) => String(s));
+  } else if (typeof sourcesRaw === "string") {
+    sources = [sourcesRaw];
+  } else if (sourcesRaw !== void 0 && sourcesRaw !== null) {
+    throw new EndpointError(
+      `Response path "${config.responsePaths.sources}" did not resolve to array or string`
+    );
+  }
+  let sourceContents;
+  if (config.responsePaths.sourceContents) {
+    const contentsRaw = getPath(raw, config.responsePaths.sourceContents);
+    if (Array.isArray(contentsRaw)) {
+      sourceContents = contentsRaw.map((c) => String(c));
+    } else if (typeof contentsRaw === "string") {
+      sourceContents = [contentsRaw];
+    } else if (contentsRaw !== void 0 && contentsRaw !== null) {
+      throw new EndpointError(
+        `Response path "${config.responsePaths.sourceContents}" did not resolve to array or string`
+      );
+    }
+  }
+  return {
+    answer: answerRaw,
+    sources,
+    sourceContents,
+    raw,
+    latencyMs: Date.now() - start
+  };
+}
+var EndpointError;
+var init_endpoint = __esm({
+  "src/adapters/endpoint.ts"() {
+    "use strict";
+    init_esm_shims();
+    EndpointError = class extends Error {
+      constructor(message, statusCode) {
+        super(message);
+        this.statusCode = statusCode;
+        this.name = "EndpointError";
+      }
+      statusCode;
+    };
+  }
+});
+
+// src/core/scorer.ts
+function scoreRetrieval(entry, response, k) {
+  const expected = Array.isArray(entry.expected_source) ? entry.expected_source : [entry.expected_source];
+  const topK = response.sources.slice(0, k);
+  let hits = 0;
+  for (const src of topK) {
+    if (expected.includes(src)) hits++;
+  }
+  const precision = topK.length > 0 ? hits / topK.length : 0;
+  const found = expected.some((e) => topK.includes(e));
+  return {
+    precision,
+    found,
+    topKSources: topK,
+    expectedSources: expected
+  };
+}
+function isValidScore(n) {
+  return typeof n === "number" && !isNaN(n);
+}
+function computeOverallScore(retrieval, judge, weights) {
+  const r = retrieval ? retrieval.found ? 1 : retrieval.precision : 0;
+  if (!judge) return r;
+  const hasFaith = isValidScore(judge.faithfulness);
+  const hasCorr = isValidScore(judge.correctness);
+  if (!hasFaith && !hasCorr) return r;
+  let totalWeight = weights.retrieval;
+  let weightedSum = r * weights.retrieval;
+  if (hasFaith) {
+    weightedSum += judge.faithfulness * weights.faithfulness;
+    totalWeight += weights.faithfulness;
+  }
+  if (hasCorr) {
+    weightedSum += judge.correctness * weights.correctness;
+    totalWeight += weights.correctness;
+  }
+  return totalWeight > 0 ? weightedSum / totalWeight : r;
+}
+var init_scorer = __esm({
+  "src/core/scorer.ts"() {
+    "use strict";
+    init_esm_shims();
+  }
+});
+
+// src/providers/prompts.ts
+function buildJudgeUserPrompt(input) {
+  const looksLikeIds = input.retrievedContext.length > 0 && input.retrievedContext.every((c) => c.length < 80 && !c.includes(" "));
+  const contextBlock = input.retrievedContext.length > 0 ? input.retrievedContext.map((c, i) => `[${i + 1}] ${c}`).join("\n\n") : "(no context retrieved)";
+  const contextNote = looksLikeIds ? "\n\n(NOTE: The retrieved context appears to be document IDs, not full text. Faithfulness cannot be reliably assessed. Return null for faithfulness.)" : "";
+  const expectedBlock = input.expectedAnswer ? `
+
+Expected answer (ground truth):
+${input.expectedAnswer}` : "\n\n(No expected_answer provided \u2014 return null for correctness.)";
+  return `Question:
+${input.question}
+
+Retrieved context:
+${contextBlock}${contextNote}
+
+RAG-generated answer:
+${input.answer}${expectedBlock}
+
+Now score and return JSON.`;
+}
+var JUDGE_SYSTEM_PROMPT;
+var init_prompts = __esm({
+  "src/providers/prompts.ts"() {
+    "use strict";
+    init_esm_shims();
+    JUDGE_SYSTEM_PROMPT = `You are a strict evaluator of RAG (retrieval-augmented generation) pipelines.
+
+Your job: given a question, a RAG-generated answer, and the retrieved context the answer was supposed to be grounded in, score two dimensions:
+
+1. FAITHFULNESS (0.0 to 1.0 or null): Is the answer supported by the retrieved context? Penalize claims that are not in the context. A score of 1.0 means every claim in the answer is grounded in the context. A score of 0.0 means the answer is entirely hallucinated. Return null if the context is uninterpretable (e.g., only opaque IDs, not readable text).
+
+2. CORRECTNESS (0.0 to 1.0 or null): Does the answer match the expected answer in substance? Be lenient on phrasing, strict on facts. If no expected_answer is provided, return null for this field. A score of 1.0 means semantically equivalent. A score of 0.0 means contradictory or unrelated.
+
+Return ONLY valid JSON in this exact shape, no markdown, no commentary:
+
+{
+  "faithfulness": <number 0.0-1.0 or null if context is uninterpretable>,
+  "correctness": <number 0.0-1.0 or null>,
+  "rationale": "<one or two sentences explaining the scores>"
+}`;
+  }
+});
+
+// src/providers/types.ts
+var JudgeError;
+var init_types = __esm({
+  "src/providers/types.ts"() {
+    "use strict";
+    init_esm_shims();
+    JudgeError = class extends Error {
+      constructor(message, code, provider, cause) {
+        super(`[judge:${provider}:${code}] ${message}`);
+        this.code = code;
+        this.provider = provider;
+        this.cause = cause;
+        this.name = "JudgeError";
+      }
+      code;
+      provider;
+      cause;
+    };
+  }
+});
+
+// src/providers/claude.ts
+var claude_exports = {};
+__export(claude_exports, {
+  claudeJudge: () => claudeJudge
+});
+import Anthropic from "@anthropic-ai/sdk";
+function getClient() {
+  const apiKey = process.env["ANTHROPIC_API_KEY"];
+  if (!apiKey) {
+    throw new JudgeError(
+      "ANTHROPIC_API_KEY not set in environment",
+      "auth",
+      "claude"
+    );
+  }
+  return new Anthropic({ apiKey });
+}
+function mapError(e) {
+  if (e instanceof JudgeError) return e;
+  const err = e;
+  if (err.status === 401 || err.status === 403) {
+    return new JudgeError("Authentication failed", "auth", "claude", e);
+  }
+  if (err.status === 429) {
+    return new JudgeError("Rate limit exceeded", "rate_limit", "claude", e);
+  }
+  if (err.status && err.status >= 500) {
+    return new JudgeError("Provider server error", "server", "claude", e);
+  }
+  if (err.status && err.status >= 400) {
+    return new JudgeError(err.message ?? "Invalid request", "invalid", "claude", e);
+  }
+  return new JudgeError(err.message ?? "Network error", "network", "claude", e);
+}
+function extractJSON(text) {
+  let cleaned = text.trim();
+  if (cleaned.startsWith("```")) {
+    cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
+  }
+  const firstBrace = cleaned.indexOf("{");
+  const lastBrace = cleaned.lastIndexOf("}");
+  if (firstBrace !== -1 && lastBrace !== -1 && lastBrace > firstBrace) {
+    cleaned = cleaned.slice(firstBrace, lastBrace + 1);
+  }
+  return JSON.parse(cleaned);
+}
+var claudeJudge;
+var init_claude = __esm({
+  "src/providers/claude.ts"() {
+    "use strict";
+    init_esm_shims();
+    init_prompts();
+    init_schema();
+    init_types();
+    claudeJudge = {
+      name: "claude",
+      async judge(input, model) {
+        const client = getClient();
+        const userPrompt = buildJudgeUserPrompt(input);
+        let response;
+        try {
+          response = await client.messages.create({
+            model,
+            max_tokens: 600,
+            temperature: 0,
+            system: JUDGE_SYSTEM_PROMPT,
+            messages: [{ role: "user", content: userPrompt }]
+          });
+        } catch (e) {
+          throw mapError(e);
+        }
+        const text = response.content.filter((b) => b.type === "text").map((b) => b.text).join("");
+        let parsed;
+        try {
+          parsed = extractJSON(text);
+        } catch (e) {
+          throw new JudgeError(
+            `Judge response was not valid JSON: ${text.slice(0, 200)}...`,
+            "parse",
+            "claude",
+            e
+          );
+        }
+        const result = JudgeOutputJSONSchema.safeParse(parsed);
+        if (!result.success) {
+          throw new JudgeError(
+            `Judge JSON failed schema validation: ${result.error.message}`,
+            "parse",
+            "claude"
+          );
+        }
+        return {
+          faithfulness: result.data.faithfulness ?? NaN,
+          correctness: result.data.correctness ?? NaN,
+          rationale: result.data.rationale,
+          rawResponse: response
+        };
+      }
+    };
+  }
+});
+
+// src/providers/openai.ts
+var openai_exports = {};
+__export(openai_exports, {
+  openaiJudge: () => openaiJudge
+});
+import OpenAI from "openai";
+function getClient2() {
+  const apiKey = process.env["OPENAI_API_KEY"];
+  if (!apiKey) {
+    throw new JudgeError(
+      "OPENAI_API_KEY not set in environment",
+      "auth",
+      "openai"
+    );
+  }
+  return new OpenAI({ apiKey });
+}
+function mapError2(e) {
+  if (e instanceof JudgeError) return e;
+  const err = e;
+  if (err.status === 401 || err.status === 403) {
+    return new JudgeError("Authentication failed", "auth", "openai", e);
+  }
+  if (err.status === 429) {
+    return new JudgeError("Rate limit exceeded", "rate_limit", "openai", e);
+  }
+  if (err.status && err.status >= 500) {
+    return new JudgeError("Provider server error", "server", "openai", e);
+  }
+  if (err.status && err.status >= 400) {
+    return new JudgeError(err.message ?? "Invalid request", "invalid", "openai", e);
+  }
+  return new JudgeError(err.message ?? "Network error", "network", "openai", e);
+}
+var OPENAI_JSON_SCHEMA, openaiJudge;
+var init_openai = __esm({
+  "src/providers/openai.ts"() {
+    "use strict";
+    init_esm_shims();
+    init_prompts();
+    init_schema();
+    init_types();
+    OPENAI_JSON_SCHEMA = {
+      type: "object",
+      properties: {
+        faithfulness: {
+          type: ["number", "null"],
+          description: "How well the answer is supported by retrieved context, null if context is uninterpretable (e.g., only IDs not text)"
+        },
+        correctness: {
+          type: ["number", "null"],
+          description: "How well the answer matches expected_answer, null if not provided"
+        },
+        rationale: {
+          type: "string",
+          description: "One or two sentences explaining the scores"
+        }
+      },
+      required: ["faithfulness", "correctness", "rationale"],
+      additionalProperties: false
+    };
+    openaiJudge = {
+      name: "openai",
+      async judge(input, model) {
+        const client = getClient2();
+        const userPrompt = buildJudgeUserPrompt(input);
+        let response;
+        try {
+          response = await client.chat.completions.create({
+            model,
+            temperature: 0,
+            max_tokens: 600,
+            messages: [
+              { role: "system", content: JUDGE_SYSTEM_PROMPT },
+              { role: "user", content: userPrompt }
+            ],
+            response_format: {
+              type: "json_schema",
+              json_schema: {
+                name: "judge_output",
+                strict: true,
+                schema: OPENAI_JSON_SCHEMA
+              }
+            }
+          });
+        } catch (e) {
+          throw mapError2(e);
+        }
+        const text = response.choices[0]?.message?.content ?? "";
+        if (!text) {
+          throw new JudgeError("Empty response from judge", "parse", "openai");
+        }
+        let parsed;
+        try {
+          parsed = JSON.parse(text);
+        } catch (e) {
+          throw new JudgeError(
+            `Judge response was not valid JSON: ${text.slice(0, 200)}`,
+            "parse",
+            "openai",
+            e
+          );
+        }
+        const result = JudgeOutputJSONSchema.safeParse(parsed);
+        if (!result.success) {
+          throw new JudgeError(
+            `Judge JSON failed schema validation: ${result.error.message}`,
+            "parse",
+            "openai"
+          );
+        }
+        return {
+          faithfulness: result.data.faithfulness ?? NaN,
+          correctness: result.data.correctness ?? NaN,
+          rationale: result.data.rationale,
+          rawResponse: response
+        };
+      }
+    };
+  }
+});
+
+// src/core/judge.ts
+async function getJudgeProvider(config) {
+  if (config.provider === "claude") {
+    const { claudeJudge: claudeJudge2 } = await Promise.resolve().then(() => (init_claude(), claude_exports));
+    return claudeJudge2;
+  }
+  if (config.provider === "openai") {
+    const { openaiJudge: openaiJudge2 } = await Promise.resolve().then(() => (init_openai(), openai_exports));
+    return openaiJudge2;
+  }
+  throw new Error(`Unknown judge provider: ${String(config.provider)}`);
+}
+async function runJudge(input, config) {
+  const provider = await getJudgeProvider(config);
+  return provider.judge(input, config.model);
+}
+var init_judge = __esm({
+  "src/core/judge.ts"() {
+    "use strict";
+    init_esm_shims();
+  }
+});
+
+// src/core/runner.ts
+import ora from "ora";
+async function runEval(opts) {
+  const startTime = Date.now();
+  const results = [];
+  const spinner = ora({ text: "Starting evaluation...", spinner: "dots" }).start();
+  for (let i = 0; i < opts.entries.length; i++) {
+    const entry = opts.entries[i];
+    spinner.text = `[${i + 1}/${opts.entries.length}] ${entry.id} \u2014 ${entry.question.slice(0, 50)}...`;
+    let result;
+    try {
+      const response = await callEndpoint(opts.config.endpoint, entry);
+      const retrieval = scoreRetrieval(entry, response, opts.config.scoring.retrievalK);
+      let judge = null;
+      if (opts.enableJudge && opts.config.judge) {
+        spinner.text = `[${i + 1}/${opts.entries.length}] ${entry.id} \u2014 judging...`;
+        try {
+          const k = opts.config.scoring.retrievalK;
+          const retrievedContext = response.sourceContents && response.sourceContents.length > 0 ? response.sourceContents.slice(0, k) : response.sources.slice(0, k);
+          const judgeOut = await runJudge(
+            {
+              question: entry.question,
+              answer: response.answer,
+              retrievedContext,
+              expectedAnswer: entry.expected_answer
+            },
+            opts.config.judge
+          );
+          judge = {
+            faithfulness: judgeOut.faithfulness,
+            correctness: judgeOut.correctness,
+            rationale: judgeOut.rationale
+          };
+        } catch (e) {
+          judge = {
+            faithfulness: NaN,
+            correctness: NaN,
+            rationale: `Judge error: ${e.message}`
+          };
+        }
+      }
+      const overallScore = computeOverallScore(retrieval, judge, opts.config.scoring.weights);
+      result = {
+        entry,
+        response,
+        error: null,
+        retrieval,
+        judge,
+        overallScore
+      };
+    } catch (e) {
+      const message = e instanceof EndpointError ? e.message : `Unexpected error: ${e.message}`;
+      result = {
+        entry,
+        response: null,
+        error: message,
+        retrieval: null,
+        judge: null,
+        overallScore: 0
+      };
+    }
+    results.push(result);
+  }
+  spinner.stop();
+  const successful = results.filter((r) => r.error === null).length;
+  const failed = results.length - successful;
+  const successResults = results.filter((r) => r.retrieval !== null);
+  const avgRetrievalPrecision = successResults.length > 0 ? successResults.reduce((s, r) => s + r.retrieval.precision, 0) / successResults.length : 0;
+  const faithScores = results.map((r) => r.judge?.faithfulness).filter((n) => typeof n === "number" && !isNaN(n));
+  const corrScores = results.map((r) => r.judge?.correctness).filter((n) => typeof n === "number" && !isNaN(n));
+  const avgFaithfulness = faithScores.length > 0 ? faithScores.reduce((s, n) => s + n, 0) / faithScores.length : NaN;
+  const avgCorrectness = corrScores.length > 0 ? corrScores.reduce((s, n) => s + n, 0) / corrScores.length : NaN;
+  const avgOverallScore = results.reduce((s, r) => s + r.overallScore, 0) / results.length;
+  const summary = {
+    total: results.length,
+    successful,
+    failed,
+    avgRetrievalPrecision,
+    avgFaithfulness,
+    avgCorrectness,
+    avgOverallScore,
+    passed: avgOverallScore >= opts.threshold,
+    durationMs: Date.now() - startTime
+  };
+  return { results, summary };
+}
+var init_runner = __esm({
+  "src/core/runner.ts"() {
+    "use strict";
+    init_esm_shims();
+    init_endpoint();
+    init_scorer();
+    init_judge();
+  }
+});
+
+// src/formatters/table.ts
+import Table from "cli-table3";
+import chalk from "chalk";
+function scoreColor(score) {
+  if (score >= 0.8) return chalk.green;
+  if (score >= 0.5) return chalk.yellow;
+  return chalk.red;
+}
+function fmtScore(s) {
+  if (s === void 0 || isNaN(s)) return chalk.gray("\u2014");
+  return scoreColor(s)(s.toFixed(2));
+}
+function renderTable(results) {
+  const table = new Table({
+    head: [
+      chalk.bold("ID"),
+      chalk.bold("Question"),
+      chalk.bold("Retr"),
+      chalk.bold("Faith"),
+      chalk.bold("Corr"),
+      chalk.bold("Score"),
+      chalk.bold("Status")
+    ],
+    colWidths: [10, 32, 8, 8, 8, 8, 20],
+    wordWrap: true
+  });
+  for (const r of results) {
+    if (r.error) {
+      table.push([
+        r.entry.id,
+        r.entry.question.slice(0, 30),
+        chalk.gray("\u2014"),
+        chalk.gray("\u2014"),
+        chalk.gray("\u2014"),
+        chalk.gray("\u2014"),
+        chalk.red(r.error.slice(0, 18))
+      ]);
+      continue;
+    }
+    const ret = r.retrieval;
+    const j = r.judge;
+    table.push([
+      r.entry.id,
+      r.entry.question.slice(0, 30),
+      ret.found ? chalk.green("\u2713") : chalk.red("\u2717"),
+      fmtScore(j?.faithfulness),
+      fmtScore(j?.correctness),
+      scoreColor(r.overallScore)(r.overallScore.toFixed(2)),
+      chalk.green("ok")
+    ]);
+  }
+  return table.toString();
+}
+function renderSummary(summary, threshold) {
+  const lines = [];
+  lines.push("");
+  lines.push(chalk.bold("Summary"));
+  lines.push(chalk.gray("\u2500".repeat(60)));
+  lines.push(`Total questions:         ${summary.total}`);
+  lines.push(`Successful:              ${chalk.green(String(summary.successful))}`);
+  if (summary.failed > 0) {
+    lines.push(`Failed:                  ${chalk.red(String(summary.failed))}`);
+  }
+  lines.push(
+    `Avg retrieval precision: ${scoreColor(summary.avgRetrievalPrecision)(summary.avgRetrievalPrecision.toFixed(3))}`
+  );
+  lines.push(`Avg faithfulness:        ${fmtScore(summary.avgFaithfulness)}`);
+  lines.push(`Avg correctness:         ${fmtScore(summary.avgCorrectness)}`);
+  lines.push(
+    `Avg overall score:       ${scoreColor(summary.avgOverallScore)(summary.avgOverallScore.toFixed(3))}`
+  );
+  lines.push(`Threshold:               ${threshold.toFixed(2)}`);
+  lines.push(`Duration:                ${(summary.durationMs / 1e3).toFixed(1)}s`);
+  lines.push(chalk.gray("\u2500".repeat(60)));
+  lines.push(
+    summary.passed ? chalk.green.bold("\u2713 PASSED") : chalk.red.bold("\u2717 FAILED \u2014 below threshold")
+  );
+  lines.push("");
+  return lines.join("\n");
+}
+var init_table = __esm({
+  "src/formatters/table.ts"() {
+    "use strict";
+    init_esm_shims();
+  }
+});
+
+// src/formatters/csv.ts
+function escape(value) {
+  if (value === void 0 || value === null) return "";
+  if (typeof value === "number") return isNaN(value) ? "" : String(value);
+  const s = String(value);
+  if (s.includes(",") || s.includes('"') || s.includes("\n")) {
+    return `"${s.replaceAll('"', '""')}"`;
+  }
+  return s;
+}
+function renderCSV(results, summary) {
+  const lines = [];
+  lines.push(
+    [
+      "id",
+      "question",
+      "expected_answer",
+      "rag_answer",
+      "expected_sources",
+      "retrieved_sources",
+      "retrieval_found",
+      "retrieval_precision",
+      "faithfulness",
+      "correctness",
+      "overall_score",
+      "judge_rationale",
+      "latency_ms",
+      "error"
+    ].map(escape).join(",")
+  );
+  for (const r of results) {
+    lines.push(
+      [
+        escape(r.entry.id),
+        escape(r.entry.question),
+        escape(r.entry.expected_answer),
+        escape(r.response?.answer),
+        escape(
+          Array.isArray(r.entry.expected_source) ? r.entry.expected_source.join("|") : r.entry.expected_source
+        ),
+        escape(r.response?.sources.join("|")),
+        escape(r.retrieval ? r.retrieval.found ? "true" : "false" : void 0),
+        escape(r.retrieval?.precision),
+        escape(r.judge?.faithfulness),
+        escape(r.judge?.correctness),
+        escape(r.overallScore),
+        escape(r.judge?.rationale),
+        escape(r.response?.latencyMs),
+        escape(r.error ?? void 0)
+      ].join(",")
+    );
+  }
+  lines.push("");
+  lines.push(
+    `# summary,total=${summary.total},successful=${summary.successful},failed=${summary.failed}`
+  );
+  lines.push(`# summary,avg_retrieval_precision=${summary.avgRetrievalPrecision.toFixed(4)}`);
+  lines.push(
+    `# summary,avg_faithfulness=${isNaN(summary.avgFaithfulness) ? "n/a" : summary.avgFaithfulness.toFixed(4)}`
+  );
+  lines.push(
+    `# summary,avg_correctness=${isNaN(summary.avgCorrectness) ? "n/a" : summary.avgCorrectness.toFixed(4)}`
+  );
+  lines.push(`# summary,avg_overall_score=${summary.avgOverallScore.toFixed(4)}`);
+  lines.push(`# summary,passed=${summary.passed}`);
+  lines.push(`# summary,duration_ms=${summary.durationMs}`);
+  return lines.join("\n");
+}
+var init_csv = __esm({
+  "src/formatters/csv.ts"() {
+    "use strict";
+    init_esm_shims();
+  }
+});
+
+// src/formatters/json.ts
+function round4(n) {
+  return Math.round(n * 1e4) / 1e4;
+}
+function numOrNull(v, formatter) {
+  if (v === void 0) return null;
+  if (isNaN(v)) return null;
+  return formatter ? formatter(v) : v;
+}
+function renderJSON(results, summary) {
+  return {
+    summary: {
+      total: summary.total,
+      successful: summary.successful,
+      failed: summary.failed,
+      avgRetrievalPrecision: round4(summary.avgRetrievalPrecision),
+      avgFaithfulness: numOrNull(summary.avgFaithfulness, round4),
+      avgCorrectness: numOrNull(summary.avgCorrectness, round4),
+      avgOverallScore: round4(summary.avgOverallScore),
+      passed: summary.passed,
+      durationMs: summary.durationMs,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString()
+    },
+    results: results.map((r) => ({
+      id: r.entry.id,
+      question: r.entry.question,
+      expectedAnswer: r.entry.expected_answer,
+      ragAnswer: r.response?.answer ?? null,
+      expectedSources: Array.isArray(r.entry.expected_source) ? r.entry.expected_source : [r.entry.expected_source],
+      retrievedSources: r.response?.sources ?? null,
+      retrieval: r.retrieval ? { found: r.retrieval.found, precision: round4(r.retrieval.precision) } : null,
+      judge: r.judge ? {
+        faithfulness: numOrNull(r.judge.faithfulness, round4),
+        correctness: numOrNull(r.judge.correctness, round4),
+        rationale: r.judge.rationale ?? ""
+      } : null,
+      overallScore: round4(r.overallScore),
+      latencyMs: r.response?.latencyMs ?? null,
+      error: r.error
+    }))
+  };
+}
+var init_json = __esm({
+  "src/formatters/json.ts"() {
+    "use strict";
+    init_esm_shims();
+  }
+});
+
+// src/commands/run.ts
+var run_exports = {};
+__export(run_exports, {
+  runCommand: () => runCommand
+});
+import { mkdirSync, writeFileSync } from "fs";
+import { join } from "path";
+import chalk2 from "chalk";
+async function runCommand(opts) {
+  try {
+    const config = loadConfig(opts.config);
+    const questionsPath = opts.questions ?? "eval-set.jsonl";
+    const entries = loadEvalSet(questionsPath);
+    const threshold = parseFloat(opts.threshold);
+    if (isNaN(threshold) || threshold < 0 || threshold > 1) {
+      console.error(chalk2.red(`Invalid threshold: ${opts.threshold} (must be 0-1)`));
+      return 2;
+    }
+    const enableJudge = opts.judge !== false;
+    if (enableJudge && typeof opts.judge === "string" && config.judge) {
+      if (opts.judge !== "claude" && opts.judge !== "openai") {
+        console.error(chalk2.red(`Invalid --judge value: "${opts.judge}" (use claude|openai)`));
+        return 2;
+      }
+      config.judge = {
+        provider: opts.judge,
+        model: opts.judge === "openai" ? "gpt-4o-mini" : "claude-sonnet-4-6"
+      };
+    }
+    console.log(chalk2.gray(`Loaded ${entries.length} questions from ${questionsPath}`));
+    console.log(chalk2.gray(`Endpoint: ${config.endpoint.url}`));
+    if (enableJudge && config.judge) {
+      console.log(chalk2.gray(`Judge: ${config.judge.provider} (${config.judge.model})`));
+    } else {
+      console.log(chalk2.gray("Judge: disabled (retrieval-only)"));
+    }
+    console.log("");
+    const { results, summary } = await runEval({ config, entries, threshold, enableJudge });
+    console.log(renderTable(results));
+    console.log(renderSummary(summary, threshold));
+    try {
+      mkdirSync(opts.output, { recursive: true });
+      const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+      const csvPath = join(opts.output, `eval-${timestamp}.csv`);
+      writeFileSync(csvPath, renderCSV(results, summary));
+      console.log(chalk2.gray(`CSV report:  ${csvPath}`));
+      const jsonPath = join(opts.output, `eval-${timestamp}.json`);
+      writeFileSync(jsonPath, JSON.stringify(renderJSON(results, summary), null, 2));
+      console.log(chalk2.gray(`JSON report: ${jsonPath}`));
+    } catch (e) {
+      console.error(chalk2.yellow(`Warning: failed to write reports: ${e.message}`));
+    }
+    return summary.passed ? 0 : 1;
+  } catch (e) {
+    if (e instanceof ConfigError) {
+      console.error(chalk2.red(e.message));
+      return 2;
+    }
+    console.error(chalk2.red(`Unexpected error: ${e.message}`));
+    if (process.env["DEBUG"]) {
+      console.error(e.stack);
+    }
+    return 3;
+  }
+}
+var init_run = __esm({
+  "src/commands/run.ts"() {
+    "use strict";
+    init_esm_shims();
+    init_loader();
+    init_runner();
+    init_table();
+    init_csv();
+    init_json();
+  }
+});
+
+// src/index.ts
+init_esm_shims();
+import { Command } from "commander";
+import { readFileSync as readFileSync2 } from "fs";
+import { fileURLToPath as fileURLToPath2 } from "url";
+import { dirname, join as join2 } from "path";
+var __filename2 = fileURLToPath2(import.meta.url);
+var __dirname2 = dirname(__filename2);
+var pkg = JSON.parse(
+  readFileSync2(join2(__dirname2, "..", "package.json"), "utf-8")
+);
+var program = new Command();
+program.name("rag-eval").description("Evaluate RAG pipelines: retrieval, faithfulness, correctness.").version(pkg.version);
+program.command("run").description("Run evaluation against a RAG endpoint").option("-c, --config <path>", "config file path", "rag-eval.config.json").option("-q, --questions <path>", "eval-set JSONL file").option("-j, --judge <provider>", "judge LLM provider: claude|openai (overrides config)").option("--no-judge", "skip judge LLM (retrieval scoring only, no API costs)").option("-o, --output <dir>", "output directory for reports", "./rag-eval-output").option("--threshold <number>", "min score to exit 0 (0-1)", "0.7").action(async (opts) => {
+  const { runCommand: runCommand2 } = await Promise.resolve().then(() => (init_run(), run_exports));
+  const code = await runCommand2(opts);
+  process.exit(code);
+});
+program.parse();