@axlsdk/eval 0.13.6 → 0.13.8

package/dist/chunk-F2AALK5U.js

@@ -1,290 +0,0 @@
-// src/compare.ts
-function round(n) {
-  return Math.round(n * 100) / 100;
-}
-function evalCompare(baseline, candidate) {
-  if (baseline.dataset !== candidate.dataset) {
-    throw new Error(
-      `Cannot compare evals from different datasets: "${baseline.dataset}" vs "${candidate.dataset}"`
-    );
-  }
-  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();
-  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();
-  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {
-    throw new Error(`Cannot compare evals with different scorers`);
-  }
-  const scorers = {};
-  for (const name of baselineScorerNames) {
-    const bMean = baseline.summary.scorers[name].mean;
-    const cMean = candidate.summary.scorers[name].mean;
-    const delta = round(cMean - bMean);
-    const deltaPercent = bMean > 0 ? round(delta / bMean * 100) : 0;
-    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };
-  }
-  const regressions = [];
-  const improvements = [];
-  const minLength = Math.min(baseline.items.length, candidate.items.length);
-  for (let i = 0; i < minLength; i++) {
-    const bItem = baseline.items[i];
-    const cItem = candidate.items[i];
-    if (bItem.error || cItem.error) continue;
-    for (const name of baselineScorerNames) {
-      const bScore = bItem.scores[name];
-      const cScore = cItem.scores[name];
-      if (bScore == null || cScore == null) continue;
-      const delta = round(cScore - bScore);
-      if (delta < -0.1)
-        regressions.push({
-          itemIndex: i,
-          input: bItem.input,
-          scorer: name,
-          baselineScore: bScore,
-          candidateScore: cScore,
-          delta
-        });
-      else if (delta > 0.1)
-        improvements.push({
-          itemIndex: i,
-          input: bItem.input,
-          scorer: name,
-          baselineScore: bScore,
-          candidateScore: cScore,
-          delta
-        });
-    }
-  }
-  let timing;
-  if (baseline.summary.timing && candidate.summary.timing) {
-    const bMean = baseline.summary.timing.mean;
-    const cMean = candidate.summary.timing.mean;
-    const delta = round(cMean - bMean);
-    const deltaPercent = bMean > 0 ? round(delta / bMean * 100) : 0;
-    timing = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };
-  }
-  let cost;
-  if (baseline.totalCost > 0 || candidate.totalCost > 0) {
-    const delta = round(candidate.totalCost - baseline.totalCost);
-    const deltaPercent = baseline.totalCost > 0 ? round(delta / baseline.totalCost * 100) : 0;
-    cost = {
-      baselineTotal: baseline.totalCost,
-      candidateTotal: candidate.totalCost,
-      delta,
-      deltaPercent
-    };
-  }
-  const parts = [];
-  for (const name of baselineScorerNames) {
-    const s = scorers[name];
-    if (Math.abs(s.delta) > 1e-3) {
-      const direction = s.delta > 0 ? "improves" : "regresses";
-      const sign = s.delta > 0 ? "+" : "";
-      parts.push(
-        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`
-      );
-    }
-  }
-  if (timing && Math.abs(timing.deltaPercent) > 1) {
-    const dir = timing.delta > 0 ? "slower" : "faster";
-    parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);
-  }
-  if (cost && Math.abs(cost.deltaPercent) > 1) {
-    const dir = cost.delta > 0 ? "more expensive" : "cheaper";
-    parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);
-  }
-  const summaryStr = `candidate ${parts.length > 0 ? parts.join(", ") : "no significant changes"} with ${regressions.length} regressions and ${improvements.length} improvements`;
-  return {
-    baseline: { id: baseline.id, metadata: baseline.metadata },
-    candidate: { id: candidate.id, metadata: candidate.metadata },
-    scorers,
-    timing,
-    cost,
-    regressions,
-    improvements,
-    summary: summaryStr
-  };
-}
-
-// src/scorer.ts
-function normalizeScorerResult(result) {
-  return typeof result === "number" ? { score: result } : result;
-}
-function scorer(config) {
-  return {
-    name: config.name,
-    description: config.description,
-    isLlm: false,
-    score: config.score
-  };
-}
-
-// src/runner.ts
-import { randomUUID } from "crypto";
-function computeStats(scores) {
-  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };
-  const sorted = [...scores].sort((a, b) => a - b);
-  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;
-  const min = sorted[0];
-  const max = sorted[sorted.length - 1];
-  const p50 = sorted[Math.floor(sorted.length * 0.5)];
-  const p95 = sorted[Math.floor(sorted.length * 0.95)];
-  return { mean: round2(mean), min: round2(min), max: round2(max), p50: round2(p50), p95: round2(p95) };
-}
-function round2(n) {
-  return Math.round(n * 1e3) / 1e3;
-}
-function parseCost(cost) {
-  const match = cost.match(/^\$?([\d.]+)$/);
-  if (!match) throw new Error(`Invalid cost format: "${cost}"`);
-  return parseFloat(match[1]);
-}
-async function runEval(config, executeWorkflow, runtime) {
-  const startTime = Date.now();
-  const id = randomUUID();
-  const items = await config.dataset.getItems();
-  const concurrency = config.concurrency ?? 5;
-  const budgetLimit = config.budget ? parseCost(config.budget) : void 0;
-  const scorerContext = {
-    resolveProvider: (uri) => {
-      if (typeof runtime.resolveProvider !== "function") {
-        throw new Error(
-          `LLM scorers require a runtime with resolveProvider(). Ensure you are using a real AxlRuntime instance, not a mock.`
-        );
-      }
-      return runtime.resolveProvider(uri);
-    }
-  };
-  const evalItems = new Array(items.length);
-  let totalCost = 0;
-  let budgetExceeded = false;
-  async function processItem(item, itemIndex) {
-    if (budgetExceeded) {
-      evalItems[itemIndex] = {
-        input: item.input,
-        annotations: item.annotations,
-        output: null,
-        error: "Budget exceeded",
-        scores: {}
-      };
-      return;
-    }
-    const evalItem = {
-      input: item.input,
-      annotations: item.annotations,
-      output: null,
-      scores: {}
-    };
-    const itemStart = Date.now();
-    try {
-      const result = await executeWorkflow(item.input, runtime);
-      evalItem.duration = Date.now() - itemStart;
-      evalItem.output = result.output;
-      evalItem.cost = result.cost;
-      if (result.cost != null) {
-        totalCost += result.cost;
-      }
-    } catch (err) {
-      evalItem.duration = Date.now() - itemStart;
-      evalItem.error = err instanceof Error ? err.message : String(err);
-      evalItems[itemIndex] = evalItem;
-      return;
-    }
-    if (budgetLimit != null && totalCost > budgetLimit) {
-      budgetExceeded = true;
-    }
-    evalItem.scoreDetails = {};
-    let itemScorerCost = 0;
-    for (const scorer2 of config.scorers) {
-      const scorerStart = Date.now();
-      try {
-        const raw = await scorer2.score(
-          evalItem.output,
-          item.input,
-          item.annotations,
-          scorerContext
-        );
-        const scorerResult = normalizeScorerResult(raw);
-        if (scorerResult.cost != null) {
-          itemScorerCost += scorerResult.cost;
-          totalCost += scorerResult.cost;
-        }
-        const scorerDuration = Date.now() - scorerStart;
-        if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
-          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];
-          evalItem.scorerErrors.push(
-            `Scorer "${scorer2.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`
-          );
-          evalItem.scores[scorer2.name] = null;
-          evalItem.scoreDetails[scorer2.name] = {
-            score: null,
-            metadata: scorerResult.metadata,
-            duration: scorerDuration,
-            cost: scorerResult.cost
-          };
-        } else {
-          evalItem.scores[scorer2.name] = round2(scorerResult.score);
-          evalItem.scoreDetails[scorer2.name] = {
-            score: round2(scorerResult.score),
-            metadata: scorerResult.metadata,
-            duration: scorerDuration,
-            cost: scorerResult.cost
-          };
-        }
-      } catch (err) {
-        const errCost = typeof err?.cost === "number" ? err.cost : void 0;
-        if (errCost != null) {
-          itemScorerCost += errCost;
-          totalCost += errCost;
-        }
-        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];
-        evalItem.scorerErrors.push(
-          `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`
-        );
-        evalItem.scores[scorer2.name] = null;
-        evalItem.scoreDetails[scorer2.name] = {
-          score: null,
-          duration: Date.now() - scorerStart,
-          cost: errCost
-        };
-      }
-    }
-    evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
-    evalItems[itemIndex] = evalItem;
-  }
-  let index = 0;
-  async function runNext() {
-    while (index < items.length) {
-      const currentIndex = index++;
-      await processItem(items[currentIndex], currentIndex);
-    }
-  }
-  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());
-  await Promise.all(workers);
-  const failures = evalItems.filter((i) => i.error).length;
-  const scorerNames = config.scorers.map((s) => s.name);
-  const scorerStats = {};
-  for (const name of scorerNames) {
-    const scores = evalItems.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
-    scorerStats[name] = computeStats(scores);
-  }
-  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
-  const timing = durations.length > 0 ? computeStats(durations) : void 0;
-  return {
-    id,
-    workflow: config.workflow,
-    dataset: config.dataset.name,
-    metadata: config.metadata ?? {},
-    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-    totalCost,
-    duration: Date.now() - startTime,
-    items: evalItems,
-    summary: { count: items.length, failures, scorers: scorerStats, timing }
-  };
-}
-
-export {
-  evalCompare,
-  normalizeScorerResult,
-  scorer,
-  runEval
-};
-//# sourceMappingURL=chunk-F2AALK5U.js.map

package/dist/chunk-F2AALK5U.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/compare.ts","../src/scorer.ts","../src/runner.ts"],"sourcesContent":["import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore == null || cScore == null) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          itemIndex: i,\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          itemIndex: i,\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  // Timing comparison (use pre-computed summary.timing when available)\n  let timing: EvalComparison['timing'];\n  if (baseline.summary.timing && candidate.summary.timing) {\n    const bMean = baseline.summary.timing.mean;\n    const cMean = candidate.summary.timing.mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    timing = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  // Cost comparison\n  let cost: EvalComparison['cost'];\n  if (baseline.totalCost > 0 || candidate.totalCost > 0) {\n    const delta = round(candidate.totalCost - baseline.totalCost);\n    const deltaPercent = baseline.totalCost > 0 ? round((delta / baseline.totalCost) * 100) : 0;\n    cost = {\n      baselineTotal: baseline.totalCost,\n      candidateTotal: candidate.totalCost,\n      delta,\n      deltaPercent,\n    };\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  if (timing && Math.abs(timing.deltaPercent) > 1) {\n    const dir = timing.delta > 0 ? 'slower' : 'faster';\n    parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);\n  }\n  if (cost && Math.abs(cost.deltaPercent) > 1) {\n    const dir = cost.delta > 0 ? 'more expensive' : 'cheaper';\n    parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    timing,\n    cost,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","/** Result from a scorer that includes metadata beyond the numeric score. */\nexport type ScorerResult = {\n  score: number;\n  metadata?: Record<string, unknown>;\n  cost?: number;\n};\n\n/** Normalize a scorer return value to a ScorerResult. */\nexport function normalizeScorerResult(result: number | ScorerResult): ScorerResult {\n  return typeof result === 'number' ? { score: result } : result;\n}\n\n/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n  /** Resolve a provider:model URI to a provider instance and model name. */\n  resolveProvider: (modelUri: string) => {\n    provider: {\n      chat(\n        messages: { role: string; content: string }[],\n        options: {\n          model: string;\n          temperature?: number;\n          responseFormat?: { type: string; json_schema?: unknown };\n        },\n      ): Promise<{ content: string; cost?: number }>;\n    };\n    model: string;\n  };\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number | ScorerResult;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(\n    output: TOutput,\n    input: TInput,\n    annotations?: TAnnotations,\n    context?: ScorerContext,\n  ): number | ScorerResult | Promise<number | ScorerResult>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n  };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { normalizeScorerResult } from './scorer.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime: AxlRuntime,\n  ) => Promise<{ output: unknown; cost?: number }>,\n  runtime: AxlRuntime,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  // Create a scorer context that LLM scorers use to resolve providers.\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n  };\n\n  const evalItems: EvalItem[] = new Array(items.length);\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0], itemIndex: number): Promise<void> {\n    if (budgetExceeded) {\n      evalItems[itemIndex] = {\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      };\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    const itemStart = Date.now();\n    try {\n      const result = await executeWorkflow(item.input, runtime);\n      evalItem.duration = Date.now() - itemStart;\n      evalItem.output = result.output;\n      evalItem.cost = result.cost;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.duration = Date.now() - itemStart;\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems[itemIndex] = evalItem;\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    evalItem.scoreDetails = {};\n    let itemScorerCost = 0;\n\n    for (const scorer of config.scorers) {\n      const scorerStart = Date.now();\n      try {\n        const raw = await scorer.score(\n          evalItem.output,\n          item.input,\n          item.annotations,\n          scorerContext,\n        );\n        const scorerResult = normalizeScorerResult(raw);\n\n        if (scorerResult.cost != null) {\n          itemScorerCost += scorerResult.cost;\n          totalCost += scorerResult.cost;\n        }\n\n        const scorerDuration = Date.now() - scorerStart;\n\n        if (\n          !Number.isFinite(scorerResult.score) ||\n          scorerResult.score < 0 ||\n          scorerResult.score > 1\n        ) {\n          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n          evalItem.scorerErrors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = null;\n          evalItem.scoreDetails[scorer.name] = {\n            score: null,\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        } else {\n          evalItem.scores[scorer.name] = round(scorerResult.score);\n          evalItem.scoreDetails[scorer.name] = {\n            score: round(scorerResult.score),\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        }\n      } catch (err) {\n        // Capture cost from error (LLM scorer attaches it)\n        const errCost = typeof (err as any)?.cost === 'number' ? (err as any).cost : undefined;\n        if (errCost != null) {\n          itemScorerCost += errCost;\n          totalCost += errCost;\n        }\n        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n        evalItem.scorerErrors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = null;\n        evalItem.scoreDetails[scorer.name] = {\n          score: null,\n          duration: Date.now() - scorerStart,\n          cost: errCost,\n        };\n      }\n    }\n    evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n    evalItems[itemIndex] = evalItem;\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex], currentIndex);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration!);\n  const timing = durations.length > 0 ? computeStats(durations) : undefined;\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats, timing },\n  };\n}\n"],"mappings":";AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,UAAU,QAAQ,UAAU,KAAM;AACtC,YAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,WAAW;AAAA,UACX,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,WAAW;AAAA,UACX,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAGA,MAAI;AACJ,MAAI,SAAS,QAAQ,UAAU,UAAU,QAAQ,QAAQ;AACvD,UAAM,QAAQ,SAAS,QAAQ,OAAO;AACtC,UAAM,QAAQ,UAAU,QAAQ,OAAO;AACvC,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,aAAS,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EAC5E;AAGA,MAAI;AACJ,MAAI,SAAS,YAAY,KAAK,UAAU,YAAY,GAAG;AACrD,UAAM,QAAQ,MAAM,UAAU,YAAY,SAAS,SAAS;AAC5D,UAAM,eAAe,SAAS,YAAY,IAAI,MAAO,QAAQ,SAAS,YAAa,GAAG,IAAI;AAC1F,WAAO;AAAA,MACL,eAAe,SAAS;AAAA,MACxB,gBAAgB,UAAU;AAAA,MAC1B;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,MAAI,UAAU,KAAK,IAAI,OAAO,YAAY,IAAI,GAAG;AAC/C,UAAM,MAAM,OAAO,QAAQ,IAAI,WAAW;AAC1C,UAAM,KAAK,GAAG,KAAK,IAAI,OAAO,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAClE;AACA,MAAI,QAAQ,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG;AAC3C,UAAM,MAAM,KAAK,QAAQ,IAAI,mBAAmB;AAChD,UAAM,KAAK,GAAG,KAAK,IAAI,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAChE;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AC5GO,SAAS,sBAAsB,QAA6C;AACjF,SAAO,OAAO,WAAW,WAAW,EAAE,OAAO,OAAO,IAAI;AAC1D;AA4CO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;AC3DA,SAAS,kBAAkB;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMA,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAEA,eAAsB,QACpB,QACA,iBAIA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,KAAK,WAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,IAAI,MAAM,MAAM,MAAM;AACpD,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAyB,WAAkC;AACpF,QAAI,gBAAgB;AAClB,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,UAAM,YAAY,KAAK,IAAI;AAC3B,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,SAAS,OAAO;AACzB,eAAS,OAAO,OAAO;AACvB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,SAAS,IAAI;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,aAAS,eAAe,CAAC;AACzB,QAAI,iBAAiB;AAErB,eAAWC,WAAU,OAAO,SAAS;AACnC,YAAM,cAAc,KAAK,IAAI;AAC7B,UAAI;AACF,cAAM,MAAM,MAAMA,QAAO;AAAA,UACvB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AACA,cAAM,eAAe,sBAAsB,GAAG;AAE9C,YAAI,aAAa,QAAQ,MAAM;AAC7B,4BAAkB,aAAa;AAC/B,uBAAa,aAAa;AAAA,QAC5B;AAEA,cAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,YACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAWA,QAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACnH;AACA,mBAAS,OAAOA,QAAO,IAAI,IAAI;AAC/B,mBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,YACnC,OAAO;AAAA,YACP,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF,OAAO;AACL,mBAAS,OAAOA,QAAO,IAAI,IAAID,OAAM,aAAa,KAAK;AACvD,mBAAS,aAAaC,QAAO,IAAI,IAAI;AAAA,YACnC,OAAOD,OAAM,aAAa,KAAK;AAAA,YAC/B,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AAEZ,cAAM,UAAU,OAAQ,KAAa,SAAS,WAAY,IAAY,OAAO;AAC7E,YAAI,WAAW,MAAM;AACnB,4BAAkB;AAClB,uBAAa;AAAA,QACf;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAWC,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAOA,QAAO,IAAI,IAAI;AAC/B,iBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,UACnC,OAAO;AAAA,UACP,UAAU,KAAK,IAAI,IAAI;AAAA,UACvB,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,aAAS,aAAa,iBAAiB,IAAI,iBAAiB;AAC5D,cAAU,SAAS,IAAI;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,GAAG,YAAY;AAAA,IACrD;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,QAAM,YAAY,UAAU,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,QAAS;AAChG,QAAM,SAAS,UAAU,SAAS,IAAI,aAAa,SAAS,IAAI;AAEhE,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,aAAa,OAAO;AAAA,EACzE;AACF;","names":["round","scorer"]}