@amodalai/core 0.1.0

package/dist/src/eval/eval-diff.d.ts

@@ -0,0 +1,11 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+import type { EvalSuiteResult, EvalDiff } from './eval-types.js';
+/**
+ * Compare current eval results against a baseline.
+ * Returns a diff for each eval case showing regressions, improvements, etc.
+ */
+export declare function diffEvalResults(current: EvalSuiteResult, baseline: EvalSuiteResult): EvalDiff[];

package/dist/src/eval/eval-diff.js

@@ -0,0 +1,97 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+/**
+ * Compare current eval results against a baseline.
+ * Returns a diff for each eval case showing regressions, improvements, etc.
+ */
+export function diffEvalResults(current, baseline) {
+    const currentMap = new Map(current.results.map((r) => [r.eval.name, r]));
+    const baselineMap = new Map(baseline.results.map((r) => [r.eval.name, r]));
+    const allNames = new Set([...currentMap.keys(), ...baselineMap.keys()]);
+    const diffs = [];
+    for (const name of allNames) {
+        const cur = currentMap.get(name);
+        const base = baselineMap.get(name);
+        if (cur && !base) {
+            diffs.push({
+                evalName: name,
+                status: 'new',
+                currentPassed: cur.passed,
+                baselinePassed: null,
+                assertionChanges: cur.assertions.map((a) => ({
+                    text: a.text,
+                    currentPassed: a.passed,
+                    baselinePassed: null,
+                    status: 'new',
+                })),
+            });
+            continue;
+        }
+        if (!cur && base) {
+            diffs.push({
+                evalName: name,
+                status: 'removed',
+                currentPassed: null,
+                baselinePassed: base.passed,
+                assertionChanges: base.assertions.map((a) => ({
+                    text: a.text,
+                    currentPassed: null,
+                    baselinePassed: a.passed,
+                    status: 'removed',
+                })),
+            });
+            continue;
+        }
+        if (cur && base) {
+            const assertionChanges = diffAssertions(cur.assertions, base.assertions);
+            let status = 'unchanged';
+            if (cur.passed && !base.passed) {
+                status = 'improved';
+            }
+            else if (!cur.passed && base.passed) {
+                status = 'regressed';
+            }
+            else if (assertionChanges.some((a) => a.status !== 'unchanged')) {
+                // Same overall pass/fail but assertion-level changes
+                status = assertionChanges.some((a) => a.status === 'regressed') ? 'regressed' : 'improved';
+            }
+            diffs.push({
+                evalName: name,
+                status,
+                currentPassed: cur.passed,
+                baselinePassed: base.passed,
+                assertionChanges,
+            });
+        }
+    }
+    return diffs;
+}
+function diffAssertions(current, baseline) {
+    const baseMap = new Map(baseline.map((a) => [a.text, a.passed]));
+    const curMap = new Map(current.map((a) => [a.text, a.passed]));
+    const allTexts = new Set([...baseMap.keys(), ...curMap.keys()]);
+    const result = [];
+    for (const text of allTexts) {
+        const curPassed = curMap.get(text) ?? null;
+        const basePassed = baseMap.get(text) ?? null;
+        let status = 'unchanged';
+        if (curPassed !== null && basePassed === null) {
+            status = 'new';
+        }
+        else if (curPassed === null && basePassed !== null) {
+            status = 'removed';
+        }
+        else if (curPassed === true && basePassed === false) {
+            status = 'improved';
+        }
+        else if (curPassed === false && basePassed === true) {
+            status = 'regressed';
+        }
+        result.push({ text, currentPassed: curPassed, baselinePassed: basePassed, status });
+    }
+    return result;
+}
+//# sourceMappingURL=eval-diff.js.map

package/dist/src/eval/eval-diff.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-diff.js","sourceRoot":"","sources":["../../../src/eval/eval-diff.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,OAAwB,EACxB,QAAyB;IAEzB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3E,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,EAAE,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,KAAK,GAAe,EAAE,CAAC;IAE7B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEnC,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YACjB,KAAK,CAAC,IAAI,CAAC;gBACT,QAAQ,EAAE,IAAI;gBACd,MAAM,EAAE,KAAK;gBACb,aAAa,EAAE,GAAG,CAAC,MAAM;gBACzB,cAAc,EAAE,IAAI;gBACpB,gBAAgB,EAAE,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC3C,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,aAAa,EAAE,CAAC,CAAC,MAAM;oBACvB,cAAc,EAAE,IAAI;oBACpB,MAAM,EAAE,KAAuB;iBAChC,CAAC,CAAC;aACJ,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC;YACjB,KAAK,CAAC,IAAI,CAAC;gBACT,QAAQ,EAAE,IAAI;gBACd,MAAM,EAAE,SAAS;gBACjB,aAAa,EAAE,IAAI;gBACnB,cAAc,EAAE,IAAI,CAAC,MAAM;gBAC3B,gBAAgB,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC5C,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,aAAa,EAAE,IAAI;oBACnB,cAAc,EAAE,CAAC,CAAC,MAAM;oBACxB,MAAM,EAAE,SAA2B;iBACpC,CAAC,CAAC;aACJ,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,MAAM,gBAAgB,GAAG,cAAc,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACzE,IAAI,MAAM,GAAmB,WAAW,CAAC;YAEzC,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC/B,MAAM,GAAG,UAAU,CAAC;YACtB,CAAC;iBAAM,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACtC,MAAM,GAAG,WAAW,CAAC;YACvB,CAAC;iBAAM,IAAI,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC,EAAE,CAAC;gBAClE,qDAAqD;gBACrD,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;YAC7F,CAAC;YAED,KAAK,CAAC,IAAI,CAAC;gBACT,QAAQ,EAAE,IAAI;gBACd,MAAM;gBACN,aAAa,EAAE,GAAG,CAAC,MAAM;gBACzB,cAAc,EAAE,IAAI,CAAC,MAAM;gBAC3B,gBAAgB;aACjB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CACrB,OAA+C,EAC/C,QAAgD;IAEhD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,EAAE,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAChE,MAAM,MAAM,GAAiH,EAAE,CAAC;IAEhI,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;QAC3C,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;QAE7C,IAAI,MAAM,GAAmB,WAAW,CAAC;QACzC,IAAI,SAAS,KAAK,IAAI,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YAC9C,MAAM,GAAG,KAAK,CAAC;QACjB,CAAC;aAAM,IAAI,SAAS,KAAK,IAAI,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YACrD,MAAM,GAAG,SAAS,CAAC;QACrB,CAAC;aAAM,IAAI,SAAS,KAAK,IAAI,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;YACtD,MAAM,GAAG,UAAU,CAAC;QACtB,CAAC;aAAM,IAAI,SAAS,KAAK,KAAK,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YACtD,MAAM,GAAG,WAAW,CAAC;QACvB,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,aAAa,EAAE,SAAS,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,EAAC,CAAC,CAAC;IACpF,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/src/eval/eval-formatter.d.ts

@@ -0,0 +1,23 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+import type { EvalSuiteResult, EvalDiff, EvalModelInfo } from './eval-types.js';
+/**
+ * Format eval results as a terminal table with token usage and cost.
+ */
+export declare function formatEvalTable(result: EvalSuiteResult, model?: EvalModelInfo): string;
+/**
+ * Format eval diff as a terminal table.
+ */
+export declare function formatDiffTable(diffs: EvalDiff[]): string;
+/**
+ * Format a side-by-side comparison of two eval suite results.
+ * Shows model, accuracy, tokens, cost, and duration for each eval.
+ */
+export declare function formatComparisonTable(current: EvalSuiteResult, baseline: EvalSuiteResult, diffs: EvalDiff[]): string;
+/**
+ * Format eval results as CI-friendly markdown.
+ */
+export declare function formatEvalMarkdown(result: EvalSuiteResult, diffs?: EvalDiff[], model?: EvalModelInfo): string;

package/dist/src/eval/eval-formatter.js

@@ -0,0 +1,221 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+import { formatCostMicros } from './eval-cost.js';
+/**
+ * Format eval results as a terminal table with token usage and cost.
+ */
+export function formatEvalTable(result, model) {
+    const lines = [];
+    const nameWidth = Math.max(20, ...result.results.map((r) => r.eval.name.length));
+    const hasUsage = result.results.some((r) => r.cost);
+    // Header
+    if (model) {
+        lines.push('');
+        lines.push(`Model: ${model.provider}/${model.model}`);
+    }
+    lines.push('');
+    if (hasUsage) {
+        lines.push(`${'Eval'.padEnd(nameWidth)}  Status   Assertions  Tokens       Cost       Duration`);
+        lines.push('-'.repeat(nameWidth + 70));
+    }
+    else {
+        lines.push(`${'Eval'.padEnd(nameWidth)}  Status   Assertions  Duration`);
+        lines.push('-'.repeat(nameWidth + 40));
+    }
+    for (const r of result.results) {
+        const status = r.passed ? 'PASS' : 'FAIL';
+        const passedCount = r.assertions.filter((a) => a.passed).length;
+        const totalCount = r.assertions.length;
+        const duration = `${r.durationMs}ms`;
+        if (hasUsage && r.cost) {
+            const tokens = `${r.cost.totalTokens.toLocaleString()}`;
+            const cost = formatCostMicros(r.cost.estimatedCostMicros);
+            lines.push(`${r.eval.name.padEnd(nameWidth)}  ${status.padEnd(7)}  ${`${passedCount}/${totalCount}`.padEnd(12)}${tokens.padEnd(13)}${cost.padEnd(11)}${duration}`);
+        }
+        else {
+            lines.push(`${r.eval.name.padEnd(nameWidth)}  ${status.padEnd(7)}  ${passedCount}/${totalCount}`.padEnd(nameWidth + 20) +
+                `         ${duration}`);
+        }
+        // Show per-assertion detail for failed evals
+        if (!r.passed) {
+            for (const a of r.assertions) {
+                const icon = a.passed ? '  ✓' : '  ✗';
+                const reason = a.reason ? ` — ${a.reason}` : '';
+                lines.push(`${icon} ${a.negated ? 'NOT ' : ''}${a.text}${reason}`);
+            }
+        }
+    }
+    lines.push('-'.repeat(hasUsage ? nameWidth + 70 : nameWidth + 40));
+    // Summary line
+    const totalAssertions = result.results.reduce((n, r) => n + r.assertions.length, 0);
+    const passedAssertions = result.results.reduce((n, r) => n + r.assertions.filter((a) => a.passed).length, 0);
+    const accuracy = totalAssertions > 0 ? ((passedAssertions / totalAssertions) * 100).toFixed(0) : '0';
+    lines.push(`Total: ${result.totalPassed} passed, ${result.totalFailed} failed (${result.totalDurationMs}ms)`);
+    lines.push(`Accuracy: ${passedAssertions}/${totalAssertions} assertions (${accuracy}%)`);
+    if (result.totalCost) {
+        const cost = formatCostMicros(result.totalCost.estimatedCostMicros);
+        lines.push(`Tokens: ${result.totalCost.totalTokens.toLocaleString()} (${result.totalCost.inputTokens.toLocaleString()} in / ${result.totalCost.outputTokens.toLocaleString()} out)`);
+        lines.push(`Cost: ${cost}`);
+    }
+    lines.push('');
+    return lines.join('\n');
+}
+/**
+ * Format eval diff as a terminal table.
+ */
+export function formatDiffTable(diffs) {
+    const lines = [];
+    const nameWidth = Math.max(20, ...diffs.map((d) => d.evalName.length));
+    lines.push('');
+    lines.push(`${'Eval'.padEnd(nameWidth)}  Status       Current  Baseline`);
+    lines.push('-'.repeat(nameWidth + 40));
+    for (const d of diffs) {
+        const icon = statusIcon(d.status);
+        const current = d.currentPassed === null ? '-' : d.currentPassed ? 'PASS' : 'FAIL';
+        const baseline = d.baselinePassed === null ? '-' : d.baselinePassed ? 'PASS' : 'FAIL';
+        lines.push(`${d.evalName.padEnd(nameWidth)}  ${icon} ${d.status.padEnd(10)}  ${current.padEnd(7)}  ${baseline}`);
+    }
+    const regressed = diffs.filter((d) => d.status === 'regressed').length;
+    const improved = diffs.filter((d) => d.status === 'improved').length;
+    const unchanged = diffs.filter((d) => d.status === 'unchanged').length;
+    lines.push('-'.repeat(nameWidth + 40));
+    lines.push(`Summary: ${regressed} regressed, ${improved} improved, ${unchanged} unchanged`);
+    lines.push('');
+    return lines.join('\n');
+}
+/**
+ * Format a side-by-side comparison of two eval suite results.
+ * Shows model, accuracy, tokens, cost, and duration for each eval.
+ */
+export function formatComparisonTable(current, baseline, diffs) {
+    const lines = [];
+    const currentModel = current.model ? `${current.model.provider}/${current.model.model}` : 'current';
+    const baselineModel = baseline.model ? `${baseline.model.provider}/${baseline.model.model}` : 'baseline';
+    const nameWidth = Math.max(20, ...diffs.map((d) => d.evalName.length));
+    const colWidth = Math.max(30, baselineModel.length + 4, currentModel.length + 4);
+    const sep = '-'.repeat(nameWidth + 4 + colWidth * 2);
+    // Header
+    lines.push('');
+    lines.push(`${''.padEnd(nameWidth + 4)}${baselineModel.padEnd(colWidth)}${currentModel.padEnd(colWidth)}`);
+    lines.push(sep);
+    // Per-eval rows
+    const baseMap = new Map(baseline.results.map((r) => [r.eval.name, r]));
+    const curMap = new Map(current.results.map((r) => [r.eval.name, r]));
+    for (const d of diffs) {
+        const base = baseMap.get(d.evalName);
+        const cur = curMap.get(d.evalName);
+        const icon = statusIcon(d.status);
+        const baseAssertions = base ? `${base.assertions.filter((a) => a.passed).length}/${base.assertions.length}` : '-';
+        const curAssertions = cur ? `${cur.assertions.filter((a) => a.passed).length}/${cur.assertions.length}` : '-';
+        const baseStatus = base ? (base.passed ? 'PASS' : 'FAIL') : '-';
+        const curStatus = cur ? (cur.passed ? 'PASS' : 'FAIL') : '-';
+        const baseTokens = base?.cost ? base.cost.totalTokens.toLocaleString() : '-';
+        const curTokens = cur?.cost ? cur.cost.totalTokens.toLocaleString() : '-';
+        const baseCost = base?.cost ? formatCostMicros(base.cost.estimatedCostMicros) : '-';
+        const curCost = cur?.cost ? formatCostMicros(cur.cost.estimatedCostMicros) : '-';
+        const baseDuration = base ? `${(base.durationMs / 1000).toFixed(1)}s` : '-';
+        const curDuration = cur ? `${(cur.durationMs / 1000).toFixed(1)}s` : '-';
+        lines.push(`${`${icon} ${d.evalName}`.padEnd(nameWidth + 4)}${`${baseStatus} ${baseAssertions}`.padEnd(colWidth)}${`${curStatus} ${curAssertions}`.padEnd(colWidth)}`);
+        lines.push(`${''.padEnd(nameWidth + 4)}${`${baseTokens} tok ${baseCost}`.padEnd(colWidth)}${`${curTokens} tok ${curCost}`.padEnd(colWidth)}`);
+        lines.push(`${''.padEnd(nameWidth + 4)}${baseDuration.padEnd(colWidth)}${curDuration.padEnd(colWidth)}`);
+    }
+    lines.push(sep);
+    // Totals
+    const baseTotal = baseline.results.length;
+    const curTotal = current.results.length;
+    const basePassed = baseline.totalPassed;
+    const curPassed = current.totalPassed;
+    const baseAssTotal = baseline.results.reduce((n, r) => n + r.assertions.length, 0);
+    const baseAssPassed = baseline.results.reduce((n, r) => n + r.assertions.filter((a) => a.passed).length, 0);
+    const curAssTotal = current.results.reduce((n, r) => n + r.assertions.length, 0);
+    const curAssPassed = current.results.reduce((n, r) => n + r.assertions.filter((a) => a.passed).length, 0);
+    const baseAccuracy = baseAssTotal > 0 ? ((baseAssPassed / baseAssTotal) * 100).toFixed(0) : '0';
+    const curAccuracy = curAssTotal > 0 ? ((curAssPassed / curAssTotal) * 100).toFixed(0) : '0';
+    lines.push(`${'Evals'.padEnd(nameWidth + 4)}${`${basePassed}/${baseTotal} passed`.padEnd(colWidth)}${`${curPassed}/${curTotal} passed`.padEnd(colWidth)}`);
+    lines.push(`${'Accuracy'.padEnd(nameWidth + 4)}${`${baseAccuracy}% (${baseAssPassed}/${baseAssTotal})`.padEnd(colWidth)}${`${curAccuracy}% (${curAssPassed}/${curAssTotal})`.padEnd(colWidth)}`);
+    if (baseline.totalCost || current.totalCost) {
+        const baseTok = baseline.totalCost ? baseline.totalCost.totalTokens.toLocaleString() : '-';
+        const curTok = current.totalCost ? current.totalCost.totalTokens.toLocaleString() : '-';
+        const baseCostTotal = baseline.totalCost ? formatCostMicros(baseline.totalCost.estimatedCostMicros) : '-';
+        const curCostTotal = current.totalCost ? formatCostMicros(current.totalCost.estimatedCostMicros) : '-';
+        lines.push(`${'Tokens'.padEnd(nameWidth + 4)}${baseTok.padEnd(colWidth)}${curTok.padEnd(colWidth)}`);
+        lines.push(`${'Cost'.padEnd(nameWidth + 4)}${baseCostTotal.padEnd(colWidth)}${curCostTotal.padEnd(colWidth)}`);
+    }
+    const baseDurTotal = `${(baseline.totalDurationMs / 1000).toFixed(1)}s`;
+    const curDurTotal = `${(current.totalDurationMs / 1000).toFixed(1)}s`;
+    lines.push(`${'Duration'.padEnd(nameWidth + 4)}${baseDurTotal.padEnd(colWidth)}${curDurTotal.padEnd(colWidth)}`);
+    lines.push('');
+    return lines.join('\n');
+}
+/**
+ * Format eval results as CI-friendly markdown.
+ */
+export function formatEvalMarkdown(result, diffs, model) {
+    const lines = [];
+    lines.push('## Eval Results');
+    if (model) {
+        lines.push(`**Model:** ${model.provider}/${model.model}`);
+    }
+    lines.push('');
+    const hasUsage = result.results.some((r) => r.cost);
+    if (hasUsage) {
+        lines.push(`| Eval | Status | Assertions | Tokens | Cost | Duration |`);
+        lines.push(`|------|--------|------------|--------|------|----------|`);
+    }
+    else {
+        lines.push(`| Eval | Status | Assertions | Duration |`);
+        lines.push(`|------|--------|------------|----------|`);
+    }
+    for (const r of result.results) {
+        const status = r.passed ? 'PASS' : 'FAIL';
+        const passedCount = r.assertions.filter((a) => a.passed).length;
+        if (hasUsage && r.cost) {
+            const cost = formatCostMicros(r.cost.estimatedCostMicros);
+            lines.push(`| ${r.eval.name} | ${status} | ${passedCount}/${r.assertions.length} | ${r.cost.totalTokens.toLocaleString()} | ${cost} | ${r.durationMs}ms |`);
+        }
+        else {
+            lines.push(`| ${r.eval.name} | ${status} | ${passedCount}/${r.assertions.length} | ${r.durationMs}ms |`);
+        }
+    }
+    const totalAssertions = result.results.reduce((n, r) => n + r.assertions.length, 0);
+    const passedAssertions = result.results.reduce((n, r) => n + r.assertions.filter((a) => a.passed).length, 0);
+    const accuracy = totalAssertions > 0 ? ((passedAssertions / totalAssertions) * 100).toFixed(0) : '0';
+    lines.push('');
+    lines.push(`**${result.totalPassed} passed, ${result.totalFailed} failed** (${result.totalDurationMs}ms) — **${accuracy}% accuracy** (${passedAssertions}/${totalAssertions})`);
+    if (result.totalCost) {
+        const cost = formatCostMicros(result.totalCost.estimatedCostMicros);
+        lines.push(`**Tokens:** ${result.totalCost.totalTokens.toLocaleString()} — **Cost:** ${cost}`);
+    }
+    if (diffs && diffs.length > 0) {
+        lines.push('');
+        lines.push('### Diff vs Baseline');
+        lines.push('');
+        lines.push('| Eval | Status | Current | Baseline |');
+        lines.push('|------|--------|---------|----------|');
+        for (const d of diffs) {
+            const icon = statusIcon(d.status);
+            const current = d.currentPassed === null ? '-' : d.currentPassed ? 'PASS' : 'FAIL';
+            const baseline = d.baselinePassed === null ? '-' : d.baselinePassed ? 'PASS' : 'FAIL';
+            lines.push(`| ${d.evalName} | ${icon} ${d.status} | ${current} | ${baseline} |`);
+        }
+        const regressed = diffs.filter((d) => d.status === 'regressed').length;
+        if (regressed > 0) {
+            lines.push('');
+            lines.push(`**${regressed} regression(s) detected**`);
+        }
+    }
+    return lines.join('\n');
+}
+function statusIcon(status) {
+    switch (status) {
+        case 'improved': return '+';
+        case 'regressed': return '!';
+        case 'new': return '*';
+        case 'removed': return '-';
+        default: return '=';
+    }
+}
+//# sourceMappingURL=eval-formatter.js.map

package/dist/src/eval/eval-formatter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-formatter.js","sourceRoot":"","sources":["../../../src/eval/eval-formatter.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAC,gBAAgB,EAAC,MAAM,gBAAgB,CAAC;AAEhD;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAuB,EAAE,KAAqB;IAC5E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IACjF,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAEpD,SAAS;IACT,IAAI,KAAK,EAAE,CAAC;QACV,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,QAAQ,EAAE,CAAC;QACb,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,yDAAyD,CAAC,CAAC;QACjG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC;IACzC,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,iCAAiC,CAAC,CAAC;QACzE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC1C,MAAM,WAAW,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAChE,MAAM,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QACvC,MAAM,QAAQ,GAAG,GAAG,CAAC,CAAC,UAAU,IAAI,CAAC;QAErC,IAAI,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,EAAE,CAAC;YACxD,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YAC1D,KAAK,CAAC,IAAI,CACR,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG,WAAW,IAAI,UAAU,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CACvJ,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CACR,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,WAAW,IAAI,UAAU,EAAE,CAAC,MAAM,CAAC,SAAS,GAAG,EAAE,CAAC;gBAC5G,YAAY,QAAQ,EAAE,CACvB,CAAC;QACJ,CAAC;QAED,6CAA6C;QAC7C,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YACd,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;gBAC7B,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBACtC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChD,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,GAAG,MAAM,EAAE,CAAC,CAAC;YACrE,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC;IAEnE,eAAe;IACf,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACpF,MAAM,gBAAgB,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC7G,MAAM,QAAQ,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACrG,KAAK,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,WAAW,YAAY,MAAM,CAAC,WAAW,YAAY,MAAM,CAAC,eAAe,KAAK,CAAC,CAAC;IAC9G,KAAK,CAAC,IAAI,CAAC,aAAa,gBAAgB,IAAI,eAAe,gBAAgB,QAAQ,IAAI,CAAC,CAAC;IAEzF,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QACpE,KAAK,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,cAAc,EAAE,KAAK,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,cAAc,EAAE,SAAS,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;QACrL,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAAiB;IAC/C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IAEvE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,kCAAkC,CAAC,CAAC;IAC1E,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC;IAEvC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAClC,MAAM,OAAO,GAAG,CAAC,CAAC,aAAa,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QACnF,MAAM,QAAQ,GAAG,CAAC,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAEtF,KAAK,CAAC,IAAI,CACR,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CACrG,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;IACvE,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;IACrE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;IAEvE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC;IACvC,KAAK,CAAC,IAAI,CAAC,YAAY,SAAS,eAAe,QAAQ,cAAc,SAAS,YAAY,CAAC,CAAC;IAC5F,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CACnC,OAAwB,EACxB,QAAyB,EACzB,KAAiB;IAEjB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACpG,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;IACzG,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACjF,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,GAAG,QAAQ,GAAG,CAAC,CAAC,CAAC;IAErD,SAAS;IACT,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3G,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEhB,gBAAgB;IAChB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACvE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAErE,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAElC,MAAM,cAAc,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAClH,MAAM,aAAa,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9G,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAChE,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAC7D,MAAM,UAAU,GAAG,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC7E,MAAM,SAAS,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC1E,MAAM,QAAQ,GAAG,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACpF,MAAM,OAAO,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACjF,MAAM,YAAY,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5E,MAAM,WAAW,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAEzE,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,GAAG,UAAU,IAAI,cAAc,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,SAAS,IAAI,aAAa,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACvK,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,GAAG,UAAU,QAAQ,QAAQ,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,SAAS,QAAQ,OAAO,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC9I,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3G,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEhB,SAAS;IACT,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;IACxC,MAAM,UAAU,GAAG,QAAQ,CAAC,WAAW,CAAC;IACxC,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC;IACtC,MAAM,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACnF,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC5G,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACjF,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1G,MAAM,YAAY,GAAG,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAChG,MAAM,WAAW,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAE5F,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,GAAG,UAAU,IAAI,SAAS,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,SAAS,IAAI,QAAQ,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3J,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,GAAG,YAAY,MAAM,aAAa,IAAI,YAAY,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,WAAW,MAAM,YAAY,IAAI,WAAW,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAEjM,IAAI,QAAQ,CAAC,SAAS,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC3F,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QACxF,MAAM,aAAa,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAC1G,MAAM,YAAY,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACvG,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACrG,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACjH,CAAC;IAED,MAAM,YAAY,GAAG,GAAG,CAAC,QAAQ,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACxE,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACtE,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACjH,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAuB,EAAE,KAAkB,EAAE,KAAqB;IACnG,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC9B,IAAI,KAAK,EAAE,CAAC;QACV,KAAK,CAAC,IAAI,CAAC,cAAc,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACpD,IAAI,QAAQ,EAAE,CAAC;QACb,KAAK,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC;QACxE,KAAK,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC;IAC1E,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;QACxD,KAAK,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC1C,MAAM,WAAW,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAChE,IAAI,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YAC1D,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,MAAM,MAAM,MAAM,WAAW,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,MAAM,IAAI,MAAM,CAAC,CAAC,UAAU,MAAM,CAAC,CAAC;QAC9J,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,MAAM,MAAM,MAAM,WAAW,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,MAAM,CAAC,CAAC,UAAU,MAAM,CAAC,CAAC;QAC3G,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACpF,MAAM,gBAAgB,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC7G,MAAM,QAAQ,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAErG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,WAAW,YAAY,MAAM,CAAC,WAAW,cAAc,MAAM,CAAC,eAAe,WAAW,QAAQ,iBAAiB,gBAAgB,IAAI,eAAe,GAAG,CAAC,CAAC;IAEhL,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QACpE,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,cAAc,EAAE,gBAAgB,IAAI,EAAE,CAAC,CAAC;IACjG,CAAC;IAED,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;QAErD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM,OAAO,GAAG,CAAC,CAAC,aAAa,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YACnF,MAAM,QAAQ,GAAG,CAAC,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YACtF,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,MAAM,IAAI,IAAI,CAAC,CAAC,MAAM,MAAM,OAAO,MAAM,QAAQ,IAAI,CAAC,CAAC;QACnF,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;QACvE,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,KAAK,SAAS,2BAA2B,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,UAAU,CAAC,MAAc;IAChC,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,UAAU,CAAC,CAAC,OAAO,GAAG,CAAC;QAC5B,KAAK,WAAW,CAAC,CAAC,OAAO,GAAG,CAAC;QAC7B,KAAK,KAAK,CAAC,CAAC,OAAO,GAAG,CAAC;QACvB,KAAK,SAAS,CAAC,CAAC,OAAO,GAAG,CAAC;QAC3B,OAAO,CAAC,CAAC,OAAO,GAAG,CAAC;IACtB,CAAC;AACH,CAAC"}

package/dist/src/eval/eval-judge.d.ts

@@ -0,0 +1,26 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+import type { AssertionResult } from './eval-types.js';
+/**
+ * Provider interface for LLM-based judgment.
+ */
+export interface JudgeProvider {
+    judge(prompt: string): Promise<string>;
+}
+/**
+ * Judge whether a response satisfies an assertion using an LLM.
+ */
+export declare function judgeAssertion(response: string, assertion: {
+    text: string;
+    negated: boolean;
+}, provider: JudgeProvider): Promise<AssertionResult>;
+/**
+ * Judge all assertions for a response.
+ */
+export declare function judgeAllAssertions(response: string, assertions: Array<{
+    text: string;
+    negated: boolean;
+}>, provider: JudgeProvider): Promise<AssertionResult[]>;

package/dist/src/eval/eval-judge.js

@@ -0,0 +1,76 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+/**
+ * Judge whether a response satisfies an assertion using an LLM.
+ */
+export async function judgeAssertion(response, assertion, provider) {
+    const direction = assertion.negated ? 'should NOT' : 'should';
+    const prompt = [
+        'You are an eval judge. Determine if the assistant response satisfies the assertion.',
+        '',
+        '## Assistant Response',
+        response,
+        '',
+        '## Assertion',
+        `The response ${direction}: ${assertion.text}`,
+        '',
+        '## Instructions',
+        'Reply with exactly one line in the format:',
+        'PASS: <brief reason>',
+        'or',
+        'FAIL: <brief reason>',
+        '',
+        'Nothing else.',
+    ].join('\n');
+    try {
+        const raw = await provider.judge(prompt);
+        const line = raw.trim().split('\n')[0];
+        const passMatch = /^PASS:\s*(.*)$/i.exec(line);
+        const failMatch = /^FAIL:\s*(.*)$/i.exec(line);
+        if (passMatch) {
+            return {
+                text: assertion.text,
+                negated: assertion.negated,
+                passed: true,
+                reason: passMatch[1].trim(),
+            };
+        }
+        if (failMatch) {
+            return {
+                text: assertion.text,
+                negated: assertion.negated,
+                passed: false,
+                reason: failMatch[1].trim(),
+            };
+        }
+        // Couldn't parse — treat as failure
+        return {
+            text: assertion.text,
+            negated: assertion.negated,
+            passed: false,
+            reason: `Judge response unparseable: ${line}`,
+        };
+    }
+    catch (err) {
+        return {
+            text: assertion.text,
+            negated: assertion.negated,
+            passed: false,
+            reason: `Judge error: ${err instanceof Error ? err.message : String(err)}`,
+        };
+    }
+}
+/**
+ * Judge all assertions for a response.
+ */
+export async function judgeAllAssertions(response, assertions, provider) {
+    const results = [];
+    for (const assertion of assertions) {
+        results.push(await judgeAssertion(response, assertion, provider));
+    }
+    return results;
+}
+//# sourceMappingURL=eval-judge.js.map

package/dist/src/eval/eval-judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-judge.js","sourceRoot":"","sources":["../../../src/eval/eval-judge.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAWH;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,SAA2C,EAC3C,QAAuB;IAEvB,MAAM,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC9D,MAAM,MAAM,GAAG;QACb,qFAAqF;QACrF,EAAE;QACF,uBAAuB;QACvB,QAAQ;QACR,EAAE;QACF,cAAc;QACd,gBAAgB,SAAS,KAAK,SAAS,CAAC,IAAI,EAAE;QAC9C,EAAE;QACF,iBAAiB;QACjB,4CAA4C;QAC5C,sBAAsB;QACtB,IAAI;QACJ,sBAAsB;QACtB,EAAE;QACF,eAAe;KAChB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE/C,IAAI,SAAS,EAAE,CAAC;YACd,OAAO;gBACL,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,OAAO,EAAE,SAAS,CAAC,OAAO;gBAC1B,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;aAC5B,CAAC;QACJ,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,OAAO;gBACL,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,OAAO,EAAE,SAAS,CAAC,OAAO;gBAC1B,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;aAC5B,CAAC;QACJ,CAAC;QAED,oCAAoC;QACpC,OAAO;YACL,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,OAAO,EAAE,SAAS,CAAC,OAAO;YAC1B,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,+BAA+B,IAAI,EAAE;SAC9C,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,OAAO,EAAE,SAAS,CAAC,OAAO;YAC1B,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,gBAAgB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;SAC3E,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAgB,EAChB,UAAmD,EACnD,QAAuB;IAEvB,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,OAAO,CAAC,IAAI,CAAC,MAAM,cAAc,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/src/eval/eval-run-builder.d.ts

@@ -0,0 +1,25 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+import type { EvalModelInfo, EvalRunRecord, EvalRunComparison, EvalSuiteResult, EvalTrendPoint } from './eval-types.js';
+/**
+ * Build an EvalRunRecord from a suite result and model info.
+ */
+export declare function buildEvalRun(suite: EvalSuiteResult, model: EvalModelInfo, options: {
+    id?: string;
+    orgId: string;
+    appId?: string;
+    gitSha?: string;
+    label?: string;
+    triggeredBy?: 'manual' | 'ci' | 'automation';
+}): EvalRunRecord;
+/**
+ * Compare two eval runs, producing diffs and cost/quality deltas.
+ */
+export declare function compareRuns(runA: EvalRunRecord, runB: EvalRunRecord): EvalRunComparison;
+/**
+ * Extract trend points from a list of eval runs for charting.
+ */
+export declare function buildTrendPoints(runs: EvalRunRecord[]): EvalTrendPoint[];

package/dist/src/eval/eval-run-builder.js

@@ -0,0 +1,78 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+import { diffEvalResults } from './eval-diff.js';
+import { aggregateRunCost } from './eval-cost.js';
+/**
+ * Build an EvalRunRecord from a suite result and model info.
+ */
+export function buildEvalRun(suite, model, options) {
+    const perCaseCosts = suite.results.map((r) => r.cost ?? { inputTokens: 0, outputTokens: 0, totalTokens: 0, estimatedCostMicros: 0 });
+    const totalCost = suite.totalCost ?? aggregateRunCost(perCaseCosts);
+    return {
+        id: options.id ?? crypto.randomUUID(),
+        orgId: options.orgId,
+        appId: options.appId,
+        model,
+        suite,
+        perCaseCosts,
+        totalCost,
+        gitSha: options.gitSha ?? suite.gitSha,
+        label: options.label,
+        triggeredBy: options.triggeredBy ?? 'manual',
+        createdAt: suite.timestamp,
+    };
+}
+/**
+ * Compare two eval runs, producing diffs and cost/quality deltas.
+ */
+export function compareRuns(runA, runB) {
+    const diff = diffEvalResults(runA.suite, runB.suite);
+    const totalA = runA.suite.results.length;
+    const totalB = runB.suite.results.length;
+    const passRateA = totalA > 0 ? runA.suite.totalPassed / totalA : 0;
+    const passRateB = totalB > 0 ? runB.suite.totalPassed / totalB : 0;
+    const maxLen = Math.max(runA.perCaseCosts.length, runB.perCaseCosts.length);
+    const perCaseDeltas = [];
+    for (let i = 0; i < maxLen; i++) {
+        const costA = runA.perCaseCosts[i]?.estimatedCostMicros ?? 0;
+        const costB = runB.perCaseCosts[i]?.estimatedCostMicros ?? 0;
+        perCaseDeltas.push(costB - costA);
+    }
+    return {
+        runA,
+        runB,
+        diff,
+        costDelta: {
+            totalMicros: runB.totalCost.estimatedCostMicros - runA.totalCost.estimatedCostMicros,
+            perCase: perCaseDeltas,
+        },
+        qualityDelta: {
+            passRateDelta: passRateB - passRateA,
+            durationDeltaMs: runB.suite.totalDurationMs - runA.suite.totalDurationMs,
+        },
+    };
+}
+/**
+ * Extract trend points from a list of eval runs for charting.
+ */
+export function buildTrendPoints(runs) {
+    return runs.map((run) => {
+        const total = run.suite.results.length;
+        const passRate = total > 0 ? run.suite.totalPassed / total : 0;
+        const avgDurationMs = total > 0 ? run.suite.totalDurationMs / total : 0;
+        return {
+            runId: run.id,
+            label: run.label,
+            gitSha: run.gitSha,
+            model: run.model,
+            passRate,
+            totalCostMicros: run.totalCost.estimatedCostMicros,
+            avgDurationMs,
+            timestamp: run.createdAt,
+        };
+    });
+}
+//# sourceMappingURL=eval-run-builder.js.map

package/dist/src/eval/eval-run-builder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-run-builder.js","sourceRoot":"","sources":["../../../src/eval/eval-run-builder.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAUH,OAAO,EAAC,eAAe,EAAC,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAC,gBAAgB,EAAC,MAAM,gBAAgB,CAAC;AAEhD;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,KAAsB,EACtB,KAAoB,EACpB,OAOC;IAED,MAAM,YAAY,GAAmB,KAAK,CAAC,OAAO,CAAC,GAAG,CACpD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAC,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAC,CAC3F,CAAC;IAEF,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,gBAAgB,CAAC,YAAY,CAAC,CAAC;IAEpE,OAAO;QACL,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,MAAM,CAAC,UAAU,EAAE;QACrC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,KAAK;QACL,KAAK;QACL,YAAY;QACZ,SAAS;QACT,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM;QACtC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,QAAQ;QAC5C,SAAS,EAAE,KAAK,CAAC,SAAS;KAC3B,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,IAAmB,EAAE,IAAmB;IAClE,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;IACzC,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAC5E,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,mBAAmB,IAAI,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,mBAAmB,IAAI,CAAC,CAAC;QAC7D,aAAa,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;IACpC,CAAC;IAED,OAAO;QACL,IAAI;QACJ,IAAI;QACJ,IAAI;QACJ,SAAS,EAAE;YACT,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,mBAAmB,GAAG,IAAI,CAAC,SAAS,CAAC,mBAAmB;YACpF,OAAO,EAAE,aAAa;SACvB;QACD,YAAY,EAAE;YACZ,aAAa,EAAE,SAAS,GAAG,SAAS;YACpC,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe;SACzE;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAqB;IACpD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACtB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QACvC,MAAM,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,aAAa,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAExE,OAAO;YACL,KAAK,EAAE,GAAG,CAAC,EAAE;YACb,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,QAAQ;YACR,eAAe,EAAE,GAAG,CAAC,SAAS,CAAC,mBAAmB;YAClD,aAAa;YACb,SAAS,EAAE,GAAG,CAAC,SAAS;SACzB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/src/eval/eval-runner.d.ts

@@ -0,0 +1,36 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+import type { AmodalRepo } from '../repo/repo-types.js';
+import type { EvalSuiteResult, EvalProgress, EvalModelInfo } from './eval-types.js';
+import type { JudgeProvider } from './eval-judge.js';
+/**
+ * Provider interface for running a query against the agent.
+ */
+export interface EvalQueryProvider {
+    query(message: string, tenantId?: string): Promise<{
+        response: string;
+        toolCalls: Array<{
+            name: string;
+            parameters: Record<string, unknown>;
+        }>;
+        usage?: {
+            inputTokens: number;
+            outputTokens: number;
+        };
+    }>;
+}
+export interface EvalRunnerOptions {
+    filter?: string;
+    queryProvider: EvalQueryProvider;
+    judgeProvider: JudgeProvider;
+    gitSha?: string;
+    model?: EvalModelInfo;
+}
+/**
+ * Run the eval suite from a loaded repo.
+ * Yields progress events and returns the full suite result.
+ */
+export declare function runEvalSuite(repo: AmodalRepo, options: EvalRunnerOptions): AsyncGenerator<EvalProgress, EvalSuiteResult>;