@lobu/cli 3.0.7 → 3.0.12

package/dist/eval/grader.js

@@ -0,0 +1,177 @@
+/**
+ * LLM-as-judge grader using the Lobu gateway.
+ *
+ * Borrows the Claude outcomes pattern: a separate evaluator context
+ * grades agent output against a markdown rubric with per-criterion scoring.
+ */
+import { createSession, deleteSession, sendAndCollect, } from "./client.js";
+const INLINE_JUDGE_PROMPT = `You are a strict evaluator. You will be given an AI agent's response and a criteria to judge it against.
+
+You MUST respond with ONLY a JSON object, no other text:
+{"passed": true, "score": 0.85, "reason": "one sentence explanation"}
+
+Rules:
+- "passed": true if the response meets the criteria, false otherwise
+- "score": a number between 0.0 and 1.0
+- "reason": a brief explanation (one sentence)
+- Return ONLY the JSON object, nothing else
+
+## Criteria
+{{criteria}}
+
+## Agent Response
+{{response}}`;
+const RUBRIC_JUDGE_PROMPT = `You are a strict evaluator. Grade the agent's conversation against each criterion in the rubric.
+
+You MUST respond with ONLY a JSON object, no other text:
+{"criteria": [{"name": "criterion name", "passed": true, "explanation": "why"}], "score": 0.85}
+
+Rules:
+- Score each criterion independently
+- "score" is the overall score 0.0-1.0
+- Return ONLY the JSON object, nothing else
+
+## Rubric
+{{rubric}}
+
+## Conversation
+{{transcript}}`;
+export async function gradeWithRubric(gatewayUrl, authToken, rubricContent, turns, timeoutMs) {
+    const transcript = turns
+        .map((t) => `User: ${t.user}\nAgent: ${t.agent}`)
+        .join("\n\n");
+    const prompt = RUBRIC_JUDGE_PROMPT.replace("{{rubric}}", rubricContent).replace("{{transcript}}", transcript);
+    const session = await createSession(gatewayUrl, authToken, {
+        forceNew: true,
+        dryRun: true,
+    });
+    try {
+        const response = await sendAndCollect(session, prompt, timeoutMs);
+        return parseGraderResponse(response);
+    }
+    finally {
+        await deleteSession(session);
+    }
+}
+export async function gradeInline(gatewayUrl, authToken, criteria, agentResponse, timeoutMs) {
+    const prompt = INLINE_JUDGE_PROMPT.replace("{{criteria}}", criteria).replace("{{response}}", agentResponse);
+    const session = await createSession(gatewayUrl, authToken, {
+        forceNew: true,
+        dryRun: true,
+    });
+    try {
+        const response = await sendAndCollect(session, prompt, timeoutMs);
+        return parseInlineResponse(response);
+    }
+    finally {
+        await deleteSession(session);
+    }
+}
+function parseGraderResponse(response) {
+    if (response.error) {
+        return {
+            score: 0,
+            criteria: [{ name: "error", passed: false, explanation: response.error }],
+        };
+    }
+    try {
+        const json = extractJSON(response.text);
+        const parsed = JSON.parse(json);
+        return {
+            score: typeof parsed.score === "number" ? parsed.score : 0,
+            criteria: Array.isArray(parsed.criteria)
+                ? parsed.criteria.map((c) => ({
+                    name: String(c.name ?? ""),
+                    passed: Boolean(c.passed),
+                    explanation: String(c.explanation ?? ""),
+                }))
+                : [],
+        };
+    }
+    catch {
+        // Fallback: try to infer from prose response
+        return inferRubricFromText(response.text);
+    }
+}
+function parseInlineResponse(response) {
+    if (response.error) {
+        return { passed: false, score: 0, reason: response.error };
+    }
+    try {
+        const json = extractJSON(response.text);
+        const parsed = JSON.parse(json);
+        return {
+            passed: Boolean(parsed.passed),
+            score: typeof parsed.score === "number" ? parsed.score : parsed.passed ? 1 : 0,
+            reason: String(parsed.reason ?? ""),
+        };
+    }
+    catch {
+        // Fallback: infer pass/fail from prose
+        return inferInlineFromText(response.text);
+    }
+}
+/** Extract JSON from text that may contain markdown fences or surrounding prose. */
+function extractJSON(text) {
+    // Try to find JSON in markdown code block
+    const fenced = text.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/);
+    if (fenced?.[1])
+        return fenced[1].trim();
+    // Try to find raw JSON object
+    const braceMatch = text.match(/\{[\s\S]*\}/);
+    if (braceMatch)
+        return braceMatch[0];
+    return text.trim();
+}
+/** Fallback: infer pass/fail from prose when JSON parsing fails. */
+function inferInlineFromText(text) {
+    const lower = text.toLowerCase();
+    const positiveSignals = [
+        "yes",
+        "pass",
+        "meets",
+        "satisfies",
+        "correct",
+        "appropriate",
+        "good",
+        "well",
+    ];
+    const negativeSignals = [
+        "no",
+        "fail",
+        "does not",
+        "doesn't",
+        "incorrect",
+        "missing",
+        "lacks",
+        "poor",
+    ];
+    const posCount = positiveSignals.filter((s) => lower.includes(s)).length;
+    const negCount = negativeSignals.filter((s) => lower.includes(s)).length;
+    const passed = posCount > negCount;
+    return {
+        passed,
+        score: passed ? 0.7 : 0.3,
+        reason: `Inferred from prose (pos=${posCount}, neg=${negCount}): ${text.slice(0, 100)}`,
+    };
+}
+/** Fallback: infer rubric result from prose when JSON parsing fails. */
+function inferRubricFromText(text) {
+    const lower = text.toLowerCase();
+    const positiveSignals = ["pass", "meets", "satisfies", "good", "correct"];
+    const negativeSignals = ["fail", "does not", "doesn't", "missing", "poor"];
+    const posCount = positiveSignals.filter((s) => lower.includes(s)).length;
+    const negCount = negativeSignals.filter((s) => lower.includes(s)).length;
+    const passed = posCount > negCount;
+    return {
+        score: passed ? 0.7 : 0.3,
+        criteria: [
+            {
+                name: "overall",
+                passed,
+                explanation: `Inferred from prose: ${text.slice(0, 200)}`,
+            },
+        ],
+    };
+}
+//# sourceMappingURL=grader.js.map

package/dist/eval/grader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grader.js","sourceRoot":"","sources":["../../src/eval/grader.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,aAAa,EACb,aAAa,EACb,cAAc,GAEf,MAAM,aAAa,CAAC;AAGrB,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;;aAef,CAAC;AAEd,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;eAcb,CAAC;AAEhB,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,UAAkB,EAClB,SAAiB,EACjB,aAAqB,EACrB,KAAmB,EACnB,SAAiB;IAEjB,MAAM,UAAU,GAAG,KAAK;SACrB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,YAAY,CAAC,CAAC,KAAK,EAAE,CAAC;SAChD,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhB,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,CACxC,YAAY,EACZ,aAAa,CACd,CAAC,OAAO,CAAC,gBAAgB,EAAE,UAAU,CAAC,CAAC;IAExC,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,UAAU,EAAE,SAAS,EAAE;QACzD,QAAQ,EAAE,IAAI;QACd,MAAM,EAAE,IAAI;KACb,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAClE,OAAO,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;YAAS,CAAC;QACT,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,UAAkB,EAClB,SAAiB,EACjB,QAAgB,EAChB,aAAqB,EACrB,SAAiB;IAEjB,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC,OAAO,CAC1E,cAAc,EACd,aAAa,CACd,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,UAAU,EAAE,SAAS,EAAE;QACzD,QAAQ,EAAE,IAAI;QACd,MAAM,EAAE,IAAI;KACb,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAClE,OAAO,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;YAAS,CAAC;QACT,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,QAA2B;IACtD,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO;YACL,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC;SAC1E,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAG7B,CAAC;QACF,OAAO;YACL,KAAK,EAAE,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1D,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;gBACtC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC1B,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;oBAC1B,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;oBACzB,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC;iBACzC,CAAC,CAAC;gBACL,CAAC,CAAC,EAAE;SACP,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,6CAA6C;QAC7C,OAAO,mBAAmB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,QAA2B;IAKtD,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC;IAC7D,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAI7B,CAAC;QACF,OAAO;YACL,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;YAC9B,KAAK,EACH,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;SACpC,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,uCAAuC;QACvC,OAAO,mBAAmB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;AACH,CAAC;AAED,oFAAoF;AACpF,SAAS,WAAW,CAAC,IAAY;IAC/B,0CAA0C;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;IAChE,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAEzC,8BAA8B;IAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC7C,IAAI,UAAU;QAAE,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC;IAErC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACrB,CAAC;AAED,oEAAoE;AACpE,SAAS,mBAAmB,CAAC,IAAY;IAKvC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,MAAM,eAAe,GAAG;QACtB,KAAK;QACL,MAAM;QACN,OAAO;QACP,WAAW;QACX,SAAS;QACT,aAAa;QACb,MAAM;QACN,MAAM;KACP,CAAC;IACF,MAAM,eAAe,GAAG;QACtB,IAAI;QACJ,MAAM;QACN,UAAU;QACV,SAAS;QACT,WAAW;QACX,SAAS;QACT,OAAO;QACP,MAAM;KACP,CAAC;IAEF,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACzE,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAEzE,MAAM,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACnC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;QACzB,MAAM,EAAE,4BAA4B,QAAQ,SAAS,QAAQ,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KACxF,CAAC;AACJ,CAAC;AAED,wEAAwE;AACxE,SAAS,mBAAmB,CAAC,IAAY;IACvC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,MAAM,eAAe,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAC1E,MAAM,eAAe,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IAE3E,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACzE,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACzE,MAAM,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAEnC,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;QACzB,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,SAAS;gBACf,MAAM;gBACN,WAAW,EAAE,wBAAwB,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;aAC1D;SACF;KACF,CAAC;AACJ,CAAC"}

package/dist/eval/reporter.d.ts

@@ -0,0 +1,8 @@
+import type { EvalReport } from "./types.js";
+export declare function printReport(report: EvalReport): void;
+export declare function saveResult(evalsDir: string, report: EvalReport): Promise<string>;
+export declare function writeJsonReport(report: EvalReport, outputPath: string): Promise<void>;
+export declare function loadSavedResults(evalsDir: string): Promise<EvalReport[]>;
+export declare function generateComparisonReport(evalsDir: string, currentReport?: EvalReport): Promise<string>;
+export declare function writeMarkdownReport(evalsDir: string, currentReport?: EvalReport): Promise<string>;
+//# sourceMappingURL=reporter.d.ts.map

package/dist/eval/reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/eval/reporter.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAA2B,MAAM,YAAY,CAAC;AAItE,wBAAgB,WAAW,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAuBpD;AA+DD,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,MAAM,CAAC,CAejB;AAID,wBAAsB,eAAe,CACnC,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC,CAGf;AAID,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,UAAU,EAAE,CAAC,CAkBvB;AAiBD,wBAAsB,wBAAwB,CAC5C,QAAQ,EAAE,MAAM,EAChB,aAAa,CAAC,EAAE,UAAU,GACzB,OAAO,CAAC,MAAM,CAAC,CA0IjB;AAED,wBAAsB,mBAAmB,CACvC,QAAQ,EAAE,MAAM,EAChB,aAAa,CAAC,EAAE,UAAU,GACzB,OAAO,CAAC,MAAM,CAAC,CAMjB"}

package/dist/eval/reporter.js

@@ -0,0 +1,242 @@
+import { readFile, readdir, writeFile, mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import chalk from "chalk";
+// ─── Console reporter ──────────────────────────────────────────────────
+export function printReport(report) {
+    console.log(chalk.bold(`\nAgent: ${report.agent}`));
+    console.log(chalk.dim(`Model: ${report.provider}/${report.model}`));
+    console.log(chalk.dim(`Evals: ${report.evals.length} total\n`));
+    for (const evalResult of report.evals) {
+        printEval(evalResult);
+    }
+    const { passed, failed, total } = report.summary;
+    const summaryColor = failed === 0 ? chalk.green : chalk.red;
+    console.log(summaryColor(`\nSummary: ${passed}/${total} evals passed`));
+    if (failed > 0) {
+        console.log(chalk.red(`  Failed: ${report.evals
+            .filter((e) => e.passRate < 1 - 0.001)
+            .map((e) => e.name)
+            .join(", ")}`));
+    }
+    console.log();
+}
+function printEval(result) {
+    const trialCount = result.trials.length;
+    const passedCount = result.trials.filter((t) => t.passed).length;
+    console.log(chalk.bold(`${result.name} (${trialCount} trials)`));
+    for (const trial of result.trials) {
+        printTrial(trial);
+    }
+    const statusColor = result.passRate >= 0.8 ? chalk.green : chalk.red;
+    const status = result.passRate >= 0.8 ? "PASS" : "FAIL";
+    const tokenInfo = result.totalTokens.totalTokens
+        ? ` tokens=${result.totalTokens.totalTokens}`
+        : "";
+    console.log(statusColor(`  ${status} ${passedCount}/${trialCount} avg=${result.avgScore.toFixed(2)} p50=${result.p50LatencyMs}ms${tokenInfo}`));
+    console.log();
+}
+function printTrial(trial) {
+    const icon = trial.passed ? chalk.green("✓") : chalk.red("✗");
+    const latency = chalk.dim(`(${(trial.durationMs / 1000).toFixed(1)}s)`);
+    console.log(`  ${icon} Trial ${trial.trial}: ${trial.score.toFixed(2)} ${latency}`);
+    for (const turn of trial.turns) {
+        for (const assertion of turn.assertions) {
+            if (!assertion.passed) {
+                console.log(chalk.red(`    └ ${assertion.type}: ${assertion.reason ?? "FAIL"}`));
+            }
+        }
+    }
+    if (trial.rubric) {
+        for (const criterion of trial.rubric.criteria) {
+            const cIcon = criterion.passed ? chalk.green("✓") : chalk.red("✗");
+            console.log(`    ${cIcon} ${criterion.name}`);
+            if (!criterion.passed) {
+                console.log(chalk.red(`      └ ${criterion.explanation}`));
+            }
+        }
+    }
+    // Show trace IDs for failed trials (for debugging in Grafana/Tempo)
+    if (!trial.passed) {
+        const traceIds = trial.turns.map((t) => t.traceId).filter(Boolean);
+        if (traceIds.length > 0) {
+            console.log(chalk.dim(`    traces: ${traceIds.join(", ")}`));
+        }
+    }
+}
+// ─── Auto-save results ─────────────────────────────────────────────────
+export async function saveResult(evalsDir, report) {
+    const resultsDir = join(evalsDir, ".results");
+    await mkdir(resultsDir, { recursive: true });
+    const slug = `${report.provider}-${report.model}`.replace(/[^a-z0-9-]/gi, "-");
+    const ts = report.timestamp.replace(/[:.]/g, "-").slice(0, 19);
+    const filename = `${slug}_${ts}.json`;
+    const filepath = join(resultsDir, filename);
+    await writeFile(filepath, JSON.stringify(report, null, 2));
+    console.log(chalk.dim(`Results saved to ${filepath}`));
+    return filepath;
+}
+// ─── JSON file output ──────────────────────────────────────────────────
+export async function writeJsonReport(report, outputPath) {
+    await writeFile(outputPath, JSON.stringify(report, null, 2));
+    console.log(chalk.dim(`Results written to ${outputPath}`));
+}
+// ─── Markdown comparison report ────────────────────────────────────────
+export async function loadSavedResults(evalsDir) {
+    const resultsDir = join(evalsDir, ".results");
+    try {
+        const files = await readdir(resultsDir);
+        const jsonFiles = files
+            .filter((f) => f.endsWith(".json"))
+            .sort()
+            .reverse(); // newest first
+        const reports = [];
+        for (const file of jsonFiles) {
+            const content = await readFile(join(resultsDir, file), "utf-8");
+            reports.push(JSON.parse(content));
+        }
+        return reports;
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Group saved results by model and pick the latest run per model.
+ */
+function latestPerModel(reports) {
+    const byModel = new Map();
+    for (const report of reports) {
+        const key = `${report.provider}/${report.model}`;
+        const existing = byModel.get(key);
+        if (!existing || report.timestamp > existing.timestamp) {
+            byModel.set(key, report);
+        }
+    }
+    return Array.from(byModel.values());
+}
+export async function generateComparisonReport(evalsDir, currentReport) {
+    const allReports = await loadSavedResults(evalsDir);
+    if (currentReport)
+        allReports.unshift(currentReport);
+    const models = latestPerModel(allReports);
+    if (models.length === 0) {
+        return "No eval results found.";
+    }
+    // Collect all eval names across all models
+    const evalNames = [
+        ...new Set(models.flatMap((m) => m.evals.map((e) => e.name))),
+    ].sort();
+    let md = "# Eval Report\n\n";
+    md += `Generated: ${new Date().toISOString()}\n`;
+    md += `Agent: ${models[0]?.agent ?? "unknown"}\n\n`;
+    // ─── Summary table ──────────────────────────────────────────────
+    md += "## Model Comparison\n\n";
+    md += `| Eval | ${models.map((m) => `${m.provider}/${m.model}`).join(" | ")} |\n`;
+    md += `| --- | ${models.map(() => "---").join(" | ")} |\n`;
+    for (const evalName of evalNames) {
+        const cells = models.map((m) => {
+            const evalResult = m.evals.find((e) => e.name === evalName);
+            if (!evalResult)
+                return "-";
+            const icon = evalResult.passRate >= 0.8 ? "PASS" : "FAIL";
+            return `${icon} ${evalResult.avgScore.toFixed(2)} (${Math.round(evalResult.passRate * 100)}%)`;
+        });
+        md += `| ${evalName} | ${cells.join(" | ")} |\n`;
+    }
+    // ─── Overall scores ──────────────────────────────────────────────
+    md += "\n## Overall Scores\n\n";
+    md += "| Model | Pass Rate | Avg Score | p50 Latency | Total Tokens |\n";
+    md += "| --- | --- | --- | --- | --- |\n";
+    for (const report of models) {
+        const overallPassRate = report.evals.length > 0
+            ? report.evals.filter((e) => e.passRate >= 0.8).length /
+                report.evals.length
+            : 0;
+        const overallAvgScore = report.evals.length > 0
+            ? report.evals.reduce((sum, e) => sum + e.avgScore, 0) /
+                report.evals.length
+            : 0;
+        const overallP50 = report.evals.length > 0
+            ? report.evals.reduce((sum, e) => sum + e.p50LatencyMs, 0) /
+                report.evals.length
+            : 0;
+        const totalTokens = report.evals.reduce((sum, e) => sum + (e.totalTokens?.totalTokens ?? 0), 0);
+        md += `| ${report.provider}/${report.model} | ${Math.round(overallPassRate * 100)}% | ${overallAvgScore.toFixed(2)} | ${Math.round(overallP50)}ms | ${totalTokens.toLocaleString()} |\n`;
+    }
+    // ─── Rubric details (latest run per model) ───────────────────────
+    for (const report of models) {
+        const rubricEvals = report.evals.filter((e) => e.trials.some((t) => t.rubric));
+        if (rubricEvals.length === 0)
+            continue;
+        md += `\n## Rubric Details: ${report.provider}/${report.model}\n\n`;
+        for (const evalResult of rubricEvals) {
+            md += `### ${evalResult.name}\n\n`;
+            // Show criteria from first trial that has rubric
+            const trial = evalResult.trials.find((t) => t.rubric);
+            if (!trial?.rubric)
+                continue;
+            for (const criterion of trial.rubric.criteria) {
+                const icon = criterion.passed ? "PASS" : "FAIL";
+                md += `- **${criterion.name}**: ${icon}`;
+                if (!criterion.passed) {
+                    md += ` -- ${criterion.explanation}`;
+                }
+                md += "\n";
+            }
+            md += "\n";
+        }
+    }
+    // ─── Failed trials with transcripts and trace IDs ─────────────────
+    for (const report of models) {
+        const failedEvals = report.evals.filter((e) => e.trials.some((t) => !t.passed));
+        if (failedEvals.length === 0)
+            continue;
+        md += `\n## Failed Trials: ${report.provider}/${report.model}\n\n`;
+        for (const evalResult of failedEvals) {
+            const failedTrials = evalResult.trials.filter((t) => !t.passed);
+            for (const trial of failedTrials) {
+                md += `### ${evalResult.name} -- Trial ${trial.trial} (score: ${trial.score.toFixed(2)})\n\n`;
+                // Trace IDs for Grafana/Tempo lookup
+                const traceIds = trial.turns.map((t) => t.traceId).filter(Boolean);
+                if (traceIds.length > 0) {
+                    md += `**Trace IDs:** ${traceIds.map((id) => `\`${id}\``).join(", ")}\n\n`;
+                }
+                // Failed assertions
+                for (const turn of trial.turns) {
+                    const failed = turn.assertions.filter((a) => !a.passed);
+                    if (failed.length === 0)
+                        continue;
+                    md += `**User:** ${turn.user}\n\n`;
+                    md += `**Agent:** ${turn.agent.slice(0, 500)}${turn.agent.length > 500 ? "..." : ""}\n\n`;
+                    for (const assertion of failed) {
+                        md += `- **${assertion.type}**: FAIL`;
+                        if (assertion.reason)
+                            md += ` -- ${assertion.reason}`;
+                        md += "\n";
+                    }
+                    md += "\n";
+                }
+                // Rubric failures
+                if (trial.rubric) {
+                    const failedCriteria = trial.rubric.criteria.filter((c) => !c.passed);
+                    if (failedCriteria.length > 0) {
+                        md += "**Rubric failures:**\n";
+                        for (const c of failedCriteria) {
+                            md += `- **${c.name}**: ${c.explanation}\n`;
+                        }
+                        md += "\n";
+                    }
+                }
+            }
+        }
+    }
+    return md;
+}
+export async function writeMarkdownReport(evalsDir, currentReport) {
+    const md = await generateComparisonReport(evalsDir, currentReport);
+    const reportPath = join(evalsDir, "evals-report.md");
+    await writeFile(reportPath, md);
+    console.log(chalk.dim(`Report written to ${reportPath}`));
+    return reportPath;
+}
+//# sourceMappingURL=reporter.js.map

package/dist/eval/reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../../src/eval/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,KAAK,MAAM,OAAO,CAAC;AAG1B,0EAA0E;AAE1E,MAAM,UAAU,WAAW,CAAC,MAAkB;IAC5C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACpE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,CAAC,CAAC,CAAC;IAEhE,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACtC,SAAS,CAAC,UAAU,CAAC,CAAC;IACxB,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;IACjD,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,cAAc,MAAM,IAAI,KAAK,eAAe,CAAC,CAAC,CAAC;IACxE,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CACP,aAAa,MAAM,CAAC,KAAK;aACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC;aACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CACF,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC;AAED,SAAS,SAAS,CAAC,MAAkB;IACnC,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;IACxC,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,UAAU,UAAU,CAAC,CAAC,CAAC;IAEjE,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,UAAU,CAAC,KAAK,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IACrE,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IACxD,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,WAAW;QAC9C,CAAC,CAAC,WAAW,MAAM,CAAC,WAAW,CAAC,WAAW,EAAE;QAC7C,CAAC,CAAC,EAAE,CAAC;IACP,OAAO,CAAC,GAAG,CACT,WAAW,CACT,KAAK,MAAM,IAAI,WAAW,IAAI,UAAU,QAAQ,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CACtH,CACF,CAAC;IACF,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,KAAkB;IACpC,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,UAAU,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,EAAE,CACvE,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC/B,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACxC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;gBACtB,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CAAC,SAAS,SAAS,CAAC,IAAI,KAAK,SAAS,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC,CACpE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnE,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;YAC9C,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;gBACtB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;IACH,CAAC;IAED,oEAAoE;IACpE,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC;AAED,0EAA0E;AAE1E,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,QAAgB,EAChB,MAAkB;IAElB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC9C,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,IAAI,GAAG,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC,OAAO,CACvD,cAAc,EACd,GAAG,CACJ,CAAC;IACF,MAAM,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,GAAG,IAAI,IAAI,EAAE,OAAO,CAAC;IACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAE5C,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,QAAQ,EAAE,CAAC,CAAC,CAAC;IACvD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,0EAA0E;AAE1E,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAkB,EAClB,UAAkB;IAElB,MAAM,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC,CAAC;AAC7D,CAAC;AAED,0EAA0E;AAE1E,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,QAAgB;IAEhB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC9C,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,KAAK;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aAClC,IAAI,EAAE;aACN,OAAO,EAAE,CAAC,CAAC,eAAe;QAE7B,MAAM,OAAO,GAAiB,EAAE,CAAC;QACjC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAChE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAe,CAAC,CAAC;QAClD,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,OAAqB;IAC3C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC9C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,SAAS,GAAG,QAAQ,CAAC,SAAS,EAAE,CAAC;YACvD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,QAAgB,EAChB,aAA0B;IAE1B,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACpD,IAAI,aAAa;QAAE,UAAU,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IAE1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,wBAAwB,CAAC;IAClC,CAAC;IAED,2CAA2C;IAC3C,MAAM,SAAS,GAAG;QAChB,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;KAC9D,CAAC,IAAI,EAAE,CAAC;IAET,IAAI,EAAE,GAAG,mBAAmB,CAAC;IAC7B,EAAE,IAAI,cAAc,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,CAAC;IACjD,EAAE,IAAI,UAAU,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,SAAS,MAAM,CAAC;IAEpD,mEAAmE;IACnE,EAAE,IAAI,yBAAyB,CAAC;IAChC,EAAE,IAAI,YAAY,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAClF,EAAE,IAAI,WAAW,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAE3D,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC7B,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;YAC5D,IAAI,CAAC,UAAU;gBAAE,OAAO,GAAG,CAAC;YAC5B,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YAC1D,OAAO,GAAG,IAAI,IAAI,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,GAAG,GAAG,CAAC,IAAI,CAAC;QACjG,CAAC,CAAC,CAAC;QACH,EAAE,IAAI,KAAK,QAAQ,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IACnD,CAAC;IAED,oEAAoE;IACpE,EAAE,IAAI,yBAAyB,CAAC;IAChC,EAAE,IAAI,kEAAkE,CAAC;IACzE,EAAE,IAAI,mCAAmC,CAAC;IAE1C,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,eAAe,GACnB,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YACrB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,GAAG,CAAC,CAAC,MAAM;gBACpD,MAAM,CAAC,KAAK,CAAC,MAAM;YACrB,CAAC,CAAC,CAAC,CAAC;QACR,MAAM,eAAe,GACnB,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YACrB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACpD,MAAM,CAAC,KAAK,CAAC,MAAM;YACrB,CAAC,CAAC,CAAC,CAAC;QACR,MAAM,UAAU,GACd,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YACrB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC;gBACxD,MAAM,CAAC,KAAK,CAAC,MAAM;YACrB,CAAC,CAAC,CAAC,CAAC;QACR,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CACrC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,WAAW,IAAI,CAAC,CAAC,EACnD,CAAC,CACF,CAAC;QAEF,EAAE,IAAI,KAAK,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,GAAG,CAAC,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,WAAW,CAAC,cAAc,EAAE,MAAM,CAAC;IAC3L,CAAC;IAED,oEAAoE;IACpE,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5C,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAC/B,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEvC,EAAE,IAAI,wBAAwB,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,MAAM,CAAC;QACpE,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACrC,EAAE,IAAI,OAAO,UAAU,CAAC,IAAI,MAAM,CAAC;YACnC,iDAAiD;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACtD,IAAI,CAAC,KAAK,EAAE,MAAM;gBAAE,SAAS;YAE7B,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;gBAC9C,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;gBAChD,EAAE,IAAI,OAAO,SAAS,CAAC,IAAI,OAAO,IAAI,EAAE,CAAC;gBACzC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;oBACtB,EAAE,IAAI,OAAO,SAAS,CAAC,WAAW,EAAE,CAAC;gBACvC,CAAC;gBACD,EAAE,IAAI,IAAI,CAAC;YACb,CAAC;YACD,EAAE,IAAI,IAAI,CAAC;QACb,CAAC;IACH,CAAC;IAED,qEAAqE;IACrE,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5C,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAChC,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEvC,EAAE,IAAI,uBAAuB,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,MAAM,CAAC;QACnE,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACrC,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAChE,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,EAAE,IAAI,OAAO,UAAU,CAAC,IAAI,aAAa,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;gBAE9F,qCAAqC;gBACrC,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBACnE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACxB,EAAE,IAAI,kBAAkB,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC7E,CAAC;gBAED,oBAAoB;gBACpB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;oBAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;oBACxD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;wBAAE,SAAS;oBAClC,EAAE,IAAI,aAAa,IAAI,CAAC,IAAI,MAAM,CAAC;oBACnC,EAAE,IAAI,cAAc,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC;oBAC1F,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;wBAC/B,EAAE,IAAI,OAAO,SAAS,CAAC,IAAI,UAAU,CAAC;wBACtC,IAAI,SAAS,CAAC,MAAM;4BAAE,EAAE,IAAI,OAAO,SAAS,CAAC,MAAM,EAAE,CAAC;wBACtD,EAAE,IAAI,IAAI,CAAC;oBACb,CAAC;oBACD,EAAE,IAAI,IAAI,CAAC;gBACb,CAAC;gBAED,kBAAkB;gBAClB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;oBACjB,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;oBACtE,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC9B,EAAE,IAAI,wBAAwB,CAAC;wBAC/B,KAAK,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;4BAC/B,EAAE,IAAI,OAAO,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,WAAW,IAAI,CAAC;wBAC9C,CAAC;wBACD,EAAE,IAAI,IAAI,CAAC;oBACb,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,QAAgB,EAChB,aAA0B;IAE1B,MAAM,EAAE,GAAG,MAAM,wBAAwB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IACnE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;IACrD,MAAM,SAAS,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAChC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC,CAAC;IAC1D,OAAO,UAAU,CAAC;AACpB,CAAC"}

package/dist/eval/runner.d.ts

@@ -0,0 +1,11 @@
+import type { EvalDefinition, EvalResult } from "./types.js";
+export interface RunOptions {
+    gatewayUrl: string;
+    authToken: string;
+    agentId?: string;
+    provider?: string;
+    model?: string;
+    trialsOverride?: number;
+}
+export declare function runEval(definition: EvalDefinition, evalFilePath: string, options: RunOptions): Promise<EvalResult>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/eval/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAGV,cAAc,EACd,UAAU,EAKX,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,wBAAsB,OAAO,CAC3B,UAAU,EAAE,cAAc,EAC1B,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,UAAU,CAAC,CAqCrB"}

package/dist/eval/runner.js

@@ -0,0 +1,172 @@
+import { readFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { createSession, deleteSession, sendAndCollect } from "./client.js";
+import { gradeInline, gradeWithRubric } from "./grader.js";
+export async function runEval(definition, evalFilePath, options) {
+    const trials = options.trialsOverride ?? definition.trials;
+    const results = [];
+    // Load rubric file if specified
+    let rubricContent;
+    if (definition.rubric) {
+        const rubricPath = join(dirname(evalFilePath), definition.rubric);
+        rubricContent = await readFile(rubricPath, "utf-8");
+    }
+    for (let i = 0; i < trials; i++) {
+        const result = await runTrial(i + 1, definition, rubricContent, options);
+        results.push(result);
+    }
+    const passedTrials = results.filter((t) => t.passed).length;
+    const latencies = results.flatMap((t) => t.turns.map((turn) => turn.latencyMs));
+    latencies.sort((a, b) => a - b);
+    // Aggregate token usage across all trials
+    const totalTokens = aggregateTokens(results);
+    return {
+        name: definition.name,
+        passRate: trials > 0 ? passedTrials / trials : 0,
+        avgScore: trials > 0 ? results.reduce((sum, t) => sum + t.score, 0) / trials : 0,
+        p50LatencyMs: latencies.length > 0
+            ? (latencies[Math.floor(latencies.length / 2)] ?? 0)
+            : 0,
+        totalTokens,
+        trials: results,
+    };
+}
+async function runTrial(trialNum, definition, rubricContent, options) {
+    const start = Date.now();
+    const timeoutMs = definition.timeout * 1000;
+    const session = await createSession(options.gatewayUrl, options.authToken, {
+        agentId: options.agentId,
+        forceNew: true,
+        dryRun: true,
+    });
+    const turnResults = [];
+    try {
+        for (const turn of definition.turns) {
+            const response = await sendAndCollect(session, turn.content, timeoutMs);
+            if (response.error) {
+                turnResults.push({
+                    user: turn.content,
+                    agent: response.text || `[Error: ${response.error}]`,
+                    latencyMs: response.latencyMs,
+                    assertions: [
+                        { type: "error", passed: false, score: 0, reason: response.error },
+                    ],
+                    tokens: response.tokens,
+                    traceId: response.traceId,
+                });
+                continue;
+            }
+            // Run assertions for this turn
+            const assertions = turn.assert
+                ? await runAssertions(turn.assert, response.text, options.gatewayUrl, options.authToken, timeoutMs)
+                : [];
+            turnResults.push({
+                user: turn.content,
+                agent: response.text,
+                latencyMs: response.latencyMs,
+                assertions,
+                tokens: response.tokens,
+                traceId: response.traceId,
+            });
+        }
+        // Run rubric grading on full transcript if rubric is specified
+        let rubric;
+        if (rubricContent) {
+            rubric = await gradeWithRubric(options.gatewayUrl, options.authToken, rubricContent, turnResults, timeoutMs);
+        }
+        // Calculate trial score
+        const score = calculateTrialScore(turnResults, rubric);
+        const passed = score >= definition.scoring.pass_threshold;
+        return {
+            trial: trialNum,
+            passed,
+            score,
+            turns: turnResults,
+            rubric,
+            durationMs: Date.now() - start,
+        };
+    }
+    finally {
+        await deleteSession(session);
+    }
+}
+async function runAssertions(assertions, agentResponse, gatewayUrl, authToken, timeoutMs) {
+    const results = [];
+    for (const assertion of assertions) {
+        switch (assertion.type) {
+            case "contains": {
+                const target = assertion.value;
+                const response = assertion.options?.case_insensitive
+                    ? agentResponse.toLowerCase()
+                    : agentResponse;
+                const search = assertion.options?.case_insensitive
+                    ? target.toLowerCase()
+                    : target;
+                const passed = response.includes(search);
+                results.push({ type: "contains", passed, score: passed ? 1 : 0 });
+                break;
+            }
+            case "regex": {
+                const regex = new RegExp(assertion.value, "i");
+                const passed = regex.test(agentResponse);
+                results.push({ type: "regex", passed, score: passed ? 1 : 0 });
+                break;
+            }
+            case "llm-rubric": {
+                const result = await gradeInline(gatewayUrl, authToken, assertion.value, agentResponse, timeoutMs);
+                results.push({
+                    type: "llm-rubric",
+                    passed: result.passed,
+                    score: result.score,
+                    reason: result.reason,
+                });
+                break;
+            }
+        }
+    }
+    return results;
+}
+function calculateTrialScore(turns, rubric) {
+    // Collect all weighted scores
+    const scores = [];
+    for (const turn of turns) {
+        if (turn.assertions.length === 0)
+            continue;
+        // If assertions have no explicit weights, weight them equally
+        const totalWeight = turn.assertions.reduce((sum, _a, _i) => sum + 1, // Default weight 1 per assertion
+        0);
+        for (const assertion of turn.assertions) {
+            scores.push({ score: assertion.score, weight: 1 / totalWeight });
+        }
+    }
+    // Add rubric score if present (weighted equally to all assertion scores combined)
+    if (rubric) {
+        const assertionWeight = scores.length > 0 ? 0.5 : 1;
+        const rubricWeight = scores.length > 0 ? 0.5 : 1;
+        const assertionAvg = scores.length > 0
+            ? scores.reduce((sum, s) => sum + s.score * s.weight, 0)
+            : 0;
+        return assertionAvg * assertionWeight + rubric.score * rubricWeight;
+    }
+    if (scores.length === 0)
+        return 1; // No assertions = pass
+    return scores.reduce((sum, s) => sum + s.score * s.weight, 0);
+}
+function aggregateTokens(trials) {
+    let inputTokens = 0;
+    let outputTokens = 0;
+    for (const trial of trials) {
+        for (const turn of trial.turns) {
+            if (turn.tokens) {
+                inputTokens += turn.tokens.inputTokens ?? 0;
+                outputTokens += turn.tokens.outputTokens ?? 0;
+            }
+        }
+    }
+    return {
+        inputTokens,
+        outputTokens,
+        totalTokens: inputTokens + outputTokens,
+    };
+}
+//# sourceMappingURL=runner.js.map