kagent-ts 0.1.4 → 0.1.5

package/dist/eval/benchmark.js

@@ -0,0 +1,292 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Benchmark = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const eval_runner_1 = require("./eval-runner");
+// ─── Regression Detection Thresholds ─────────────────────────────────────
+/** Minimum drop in success rate to flag as a regression (percentage points). */
+const SUCCESS_RATE_REGRESSION_THRESHOLD = 0.05;
+/** Minimum increase in failures to flag as a regression (absolute count). */
+const FAILURE_COUNT_REGRESSION_THRESHOLD = 2;
+/** Factor by which latency must increase to flag as a regression. */
+const LATENCY_REGRESSION_FACTOR = 1.5;
+/** Minimum absolute latency increase to flag (ms). */
+const LATENCY_MIN_ABSOLUTE_INCREASE_MS = 1000;
+/**
+ * Benchmark — runs evaluation cases against an agent, compares with
+ * baseline results, and flags regressions & improvements.
+ *
+ * Usage:
+ * ```ts
+ * const benchmark = new Benchmark({
+ *   name: "tool-calling-v2",
+ *   agentFactory: (evaluator) => new ReActAgent({ llm, hooks: [evaluator] }),
+ *   cases: myEvalCases,
+ *   baselinePath: ".kagent-benchmarks/tool-calling-v1.json",
+ * });
+ *
+ * const result = await benchmark.run();
+ * console.log(benchmark.generateReport(result));
+ * ```
+ */
+class Benchmark {
+    config;
+    outputDir;
+    constructor(config) {
+        this.config = config;
+        this.outputDir = path.resolve(config.outputDir ?? ".kagent-benchmarks");
+    }
+    /**
+     * Run the benchmark and return results.
+     *
+     * Results are automatically persisted to disk so they can serve
+     * as the baseline for future runs.
+     */
+    async run() {
+        const runner = this.config.runner ?? new eval_runner_1.EvalRunner();
+        const cases = this.config.cases;
+        const timestamp = new Date().toISOString();
+        // Run all cases
+        const results = await runner.run(this.config.agentFactory, cases);
+        // Build summary
+        const summary = this.buildSummary(results, timestamp);
+        // Compare against baseline if available
+        if (this.config.baselinePath) {
+            const baseline = this.loadBaseline();
+            if (baseline) {
+                this.compareWithBaseline(summary, baseline, results);
+            }
+        }
+        const result = { summary, cases: results };
+        // Persist for future baseline use
+        this.persistResult(result);
+        return result;
+    }
+    /**
+     * Generate a Markdown comparison report.
+     */
+    generateReport(result) {
+        const s = result.summary;
+        const passRate = (s.passRate * 100).toFixed(1);
+        let report = `# Benchmark: ${s.name}\n\n`;
+        report += `**Run at:** ${s.timestamp}\n\n`;
+        report += `## Summary\n\n`;
+        report += `| Metric | Value |\n`;
+        report += `|--------|-------|\n`;
+        report += `| Pass Rate | ${passRate}% (${s.passed}/${s.total}) |\n`;
+        report += `| Avg Tool Calls / Case | ${s.avgToolCallsPerCase.toFixed(1)} |\n`;
+        report += `| Avg Duration | ${s.avgLatencyMs}ms |\n\n`;
+        // Regressions (most important — show first)
+        if (s.regressions.length > 0) {
+            report += `## ⚠️ Regressions\n\n`;
+            report += `| Target | Metric | Baseline | Current | Details |\n`;
+            report += `|--------|--------|----------|---------|----------|\n`;
+            for (const r of s.regressions) {
+                report += `| ${r.target} | ${r.metric} | ${r.baseline} | ${r.current} | ${r.description} |\n`;
+            }
+            report += `\n`;
+        }
+        else {
+            report += `## ✅ No Regressions\n\n`;
+        }
+        // Improvements
+        if (s.improvements.length > 0) {
+            report += `## 📈 Improvements\n\n`;
+            report += `| Target | Metric | Baseline | Current |\n`;
+            report += `|--------|--------|----------|--------|\n`;
+            for (const imp of s.improvements) {
+                report += `| ${imp.target} | ${imp.metric} | ${imp.baseline} | ${imp.current} |\n`;
+            }
+            report += `\n`;
+        }
+        // Per-case results
+        report += `## Per-Case Results\n\n`;
+        report += `| Case | Status | Duration | Tool Calls | Success Rate |\n`;
+        report += `|------|--------|----------|------------|-------------|\n`;
+        for (const r of result.cases) {
+            const icon = r.passed ? "✅" : "❌";
+            const sr = (r.scorecard.overallSuccessRate * 100).toFixed(0);
+            report += `| ${icon} ${r.caseName} | ${r.passed ? "PASS" : "FAIL"} | ${r.durationMs}ms | ${r.toolCalls.length} | ${sr}% |\n`;
+        }
+        report += `\n`;
+        // Failure details
+        const failures = result.cases.filter((r) => !r.passed);
+        if (failures.length > 0) {
+            report += `## Failure Details\n\n`;
+            for (const f of failures) {
+                report += `### ❌ ${f.caseName}\n\n`;
+                for (const reason of f.failures) {
+                    report += `- ${reason}\n`;
+                }
+                report += `\n`;
+            }
+        }
+        report += `---\n*Generated at ${new Date().toISOString()}*\n`;
+        return report;
+    }
+    // ─── Private ────────────────────────────────────────────────────────────
+    buildSummary(results, timestamp) {
+        const total = results.length;
+        const passed = results.filter((r) => r.passed).length;
+        const avgToolCalls = total > 0
+            ? results.reduce((s, r) => s + r.toolCalls.length, 0) / total
+            : 0;
+        const avgLatencyMs = total > 0
+            ? Math.round(results.reduce((s, r) => s + r.durationMs, 0) / total)
+            : 0;
+        return {
+            name: this.config.name,
+            timestamp,
+            passed,
+            total,
+            passRate: total > 0 ? passed / total : 1,
+            avgToolCallsPerCase: avgToolCalls,
+            avgLatencyMs,
+            regressions: [],
+            improvements: [],
+        };
+    }
+    /**
+     * Compare current results against a baseline and populate
+     * regressions / improvements in the summary.
+     */
+    compareWithBaseline(summary, baseline, currentResults) {
+        const bl = baseline.summary;
+        // ── Pass rate regression ──────────────────────────────────────────
+        if (bl.passRate - summary.passRate >= SUCCESS_RATE_REGRESSION_THRESHOLD) {
+            const drop = ((bl.passRate - summary.passRate) * 100).toFixed(1);
+            summary.regressions.push({
+                target: "overall",
+                metric: "passRate",
+                baseline: `${(bl.passRate * 100).toFixed(1)}%`,
+                current: `${(summary.passRate * 100).toFixed(1)}%`,
+                description: `Pass rate dropped by ${drop} percentage points.`,
+            });
+        }
+        else if (summary.passRate - bl.passRate >=
+            SUCCESS_RATE_REGRESSION_THRESHOLD) {
+            summary.improvements.push({
+                target: "overall",
+                metric: "passRate",
+                baseline: `${(bl.passRate * 100).toFixed(1)}%`,
+                current: `${(summary.passRate * 100).toFixed(1)}%`,
+            });
+        }
+        // ── Latency regression ────────────────────────────────────────────
+        if (summary.avgLatencyMs >
+            bl.avgLatencyMs * LATENCY_REGRESSION_FACTOR &&
+            summary.avgLatencyMs - bl.avgLatencyMs > LATENCY_MIN_ABSOLUTE_INCREASE_MS) {
+            summary.regressions.push({
+                target: "overall",
+                metric: "avgLatencyMs",
+                baseline: `${bl.avgLatencyMs}ms`,
+                current: `${summary.avgLatencyMs}ms`,
+                description: `Average latency increased by ${summary.avgLatencyMs - bl.avgLatencyMs}ms.`,
+            });
+        }
+        // ── Per-case comparison ───────────────────────────────────────────
+        const baselineCases = new Map(baseline.cases.map((c) => [c.caseName, c]));
+        for (const current of currentResults) {
+            const prev = baselineCases.get(current.caseName);
+            if (!prev)
+                continue; // New case — no baseline to compare
+            // Case flipped from pass to fail
+            if (prev.passed && !current.passed) {
+                summary.regressions.push({
+                    target: current.caseName,
+                    metric: "passed",
+                    baseline: "true",
+                    current: "false",
+                    description: `"${current.caseName}" went from PASS to FAIL. Failures: ${current.failures.join("; ")}`,
+                });
+            }
+            // Case flipped from fail to pass
+            if (!prev.passed && current.passed) {
+                summary.improvements.push({
+                    target: current.caseName,
+                    metric: "passed",
+                    baseline: "false",
+                    current: "true",
+                });
+            }
+            // Failure count increased significantly
+            const prevFailCount = prev.failures.length;
+            const currFailCount = current.failures.length;
+            if (currFailCount - prevFailCount >= FAILURE_COUNT_REGRESSION_THRESHOLD) {
+                summary.regressions.push({
+                    target: current.caseName,
+                    metric: "failureCount",
+                    baseline: String(prevFailCount),
+                    current: String(currFailCount),
+                    description: `Failure count increased from ${prevFailCount} to ${currFailCount}.`,
+                });
+            }
+        }
+    }
+    /**
+     * Load baseline results from disk.
+     */
+    loadBaseline() {
+        if (!this.config.baselinePath)
+            return null;
+        try {
+            const raw = fs.readFileSync(this.config.baselinePath, "utf-8");
+            return JSON.parse(raw);
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Persist results to disk for future baseline use.
+     */
+    persistResult(result) {
+        try {
+            fs.mkdirSync(this.outputDir, { recursive: true });
+            const safeName = this.config.name.replace(/[^a-zA-Z0-9_-]/g, "_");
+            const timestamp = result.summary.timestamp.replace(/[:.]/g, "-");
+            const filename = `${safeName}_${timestamp}.json`;
+            const filePath = path.join(this.outputDir, filename);
+            fs.writeFileSync(filePath, JSON.stringify(result, null, 2), "utf-8");
+        }
+        catch {
+            // Best-effort persistence — never throw from persist
+        }
+    }
+}
+exports.Benchmark = Benchmark;
+//# sourceMappingURL=benchmark.js.map

package/dist/eval/benchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/eval/benchmark.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAyD;AAwCzD,4EAA4E;AAE5E,gFAAgF;AAChF,MAAM,iCAAiC,GAAG,IAAI,CAAC;AAE/C,6EAA6E;AAC7E,MAAM,kCAAkC,GAAG,CAAC,CAAC;AAE7C,qEAAqE;AACrE,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAEtC,sDAAsD;AACtD,MAAM,gCAAgC,GAAG,IAAI,CAAC;AAE9C;;;;;;;;;;;;;;;;GAgBG;AACH,MAAa,SAAS;IACZ,MAAM,CAAkB;IACxB,SAAS,CAAS;IAE1B,YAAY,MAAuB;QACjC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,IAAI,oBAAoB,CAAC,CAAC;IAC1E,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,GAAG;QACP,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,wBAAU,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAChC,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAE3C,gBAAgB;QAChB,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;QAElE,gBAAgB;QAChB,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAEtD,wCAAwC;QACxC,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;YACrC,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAoB,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAE5D,kCAAkC;QAClC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAE3B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,MAAuB;QACpC,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC;QACzB,MAAM,QAAQ,GAAG,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAE/C,IAAI,MAAM,GAAG,gBAAgB,CAAC,CAAC,IAAI,MAAM,CAAC;QAC1C,MAAM,IAAI,eAAe,CAAC,CAAC,SAAS,MAAM,CAAC;QAE3C,MAAM,IAAI,gBAAgB,CAAC;QAC3B,MAAM,IAAI,sBAAsB,CAAC;QACjC,MAAM,IAAI,sBAAsB,CAAC;QACjC,MAAM,IAAI,iBAAiB,QAAQ,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,OAAO,CAAC;QACpE,MAAM,IAAI,6BAA6B,CAAC,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9E,MAAM,IAAI,oBAAoB,CAAC,CAAC,YAAY,UAAU,CAAC;QAEvD,4CAA4C;QAC5C,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,uBAAuB,CAAC;YAClC,MAAM,IAAI,sDAAsD,CAAC;YACjE,MAAM,IAAI,uDAAuD,CAAC;YAClE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,CAAC,MAAM,MAAM,CAAC,CAAC,MAAM,MAAM,CAAC,CAAC,QAAQ,MAAM,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,WAAW,MAAM,CAAC;YAChG,CAAC;YACD,MAAM,IAAI,IAAI,CAAC;QACjB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,yBAAyB,CAAC;QACtC,CAAC;QAED,eAAe;QACf,IAAI,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,wBAAwB,CAAC;YACnC,MAAM,IAAI,4CAA4C,CAAC;YACvD,MAAM,IAAI,2CAA2C,CAAC;YACtD,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,GAAG,CAAC,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,GAAG,CAAC,QAAQ,MAAM,GAAG,CAAC,OAAO,MAAM,CAAC;YACrF,CAAC;YACD,MAAM,IAAI,IAAI,CAAC;QACjB,CAAC;QAED,mBAAmB;QACnB,MAAM,IAAI,yBAAyB,CAAC;QACpC,MAAM,IAAI,4DAA4D,CAAC;QACvE,MAAM,IAAI,2DAA2D,CAAC;QACtE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAClC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC7D,MAAM,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,QAAQ,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,CAAC,UAAU,QAAQ,CAAC,CAAC,SAAS,CAAC,MAAM,MAAM,EAAE,OAAO,CAAC;QAC/H,CAAC;QACD,MAAM,IAAI,IAAI,CAAC;QAEf,kBAAkB;QAClB,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACvD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,wBAAwB,CAAC;YACnC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACzB,MAAM,IAAI,SAAS,CAAC,CAAC,QAAQ,MAAM,CAAC;gBACpC,KAAK,MAAM,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;oBAChC,MAAM,IAAI,KAAK,MAAM,IAAI,CAAC;gBAC5B,CAAC;gBACD,MAAM,IAAI,IAAI,CAAC;YACjB,CAAC;QACH,CAAC;QAED,MAAM,IAAI,sBAAsB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC;QAC9D,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,2EAA2E;IAEnE,YAAY,CAClB,OAAqB,EACrB,SAAiB;QAEjB,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACtD,MAAM,YAAY,GAChB,KAAK,GAAG,CAAC;YACP,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK;YAC7D,CAAC,CAAC,CAAC,CAAC;QACR,MAAM,YAAY,GAChB,KAAK,GAAG,CAAC;YACP,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC;YACnE,CAAC,CAAC,CAAC,CAAC;QAER,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;YACtB,SAAS;YACT,MAAM;YACN,KAAK;YACL,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACxC,mBAAmB,EAAE,YAAY;YACjC,YAAY;YACZ,WAAW,EAAE,EAAE;YACf,YAAY,EAAE,EAAE;SACjB,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,mBAAmB,CACzB,OAAyB,EACzB,QAAyB,EACzB,cAA4B;QAE5B,MAAM,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC;QAE5B,qEAAqE;QACrE,IAAI,EAAE,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,iCAAiC,EAAE,CAAC;YACxE,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACjE,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC;gBACvB,MAAM,EAAE,SAAS;gBACjB,MAAM,EAAE,UAAU;gBAClB,QAAQ,EAAE,GAAG,CAAC,EAAE,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;gBAC9C,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;gBAClD,WAAW,EAAE,wBAAwB,IAAI,qBAAqB;aAC/D,CAAC,CAAC;QACL,CAAC;aAAM,IACL,OAAO,CAAC,QAAQ,GAAG,EAAE,CAAC,QAAQ;YAC9B,iCAAiC,EACjC,CAAC;YACD,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC;gBACxB,MAAM,EAAE,SAAS;gBACjB,MAAM,EAAE,UAAU;gBAClB,QAAQ,EAAE,GAAG,CAAC,EAAE,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;gBAC9C,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;aACnD,CAAC,CAAC;QACL,CAAC;QAED,qEAAqE;QACrE,IACE,OAAO,CAAC,YAAY;YAClB,EAAE,CAAC,YAAY,GAAG,yBAAyB;YAC7C,OAAO,CAAC,YAAY,GAAG,EAAE,CAAC,YAAY,GAAG,gCAAgC,EACzE,CAAC;YACD,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC;gBACvB,MAAM,EAAE,SAAS;gBACjB,MAAM,EAAE,cAAc;gBACtB,QAAQ,EAAE,GAAG,EAAE,CAAC,YAAY,IAAI;gBAChC,OAAO,EAAE,GAAG,OAAO,CAAC,YAAY,IAAI;gBACpC,WAAW,EAAE,gCACX,OAAO,CAAC,YAAY,GAAG,EAAE,CAAC,YAC5B,KAAK;aACN,CAAC,CAAC;QACL,CAAC;QAED,qEAAqE;QACrE,MAAM,aAAa,GAAG,IAAI,GAAG,CAC3B,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAC3C,CAAC;QAEF,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACjD,IAAI,CAAC,IAAI;gBAAE,SAAS,CAAC,oCAAoC;YAEzD,iCAAiC;YACjC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBACnC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC;oBACvB,MAAM,EAAE,OAAO,CAAC,QAAQ;oBACxB,MAAM,EAAE,QAAQ;oBAChB,QAAQ,EAAE,MAAM;oBAChB,OAAO,EAAE,OAAO;oBAChB,WAAW,EAAE,IAAI,OAAO,CAAC,QAAQ,uCAAuC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;iBACtG,CAAC,CAAC;YACL,CAAC;YAED,iCAAiC;YACjC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACnC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC;oBACxB,MAAM,EAAE,OAAO,CAAC,QAAQ;oBACxB,MAAM,EAAE,QAAQ;oBAChB,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,MAAM;iBAChB,CAAC,CAAC;YACL,CAAC;YAED,wCAAwC;YACxC,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC3C,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC9C,IAAI,aAAa,GAAG,aAAa,IAAI,kCAAkC,EAAE,CAAC;gBACxE,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC;oBACvB,MAAM,EAAE,OAAO,CAAC,QAAQ;oBACxB,MAAM,EAAE,cAAc;oBACtB,QAAQ,EAAE,MAAM,CAAC,aAAa,CAAC;oBAC/B,OAAO,EAAE,MAAM,CAAC,aAAa,CAAC;oBAC9B,WAAW,EAAE,gCAAgC,aAAa,OAAO,aAAa,GAAG;iBAClF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,YAAY;QAClB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC;QAE3C,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,MAAuB;QAC3C,IAAI,CAAC;YACH,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAElD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;YAClE,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YACjE,MAAM,QAAQ,GAAG,GAAG,QAAQ,IAAI,SAAS,OAAO,CAAC;YACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YAErD,EAAE,CAAC,aAAa,CACd,QAAQ,EACR,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAC/B,OAAO,CACR,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,qDAAqD;QACvD,CAAC;IACH,CAAC;CACF;AA/QD,8BA+QC"}

package/dist/eval/eval-runner.d.ts

@@ -0,0 +1,79 @@
+import type { LLMProvider } from "../llm/interface";
+import { ToolCallEvaluator } from "./tool-call-evaluator";
+import type { EvalCase, EvalResult } from "./types";
+export type { EvalCase, EvalResult, LLMEvalJudgment } from "./types";
+/**
+ * An agent factory — called once per eval case to create a fresh agent
+ * with a ToolCallEvaluator hook attached.
+ */
+export type AgentFactory = (evaluator: ToolCallEvaluator) => {
+    run(input: string): Promise<string>;
+    cancel(): void;
+};
+/**
+ * Configuration for the EvalRunner.
+ */
+export interface EvalRunnerConfig {
+    /**
+     * Default timeout per case in milliseconds.
+     * Cases can override with their own `timeoutMs`.
+     * Default: 120_000 (2 minutes).
+     */
+    defaultTimeoutMs?: number;
+    /**
+     * Optional LLM provider for answer quality judging.
+     * When set, each case's final answer is independently evaluated.
+     *
+     * Using a different model than the agent's own LLM provides an
+     * unbiased quality assessment. Pass `router.forReflection()` from
+     * a ModelRouter for this purpose.
+     */
+    judgeLLM?: LLMProvider;
+}
+/**
+ * EvalRunner — runs test cases against an agent and produces pass/fail results.
+ *
+ * Uses an agent factory so each case starts fresh (no context pollution between
+ * cases). The factory receives a ToolCallEvaluator hook to collect metrics.
+ *
+ * Usage:
+ * ```ts
+ * const runner = new EvalRunner({ judgeLLM: router.forReflection() });
+ *
+ * const results = await runner.run(
+ *   (evaluator) => new ReActAgent({ llm, hooks: [evaluator] }),
+ *   [
+ *     { name: "basic math", input: "2+2=?", expectedTools: ["calculator"] },
+ *     { name: "file read", input: "read README.md", expectedTools: ["read_file"] },
+ *   ],
+ * );
+ *
+ * console.log(runner.generateReport(results));
+ * ```
+ */
+export declare class EvalRunner {
+    private defaultTimeoutMs;
+    private judgeLLM?;
+    constructor(config?: EvalRunnerConfig);
+    /**
+     * Run a batch of evaluation cases.
+     *
+     * @param factory Creates a fresh agent for each case. Receives a
+     *                ToolCallEvaluator hook that MUST be attached to the
+     *                agent's hooks array.
+     * @param cases   The test cases to run.
+     * @returns One EvalResult per case.
+     */
+    run(factory: AgentFactory, cases: EvalCase[]): Promise<EvalResult[]>;
+    /**
+     * Run a single case and return the result (convenience method).
+     */
+    runCase(factory: AgentFactory, c: EvalCase): Promise<EvalResult>;
+    /**
+     * Generate a Markdown report from evaluation results.
+     */
+    generateReport(results: EvalResult[]): string;
+    private judgeAnswer;
+    private parseJudgment;
+}
+//# sourceMappingURL=eval-runner.d.ts.map

package/dist/eval/eval-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-runner.d.ts","sourceRoot":"","sources":["../../src/eval/eval-runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAe,MAAM,kBAAkB,CAAC;AAGjE,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAmB,MAAM,SAAS,CAAC;AAIrE,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAErE;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,SAAS,EAAE,iBAAiB,KAAK;IAC3D,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,IAAI,IAAI,CAAC;CAChB,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;;;;OAOG;IACH,QAAQ,CAAC,EAAE,WAAW,CAAC;CACxB;AAqBD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,CAAc;gBAEnB,MAAM,CAAC,EAAE,gBAAgB;IAKrC;;;;;;;;OAQG;IACG,GAAG,CACP,OAAO,EAAE,YAAY,EACrB,KAAK,EAAE,QAAQ,EAAE,GAChB,OAAO,CAAC,UAAU,EAAE,CAAC;IAiGxB;;OAEG;IACG,OAAO,CACX,OAAO,EAAE,YAAY,EACrB,CAAC,EAAE,QAAQ,GACV,OAAO,CAAC,UAAU,CAAC;IAKtB;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,MAAM;YAwD/B,WAAW;IA0BzB,OAAO,CAAC,aAAa;CAyBtB"}

package/dist/eval/eval-runner.js

@@ -0,0 +1,252 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EvalRunner = void 0;
+const types_1 = require("../messages/types");
+const tool_call_evaluator_1 = require("./tool-call-evaluator");
+// ─── LLM Judge System Prompt ──────────────────────────────────────────────
+const JUDGE_SYSTEM_PROMPT = `You are an impartial evaluation judge. Your job is to assess the quality
+of an AI agent's answer to a user query.
+
+Evaluate the answer across these dimensions:
+- **Correctness**: Is the answer factually correct?
+- **Completeness**: Does it fully address the user's query?
+- **Clarity**: Is the answer well-structured and easy to understand?
+- **Efficiency**: Did the agent use a reasonable approach? Any obvious wasted effort?
+
+Output a JSON object:
+{
+  "passed": true/false,
+  "score": 0-100,
+  "reasoning": "brief explanation (1-3 sentences)",
+  "issues": ["issue 1", "issue 2"]  // empty array if no issues
+}`;
+/**
+ * EvalRunner — runs test cases against an agent and produces pass/fail results.
+ *
+ * Uses an agent factory so each case starts fresh (no context pollution between
+ * cases). The factory receives a ToolCallEvaluator hook to collect metrics.
+ *
+ * Usage:
+ * ```ts
+ * const runner = new EvalRunner({ judgeLLM: router.forReflection() });
+ *
+ * const results = await runner.run(
+ *   (evaluator) => new ReActAgent({ llm, hooks: [evaluator] }),
+ *   [
+ *     { name: "basic math", input: "2+2=?", expectedTools: ["calculator"] },
+ *     { name: "file read", input: "read README.md", expectedTools: ["read_file"] },
+ *   ],
+ * );
+ *
+ * console.log(runner.generateReport(results));
+ * ```
+ */
+class EvalRunner {
+    defaultTimeoutMs;
+    judgeLLM;
+    constructor(config) {
+        this.defaultTimeoutMs = config?.defaultTimeoutMs ?? 120_000;
+        this.judgeLLM = config?.judgeLLM;
+    }
+    /**
+     * Run a batch of evaluation cases.
+     *
+     * @param factory Creates a fresh agent for each case. Receives a
+     *                ToolCallEvaluator hook that MUST be attached to the
+     *                agent's hooks array.
+     * @param cases   The test cases to run.
+     * @returns One EvalResult per case.
+     */
+    async run(factory, cases) {
+        const results = [];
+        for (const c of cases) {
+            const evaluator = new tool_call_evaluator_1.ToolCallEvaluator();
+            const agent = factory(evaluator);
+            const caseTimeout = c.timeoutMs ?? this.defaultTimeoutMs;
+            const startedAt = Date.now();
+            let answer;
+            let iterations = 0;
+            const toolCalls = [];
+            const failures = [];
+            try {
+                answer = await withTimeout(agent.run(c.input), caseTimeout);
+                // Collect tool calls from the evaluator
+                for (const r of evaluator.getRecords()) {
+                    if (r.endTime)
+                        toolCalls.push(r.toolName);
+                }
+                iterations = toolCalls.length;
+                // ── Checks ──────────────────────────────────────────────────
+                if (c.expectedTools && c.expectedTools.length > 0) {
+                    for (const expected of c.expectedTools) {
+                        if (!toolCalls.includes(expected)) {
+                            failures.push(`Expected tool "${expected}" was not called.`);
+                        }
+                    }
+                }
+                if (c.forbiddenTools && c.forbiddenTools.length > 0) {
+                    for (const forbidden of c.forbiddenTools) {
+                        if (toolCalls.includes(forbidden)) {
+                            failures.push(`Forbidden tool "${forbidden}" was called.`);
+                        }
+                    }
+                }
+                if (c.expectedOutput) {
+                    const pattern = c.expectedOutput;
+                    const matches = typeof pattern === "string"
+                        ? answer.includes(pattern)
+                        : pattern.test(answer);
+                    if (!matches) {
+                        failures.push(`Answer does not match expected pattern: "${pattern}".`);
+                    }
+                }
+            }
+            catch (err) {
+                answer = err instanceof Error ? err.message : String(err);
+                failures.push(`Execution error: ${answer}`);
+            }
+            const durationMs = Date.now() - startedAt;
+            const scorecard = evaluator.getScorecard();
+            // ── LLM Judging ───────────────────────────────────────────────
+            let llmJudgment;
+            if (this.judgeLLM && failures.length === 0) {
+                try {
+                    llmJudgment = await this.judgeAnswer(c.input, answer);
+                }
+                catch {
+                    // Judge failed — leave judgment undefined
+                }
+            }
+            if (llmJudgment && !llmJudgment.passed) {
+                failures.push(`LLM judge (score ${llmJudgment.score}/100): ${llmJudgment.reasoning}`);
+            }
+            results.push({
+                caseName: c.name,
+                passed: failures.length === 0,
+                answer,
+                toolCalls,
+                iterations,
+                durationMs,
+                scorecard,
+                llmJudgment,
+                failures,
+            });
+        }
+        return results;
+    }
+    /**
+     * Run a single case and return the result (convenience method).
+     */
+    async runCase(factory, c) {
+        const results = await this.run(factory, [c]);
+        return results[0];
+    }
+    /**
+     * Generate a Markdown report from evaluation results.
+     */
+    generateReport(results) {
+        const passed = results.filter((r) => r.passed).length;
+        const total = results.length;
+        const passRate = total > 0 ? ((passed / total) * 100).toFixed(1) : "0.0";
+        const avgLatency = total > 0
+            ? Math.round(results.reduce((s, r) => s + r.durationMs, 0) / total)
+            : 0;
+        let report = `# Evaluation Report\n\n`;
+        report += `## Summary\n\n`;
+        report += `| Metric | Value |\n`;
+        report += `|--------|-------|\n`;
+        report += `| Cases | ${total} |\n`;
+        report += `| Passed | ${passed} |\n`;
+        report += `| Failed | ${total - passed} |\n`;
+        report += `| Pass Rate | ${passRate}% |\n`;
+        report += `| Avg Duration | ${avgLatency}ms |\n\n`;
+        report += `## Results\n\n`;
+        for (const r of results) {
+            const icon = r.passed ? "✅" : "❌";
+            report += `### ${icon} ${r.caseName}\n\n`;
+            report += `- **Duration:** ${r.durationMs}ms\n`;
+            report += `- **Tool calls:** ${r.toolCalls.join(", ") || "(none)"}\n`;
+            report += `- **Tool success rate:** ${(r.scorecard.overallSuccessRate * 100).toFixed(1)}%\n`;
+            if (r.llmJudgment) {
+                report += `- **Judge score:** ${r.llmJudgment.score}/100\n`;
+                if (r.llmJudgment.issues.length > 0) {
+                    report += `- **Issues:**\n`;
+                    for (const issue of r.llmJudgment.issues) {
+                        report += `  - ${issue}\n`;
+                    }
+                }
+            }
+            if (r.failures.length > 0) {
+                report += `- **Failures:**\n`;
+                for (const f of r.failures) {
+                    report += `  - ${f}\n`;
+                }
+            }
+            report += `\n<details>\n<summary>Answer</summary>\n\n${r.answer}\n\n</details>\n\n`;
+        }
+        report += `---\n*Generated at ${new Date().toISOString()}*\n`;
+        return report;
+    }
+    // ─── Private ────────────────────────────────────────────────────────────
+    async judgeAnswer(query, answer) {
+        if (!this.judgeLLM) {
+            return { passed: true, score: 100, reasoning: "", issues: [] };
+        }
+        const messages = [
+            { role: types_1.Role.System, content: JUDGE_SYSTEM_PROMPT },
+            {
+                role: types_1.Role.User,
+                content: [
+                    `User query: ${query}`,
+                    ``,
+                    `Agent answer: ${answer}`,
+                    ``,
+                    `Please evaluate the answer quality. Output JSON only.`,
+                ].join("\n"),
+            },
+        ];
+        const response = await this.judgeLLM.chat(messages);
+        return this.parseJudgment(response.content);
+    }
+    parseJudgment(raw) {
+        try {
+            let json = raw.trim();
+            const fenceMatch = json.match(/```(?:json)?\s*([\s\S]*?)\s*```/);
+            if (fenceMatch)
+                json = fenceMatch[1];
+            const parsed = JSON.parse(json);
+            return {
+                passed: Boolean(parsed.passed),
+                score: Math.max(0, Math.min(100, Number(parsed.score) || 0)),
+                reasoning: String(parsed.reasoning ?? ""),
+                issues: Array.isArray(parsed.issues)
+                    ? parsed.issues.map(String)
+                    : [],
+            };
+        }
+        catch {
+            return {
+                passed: true,
+                score: 50,
+                reasoning: "Could not parse judge response.",
+                issues: [],
+            };
+        }
+    }
+}
+exports.EvalRunner = EvalRunner;
+// ─── Helpers ───────────────────────────────────────────────────────────────
+async function withTimeout(promise, timeoutMs) {
+    let timer;
+    const timeoutPromise = new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new Error(`Timed out after ${timeoutMs / 1000}s.`)), timeoutMs);
+    });
+    try {
+        return await Promise.race([promise, timeoutPromise]);
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+//# sourceMappingURL=eval-runner.js.map

package/dist/eval/eval-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-runner.js","sourceRoot":"","sources":["../../src/eval/eval-runner.ts"],"names":[],"mappings":";;;AAEA,6CAAyC;AACzC,+DAA0D;AAsC1D,6EAA6E;AAE7E,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;;EAe1B,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAa,UAAU;IACb,gBAAgB,CAAS;IACzB,QAAQ,CAAe;IAE/B,YAAY,MAAyB;QACnC,IAAI,CAAC,gBAAgB,GAAG,MAAM,EAAE,gBAAgB,IAAI,OAAO,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,MAAM,EAAE,QAAQ,CAAC;IACnC,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CACP,OAAqB,EACrB,KAAiB;QAEjB,MAAM,OAAO,GAAiB,EAAE,CAAC;QAEjC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,SAAS,GAAG,IAAI,uCAAiB,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;YACjC,MAAM,WAAW,GAAG,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,gBAAgB,CAAC;YACzD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE7B,IAAI,MAAc,CAAC;YACnB,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,MAAM,SAAS,GAAa,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAa,EAAE,CAAC;YAE9B,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC,CAAC;gBAE5D,wCAAwC;gBACxC,KAAK,MAAM,CAAC,IAAI,SAAS,CAAC,UAAU,EAAE,EAAE,CAAC;oBACvC,IAAI,CAAC,CAAC,OAAO;wBAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;gBAC5C,CAAC;gBACD,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC;gBAE9B,+DAA+D;gBAE/D,IAAI,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClD,KAAK,MAAM,QAAQ,IAAI,CAAC,CAAC,aAAa,EAAE,CAAC;wBACvC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;4BAClC,QAAQ,CAAC,IAAI,CACX,kBAAkB,QAAQ,mBAAmB,CAC9C,CAAC;wBACJ,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACpD,KAAK,MAAM,SAAS,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;wBACzC,IAAI,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;4BAClC,QAAQ,CAAC,IAAI,CACX,mBAAmB,SAAS,eAAe,CAC5C,CAAC;wBACJ,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;oBACrB,MAAM,OAAO,GAAG,CAAC,CAAC,cAAc,CAAC;oBACjC,MAAM,OAAO,GACX,OAAO,OAAO,KAAK,QAAQ;wBACzB,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC;wBAC1B,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,QAAQ,CAAC,IAAI,CACX,4CAA4C,OAAO,IAAI,CACxD,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,MAAM,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC1D,QAAQ,CAAC,IAAI,CAAC,oBAAoB,MAAM,EAAE,CAAC,CAAC;YAC9C,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,YAAY,EAAE,CAAC;YAE3C,iEAAiE;YACjE,IAAI,WAAwC,CAAC;YAC7C,IAAI,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3C,IAAI,CAAC;oBACH,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBACxD,CAAC;gBAAC,MAAM,CAAC;oBACP,0CAA0C;gBAC5C,CAAC;YACH,CAAC;YAED,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;gBACvC,QAAQ,CAAC,IAAI,CACX,oBAAoB,WAAW,CAAC,KAAK,UAAU,WAAW,CAAC,SAAS,EAAE,CACvE,CAAC;YACJ,CAAC;YAED,OAAO,CAAC,IAAI,CAAC;gBACX,QAAQ,EAAE,CAAC,CAAC,IAAI;gBAChB,MAAM,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;gBAC7B,MAAM;gBACN,SAAS;gBACT,UAAU;gBACV,UAAU;gBACV,SAAS;gBACT,WAAW;gBACX,QAAQ;aACT,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CACX,OAAqB,EACrB,CAAW;QAEX,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,OAAqB;QAClC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACtD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,MAAM,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACzE,MAAM,UAAU,GACd,KAAK,GAAG,CAAC;YACP,CAAC,CAAC,IAAI,CAAC,KAAK,CACR,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,KAAK,CACtD;YACH,CAAC,CAAC,CAAC,CAAC;QAER,IAAI,MAAM,GAAG,yBAAyB,CAAC;QACvC,MAAM,IAAI,gBAAgB,CAAC;QAC3B,MAAM,IAAI,sBAAsB,CAAC;QACjC,MAAM,IAAI,sBAAsB,CAAC;QACjC,MAAM,IAAI,aAAa,KAAK,MAAM,CAAC;QACnC,MAAM,IAAI,cAAc,MAAM,MAAM,CAAC;QACrC,MAAM,IAAI,cAAc,KAAK,GAAG,MAAM,MAAM,CAAC;QAC7C,MAAM,IAAI,iBAAiB,QAAQ,OAAO,CAAC;QAC3C,MAAM,IAAI,oBAAoB,UAAU,UAAU,CAAC;QAEnD,MAAM,IAAI,gBAAgB,CAAC;QAE3B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAClC,MAAM,IAAI,OAAO,IAAI,IAAI,CAAC,CAAC,QAAQ,MAAM,CAAC;YAC1C,MAAM,IAAI,mBAAmB,CAAC,CAAC,UAAU,MAAM,CAAC;YAChD,MAAM,IAAI,qBAAqB,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,IAAI,CAAC;YACtE,MAAM,IAAI,4BAA4B,CAAC,CAAC,CAAC,SAAS,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;YAE7F,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;gBAClB,MAAM,IAAI,sBAAsB,CAAC,CAAC,WAAW,CAAC,KAAK,QAAQ,CAAC;gBAC5D,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACpC,MAAM,IAAI,iBAAiB,CAAC;oBAC5B,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;wBACzC,MAAM,IAAI,OAAO,KAAK,IAAI,CAAC;oBAC7B,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,mBAAmB,CAAC;gBAC9B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;oBAC3B,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;gBACzB,CAAC;YACH,CAAC;YAED,MAAM,IAAI,6CAA6C,CAAC,CAAC,MAAM,oBAAoB,CAAC;QACtF,CAAC;QAED,MAAM,IAAI,sBAAsB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC;QAC9D,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,2EAA2E;IAEnE,KAAK,CAAC,WAAW,CACvB,KAAa,EACb,MAAc;QAEd,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QACjE,CAAC;QAED,MAAM,QAAQ,GAAkB;YAC9B,EAAE,IAAI,EAAE,YAAI,CAAC,MAAM,EAAE,OAAO,EAAE,mBAAmB,EAAE;YACnD;gBACE,IAAI,EAAE,YAAI,CAAC,IAAI;gBACf,OAAO,EAAE;oBACP,eAAe,KAAK,EAAE;oBACtB,EAAE;oBACF,iBAAiB,MAAM,EAAE;oBACzB,EAAE;oBACF,uDAAuD;iBACxD,CAAC,IAAI,CAAC,IAAI,CAAC;aACb;SACF,CAAC;QAEF,MAAM,QAAQ,GAAgB,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjE,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC;IAEO,aAAa,CAAC,GAAW;QAC/B,IAAI,CAAC;YACH,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;YACtB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;YACjE,IAAI,UAAU;gBAAE,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAErC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEhC,OAAO;gBACL,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;gBAC9B,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC5D,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;gBACzC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;oBAClC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;oBAC3B,CAAC,CAAC,EAAE;aACP,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,EAAE;gBACT,SAAS,EAAE,iCAAiC;gBAC5C,MAAM,EAAE,EAAE;aACX,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AA/OD,gCA+OC;AAED,8EAA8E;AAE9E,KAAK,UAAU,WAAW,CACxB,OAAmB,EACnB,SAAiB;IAEjB,IAAI,KAAgD,CAAC;IAErD,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;QACtD,KAAK,GAAG,UAAU,CAChB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,SAAS,GAAG,IAAI,IAAI,CAAC,CAAC,EAChE,SAAS,CACV,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC;IACvD,CAAC;YAAS,CAAC;QACT,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;AACH,CAAC"}

package/dist/eval/index.d.ts

@@ -0,0 +1,7 @@
+export { ToolCallEvaluator } from "./tool-call-evaluator";
+export { EvalRunner } from "./eval-runner";
+export type { EvalRunnerConfig, AgentFactory, EvalCase, EvalResult, LLMEvalJudgment } from "./eval-runner";
+export { Benchmark } from "./benchmark";
+export type { BenchmarkConfig } from "./benchmark";
+export type { ToolCallRecord, ToolCallStats, ToolCallScorecard, Regression, Improvement, BenchmarkSummary, BenchmarkResult, } from "./types";
+//# sourceMappingURL=index.d.ts.map

package/dist/eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAG1D,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAG3G,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAGnD,YAAY,EACV,cAAc,EACd,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,WAAW,EACX,gBAAgB,EAChB,eAAe,GAChB,MAAM,SAAS,CAAC"}