@databricks/appkit 0.73.0 → 0.74.1

package/dist/evals/index.d.ts

@@ -1,14 +1,14 @@
 import { DatabricksAuth, ResolveDatabricksAuthOptions, resolveDatabricksAuth, resolveWorkspaceClient } from "../connectors/mlflow/auth.js";
 import { MlflowClient, PostResult, normalizeHost } from "../connectors/mlflow/client.js";
 import "../connectors/mlflow/index.js";
-import { DatasetRow, ReadEvalDatasetOptions, readEvalDataset } from "./dataset.js";
+import { DatasetRow, ReadEvalDatasetOptions, readEvalDataset, userTurns } from "./dataset.js";
 import { AssertionHandle, AssertionResult, CustomJudgeSpec, DriveResult, EvalDefinition, EvalDriver, EvalResult, MatchResult, Matcher, Severity, TestContext } from "./types.js";
-import { defineEval } from "./define-eval.js";
-import { DiscoveredEval, discoverEvalFiles } from "./discover.js";
+import { defineEval, defineEvalConfig } from "./define-eval.js";
+import { DiscoveredEval, DiscoveredEvalConfig, discoverEvalConfigs, discoverEvalFiles } from "./discover.js";
 import { HttpDriverOptions, createHttpDriver } from "./http-driver.js";
 import { JudgeConfig, JudgeScore, configureJudge, isJudgeConfigured } from "./judge.js";
 import { equals, includes, matches } from "./matchers.js";
 import { Assessment, ReportOutcome, buildAssessments, reportToMlflow } from "./mlflow-report.js";
-import { EvalSummary, evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, summarize } from "./report.js";
+import { EvalSummary, evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, summarize } from "./report.js";
 import { RunEvalOptions, runEval } from "./run-eval.js";
-import { EvalProgress, EvalRunSummary, RunEvalsOptions, runEvalsInDir } from "./run-evals.js";
+import { EvalProgress, EvalRunSummary, RunEvalsOptions, runEvalsInDir, runWithRetries } from "./run-evals.js";

package/dist/evals/index.js

@@ -1,14 +1,14 @@
 import { resolveDatabricksAuth, resolveWorkspaceClient } from "../connectors/mlflow/auth.js";
 import { MlflowClient, normalizeHost } from "../connectors/mlflow/client.js";
-import { readEvalDataset } from "./dataset.js";
-import { defineEval } from "./define-eval.js";
-import { discoverEvalFiles } from "./discover.js";
+import { readEvalDataset, userTurns } from "./dataset.js";
+import { defineEval, defineEvalConfig } from "./define-eval.js";
+import { discoverEvalConfigs, discoverEvalFiles } from "./discover.js";
 import { createHttpDriver } from "./http-driver.js";
 import { configureJudge, isJudgeConfigured } from "./judge.js";
 import { equals, includes, matches } from "./matchers.js";
 import { buildAssessments, reportToMlflow } from "./mlflow-report.js";
-import { evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, summarize } from "./report.js";
+import { evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, summarize } from "./report.js";
 import { runEval } from "./run-eval.js";
-import { runEvalsInDir } from "./run-evals.js";
+import { runEvalsInDir, runWithRetries } from "./run-evals.js";
 
 export {  };

package/dist/evals/report.d.ts

@@ -8,6 +8,8 @@ interface EvalSummary {
   skipped: number;
   /** True when no eval failed (skips don't count as failures). */
   allPassed: boolean;
+  /** Fraction of scored (non-skipped) evals that passed, 0..1 (1 when none scored). */
+  passRate: number;
 }
 declare function summarize(results: EvalResult[]): EvalSummary;
 /** Status glyph for a single eval result. */
@@ -20,6 +22,19 @@ declare function formatEvalDetail(result: EvalResult): string[];
 declare function formatSummaryLine(results: EvalResult[]): string;
 /** Render all results as a human-readable console report (non-streaming). */
 declare function formatEvalResults(results: EvalResult[]): string;
+/**
+ * Render results as a machine-readable JSON report (2-space indented):
+ * `{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types —
+ * every field present on a result round-trips.
+ */
+declare function formatResultsJson(results: EvalResult[]): string;
+/**
+ * Render results as JUnit XML for standard CI test reporters: a single
+ * `<testsuite name="appkit-agent-evals">` with one `<testcase>` per result.
+ * Failures carry a `<failure>` (error or failing-gate summary); skips a
+ * `<skipped>`. All attribute/text values are XML-escaped.
+ */
+declare function formatResultsJUnit(results: EvalResult[]): string;
 //#endregion
-export { EvalSummary, evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, summarize };
+export { EvalSummary, evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, summarize };
 //# sourceMappingURL=report.d.ts.map

package/dist/evals/report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report.d.ts","names":[],"sources":["../../src/evals/report.ts"],"mappings":";;;UAEiB,WAAA;EACf,KAAA;EACA,MAAA;EACA,MAAA;EACA,OAAA;EAJ0B;EAM1B,SAAA;AAAA;AAAA,iBAGc,SAAA,CAAU,OAAA,EAAS,UAAA,KAAe,WAAA;;iBAmBlC,SAAA,CAAU,MAAA,EAAQ,UAAA;;iBAMlB,kBAAA,CAAmB,MAAA,EAAQ,UAAA;AAzB3C;AAAA,iBAqCgB,gBAAA,CAAiB,MAAA,EAAQ,UAAA;;iBAYzB,iBAAA,CAAkB,OAAA,EAAS,UAAA;;iBAM3B,iBAAA,CAAkB,OAAA,EAAS,UAAA"}
+{"version":3,"file":"report.d.ts","names":[],"sources":["../../src/evals/report.ts"],"mappings":";;;UAEiB,WAAA;EACf,KAAA;EACA,MAAA;EACA,MAAA;EACA,OAAA;EAJ0B;EAM1B,SAAA;EAJA;EAMA,QAAA;AAAA;AAAA,iBAGc,SAAA,CAAU,OAAA,EAAS,UAAA,KAAe,WAAA;;iBAqBlC,SAAA,CAAU,MAAA,EAAQ,UAAA;;iBAMlB,kBAAA,CAAmB,MAAA,EAAQ,UAAA;;iBAY3B,gBAAA,CAAiB,MAAA,EAAQ,UAAA;;iBAYzB,iBAAA,CAAkB,OAAA,EAAS,UAAA;;iBAM3B,iBAAA,CAAkB,OAAA,EAAS,UAAA;;;AApC3C;;;iBAoDgB,iBAAA,CAAkB,OAAA,EAAS,UAAA;;AA9C3C;;;;;iBA2FgB,kBAAA,CAAmB,OAAA,EAAS,UAAA"}

package/dist/evals/report.js

@@ -6,12 +6,14 @@ function summarize(results) {
 	for (const r of results) if (r.skipped) skipped++;
 	else if (r.passed) passed++;
 	else failed++;
+	const scored = passed + failed;
 	return {
 		total: results.length,
 		passed,
 		failed,
 		skipped,
-		allPassed: failed === 0
+		allPassed: failed === 0,
+		passRate: scored === 0 ? 1 : passed / scored
 	};
 }
 /** Status glyph for a single eval result. */
@@ -51,7 +53,67 @@ function formatEvalResults(results) {
 	lines.push(formatSummaryLine(results));
 	return lines.join("\n");
 }
+/**
+* Render results as a machine-readable JSON report (2-space indented):
+* `{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types —
+* every field present on a result round-trips.
+*/
+function formatResultsJson(results) {
+	return JSON.stringify({
+		summary: summarize(results),
+		results
+	}, null, 2);
+}
+/**
+* Drop the C0 control chars XML 1.0 forbids even when escaped (except tab, LF,
+* CR) — a raw NUL or ANSI escape would otherwise make the JUnit doc reject on parse.
+*/
+function stripXmlControlChars(value) {
+	let out = "";
+	for (const ch of value) {
+		const code = ch.codePointAt(0);
+		if (code >= 32 || code === 9 || code === 10 || code === 13) out += ch;
+	}
+	return out;
+}
+/** Escape a value for use in XML text/attribute content (control chars dropped). */
+function escapeXml(value) {
+	return stripXmlControlChars(value).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+}
+/** One-line reason a result failed: its error, else its failing gate labels. */
+function failureMessage(result) {
+	if (result.error) return result.error;
+	const gates = result.assertions.filter((a) => !a.pass && a.severity === "gate").map((a) => a.detail ? `${a.label} — ${a.detail}` : a.label);
+	return gates.length ? gates.join("; ") : "eval failed";
+}
+/**
+* Render results as JUnit XML for standard CI test reporters: a single
+* `<testsuite name="appkit-agent-evals">` with one `<testcase>` per result.
+* Failures carry a `<failure>` (error or failing-gate summary); skips a
+* `<skipped>`. All attribute/text values are XML-escaped.
+*/
+function formatResultsJUnit(results) {
+	const s = summarize(results);
+	const lines = [];
+	lines.push("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+	lines.push(`<testsuite name="appkit-agent-evals" tests="${s.total}" failures="${s.failed}" skipped="${s.skipped}">`);
+	for (const r of results) {
+		const open = `  <testcase name="${escapeXml(r.id)}" classname="agent-eval"`;
+		if (r.skipped) {
+			lines.push(`${open}>`);
+			lines.push(r.skipped.reason ? `    <skipped message="${escapeXml(r.skipped.reason)}"/>` : "    <skipped/>");
+			lines.push("  </testcase>");
+		} else if (!r.passed) {
+			const message = failureMessage(r);
+			lines.push(`${open}>`);
+			lines.push(`    <failure message="${escapeXml(message)}"/>`);
+			lines.push("  </testcase>");
+		} else lines.push(`${open}/>`);
+	}
+	lines.push("</testsuite>");
+	return lines.join("\n");
+}
 
 //#endregion
-export { evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, summarize };
+export { evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, summarize };
 //# sourceMappingURL=report.js.map

package/dist/evals/report.js.map

@@ -1 +1 @@
-{"version":3,"file":"report.js","names":[],"sources":["../../src/evals/report.ts"],"sourcesContent":["import type { EvalResult } from \"./types\";\n\nexport interface EvalSummary {\n  total: number;\n  passed: number;\n  failed: number;\n  skipped: number;\n  /** True when no eval failed (skips don't count as failures). */\n  allPassed: boolean;\n}\n\nexport function summarize(results: EvalResult[]): EvalSummary {\n  let passed = 0;\n  let failed = 0;\n  let skipped = 0;\n  for (const r of results) {\n    if (r.skipped) skipped++;\n    else if (r.passed) passed++;\n    else failed++;\n  }\n  return {\n    total: results.length,\n    passed,\n    failed,\n    skipped,\n    allPassed: failed === 0,\n  };\n}\n\n/** Status glyph for a single eval result. */\nexport function evalGlyph(result: EvalResult): string {\n  if (result.skipped) return \"−\";\n  return result.passed ? \"✓\" : \"✗\";\n}\n\n/** The one-line header for a single eval result (no failure detail). */\nexport function formatEvalHeadline(result: EvalResult): string {\n  if (result.skipped) {\n    return `− ${result.id} (skipped${\n      result.skipped.reason ? `: ${result.skipped.reason}` : \"\"\n    })`;\n  }\n  return `${evalGlyph(result)} ${result.id}${\n    result.description ? ` — ${result.description}` : \"\"\n  }`;\n}\n\n/** Indented detail lines for a failing eval (error + failing assertions). */\nexport function formatEvalDetail(result: EvalResult): string[] {\n  const lines: string[] = [];\n  if (result.error) lines.push(`    error: ${result.error}`);\n  for (const a of result.assertions) {\n    if (a.pass) continue;\n    const tag = a.severity === \"soft\" ? \"soft\" : \"gate\";\n    lines.push(`    ✗ [${tag}] ${a.label}${a.detail ? ` — ${a.detail}` : \"\"}`);\n  }\n  return lines;\n}\n\n/** The final PASS/FAIL summary line. */\nexport function formatSummaryLine(results: EvalResult[]): string {\n  const s = summarize(results);\n  return `${s.allPassed ? \"PASS\" : \"FAIL\"} — ${s.passed} passed, ${s.failed} failed, ${s.skipped} skipped (${s.total} total)`;\n}\n\n/** Render all results as a human-readable console report (non-streaming). */\nexport function formatEvalResults(results: EvalResult[]): string {\n  const lines: string[] = [];\n  for (const r of results) {\n    lines.push(formatEvalHeadline(r));\n    lines.push(...formatEvalDetail(r));\n  }\n  lines.push(\"\");\n  lines.push(formatSummaryLine(results));\n  return lines.join(\"\\n\");\n}\n"],"mappings":";AAWA,SAAgB,UAAU,SAAoC;CAC5D,IAAI,SAAS;CACb,IAAI,SAAS;CACb,IAAI,UAAU;AACd,MAAK,MAAM,KAAK,QACd,KAAI,EAAE,QAAS;UACN,EAAE,OAAQ;KACd;AAEP,QAAO;EACL,OAAO,QAAQ;EACf;EACA;EACA;EACA,WAAW,WAAW;EACvB;;;AAIH,SAAgB,UAAU,QAA4B;AACpD,KAAI,OAAO,QAAS,QAAO;AAC3B,QAAO,OAAO,SAAS,MAAM;;;AAI/B,SAAgB,mBAAmB,QAA4B;AAC7D,KAAI,OAAO,QACT,QAAO,KAAK,OAAO,GAAG,WACpB,OAAO,QAAQ,SAAS,KAAK,OAAO,QAAQ,WAAW,GACxD;AAEH,QAAO,GAAG,UAAU,OAAO,CAAC,GAAG,OAAO,KACpC,OAAO,cAAc,MAAM,OAAO,gBAAgB;;;AAKtD,SAAgB,iBAAiB,QAA8B;CAC7D,MAAM,QAAkB,EAAE;AAC1B,KAAI,OAAO,MAAO,OAAM,KAAK,cAAc,OAAO,QAAQ;AAC1D,MAAK,MAAM,KAAK,OAAO,YAAY;AACjC,MAAI,EAAE,KAAM;EACZ,MAAM,MAAM,EAAE,aAAa,SAAS,SAAS;AAC7C,QAAM,KAAK,UAAU,IAAI,IAAI,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,WAAW,KAAK;;AAE5E,QAAO;;;AAIT,SAAgB,kBAAkB,SAA+B;CAC/D,MAAM,IAAI,UAAU,QAAQ;AAC5B,QAAO,GAAG,EAAE,YAAY,SAAS,OAAO,KAAK,EAAE,OAAO,WAAW,EAAE,OAAO,WAAW,EAAE,QAAQ,YAAY,EAAE,MAAM;;;AAIrH,SAAgB,kBAAkB,SAA+B;CAC/D,MAAM,QAAkB,EAAE;AAC1B,MAAK,MAAM,KAAK,SAAS;AACvB,QAAM,KAAK,mBAAmB,EAAE,CAAC;AACjC,QAAM,KAAK,GAAG,iBAAiB,EAAE,CAAC;;AAEpC,OAAM,KAAK,GAAG;AACd,OAAM,KAAK,kBAAkB,QAAQ,CAAC;AACtC,QAAO,MAAM,KAAK,KAAK"}
+{"version":3,"file":"report.js","names":[],"sources":["../../src/evals/report.ts"],"sourcesContent":["import type { EvalResult } from \"./types\";\n\nexport interface EvalSummary {\n  total: number;\n  passed: number;\n  failed: number;\n  skipped: number;\n  /** True when no eval failed (skips don't count as failures). */\n  allPassed: boolean;\n  /** Fraction of scored (non-skipped) evals that passed, 0..1 (1 when none scored). */\n  passRate: number;\n}\n\nexport function summarize(results: EvalResult[]): EvalSummary {\n  let passed = 0;\n  let failed = 0;\n  let skipped = 0;\n  for (const r of results) {\n    if (r.skipped) skipped++;\n    else if (r.passed) passed++;\n    else failed++;\n  }\n  const scored = passed + failed;\n  return {\n    total: results.length,\n    passed,\n    failed,\n    skipped,\n    allPassed: failed === 0,\n    passRate: scored === 0 ? 1 : passed / scored,\n  };\n}\n\n/** Status glyph for a single eval result. */\nexport function evalGlyph(result: EvalResult): string {\n  if (result.skipped) return \"−\";\n  return result.passed ? \"✓\" : \"✗\";\n}\n\n/** The one-line header for a single eval result (no failure detail). */\nexport function formatEvalHeadline(result: EvalResult): string {\n  if (result.skipped) {\n    return `− ${result.id} (skipped${\n      result.skipped.reason ? `: ${result.skipped.reason}` : \"\"\n    })`;\n  }\n  return `${evalGlyph(result)} ${result.id}${\n    result.description ? ` — ${result.description}` : \"\"\n  }`;\n}\n\n/** Indented detail lines for a failing eval (error + failing assertions). */\nexport function formatEvalDetail(result: EvalResult): string[] {\n  const lines: string[] = [];\n  if (result.error) lines.push(`    error: ${result.error}`);\n  for (const a of result.assertions) {\n    if (a.pass) continue;\n    const tag = a.severity === \"soft\" ? \"soft\" : \"gate\";\n    lines.push(`    ✗ [${tag}] ${a.label}${a.detail ? ` — ${a.detail}` : \"\"}`);\n  }\n  return lines;\n}\n\n/** The final PASS/FAIL summary line. */\nexport function formatSummaryLine(results: EvalResult[]): string {\n  const s = summarize(results);\n  return `${s.allPassed ? \"PASS\" : \"FAIL\"} — ${s.passed} passed, ${s.failed} failed, ${s.skipped} skipped (${s.total} total)`;\n}\n\n/** Render all results as a human-readable console report (non-streaming). */\nexport function formatEvalResults(results: EvalResult[]): string {\n  const lines: string[] = [];\n  for (const r of results) {\n    lines.push(formatEvalHeadline(r));\n    lines.push(...formatEvalDetail(r));\n  }\n  lines.push(\"\");\n  lines.push(formatSummaryLine(results));\n  return lines.join(\"\\n\");\n}\n\n/**\n * Render results as a machine-readable JSON report (2-space indented):\n * `{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types —\n * every field present on a result round-trips.\n */\nexport function formatResultsJson(results: EvalResult[]): string {\n  return JSON.stringify({ summary: summarize(results), results }, null, 2);\n}\n\n/**\n * Drop the C0 control chars XML 1.0 forbids even when escaped (except tab, LF,\n * CR) — a raw NUL or ANSI escape would otherwise make the JUnit doc reject on parse.\n */\nfunction stripXmlControlChars(value: string): string {\n  // Code-point filter, not a regex: oxlint `no-control-regex` rejects the class.\n  let out = \"\";\n  for (const ch of value) {\n    const code = ch.codePointAt(0) as number;\n    if (code >= 0x20 || code === 0x09 || code === 0x0a || code === 0x0d) {\n      out += ch;\n    }\n  }\n  return out;\n}\n\n/** Escape a value for use in XML text/attribute content (control chars dropped). */\nfunction escapeXml(value: string): string {\n  return stripXmlControlChars(value)\n    .replace(/&/g, \"&amp;\")\n    .replace(/</g, \"&lt;\")\n    .replace(/>/g, \"&gt;\")\n    .replace(/\"/g, \"&quot;\")\n    .replace(/'/g, \"&apos;\");\n}\n\n/** One-line reason a result failed: its error, else its failing gate labels. */\nfunction failureMessage(result: EvalResult): string {\n  if (result.error) return result.error;\n  const gates = result.assertions\n    .filter((a) => !a.pass && a.severity === \"gate\")\n    .map((a) => (a.detail ? `${a.label} — ${a.detail}` : a.label));\n  return gates.length ? gates.join(\"; \") : \"eval failed\";\n}\n\n/**\n * Render results as JUnit XML for standard CI test reporters: a single\n * `<testsuite name=\"appkit-agent-evals\">` with one `<testcase>` per result.\n * Failures carry a `<failure>` (error or failing-gate summary); skips a\n * `<skipped>`. All attribute/text values are XML-escaped.\n */\nexport function formatResultsJUnit(results: EvalResult[]): string {\n  const s = summarize(results);\n  const lines: string[] = [];\n  lines.push('<?xml version=\"1.0\" encoding=\"UTF-8\"?>');\n  lines.push(\n    `<testsuite name=\"appkit-agent-evals\" tests=\"${s.total}\" failures=\"${s.failed}\" skipped=\"${s.skipped}\">`,\n  );\n  for (const r of results) {\n    const open = `  <testcase name=\"${escapeXml(r.id)}\" classname=\"agent-eval\"`;\n    if (r.skipped) {\n      lines.push(`${open}>`);\n      lines.push(\n        r.skipped.reason\n          ? `    <skipped message=\"${escapeXml(r.skipped.reason)}\"/>`\n          : \"    <skipped/>\",\n      );\n      lines.push(\"  </testcase>\");\n    } else if (!r.passed) {\n      const message = failureMessage(r);\n      lines.push(`${open}>`);\n      lines.push(`    <failure message=\"${escapeXml(message)}\"/>`);\n      lines.push(\"  </testcase>\");\n    } else {\n      lines.push(`${open}/>`);\n    }\n  }\n  lines.push(\"</testsuite>\");\n  return lines.join(\"\\n\");\n}\n"],"mappings":";AAaA,SAAgB,UAAU,SAAoC;CAC5D,IAAI,SAAS;CACb,IAAI,SAAS;CACb,IAAI,UAAU;AACd,MAAK,MAAM,KAAK,QACd,KAAI,EAAE,QAAS;UACN,EAAE,OAAQ;KACd;CAEP,MAAM,SAAS,SAAS;AACxB,QAAO;EACL,OAAO,QAAQ;EACf;EACA;EACA;EACA,WAAW,WAAW;EACtB,UAAU,WAAW,IAAI,IAAI,SAAS;EACvC;;;AAIH,SAAgB,UAAU,QAA4B;AACpD,KAAI,OAAO,QAAS,QAAO;AAC3B,QAAO,OAAO,SAAS,MAAM;;;AAI/B,SAAgB,mBAAmB,QAA4B;AAC7D,KAAI,OAAO,QACT,QAAO,KAAK,OAAO,GAAG,WACpB,OAAO,QAAQ,SAAS,KAAK,OAAO,QAAQ,WAAW,GACxD;AAEH,QAAO,GAAG,UAAU,OAAO,CAAC,GAAG,OAAO,KACpC,OAAO,cAAc,MAAM,OAAO,gBAAgB;;;AAKtD,SAAgB,iBAAiB,QAA8B;CAC7D,MAAM,QAAkB,EAAE;AAC1B,KAAI,OAAO,MAAO,OAAM,KAAK,cAAc,OAAO,QAAQ;AAC1D,MAAK,MAAM,KAAK,OAAO,YAAY;AACjC,MAAI,EAAE,KAAM;EACZ,MAAM,MAAM,EAAE,aAAa,SAAS,SAAS;AAC7C,QAAM,KAAK,UAAU,IAAI,IAAI,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,WAAW,KAAK;;AAE5E,QAAO;;;AAIT,SAAgB,kBAAkB,SAA+B;CAC/D,MAAM,IAAI,UAAU,QAAQ;AAC5B,QAAO,GAAG,EAAE,YAAY,SAAS,OAAO,KAAK,EAAE,OAAO,WAAW,EAAE,OAAO,WAAW,EAAE,QAAQ,YAAY,EAAE,MAAM;;;AAIrH,SAAgB,kBAAkB,SAA+B;CAC/D,MAAM,QAAkB,EAAE;AAC1B,MAAK,MAAM,KAAK,SAAS;AACvB,QAAM,KAAK,mBAAmB,EAAE,CAAC;AACjC,QAAM,KAAK,GAAG,iBAAiB,EAAE,CAAC;;AAEpC,OAAM,KAAK,GAAG;AACd,OAAM,KAAK,kBAAkB,QAAQ,CAAC;AACtC,QAAO,MAAM,KAAK,KAAK;;;;;;;AAQzB,SAAgB,kBAAkB,SAA+B;AAC/D,QAAO,KAAK,UAAU;EAAE,SAAS,UAAU,QAAQ;EAAE;EAAS,EAAE,MAAM,EAAE;;;;;;AAO1E,SAAS,qBAAqB,OAAuB;CAEnD,IAAI,MAAM;AACV,MAAK,MAAM,MAAM,OAAO;EACtB,MAAM,OAAO,GAAG,YAAY,EAAE;AAC9B,MAAI,QAAQ,MAAQ,SAAS,KAAQ,SAAS,MAAQ,SAAS,GAC7D,QAAO;;AAGX,QAAO;;;AAIT,SAAS,UAAU,OAAuB;AACxC,QAAO,qBAAqB,MAAM,CAC/B,QAAQ,MAAM,QAAQ,CACtB,QAAQ,MAAM,OAAO,CACrB,QAAQ,MAAM,OAAO,CACrB,QAAQ,MAAM,SAAS,CACvB,QAAQ,MAAM,SAAS;;;AAI5B,SAAS,eAAe,QAA4B;AAClD,KAAI,OAAO,MAAO,QAAO,OAAO;CAChC,MAAM,QAAQ,OAAO,WAClB,QAAQ,MAAM,CAAC,EAAE,QAAQ,EAAE,aAAa,OAAO,CAC/C,KAAK,MAAO,EAAE,SAAS,GAAG,EAAE,MAAM,KAAK,EAAE,WAAW,EAAE,MAAO;AAChE,QAAO,MAAM,SAAS,MAAM,KAAK,KAAK,GAAG;;;;;;;;AAS3C,SAAgB,mBAAmB,SAA+B;CAChE,MAAM,IAAI,UAAU,QAAQ;CAC5B,MAAM,QAAkB,EAAE;AAC1B,OAAM,KAAK,6CAAyC;AACpD,OAAM,KACJ,+CAA+C,EAAE,MAAM,cAAc,EAAE,OAAO,aAAa,EAAE,QAAQ,IACtG;AACD,MAAK,MAAM,KAAK,SAAS;EACvB,MAAM,OAAO,qBAAqB,UAAU,EAAE,GAAG,CAAC;AAClD,MAAI,EAAE,SAAS;AACb,SAAM,KAAK,GAAG,KAAK,GAAG;AACtB,SAAM,KACJ,EAAE,QAAQ,SACN,yBAAyB,UAAU,EAAE,QAAQ,OAAO,CAAC,OACrD,iBACL;AACD,SAAM,KAAK,gBAAgB;aAClB,CAAC,EAAE,QAAQ;GACpB,MAAM,UAAU,eAAe,EAAE;AACjC,SAAM,KAAK,GAAG,KAAK,GAAG;AACtB,SAAM,KAAK,yBAAyB,UAAU,QAAQ,CAAC,KAAK;AAC5D,SAAM,KAAK,gBAAgB;QAE3B,OAAM,KAAK,GAAG,KAAK,IAAI;;AAG3B,OAAM,KAAK,eAAe;AAC1B,QAAO,MAAM,KAAK,KAAK"}

package/dist/evals/run-eval.d.ts

@@ -11,6 +11,11 @@ interface RunEvalOptions {
   strict?: boolean;
   /** Dataset row bound to `t.input`/`t.expected` for dataset-driven evals. */
   row?: DatasetRow;
+  /**
+   * Runner-level default per-eval timeout (ms). `def.timeoutMs` wins over this;
+   * when both are unset the eval runs unbounded (current behavior).
+   */
+  timeoutMs?: number;
 }
 /**
  * Run a single eval against a driver. Never throws for assertion or agent

package/dist/evals/run-eval.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-eval.d.ts","names":[],"sources":["../../src/evals/run-eval.ts"],"mappings":";;;;UAuBiB,cAAA;;EAEf,EAAA;EAF6B;EAI7B,MAAA,EAAQ,UAAA;EAIQ;EAFhB,MAAA;EAFA;EAIA,GAAA,GAAM,UAAA;AAAA;;;;;AAQR;iBAAsB,OAAA,CACpB,GAAA,EAAK,cAAA,EACL,OAAA,EAAS,cAAA,GACR,OAAA,CAAQ,UAAA"}
+{"version":3,"file":"run-eval.d.ts","names":[],"sources":["../../src/evals/run-eval.ts"],"mappings":";;;;UAoDiB,cAAA;;EAEf,EAAA;EAF6B;EAI7B,MAAA,EAAQ,UAAA;EAIQ;EAFhB,MAAA;EAFA;EAIA,GAAA,GAAM,UAAA;EAFN;;;;EAOA,SAAA;AAAA;AAQF;;;;;AAAA,iBAAsB,OAAA,CACpB,GAAA,EAAK,cAAA,EACL,OAAA,EAAS,cAAA,GACR,OAAA,CAAQ,UAAA"}

package/dist/evals/run-eval.js

@@ -12,6 +12,22 @@ var SkipSignal = class extends Error {
 	}
 };
 /**
+* Deep partial match: every key in `expected` is present in `actual` and equal,
+* recursing into nested plain objects so extra actual keys are ignored. Arrays
+* match element-for-element (same length, deep-equal items).
+*/
+function deepContains(actual, expected) {
+	if (Array.isArray(expected)) return Array.isArray(actual) && actual.length === expected.length && expected.every((item, i) => deepContains(actual[i], item));
+	if (isPlainObject(expected)) {
+		if (!isPlainObject(actual)) return false;
+		return Object.keys(expected).every((key) => Object.hasOwn(actual, key) && deepContains(actual[key], expected[key]));
+	}
+	return actual === expected;
+}
+function isPlainObject(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+/**
 * Run a single eval against a driver. Never throws for assertion or agent
 * failures — those become a non-passing {@link EvalResult}. Only a malformed
 * eval definition surfaces as `result.error`.
@@ -21,9 +37,12 @@ async function runEval(def, options) {
 	let reply = "";
 	let lastInput = "";
 	let toolCalls = [];
+	let toolCallDetails = [];
 	let sessionId;
 	let lastTraceId;
 	let lastSucceeded = false;
+	let turnFailed = false;
+	const controller = new AbortController();
 	const record = (label, pass, score, detail) => {
 		const result = {
 			label,
@@ -53,11 +72,13 @@ async function runEval(def, options) {
 	const t = {
 		async send(message) {
 			lastInput = message;
-			const r = await options.driver.send(message);
+			const r = await options.driver.send(message, { signal: controller.signal });
 			reply = r.reply;
 			toolCalls = r.toolCalls;
+			toolCallDetails = r.toolCallDetails;
 			sessionId = r.sessionId;
 			lastSucceeded = r.succeeded;
+			if (!r.succeeded) turnFailed = true;
 			if (r.traceId) lastTraceId = r.traceId;
 		},
 		reset() {
@@ -84,6 +105,12 @@ async function runEval(def, options) {
 		calledTool(name) {
 			return record(`calledTool(${name})`, toolCalls.includes(name), void 0, `expected tool "${name}" to be called (called: ${toolCalls.length ? toolCalls.join(", ") : "none"})`);
 		},
+		calledToolWith(name, expected) {
+			const matching = toolCallDetails.filter((c) => c.name === name);
+			const pass = matching.some((c) => deepContains(c.args, expected));
+			const seen = matching.length ? matching.map((c) => `{${Object.keys(c.args).sort().join(", ")}}`).join(", ") : "not called";
+			return record(`calledToolWith(${name})`, pass, void 0, `expected tool "${name}" to be called with ${JSON.stringify(expected)} (arg keys seen: ${seen})`);
+		},
 		check(value, matcher) {
 			const m = matcher(value);
 			return record("check", m.pass, m.score, m.detail);
@@ -117,8 +144,19 @@ async function runEval(def, options) {
 			throw new SkipSignal(reason);
 		}
 	};
+	const timeoutMs = def.timeoutMs ?? options.timeoutMs;
+	let timer;
 	try {
-		await def.test(t);
+		if (timeoutMs === void 0) await def.test(t);
+		else {
+			const timeout = new Promise((_, reject) => {
+				timer = setTimeout(() => {
+					controller.abort();
+					reject(/* @__PURE__ */ new Error(`eval timed out after ${timeoutMs}ms`));
+				}, timeoutMs);
+			});
+			await Promise.race([Promise.resolve(def.test(t)), timeout]);
+		}
 	} catch (err) {
 		if (err instanceof SkipSignal) return {
 			id: options.id,
@@ -136,6 +174,8 @@ async function runEval(def, options) {
 			error: err instanceof Error ? err.message : String(err),
 			traceId: lastTraceId
 		};
+	} finally {
+		if (timer) clearTimeout(timer);
 	}
 	const passed = assertions.every((a) => a.pass || a.severity === "soft" && !options.strict);
 	return {
@@ -143,7 +183,8 @@ async function runEval(def, options) {
 		description: def.description,
 		assertions,
 		passed,
-		traceId: lastTraceId
+		traceId: lastTraceId,
+		infraFailure: turnFailed && !passed || void 0
 	};
 }
 

package/dist/evals/run-eval.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-eval.js","names":[],"sources":["../../src/evals/run-eval.ts"],"sourcesContent":["import type { DatasetRow } from \"./dataset\";\nimport { judgeClosedQA, judgeCustom, judgeFactuality } from \"./judge\";\nimport type {\n  AssertionHandle,\n  AssertionResult,\n  EvalDefinition,\n  EvalDriver,\n  EvalResult,\n  Matcher,\n  TestContext,\n} from \"./types\";\n\n/** Default pass threshold for an LLM-judge score (0..1) before `.atLeast()`. */\nconst DEFAULT_JUDGE_THRESHOLD = 0.5;\n\n/** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */\nclass SkipSignal extends Error {\n  constructor(public reason?: string) {\n    super(\"eval skipped\");\n    this.name = \"SkipSignal\";\n  }\n}\n\nexport interface RunEvalOptions {\n  /** Stable id for the eval (e.g. its file path relative to the evals dir). */\n  id: string;\n  /** Drives the agent and returns reply/tool-calls/success per `send`. */\n  driver: EvalDriver;\n  /** When true, soft assertion failures also fail the eval. */\n  strict?: boolean;\n  /** Dataset row bound to `t.input`/`t.expected` for dataset-driven evals. */\n  row?: DatasetRow;\n}\n\n/**\n * Run a single eval against a driver. Never throws for assertion or agent\n * failures — those become a non-passing {@link EvalResult}. Only a malformed\n * eval definition surfaces as `result.error`.\n */\nexport async function runEval(\n  def: EvalDefinition,\n  options: RunEvalOptions,\n): Promise<EvalResult> {\n  const assertions: AssertionResult[] = [];\n  let reply = \"\";\n  let lastInput = \"\";\n  let toolCalls: string[] = [];\n  let sessionId: string | undefined;\n  let lastTraceId: string | undefined;\n  let lastSucceeded = false;\n\n  const record = (\n    label: string,\n    pass: boolean,\n    score?: number,\n    detail?: string,\n  ): AssertionHandle => {\n    const result: AssertionResult = {\n      label,\n      severity: \"gate\",\n      pass,\n      score,\n      detail,\n    };\n    assertions.push(result);\n    const handle: AssertionHandle = {\n      gate() {\n        result.severity = \"gate\";\n        return handle;\n      },\n      soft() {\n        result.severity = \"soft\";\n        return handle;\n      },\n      atLeast(threshold: number) {\n        // Re-threshold the score; keep the current severity (gate unless the\n        // caller also chained `.soft()`).\n        result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold;\n        return handle;\n      },\n    };\n    return handle;\n  };\n\n  // LLM-judge assertions are scored and gate by default (a miss fails the eval,\n  // like other assertions). The caller chains `.atLeast(n)` to change the pass\n  // threshold, or `.soft()` to demote to a tracked-only metric.\n  const recordJudge = (\n    label: string,\n    score: number,\n    rationale?: string,\n  ): AssertionHandle =>\n    record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale);\n\n  const t: TestContext = {\n    async send(message) {\n      lastInput = message;\n      const r = await options.driver.send(message);\n      reply = r.reply;\n      toolCalls = r.toolCalls;\n      sessionId = r.sessionId;\n      lastSucceeded = r.succeeded;\n      if (r.traceId) lastTraceId = r.traceId;\n    },\n    reset() {\n      options.driver.reset?.();\n    },\n    get reply() {\n      return reply;\n    },\n    get toolCalls() {\n      return toolCalls;\n    },\n    get sessionId() {\n      return sessionId;\n    },\n    get input() {\n      return options.row?.inputs ?? {};\n    },\n    get expected() {\n      return options.row?.expectations;\n    },\n    succeeded() {\n      return record(\n        \"succeeded\",\n        lastSucceeded,\n        undefined,\n        lastSucceeded ? undefined : \"agent turn did not complete successfully\",\n      );\n    },\n    calledTool(name) {\n      return record(\n        `calledTool(${name})`,\n        toolCalls.includes(name),\n        undefined,\n        `expected tool \"${name}\" to be called (called: ${\n          toolCalls.length ? toolCalls.join(\", \") : \"none\"\n        })`,\n      );\n    },\n    check(value: string, matcher: Matcher) {\n      const m = matcher(value);\n      return record(\"check\", m.pass, m.score, m.detail);\n    },\n    judge: {\n      async factuality(expected) {\n        const { score, rationale } = await judgeFactuality({\n          input: lastInput,\n          output: reply,\n          expected,\n        });\n        return recordJudge(\"judge.factuality\", score, rationale);\n      },\n      async closedQA(criteria) {\n        const { score, rationale } = await judgeClosedQA({\n          input: lastInput,\n          output: reply,\n          criteria,\n        });\n        return recordJudge(\"judge.closedQA\", score, rationale);\n      },\n      async custom(spec) {\n        const { score, rationale } = await judgeCustom(spec, {\n          input: lastInput,\n          output: reply,\n        });\n        return recordJudge(`judge.${spec.name}`, score, rationale);\n      },\n    },\n    skip(reason) {\n      throw new SkipSignal(reason);\n    },\n  };\n\n  try {\n    await def.test(t);\n  } catch (err) {\n    if (err instanceof SkipSignal) {\n      return {\n        id: options.id,\n        description: def.description,\n        skipped: { reason: err.reason },\n        assertions,\n        passed: true,\n        traceId: lastTraceId,\n      };\n    }\n    return {\n      id: options.id,\n      description: def.description,\n      assertions,\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n      traceId: lastTraceId,\n    };\n  }\n\n  const passed = assertions.every(\n    (a) => a.pass || (a.severity === \"soft\" && !options.strict),\n  );\n\n  return {\n    id: options.id,\n    description: def.description,\n    assertions,\n    passed,\n    traceId: lastTraceId,\n  };\n}\n"],"mappings":";;;;AAaA,MAAM,0BAA0B;;AAGhC,IAAM,aAAN,cAAyB,MAAM;CAC7B,YAAY,AAAO,QAAiB;AAClC,QAAM,eAAe;EADJ;AAEjB,OAAK,OAAO;;;;;;;;AAoBhB,eAAsB,QACpB,KACA,SACqB;CACrB,MAAM,aAAgC,EAAE;CACxC,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,YAAsB,EAAE;CAC5B,IAAI;CACJ,IAAI;CACJ,IAAI,gBAAgB;CAEpB,MAAM,UACJ,OACA,MACA,OACA,WACoB;EACpB,MAAM,SAA0B;GAC9B;GACA,UAAU;GACV;GACA;GACA;GACD;AACD,aAAW,KAAK,OAAO;EACvB,MAAM,SAA0B;GAC9B,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,QAAQ,WAAmB;AAGzB,WAAO,QAAQ,OAAO,UAAU,OAAO,OAAO,IAAI,OAAO;AACzD,WAAO;;GAEV;AACD,SAAO;;CAMT,MAAM,eACJ,OACA,OACA,cAEA,OAAO,OAAO,SAAS,yBAAyB,OAAO,UAAU;CAEnE,MAAM,IAAiB;EACrB,MAAM,KAAK,SAAS;AAClB,eAAY;GACZ,MAAM,IAAI,MAAM,QAAQ,OAAO,KAAK,QAAQ;AAC5C,WAAQ,EAAE;AACV,eAAY,EAAE;AACd,eAAY,EAAE;AACd,mBAAgB,EAAE;AAClB,OAAI,EAAE,QAAS,eAAc,EAAE;;EAEjC,QAAQ;AACN,WAAQ,OAAO,SAAS;;EAE1B,IAAI,QAAQ;AACV,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,QAAQ;AACV,UAAO,QAAQ,KAAK,UAAU,EAAE;;EAElC,IAAI,WAAW;AACb,UAAO,QAAQ,KAAK;;EAEtB,YAAY;AACV,UAAO,OACL,aACA,eACA,QACA,gBAAgB,SAAY,2CAC7B;;EAEH,WAAW,MAAM;AACf,UAAO,OACL,cAAc,KAAK,IACnB,UAAU,SAAS,KAAK,EACxB,QACA,kBAAkB,KAAK,0BACrB,UAAU,SAAS,UAAU,KAAK,KAAK,GAAG,OAC3C,GACF;;EAEH,MAAM,OAAe,SAAkB;GACrC,MAAM,IAAI,QAAQ,MAAM;AACxB,UAAO,OAAO,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;;EAEnD,OAAO;GACL,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,OAAO,cAAc,MAAM,gBAAgB;KACjD,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,oBAAoB,OAAO,UAAU;;GAE1D,MAAM,SAAS,UAAU;IACvB,MAAM,EAAE,OAAO,cAAc,MAAM,cAAc;KAC/C,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,kBAAkB,OAAO,UAAU;;GAExD,MAAM,OAAO,MAAM;IACjB,MAAM,EAAE,OAAO,cAAc,MAAM,YAAY,MAAM;KACnD,OAAO;KACP,QAAQ;KACT,CAAC;AACF,WAAO,YAAY,SAAS,KAAK,QAAQ,OAAO,UAAU;;GAE7D;EACD,KAAK,QAAQ;AACX,SAAM,IAAI,WAAW,OAAO;;EAE/B;AAED,KAAI;AACF,QAAM,IAAI,KAAK,EAAE;UACV,KAAK;AACZ,MAAI,eAAe,WACjB,QAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB,SAAS,EAAE,QAAQ,IAAI,QAAQ;GAC/B;GACA,QAAQ;GACR,SAAS;GACV;AAEH,SAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB;GACA,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACvD,SAAS;GACV;;CAGH,MAAM,SAAS,WAAW,OACvB,MAAM,EAAE,QAAS,EAAE,aAAa,UAAU,CAAC,QAAQ,OACrD;AAED,QAAO;EACL,IAAI,QAAQ;EACZ,aAAa,IAAI;EACjB;EACA;EACA,SAAS;EACV"}
+{"version":3,"file":"run-eval.js","names":[],"sources":["../../src/evals/run-eval.ts"],"sourcesContent":["import type { DatasetRow } from \"./dataset\";\nimport { judgeClosedQA, judgeCustom, judgeFactuality } from \"./judge\";\nimport type {\n  AssertionHandle,\n  AssertionResult,\n  DriveResult,\n  EvalDefinition,\n  EvalDriver,\n  EvalResult,\n  Matcher,\n  TestContext,\n} from \"./types\";\n\n/** Default pass threshold for an LLM-judge score (0..1) before `.atLeast()`. */\nconst DEFAULT_JUDGE_THRESHOLD = 0.5;\n\n/** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */\nclass SkipSignal extends Error {\n  constructor(public reason?: string) {\n    super(\"eval skipped\");\n    this.name = \"SkipSignal\";\n  }\n}\n\n/**\n * Deep partial match: every key in `expected` is present in `actual` and equal,\n * recursing into nested plain objects so extra actual keys are ignored. Arrays\n * match element-for-element (same length, deep-equal items).\n */\nfunction deepContains(actual: unknown, expected: unknown): boolean {\n  if (Array.isArray(expected)) {\n    return (\n      Array.isArray(actual) &&\n      actual.length === expected.length &&\n      expected.every((item, i) => deepContains(actual[i], item))\n    );\n  }\n  if (isPlainObject(expected)) {\n    if (!isPlainObject(actual)) return false;\n    // Require the key present: an expected `undefined` must not match an omitted key.\n    return Object.keys(expected).every(\n      (key) =>\n        Object.hasOwn(actual, key) && deepContains(actual[key], expected[key]),\n    );\n  }\n  return actual === expected;\n}\n\nfunction isPlainObject(value: unknown): value is Record<string, unknown> {\n  return typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n\nexport interface RunEvalOptions {\n  /** Stable id for the eval (e.g. its file path relative to the evals dir). */\n  id: string;\n  /** Drives the agent and returns reply/tool-calls/success per `send`. */\n  driver: EvalDriver;\n  /** When true, soft assertion failures also fail the eval. */\n  strict?: boolean;\n  /** Dataset row bound to `t.input`/`t.expected` for dataset-driven evals. */\n  row?: DatasetRow;\n  /**\n   * Runner-level default per-eval timeout (ms). `def.timeoutMs` wins over this;\n   * when both are unset the eval runs unbounded (current behavior).\n   */\n  timeoutMs?: number;\n}\n\n/**\n * Run a single eval against a driver. Never throws for assertion or agent\n * failures — those become a non-passing {@link EvalResult}. Only a malformed\n * eval definition surfaces as `result.error`.\n */\nexport async function runEval(\n  def: EvalDefinition,\n  options: RunEvalOptions,\n): Promise<EvalResult> {\n  const assertions: AssertionResult[] = [];\n  let reply = \"\";\n  let lastInput = \"\";\n  let toolCalls: string[] = [];\n  let toolCallDetails: DriveResult[\"toolCallDetails\"] = [];\n  let sessionId: string | undefined;\n  let lastTraceId: string | undefined;\n  let lastSucceeded = false;\n  // Any transport/agent turn failure (driver `succeeded: false`) → `infraFailure`, for retry.\n  let turnFailed = false;\n  // Aborted on per-eval timeout to cancel the in-flight turn (no leaked stream).\n  const controller = new AbortController();\n\n  const record = (\n    label: string,\n    pass: boolean,\n    score?: number,\n    detail?: string,\n  ): AssertionHandle => {\n    const result: AssertionResult = {\n      label,\n      severity: \"gate\",\n      pass,\n      score,\n      detail,\n    };\n    assertions.push(result);\n    const handle: AssertionHandle = {\n      gate() {\n        result.severity = \"gate\";\n        return handle;\n      },\n      soft() {\n        result.severity = \"soft\";\n        return handle;\n      },\n      atLeast(threshold: number) {\n        // Re-threshold the score; keep the current severity (gate unless the\n        // caller also chained `.soft()`).\n        result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold;\n        return handle;\n      },\n    };\n    return handle;\n  };\n\n  // LLM-judge assertions are scored and gate by default (a miss fails the eval,\n  // like other assertions). The caller chains `.atLeast(n)` to change the pass\n  // threshold, or `.soft()` to demote to a tracked-only metric.\n  const recordJudge = (\n    label: string,\n    score: number,\n    rationale?: string,\n  ): AssertionHandle =>\n    record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale);\n\n  const t: TestContext = {\n    async send(message) {\n      lastInput = message;\n      const r = await options.driver.send(message, {\n        signal: controller.signal,\n      });\n      reply = r.reply;\n      toolCalls = r.toolCalls;\n      toolCallDetails = r.toolCallDetails;\n      sessionId = r.sessionId;\n      lastSucceeded = r.succeeded;\n      if (!r.succeeded) turnFailed = true;\n      if (r.traceId) lastTraceId = r.traceId;\n    },\n    reset() {\n      options.driver.reset?.();\n    },\n    get reply() {\n      return reply;\n    },\n    get toolCalls() {\n      return toolCalls;\n    },\n    get sessionId() {\n      return sessionId;\n    },\n    get input() {\n      return options.row?.inputs ?? {};\n    },\n    get expected() {\n      return options.row?.expectations;\n    },\n    succeeded() {\n      return record(\n        \"succeeded\",\n        lastSucceeded,\n        undefined,\n        lastSucceeded ? undefined : \"agent turn did not complete successfully\",\n      );\n    },\n    calledTool(name) {\n      return record(\n        `calledTool(${name})`,\n        toolCalls.includes(name),\n        undefined,\n        `expected tool \"${name}\" to be called (called: ${\n          toolCalls.length ? toolCalls.join(\", \") : \"none\"\n        })`,\n      );\n    },\n    calledToolWith(name, expected) {\n      const matching = toolCallDetails.filter((c) => c.name === name);\n      const pass = matching.some((c) => deepContains(c.args, expected));\n      // Keys only, never values: actual args may hold PII/secrets and are persisted to reports (CWE-532).\n      const seen = matching.length\n        ? matching\n            .map((c) => `{${Object.keys(c.args).sort().join(\", \")}}`)\n            .join(\", \")\n        : \"not called\";\n      return record(\n        `calledToolWith(${name})`,\n        pass,\n        undefined,\n        `expected tool \"${name}\" to be called with ${JSON.stringify(\n          expected,\n        )} (arg keys seen: ${seen})`,\n      );\n    },\n    check(value: string, matcher: Matcher) {\n      const m = matcher(value);\n      return record(\"check\", m.pass, m.score, m.detail);\n    },\n    judge: {\n      async factuality(expected) {\n        const { score, rationale } = await judgeFactuality({\n          input: lastInput,\n          output: reply,\n          expected,\n        });\n        return recordJudge(\"judge.factuality\", score, rationale);\n      },\n      async closedQA(criteria) {\n        const { score, rationale } = await judgeClosedQA({\n          input: lastInput,\n          output: reply,\n          criteria,\n        });\n        return recordJudge(\"judge.closedQA\", score, rationale);\n      },\n      async custom(spec) {\n        const { score, rationale } = await judgeCustom(spec, {\n          input: lastInput,\n          output: reply,\n        });\n        return recordJudge(`judge.${spec.name}`, score, rationale);\n      },\n    },\n    skip(reason) {\n      throw new SkipSignal(reason);\n    },\n  };\n\n  // `def.timeoutMs` (per-eval) wins over the runner default; when both are\n  // unset the eval runs unbounded (undefined = no timeout).\n  const timeoutMs = def.timeoutMs ?? options.timeoutMs;\n  let timer: ReturnType<typeof setTimeout> | undefined;\n\n  try {\n    if (timeoutMs === undefined) {\n      await def.test(t);\n    } else {\n      // Race the test against the timeout; on elapse, abort the turn and settle\n      // non-passing. Only the driver turn cancels — non-driver hangs (judge, sleep) run on.\n      const timeout = new Promise<never>((_, reject) => {\n        timer = setTimeout(() => {\n          controller.abort();\n          reject(new Error(`eval timed out after ${timeoutMs}ms`));\n        }, timeoutMs);\n      });\n      await Promise.race([Promise.resolve(def.test(t)), timeout]);\n    }\n  } catch (err) {\n    if (err instanceof SkipSignal) {\n      return {\n        id: options.id,\n        description: def.description,\n        skipped: { reason: err.reason },\n        assertions,\n        passed: true,\n        traceId: lastTraceId,\n      };\n    }\n    return {\n      id: options.id,\n      description: def.description,\n      assertions,\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n      traceId: lastTraceId,\n    };\n  } finally {\n    if (timer) clearTimeout(timer);\n  }\n\n  const passed = assertions.every(\n    (a) => a.pass || (a.severity === \"soft\" && !options.strict),\n  );\n\n  return {\n    id: options.id,\n    description: def.description,\n    assertions,\n    passed,\n    traceId: lastTraceId,\n    // Only a *failing* eval whose turn broke at the transport/agent level is a\n    // retryable infra flake; a passing eval (or a pure assertion mismatch, where\n    // the turn itself succeeded) is real signal and must not be retried.\n    infraFailure: (turnFailed && !passed) || undefined,\n  };\n}\n"],"mappings":";;;;AAcA,MAAM,0BAA0B;;AAGhC,IAAM,aAAN,cAAyB,MAAM;CAC7B,YAAY,AAAO,QAAiB;AAClC,QAAM,eAAe;EADJ;AAEjB,OAAK,OAAO;;;;;;;;AAShB,SAAS,aAAa,QAAiB,UAA4B;AACjE,KAAI,MAAM,QAAQ,SAAS,CACzB,QACE,MAAM,QAAQ,OAAO,IACrB,OAAO,WAAW,SAAS,UAC3B,SAAS,OAAO,MAAM,MAAM,aAAa,OAAO,IAAI,KAAK,CAAC;AAG9D,KAAI,cAAc,SAAS,EAAE;AAC3B,MAAI,CAAC,cAAc,OAAO,CAAE,QAAO;AAEnC,SAAO,OAAO,KAAK,SAAS,CAAC,OAC1B,QACC,OAAO,OAAO,QAAQ,IAAI,IAAI,aAAa,OAAO,MAAM,SAAS,KAAK,CACzE;;AAEH,QAAO,WAAW;;AAGpB,SAAS,cAAc,OAAkD;AACvE,QAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,MAAM;;;;;;;AAwB7E,eAAsB,QACpB,KACA,SACqB;CACrB,MAAM,aAAgC,EAAE;CACxC,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,YAAsB,EAAE;CAC5B,IAAI,kBAAkD,EAAE;CACxD,IAAI;CACJ,IAAI;CACJ,IAAI,gBAAgB;CAEpB,IAAI,aAAa;CAEjB,MAAM,aAAa,IAAI,iBAAiB;CAExC,MAAM,UACJ,OACA,MACA,OACA,WACoB;EACpB,MAAM,SAA0B;GAC9B;GACA,UAAU;GACV;GACA;GACA;GACD;AACD,aAAW,KAAK,OAAO;EACvB,MAAM,SAA0B;GAC9B,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,QAAQ,WAAmB;AAGzB,WAAO,QAAQ,OAAO,UAAU,OAAO,OAAO,IAAI,OAAO;AACzD,WAAO;;GAEV;AACD,SAAO;;CAMT,MAAM,eACJ,OACA,OACA,cAEA,OAAO,OAAO,SAAS,yBAAyB,OAAO,UAAU;CAEnE,MAAM,IAAiB;EACrB,MAAM,KAAK,SAAS;AAClB,eAAY;GACZ,MAAM,IAAI,MAAM,QAAQ,OAAO,KAAK,SAAS,EAC3C,QAAQ,WAAW,QACpB,CAAC;AACF,WAAQ,EAAE;AACV,eAAY,EAAE;AACd,qBAAkB,EAAE;AACpB,eAAY,EAAE;AACd,mBAAgB,EAAE;AAClB,OAAI,CAAC,EAAE,UAAW,cAAa;AAC/B,OAAI,EAAE,QAAS,eAAc,EAAE;;EAEjC,QAAQ;AACN,WAAQ,OAAO,SAAS;;EAE1B,IAAI,QAAQ;AACV,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,QAAQ;AACV,UAAO,QAAQ,KAAK,UAAU,EAAE;;EAElC,IAAI,WAAW;AACb,UAAO,QAAQ,KAAK;;EAEtB,YAAY;AACV,UAAO,OACL,aACA,eACA,QACA,gBAAgB,SAAY,2CAC7B;;EAEH,WAAW,MAAM;AACf,UAAO,OACL,cAAc,KAAK,IACnB,UAAU,SAAS,KAAK,EACxB,QACA,kBAAkB,KAAK,0BACrB,UAAU,SAAS,UAAU,KAAK,KAAK,GAAG,OAC3C,GACF;;EAEH,eAAe,MAAM,UAAU;GAC7B,MAAM,WAAW,gBAAgB,QAAQ,MAAM,EAAE,SAAS,KAAK;GAC/D,MAAM,OAAO,SAAS,MAAM,MAAM,aAAa,EAAE,MAAM,SAAS,CAAC;GAEjE,MAAM,OAAO,SAAS,SAClB,SACG,KAAK,MAAM,IAAI,OAAO,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,KAAK,CAAC,GAAG,CACxD,KAAK,KAAK,GACb;AACJ,UAAO,OACL,kBAAkB,KAAK,IACvB,MACA,QACA,kBAAkB,KAAK,sBAAsB,KAAK,UAChD,SACD,CAAC,mBAAmB,KAAK,GAC3B;;EAEH,MAAM,OAAe,SAAkB;GACrC,MAAM,IAAI,QAAQ,MAAM;AACxB,UAAO,OAAO,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;;EAEnD,OAAO;GACL,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,OAAO,cAAc,MAAM,gBAAgB;KACjD,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,oBAAoB,OAAO,UAAU;;GAE1D,MAAM,SAAS,UAAU;IACvB,MAAM,EAAE,OAAO,cAAc,MAAM,cAAc;KAC/C,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,kBAAkB,OAAO,UAAU;;GAExD,MAAM,OAAO,MAAM;IACjB,MAAM,EAAE,OAAO,cAAc,MAAM,YAAY,MAAM;KACnD,OAAO;KACP,QAAQ;KACT,CAAC;AACF,WAAO,YAAY,SAAS,KAAK,QAAQ,OAAO,UAAU;;GAE7D;EACD,KAAK,QAAQ;AACX,SAAM,IAAI,WAAW,OAAO;;EAE/B;CAID,MAAM,YAAY,IAAI,aAAa,QAAQ;CAC3C,IAAI;AAEJ,KAAI;AACF,MAAI,cAAc,OAChB,OAAM,IAAI,KAAK,EAAE;OACZ;GAGL,MAAM,UAAU,IAAI,SAAgB,GAAG,WAAW;AAChD,YAAQ,iBAAiB;AACvB,gBAAW,OAAO;AAClB,4BAAO,IAAI,MAAM,wBAAwB,UAAU,IAAI,CAAC;OACvD,UAAU;KACb;AACF,SAAM,QAAQ,KAAK,CAAC,QAAQ,QAAQ,IAAI,KAAK,EAAE,CAAC,EAAE,QAAQ,CAAC;;UAEtD,KAAK;AACZ,MAAI,eAAe,WACjB,QAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB,SAAS,EAAE,QAAQ,IAAI,QAAQ;GAC/B;GACA,QAAQ;GACR,SAAS;GACV;AAEH,SAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB;GACA,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACvD,SAAS;GACV;WACO;AACR,MAAI,MAAO,cAAa,MAAM;;CAGhC,MAAM,SAAS,WAAW,OACvB,MAAM,EAAE,QAAS,EAAE,aAAa,UAAU,CAAC,QAAQ,OACrD;AAED,QAAO;EACL,IAAI,QAAQ;EACZ,aAAa,IAAI;EACjB;EACA;EACA,SAAS;EAIT,cAAe,cAAc,CAAC,UAAW;EAC1C"}

package/dist/evals/run-evals.d.ts

@@ -12,12 +12,15 @@ interface RunEvalsOptions {
   baseUrl: string;
   /** Substring filter on `<agent>/<id>` (or an exact agent id). */
   filter?: string;
+  /**
+   * Only run evals whose `tags` intersect this list. Empty/undefined runs all.
+   * Tags live on the eval def, so filtering happens after each file is loaded.
+   */
+  tags?: string[];
   /** Soft assertion failures also fail the eval. */
   strict?: boolean;
   /** Extra request headers for the driver (e.g. auth for a deployed app). */
   headers?: Record<string, string>;
-  /** Per-turn wall-clock timeout (ms) before a turn is failed. Defaults to 120s. */
-  timeoutMs?: number;
   /**
    * Max evals to drive concurrently. Each eval opens one stream to the app as
    * the same user, so keep this at or below the app's
@@ -54,6 +57,18 @@ interface RunEvalsOptions {
   warehouseId?: string;
   /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */
   now?: number;
+  /**
+   * Default per-eval timeout (ms): `runEval` races the whole test against it and
+   * it also caps each driver turn. A per-eval `def.timeoutMs` overrides it, and
+   * it wins over an agent's `evals.config.ts` `timeoutMs`. Unbounded when unset.
+   */
+  timeoutMs?: number;
+  /**
+   * Re-run an eval up to this many extra times when it fails on infrastructure —
+   * a thrown error/timeout (`result.error`) or a transport/agent turn failure
+   * (`result.infraFailure`). Assertion failures are never retried. Defaults to `0`.
+   */
+  retries?: number;
   /** Progress callback, invoked as evals are discovered, started, and finished. */
   onEvent?: (event: EvalProgress) => void;
 }
@@ -83,6 +98,20 @@ interface EvalRunSummary {
     finish: FinishOutcome;
   };
 }
+/**
+ * Run `attempt` up to `1 + retries` times, stopping as soon as it returns a
+ * result that is neither a thrown error / per-eval timeout (`error`) nor a
+ * transport/agent turn failure (`infraFailure`). Assertion failures set
+ * neither, so a failed-but-completed eval is returned on the first try and
+ * never retried. Returns the last result when every attempt failed on infra.
+ *
+ * Between attempts it waits a full-jittered exponential backoff (infra flakes
+ * are overload-correlated). `retries` is coerced to a finite non-negative
+ * integer; `baseDelayMs: 0` disables the wait (tests).
+ */
+declare function runWithRetries(retries: number, attempt: (attemptNumber: number) => Promise<EvalResult>, options?: {
+  baseDelayMs?: number;
+}): Promise<EvalResult>;
 /**
  * Discover, load, and run every eval under each agent's `evals/` dir, driving
  * the agents on a running app. Never throws for an individual eval — load/run
@@ -90,5 +119,5 @@ interface EvalRunSummary {
  */
 declare function runEvalsInDir(options: RunEvalsOptions): Promise<EvalRunSummary>;
 //#endregion
-export { EvalProgress, EvalRunSummary, RunEvalsOptions, runEvalsInDir };
+export { EvalProgress, EvalRunSummary, RunEvalsOptions, runEvalsInDir, runWithRetries };
 //# sourceMappingURL=run-evals.d.ts.map

package/dist/evals/run-evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-evals.d.ts","names":[],"sources":["../../src/evals/run-evals.ts"],"mappings":";;;;;;;UAciB,eAAA;;EAEf,OAAA;EAF8B;EAI9B,OAAA;EAMU;EAJV,MAAA;EAyCkB;EAvClB,MAAA;EAuC8B;EArC9B,OAAA,GAAU,MAAA;EANV;EAQA,SAAA;EAJA;;;;;;EAWA,WAAA;EAQE;;;;;EAFF,MAAA;IACE,IAAA;IACA,KAAA;IACA,YAAA,UAeF;IAbE,cAAA;EAAA;EAiBgB;;;;EAXlB,KAAA;IAAU,IAAA;IAAc,KAAA;IAAe,KAAA;EAAA;EAef;;;;EAVxB,eAAA,GAAkB,eAAA;EAYa;EAV/B,WAAA;EAWI;EATJ,GAAA;EAS4B;EAP5B,OAAA,IAAW,KAAA,EAAO,YAAA;AAAA;AAAA,KAGR,YAAA;EACN,IAAA;EAAoB,KAAA;AAAA;EACpB,IAAA;EAAqB,KAAA;AAAA;EACrB,IAAA;EAAe,EAAA;EAAY,KAAA;EAAe,KAAA;AAAA;EAC1C,IAAA;EAAgB,MAAA,EAAQ,UAAA;EAAY,KAAA;EAAe,KAAA;AAAA;AAAA,UAExC,cAAA;EACf,OAAA,EAAS,UAAA;EAE6D;EAAtE,MAAA;IAAW,KAAA;IAAe,MAAA,EAAQ,aAAA;IAAe,MAAA,EAAQ,aAAA;EAAA;AAAA;;;;;;iBAoOrC,aAAA,CACpB,OAAA,EAAS,eAAA,GACR,OAAA,CAAQ,cAAA"}
+{"version":3,"file":"run-evals.d.ts","names":[],"sources":["../../src/evals/run-evals.ts"],"mappings":";;;;;;;UAmBiB,eAAA;;EAEf,OAAA;EAF8B;EAI9B,OAAA;EAWU;EATV,MAAA;EAwDkB;;;;EAnDlB,IAAA;EALA;EAOA,MAAA;EAAA;EAEA,OAAA,GAAU,MAAA;EAAA;;;;;;EAOV,WAAA;EAiBA;;;;;EAXA,MAAA;IACE,IAAA;IACA,KAAA;IACA,YAAA,UA6BF;IA3BE,cAAA;EAAA;EA6BS;;;AAGb;EA1BE,KAAA;IAAU,IAAA;IAAc,KAAA;IAAe,KAAA;EAAA;EA4BnC;;;;EAvBJ,eAAA,GAAkB,eAAA;EAwB4B;EAtB9C,WAAA;EAuBoB;EArBpB,GAAA;EAqBwC;;;;AAE1C;EAjBE,SAAA;;;;;;EAMA,OAAA;EAYA;EAVA,OAAA,IAAW,KAAA,EAAO,YAAA;AAAA;AAAA,KAGR,YAAA;EACN,IAAA;EAAoB,KAAA;AAAA;EACpB,IAAA;EAAqB,KAAA;AAAA;EACrB,IAAA;EAAe,EAAA;EAAY,KAAA;EAAe,KAAA;AAAA;EAC1C,IAAA;EAAgB,MAAA,EAAQ,UAAA;EAAY,KAAA;EAAe,KAAA;AAAA;AAAA,UAExC,cAAA;EACf,OAAA,EAAS,UAAA;EA2OmC;EAzO5C,MAAA;IAAW,KAAA;IAAe,MAAA,EAAQ,aAAA;IAAe,MAAA,EAAQ,aAAA;EAAA;AAAA;;;;;;;;;;;;iBAuOrC,cAAA,CACpB,OAAA,UACA,OAAA,GAAU,aAAA,aAA0B,OAAA,CAAQ,UAAA,GAC5C,OAAA;EAAW,WAAA;AAAA,IACV,OAAA,CAAQ,UAAA;;;;;;iBA8GW,aAAA,CACpB,OAAA,EAAS,eAAA,GACR,OAAA,CAAQ,cAAA"}