@databricks/appkit 0.71.0 → 0.73.0

package/dist/evals/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","names":[],"sources":["../../src/evals/report.ts"],"sourcesContent":["import type { EvalResult } from \"./types\";\n\nexport interface EvalSummary {\n  total: number;\n  passed: number;\n  failed: number;\n  skipped: number;\n  /** True when no eval failed (skips don't count as failures). */\n  allPassed: boolean;\n}\n\nexport function summarize(results: EvalResult[]): EvalSummary {\n  let passed = 0;\n  let failed = 0;\n  let skipped = 0;\n  for (const r of results) {\n    if (r.skipped) skipped++;\n    else if (r.passed) passed++;\n    else failed++;\n  }\n  return {\n    total: results.length,\n    passed,\n    failed,\n    skipped,\n    allPassed: failed === 0,\n  };\n}\n\n/** Status glyph for a single eval result. */\nexport function evalGlyph(result: EvalResult): string {\n  if (result.skipped) return \"−\";\n  return result.passed ? \"✓\" : \"✗\";\n}\n\n/** The one-line header for a single eval result (no failure detail). */\nexport function formatEvalHeadline(result: EvalResult): string {\n  if (result.skipped) {\n    return `− ${result.id} (skipped${\n      result.skipped.reason ? `: ${result.skipped.reason}` : \"\"\n    })`;\n  }\n  return `${evalGlyph(result)} ${result.id}${\n    result.description ? ` — ${result.description}` : \"\"\n  }`;\n}\n\n/** Indented detail lines for a failing eval (error + failing assertions). */\nexport function formatEvalDetail(result: EvalResult): string[] {\n  const lines: string[] = [];\n  if (result.error) lines.push(`    error: ${result.error}`);\n  for (const a of result.assertions) {\n    if (a.pass) continue;\n    const tag = a.severity === \"soft\" ? \"soft\" : \"gate\";\n    lines.push(`    ✗ [${tag}] ${a.label}${a.detail ? ` — ${a.detail}` : \"\"}`);\n  }\n  return lines;\n}\n\n/** The final PASS/FAIL summary line. */\nexport function formatSummaryLine(results: EvalResult[]): string {\n  const s = summarize(results);\n  return `${s.allPassed ? \"PASS\" : \"FAIL\"} — ${s.passed} passed, ${s.failed} failed, ${s.skipped} skipped (${s.total} total)`;\n}\n\n/** Render all results as a human-readable console report (non-streaming). */\nexport function formatEvalResults(results: EvalResult[]): string {\n  const lines: string[] = [];\n  for (const r of results) {\n    lines.push(formatEvalHeadline(r));\n    lines.push(...formatEvalDetail(r));\n  }\n  lines.push(\"\");\n  lines.push(formatSummaryLine(results));\n  return lines.join(\"\\n\");\n}\n"],"mappings":";AAWA,SAAgB,UAAU,SAAoC;CAC5D,IAAI,SAAS;CACb,IAAI,SAAS;CACb,IAAI,UAAU;AACd,MAAK,MAAM,KAAK,QACd,KAAI,EAAE,QAAS;UACN,EAAE,OAAQ;KACd;AAEP,QAAO;EACL,OAAO,QAAQ;EACf;EACA;EACA;EACA,WAAW,WAAW;EACvB;;;AAIH,SAAgB,UAAU,QAA4B;AACpD,KAAI,OAAO,QAAS,QAAO;AAC3B,QAAO,OAAO,SAAS,MAAM;;;AAI/B,SAAgB,mBAAmB,QAA4B;AAC7D,KAAI,OAAO,QACT,QAAO,KAAK,OAAO,GAAG,WACpB,OAAO,QAAQ,SAAS,KAAK,OAAO,QAAQ,WAAW,GACxD;AAEH,QAAO,GAAG,UAAU,OAAO,CAAC,GAAG,OAAO,KACpC,OAAO,cAAc,MAAM,OAAO,gBAAgB;;;AAKtD,SAAgB,iBAAiB,QAA8B;CAC7D,MAAM,QAAkB,EAAE;AAC1B,KAAI,OAAO,MAAO,OAAM,KAAK,cAAc,OAAO,QAAQ;AAC1D,MAAK,MAAM,KAAK,OAAO,YAAY;AACjC,MAAI,EAAE,KAAM;EACZ,MAAM,MAAM,EAAE,aAAa,SAAS,SAAS;AAC7C,QAAM,KAAK,UAAU,IAAI,IAAI,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,WAAW,KAAK;;AAE5E,QAAO;;;AAIT,SAAgB,kBAAkB,SAA+B;CAC/D,MAAM,IAAI,UAAU,QAAQ;AAC5B,QAAO,GAAG,EAAE,YAAY,SAAS,OAAO,KAAK,EAAE,OAAO,WAAW,EAAE,OAAO,WAAW,EAAE,QAAQ,YAAY,EAAE,MAAM;;;AAIrH,SAAgB,kBAAkB,SAA+B;CAC/D,MAAM,QAAkB,EAAE;AAC1B,MAAK,MAAM,KAAK,SAAS;AACvB,QAAM,KAAK,mBAAmB,EAAE,CAAC;AACjC,QAAM,KAAK,GAAG,iBAAiB,EAAE,CAAC;;AAEpC,OAAM,KAAK,GAAG;AACd,OAAM,KAAK,kBAAkB,QAAQ,CAAC;AACtC,QAAO,MAAM,KAAK,KAAK"}

package/dist/evals/run-eval.d.ts

@@ -0,0 +1,23 @@
+import { DatasetRow } from "./dataset.js";
+import { EvalDefinition, EvalDriver, EvalResult } from "./types.js";
+
+//#region src/evals/run-eval.d.ts
+interface RunEvalOptions {
+  /** Stable id for the eval (e.g. its file path relative to the evals dir). */
+  id: string;
+  /** Drives the agent and returns reply/tool-calls/success per `send`. */
+  driver: EvalDriver;
+  /** When true, soft assertion failures also fail the eval. */
+  strict?: boolean;
+  /** Dataset row bound to `t.input`/`t.expected` for dataset-driven evals. */
+  row?: DatasetRow;
+}
+/**
+ * Run a single eval against a driver. Never throws for assertion or agent
+ * failures — those become a non-passing {@link EvalResult}. Only a malformed
+ * eval definition surfaces as `result.error`.
+ */
+declare function runEval(def: EvalDefinition, options: RunEvalOptions): Promise<EvalResult>;
+//#endregion
+export { RunEvalOptions, runEval };
+//# sourceMappingURL=run-eval.d.ts.map

package/dist/evals/run-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-eval.d.ts","names":[],"sources":["../../src/evals/run-eval.ts"],"mappings":";;;;UAuBiB,cAAA;;EAEf,EAAA;EAF6B;EAI7B,MAAA,EAAQ,UAAA;EAIQ;EAFhB,MAAA;EAFA;EAIA,GAAA,GAAM,UAAA;AAAA;;;;;AAQR;iBAAsB,OAAA,CACpB,GAAA,EAAK,cAAA,EACL,OAAA,EAAS,cAAA,GACR,OAAA,CAAQ,UAAA"}

package/dist/evals/run-eval.js

@@ -0,0 +1,152 @@
+import { judgeClosedQA, judgeCustom, judgeFactuality } from "./judge.js";
+
+//#region src/evals/run-eval.ts
+/** Default pass threshold for an LLM-judge score (0..1) before `.atLeast()`. */
+const DEFAULT_JUDGE_THRESHOLD = .5;
+/** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */
+var SkipSignal = class extends Error {
+	constructor(reason) {
+		super("eval skipped");
+		this.reason = reason;
+		this.name = "SkipSignal";
+	}
+};
+/**
+* Run a single eval against a driver. Never throws for assertion or agent
+* failures — those become a non-passing {@link EvalResult}. Only a malformed
+* eval definition surfaces as `result.error`.
+*/
+async function runEval(def, options) {
+	const assertions = [];
+	let reply = "";
+	let lastInput = "";
+	let toolCalls = [];
+	let sessionId;
+	let lastTraceId;
+	let lastSucceeded = false;
+	const record = (label, pass, score, detail) => {
+		const result = {
+			label,
+			severity: "gate",
+			pass,
+			score,
+			detail
+		};
+		assertions.push(result);
+		const handle = {
+			gate() {
+				result.severity = "gate";
+				return handle;
+			},
+			soft() {
+				result.severity = "soft";
+				return handle;
+			},
+			atLeast(threshold) {
+				result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold;
+				return handle;
+			}
+		};
+		return handle;
+	};
+	const recordJudge = (label, score, rationale) => record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale);
+	const t = {
+		async send(message) {
+			lastInput = message;
+			const r = await options.driver.send(message);
+			reply = r.reply;
+			toolCalls = r.toolCalls;
+			sessionId = r.sessionId;
+			lastSucceeded = r.succeeded;
+			if (r.traceId) lastTraceId = r.traceId;
+		},
+		reset() {
+			options.driver.reset?.();
+		},
+		get reply() {
+			return reply;
+		},
+		get toolCalls() {
+			return toolCalls;
+		},
+		get sessionId() {
+			return sessionId;
+		},
+		get input() {
+			return options.row?.inputs ?? {};
+		},
+		get expected() {
+			return options.row?.expectations;
+		},
+		succeeded() {
+			return record("succeeded", lastSucceeded, void 0, lastSucceeded ? void 0 : "agent turn did not complete successfully");
+		},
+		calledTool(name) {
+			return record(`calledTool(${name})`, toolCalls.includes(name), void 0, `expected tool "${name}" to be called (called: ${toolCalls.length ? toolCalls.join(", ") : "none"})`);
+		},
+		check(value, matcher) {
+			const m = matcher(value);
+			return record("check", m.pass, m.score, m.detail);
+		},
+		judge: {
+			async factuality(expected) {
+				const { score, rationale } = await judgeFactuality({
+					input: lastInput,
+					output: reply,
+					expected
+				});
+				return recordJudge("judge.factuality", score, rationale);
+			},
+			async closedQA(criteria) {
+				const { score, rationale } = await judgeClosedQA({
+					input: lastInput,
+					output: reply,
+					criteria
+				});
+				return recordJudge("judge.closedQA", score, rationale);
+			},
+			async custom(spec) {
+				const { score, rationale } = await judgeCustom(spec, {
+					input: lastInput,
+					output: reply
+				});
+				return recordJudge(`judge.${spec.name}`, score, rationale);
+			}
+		},
+		skip(reason) {
+			throw new SkipSignal(reason);
+		}
+	};
+	try {
+		await def.test(t);
+	} catch (err) {
+		if (err instanceof SkipSignal) return {
+			id: options.id,
+			description: def.description,
+			skipped: { reason: err.reason },
+			assertions,
+			passed: true,
+			traceId: lastTraceId
+		};
+		return {
+			id: options.id,
+			description: def.description,
+			assertions,
+			passed: false,
+			error: err instanceof Error ? err.message : String(err),
+			traceId: lastTraceId
+		};
+	}
+	const passed = assertions.every((a) => a.pass || a.severity === "soft" && !options.strict);
+	return {
+		id: options.id,
+		description: def.description,
+		assertions,
+		passed,
+		traceId: lastTraceId
+	};
+}
+
+//#endregion
+export { runEval };
+//# sourceMappingURL=run-eval.js.map

package/dist/evals/run-eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-eval.js","names":[],"sources":["../../src/evals/run-eval.ts"],"sourcesContent":["import type { DatasetRow } from \"./dataset\";\nimport { judgeClosedQA, judgeCustom, judgeFactuality } from \"./judge\";\nimport type {\n  AssertionHandle,\n  AssertionResult,\n  EvalDefinition,\n  EvalDriver,\n  EvalResult,\n  Matcher,\n  TestContext,\n} from \"./types\";\n\n/** Default pass threshold for an LLM-judge score (0..1) before `.atLeast()`. */\nconst DEFAULT_JUDGE_THRESHOLD = 0.5;\n\n/** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */\nclass SkipSignal extends Error {\n  constructor(public reason?: string) {\n    super(\"eval skipped\");\n    this.name = \"SkipSignal\";\n  }\n}\n\nexport interface RunEvalOptions {\n  /** Stable id for the eval (e.g. its file path relative to the evals dir). */\n  id: string;\n  /** Drives the agent and returns reply/tool-calls/success per `send`. */\n  driver: EvalDriver;\n  /** When true, soft assertion failures also fail the eval. */\n  strict?: boolean;\n  /** Dataset row bound to `t.input`/`t.expected` for dataset-driven evals. */\n  row?: DatasetRow;\n}\n\n/**\n * Run a single eval against a driver. Never throws for assertion or agent\n * failures — those become a non-passing {@link EvalResult}. Only a malformed\n * eval definition surfaces as `result.error`.\n */\nexport async function runEval(\n  def: EvalDefinition,\n  options: RunEvalOptions,\n): Promise<EvalResult> {\n  const assertions: AssertionResult[] = [];\n  let reply = \"\";\n  let lastInput = \"\";\n  let toolCalls: string[] = [];\n  let sessionId: string | undefined;\n  let lastTraceId: string | undefined;\n  let lastSucceeded = false;\n\n  const record = (\n    label: string,\n    pass: boolean,\n    score?: number,\n    detail?: string,\n  ): AssertionHandle => {\n    const result: AssertionResult = {\n      label,\n      severity: \"gate\",\n      pass,\n      score,\n      detail,\n    };\n    assertions.push(result);\n    const handle: AssertionHandle = {\n      gate() {\n        result.severity = \"gate\";\n        return handle;\n      },\n      soft() {\n        result.severity = \"soft\";\n        return handle;\n      },\n      atLeast(threshold: number) {\n        // Re-threshold the score; keep the current severity (gate unless the\n        // caller also chained `.soft()`).\n        result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold;\n        return handle;\n      },\n    };\n    return handle;\n  };\n\n  // LLM-judge assertions are scored and gate by default (a miss fails the eval,\n  // like other assertions). The caller chains `.atLeast(n)` to change the pass\n  // threshold, or `.soft()` to demote to a tracked-only metric.\n  const recordJudge = (\n    label: string,\n    score: number,\n    rationale?: string,\n  ): AssertionHandle =>\n    record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale);\n\n  const t: TestContext = {\n    async send(message) {\n      lastInput = message;\n      const r = await options.driver.send(message);\n      reply = r.reply;\n      toolCalls = r.toolCalls;\n      sessionId = r.sessionId;\n      lastSucceeded = r.succeeded;\n      if (r.traceId) lastTraceId = r.traceId;\n    },\n    reset() {\n      options.driver.reset?.();\n    },\n    get reply() {\n      return reply;\n    },\n    get toolCalls() {\n      return toolCalls;\n    },\n    get sessionId() {\n      return sessionId;\n    },\n    get input() {\n      return options.row?.inputs ?? {};\n    },\n    get expected() {\n      return options.row?.expectations;\n    },\n    succeeded() {\n      return record(\n        \"succeeded\",\n        lastSucceeded,\n        undefined,\n        lastSucceeded ? undefined : \"agent turn did not complete successfully\",\n      );\n    },\n    calledTool(name) {\n      return record(\n        `calledTool(${name})`,\n        toolCalls.includes(name),\n        undefined,\n        `expected tool \"${name}\" to be called (called: ${\n          toolCalls.length ? toolCalls.join(\", \") : \"none\"\n        })`,\n      );\n    },\n    check(value: string, matcher: Matcher) {\n      const m = matcher(value);\n      return record(\"check\", m.pass, m.score, m.detail);\n    },\n    judge: {\n      async factuality(expected) {\n        const { score, rationale } = await judgeFactuality({\n          input: lastInput,\n          output: reply,\n          expected,\n        });\n        return recordJudge(\"judge.factuality\", score, rationale);\n      },\n      async closedQA(criteria) {\n        const { score, rationale } = await judgeClosedQA({\n          input: lastInput,\n          output: reply,\n          criteria,\n        });\n        return recordJudge(\"judge.closedQA\", score, rationale);\n      },\n      async custom(spec) {\n        const { score, rationale } = await judgeCustom(spec, {\n          input: lastInput,\n          output: reply,\n        });\n        return recordJudge(`judge.${spec.name}`, score, rationale);\n      },\n    },\n    skip(reason) {\n      throw new SkipSignal(reason);\n    },\n  };\n\n  try {\n    await def.test(t);\n  } catch (err) {\n    if (err instanceof SkipSignal) {\n      return {\n        id: options.id,\n        description: def.description,\n        skipped: { reason: err.reason },\n        assertions,\n        passed: true,\n        traceId: lastTraceId,\n      };\n    }\n    return {\n      id: options.id,\n      description: def.description,\n      assertions,\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n      traceId: lastTraceId,\n    };\n  }\n\n  const passed = assertions.every(\n    (a) => a.pass || (a.severity === \"soft\" && !options.strict),\n  );\n\n  return {\n    id: options.id,\n    description: def.description,\n    assertions,\n    passed,\n    traceId: lastTraceId,\n  };\n}\n"],"mappings":";;;;AAaA,MAAM,0BAA0B;;AAGhC,IAAM,aAAN,cAAyB,MAAM;CAC7B,YAAY,AAAO,QAAiB;AAClC,QAAM,eAAe;EADJ;AAEjB,OAAK,OAAO;;;;;;;;AAoBhB,eAAsB,QACpB,KACA,SACqB;CACrB,MAAM,aAAgC,EAAE;CACxC,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,YAAsB,EAAE;CAC5B,IAAI;CACJ,IAAI;CACJ,IAAI,gBAAgB;CAEpB,MAAM,UACJ,OACA,MACA,OACA,WACoB;EACpB,MAAM,SAA0B;GAC9B;GACA,UAAU;GACV;GACA;GACA;GACD;AACD,aAAW,KAAK,OAAO;EACvB,MAAM,SAA0B;GAC9B,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,QAAQ,WAAmB;AAGzB,WAAO,QAAQ,OAAO,UAAU,OAAO,OAAO,IAAI,OAAO;AACzD,WAAO;;GAEV;AACD,SAAO;;CAMT,MAAM,eACJ,OACA,OACA,cAEA,OAAO,OAAO,SAAS,yBAAyB,OAAO,UAAU;CAEnE,MAAM,IAAiB;EACrB,MAAM,KAAK,SAAS;AAClB,eAAY;GACZ,MAAM,IAAI,MAAM,QAAQ,OAAO,KAAK,QAAQ;AAC5C,WAAQ,EAAE;AACV,eAAY,EAAE;AACd,eAAY,EAAE;AACd,mBAAgB,EAAE;AAClB,OAAI,EAAE,QAAS,eAAc,EAAE;;EAEjC,QAAQ;AACN,WAAQ,OAAO,SAAS;;EAE1B,IAAI,QAAQ;AACV,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,QAAQ;AACV,UAAO,QAAQ,KAAK,UAAU,EAAE;;EAElC,IAAI,WAAW;AACb,UAAO,QAAQ,KAAK;;EAEtB,YAAY;AACV,UAAO,OACL,aACA,eACA,QACA,gBAAgB,SAAY,2CAC7B;;EAEH,WAAW,MAAM;AACf,UAAO,OACL,cAAc,KAAK,IACnB,UAAU,SAAS,KAAK,EACxB,QACA,kBAAkB,KAAK,0BACrB,UAAU,SAAS,UAAU,KAAK,KAAK,GAAG,OAC3C,GACF;;EAEH,MAAM,OAAe,SAAkB;GACrC,MAAM,IAAI,QAAQ,MAAM;AACxB,UAAO,OAAO,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;;EAEnD,OAAO;GACL,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,OAAO,cAAc,MAAM,gBAAgB;KACjD,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,oBAAoB,OAAO,UAAU;;GAE1D,MAAM,SAAS,UAAU;IACvB,MAAM,EAAE,OAAO,cAAc,MAAM,cAAc;KAC/C,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,kBAAkB,OAAO,UAAU;;GAExD,MAAM,OAAO,MAAM;IACjB,MAAM,EAAE,OAAO,cAAc,MAAM,YAAY,MAAM;KACnD,OAAO;KACP,QAAQ;KACT,CAAC;AACF,WAAO,YAAY,SAAS,KAAK,QAAQ,OAAO,UAAU;;GAE7D;EACD,KAAK,QAAQ;AACX,SAAM,IAAI,WAAW,OAAO;;EAE/B;AAED,KAAI;AACF,QAAM,IAAI,KAAK,EAAE;UACV,KAAK;AACZ,MAAI,eAAe,WACjB,QAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB,SAAS,EAAE,QAAQ,IAAI,QAAQ;GAC/B;GACA,QAAQ;GACR,SAAS;GACV;AAEH,SAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB;GACA,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACvD,SAAS;GACV;;CAGH,MAAM,SAAS,WAAW,OACvB,MAAM,EAAE,QAAS,EAAE,aAAa,UAAU,CAAC,QAAQ,OACrD;AAED,QAAO;EACL,IAAI,QAAQ;EACZ,aAAa,IAAI;EACjB;EACA;EACA,SAAS;EACV"}

package/dist/evals/run-evals.d.ts

@@ -0,0 +1,94 @@
+import { WorkspaceClient } from "../workspace-client/index.js";
+import "./dataset.js";
+import { EvalResult } from "./types.js";
+import { ReportOutcome } from "./mlflow-report.js";
+import { FinishOutcome } from "./mlflow-run.js";
+
+//#region src/evals/run-evals.d.ts
+interface RunEvalsOptions {
+  /** Project root containing `server/agents/`. Defaults to `process.cwd()`. */
+  rootDir?: string;
+  /** Base URL of the running app to drive, e.g. `http://localhost:3000`. */
+  baseUrl: string;
+  /** Substring filter on `<agent>/<id>` (or an exact agent id). */
+  filter?: string;
+  /** Soft assertion failures also fail the eval. */
+  strict?: boolean;
+  /** Extra request headers for the driver (e.g. auth for a deployed app). */
+  headers?: Record<string, string>;
+  /** Per-turn wall-clock timeout (ms) before a turn is failed. Defaults to 120s. */
+  timeoutMs?: number;
+  /**
+   * Max evals to drive concurrently. Each eval opens one stream to the app as
+   * the same user, so keep this at or below the app's
+   * `maxConcurrentStreamsPerUser` (default 5) or the surplus streams hit the
+   * 429 guard. Defaults to 4; clamped to `[1, total]`.
+   */
+  concurrency?: number;
+  /**
+   * When set, create a native MLflow "Evaluation run": each eval's trace is
+   * linked to the run, pass/fail is written as feedback, and aggregate metrics
+   * are logged. Requires Databricks creds + the target experiment.
+   */
+  mlflow?: {
+    host: string;
+    token: string;
+    experimentId: string; /** SQL warehouse id for writing assessments to UC-backed (V4) traces. */
+    sqlWarehouseId?: string;
+  };
+  /**
+   * When set, enable `t.judge.*` LLM-as-judge scoring via autoevals against a
+   * Databricks serving endpoint (`model`).
+   */
+  judge?: {
+    host: string;
+    token: string;
+    model: string;
+  };
+  /**
+   * Workspace client used to read managed evaluation datasets (for evals that
+   * declare `dataset`). Required alongside {@link warehouseId} for those evals.
+   */
+  workspaceClient?: WorkspaceClient;
+  /** SQL warehouse id used to read managed evaluation datasets. */
+  warehouseId?: string;
+  /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */
+  now?: number;
+  /** Progress callback, invoked as evals are discovered, started, and finished. */
+  onEvent?: (event: EvalProgress) => void;
+}
+type EvalProgress = {
+  type: "discovered";
+  total: number;
+} | {
+  type: "run-created";
+  runId: string;
+} | {
+  type: "start";
+  id: string;
+  index: number;
+  total: number;
+} | {
+  type: "result";
+  result: EvalResult;
+  index: number;
+  total: number;
+};
+interface EvalRunSummary {
+  results: EvalResult[];
+  /** Present when an MLflow evaluation run was created. */
+  mlflow?: {
+    runId: string;
+    report: ReportOutcome;
+    finish: FinishOutcome;
+  };
+}
+/**
+ * Discover, load, and run every eval under each agent's `evals/` dir, driving
+ * the agents on a running app. Never throws for an individual eval — load/run
+ * failures become non-passing {@link EvalResult}s.
+ */
+declare function runEvalsInDir(options: RunEvalsOptions): Promise<EvalRunSummary>;
+//#endregion
+export { EvalProgress, EvalRunSummary, RunEvalsOptions, runEvalsInDir };
+//# sourceMappingURL=run-evals.d.ts.map

package/dist/evals/run-evals.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-evals.d.ts","names":[],"sources":["../../src/evals/run-evals.ts"],"mappings":";;;;;;;UAciB,eAAA;;EAEf,OAAA;EAF8B;EAI9B,OAAA;EAMU;EAJV,MAAA;EAyCkB;EAvClB,MAAA;EAuC8B;EArC9B,OAAA,GAAU,MAAA;EANV;EAQA,SAAA;EAJA;;;;;;EAWA,WAAA;EAQE;;;;;EAFF,MAAA;IACE,IAAA;IACA,KAAA;IACA,YAAA,UAeF;IAbE,cAAA;EAAA;EAiBgB;;;;EAXlB,KAAA;IAAU,IAAA;IAAc,KAAA;IAAe,KAAA;EAAA;EAef;;;;EAVxB,eAAA,GAAkB,eAAA;EAYa;EAV/B,WAAA;EAWI;EATJ,GAAA;EAS4B;EAP5B,OAAA,IAAW,KAAA,EAAO,YAAA;AAAA;AAAA,KAGR,YAAA;EACN,IAAA;EAAoB,KAAA;AAAA;EACpB,IAAA;EAAqB,KAAA;AAAA;EACrB,IAAA;EAAe,EAAA;EAAY,KAAA;EAAe,KAAA;AAAA;EAC1C,IAAA;EAAgB,MAAA,EAAQ,UAAA;EAAY,KAAA;EAAe,KAAA;AAAA;AAAA,UAExC,cAAA;EACf,OAAA,EAAS,UAAA;EAE6D;EAAtE,MAAA;IAAW,KAAA;IAAe,MAAA,EAAQ,aAAA;IAAe,MAAA,EAAQ,aAAA;EAAA;AAAA;;;;;;iBAoOrC,aAAA,CACpB,OAAA,EAAS,eAAA,GACR,OAAA,CAAQ,cAAA"}

package/dist/evals/run-evals.js

@@ -0,0 +1,257 @@
+import { MlflowClient } from "../connectors/mlflow/client.js";
+import { readEvalDataset } from "./dataset.js";
+import { discoverEvalFiles } from "./discover.js";
+import { createHttpDriver } from "./http-driver.js";
+import { configureJudge, teardownJudge } from "./judge.js";
+import { mapPool } from "./pool.js";
+import { reportToMlflow } from "./mlflow-report.js";
+import { runEval } from "./run-eval.js";
+import { createEvalRun, finishEvalRun } from "./mlflow-run.js";
+import { pathToFileURL } from "node:url";
+
+//#region src/evals/run-evals.ts
+/**
+* Load a `*.eval.ts` file and return its default-exported {@link EvalDefinition}.
+* Uses tsx's programmatic loader so TypeScript eval files run without a build
+* step. The specifier is indirected so the type checker doesn't try to resolve
+* tsx's internal entry.
+*/
+async function loadEval(file) {
+	const tsxApi = "tsx/esm/api";
+	let tsImport;
+	try {
+		({tsImport} = await import(tsxApi));
+	} catch {
+		throw new Error("Running .eval.ts files requires `tsx`. Install it as a dev dependency (`pnpm add -D tsx`).");
+	}
+	const def = resolveEvalDefault(await tsImport(pathToFileURL(file).href, import.meta.url));
+	if (!def) throw new Error(`${file}: must default-export defineEval({ test })`);
+	return def;
+}
+/**
+* Unwrap the eval default export across module-interop shapes. Depending on
+* whether the eval file is treated as ESM or CJS, the value lands at
+* `mod.default` (ESM), `mod.default.default` (CJS `__esModule` double-wrap), or
+* `mod` itself. Returns the first candidate that looks like an eval.
+*/
+function resolveEvalDefault(mod) {
+	let candidate = mod;
+	for (let i = 0; i < 4 && candidate; i++) {
+		if (typeof candidate.test === "function") return candidate;
+		candidate = candidate.default;
+	}
+}
+/**
+* Run one eval turn against a fresh driver. Never throws — a run failure becomes
+* a non-passing {@link EvalResult} so one bad eval can't abort the run. `row`
+* binds the current managed-dataset row (see {@link resolveDatasetRows}), or is
+* `undefined` for a plain single-run eval.
+*/
+async function runOne(d, id, def, row, runId, options) {
+	try {
+		return await runEval(def, {
+			id,
+			driver: createHttpDriver({
+				baseUrl: options.baseUrl,
+				agent: def.agent ?? d.agent,
+				headers: options.headers,
+				mlflowRunId: runId,
+				timeoutMs: options.timeoutMs
+			}),
+			strict: options.strict,
+			row
+		});
+	} catch (err) {
+		return {
+			id,
+			assertions: [],
+			passed: false,
+			error: err instanceof Error ? err.message : String(err)
+		};
+	}
+}
+/**
+* Resolve the rows a (possibly dataset-driven) eval runs over. A plain eval
+* yields a single `undefined` row; a dataset eval reads its Unity Catalog table
+* via {@link readEvalDataset}. On misconfiguration or read failure, returns a
+* single `undefined` row plus an `error`, so the eval still surfaces one result.
+*/
+async function resolveDatasetRows(def, options) {
+	if (!def.dataset) return { rows: [void 0] };
+	if (!options.workspaceClient || !options.warehouseId) return {
+		rows: [void 0],
+		error: "dataset eval requires a workspace client and warehouse (pass --warehouse-id)"
+	};
+	try {
+		const rows = await readEvalDataset(options.workspaceClient, {
+			table: def.dataset.table,
+			warehouseId: options.warehouseId,
+			limit: def.dataset.limit
+		});
+		if (rows.length === 0) return {
+			rows: [void 0],
+			error: `dataset "${def.dataset.table}" returned no rows`
+		};
+		return { rows };
+	} catch (err) {
+		return {
+			rows: [void 0],
+			error: err instanceof Error ? err.message : String(err)
+		};
+	}
+}
+/**
+* Load one discovered eval and run it, expanding a dataset-driven eval into one
+* run per row. Appends one result per row to `results`, emitting `start`/
+* `result` around each. Never throws: a load or dataset-read failure surfaces as
+* a non-passing result. `total` counts eval files, not rows — per-row detail is
+* carried in the result id (`[row i/n]`).
+*/
+async function runDiscovered(d, index, total, runId, options, emit, results) {
+	const id = `${d.agent}/${d.id}`;
+	let def;
+	try {
+		def = await loadEval(d.file);
+	} catch (err) {
+		emit({
+			type: "start",
+			id,
+			index,
+			total
+		});
+		const result = {
+			id,
+			assertions: [],
+			passed: false,
+			error: err instanceof Error ? err.message : String(err)
+		};
+		results.push(result);
+		emit({
+			type: "result",
+			result,
+			index,
+			total
+		});
+		return;
+	}
+	const { rows, error: datasetError } = await resolveDatasetRows(def, options);
+	for (let r = 0; r < rows.length; r++) {
+		const rowId = def.dataset && rows.length > 1 ? `${id} [row ${r + 1}/${rows.length}]` : id;
+		emit({
+			type: "start",
+			id: rowId,
+			index,
+			total
+		});
+		const result = datasetError ? {
+			id: rowId,
+			assertions: [],
+			passed: false,
+			error: datasetError
+		} : await runOne(d, rowId, def, rows[r], runId, options);
+		results.push(result);
+		emit({
+			type: "result",
+			result,
+			index,
+			total
+		});
+	}
+}
+/** Configure the LLM judge when judge creds were supplied; otherwise a no-op. */
+async function maybeConfigureJudge(options) {
+	if (!options.judge) return;
+	await configureJudge({
+		client: new MlflowClient(options.judge.host, options.judge.token),
+		token: options.judge.token,
+		model: options.judge.model
+	});
+}
+/**
+* Report per-eval assessments and finish the MLflow run, when one was created.
+* Returns the run summary, or `undefined` when there was no run to finalize.
+*/
+async function finalizeMlflow(client, runId, results, options) {
+	if (!client || !runId) return void 0;
+	let report = {
+		written: 0,
+		skipped: 0,
+		failures: []
+	};
+	try {
+		report = await reportToMlflow(client, results, options.mlflow?.sqlWarehouseId);
+	} catch (err) {
+		report.failures.push({
+			traceId: "(report)",
+			error: err instanceof Error ? err.message : String(err)
+		});
+	}
+	const finish = await finishEvalRun(client, {
+		runId,
+		results,
+		endTime: options.now ?? Date.now()
+	});
+	return {
+		runId,
+		report,
+		finish
+	};
+}
+/**
+* Default max evals in flight. Each eval opens one stream to the app as the
+* same user; the server caps concurrent streams per user at 5 by default
+* (`maxConcurrentStreamsPerUser`), so 4 leaves headroom under that limit.
+*/
+const DEFAULT_CONCURRENCY = 4;
+/**
+* Discover, load, and run every eval under each agent's `evals/` dir, driving
+* the agents on a running app. Never throws for an individual eval — load/run
+* failures become non-passing {@link EvalResult}s.
+*/
+async function runEvalsInDir(options) {
+	const root = options.rootDir ?? process.cwd();
+	const now = options.now ?? Date.now();
+	let discovered = discoverEvalFiles(root);
+	if (options.filter) {
+		const f = options.filter;
+		discovered = discovered.filter((d) => d.agent === f || `${d.agent}/${d.id}`.includes(f));
+	}
+	const emit = options.onEvent ?? (() => {});
+	const total = discovered.length;
+	emit({
+		type: "discovered",
+		total
+	});
+	await maybeConfigureJudge(options);
+	try {
+		let runId;
+		let mlflowClient;
+		if (options.mlflow) {
+			mlflowClient = new MlflowClient(options.mlflow.host, options.mlflow.token);
+			runId = await createEvalRun(mlflowClient, {
+				experimentId: options.mlflow.experimentId,
+				runName: `appkit-eval ${new Date(now).toISOString()}`,
+				startTime: now
+			});
+			emit({
+				type: "run-created",
+				runId
+			});
+		}
+		const results = (await mapPool(discovered, options.concurrency ?? DEFAULT_CONCURRENCY, async (d, index) => {
+			const fileResults = [];
+			await runDiscovered(d, index, total, runId, options, emit, fileResults);
+			return fileResults;
+		})).flat();
+		const summary = { results };
+		const mlflow = await finalizeMlflow(mlflowClient, runId, results, options);
+		if (mlflow) summary.mlflow = mlflow;
+		return summary;
+	} finally {
+		teardownJudge();
+	}
+}
+
+//#endregion
+export { runEvalsInDir };
+//# sourceMappingURL=run-evals.js.map

package/dist/evals/run-evals.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-evals.js","names":[],"sources":["../../src/evals/run-evals.ts"],"sourcesContent":["import { pathToFileURL } from \"node:url\";\n\nimport { MlflowClient } from \"../connectors/mlflow\";\nimport type { WorkspaceClient } from \"../workspace-client\";\nimport { type DatasetRow, readEvalDataset } from \"./dataset\";\nimport { type DiscoveredEval, discoverEvalFiles } from \"./discover\";\nimport { createHttpDriver } from \"./http-driver\";\nimport { configureJudge, teardownJudge } from \"./judge\";\nimport { type ReportOutcome, reportToMlflow } from \"./mlflow-report\";\nimport { createEvalRun, type FinishOutcome, finishEvalRun } from \"./mlflow-run\";\nimport { mapPool } from \"./pool\";\nimport { runEval } from \"./run-eval\";\nimport type { EvalDefinition, EvalResult } from \"./types\";\n\nexport interface RunEvalsOptions {\n  /** Project root containing `server/agents/`. Defaults to `process.cwd()`. */\n  rootDir?: string;\n  /** Base URL of the running app to drive, e.g. `http://localhost:3000`. */\n  baseUrl: string;\n  /** Substring filter on `<agent>/<id>` (or an exact agent id). */\n  filter?: string;\n  /** Soft assertion failures also fail the eval. */\n  strict?: boolean;\n  /** Extra request headers for the driver (e.g. auth for a deployed app). */\n  headers?: Record<string, string>;\n  /** Per-turn wall-clock timeout (ms) before a turn is failed. Defaults to 120s. */\n  timeoutMs?: number;\n  /**\n   * Max evals to drive concurrently. Each eval opens one stream to the app as\n   * the same user, so keep this at or below the app's\n   * `maxConcurrentStreamsPerUser` (default 5) or the surplus streams hit the\n   * 429 guard. Defaults to 4; clamped to `[1, total]`.\n   */\n  concurrency?: number;\n  /**\n   * When set, create a native MLflow \"Evaluation run\": each eval's trace is\n   * linked to the run, pass/fail is written as feedback, and aggregate metrics\n   * are logged. Requires Databricks creds + the target experiment.\n   */\n  mlflow?: {\n    host: string;\n    token: string;\n    experimentId: string;\n    /** SQL warehouse id for writing assessments to UC-backed (V4) traces. */\n    sqlWarehouseId?: string;\n  };\n  /**\n   * When set, enable `t.judge.*` LLM-as-judge scoring via autoevals against a\n   * Databricks serving endpoint (`model`).\n   */\n  judge?: { host: string; token: string; model: string };\n  /**\n   * Workspace client used to read managed evaluation datasets (for evals that\n   * declare `dataset`). Required alongside {@link warehouseId} for those evals.\n   */\n  workspaceClient?: WorkspaceClient;\n  /** SQL warehouse id used to read managed evaluation datasets. */\n  warehouseId?: string;\n  /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */\n  now?: number;\n  /** Progress callback, invoked as evals are discovered, started, and finished. */\n  onEvent?: (event: EvalProgress) => void;\n}\n\nexport type EvalProgress =\n  | { type: \"discovered\"; total: number }\n  | { type: \"run-created\"; runId: string }\n  | { type: \"start\"; id: string; index: number; total: number }\n  | { type: \"result\"; result: EvalResult; index: number; total: number };\n\nexport interface EvalRunSummary {\n  results: EvalResult[];\n  /** Present when an MLflow evaluation run was created. */\n  mlflow?: { runId: string; report: ReportOutcome; finish: FinishOutcome };\n}\n\n/**\n * Load a `*.eval.ts` file and return its default-exported {@link EvalDefinition}.\n * Uses tsx's programmatic loader so TypeScript eval files run without a build\n * step. The specifier is indirected so the type checker doesn't try to resolve\n * tsx's internal entry.\n */\nasync function loadEval(file: string): Promise<EvalDefinition> {\n  const tsxApi = \"tsx/esm/api\";\n  let tsImport: (specifier: string, parentURL: string) => Promise<unknown>;\n  try {\n    ({ tsImport } = (await import(tsxApi)) as {\n      tsImport: (specifier: string, parentURL: string) => Promise<unknown>;\n    });\n  } catch {\n    throw new Error(\n      \"Running .eval.ts files requires `tsx`. Install it as a dev dependency (`pnpm add -D tsx`).\",\n    );\n  }\n\n  const mod = await tsImport(pathToFileURL(file).href, import.meta.url);\n  const def = resolveEvalDefault(mod);\n  if (!def) {\n    throw new Error(`${file}: must default-export defineEval({ test })`);\n  }\n  return def;\n}\n\n/**\n * Unwrap the eval default export across module-interop shapes. Depending on\n * whether the eval file is treated as ESM or CJS, the value lands at\n * `mod.default` (ESM), `mod.default.default` (CJS `__esModule` double-wrap), or\n * `mod` itself. Returns the first candidate that looks like an eval.\n */\nexport function resolveEvalDefault(mod: unknown): EvalDefinition | undefined {\n  let candidate: unknown = mod;\n  for (let i = 0; i < 4 && candidate; i++) {\n    if (typeof (candidate as EvalDefinition).test === \"function\") {\n      return candidate as EvalDefinition;\n    }\n    candidate = (candidate as { default?: unknown }).default;\n  }\n  return undefined;\n}\n\n/**\n * Run one eval turn against a fresh driver. Never throws — a run failure becomes\n * a non-passing {@link EvalResult} so one bad eval can't abort the run. `row`\n * binds the current managed-dataset row (see {@link resolveDatasetRows}), or is\n * `undefined` for a plain single-run eval.\n */\nasync function runOne(\n  d: DiscoveredEval,\n  id: string,\n  def: EvalDefinition,\n  row: DatasetRow | undefined,\n  runId: string | undefined,\n  options: RunEvalsOptions,\n): Promise<EvalResult> {\n  try {\n    // A fresh driver per row: each row is an independent conversation whose\n    // thread must not carry over the previous row's history. (Multiple\n    // `t.send`s within one row still share the thread — the driver's behavior.)\n    const driver = createHttpDriver({\n      baseUrl: options.baseUrl,\n      agent: def.agent ?? d.agent,\n      headers: options.headers,\n      mlflowRunId: runId,\n      timeoutMs: options.timeoutMs,\n    });\n    return await runEval(def, { id, driver, strict: options.strict, row });\n  } catch (err) {\n    return {\n      id,\n      assertions: [],\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n    };\n  }\n}\n\n/**\n * Resolve the rows a (possibly dataset-driven) eval runs over. A plain eval\n * yields a single `undefined` row; a dataset eval reads its Unity Catalog table\n * via {@link readEvalDataset}. On misconfiguration or read failure, returns a\n * single `undefined` row plus an `error`, so the eval still surfaces one result.\n */\nexport async function resolveDatasetRows(\n  def: EvalDefinition,\n  options: RunEvalsOptions,\n): Promise<{ rows: Array<DatasetRow | undefined>; error?: string }> {\n  if (!def.dataset) return { rows: [undefined] };\n  if (!options.workspaceClient || !options.warehouseId) {\n    return {\n      rows: [undefined],\n      error:\n        \"dataset eval requires a workspace client and warehouse (pass --warehouse-id)\",\n    };\n  }\n  try {\n    const rows = await readEvalDataset(options.workspaceClient, {\n      table: def.dataset.table,\n      warehouseId: options.warehouseId,\n      limit: def.dataset.limit,\n    });\n    if (rows.length === 0) {\n      return {\n        rows: [undefined],\n        error: `dataset \"${def.dataset.table}\" returned no rows`,\n      };\n    }\n    return { rows };\n  } catch (err) {\n    return {\n      rows: [undefined],\n      error: err instanceof Error ? err.message : String(err),\n    };\n  }\n}\n\n/**\n * Load one discovered eval and run it, expanding a dataset-driven eval into one\n * run per row. Appends one result per row to `results`, emitting `start`/\n * `result` around each. Never throws: a load or dataset-read failure surfaces as\n * a non-passing result. `total` counts eval files, not rows — per-row detail is\n * carried in the result id (`[row i/n]`).\n */\nasync function runDiscovered(\n  d: DiscoveredEval,\n  index: number,\n  total: number,\n  runId: string | undefined,\n  options: RunEvalsOptions,\n  emit: (event: EvalProgress) => void,\n  results: EvalResult[],\n): Promise<void> {\n  const id = `${d.agent}/${d.id}`;\n\n  let def: EvalDefinition;\n  try {\n    def = await loadEval(d.file);\n  } catch (err) {\n    emit({ type: \"start\", id, index, total });\n    const result: EvalResult = {\n      id,\n      assertions: [],\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n    };\n    results.push(result);\n    emit({ type: \"result\", result, index, total });\n    return;\n  }\n\n  const { rows, error: datasetError } = await resolveDatasetRows(def, options);\n\n  for (let r = 0; r < rows.length; r++) {\n    const rowId =\n      def.dataset && rows.length > 1\n        ? `${id} [row ${r + 1}/${rows.length}]`\n        : id;\n    emit({ type: \"start\", id: rowId, index, total });\n    const result: EvalResult = datasetError\n      ? { id: rowId, assertions: [], passed: false, error: datasetError }\n      : await runOne(d, rowId, def, rows[r], runId, options);\n    results.push(result);\n    emit({ type: \"result\", result, index, total });\n  }\n}\n\n/** Configure the LLM judge when judge creds were supplied; otherwise a no-op. */\nasync function maybeConfigureJudge(options: RunEvalsOptions): Promise<void> {\n  if (!options.judge) return;\n  await configureJudge({\n    client: new MlflowClient(options.judge.host, options.judge.token),\n    token: options.judge.token,\n    model: options.judge.model,\n  });\n}\n\n/**\n * Report per-eval assessments and finish the MLflow run, when one was created.\n * Returns the run summary, or `undefined` when there was no run to finalize.\n */\nasync function finalizeMlflow(\n  client: MlflowClient | undefined,\n  runId: string | undefined,\n  results: EvalResult[],\n  options: RunEvalsOptions,\n): Promise<EvalRunSummary[\"mlflow\"]> {\n  if (!client || !runId) return undefined;\n  // reportToMlflow is not supposed to throw, but if it ever does the run must\n  // still be finished — otherwise it hangs in RUNNING forever.\n  let report: ReportOutcome = { written: 0, skipped: 0, failures: [] };\n  try {\n    report = await reportToMlflow(\n      client,\n      results,\n      options.mlflow?.sqlWarehouseId,\n    );\n  } catch (err) {\n    report.failures.push({\n      traceId: \"(report)\",\n      error: err instanceof Error ? err.message : String(err),\n    });\n  }\n  const finish = await finishEvalRun(client, {\n    runId,\n    results,\n    endTime: options.now ?? Date.now(),\n  });\n  return { runId, report, finish };\n}\n\n/**\n * Default max evals in flight. Each eval opens one stream to the app as the\n * same user; the server caps concurrent streams per user at 5 by default\n * (`maxConcurrentStreamsPerUser`), so 4 leaves headroom under that limit.\n */\nconst DEFAULT_CONCURRENCY = 4;\n\n/**\n * Discover, load, and run every eval under each agent's `evals/` dir, driving\n * the agents on a running app. Never throws for an individual eval — load/run\n * failures become non-passing {@link EvalResult}s.\n */\nexport async function runEvalsInDir(\n  options: RunEvalsOptions,\n): Promise<EvalRunSummary> {\n  const root = options.rootDir ?? process.cwd();\n  const now = options.now ?? Date.now();\n  let discovered = discoverEvalFiles(root);\n\n  if (options.filter) {\n    const f = options.filter;\n    discovered = discovered.filter(\n      (d) => d.agent === f || `${d.agent}/${d.id}`.includes(f),\n    );\n  }\n\n  const emit = options.onEvent ?? (() => {});\n  const total = discovered.length;\n  emit({ type: \"discovered\", total });\n\n  // The judge sets OPENAI_* env vars globally (autoevals reads them per call),\n  // so tear them down in `finally` once the run is over — pass or throw — so\n  // the bearer doesn't linger in process.env.\n  await maybeConfigureJudge(options);\n  try {\n    // Create the MLflow evaluation run up front so each eval's trace can be\n    // linked to it as it runs. One client is shared by run create/finish and\n    // the per-trace assessment writes.\n    let runId: string | undefined;\n    let mlflowClient: MlflowClient | undefined;\n    if (options.mlflow) {\n      mlflowClient = new MlflowClient(\n        options.mlflow.host,\n        options.mlflow.token,\n      );\n      runId = await createEvalRun(mlflowClient, {\n        experimentId: options.mlflow.experimentId,\n        runName: `appkit-eval ${new Date(now).toISOString()}`,\n        startTime: now,\n      });\n      emit({ type: \"run-created\", runId });\n    }\n\n    // Run each eval through the bounded pool — one in-flight stream per eval, so\n    // the pool respects the server's per-user stream cap (see mapPool/concurrency).\n    // A dataset eval expands into per-row runs that execute serially within its\n    // slot; results preserve discovery order (mapPool writes by index) and row\n    // order within each file. `total` counts eval files, not dataset rows — per-row\n    // detail is carried in the result id (`[row i/n]`).\n    const perFile = await mapPool(\n      discovered,\n      options.concurrency ?? DEFAULT_CONCURRENCY,\n      async (d, index) => {\n        const fileResults: EvalResult[] = [];\n        await runDiscovered(d, index, total, runId, options, emit, fileResults);\n        return fileResults;\n      },\n    );\n    const results = perFile.flat();\n\n    const summary: EvalRunSummary = { results };\n    const mlflow = await finalizeMlflow(mlflowClient, runId, results, options);\n    if (mlflow) summary.mlflow = mlflow;\n    return summary;\n  } finally {\n    teardownJudge();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAkFA,eAAe,SAAS,MAAuC;CAC7D,MAAM,SAAS;CACf,IAAI;AACJ,KAAI;AACF,GAAC,CAAE,YAAc,MAAM,OAAO;SAGxB;AACN,QAAM,IAAI,MACR,6FACD;;CAIH,MAAM,MAAM,mBADA,MAAM,SAAS,cAAc,KAAK,CAAC,MAAM,OAAO,KAAK,IAAI,CAClC;AACnC,KAAI,CAAC,IACH,OAAM,IAAI,MAAM,GAAG,KAAK,4CAA4C;AAEtE,QAAO;;;;;;;;AAST,SAAgB,mBAAmB,KAA0C;CAC3E,IAAI,YAAqB;AACzB,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,WAAW,KAAK;AACvC,MAAI,OAAQ,UAA6B,SAAS,WAChD,QAAO;AAET,cAAa,UAAoC;;;;;;;;;AAWrD,eAAe,OACb,GACA,IACA,KACA,KACA,OACA,SACqB;AACrB,KAAI;AAWF,SAAO,MAAM,QAAQ,KAAK;GAAE;GAAI,QAPjB,iBAAiB;IAC9B,SAAS,QAAQ;IACjB,OAAO,IAAI,SAAS,EAAE;IACtB,SAAS,QAAQ;IACjB,aAAa;IACb,WAAW,QAAQ;IACpB,CAAC;GACsC,QAAQ,QAAQ;GAAQ;GAAK,CAAC;UAC/D,KAAK;AACZ,SAAO;GACL;GACA,YAAY,EAAE;GACd,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACxD;;;;;;;;;AAUL,eAAsB,mBACpB,KACA,SACkE;AAClE,KAAI,CAAC,IAAI,QAAS,QAAO,EAAE,MAAM,CAAC,OAAU,EAAE;AAC9C,KAAI,CAAC,QAAQ,mBAAmB,CAAC,QAAQ,YACvC,QAAO;EACL,MAAM,CAAC,OAAU;EACjB,OACE;EACH;AAEH,KAAI;EACF,MAAM,OAAO,MAAM,gBAAgB,QAAQ,iBAAiB;GAC1D,OAAO,IAAI,QAAQ;GACnB,aAAa,QAAQ;GACrB,OAAO,IAAI,QAAQ;GACpB,CAAC;AACF,MAAI,KAAK,WAAW,EAClB,QAAO;GACL,MAAM,CAAC,OAAU;GACjB,OAAO,YAAY,IAAI,QAAQ,MAAM;GACtC;AAEH,SAAO,EAAE,MAAM;UACR,KAAK;AACZ,SAAO;GACL,MAAM,CAAC,OAAU;GACjB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACxD;;;;;;;;;;AAWL,eAAe,cACb,GACA,OACA,OACA,OACA,SACA,MACA,SACe;CACf,MAAM,KAAK,GAAG,EAAE,MAAM,GAAG,EAAE;CAE3B,IAAI;AACJ,KAAI;AACF,QAAM,MAAM,SAAS,EAAE,KAAK;UACrB,KAAK;AACZ,OAAK;GAAE,MAAM;GAAS;GAAI;GAAO;GAAO,CAAC;EACzC,MAAM,SAAqB;GACzB;GACA,YAAY,EAAE;GACd,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACxD;AACD,UAAQ,KAAK,OAAO;AACpB,OAAK;GAAE,MAAM;GAAU;GAAQ;GAAO;GAAO,CAAC;AAC9C;;CAGF,MAAM,EAAE,MAAM,OAAO,iBAAiB,MAAM,mBAAmB,KAAK,QAAQ;AAE5E,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,QACJ,IAAI,WAAW,KAAK,SAAS,IACzB,GAAG,GAAG,QAAQ,IAAI,EAAE,GAAG,KAAK,OAAO,KACnC;AACN,OAAK;GAAE,MAAM;GAAS,IAAI;GAAO;GAAO;GAAO,CAAC;EAChD,MAAM,SAAqB,eACvB;GAAE,IAAI;GAAO,YAAY,EAAE;GAAE,QAAQ;GAAO,OAAO;GAAc,GACjE,MAAM,OAAO,GAAG,OAAO,KAAK,KAAK,IAAI,OAAO,QAAQ;AACxD,UAAQ,KAAK,OAAO;AACpB,OAAK;GAAE,MAAM;GAAU;GAAQ;GAAO;GAAO,CAAC;;;;AAKlD,eAAe,oBAAoB,SAAyC;AAC1E,KAAI,CAAC,QAAQ,MAAO;AACpB,OAAM,eAAe;EACnB,QAAQ,IAAI,aAAa,QAAQ,MAAM,MAAM,QAAQ,MAAM,MAAM;EACjE,OAAO,QAAQ,MAAM;EACrB,OAAO,QAAQ,MAAM;EACtB,CAAC;;;;;;AAOJ,eAAe,eACb,QACA,OACA,SACA,SACmC;AACnC,KAAI,CAAC,UAAU,CAAC,MAAO,QAAO;CAG9B,IAAI,SAAwB;EAAE,SAAS;EAAG,SAAS;EAAG,UAAU,EAAE;EAAE;AACpE,KAAI;AACF,WAAS,MAAM,eACb,QACA,SACA,QAAQ,QAAQ,eACjB;UACM,KAAK;AACZ,SAAO,SAAS,KAAK;GACnB,SAAS;GACT,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACxD,CAAC;;CAEJ,MAAM,SAAS,MAAM,cAAc,QAAQ;EACzC;EACA;EACA,SAAS,QAAQ,OAAO,KAAK,KAAK;EACnC,CAAC;AACF,QAAO;EAAE;EAAO;EAAQ;EAAQ;;;;;;;AAQlC,MAAM,sBAAsB;;;;;;AAO5B,eAAsB,cACpB,SACyB;CACzB,MAAM,OAAO,QAAQ,WAAW,QAAQ,KAAK;CAC7C,MAAM,MAAM,QAAQ,OAAO,KAAK,KAAK;CACrC,IAAI,aAAa,kBAAkB,KAAK;AAExC,KAAI,QAAQ,QAAQ;EAClB,MAAM,IAAI,QAAQ;AAClB,eAAa,WAAW,QACrB,MAAM,EAAE,UAAU,KAAK,GAAG,EAAE,MAAM,GAAG,EAAE,KAAK,SAAS,EAAE,CACzD;;CAGH,MAAM,OAAO,QAAQ,kBAAkB;CACvC,MAAM,QAAQ,WAAW;AACzB,MAAK;EAAE,MAAM;EAAc;EAAO,CAAC;AAKnC,OAAM,oBAAoB,QAAQ;AAClC,KAAI;EAIF,IAAI;EACJ,IAAI;AACJ,MAAI,QAAQ,QAAQ;AAClB,kBAAe,IAAI,aACjB,QAAQ,OAAO,MACf,QAAQ,OAAO,MAChB;AACD,WAAQ,MAAM,cAAc,cAAc;IACxC,cAAc,QAAQ,OAAO;IAC7B,SAAS,eAAe,IAAI,KAAK,IAAI,CAAC,aAAa;IACnD,WAAW;IACZ,CAAC;AACF,QAAK;IAAE,MAAM;IAAe;IAAO,CAAC;;EAkBtC,MAAM,WATU,MAAM,QACpB,YACA,QAAQ,eAAe,qBACvB,OAAO,GAAG,UAAU;GAClB,MAAM,cAA4B,EAAE;AACpC,SAAM,cAAc,GAAG,OAAO,OAAO,OAAO,SAAS,MAAM,YAAY;AACvE,UAAO;IAEV,EACuB,MAAM;EAE9B,MAAM,UAA0B,EAAE,SAAS;EAC3C,MAAM,SAAS,MAAM,eAAe,cAAc,OAAO,SAAS,QAAQ;AAC1E,MAAI,OAAQ,SAAQ,SAAS;AAC7B,SAAO;WACC;AACR,iBAAe"}