@databricks/appkit 0.72.0 → 0.74.0

package/dist/evals/run-eval.js

@@ -12,6 +12,22 @@ var SkipSignal = class extends Error {
 	}
 };
 /**
+* Deep partial match: every key in `expected` is present in `actual` and equal,
+* recursing into nested plain objects so extra actual keys are ignored. Arrays
+* match element-for-element (same length, deep-equal items).
+*/
+function deepContains(actual, expected) {
+	if (Array.isArray(expected)) return Array.isArray(actual) && actual.length === expected.length && expected.every((item, i) => deepContains(actual[i], item));
+	if (isPlainObject(expected)) {
+		if (!isPlainObject(actual)) return false;
+		return Object.keys(expected).every((key) => Object.hasOwn(actual, key) && deepContains(actual[key], expected[key]));
+	}
+	return actual === expected;
+}
+function isPlainObject(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+/**
 * Run a single eval against a driver. Never throws for assertion or agent
 * failures — those become a non-passing {@link EvalResult}. Only a malformed
 * eval definition surfaces as `result.error`.
@@ -21,9 +37,12 @@ async function runEval(def, options) {
 	let reply = "";
 	let lastInput = "";
 	let toolCalls = [];
+	let toolCallDetails = [];
 	let sessionId;
 	let lastTraceId;
 	let lastSucceeded = false;
+	let turnFailed = false;
+	const controller = new AbortController();
 	const record = (label, pass, score, detail) => {
 		const result = {
 			label,
@@ -43,24 +62,28 @@ async function runEval(def, options) {
 				return handle;
 			},
 			atLeast(threshold) {
-				result.severity = "soft";
 				result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold;
 				return handle;
 			}
 		};
 		return handle;
 	};
-	const recordJudge = (label, score, rationale) => record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale).soft();
+	const recordJudge = (label, score, rationale) => record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale);
 	const t = {
 		async send(message) {
 			lastInput = message;
-			const r = await options.driver.send(message);
+			const r = await options.driver.send(message, { signal: controller.signal });
 			reply = r.reply;
 			toolCalls = r.toolCalls;
+			toolCallDetails = r.toolCallDetails;
 			sessionId = r.sessionId;
 			lastSucceeded = r.succeeded;
+			if (!r.succeeded) turnFailed = true;
 			if (r.traceId) lastTraceId = r.traceId;
 		},
+		reset() {
+			options.driver.reset?.();
+		},
 		get reply() {
 			return reply;
 		},
@@ -70,12 +93,24 @@ async function runEval(def, options) {
 		get sessionId() {
 			return sessionId;
 		},
+		get input() {
+			return options.row?.inputs ?? {};
+		},
+		get expected() {
+			return options.row?.expectations;
+		},
 		succeeded() {
 			return record("succeeded", lastSucceeded, void 0, lastSucceeded ? void 0 : "agent turn did not complete successfully");
 		},
 		calledTool(name) {
 			return record(`calledTool(${name})`, toolCalls.includes(name), void 0, `expected tool "${name}" to be called (called: ${toolCalls.length ? toolCalls.join(", ") : "none"})`);
 		},
+		calledToolWith(name, expected) {
+			const matching = toolCallDetails.filter((c) => c.name === name);
+			const pass = matching.some((c) => deepContains(c.args, expected));
+			const seen = matching.length ? matching.map((c) => `{${Object.keys(c.args).sort().join(", ")}}`).join(", ") : "not called";
+			return record(`calledToolWith(${name})`, pass, void 0, `expected tool "${name}" to be called with ${JSON.stringify(expected)} (arg keys seen: ${seen})`);
+		},
 		check(value, matcher) {
 			const m = matcher(value);
 			return record("check", m.pass, m.score, m.detail);
@@ -109,8 +144,19 @@ async function runEval(def, options) {
 			throw new SkipSignal(reason);
 		}
 	};
+	const timeoutMs = def.timeoutMs ?? options.timeoutMs;
+	let timer;
 	try {
-		await def.test(t);
+		if (timeoutMs === void 0) await def.test(t);
+		else {
+			const timeout = new Promise((_, reject) => {
+				timer = setTimeout(() => {
+					controller.abort();
+					reject(/* @__PURE__ */ new Error(`eval timed out after ${timeoutMs}ms`));
+				}, timeoutMs);
+			});
+			await Promise.race([Promise.resolve(def.test(t)), timeout]);
+		}
 	} catch (err) {
 		if (err instanceof SkipSignal) return {
 			id: options.id,
@@ -128,6 +174,8 @@ async function runEval(def, options) {
 			error: err instanceof Error ? err.message : String(err),
 			traceId: lastTraceId
 		};
+	} finally {
+		if (timer) clearTimeout(timer);
 	}
 	const passed = assertions.every((a) => a.pass || a.severity === "soft" && !options.strict);
 	return {
@@ -135,7 +183,8 @@ async function runEval(def, options) {
 		description: def.description,
 		assertions,
 		passed,
-		traceId: lastTraceId
+		traceId: lastTraceId,
+		infraFailure: turnFailed && !passed || void 0
 	};
 }
 

package/dist/evals/run-eval.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-eval.js","names":[],"sources":["../../src/evals/run-eval.ts"],"sourcesContent":["import { judgeClosedQA, judgeCustom, judgeFactuality } from \"./judge\";\nimport type {\n  AssertionHandle,\n  AssertionResult,\n  EvalDefinition,\n  EvalDriver,\n  EvalResult,\n  Matcher,\n  TestContext,\n} from \"./types\";\n\n/** Default pass threshold for an LLM-judge score (0..1) before `.atLeast()`. */\nconst DEFAULT_JUDGE_THRESHOLD = 0.5;\n\n/** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */\nclass SkipSignal extends Error {\n  constructor(public reason?: string) {\n    super(\"eval skipped\");\n    this.name = \"SkipSignal\";\n  }\n}\n\nexport interface RunEvalOptions {\n  /** Stable id for the eval (e.g. its file path relative to the evals dir). */\n  id: string;\n  /** Drives the agent and returns reply/tool-calls/success per `send`. */\n  driver: EvalDriver;\n  /** When true, soft assertion failures also fail the eval. */\n  strict?: boolean;\n}\n\n/**\n * Run a single eval against a driver. Never throws for assertion or agent\n * failures — those become a non-passing {@link EvalResult}. Only a malformed\n * eval definition surfaces as `result.error`.\n */\nexport async function runEval(\n  def: EvalDefinition,\n  options: RunEvalOptions,\n): Promise<EvalResult> {\n  const assertions: AssertionResult[] = [];\n  let reply = \"\";\n  let lastInput = \"\";\n  let toolCalls: string[] = [];\n  let sessionId: string | undefined;\n  let lastTraceId: string | undefined;\n  let lastSucceeded = false;\n\n  const record = (\n    label: string,\n    pass: boolean,\n    score?: number,\n    detail?: string,\n  ): AssertionHandle => {\n    const result: AssertionResult = {\n      label,\n      severity: \"gate\",\n      pass,\n      score,\n      detail,\n    };\n    assertions.push(result);\n    const handle: AssertionHandle = {\n      gate() {\n        result.severity = \"gate\";\n        return handle;\n      },\n      soft() {\n        result.severity = \"soft\";\n        return handle;\n      },\n      atLeast(threshold: number) {\n        result.severity = \"soft\";\n        result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold;\n        return handle;\n      },\n    };\n    return handle;\n  };\n\n  // LLM-judge assertions are scored and soft by default; the caller chains\n  // `.atLeast(n)` to set the pass threshold or `.gate()` to promote.\n  const recordJudge = (\n    label: string,\n    score: number,\n    rationale?: string,\n  ): AssertionHandle =>\n    record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale).soft();\n\n  const t: TestContext = {\n    async send(message) {\n      lastInput = message;\n      const r = await options.driver.send(message);\n      reply = r.reply;\n      toolCalls = r.toolCalls;\n      sessionId = r.sessionId;\n      lastSucceeded = r.succeeded;\n      if (r.traceId) lastTraceId = r.traceId;\n    },\n    get reply() {\n      return reply;\n    },\n    get toolCalls() {\n      return toolCalls;\n    },\n    get sessionId() {\n      return sessionId;\n    },\n    succeeded() {\n      return record(\n        \"succeeded\",\n        lastSucceeded,\n        undefined,\n        lastSucceeded ? undefined : \"agent turn did not complete successfully\",\n      );\n    },\n    calledTool(name) {\n      return record(\n        `calledTool(${name})`,\n        toolCalls.includes(name),\n        undefined,\n        `expected tool \"${name}\" to be called (called: ${\n          toolCalls.length ? toolCalls.join(\", \") : \"none\"\n        })`,\n      );\n    },\n    check(value: string, matcher: Matcher) {\n      const m = matcher(value);\n      return record(\"check\", m.pass, m.score, m.detail);\n    },\n    judge: {\n      async factuality(expected) {\n        const { score, rationale } = await judgeFactuality({\n          input: lastInput,\n          output: reply,\n          expected,\n        });\n        return recordJudge(\"judge.factuality\", score, rationale);\n      },\n      async closedQA(criteria) {\n        const { score, rationale } = await judgeClosedQA({\n          input: lastInput,\n          output: reply,\n          criteria,\n        });\n        return recordJudge(\"judge.closedQA\", score, rationale);\n      },\n      async custom(spec) {\n        const { score, rationale } = await judgeCustom(spec, {\n          input: lastInput,\n          output: reply,\n        });\n        return recordJudge(`judge.${spec.name}`, score, rationale);\n      },\n    },\n    skip(reason) {\n      throw new SkipSignal(reason);\n    },\n  };\n\n  try {\n    await def.test(t);\n  } catch (err) {\n    if (err instanceof SkipSignal) {\n      return {\n        id: options.id,\n        description: def.description,\n        skipped: { reason: err.reason },\n        assertions,\n        passed: true,\n        traceId: lastTraceId,\n      };\n    }\n    return {\n      id: options.id,\n      description: def.description,\n      assertions,\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n      traceId: lastTraceId,\n    };\n  }\n\n  const passed = assertions.every(\n    (a) => a.pass || (a.severity === \"soft\" && !options.strict),\n  );\n\n  return {\n    id: options.id,\n    description: def.description,\n    assertions,\n    passed,\n    traceId: lastTraceId,\n  };\n}\n"],"mappings":";;;;AAYA,MAAM,0BAA0B;;AAGhC,IAAM,aAAN,cAAyB,MAAM;CAC7B,YAAY,AAAO,QAAiB;AAClC,QAAM,eAAe;EADJ;AAEjB,OAAK,OAAO;;;;;;;;AAkBhB,eAAsB,QACpB,KACA,SACqB;CACrB,MAAM,aAAgC,EAAE;CACxC,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,YAAsB,EAAE;CAC5B,IAAI;CACJ,IAAI;CACJ,IAAI,gBAAgB;CAEpB,MAAM,UACJ,OACA,MACA,OACA,WACoB;EACpB,MAAM,SAA0B;GAC9B;GACA,UAAU;GACV;GACA;GACA;GACD;AACD,aAAW,KAAK,OAAO;EACvB,MAAM,SAA0B;GAC9B,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,QAAQ,WAAmB;AACzB,WAAO,WAAW;AAClB,WAAO,QAAQ,OAAO,UAAU,OAAO,OAAO,IAAI,OAAO;AACzD,WAAO;;GAEV;AACD,SAAO;;CAKT,MAAM,eACJ,OACA,OACA,cAEA,OAAO,OAAO,SAAS,yBAAyB,OAAO,UAAU,CAAC,MAAM;CAE1E,MAAM,IAAiB;EACrB,MAAM,KAAK,SAAS;AAClB,eAAY;GACZ,MAAM,IAAI,MAAM,QAAQ,OAAO,KAAK,QAAQ;AAC5C,WAAQ,EAAE;AACV,eAAY,EAAE;AACd,eAAY,EAAE;AACd,mBAAgB,EAAE;AAClB,OAAI,EAAE,QAAS,eAAc,EAAE;;EAEjC,IAAI,QAAQ;AACV,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,YAAY;AACV,UAAO,OACL,aACA,eACA,QACA,gBAAgB,SAAY,2CAC7B;;EAEH,WAAW,MAAM;AACf,UAAO,OACL,cAAc,KAAK,IACnB,UAAU,SAAS,KAAK,EACxB,QACA,kBAAkB,KAAK,0BACrB,UAAU,SAAS,UAAU,KAAK,KAAK,GAAG,OAC3C,GACF;;EAEH,MAAM,OAAe,SAAkB;GACrC,MAAM,IAAI,QAAQ,MAAM;AACxB,UAAO,OAAO,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;;EAEnD,OAAO;GACL,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,OAAO,cAAc,MAAM,gBAAgB;KACjD,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,oBAAoB,OAAO,UAAU;;GAE1D,MAAM,SAAS,UAAU;IACvB,MAAM,EAAE,OAAO,cAAc,MAAM,cAAc;KAC/C,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,kBAAkB,OAAO,UAAU;;GAExD,MAAM,OAAO,MAAM;IACjB,MAAM,EAAE,OAAO,cAAc,MAAM,YAAY,MAAM;KACnD,OAAO;KACP,QAAQ;KACT,CAAC;AACF,WAAO,YAAY,SAAS,KAAK,QAAQ,OAAO,UAAU;;GAE7D;EACD,KAAK,QAAQ;AACX,SAAM,IAAI,WAAW,OAAO;;EAE/B;AAED,KAAI;AACF,QAAM,IAAI,KAAK,EAAE;UACV,KAAK;AACZ,MAAI,eAAe,WACjB,QAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB,SAAS,EAAE,QAAQ,IAAI,QAAQ;GAC/B;GACA,QAAQ;GACR,SAAS;GACV;AAEH,SAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB;GACA,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACvD,SAAS;GACV;;CAGH,MAAM,SAAS,WAAW,OACvB,MAAM,EAAE,QAAS,EAAE,aAAa,UAAU,CAAC,QAAQ,OACrD;AAED,QAAO;EACL,IAAI,QAAQ;EACZ,aAAa,IAAI;EACjB;EACA;EACA,SAAS;EACV"}
+{"version":3,"file":"run-eval.js","names":[],"sources":["../../src/evals/run-eval.ts"],"sourcesContent":["import type { DatasetRow } from \"./dataset\";\nimport { judgeClosedQA, judgeCustom, judgeFactuality } from \"./judge\";\nimport type {\n  AssertionHandle,\n  AssertionResult,\n  DriveResult,\n  EvalDefinition,\n  EvalDriver,\n  EvalResult,\n  Matcher,\n  TestContext,\n} from \"./types\";\n\n/** Default pass threshold for an LLM-judge score (0..1) before `.atLeast()`. */\nconst DEFAULT_JUDGE_THRESHOLD = 0.5;\n\n/** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */\nclass SkipSignal extends Error {\n  constructor(public reason?: string) {\n    super(\"eval skipped\");\n    this.name = \"SkipSignal\";\n  }\n}\n\n/**\n * Deep partial match: every key in `expected` is present in `actual` and equal,\n * recursing into nested plain objects so extra actual keys are ignored. Arrays\n * match element-for-element (same length, deep-equal items).\n */\nfunction deepContains(actual: unknown, expected: unknown): boolean {\n  if (Array.isArray(expected)) {\n    return (\n      Array.isArray(actual) &&\n      actual.length === expected.length &&\n      expected.every((item, i) => deepContains(actual[i], item))\n    );\n  }\n  if (isPlainObject(expected)) {\n    if (!isPlainObject(actual)) return false;\n    // Require the key present: an expected `undefined` must not match an omitted key.\n    return Object.keys(expected).every(\n      (key) =>\n        Object.hasOwn(actual, key) && deepContains(actual[key], expected[key]),\n    );\n  }\n  return actual === expected;\n}\n\nfunction isPlainObject(value: unknown): value is Record<string, unknown> {\n  return typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n\nexport interface RunEvalOptions {\n  /** Stable id for the eval (e.g. its file path relative to the evals dir). */\n  id: string;\n  /** Drives the agent and returns reply/tool-calls/success per `send`. */\n  driver: EvalDriver;\n  /** When true, soft assertion failures also fail the eval. */\n  strict?: boolean;\n  /** Dataset row bound to `t.input`/`t.expected` for dataset-driven evals. */\n  row?: DatasetRow;\n  /**\n   * Runner-level default per-eval timeout (ms). `def.timeoutMs` wins over this;\n   * when both are unset the eval runs unbounded (current behavior).\n   */\n  timeoutMs?: number;\n}\n\n/**\n * Run a single eval against a driver. Never throws for assertion or agent\n * failures — those become a non-passing {@link EvalResult}. Only a malformed\n * eval definition surfaces as `result.error`.\n */\nexport async function runEval(\n  def: EvalDefinition,\n  options: RunEvalOptions,\n): Promise<EvalResult> {\n  const assertions: AssertionResult[] = [];\n  let reply = \"\";\n  let lastInput = \"\";\n  let toolCalls: string[] = [];\n  let toolCallDetails: DriveResult[\"toolCallDetails\"] = [];\n  let sessionId: string | undefined;\n  let lastTraceId: string | undefined;\n  let lastSucceeded = false;\n  // Any transport/agent turn failure (driver `succeeded: false`) → `infraFailure`, for retry.\n  let turnFailed = false;\n  // Aborted on per-eval timeout to cancel the in-flight turn (no leaked stream).\n  const controller = new AbortController();\n\n  const record = (\n    label: string,\n    pass: boolean,\n    score?: number,\n    detail?: string,\n  ): AssertionHandle => {\n    const result: AssertionResult = {\n      label,\n      severity: \"gate\",\n      pass,\n      score,\n      detail,\n    };\n    assertions.push(result);\n    const handle: AssertionHandle = {\n      gate() {\n        result.severity = \"gate\";\n        return handle;\n      },\n      soft() {\n        result.severity = \"soft\";\n        return handle;\n      },\n      atLeast(threshold: number) {\n        // Re-threshold the score; keep the current severity (gate unless the\n        // caller also chained `.soft()`).\n        result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold;\n        return handle;\n      },\n    };\n    return handle;\n  };\n\n  // LLM-judge assertions are scored and gate by default (a miss fails the eval,\n  // like other assertions). The caller chains `.atLeast(n)` to change the pass\n  // threshold, or `.soft()` to demote to a tracked-only metric.\n  const recordJudge = (\n    label: string,\n    score: number,\n    rationale?: string,\n  ): AssertionHandle =>\n    record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale);\n\n  const t: TestContext = {\n    async send(message) {\n      lastInput = message;\n      const r = await options.driver.send(message, {\n        signal: controller.signal,\n      });\n      reply = r.reply;\n      toolCalls = r.toolCalls;\n      toolCallDetails = r.toolCallDetails;\n      sessionId = r.sessionId;\n      lastSucceeded = r.succeeded;\n      if (!r.succeeded) turnFailed = true;\n      if (r.traceId) lastTraceId = r.traceId;\n    },\n    reset() {\n      options.driver.reset?.();\n    },\n    get reply() {\n      return reply;\n    },\n    get toolCalls() {\n      return toolCalls;\n    },\n    get sessionId() {\n      return sessionId;\n    },\n    get input() {\n      return options.row?.inputs ?? {};\n    },\n    get expected() {\n      return options.row?.expectations;\n    },\n    succeeded() {\n      return record(\n        \"succeeded\",\n        lastSucceeded,\n        undefined,\n        lastSucceeded ? undefined : \"agent turn did not complete successfully\",\n      );\n    },\n    calledTool(name) {\n      return record(\n        `calledTool(${name})`,\n        toolCalls.includes(name),\n        undefined,\n        `expected tool \"${name}\" to be called (called: ${\n          toolCalls.length ? toolCalls.join(\", \") : \"none\"\n        })`,\n      );\n    },\n    calledToolWith(name, expected) {\n      const matching = toolCallDetails.filter((c) => c.name === name);\n      const pass = matching.some((c) => deepContains(c.args, expected));\n      // Keys only, never values: actual args may hold PII/secrets and are persisted to reports (CWE-532).\n      const seen = matching.length\n        ? matching\n            .map((c) => `{${Object.keys(c.args).sort().join(\", \")}}`)\n            .join(\", \")\n        : \"not called\";\n      return record(\n        `calledToolWith(${name})`,\n        pass,\n        undefined,\n        `expected tool \"${name}\" to be called with ${JSON.stringify(\n          expected,\n        )} (arg keys seen: ${seen})`,\n      );\n    },\n    check(value: string, matcher: Matcher) {\n      const m = matcher(value);\n      return record(\"check\", m.pass, m.score, m.detail);\n    },\n    judge: {\n      async factuality(expected) {\n        const { score, rationale } = await judgeFactuality({\n          input: lastInput,\n          output: reply,\n          expected,\n        });\n        return recordJudge(\"judge.factuality\", score, rationale);\n      },\n      async closedQA(criteria) {\n        const { score, rationale } = await judgeClosedQA({\n          input: lastInput,\n          output: reply,\n          criteria,\n        });\n        return recordJudge(\"judge.closedQA\", score, rationale);\n      },\n      async custom(spec) {\n        const { score, rationale } = await judgeCustom(spec, {\n          input: lastInput,\n          output: reply,\n        });\n        return recordJudge(`judge.${spec.name}`, score, rationale);\n      },\n    },\n    skip(reason) {\n      throw new SkipSignal(reason);\n    },\n  };\n\n  // `def.timeoutMs` (per-eval) wins over the runner default; when both are\n  // unset the eval runs unbounded (undefined = no timeout).\n  const timeoutMs = def.timeoutMs ?? options.timeoutMs;\n  let timer: ReturnType<typeof setTimeout> | undefined;\n\n  try {\n    if (timeoutMs === undefined) {\n      await def.test(t);\n    } else {\n      // Race the test against the timeout; on elapse, abort the turn and settle\n      // non-passing. Only the driver turn cancels — non-driver hangs (judge, sleep) run on.\n      const timeout = new Promise<never>((_, reject) => {\n        timer = setTimeout(() => {\n          controller.abort();\n          reject(new Error(`eval timed out after ${timeoutMs}ms`));\n        }, timeoutMs);\n      });\n      await Promise.race([Promise.resolve(def.test(t)), timeout]);\n    }\n  } catch (err) {\n    if (err instanceof SkipSignal) {\n      return {\n        id: options.id,\n        description: def.description,\n        skipped: { reason: err.reason },\n        assertions,\n        passed: true,\n        traceId: lastTraceId,\n      };\n    }\n    return {\n      id: options.id,\n      description: def.description,\n      assertions,\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n      traceId: lastTraceId,\n    };\n  } finally {\n    if (timer) clearTimeout(timer);\n  }\n\n  const passed = assertions.every(\n    (a) => a.pass || (a.severity === \"soft\" && !options.strict),\n  );\n\n  return {\n    id: options.id,\n    description: def.description,\n    assertions,\n    passed,\n    traceId: lastTraceId,\n    // Only a *failing* eval whose turn broke at the transport/agent level is a\n    // retryable infra flake; a passing eval (or a pure assertion mismatch, where\n    // the turn itself succeeded) is real signal and must not be retried.\n    infraFailure: (turnFailed && !passed) || undefined,\n  };\n}\n"],"mappings":";;;;AAcA,MAAM,0BAA0B;;AAGhC,IAAM,aAAN,cAAyB,MAAM;CAC7B,YAAY,AAAO,QAAiB;AAClC,QAAM,eAAe;EADJ;AAEjB,OAAK,OAAO;;;;;;;;AAShB,SAAS,aAAa,QAAiB,UAA4B;AACjE,KAAI,MAAM,QAAQ,SAAS,CACzB,QACE,MAAM,QAAQ,OAAO,IACrB,OAAO,WAAW,SAAS,UAC3B,SAAS,OAAO,MAAM,MAAM,aAAa,OAAO,IAAI,KAAK,CAAC;AAG9D,KAAI,cAAc,SAAS,EAAE;AAC3B,MAAI,CAAC,cAAc,OAAO,CAAE,QAAO;AAEnC,SAAO,OAAO,KAAK,SAAS,CAAC,OAC1B,QACC,OAAO,OAAO,QAAQ,IAAI,IAAI,aAAa,OAAO,MAAM,SAAS,KAAK,CACzE;;AAEH,QAAO,WAAW;;AAGpB,SAAS,cAAc,OAAkD;AACvE,QAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,MAAM;;;;;;;AAwB7E,eAAsB,QACpB,KACA,SACqB;CACrB,MAAM,aAAgC,EAAE;CACxC,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,YAAsB,EAAE;CAC5B,IAAI,kBAAkD,EAAE;CACxD,IAAI;CACJ,IAAI;CACJ,IAAI,gBAAgB;CAEpB,IAAI,aAAa;CAEjB,MAAM,aAAa,IAAI,iBAAiB;CAExC,MAAM,UACJ,OACA,MACA,OACA,WACoB;EACpB,MAAM,SAA0B;GAC9B;GACA,UAAU;GACV;GACA;GACA;GACD;AACD,aAAW,KAAK,OAAO;EACvB,MAAM,SAA0B;GAC9B,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,QAAQ,WAAmB;AAGzB,WAAO,QAAQ,OAAO,UAAU,OAAO,OAAO,IAAI,OAAO;AACzD,WAAO;;GAEV;AACD,SAAO;;CAMT,MAAM,eACJ,OACA,OACA,cAEA,OAAO,OAAO,SAAS,yBAAyB,OAAO,UAAU;CAEnE,MAAM,IAAiB;EACrB,MAAM,KAAK,SAAS;AAClB,eAAY;GACZ,MAAM,IAAI,MAAM,QAAQ,OAAO,KAAK,SAAS,EAC3C,QAAQ,WAAW,QACpB,CAAC;AACF,WAAQ,EAAE;AACV,eAAY,EAAE;AACd,qBAAkB,EAAE;AACpB,eAAY,EAAE;AACd,mBAAgB,EAAE;AAClB,OAAI,CAAC,EAAE,UAAW,cAAa;AAC/B,OAAI,EAAE,QAAS,eAAc,EAAE;;EAEjC,QAAQ;AACN,WAAQ,OAAO,SAAS;;EAE1B,IAAI,QAAQ;AACV,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,QAAQ;AACV,UAAO,QAAQ,KAAK,UAAU,EAAE;;EAElC,IAAI,WAAW;AACb,UAAO,QAAQ,KAAK;;EAEtB,YAAY;AACV,UAAO,OACL,aACA,eACA,QACA,gBAAgB,SAAY,2CAC7B;;EAEH,WAAW,MAAM;AACf,UAAO,OACL,cAAc,KAAK,IACnB,UAAU,SAAS,KAAK,EACxB,QACA,kBAAkB,KAAK,0BACrB,UAAU,SAAS,UAAU,KAAK,KAAK,GAAG,OAC3C,GACF;;EAEH,eAAe,MAAM,UAAU;GAC7B,MAAM,WAAW,gBAAgB,QAAQ,MAAM,EAAE,SAAS,KAAK;GAC/D,MAAM,OAAO,SAAS,MAAM,MAAM,aAAa,EAAE,MAAM,SAAS,CAAC;GAEjE,MAAM,OAAO,SAAS,SAClB,SACG,KAAK,MAAM,IAAI,OAAO,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,KAAK,CAAC,GAAG,CACxD,KAAK,KAAK,GACb;AACJ,UAAO,OACL,kBAAkB,KAAK,IACvB,MACA,QACA,kBAAkB,KAAK,sBAAsB,KAAK,UAChD,SACD,CAAC,mBAAmB,KAAK,GAC3B;;EAEH,MAAM,OAAe,SAAkB;GACrC,MAAM,IAAI,QAAQ,MAAM;AACxB,UAAO,OAAO,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;;EAEnD,OAAO;GACL,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,OAAO,cAAc,MAAM,gBAAgB;KACjD,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,oBAAoB,OAAO,UAAU;;GAE1D,MAAM,SAAS,UAAU;IACvB,MAAM,EAAE,OAAO,cAAc,MAAM,cAAc;KAC/C,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,kBAAkB,OAAO,UAAU;;GAExD,MAAM,OAAO,MAAM;IACjB,MAAM,EAAE,OAAO,cAAc,MAAM,YAAY,MAAM;KACnD,OAAO;KACP,QAAQ;KACT,CAAC;AACF,WAAO,YAAY,SAAS,KAAK,QAAQ,OAAO,UAAU;;GAE7D;EACD,KAAK,QAAQ;AACX,SAAM,IAAI,WAAW,OAAO;;EAE/B;CAID,MAAM,YAAY,IAAI,aAAa,QAAQ;CAC3C,IAAI;AAEJ,KAAI;AACF,MAAI,cAAc,OAChB,OAAM,IAAI,KAAK,EAAE;OACZ;GAGL,MAAM,UAAU,IAAI,SAAgB,GAAG,WAAW;AAChD,YAAQ,iBAAiB;AACvB,gBAAW,OAAO;AAClB,4BAAO,IAAI,MAAM,wBAAwB,UAAU,IAAI,CAAC;OACvD,UAAU;KACb;AACF,SAAM,QAAQ,KAAK,CAAC,QAAQ,QAAQ,IAAI,KAAK,EAAE,CAAC,EAAE,QAAQ,CAAC;;UAEtD,KAAK;AACZ,MAAI,eAAe,WACjB,QAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB,SAAS,EAAE,QAAQ,IAAI,QAAQ;GAC/B;GACA,QAAQ;GACR,SAAS;GACV;AAEH,SAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB;GACA,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACvD,SAAS;GACV;WACO;AACR,MAAI,MAAO,cAAa,MAAM;;CAGhC,MAAM,SAAS,WAAW,OACvB,MAAM,EAAE,QAAS,EAAE,aAAa,UAAU,CAAC,QAAQ,OACrD;AAED,QAAO;EACL,IAAI,QAAQ;EACZ,aAAa,IAAI;EACjB;EACA;EACA,SAAS;EAIT,cAAe,cAAc,CAAC,UAAW;EAC1C"}

package/dist/evals/run-evals.d.ts

@@ -1,3 +1,5 @@
+import { WorkspaceClient } from "../workspace-client/index.js";
+import "./dataset.js";
 import { EvalResult } from "./types.js";
 import { ReportOutcome } from "./mlflow-report.js";
 import { FinishOutcome } from "./mlflow-run.js";
@@ -10,12 +12,15 @@ interface RunEvalsOptions {
   baseUrl: string;
   /** Substring filter on `<agent>/<id>` (or an exact agent id). */
   filter?: string;
+  /**
+   * Only run evals whose `tags` intersect this list. Empty/undefined runs all.
+   * Tags live on the eval def, so filtering happens after each file is loaded.
+   */
+  tags?: string[];
   /** Soft assertion failures also fail the eval. */
   strict?: boolean;
   /** Extra request headers for the driver (e.g. auth for a deployed app). */
   headers?: Record<string, string>;
-  /** Per-turn wall-clock timeout (ms) before a turn is failed. Defaults to 120s. */
-  timeoutMs?: number;
   /**
    * Max evals to drive concurrently. Each eval opens one stream to the app as
    * the same user, so keep this at or below the app's
@@ -43,8 +48,27 @@ interface RunEvalsOptions {
     token: string;
     model: string;
   };
+  /**
+   * Workspace client used to read managed evaluation datasets (for evals that
+   * declare `dataset`). Required alongside {@link warehouseId} for those evals.
+   */
+  workspaceClient?: WorkspaceClient;
+  /** SQL warehouse id used to read managed evaluation datasets. */
+  warehouseId?: string;
   /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */
   now?: number;
+  /**
+   * Default per-eval timeout (ms): `runEval` races the whole test against it and
+   * it also caps each driver turn. A per-eval `def.timeoutMs` overrides it, and
+   * it wins over an agent's `evals.config.ts` `timeoutMs`. Unbounded when unset.
+   */
+  timeoutMs?: number;
+  /**
+   * Re-run an eval up to this many extra times when it fails on infrastructure —
+   * a thrown error/timeout (`result.error`) or a transport/agent turn failure
+   * (`result.infraFailure`). Assertion failures are never retried. Defaults to `0`.
+   */
+  retries?: number;
   /** Progress callback, invoked as evals are discovered, started, and finished. */
   onEvent?: (event: EvalProgress) => void;
 }
@@ -74,6 +98,20 @@ interface EvalRunSummary {
     finish: FinishOutcome;
   };
 }
+/**
+ * Run `attempt` up to `1 + retries` times, stopping as soon as it returns a
+ * result that is neither a thrown error / per-eval timeout (`error`) nor a
+ * transport/agent turn failure (`infraFailure`). Assertion failures set
+ * neither, so a failed-but-completed eval is returned on the first try and
+ * never retried. Returns the last result when every attempt failed on infra.
+ *
+ * Between attempts it waits a full-jittered exponential backoff (infra flakes
+ * are overload-correlated). `retries` is coerced to a finite non-negative
+ * integer; `baseDelayMs: 0` disables the wait (tests).
+ */
+declare function runWithRetries(retries: number, attempt: (attemptNumber: number) => Promise<EvalResult>, options?: {
+  baseDelayMs?: number;
+}): Promise<EvalResult>;
 /**
  * Discover, load, and run every eval under each agent's `evals/` dir, driving
  * the agents on a running app. Never throws for an individual eval — load/run
@@ -81,5 +119,5 @@ interface EvalRunSummary {
  */
 declare function runEvalsInDir(options: RunEvalsOptions): Promise<EvalRunSummary>;
 //#endregion
-export { EvalProgress, EvalRunSummary, RunEvalsOptions, runEvalsInDir };
+export { EvalProgress, EvalRunSummary, RunEvalsOptions, runEvalsInDir, runWithRetries };
 //# sourceMappingURL=run-evals.d.ts.map

package/dist/evals/run-evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-evals.d.ts","names":[],"sources":["../../src/evals/run-evals.ts"],"mappings":";;;;;UAYiB,eAAA;;EAEf,OAAA;EAF8B;EAI9B,OAAA;EAoC8B;EAlC9B,MAAA;EAFA;EAIA,MAAA;EAAA;EAEA,OAAA,GAAU,MAAA;EAAA;EAEV,SAAA;EAOA;;;;;;EAAA,WAAA;EAiBU;;;;;EAXV,MAAA;IACE,IAAA;IACA,KAAA;IACA,YAAA,UAeQ;IAbR,cAAA;EAAA;EAiBoC;;;;EAXtC,KAAA;IAAU,IAAA;IAAc,KAAA;IAAe,KAAA;EAAA;EAWnC;EATJ,GAAA;EAS4B;EAP5B,OAAA,IAAW,KAAA,EAAO,YAAA;AAAA;AAAA,KAGR,YAAA;EACN,IAAA;EAAoB,KAAA;AAAA;EACpB,IAAA;EAAqB,KAAA;AAAA;EACrB,IAAA;EAAe,EAAA;EAAY,KAAA;EAAe,KAAA;AAAA;EAC1C,IAAA;EAAgB,MAAA,EAAQ,UAAA;EAAY,KAAA;EAAe,KAAA;AAAA;AAAA,UAExC,cAAA;EACf,OAAA,EAAS,UAAA;EAE6D;EAAtE,MAAA;IAAW,KAAA;IAAe,MAAA,EAAQ,aAAA;IAAe,MAAA,EAAQ,aAAA;EAAA;AAAA;;;;;;iBAqIrC,aAAA,CACpB,OAAA,EAAS,eAAA,GACR,OAAA,CAAQ,cAAA"}
+{"version":3,"file":"run-evals.d.ts","names":[],"sources":["../../src/evals/run-evals.ts"],"mappings":";;;;;;;UAmBiB,eAAA;;EAEf,OAAA;EAF8B;EAI9B,OAAA;EAWU;EATV,MAAA;EAwDkB;;;;EAnDlB,IAAA;EALA;EAOA,MAAA;EAAA;EAEA,OAAA,GAAU,MAAA;EAAA;;;;;;EAOV,WAAA;EAiBA;;;;;EAXA,MAAA;IACE,IAAA;IACA,KAAA;IACA,YAAA,UA6BF;IA3BE,cAAA;EAAA;EA6BS;;;AAGb;EA1BE,KAAA;IAAU,IAAA;IAAc,KAAA;IAAe,KAAA;EAAA;EA4BnC;;;;EAvBJ,eAAA,GAAkB,eAAA;EAwB4B;EAtB9C,WAAA;EAuBoB;EArBpB,GAAA;EAqBwC;;;;AAE1C;EAjBE,SAAA;;;;;;EAMA,OAAA;EAYA;EAVA,OAAA,IAAW,KAAA,EAAO,YAAA;AAAA;AAAA,KAGR,YAAA;EACN,IAAA;EAAoB,KAAA;AAAA;EACpB,IAAA;EAAqB,KAAA;AAAA;EACrB,IAAA;EAAe,EAAA;EAAY,KAAA;EAAe,KAAA;AAAA;EAC1C,IAAA;EAAgB,MAAA,EAAQ,UAAA;EAAY,KAAA;EAAe,KAAA;AAAA;AAAA,UAExC,cAAA;EACf,OAAA,EAAS,UAAA;EA2OmC;EAzO5C,MAAA;IAAW,KAAA;IAAe,MAAA,EAAQ,aAAA;IAAe,MAAA,EAAQ,aAAA;EAAA;AAAA;;;;;;;;;;;;iBAuOrC,cAAA,CACpB,OAAA,UACA,OAAA,GAAU,aAAA,aAA0B,OAAA,CAAQ,UAAA,GAC5C,OAAA;EAAW,WAAA;AAAA,IACV,OAAA,CAAQ,UAAA;;;;;;iBA8GW,aAAA,CACpB,OAAA,EAAS,eAAA,GACR,OAAA,CAAQ,cAAA"}

package/dist/evals/run-evals.js

@@ -1,5 +1,6 @@
 import { MlflowClient } from "../connectors/mlflow/client.js";
-import { discoverEvalFiles } from "./discover.js";
+import { readEvalDataset } from "./dataset.js";
+import { discoverEvalConfigs, discoverEvalFiles } from "./discover.js";
 import { createHttpDriver } from "./http-driver.js";
 import { configureJudge, teardownJudge } from "./judge.js";
 import { mapPool } from "./pool.js";
@@ -7,15 +8,15 @@ import { reportToMlflow } from "./mlflow-report.js";
 import { runEval } from "./run-eval.js";
 import { createEvalRun, finishEvalRun } from "./mlflow-run.js";
 import { pathToFileURL } from "node:url";
+import { setTimeout } from "node:timers/promises";
 
 //#region src/evals/run-evals.ts
 /**
-* Load a `*.eval.ts` file and return its default-exported {@link EvalDefinition}.
-* Uses tsx's programmatic loader so TypeScript eval files run without a build
-* step. The specifier is indirected so the type checker doesn't try to resolve
-* tsx's internal entry.
+* Import a TypeScript file with tsx's programmatic loader so eval files run
+* without a build step. The specifier is indirected so the type checker doesn't
+* try to resolve tsx's internal entry.
 */
-async function loadEval(file) {
+async function tsImportFile(file) {
 	const tsxApi = "tsx/esm/api";
 	let tsImport;
 	try {
@@ -23,11 +24,38 @@ async function loadEval(file) {
 	} catch {
 		throw new Error("Running .eval.ts files requires `tsx`. Install it as a dev dependency (`pnpm add -D tsx`).");
 	}
-	const def = resolveEvalDefault(await tsImport(pathToFileURL(file).href, import.meta.url));
+	return tsImport(pathToFileURL(file).href, import.meta.url);
+}
+/**
+* Load a `*.eval.ts` file and return its default-exported {@link EvalDefinition}.
+*/
+async function loadEval(file) {
+	const def = resolveEvalDefault(await tsImportFile(file));
 	if (!def) throw new Error(`${file}: must default-export defineEval({ test })`);
 	return def;
 }
 /**
+* Load an `evals.config.ts` file and return its default-exported
+* {@link EvalConfig}. A malformed/missing default surfaces as `undefined` so a
+* bad config never aborts a whole run.
+*/
+async function loadEvalConfig(file) {
+	return resolveConfigDefault(await tsImportFile(file));
+}
+/**
+* Unwrap the config default export across module-interop shapes (see
+* {@link resolveEvalDefault}). A config has no `.test`, so the first plain
+* object reached through the `default` chain is taken as the config.
+*/
+function resolveConfigDefault(mod) {
+	let candidate = mod;
+	for (let i = 0; i < 4 && candidate; i++) {
+		const next = candidate.default;
+		if (next === void 0) return typeof candidate === "object" ? candidate : void 0;
+		candidate = next;
+	}
+}
+/**
 * Unwrap the eval default export across module-interop shapes. Depending on
 * whether the eval file is treated as ESM or CJS, the value lands at
 * `mod.default` (ESM), `mod.default.default` (CJS `__esModule` double-wrap), or
@@ -41,23 +69,26 @@ function resolveEvalDefault(mod) {
 	}
 }
 /**
-* Load and run a single discovered eval. Never throws — a load/run failure
-* becomes a non-passing {@link EvalResult} so one bad eval can't abort the run.
+* Run one eval turn against a fresh driver. Never throws — a run failure becomes
+* a non-passing {@link EvalResult} so one bad eval can't abort the run. `row`
+* binds the current managed-dataset row (see {@link resolveDatasetRows}), or is
+* `undefined` for a plain single-run eval.
 */
-async function runOne(d, id, runId, options) {
+async function runOne(d, id, def, row, runId, options) {
 	try {
-		const def = await loadEval(d.file);
-		return await runEval(def, {
+		return await runWithRetries(options.retries ?? 0, () => runEval(def, {
 			id,
 			driver: createHttpDriver({
 				baseUrl: options.baseUrl,
 				agent: def.agent ?? d.agent,
 				headers: options.headers,
 				mlflowRunId: runId,
-				timeoutMs: options.timeoutMs
+				timeoutMs: def.timeoutMs ?? options.timeoutMs
 			}),
-			strict: options.strict
-		});
+			strict: options.strict,
+			row,
+			timeoutMs: options.timeoutMs
+		}));
 	} catch (err) {
 		return {
 			id,
@@ -67,6 +98,129 @@ async function runOne(d, id, runId, options) {
 		};
 	}
 }
+/**
+* Resolve the rows a (possibly dataset-driven) eval runs over. A plain eval
+* yields a single `undefined` row; a dataset eval reads its Unity Catalog table
+* via {@link readEvalDataset}. On misconfiguration or read failure, returns a
+* single `undefined` row plus an `error`, so the eval still surfaces one result.
+*/
+async function resolveDatasetRows(def, options) {
+	if (!def.dataset) return { rows: [void 0] };
+	if (!options.workspaceClient || !options.warehouseId) return {
+		rows: [void 0],
+		error: "dataset eval requires a workspace client and warehouse (pass --warehouse-id)"
+	};
+	try {
+		const rows = await readEvalDataset(options.workspaceClient, {
+			table: def.dataset.table,
+			warehouseId: options.warehouseId,
+			limit: def.dataset.limit
+		});
+		if (rows.length === 0) return {
+			rows: [void 0],
+			error: `dataset "${def.dataset.table}" returned no rows`
+		};
+		return { rows };
+	} catch (err) {
+		return {
+			rows: [void 0],
+			error: err instanceof Error ? err.message : String(err)
+		};
+	}
+}
+/**
+* Run one already-loaded eval (from the `loaded` pre-pass), expanding a
+* dataset-driven eval into one run per row. Appends one result per row to
+* `results`, emitting `start`/`result` around each. Never throws: a load error
+* (carried in `loadError`) or a dataset-read failure surfaces as a non-passing
+* result. `total` counts eval files, not rows — per-row detail is carried in the
+* result id (`[row i/n]`).
+*/
+async function runDiscovered(d, def, loadError, index, total, runId, options, emit, results) {
+	const id = `${d.agent}/${d.id}`;
+	if (loadError) {
+		emit({
+			type: "start",
+			id,
+			index,
+			total
+		});
+		const result = {
+			id,
+			assertions: [],
+			passed: false,
+			error: loadError
+		};
+		results.push(result);
+		emit({
+			type: "result",
+			result,
+			index,
+			total
+		});
+		return;
+	}
+	const { rows, error: datasetError } = await resolveDatasetRows(def, options);
+	for (let r = 0; r < rows.length; r++) {
+		const rowId = def.dataset && rows.length > 1 ? `${id} [row ${r + 1}/${rows.length}]` : id;
+		emit({
+			type: "start",
+			id: rowId,
+			index,
+			total
+		});
+		const result = datasetError ? {
+			id: rowId,
+			assertions: [],
+			passed: false,
+			error: datasetError
+		} : await runOne(d, rowId, def, rows[r], runId, options);
+		results.push(result);
+		emit({
+			type: "result",
+			result,
+			index,
+			total
+		});
+	}
+}
+/** Base delay (ms) before the first retry; doubled per attempt, full-jittered, capped. */
+const DEFAULT_RETRY_BASE_DELAY_MS = 250;
+/** Ceiling for a single retry backoff wait (ms). */
+const MAX_RETRY_DELAY_MS = 5e3;
+/**
+* Run `attempt` up to `1 + retries` times, stopping as soon as it returns a
+* result that is neither a thrown error / per-eval timeout (`error`) nor a
+* transport/agent turn failure (`infraFailure`). Assertion failures set
+* neither, so a failed-but-completed eval is returned on the first try and
+* never retried. Returns the last result when every attempt failed on infra.
+*
+* Between attempts it waits a full-jittered exponential backoff (infra flakes
+* are overload-correlated). `retries` is coerced to a finite non-negative
+* integer; `baseDelayMs: 0` disables the wait (tests).
+*/
+async function runWithRetries(retries, attempt, options = {}) {
+	const baseDelayMs = options.baseDelayMs ?? DEFAULT_RETRY_BASE_DELAY_MS;
+	const maxAttempts = 1 + (Number.isFinite(retries) ? Math.max(0, Math.floor(retries)) : 0);
+	let result;
+	for (let n = 1;; n++) {
+		result = await attempt(n);
+		if (!(result.error !== void 0 || result.infraFailure) || n >= maxAttempts) return result;
+		if (baseDelayMs > 0) {
+			const ceiling = Math.min(baseDelayMs * 2 ** (n - 1), MAX_RETRY_DELAY_MS);
+			await setTimeout(Math.random() * ceiling);
+		}
+	}
+}
+/**
+* Whether an eval's `tags` satisfy a `--tag` filter: `true` when the filter is
+* empty/undefined (no filtering), otherwise only when the eval shares at least
+* one tag with it. An eval with no tags never matches a non-empty filter.
+*/
+function matchesTags(defTags, filterTags) {
+	if (!filterTags || filterTags.length === 0) return true;
+	return defTags?.some((t) => filterTags.includes(t)) ?? false;
+}
 /** Configure the LLM judge when judge creds were supplied; otherwise a no-op. */
 async function maybeConfigureJudge(options) {
 	if (!options.judge) return;
@@ -113,6 +267,16 @@ async function finalizeMlflow(client, runId, results, options) {
 */
 const DEFAULT_CONCURRENCY = 4;
 /**
+* Resolve the work-pool width: `--concurrency` wins; else the lowest
+* `maxConcurrency` any *participating* agent's `evals.config.ts` requests (all
+* evals share one per-user stream budget, so the most conservative ceiling
+* governs); else {@link DEFAULT_CONCURRENCY}.
+*/
+function deriveConcurrency(activeAgents, configs, cliConcurrency) {
+	const configMin = [...configs.entries()].filter(([agent]) => activeAgents.has(agent)).map(([, c]) => c.maxConcurrency).filter((n) => typeof n === "number").reduce((min, n) => min === void 0 ? n : Math.min(min, n), void 0);
+	return cliConcurrency ?? configMin ?? DEFAULT_CONCURRENCY;
+}
+/**
 * Discover, load, and run every eval under each agent's `evals/` dir, driving
 * the agents on a running app. Never throws for an individual eval — load/run
 * failures become non-passing {@link EvalResult}s.
@@ -126,7 +290,32 @@ async function runEvalsInDir(options) {
 		discovered = discovered.filter((d) => d.agent === f || `${d.agent}/${d.id}`.includes(f));
 	}
 	const emit = options.onEvent ?? (() => {});
-	const total = discovered.length;
+	const configs = /* @__PURE__ */ new Map();
+	for (const c of discoverEvalConfigs(root)) try {
+		const cfg = await loadEvalConfig(c.file);
+		if (cfg) configs.set(c.agent, cfg);
+	} catch {}
+	const loaded = [];
+	for (const d of discovered) {
+		let def;
+		try {
+			def = await loadEval(d.file);
+		} catch (err) {
+			loaded.push({
+				d,
+				def: { test: () => {} },
+				loadError: err instanceof Error ? err.message : String(err)
+			});
+			continue;
+		}
+		if (!matchesTags(def.tags, options.tags)) continue;
+		loaded.push({
+			d,
+			def
+		});
+	}
+	const concurrency = deriveConcurrency(new Set(loaded.map((l) => l.d.agent)), configs, options.concurrency);
+	const total = loaded.length;
 	emit({
 		type: "discovered",
 		total
@@ -147,23 +336,15 @@ async function runEvalsInDir(options) {
 				runId
 			});
 		}
-		const results = await mapPool(discovered, options.concurrency ?? DEFAULT_CONCURRENCY, async (d, index) => {
-			const id = `${d.agent}/${d.id}`;
-			emit({
-				type: "start",
-				id,
-				index,
-				total
-			});
-			const result = await runOne(d, id, runId, options);
-			emit({
-				type: "result",
-				result,
-				index,
-				total
-			});
-			return result;
-		});
+		const results = (await mapPool(loaded, concurrency, async ({ d, def, loadError }, index) => {
+			const fileResults = [];
+			const fileOptions = {
+				...options,
+				timeoutMs: options.timeoutMs ?? configs.get(d.agent)?.timeoutMs
+			};
+			await runDiscovered(d, def, loadError, index, total, runId, fileOptions, emit, fileResults);
+			return fileResults;
+		})).flat();
 		const summary = { results };
 		const mlflow = await finalizeMlflow(mlflowClient, runId, results, options);
 		if (mlflow) summary.mlflow = mlflow;
@@ -174,5 +355,5 @@ async function runEvalsInDir(options) {
 }
 
 //#endregion
-export { runEvalsInDir };
+export { runEvalsInDir, runWithRetries };
 //# sourceMappingURL=run-evals.js.map