@databricks/appkit 0.73.0 → 0.74.0

package/dist/evals/discover.js.map

@@ -1 +1 @@
-{"version":3,"file":"discover.js","names":[],"sources":["../../src/evals/discover.ts"],"sourcesContent":["import { type Dirent, readdirSync } from \"node:fs\";\nimport path from \"node:path\";\n\nimport { agentDirNames } from \"../core/agent/agent-dirs\";\nimport { CODE_AGENTS_SOURCE_DIR } from \"../core/agent/load-code-agents\";\n\n/** An eval file found under `server/agents/<agent>/evals/`. */\nexport interface DiscoveredEval {\n  /** Absolute path to the `*.eval.ts` file. */\n  file: string;\n  /** Id relative to the agent's evals dir, without `.eval.ts` (e.g. `weather/basic`). */\n  id: string;\n  /** The agent id (the `server/agents/<agent>` directory name). */\n  agent: string;\n}\n\n/** Recursively collect `*.eval.ts` files under `dir`. Empty when `dir` is absent. */\nfunction evalFilesIn(dir: string): string[] {\n  try {\n    return readdirSync(dir, { recursive: true, withFileTypes: true })\n      .filter((e) => e.isFile() && e.name.endsWith(\".eval.ts\"))\n      .map((e) => path.join(e.parentPath, e.name));\n  } catch {\n    return [];\n  }\n}\n\n/**\n * Discover evals under `<rootDir>/server/agents/<agent>/evals/` — co-located\n * with each agent's `agent.{md,ts}` (same folder-per-agent layout the agents\n * plugin discovers). The agent id is the folder name; the eval id is the file\n * path relative to that evals dir with `.eval.ts` stripped. Sorted + stable.\n */\nexport function discoverEvalFiles(rootDir: string): DiscoveredEval[] {\n  const agentsDir = path.join(rootDir, CODE_AGENTS_SOURCE_DIR);\n  const out: DiscoveredEval[] = [];\n\n  let entries: Dirent[];\n  try {\n    entries = readdirSync(agentsDir, { withFileTypes: true });\n  } catch {\n    return out;\n  }\n\n  for (const agent of agentDirNames(entries)) {\n    const evalsDir = path.join(agentsDir, agent, \"evals\");\n    for (const file of evalFilesIn(evalsDir)) {\n      const id = path\n        .relative(evalsDir, file)\n        .replace(/\\.eval\\.ts$/, \"\")\n        .split(path.sep)\n        .join(\"/\");\n      out.push({ file, id, agent });\n    }\n  }\n\n  return out.sort(\n    (a, b) => a.agent.localeCompare(b.agent) || a.id.localeCompare(b.id),\n  );\n}\n"],"mappings":";;;;;;;AAiBA,SAAS,YAAY,KAAuB;AAC1C,KAAI;AACF,SAAO,YAAY,KAAK;GAAE,WAAW;GAAM,eAAe;GAAM,CAAC,CAC9D,QAAQ,MAAM,EAAE,QAAQ,IAAI,EAAE,KAAK,SAAS,WAAW,CAAC,CACxD,KAAK,MAAM,KAAK,KAAK,EAAE,YAAY,EAAE,KAAK,CAAC;SACxC;AACN,SAAO,EAAE;;;;;;;;;AAUb,SAAgB,kBAAkB,SAAmC;CACnE,MAAM,YAAY,KAAK,KAAK,SAAS,uBAAuB;CAC5D,MAAM,MAAwB,EAAE;CAEhC,IAAI;AACJ,KAAI;AACF,YAAU,YAAY,WAAW,EAAE,eAAe,MAAM,CAAC;SACnD;AACN,SAAO;;AAGT,MAAK,MAAM,SAAS,cAAc,QAAQ,EAAE;EAC1C,MAAM,WAAW,KAAK,KAAK,WAAW,OAAO,QAAQ;AACrD,OAAK,MAAM,QAAQ,YAAY,SAAS,EAAE;GACxC,MAAM,KAAK,KACR,SAAS,UAAU,KAAK,CACxB,QAAQ,eAAe,GAAG,CAC1B,MAAM,KAAK,IAAI,CACf,KAAK,IAAI;AACZ,OAAI,KAAK;IAAE;IAAM;IAAI;IAAO,CAAC;;;AAIjC,QAAO,IAAI,MACR,GAAG,MAAM,EAAE,MAAM,cAAc,EAAE,MAAM,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,CACrE"}
+{"version":3,"file":"discover.js","names":[],"sources":["../../src/evals/discover.ts"],"sourcesContent":["import { type Dirent, existsSync, readdirSync } from \"node:fs\";\nimport path from \"node:path\";\n\nimport { agentDirNames } from \"../core/agent/agent-dirs\";\nimport { CODE_AGENTS_SOURCE_DIR } from \"../core/agent/load-code-agents\";\n\n/** An eval file found under `server/agents/<agent>/evals/`. */\nexport interface DiscoveredEval {\n  /** Absolute path to the `*.eval.ts` file. */\n  file: string;\n  /** Id relative to the agent's evals dir, without `.eval.ts` (e.g. `weather/basic`). */\n  id: string;\n  /** The agent id (the `server/agents/<agent>` directory name). */\n  agent: string;\n}\n\n/** A per-agent `evals.config.ts` found under `server/agents/<agent>/evals/`. */\nexport interface DiscoveredEvalConfig {\n  /** Absolute path to the `evals.config.ts` file. */\n  file: string;\n  /** The agent id whose evals this config applies to. */\n  agent: string;\n}\n\n/** Recursively collect `*.eval.ts` files under `dir`. Empty when `dir` is absent. */\nfunction evalFilesIn(dir: string): string[] {\n  try {\n    return readdirSync(dir, { recursive: true, withFileTypes: true })\n      .filter((e) => e.isFile() && e.name.endsWith(\".eval.ts\"))\n      .map((e) => path.join(e.parentPath, e.name));\n  } catch {\n    return [];\n  }\n}\n\n/**\n * Discover evals under `<rootDir>/server/agents/<agent>/evals/` — co-located\n * with each agent's `agent.{md,ts}` (same folder-per-agent layout the agents\n * plugin discovers). The agent id is the folder name; the eval id is the file\n * path relative to that evals dir with `.eval.ts` stripped. Sorted + stable.\n */\nexport function discoverEvalFiles(rootDir: string): DiscoveredEval[] {\n  const agentsDir = path.join(rootDir, CODE_AGENTS_SOURCE_DIR);\n  const out: DiscoveredEval[] = [];\n\n  let entries: Dirent[];\n  try {\n    entries = readdirSync(agentsDir, { withFileTypes: true });\n  } catch {\n    return out;\n  }\n\n  for (const agent of agentDirNames(entries)) {\n    const evalsDir = path.join(agentsDir, agent, \"evals\");\n    for (const file of evalFilesIn(evalsDir)) {\n      const id = path\n        .relative(evalsDir, file)\n        .replace(/\\.eval\\.ts$/, \"\")\n        .split(path.sep)\n        .join(\"/\");\n      out.push({ file, id, agent });\n    }\n  }\n\n  return out.sort(\n    (a, b) => a.agent.localeCompare(b.agent) || a.id.localeCompare(b.id),\n  );\n}\n\n/**\n * Discover the per-agent `evals.config.ts` (from {@link defineEvalConfig}) at\n * `<rootDir>/server/agents/<agent>/evals/evals.config.ts`. Config is per-agent:\n * each agent's config applies only to that agent's evals. Agents without a\n * config file are omitted. Returns a stable, sorted list.\n */\nexport function discoverEvalConfigs(rootDir: string): DiscoveredEvalConfig[] {\n  const agentsDir = path.join(rootDir, CODE_AGENTS_SOURCE_DIR);\n  const out: DiscoveredEvalConfig[] = [];\n\n  let entries: Dirent[];\n  try {\n    entries = readdirSync(agentsDir, { withFileTypes: true });\n  } catch {\n    return out;\n  }\n\n  for (const agent of agentDirNames(entries)) {\n    const file = path.join(agentsDir, agent, \"evals\", \"evals.config.ts\");\n    if (existsSync(file)) out.push({ file, agent });\n  }\n\n  return out.sort((a, b) => a.agent.localeCompare(b.agent));\n}\n"],"mappings":";;;;;;;AAyBA,SAAS,YAAY,KAAuB;AAC1C,KAAI;AACF,SAAO,YAAY,KAAK;GAAE,WAAW;GAAM,eAAe;GAAM,CAAC,CAC9D,QAAQ,MAAM,EAAE,QAAQ,IAAI,EAAE,KAAK,SAAS,WAAW,CAAC,CACxD,KAAK,MAAM,KAAK,KAAK,EAAE,YAAY,EAAE,KAAK,CAAC;SACxC;AACN,SAAO,EAAE;;;;;;;;;AAUb,SAAgB,kBAAkB,SAAmC;CACnE,MAAM,YAAY,KAAK,KAAK,SAAS,uBAAuB;CAC5D,MAAM,MAAwB,EAAE;CAEhC,IAAI;AACJ,KAAI;AACF,YAAU,YAAY,WAAW,EAAE,eAAe,MAAM,CAAC;SACnD;AACN,SAAO;;AAGT,MAAK,MAAM,SAAS,cAAc,QAAQ,EAAE;EAC1C,MAAM,WAAW,KAAK,KAAK,WAAW,OAAO,QAAQ;AACrD,OAAK,MAAM,QAAQ,YAAY,SAAS,EAAE;GACxC,MAAM,KAAK,KACR,SAAS,UAAU,KAAK,CACxB,QAAQ,eAAe,GAAG,CAC1B,MAAM,KAAK,IAAI,CACf,KAAK,IAAI;AACZ,OAAI,KAAK;IAAE;IAAM;IAAI;IAAO,CAAC;;;AAIjC,QAAO,IAAI,MACR,GAAG,MAAM,EAAE,MAAM,cAAc,EAAE,MAAM,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,CACrE;;;;;;;;AASH,SAAgB,oBAAoB,SAAyC;CAC3E,MAAM,YAAY,KAAK,KAAK,SAAS,uBAAuB;CAC5D,MAAM,MAA8B,EAAE;CAEtC,IAAI;AACJ,KAAI;AACF,YAAU,YAAY,WAAW,EAAE,eAAe,MAAM,CAAC;SACnD;AACN,SAAO;;AAGT,MAAK,MAAM,SAAS,cAAc,QAAQ,EAAE;EAC1C,MAAM,OAAO,KAAK,KAAK,WAAW,OAAO,SAAS,kBAAkB;AACpE,MAAI,WAAW,KAAK,CAAE,KAAI,KAAK;GAAE;GAAM;GAAO,CAAC;;AAGjD,QAAO,IAAI,MAAM,GAAG,MAAM,EAAE,MAAM,cAAc,EAAE,MAAM,CAAC"}

package/dist/evals/http-driver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"http-driver.d.ts","names":[],"sources":["../../src/evals/http-driver.ts"],"mappings":";;;UAGiB,iBAAA;;EAEf,OAAA;EAFgC;EAIhC,KAAA;EAEgB;EAAhB,OAAA,GAAU,MAAA;EAFV;EAIA,IAAA;EAFU;EAIV,WAAA;EAAA;;;;AA6GF;;;EApGE,SAAA;AAAA;;;;;;;iBAoGc,gBAAA,CAAiB,OAAA,EAAS,iBAAA,GAAoB,UAAA"}
+{"version":3,"file":"http-driver.d.ts","names":[],"sources":["../../src/evals/http-driver.ts"],"mappings":";;;UAGiB,iBAAA;;EAEf,OAAA;EAFgC;EAIhC,KAAA;EAEgB;EAAhB,OAAA,GAAU,MAAA;EAFV;EAIA,IAAA;EAFU;EAIV,WAAA;EAAA;;;;AA4IF;;;EAnIE,SAAA;AAAA;;;;;;;iBAmIc,gBAAA,CAAiB,OAAA,EAAS,iBAAA,GAAoB,UAAA"}

package/dist/evals/http-driver.js

@@ -1,19 +1,37 @@
 import { readSseEvents } from "../stream/sse-reader.js";
 
 //#region src/evals/http-driver.ts
-/** Record a `function_call` output item once per call id (deduped). */
+/**
+* Record a `function_call` output item, keyed by `call_id ?? name`, capturing
+* its parsed arguments. A later `done` event's fuller args win over the initial
+* `added` event's (often empty) args.
+*/
 function recordToolCall(item, state) {
 	if (item?.type !== "function_call" || !item.name) return;
 	const key = item.call_id ?? item.name;
-	if (state.seen.has(key)) return;
-	state.seen.add(key);
-	state.toolCalls.push(item.name);
+	const args = parseArgs(item.arguments);
+	const existing = state.toolCalls.get(key);
+	if (!existing) state.toolCalls.set(key, {
+		name: item.name,
+		args
+	});
+	else if (Object.keys(args).length > 0) existing.args = args;
 }
 /** Apply an `appkit.metadata` event's thread/trace ids. */
 function applyMetadata(data, state, setThread) {
 	if (data?.threadId) setThread(data.threadId);
 	if (data?.mlflowTraceId) state.traceId = data.mlflowTraceId;
 }
+/** Parse a function-call `arguments` JSON string; `{}` on missing/invalid. */
+function parseArgs(raw) {
+	if (typeof raw !== "string" || raw.trim() === "") return {};
+	try {
+		const parsed = JSON.parse(raw);
+		return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
+	} catch {
+		return {};
+	}
+}
 /** Parse a single Responses-API SSE `data:` payload into the running totals. */
 function applyEvent(event, state, setThread) {
 	const type = event.type;
@@ -59,8 +77,9 @@ function createHttpDriver(options) {
 		reset() {
 			threadId = void 0;
 		},
-		async send(message) {
-			const signal = AbortSignal.timeout(timeoutMs);
+		async send(message, opts) {
+			const timeout = AbortSignal.timeout(timeoutMs);
+			const signal = opts?.signal ? AbortSignal.any([timeout, opts.signal]) : timeout;
 			let res;
 			try {
 				res = await fetch(`${options.baseUrl}${chatPath}`, {
@@ -77,19 +96,20 @@ function createHttpDriver(options) {
 				return {
 					reply: "",
 					toolCalls: [],
+					toolCallDetails: [],
 					succeeded: false
 				};
 			}
 			if (!res.ok || !res.body) return {
 				reply: "",
 				toolCalls: [],
+				toolCallDetails: [],
 				succeeded: false,
 				sessionId: threadId
 			};
 			const state = {
 				reply: "",
-				toolCalls: [],
-				seen: /* @__PURE__ */ new Set(),
+				toolCalls: /* @__PURE__ */ new Map(),
 				ok: true
 			};
 			const setThread = (id) => {
@@ -107,9 +127,11 @@ function createHttpDriver(options) {
 				state.ok = false;
 			}
 			if (signal.aborted) state.ok = false;
+			const toolCallDetails = [...state.toolCalls.values()];
 			return {
 				reply: state.reply,
-				toolCalls: state.toolCalls,
+				toolCalls: toolCallDetails.map((c) => c.name),
+				toolCallDetails,
 				succeeded: state.ok,
 				sessionId: threadId,
 				traceId: state.traceId

package/dist/evals/http-driver.js.map

@@ -1 +1 @@
-{"version":3,"file":"http-driver.js","names":[],"sources":["../../src/evals/http-driver.ts"],"sourcesContent":["import { readSseEvents } from \"../stream/sse-reader\";\nimport type { DriveResult, EvalDriver } from \"./types\";\n\nexport interface HttpDriverOptions {\n  /** Base URL of the running app, e.g. `http://localhost:3000`. */\n  baseUrl: string;\n  /** Agent alias to target. Omit to use the app's default agent. */\n  agent?: string;\n  /** Extra request headers (e.g. auth for a deployed app). */\n  headers?: Record<string, string>;\n  /** Chat endpoint path. Defaults to `/api/agents/chat`. */\n  path?: string;\n  /** MLflow run id to link each turn's trace to (for evaluation runs). */\n  mlflowRunId?: string;\n  /**\n   * Max wall-clock time for a single turn before it is abandoned as a failed\n   * turn (`succeeded: false`). Without this a hung agent — a blocked tool, a\n   * stalled model — never ends the SSE stream (heartbeats keep it alive), so\n   * the read loop spins forever and wedges the whole sequential suite.\n   * Defaults to 120s.\n   */\n  // ponytail: total per-turn cap; switch to an idle timeout if long legit turns get killed.\n  timeoutMs?: number;\n}\n\n/** Mutable running totals accumulated while draining one turn's SSE stream. */\ninterface DriveState {\n  reply: string;\n  toolCalls: string[];\n  seen: Set<string>;\n  ok: boolean;\n  traceId?: string;\n}\n\n/** Record a `function_call` output item once per call id (deduped). */\nfunction recordToolCall(\n  item: { type?: string; name?: string; call_id?: string } | undefined,\n  state: DriveState,\n): void {\n  if (item?.type !== \"function_call\" || !item.name) return;\n  const key = item.call_id ?? item.name;\n  if (state.seen.has(key)) return;\n  state.seen.add(key);\n  state.toolCalls.push(item.name);\n}\n\n/** Apply an `appkit.metadata` event's thread/trace ids. */\nfunction applyMetadata(\n  data: { threadId?: string; mlflowTraceId?: string } | undefined,\n  state: DriveState,\n  setThread: (id: string) => void,\n): void {\n  if (data?.threadId) setThread(data.threadId);\n  if (data?.mlflowTraceId) state.traceId = data.mlflowTraceId;\n}\n\n/** Parse a single Responses-API SSE `data:` payload into the running totals. */\nfunction applyEvent(\n  event: Record<string, unknown>,\n  state: DriveState,\n  setThread: (id: string) => void,\n): void {\n  const type = event.type;\n  if (type === \"response.output_text.delta\") {\n    if (typeof event.delta === \"string\") state.reply += event.delta;\n    return;\n  }\n  if (\n    type === \"response.output_item.added\" ||\n    type === \"response.output_item.done\"\n  ) {\n    const item = event.item as {\n      type?: string;\n      name?: string;\n      call_id?: string;\n      content?: Array<{ text?: string }>;\n    };\n    recordToolCall(item, state);\n    // A terminal `message` item carries the full reply text and *replaces*\n    // the accumulated deltas — the server emits no delta for a full message\n    // (event-translator `handleFullMessage`), so a non-streaming adapter's\n    // whole reply arrives only here. Guard on non-empty so a streaming turn's\n    // trailing done event can't clobber the deltas with \"\".\n    if (type === \"response.output_item.done\" && item.type === \"message\") {\n      const text = (item.content ?? []).map((c) => c.text ?? \"\").join(\"\");\n      if (text) state.reply = text;\n    }\n    return;\n  }\n  if (type === \"error\" || type === \"response.failed\") {\n    state.ok = false;\n    return;\n  }\n  if (type === \"appkit.metadata\") {\n    applyMetadata(\n      event.data as { threadId?: string; mlflowTraceId?: string },\n      state,\n      setThread,\n    );\n  }\n}\n\n/** Build the chat request payload, including only the optional fields that are set. */\nfunction buildRequestBody(\n  message: string,\n  options: HttpDriverOptions,\n  threadId: string | undefined,\n): string {\n  return JSON.stringify({\n    message,\n    ...(options.agent ? { agent: options.agent } : {}),\n    ...(threadId ? { threadId } : {}),\n    ...(options.mlflowRunId ? { mlflowRunId: options.mlflowRunId } : {}),\n  });\n}\n\n/**\n * Drives an agent by POSTing to a running app's chat endpoint and parsing the\n * SSE response. Keeps the thread id across `send`s so multi-turn evals share a\n * conversation. Agent/stream errors surface as `succeeded: false` rather than\n * throwing, so `t.succeeded()` can assert on them.\n */\nexport function createHttpDriver(options: HttpDriverOptions): EvalDriver {\n  const chatPath = options.path ?? \"/api/agents/chat\";\n  const timeoutMs = options.timeoutMs ?? 120_000;\n  let threadId: string | undefined;\n\n  return {\n    reset(): void {\n      threadId = undefined;\n    },\n    async send(message: string): Promise<DriveResult> {\n      // Bounds connect + the entire read below. Passed to both the fetch and\n      // the SSE reader: on expiry the reader is cancelled and the turn fails.\n      const signal = AbortSignal.timeout(timeoutMs);\n      let res: Response;\n      try {\n        res = await fetch(`${options.baseUrl}${chatPath}`, {\n          method: \"POST\",\n          headers: { \"content-type\": \"application/json\", ...options.headers },\n          body: buildRequestBody(message, options, threadId),\n          // A chat endpoint never needs a redirect; following one would replay\n          // custom auth headers (`options.headers`) to the redirect target.\n          // A 3xx becomes an opaque response (res.ok === false), handled as a\n          // failed turn by the !res.ok guard below.\n          redirect: \"manual\",\n          signal,\n        });\n      } catch {\n        return { reply: \"\", toolCalls: [], succeeded: false };\n      }\n\n      if (!res.ok || !res.body) {\n        return {\n          reply: \"\",\n          toolCalls: [],\n          succeeded: false,\n          sessionId: threadId,\n        };\n      }\n\n      const state: DriveState = {\n        reply: \"\",\n        toolCalls: [],\n        seen: new Set<string>(),\n        ok: true,\n      };\n      const setThread = (id: string) => {\n        threadId = id;\n      };\n      try {\n        for await (const { event, data } of readSseEvents(res.body, signal)) {\n          if (data === \"[DONE]\") continue;\n          // A stream-level error frame (a thrown exception in the generator)\n          // is framed as `event: error` with a payload that carries no `type`,\n          // so the SSE event name — not the payload — is the real signal.\n          if (event === \"error\") state.ok = false;\n          try {\n            applyEvent(\n              JSON.parse(data) as Record<string, unknown>,\n              state,\n              setThread,\n            );\n          } catch {\n            // skip malformed event payloads\n          }\n        }\n      } catch {\n        // mid-stream transport error or DoS-guard throw: a broken turn.\n        state.ok = false;\n      }\n      // A timeout ends the iterator cleanly (reader cancel) rather than\n      // throwing, so mark an aborted turn failed explicitly.\n      if (signal.aborted) state.ok = false;\n\n      return {\n        reply: state.reply,\n        toolCalls: state.toolCalls,\n        succeeded: state.ok,\n        sessionId: threadId,\n        traceId: state.traceId,\n      };\n    },\n  };\n}\n"],"mappings":";;;;AAmCA,SAAS,eACP,MACA,OACM;AACN,KAAI,MAAM,SAAS,mBAAmB,CAAC,KAAK,KAAM;CAClD,MAAM,MAAM,KAAK,WAAW,KAAK;AACjC,KAAI,MAAM,KAAK,IAAI,IAAI,CAAE;AACzB,OAAM,KAAK,IAAI,IAAI;AACnB,OAAM,UAAU,KAAK,KAAK,KAAK;;;AAIjC,SAAS,cACP,MACA,OACA,WACM;AACN,KAAI,MAAM,SAAU,WAAU,KAAK,SAAS;AAC5C,KAAI,MAAM,cAAe,OAAM,UAAU,KAAK;;;AAIhD,SAAS,WACP,OACA,OACA,WACM;CACN,MAAM,OAAO,MAAM;AACnB,KAAI,SAAS,8BAA8B;AACzC,MAAI,OAAO,MAAM,UAAU,SAAU,OAAM,SAAS,MAAM;AAC1D;;AAEF,KACE,SAAS,gCACT,SAAS,6BACT;EACA,MAAM,OAAO,MAAM;AAMnB,iBAAe,MAAM,MAAM;AAM3B,MAAI,SAAS,+BAA+B,KAAK,SAAS,WAAW;GACnE,MAAM,QAAQ,KAAK,WAAW,EAAE,EAAE,KAAK,MAAM,EAAE,QAAQ,GAAG,CAAC,KAAK,GAAG;AACnE,OAAI,KAAM,OAAM,QAAQ;;AAE1B;;AAEF,KAAI,SAAS,WAAW,SAAS,mBAAmB;AAClD,QAAM,KAAK;AACX;;AAEF,KAAI,SAAS,kBACX,eACE,MAAM,MACN,OACA,UACD;;;AAKL,SAAS,iBACP,SACA,SACA,UACQ;AACR,QAAO,KAAK,UAAU;EACpB;EACA,GAAI,QAAQ,QAAQ,EAAE,OAAO,QAAQ,OAAO,GAAG,EAAE;EACjD,GAAI,WAAW,EAAE,UAAU,GAAG,EAAE;EAChC,GAAI,QAAQ,cAAc,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;EACpE,CAAC;;;;;;;;AASJ,SAAgB,iBAAiB,SAAwC;CACvE,MAAM,WAAW,QAAQ,QAAQ;CACjC,MAAM,YAAY,QAAQ,aAAa;CACvC,IAAI;AAEJ,QAAO;EACL,QAAc;AACZ,cAAW;;EAEb,MAAM,KAAK,SAAuC;GAGhD,MAAM,SAAS,YAAY,QAAQ,UAAU;GAC7C,IAAI;AACJ,OAAI;AACF,UAAM,MAAM,MAAM,GAAG,QAAQ,UAAU,YAAY;KACjD,QAAQ;KACR,SAAS;MAAE,gBAAgB;MAAoB,GAAG,QAAQ;MAAS;KACnE,MAAM,iBAAiB,SAAS,SAAS,SAAS;KAKlD,UAAU;KACV;KACD,CAAC;WACI;AACN,WAAO;KAAE,OAAO;KAAI,WAAW,EAAE;KAAE,WAAW;KAAO;;AAGvD,OAAI,CAAC,IAAI,MAAM,CAAC,IAAI,KAClB,QAAO;IACL,OAAO;IACP,WAAW,EAAE;IACb,WAAW;IACX,WAAW;IACZ;GAGH,MAAM,QAAoB;IACxB,OAAO;IACP,WAAW,EAAE;IACb,sBAAM,IAAI,KAAa;IACvB,IAAI;IACL;GACD,MAAM,aAAa,OAAe;AAChC,eAAW;;AAEb,OAAI;AACF,eAAW,MAAM,EAAE,OAAO,UAAU,cAAc,IAAI,MAAM,OAAO,EAAE;AACnE,SAAI,SAAS,SAAU;AAIvB,SAAI,UAAU,QAAS,OAAM,KAAK;AAClC,SAAI;AACF,iBACE,KAAK,MAAM,KAAK,EAChB,OACA,UACD;aACK;;WAIJ;AAEN,UAAM,KAAK;;AAIb,OAAI,OAAO,QAAS,OAAM,KAAK;AAE/B,UAAO;IACL,OAAO,MAAM;IACb,WAAW,MAAM;IACjB,WAAW,MAAM;IACjB,WAAW;IACX,SAAS,MAAM;IAChB;;EAEJ"}
+{"version":3,"file":"http-driver.js","names":[],"sources":["../../src/evals/http-driver.ts"],"sourcesContent":["import { readSseEvents } from \"../stream/sse-reader\";\nimport type { DriveResult, EvalDriver } from \"./types\";\n\nexport interface HttpDriverOptions {\n  /** Base URL of the running app, e.g. `http://localhost:3000`. */\n  baseUrl: string;\n  /** Agent alias to target. Omit to use the app's default agent. */\n  agent?: string;\n  /** Extra request headers (e.g. auth for a deployed app). */\n  headers?: Record<string, string>;\n  /** Chat endpoint path. Defaults to `/api/agents/chat`. */\n  path?: string;\n  /** MLflow run id to link each turn's trace to (for evaluation runs). */\n  mlflowRunId?: string;\n  /**\n   * Max wall-clock time for a single turn before it is abandoned as a failed\n   * turn (`succeeded: false`). Without this a hung agent — a blocked tool, a\n   * stalled model — never ends the SSE stream (heartbeats keep it alive), so\n   * the read loop spins forever and wedges the whole sequential suite.\n   * Defaults to 120s.\n   */\n  // ponytail: total per-turn cap; switch to an idle timeout if long legit turns get killed.\n  timeoutMs?: number;\n}\n\n/** Mutable running totals accumulated while draining one turn's SSE stream. */\ninterface DriveState {\n  reply: string;\n  /** Captured tool calls, keyed by `call_id ?? name` (dedupe). */\n  toolCalls: Map<string, ToolCall>;\n  ok: boolean;\n  traceId?: string;\n}\n\n/**\n * Record a `function_call` output item, keyed by `call_id ?? name`, capturing\n * its parsed arguments. A later `done` event's fuller args win over the initial\n * `added` event's (often empty) args.\n */\nfunction recordToolCall(\n  item:\n    | { type?: string; name?: string; call_id?: string; arguments?: string }\n    | undefined,\n  state: DriveState,\n): void {\n  if (item?.type !== \"function_call\" || !item.name) return;\n  const key = item.call_id ?? item.name;\n  const args = parseArgs(item.arguments);\n  const existing = state.toolCalls.get(key);\n  if (!existing) {\n    state.toolCalls.set(key, { name: item.name, args });\n  } else if (Object.keys(args).length > 0) {\n    // The initial `added` event may carry empty args while the later `done`\n    // carries the full arguments — keep the fuller set.\n    existing.args = args;\n  }\n}\n\n/** Apply an `appkit.metadata` event's thread/trace ids. */\nfunction applyMetadata(\n  data: { threadId?: string; mlflowTraceId?: string } | undefined,\n  state: DriveState,\n  setThread: (id: string) => void,\n): void {\n  if (data?.threadId) setThread(data.threadId);\n  if (data?.mlflowTraceId) state.traceId = data.mlflowTraceId;\n}\n\n/** A single captured tool call, deduped by `call_id ?? name`. */\ntype ToolCall = { name: string; args: Record<string, unknown> };\n\n/** Parse a function-call `arguments` JSON string; `{}` on missing/invalid. */\nfunction parseArgs(raw: unknown): Record<string, unknown> {\n  if (typeof raw !== \"string\" || raw.trim() === \"\") return {};\n  try {\n    const parsed = JSON.parse(raw);\n    return parsed !== null &&\n      typeof parsed === \"object\" &&\n      !Array.isArray(parsed)\n      ? (parsed as Record<string, unknown>)\n      : {};\n  } catch {\n    return {};\n  }\n}\n\n/** Parse a single Responses-API SSE `data:` payload into the running totals. */\nfunction applyEvent(\n  event: Record<string, unknown>,\n  state: DriveState,\n  setThread: (id: string) => void,\n): void {\n  const type = event.type;\n  if (type === \"response.output_text.delta\") {\n    if (typeof event.delta === \"string\") state.reply += event.delta;\n    return;\n  }\n  if (\n    type === \"response.output_item.added\" ||\n    type === \"response.output_item.done\"\n  ) {\n    const item = event.item as {\n      type?: string;\n      name?: string;\n      call_id?: string;\n      arguments?: string;\n      content?: Array<{ text?: string }>;\n    };\n    recordToolCall(item, state);\n    // A terminal `message` item carries the full reply text and *replaces*\n    // the accumulated deltas — the server emits no delta for a full message\n    // (event-translator `handleFullMessage`), so a non-streaming adapter's\n    // whole reply arrives only here. Guard on non-empty so a streaming turn's\n    // trailing done event can't clobber the deltas with \"\".\n    if (type === \"response.output_item.done\" && item.type === \"message\") {\n      const text = (item.content ?? []).map((c) => c.text ?? \"\").join(\"\");\n      if (text) state.reply = text;\n    }\n    return;\n  }\n  if (type === \"error\" || type === \"response.failed\") {\n    state.ok = false;\n    return;\n  }\n  if (type === \"appkit.metadata\") {\n    applyMetadata(\n      event.data as { threadId?: string; mlflowTraceId?: string },\n      state,\n      setThread,\n    );\n  }\n}\n\n/** Build the chat request payload, including only the optional fields that are set. */\nfunction buildRequestBody(\n  message: string,\n  options: HttpDriverOptions,\n  threadId: string | undefined,\n): string {\n  return JSON.stringify({\n    message,\n    ...(options.agent ? { agent: options.agent } : {}),\n    ...(threadId ? { threadId } : {}),\n    ...(options.mlflowRunId ? { mlflowRunId: options.mlflowRunId } : {}),\n  });\n}\n\n/**\n * Drives an agent by POSTing to a running app's chat endpoint and parsing the\n * SSE response. Keeps the thread id across `send`s so multi-turn evals share a\n * conversation. Agent/stream errors surface as `succeeded: false` rather than\n * throwing, so `t.succeeded()` can assert on them.\n */\nexport function createHttpDriver(options: HttpDriverOptions): EvalDriver {\n  const chatPath = options.path ?? \"/api/agents/chat\";\n  const timeoutMs = options.timeoutMs ?? 120_000;\n  let threadId: string | undefined;\n\n  return {\n    reset(): void {\n      threadId = undefined;\n    },\n    async send(\n      message: string,\n      opts?: { signal?: AbortSignal },\n    ): Promise<DriveResult> {\n      // Bounds connect + the entire read below. Passed to both the fetch and\n      // the SSE reader: on expiry the reader is cancelled and the turn fails.\n      // Composed with the caller's signal so a per-eval timeout also aborts this turn.\n      const timeout = AbortSignal.timeout(timeoutMs);\n      const signal = opts?.signal\n        ? AbortSignal.any([timeout, opts.signal])\n        : timeout;\n      let res: Response;\n      try {\n        res = await fetch(`${options.baseUrl}${chatPath}`, {\n          method: \"POST\",\n          headers: { \"content-type\": \"application/json\", ...options.headers },\n          body: buildRequestBody(message, options, threadId),\n          // A chat endpoint never needs a redirect; following one would replay\n          // custom auth headers (`options.headers`) to the redirect target.\n          // A 3xx becomes an opaque response (res.ok === false), handled as a\n          // failed turn by the !res.ok guard below.\n          redirect: \"manual\",\n          signal,\n        });\n      } catch {\n        return {\n          reply: \"\",\n          toolCalls: [],\n          toolCallDetails: [],\n          succeeded: false,\n        };\n      }\n\n      if (!res.ok || !res.body) {\n        return {\n          reply: \"\",\n          toolCalls: [],\n          toolCallDetails: [],\n          succeeded: false,\n          sessionId: threadId,\n        };\n      }\n\n      const state: DriveState = {\n        reply: \"\",\n        toolCalls: new Map<string, ToolCall>(),\n        ok: true,\n      };\n      const setThread = (id: string) => {\n        threadId = id;\n      };\n      try {\n        for await (const { event, data } of readSseEvents(res.body, signal)) {\n          if (data === \"[DONE]\") continue;\n          // A stream-level error frame (a thrown exception in the generator)\n          // is framed as `event: error` with a payload that carries no `type`,\n          // so the SSE event name — not the payload — is the real signal.\n          if (event === \"error\") state.ok = false;\n          try {\n            applyEvent(\n              JSON.parse(data) as Record<string, unknown>,\n              state,\n              setThread,\n            );\n          } catch {\n            // skip malformed event payloads\n          }\n        }\n      } catch {\n        // mid-stream transport error or DoS-guard throw: a broken turn.\n        state.ok = false;\n      }\n      // A timeout ends the iterator cleanly (reader cancel) rather than\n      // throwing, so mark an aborted turn failed explicitly.\n      if (signal.aborted) state.ok = false;\n\n      const toolCallDetails = [...state.toolCalls.values()];\n      return {\n        reply: state.reply,\n        toolCalls: toolCallDetails.map((c) => c.name),\n        toolCallDetails,\n        succeeded: state.ok,\n        sessionId: threadId,\n        traceId: state.traceId,\n      };\n    },\n  };\n}\n"],"mappings":";;;;;;;;AAuCA,SAAS,eACP,MAGA,OACM;AACN,KAAI,MAAM,SAAS,mBAAmB,CAAC,KAAK,KAAM;CAClD,MAAM,MAAM,KAAK,WAAW,KAAK;CACjC,MAAM,OAAO,UAAU,KAAK,UAAU;CACtC,MAAM,WAAW,MAAM,UAAU,IAAI,IAAI;AACzC,KAAI,CAAC,SACH,OAAM,UAAU,IAAI,KAAK;EAAE,MAAM,KAAK;EAAM;EAAM,CAAC;UAC1C,OAAO,KAAK,KAAK,CAAC,SAAS,EAGpC,UAAS,OAAO;;;AAKpB,SAAS,cACP,MACA,OACA,WACM;AACN,KAAI,MAAM,SAAU,WAAU,KAAK,SAAS;AAC5C,KAAI,MAAM,cAAe,OAAM,UAAU,KAAK;;;AAOhD,SAAS,UAAU,KAAuC;AACxD,KAAI,OAAO,QAAQ,YAAY,IAAI,MAAM,KAAK,GAAI,QAAO,EAAE;AAC3D,KAAI;EACF,MAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,SAAO,WAAW,QAChB,OAAO,WAAW,YAClB,CAAC,MAAM,QAAQ,OAAO,GACnB,SACD,EAAE;SACA;AACN,SAAO,EAAE;;;;AAKb,SAAS,WACP,OACA,OACA,WACM;CACN,MAAM,OAAO,MAAM;AACnB,KAAI,SAAS,8BAA8B;AACzC,MAAI,OAAO,MAAM,UAAU,SAAU,OAAM,SAAS,MAAM;AAC1D;;AAEF,KACE,SAAS,gCACT,SAAS,6BACT;EACA,MAAM,OAAO,MAAM;AAOnB,iBAAe,MAAM,MAAM;AAM3B,MAAI,SAAS,+BAA+B,KAAK,SAAS,WAAW;GACnE,MAAM,QAAQ,KAAK,WAAW,EAAE,EAAE,KAAK,MAAM,EAAE,QAAQ,GAAG,CAAC,KAAK,GAAG;AACnE,OAAI,KAAM,OAAM,QAAQ;;AAE1B;;AAEF,KAAI,SAAS,WAAW,SAAS,mBAAmB;AAClD,QAAM,KAAK;AACX;;AAEF,KAAI,SAAS,kBACX,eACE,MAAM,MACN,OACA,UACD;;;AAKL,SAAS,iBACP,SACA,SACA,UACQ;AACR,QAAO,KAAK,UAAU;EACpB;EACA,GAAI,QAAQ,QAAQ,EAAE,OAAO,QAAQ,OAAO,GAAG,EAAE;EACjD,GAAI,WAAW,EAAE,UAAU,GAAG,EAAE;EAChC,GAAI,QAAQ,cAAc,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;EACpE,CAAC;;;;;;;;AASJ,SAAgB,iBAAiB,SAAwC;CACvE,MAAM,WAAW,QAAQ,QAAQ;CACjC,MAAM,YAAY,QAAQ,aAAa;CACvC,IAAI;AAEJ,QAAO;EACL,QAAc;AACZ,cAAW;;EAEb,MAAM,KACJ,SACA,MACsB;GAItB,MAAM,UAAU,YAAY,QAAQ,UAAU;GAC9C,MAAM,SAAS,MAAM,SACjB,YAAY,IAAI,CAAC,SAAS,KAAK,OAAO,CAAC,GACvC;GACJ,IAAI;AACJ,OAAI;AACF,UAAM,MAAM,MAAM,GAAG,QAAQ,UAAU,YAAY;KACjD,QAAQ;KACR,SAAS;MAAE,gBAAgB;MAAoB,GAAG,QAAQ;MAAS;KACnE,MAAM,iBAAiB,SAAS,SAAS,SAAS;KAKlD,UAAU;KACV;KACD,CAAC;WACI;AACN,WAAO;KACL,OAAO;KACP,WAAW,EAAE;KACb,iBAAiB,EAAE;KACnB,WAAW;KACZ;;AAGH,OAAI,CAAC,IAAI,MAAM,CAAC,IAAI,KAClB,QAAO;IACL,OAAO;IACP,WAAW,EAAE;IACb,iBAAiB,EAAE;IACnB,WAAW;IACX,WAAW;IACZ;GAGH,MAAM,QAAoB;IACxB,OAAO;IACP,2BAAW,IAAI,KAAuB;IACtC,IAAI;IACL;GACD,MAAM,aAAa,OAAe;AAChC,eAAW;;AAEb,OAAI;AACF,eAAW,MAAM,EAAE,OAAO,UAAU,cAAc,IAAI,MAAM,OAAO,EAAE;AACnE,SAAI,SAAS,SAAU;AAIvB,SAAI,UAAU,QAAS,OAAM,KAAK;AAClC,SAAI;AACF,iBACE,KAAK,MAAM,KAAK,EAChB,OACA,UACD;aACK;;WAIJ;AAEN,UAAM,KAAK;;AAIb,OAAI,OAAO,QAAS,OAAM,KAAK;GAE/B,MAAM,kBAAkB,CAAC,GAAG,MAAM,UAAU,QAAQ,CAAC;AACrD,UAAO;IACL,OAAO,MAAM;IACb,WAAW,gBAAgB,KAAK,MAAM,EAAE,KAAK;IAC7C;IACA,WAAW,MAAM;IACjB,WAAW;IACX,SAAS,MAAM;IAChB;;EAEJ"}

package/dist/evals/index.d.ts

@@ -1,14 +1,14 @@
 import { DatabricksAuth, ResolveDatabricksAuthOptions, resolveDatabricksAuth, resolveWorkspaceClient } from "../connectors/mlflow/auth.js";
 import { MlflowClient, PostResult, normalizeHost } from "../connectors/mlflow/client.js";
 import "../connectors/mlflow/index.js";
-import { DatasetRow, ReadEvalDatasetOptions, readEvalDataset } from "./dataset.js";
+import { DatasetRow, ReadEvalDatasetOptions, readEvalDataset, userTurns } from "./dataset.js";
 import { AssertionHandle, AssertionResult, CustomJudgeSpec, DriveResult, EvalDefinition, EvalDriver, EvalResult, MatchResult, Matcher, Severity, TestContext } from "./types.js";
-import { defineEval } from "./define-eval.js";
-import { DiscoveredEval, discoverEvalFiles } from "./discover.js";
+import { defineEval, defineEvalConfig } from "./define-eval.js";
+import { DiscoveredEval, DiscoveredEvalConfig, discoverEvalConfigs, discoverEvalFiles } from "./discover.js";
 import { HttpDriverOptions, createHttpDriver } from "./http-driver.js";
 import { JudgeConfig, JudgeScore, configureJudge, isJudgeConfigured } from "./judge.js";
 import { equals, includes, matches } from "./matchers.js";
 import { Assessment, ReportOutcome, buildAssessments, reportToMlflow } from "./mlflow-report.js";
-import { EvalSummary, evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, summarize } from "./report.js";
+import { EvalSummary, evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, summarize } from "./report.js";
 import { RunEvalOptions, runEval } from "./run-eval.js";
-import { EvalProgress, EvalRunSummary, RunEvalsOptions, runEvalsInDir } from "./run-evals.js";
+import { EvalProgress, EvalRunSummary, RunEvalsOptions, runEvalsInDir, runWithRetries } from "./run-evals.js";

package/dist/evals/index.js

@@ -1,14 +1,14 @@
 import { resolveDatabricksAuth, resolveWorkspaceClient } from "../connectors/mlflow/auth.js";
 import { MlflowClient, normalizeHost } from "../connectors/mlflow/client.js";
-import { readEvalDataset } from "./dataset.js";
-import { defineEval } from "./define-eval.js";
-import { discoverEvalFiles } from "./discover.js";
+import { readEvalDataset, userTurns } from "./dataset.js";
+import { defineEval, defineEvalConfig } from "./define-eval.js";
+import { discoverEvalConfigs, discoverEvalFiles } from "./discover.js";
 import { createHttpDriver } from "./http-driver.js";
 import { configureJudge, isJudgeConfigured } from "./judge.js";
 import { equals, includes, matches } from "./matchers.js";
 import { buildAssessments, reportToMlflow } from "./mlflow-report.js";
-import { evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, summarize } from "./report.js";
+import { evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, summarize } from "./report.js";
 import { runEval } from "./run-eval.js";
-import { runEvalsInDir } from "./run-evals.js";
+import { runEvalsInDir, runWithRetries } from "./run-evals.js";
 
 export {  };

package/dist/evals/report.d.ts

@@ -8,6 +8,8 @@ interface EvalSummary {
   skipped: number;
   /** True when no eval failed (skips don't count as failures). */
   allPassed: boolean;
+  /** Fraction of scored (non-skipped) evals that passed, 0..1 (1 when none scored). */
+  passRate: number;
 }
 declare function summarize(results: EvalResult[]): EvalSummary;
 /** Status glyph for a single eval result. */
@@ -20,6 +22,19 @@ declare function formatEvalDetail(result: EvalResult): string[];
 declare function formatSummaryLine(results: EvalResult[]): string;
 /** Render all results as a human-readable console report (non-streaming). */
 declare function formatEvalResults(results: EvalResult[]): string;
+/**
+ * Render results as a machine-readable JSON report (2-space indented):
+ * `{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types —
+ * every field present on a result round-trips.
+ */
+declare function formatResultsJson(results: EvalResult[]): string;
+/**
+ * Render results as JUnit XML for standard CI test reporters: a single
+ * `<testsuite name="appkit-agent-evals">` with one `<testcase>` per result.
+ * Failures carry a `<failure>` (error or failing-gate summary); skips a
+ * `<skipped>`. All attribute/text values are XML-escaped.
+ */
+declare function formatResultsJUnit(results: EvalResult[]): string;
 //#endregion
-export { EvalSummary, evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, summarize };
+export { EvalSummary, evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, summarize };
 //# sourceMappingURL=report.d.ts.map

package/dist/evals/report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report.d.ts","names":[],"sources":["../../src/evals/report.ts"],"mappings":";;;UAEiB,WAAA;EACf,KAAA;EACA,MAAA;EACA,MAAA;EACA,OAAA;EAJ0B;EAM1B,SAAA;AAAA;AAAA,iBAGc,SAAA,CAAU,OAAA,EAAS,UAAA,KAAe,WAAA;;iBAmBlC,SAAA,CAAU,MAAA,EAAQ,UAAA;;iBAMlB,kBAAA,CAAmB,MAAA,EAAQ,UAAA;AAzB3C;AAAA,iBAqCgB,gBAAA,CAAiB,MAAA,EAAQ,UAAA;;iBAYzB,iBAAA,CAAkB,OAAA,EAAS,UAAA;;iBAM3B,iBAAA,CAAkB,OAAA,EAAS,UAAA"}
+{"version":3,"file":"report.d.ts","names":[],"sources":["../../src/evals/report.ts"],"mappings":";;;UAEiB,WAAA;EACf,KAAA;EACA,MAAA;EACA,MAAA;EACA,OAAA;EAJ0B;EAM1B,SAAA;EAJA;EAMA,QAAA;AAAA;AAAA,iBAGc,SAAA,CAAU,OAAA,EAAS,UAAA,KAAe,WAAA;;iBAqBlC,SAAA,CAAU,MAAA,EAAQ,UAAA;;iBAMlB,kBAAA,CAAmB,MAAA,EAAQ,UAAA;;iBAY3B,gBAAA,CAAiB,MAAA,EAAQ,UAAA;;iBAYzB,iBAAA,CAAkB,OAAA,EAAS,UAAA;;iBAM3B,iBAAA,CAAkB,OAAA,EAAS,UAAA;;;AApC3C;;;iBAoDgB,iBAAA,CAAkB,OAAA,EAAS,UAAA;;AA9C3C;;;;;iBA2FgB,kBAAA,CAAmB,OAAA,EAAS,UAAA"}

package/dist/evals/report.js

@@ -6,12 +6,14 @@ function summarize(results) {
 	for (const r of results) if (r.skipped) skipped++;
 	else if (r.passed) passed++;
 	else failed++;
+	const scored = passed + failed;
 	return {
 		total: results.length,
 		passed,
 		failed,
 		skipped,
-		allPassed: failed === 0
+		allPassed: failed === 0,
+		passRate: scored === 0 ? 1 : passed / scored
 	};
 }
 /** Status glyph for a single eval result. */
@@ -51,7 +53,67 @@ function formatEvalResults(results) {
 	lines.push(formatSummaryLine(results));
 	return lines.join("\n");
 }
+/**
+* Render results as a machine-readable JSON report (2-space indented):
+* `{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types —
+* every field present on a result round-trips.
+*/
+function formatResultsJson(results) {
+	return JSON.stringify({
+		summary: summarize(results),
+		results
+	}, null, 2);
+}
+/**
+* Drop the C0 control chars XML 1.0 forbids even when escaped (except tab, LF,
+* CR) — a raw NUL or ANSI escape would otherwise make the JUnit doc reject on parse.
+*/
+function stripXmlControlChars(value) {
+	let out = "";
+	for (const ch of value) {
+		const code = ch.codePointAt(0);
+		if (code >= 32 || code === 9 || code === 10 || code === 13) out += ch;
+	}
+	return out;
+}
+/** Escape a value for use in XML text/attribute content (control chars dropped). */
+function escapeXml(value) {
+	return stripXmlControlChars(value).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+}
+/** One-line reason a result failed: its error, else its failing gate labels. */
+function failureMessage(result) {
+	if (result.error) return result.error;
+	const gates = result.assertions.filter((a) => !a.pass && a.severity === "gate").map((a) => a.detail ? `${a.label} — ${a.detail}` : a.label);
+	return gates.length ? gates.join("; ") : "eval failed";
+}
+/**
+* Render results as JUnit XML for standard CI test reporters: a single
+* `<testsuite name="appkit-agent-evals">` with one `<testcase>` per result.
+* Failures carry a `<failure>` (error or failing-gate summary); skips a
+* `<skipped>`. All attribute/text values are XML-escaped.
+*/
+function formatResultsJUnit(results) {
+	const s = summarize(results);
+	const lines = [];
+	lines.push("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+	lines.push(`<testsuite name="appkit-agent-evals" tests="${s.total}" failures="${s.failed}" skipped="${s.skipped}">`);
+	for (const r of results) {
+		const open = `  <testcase name="${escapeXml(r.id)}" classname="agent-eval"`;
+		if (r.skipped) {
+			lines.push(`${open}>`);
+			lines.push(r.skipped.reason ? `    <skipped message="${escapeXml(r.skipped.reason)}"/>` : "    <skipped/>");
+			lines.push("  </testcase>");
+		} else if (!r.passed) {
+			const message = failureMessage(r);
+			lines.push(`${open}>`);
+			lines.push(`    <failure message="${escapeXml(message)}"/>`);
+			lines.push("  </testcase>");
+		} else lines.push(`${open}/>`);
+	}
+	lines.push("</testsuite>");
+	return lines.join("\n");
+}
 
 //#endregion
-export { evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, summarize };
+export { evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, summarize };
 //# sourceMappingURL=report.js.map

package/dist/evals/report.js.map

@@ -1 +1 @@
-{"version":3,"file":"report.js","names":[],"sources":["../../src/evals/report.ts"],"sourcesContent":["import type { EvalResult } from \"./types\";\n\nexport interface EvalSummary {\n  total: number;\n  passed: number;\n  failed: number;\n  skipped: number;\n  /** True when no eval failed (skips don't count as failures). */\n  allPassed: boolean;\n}\n\nexport function summarize(results: EvalResult[]): EvalSummary {\n  let passed = 0;\n  let failed = 0;\n  let skipped = 0;\n  for (const r of results) {\n    if (r.skipped) skipped++;\n    else if (r.passed) passed++;\n    else failed++;\n  }\n  return {\n    total: results.length,\n    passed,\n    failed,\n    skipped,\n    allPassed: failed === 0,\n  };\n}\n\n/** Status glyph for a single eval result. */\nexport function evalGlyph(result: EvalResult): string {\n  if (result.skipped) return \"−\";\n  return result.passed ? \"✓\" : \"✗\";\n}\n\n/** The one-line header for a single eval result (no failure detail). */\nexport function formatEvalHeadline(result: EvalResult): string {\n  if (result.skipped) {\n    return `− ${result.id} (skipped${\n      result.skipped.reason ? `: ${result.skipped.reason}` : \"\"\n    })`;\n  }\n  return `${evalGlyph(result)} ${result.id}${\n    result.description ? ` — ${result.description}` : \"\"\n  }`;\n}\n\n/** Indented detail lines for a failing eval (error + failing assertions). */\nexport function formatEvalDetail(result: EvalResult): string[] {\n  const lines: string[] = [];\n  if (result.error) lines.push(`    error: ${result.error}`);\n  for (const a of result.assertions) {\n    if (a.pass) continue;\n    const tag = a.severity === \"soft\" ? \"soft\" : \"gate\";\n    lines.push(`    ✗ [${tag}] ${a.label}${a.detail ? ` — ${a.detail}` : \"\"}`);\n  }\n  return lines;\n}\n\n/** The final PASS/FAIL summary line. */\nexport function formatSummaryLine(results: EvalResult[]): string {\n  const s = summarize(results);\n  return `${s.allPassed ? \"PASS\" : \"FAIL\"} — ${s.passed} passed, ${s.failed} failed, ${s.skipped} skipped (${s.total} total)`;\n}\n\n/** Render all results as a human-readable console report (non-streaming). */\nexport function formatEvalResults(results: EvalResult[]): string {\n  const lines: string[] = [];\n  for (const r of results) {\n    lines.push(formatEvalHeadline(r));\n    lines.push(...formatEvalDetail(r));\n  }\n  lines.push(\"\");\n  lines.push(formatSummaryLine(results));\n  return lines.join(\"\\n\");\n}\n"],"mappings":";AAWA,SAAgB,UAAU,SAAoC;CAC5D,IAAI,SAAS;CACb,IAAI,SAAS;CACb,IAAI,UAAU;AACd,MAAK,MAAM,KAAK,QACd,KAAI,EAAE,QAAS;UACN,EAAE,OAAQ;KACd;AAEP,QAAO;EACL,OAAO,QAAQ;EACf;EACA;EACA;EACA,WAAW,WAAW;EACvB;;;AAIH,SAAgB,UAAU,QAA4B;AACpD,KAAI,OAAO,QAAS,QAAO;AAC3B,QAAO,OAAO,SAAS,MAAM;;;AAI/B,SAAgB,mBAAmB,QAA4B;AAC7D,KAAI,OAAO,QACT,QAAO,KAAK,OAAO,GAAG,WACpB,OAAO,QAAQ,SAAS,KAAK,OAAO,QAAQ,WAAW,GACxD;AAEH,QAAO,GAAG,UAAU,OAAO,CAAC,GAAG,OAAO,KACpC,OAAO,cAAc,MAAM,OAAO,gBAAgB;;;AAKtD,SAAgB,iBAAiB,QAA8B;CAC7D,MAAM,QAAkB,EAAE;AAC1B,KAAI,OAAO,MAAO,OAAM,KAAK,cAAc,OAAO,QAAQ;AAC1D,MAAK,MAAM,KAAK,OAAO,YAAY;AACjC,MAAI,EAAE,KAAM;EACZ,MAAM,MAAM,EAAE,aAAa,SAAS,SAAS;AAC7C,QAAM,KAAK,UAAU,IAAI,IAAI,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,WAAW,KAAK;;AAE5E,QAAO;;;AAIT,SAAgB,kBAAkB,SAA+B;CAC/D,MAAM,IAAI,UAAU,QAAQ;AAC5B,QAAO,GAAG,EAAE,YAAY,SAAS,OAAO,KAAK,EAAE,OAAO,WAAW,EAAE,OAAO,WAAW,EAAE,QAAQ,YAAY,EAAE,MAAM;;;AAIrH,SAAgB,kBAAkB,SAA+B;CAC/D,MAAM,QAAkB,EAAE;AAC1B,MAAK,MAAM,KAAK,SAAS;AACvB,QAAM,KAAK,mBAAmB,EAAE,CAAC;AACjC,QAAM,KAAK,GAAG,iBAAiB,EAAE,CAAC;;AAEpC,OAAM,KAAK,GAAG;AACd,OAAM,KAAK,kBAAkB,QAAQ,CAAC;AACtC,QAAO,MAAM,KAAK,KAAK"}
+{"version":3,"file":"report.js","names":[],"sources":["../../src/evals/report.ts"],"sourcesContent":["import type { EvalResult } from \"./types\";\n\nexport interface EvalSummary {\n  total: number;\n  passed: number;\n  failed: number;\n  skipped: number;\n  /** True when no eval failed (skips don't count as failures). */\n  allPassed: boolean;\n  /** Fraction of scored (non-skipped) evals that passed, 0..1 (1 when none scored). */\n  passRate: number;\n}\n\nexport function summarize(results: EvalResult[]): EvalSummary {\n  let passed = 0;\n  let failed = 0;\n  let skipped = 0;\n  for (const r of results) {\n    if (r.skipped) skipped++;\n    else if (r.passed) passed++;\n    else failed++;\n  }\n  const scored = passed + failed;\n  return {\n    total: results.length,\n    passed,\n    failed,\n    skipped,\n    allPassed: failed === 0,\n    passRate: scored === 0 ? 1 : passed / scored,\n  };\n}\n\n/** Status glyph for a single eval result. */\nexport function evalGlyph(result: EvalResult): string {\n  if (result.skipped) return \"−\";\n  return result.passed ? \"✓\" : \"✗\";\n}\n\n/** The one-line header for a single eval result (no failure detail). */\nexport function formatEvalHeadline(result: EvalResult): string {\n  if (result.skipped) {\n    return `− ${result.id} (skipped${\n      result.skipped.reason ? `: ${result.skipped.reason}` : \"\"\n    })`;\n  }\n  return `${evalGlyph(result)} ${result.id}${\n    result.description ? ` — ${result.description}` : \"\"\n  }`;\n}\n\n/** Indented detail lines for a failing eval (error + failing assertions). */\nexport function formatEvalDetail(result: EvalResult): string[] {\n  const lines: string[] = [];\n  if (result.error) lines.push(`    error: ${result.error}`);\n  for (const a of result.assertions) {\n    if (a.pass) continue;\n    const tag = a.severity === \"soft\" ? \"soft\" : \"gate\";\n    lines.push(`    ✗ [${tag}] ${a.label}${a.detail ? ` — ${a.detail}` : \"\"}`);\n  }\n  return lines;\n}\n\n/** The final PASS/FAIL summary line. */\nexport function formatSummaryLine(results: EvalResult[]): string {\n  const s = summarize(results);\n  return `${s.allPassed ? \"PASS\" : \"FAIL\"} — ${s.passed} passed, ${s.failed} failed, ${s.skipped} skipped (${s.total} total)`;\n}\n\n/** Render all results as a human-readable console report (non-streaming). */\nexport function formatEvalResults(results: EvalResult[]): string {\n  const lines: string[] = [];\n  for (const r of results) {\n    lines.push(formatEvalHeadline(r));\n    lines.push(...formatEvalDetail(r));\n  }\n  lines.push(\"\");\n  lines.push(formatSummaryLine(results));\n  return lines.join(\"\\n\");\n}\n\n/**\n * Render results as a machine-readable JSON report (2-space indented):\n * `{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types —\n * every field present on a result round-trips.\n */\nexport function formatResultsJson(results: EvalResult[]): string {\n  return JSON.stringify({ summary: summarize(results), results }, null, 2);\n}\n\n/**\n * Drop the C0 control chars XML 1.0 forbids even when escaped (except tab, LF,\n * CR) — a raw NUL or ANSI escape would otherwise make the JUnit doc reject on parse.\n */\nfunction stripXmlControlChars(value: string): string {\n  // Code-point filter, not a regex: oxlint `no-control-regex` rejects the class.\n  let out = \"\";\n  for (const ch of value) {\n    const code = ch.codePointAt(0) as number;\n    if (code >= 0x20 || code === 0x09 || code === 0x0a || code === 0x0d) {\n      out += ch;\n    }\n  }\n  return out;\n}\n\n/** Escape a value for use in XML text/attribute content (control chars dropped). */\nfunction escapeXml(value: string): string {\n  return stripXmlControlChars(value)\n    .replace(/&/g, \"&amp;\")\n    .replace(/</g, \"&lt;\")\n    .replace(/>/g, \"&gt;\")\n    .replace(/\"/g, \"&quot;\")\n    .replace(/'/g, \"&apos;\");\n}\n\n/** One-line reason a result failed: its error, else its failing gate labels. */\nfunction failureMessage(result: EvalResult): string {\n  if (result.error) return result.error;\n  const gates = result.assertions\n    .filter((a) => !a.pass && a.severity === \"gate\")\n    .map((a) => (a.detail ? `${a.label} — ${a.detail}` : a.label));\n  return gates.length ? gates.join(\"; \") : \"eval failed\";\n}\n\n/**\n * Render results as JUnit XML for standard CI test reporters: a single\n * `<testsuite name=\"appkit-agent-evals\">` with one `<testcase>` per result.\n * Failures carry a `<failure>` (error or failing-gate summary); skips a\n * `<skipped>`. All attribute/text values are XML-escaped.\n */\nexport function formatResultsJUnit(results: EvalResult[]): string {\n  const s = summarize(results);\n  const lines: string[] = [];\n  lines.push('<?xml version=\"1.0\" encoding=\"UTF-8\"?>');\n  lines.push(\n    `<testsuite name=\"appkit-agent-evals\" tests=\"${s.total}\" failures=\"${s.failed}\" skipped=\"${s.skipped}\">`,\n  );\n  for (const r of results) {\n    const open = `  <testcase name=\"${escapeXml(r.id)}\" classname=\"agent-eval\"`;\n    if (r.skipped) {\n      lines.push(`${open}>`);\n      lines.push(\n        r.skipped.reason\n          ? `    <skipped message=\"${escapeXml(r.skipped.reason)}\"/>`\n          : \"    <skipped/>\",\n      );\n      lines.push(\"  </testcase>\");\n    } else if (!r.passed) {\n      const message = failureMessage(r);\n      lines.push(`${open}>`);\n      lines.push(`    <failure message=\"${escapeXml(message)}\"/>`);\n      lines.push(\"  </testcase>\");\n    } else {\n      lines.push(`${open}/>`);\n    }\n  }\n  lines.push(\"</testsuite>\");\n  return lines.join(\"\\n\");\n}\n"],"mappings":";AAaA,SAAgB,UAAU,SAAoC;CAC5D,IAAI,SAAS;CACb,IAAI,SAAS;CACb,IAAI,UAAU;AACd,MAAK,MAAM,KAAK,QACd,KAAI,EAAE,QAAS;UACN,EAAE,OAAQ;KACd;CAEP,MAAM,SAAS,SAAS;AACxB,QAAO;EACL,OAAO,QAAQ;EACf;EACA;EACA;EACA,WAAW,WAAW;EACtB,UAAU,WAAW,IAAI,IAAI,SAAS;EACvC;;;AAIH,SAAgB,UAAU,QAA4B;AACpD,KAAI,OAAO,QAAS,QAAO;AAC3B,QAAO,OAAO,SAAS,MAAM;;;AAI/B,SAAgB,mBAAmB,QAA4B;AAC7D,KAAI,OAAO,QACT,QAAO,KAAK,OAAO,GAAG,WACpB,OAAO,QAAQ,SAAS,KAAK,OAAO,QAAQ,WAAW,GACxD;AAEH,QAAO,GAAG,UAAU,OAAO,CAAC,GAAG,OAAO,KACpC,OAAO,cAAc,MAAM,OAAO,gBAAgB;;;AAKtD,SAAgB,iBAAiB,QAA8B;CAC7D,MAAM,QAAkB,EAAE;AAC1B,KAAI,OAAO,MAAO,OAAM,KAAK,cAAc,OAAO,QAAQ;AAC1D,MAAK,MAAM,KAAK,OAAO,YAAY;AACjC,MAAI,EAAE,KAAM;EACZ,MAAM,MAAM,EAAE,aAAa,SAAS,SAAS;AAC7C,QAAM,KAAK,UAAU,IAAI,IAAI,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,WAAW,KAAK;;AAE5E,QAAO;;;AAIT,SAAgB,kBAAkB,SAA+B;CAC/D,MAAM,IAAI,UAAU,QAAQ;AAC5B,QAAO,GAAG,EAAE,YAAY,SAAS,OAAO,KAAK,EAAE,OAAO,WAAW,EAAE,OAAO,WAAW,EAAE,QAAQ,YAAY,EAAE,MAAM;;;AAIrH,SAAgB,kBAAkB,SAA+B;CAC/D,MAAM,QAAkB,EAAE;AAC1B,MAAK,MAAM,KAAK,SAAS;AACvB,QAAM,KAAK,mBAAmB,EAAE,CAAC;AACjC,QAAM,KAAK,GAAG,iBAAiB,EAAE,CAAC;;AAEpC,OAAM,KAAK,GAAG;AACd,OAAM,KAAK,kBAAkB,QAAQ,CAAC;AACtC,QAAO,MAAM,KAAK,KAAK;;;;;;;AAQzB,SAAgB,kBAAkB,SAA+B;AAC/D,QAAO,KAAK,UAAU;EAAE,SAAS,UAAU,QAAQ;EAAE;EAAS,EAAE,MAAM,EAAE;;;;;;AAO1E,SAAS,qBAAqB,OAAuB;CAEnD,IAAI,MAAM;AACV,MAAK,MAAM,MAAM,OAAO;EACtB,MAAM,OAAO,GAAG,YAAY,EAAE;AAC9B,MAAI,QAAQ,MAAQ,SAAS,KAAQ,SAAS,MAAQ,SAAS,GAC7D,QAAO;;AAGX,QAAO;;;AAIT,SAAS,UAAU,OAAuB;AACxC,QAAO,qBAAqB,MAAM,CAC/B,QAAQ,MAAM,QAAQ,CACtB,QAAQ,MAAM,OAAO,CACrB,QAAQ,MAAM,OAAO,CACrB,QAAQ,MAAM,SAAS,CACvB,QAAQ,MAAM,SAAS;;;AAI5B,SAAS,eAAe,QAA4B;AAClD,KAAI,OAAO,MAAO,QAAO,OAAO;CAChC,MAAM,QAAQ,OAAO,WAClB,QAAQ,MAAM,CAAC,EAAE,QAAQ,EAAE,aAAa,OAAO,CAC/C,KAAK,MAAO,EAAE,SAAS,GAAG,EAAE,MAAM,KAAK,EAAE,WAAW,EAAE,MAAO;AAChE,QAAO,MAAM,SAAS,MAAM,KAAK,KAAK,GAAG;;;;;;;;AAS3C,SAAgB,mBAAmB,SAA+B;CAChE,MAAM,IAAI,UAAU,QAAQ;CAC5B,MAAM,QAAkB,EAAE;AAC1B,OAAM,KAAK,6CAAyC;AACpD,OAAM,KACJ,+CAA+C,EAAE,MAAM,cAAc,EAAE,OAAO,aAAa,EAAE,QAAQ,IACtG;AACD,MAAK,MAAM,KAAK,SAAS;EACvB,MAAM,OAAO,qBAAqB,UAAU,EAAE,GAAG,CAAC;AAClD,MAAI,EAAE,SAAS;AACb,SAAM,KAAK,GAAG,KAAK,GAAG;AACtB,SAAM,KACJ,EAAE,QAAQ,SACN,yBAAyB,UAAU,EAAE,QAAQ,OAAO,CAAC,OACrD,iBACL;AACD,SAAM,KAAK,gBAAgB;aAClB,CAAC,EAAE,QAAQ;GACpB,MAAM,UAAU,eAAe,EAAE;AACjC,SAAM,KAAK,GAAG,KAAK,GAAG;AACtB,SAAM,KAAK,yBAAyB,UAAU,QAAQ,CAAC,KAAK;AAC5D,SAAM,KAAK,gBAAgB;QAE3B,OAAM,KAAK,GAAG,KAAK,IAAI;;AAG3B,OAAM,KAAK,eAAe;AAC1B,QAAO,MAAM,KAAK,KAAK"}

package/dist/evals/run-eval.d.ts

@@ -11,6 +11,11 @@ interface RunEvalOptions {
   strict?: boolean;
   /** Dataset row bound to `t.input`/`t.expected` for dataset-driven evals. */
   row?: DatasetRow;
+  /**
+   * Runner-level default per-eval timeout (ms). `def.timeoutMs` wins over this;
+   * when both are unset the eval runs unbounded (current behavior).
+   */
+  timeoutMs?: number;
 }
 /**
  * Run a single eval against a driver. Never throws for assertion or agent

package/dist/evals/run-eval.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-eval.d.ts","names":[],"sources":["../../src/evals/run-eval.ts"],"mappings":";;;;UAuBiB,cAAA;;EAEf,EAAA;EAF6B;EAI7B,MAAA,EAAQ,UAAA;EAIQ;EAFhB,MAAA;EAFA;EAIA,GAAA,GAAM,UAAA;AAAA;;;;;AAQR;iBAAsB,OAAA,CACpB,GAAA,EAAK,cAAA,EACL,OAAA,EAAS,cAAA,GACR,OAAA,CAAQ,UAAA"}
+{"version":3,"file":"run-eval.d.ts","names":[],"sources":["../../src/evals/run-eval.ts"],"mappings":";;;;UAoDiB,cAAA;;EAEf,EAAA;EAF6B;EAI7B,MAAA,EAAQ,UAAA;EAIQ;EAFhB,MAAA;EAFA;EAIA,GAAA,GAAM,UAAA;EAFN;;;;EAOA,SAAA;AAAA;AAQF;;;;;AAAA,iBAAsB,OAAA,CACpB,GAAA,EAAK,cAAA,EACL,OAAA,EAAS,cAAA,GACR,OAAA,CAAQ,UAAA"}

package/dist/evals/run-eval.js

@@ -12,6 +12,22 @@ var SkipSignal = class extends Error {
 	}
 };
 /**
+* Deep partial match: every key in `expected` is present in `actual` and equal,
+* recursing into nested plain objects so extra actual keys are ignored. Arrays
+* match element-for-element (same length, deep-equal items).
+*/
+function deepContains(actual, expected) {
+	if (Array.isArray(expected)) return Array.isArray(actual) && actual.length === expected.length && expected.every((item, i) => deepContains(actual[i], item));
+	if (isPlainObject(expected)) {
+		if (!isPlainObject(actual)) return false;
+		return Object.keys(expected).every((key) => Object.hasOwn(actual, key) && deepContains(actual[key], expected[key]));
+	}
+	return actual === expected;
+}
+function isPlainObject(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+/**
 * Run a single eval against a driver. Never throws for assertion or agent
 * failures — those become a non-passing {@link EvalResult}. Only a malformed
 * eval definition surfaces as `result.error`.
@@ -21,9 +37,12 @@ async function runEval(def, options) {
 	let reply = "";
 	let lastInput = "";
 	let toolCalls = [];
+	let toolCallDetails = [];
 	let sessionId;
 	let lastTraceId;
 	let lastSucceeded = false;
+	let turnFailed = false;
+	const controller = new AbortController();
 	const record = (label, pass, score, detail) => {
 		const result = {
 			label,
@@ -53,11 +72,13 @@ async function runEval(def, options) {
 	const t = {
 		async send(message) {
 			lastInput = message;
-			const r = await options.driver.send(message);
+			const r = await options.driver.send(message, { signal: controller.signal });
 			reply = r.reply;
 			toolCalls = r.toolCalls;
+			toolCallDetails = r.toolCallDetails;
 			sessionId = r.sessionId;
 			lastSucceeded = r.succeeded;
+			if (!r.succeeded) turnFailed = true;
 			if (r.traceId) lastTraceId = r.traceId;
 		},
 		reset() {
@@ -84,6 +105,12 @@ async function runEval(def, options) {
 		calledTool(name) {
 			return record(`calledTool(${name})`, toolCalls.includes(name), void 0, `expected tool "${name}" to be called (called: ${toolCalls.length ? toolCalls.join(", ") : "none"})`);
 		},
+		calledToolWith(name, expected) {
+			const matching = toolCallDetails.filter((c) => c.name === name);
+			const pass = matching.some((c) => deepContains(c.args, expected));
+			const seen = matching.length ? matching.map((c) => `{${Object.keys(c.args).sort().join(", ")}}`).join(", ") : "not called";
+			return record(`calledToolWith(${name})`, pass, void 0, `expected tool "${name}" to be called with ${JSON.stringify(expected)} (arg keys seen: ${seen})`);
+		},
 		check(value, matcher) {
 			const m = matcher(value);
 			return record("check", m.pass, m.score, m.detail);
@@ -117,8 +144,19 @@ async function runEval(def, options) {
 			throw new SkipSignal(reason);
 		}
 	};
+	const timeoutMs = def.timeoutMs ?? options.timeoutMs;
+	let timer;
 	try {
-		await def.test(t);
+		if (timeoutMs === void 0) await def.test(t);
+		else {
+			const timeout = new Promise((_, reject) => {
+				timer = setTimeout(() => {
+					controller.abort();
+					reject(/* @__PURE__ */ new Error(`eval timed out after ${timeoutMs}ms`));
+				}, timeoutMs);
+			});
+			await Promise.race([Promise.resolve(def.test(t)), timeout]);
+		}
 	} catch (err) {
 		if (err instanceof SkipSignal) return {
 			id: options.id,
@@ -136,6 +174,8 @@ async function runEval(def, options) {
 			error: err instanceof Error ? err.message : String(err),
 			traceId: lastTraceId
 		};
+	} finally {
+		if (timer) clearTimeout(timer);
 	}
 	const passed = assertions.every((a) => a.pass || a.severity === "soft" && !options.strict);
 	return {
@@ -143,7 +183,8 @@ async function runEval(def, options) {
 		description: def.description,
 		assertions,
 		passed,
-		traceId: lastTraceId
+		traceId: lastTraceId,
+		infraFailure: turnFailed && !passed || void 0
 	};
 }
 

package/dist/evals/run-eval.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-eval.js","names":[],"sources":["../../src/evals/run-eval.ts"],"sourcesContent":["import type { DatasetRow } from \"./dataset\";\nimport { judgeClosedQA, judgeCustom, judgeFactuality } from \"./judge\";\nimport type {\n  AssertionHandle,\n  AssertionResult,\n  EvalDefinition,\n  EvalDriver,\n  EvalResult,\n  Matcher,\n  TestContext,\n} from \"./types\";\n\n/** Default pass threshold for an LLM-judge score (0..1) before `.atLeast()`. */\nconst DEFAULT_JUDGE_THRESHOLD = 0.5;\n\n/** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */\nclass SkipSignal extends Error {\n  constructor(public reason?: string) {\n    super(\"eval skipped\");\n    this.name = \"SkipSignal\";\n  }\n}\n\nexport interface RunEvalOptions {\n  /** Stable id for the eval (e.g. its file path relative to the evals dir). */\n  id: string;\n  /** Drives the agent and returns reply/tool-calls/success per `send`. */\n  driver: EvalDriver;\n  /** When true, soft assertion failures also fail the eval. */\n  strict?: boolean;\n  /** Dataset row bound to `t.input`/`t.expected` for dataset-driven evals. */\n  row?: DatasetRow;\n}\n\n/**\n * Run a single eval against a driver. Never throws for assertion or agent\n * failures — those become a non-passing {@link EvalResult}. Only a malformed\n * eval definition surfaces as `result.error`.\n */\nexport async function runEval(\n  def: EvalDefinition,\n  options: RunEvalOptions,\n): Promise<EvalResult> {\n  const assertions: AssertionResult[] = [];\n  let reply = \"\";\n  let lastInput = \"\";\n  let toolCalls: string[] = [];\n  let sessionId: string | undefined;\n  let lastTraceId: string | undefined;\n  let lastSucceeded = false;\n\n  const record = (\n    label: string,\n    pass: boolean,\n    score?: number,\n    detail?: string,\n  ): AssertionHandle => {\n    const result: AssertionResult = {\n      label,\n      severity: \"gate\",\n      pass,\n      score,\n      detail,\n    };\n    assertions.push(result);\n    const handle: AssertionHandle = {\n      gate() {\n        result.severity = \"gate\";\n        return handle;\n      },\n      soft() {\n        result.severity = \"soft\";\n        return handle;\n      },\n      atLeast(threshold: number) {\n        // Re-threshold the score; keep the current severity (gate unless the\n        // caller also chained `.soft()`).\n        result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold;\n        return handle;\n      },\n    };\n    return handle;\n  };\n\n  // LLM-judge assertions are scored and gate by default (a miss fails the eval,\n  // like other assertions). The caller chains `.atLeast(n)` to change the pass\n  // threshold, or `.soft()` to demote to a tracked-only metric.\n  const recordJudge = (\n    label: string,\n    score: number,\n    rationale?: string,\n  ): AssertionHandle =>\n    record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale);\n\n  const t: TestContext = {\n    async send(message) {\n      lastInput = message;\n      const r = await options.driver.send(message);\n      reply = r.reply;\n      toolCalls = r.toolCalls;\n      sessionId = r.sessionId;\n      lastSucceeded = r.succeeded;\n      if (r.traceId) lastTraceId = r.traceId;\n    },\n    reset() {\n      options.driver.reset?.();\n    },\n    get reply() {\n      return reply;\n    },\n    get toolCalls() {\n      return toolCalls;\n    },\n    get sessionId() {\n      return sessionId;\n    },\n    get input() {\n      return options.row?.inputs ?? {};\n    },\n    get expected() {\n      return options.row?.expectations;\n    },\n    succeeded() {\n      return record(\n        \"succeeded\",\n        lastSucceeded,\n        undefined,\n        lastSucceeded ? undefined : \"agent turn did not complete successfully\",\n      );\n    },\n    calledTool(name) {\n      return record(\n        `calledTool(${name})`,\n        toolCalls.includes(name),\n        undefined,\n        `expected tool \"${name}\" to be called (called: ${\n          toolCalls.length ? toolCalls.join(\", \") : \"none\"\n        })`,\n      );\n    },\n    check(value: string, matcher: Matcher) {\n      const m = matcher(value);\n      return record(\"check\", m.pass, m.score, m.detail);\n    },\n    judge: {\n      async factuality(expected) {\n        const { score, rationale } = await judgeFactuality({\n          input: lastInput,\n          output: reply,\n          expected,\n        });\n        return recordJudge(\"judge.factuality\", score, rationale);\n      },\n      async closedQA(criteria) {\n        const { score, rationale } = await judgeClosedQA({\n          input: lastInput,\n          output: reply,\n          criteria,\n        });\n        return recordJudge(\"judge.closedQA\", score, rationale);\n      },\n      async custom(spec) {\n        const { score, rationale } = await judgeCustom(spec, {\n          input: lastInput,\n          output: reply,\n        });\n        return recordJudge(`judge.${spec.name}`, score, rationale);\n      },\n    },\n    skip(reason) {\n      throw new SkipSignal(reason);\n    },\n  };\n\n  try {\n    await def.test(t);\n  } catch (err) {\n    if (err instanceof SkipSignal) {\n      return {\n        id: options.id,\n        description: def.description,\n        skipped: { reason: err.reason },\n        assertions,\n        passed: true,\n        traceId: lastTraceId,\n      };\n    }\n    return {\n      id: options.id,\n      description: def.description,\n      assertions,\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n      traceId: lastTraceId,\n    };\n  }\n\n  const passed = assertions.every(\n    (a) => a.pass || (a.severity === \"soft\" && !options.strict),\n  );\n\n  return {\n    id: options.id,\n    description: def.description,\n    assertions,\n    passed,\n    traceId: lastTraceId,\n  };\n}\n"],"mappings":";;;;AAaA,MAAM,0BAA0B;;AAGhC,IAAM,aAAN,cAAyB,MAAM;CAC7B,YAAY,AAAO,QAAiB;AAClC,QAAM,eAAe;EADJ;AAEjB,OAAK,OAAO;;;;;;;;AAoBhB,eAAsB,QACpB,KACA,SACqB;CACrB,MAAM,aAAgC,EAAE;CACxC,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,YAAsB,EAAE;CAC5B,IAAI;CACJ,IAAI;CACJ,IAAI,gBAAgB;CAEpB,MAAM,UACJ,OACA,MACA,OACA,WACoB;EACpB,MAAM,SAA0B;GAC9B;GACA,UAAU;GACV;GACA;GACA;GACD;AACD,aAAW,KAAK,OAAO;EACvB,MAAM,SAA0B;GAC9B,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,QAAQ,WAAmB;AAGzB,WAAO,QAAQ,OAAO,UAAU,OAAO,OAAO,IAAI,OAAO;AACzD,WAAO;;GAEV;AACD,SAAO;;CAMT,MAAM,eACJ,OACA,OACA,cAEA,OAAO,OAAO,SAAS,yBAAyB,OAAO,UAAU;CAEnE,MAAM,IAAiB;EACrB,MAAM,KAAK,SAAS;AAClB,eAAY;GACZ,MAAM,IAAI,MAAM,QAAQ,OAAO,KAAK,QAAQ;AAC5C,WAAQ,EAAE;AACV,eAAY,EAAE;AACd,eAAY,EAAE;AACd,mBAAgB,EAAE;AAClB,OAAI,EAAE,QAAS,eAAc,EAAE;;EAEjC,QAAQ;AACN,WAAQ,OAAO,SAAS;;EAE1B,IAAI,QAAQ;AACV,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,QAAQ;AACV,UAAO,QAAQ,KAAK,UAAU,EAAE;;EAElC,IAAI,WAAW;AACb,UAAO,QAAQ,KAAK;;EAEtB,YAAY;AACV,UAAO,OACL,aACA,eACA,QACA,gBAAgB,SAAY,2CAC7B;;EAEH,WAAW,MAAM;AACf,UAAO,OACL,cAAc,KAAK,IACnB,UAAU,SAAS,KAAK,EACxB,QACA,kBAAkB,KAAK,0BACrB,UAAU,SAAS,UAAU,KAAK,KAAK,GAAG,OAC3C,GACF;;EAEH,MAAM,OAAe,SAAkB;GACrC,MAAM,IAAI,QAAQ,MAAM;AACxB,UAAO,OAAO,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;;EAEnD,OAAO;GACL,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,OAAO,cAAc,MAAM,gBAAgB;KACjD,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,oBAAoB,OAAO,UAAU;;GAE1D,MAAM,SAAS,UAAU;IACvB,MAAM,EAAE,OAAO,cAAc,MAAM,cAAc;KAC/C,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,kBAAkB,OAAO,UAAU;;GAExD,MAAM,OAAO,MAAM;IACjB,MAAM,EAAE,OAAO,cAAc,MAAM,YAAY,MAAM;KACnD,OAAO;KACP,QAAQ;KACT,CAAC;AACF,WAAO,YAAY,SAAS,KAAK,QAAQ,OAAO,UAAU;;GAE7D;EACD,KAAK,QAAQ;AACX,SAAM,IAAI,WAAW,OAAO;;EAE/B;AAED,KAAI;AACF,QAAM,IAAI,KAAK,EAAE;UACV,KAAK;AACZ,MAAI,eAAe,WACjB,QAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB,SAAS,EAAE,QAAQ,IAAI,QAAQ;GAC/B;GACA,QAAQ;GACR,SAAS;GACV;AAEH,SAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB;GACA,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACvD,SAAS;GACV;;CAGH,MAAM,SAAS,WAAW,OACvB,MAAM,EAAE,QAAS,EAAE,aAAa,UAAU,CAAC,QAAQ,OACrD;AAED,QAAO;EACL,IAAI,QAAQ;EACZ,aAAa,IAAI;EACjB;EACA;EACA,SAAS;EACV"}
+{"version":3,"file":"run-eval.js","names":[],"sources":["../../src/evals/run-eval.ts"],"sourcesContent":["import type { DatasetRow } from \"./dataset\";\nimport { judgeClosedQA, judgeCustom, judgeFactuality } from \"./judge\";\nimport type {\n  AssertionHandle,\n  AssertionResult,\n  DriveResult,\n  EvalDefinition,\n  EvalDriver,\n  EvalResult,\n  Matcher,\n  TestContext,\n} from \"./types\";\n\n/** Default pass threshold for an LLM-judge score (0..1) before `.atLeast()`. */\nconst DEFAULT_JUDGE_THRESHOLD = 0.5;\n\n/** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */\nclass SkipSignal extends Error {\n  constructor(public reason?: string) {\n    super(\"eval skipped\");\n    this.name = \"SkipSignal\";\n  }\n}\n\n/**\n * Deep partial match: every key in `expected` is present in `actual` and equal,\n * recursing into nested plain objects so extra actual keys are ignored. Arrays\n * match element-for-element (same length, deep-equal items).\n */\nfunction deepContains(actual: unknown, expected: unknown): boolean {\n  if (Array.isArray(expected)) {\n    return (\n      Array.isArray(actual) &&\n      actual.length === expected.length &&\n      expected.every((item, i) => deepContains(actual[i], item))\n    );\n  }\n  if (isPlainObject(expected)) {\n    if (!isPlainObject(actual)) return false;\n    // Require the key present: an expected `undefined` must not match an omitted key.\n    return Object.keys(expected).every(\n      (key) =>\n        Object.hasOwn(actual, key) && deepContains(actual[key], expected[key]),\n    );\n  }\n  return actual === expected;\n}\n\nfunction isPlainObject(value: unknown): value is Record<string, unknown> {\n  return typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n\nexport interface RunEvalOptions {\n  /** Stable id for the eval (e.g. its file path relative to the evals dir). */\n  id: string;\n  /** Drives the agent and returns reply/tool-calls/success per `send`. */\n  driver: EvalDriver;\n  /** When true, soft assertion failures also fail the eval. */\n  strict?: boolean;\n  /** Dataset row bound to `t.input`/`t.expected` for dataset-driven evals. */\n  row?: DatasetRow;\n  /**\n   * Runner-level default per-eval timeout (ms). `def.timeoutMs` wins over this;\n   * when both are unset the eval runs unbounded (current behavior).\n   */\n  timeoutMs?: number;\n}\n\n/**\n * Run a single eval against a driver. Never throws for assertion or agent\n * failures — those become a non-passing {@link EvalResult}. Only a malformed\n * eval definition surfaces as `result.error`.\n */\nexport async function runEval(\n  def: EvalDefinition,\n  options: RunEvalOptions,\n): Promise<EvalResult> {\n  const assertions: AssertionResult[] = [];\n  let reply = \"\";\n  let lastInput = \"\";\n  let toolCalls: string[] = [];\n  let toolCallDetails: DriveResult[\"toolCallDetails\"] = [];\n  let sessionId: string | undefined;\n  let lastTraceId: string | undefined;\n  let lastSucceeded = false;\n  // Any transport/agent turn failure (driver `succeeded: false`) → `infraFailure`, for retry.\n  let turnFailed = false;\n  // Aborted on per-eval timeout to cancel the in-flight turn (no leaked stream).\n  const controller = new AbortController();\n\n  const record = (\n    label: string,\n    pass: boolean,\n    score?: number,\n    detail?: string,\n  ): AssertionHandle => {\n    const result: AssertionResult = {\n      label,\n      severity: \"gate\",\n      pass,\n      score,\n      detail,\n    };\n    assertions.push(result);\n    const handle: AssertionHandle = {\n      gate() {\n        result.severity = \"gate\";\n        return handle;\n      },\n      soft() {\n        result.severity = \"soft\";\n        return handle;\n      },\n      atLeast(threshold: number) {\n        // Re-threshold the score; keep the current severity (gate unless the\n        // caller also chained `.soft()`).\n        result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold;\n        return handle;\n      },\n    };\n    return handle;\n  };\n\n  // LLM-judge assertions are scored and gate by default (a miss fails the eval,\n  // like other assertions). The caller chains `.atLeast(n)` to change the pass\n  // threshold, or `.soft()` to demote to a tracked-only metric.\n  const recordJudge = (\n    label: string,\n    score: number,\n    rationale?: string,\n  ): AssertionHandle =>\n    record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale);\n\n  const t: TestContext = {\n    async send(message) {\n      lastInput = message;\n      const r = await options.driver.send(message, {\n        signal: controller.signal,\n      });\n      reply = r.reply;\n      toolCalls = r.toolCalls;\n      toolCallDetails = r.toolCallDetails;\n      sessionId = r.sessionId;\n      lastSucceeded = r.succeeded;\n      if (!r.succeeded) turnFailed = true;\n      if (r.traceId) lastTraceId = r.traceId;\n    },\n    reset() {\n      options.driver.reset?.();\n    },\n    get reply() {\n      return reply;\n    },\n    get toolCalls() {\n      return toolCalls;\n    },\n    get sessionId() {\n      return sessionId;\n    },\n    get input() {\n      return options.row?.inputs ?? {};\n    },\n    get expected() {\n      return options.row?.expectations;\n    },\n    succeeded() {\n      return record(\n        \"succeeded\",\n        lastSucceeded,\n        undefined,\n        lastSucceeded ? undefined : \"agent turn did not complete successfully\",\n      );\n    },\n    calledTool(name) {\n      return record(\n        `calledTool(${name})`,\n        toolCalls.includes(name),\n        undefined,\n        `expected tool \"${name}\" to be called (called: ${\n          toolCalls.length ? toolCalls.join(\", \") : \"none\"\n        })`,\n      );\n    },\n    calledToolWith(name, expected) {\n      const matching = toolCallDetails.filter((c) => c.name === name);\n      const pass = matching.some((c) => deepContains(c.args, expected));\n      // Keys only, never values: actual args may hold PII/secrets and are persisted to reports (CWE-532).\n      const seen = matching.length\n        ? matching\n            .map((c) => `{${Object.keys(c.args).sort().join(\", \")}}`)\n            .join(\", \")\n        : \"not called\";\n      return record(\n        `calledToolWith(${name})`,\n        pass,\n        undefined,\n        `expected tool \"${name}\" to be called with ${JSON.stringify(\n          expected,\n        )} (arg keys seen: ${seen})`,\n      );\n    },\n    check(value: string, matcher: Matcher) {\n      const m = matcher(value);\n      return record(\"check\", m.pass, m.score, m.detail);\n    },\n    judge: {\n      async factuality(expected) {\n        const { score, rationale } = await judgeFactuality({\n          input: lastInput,\n          output: reply,\n          expected,\n        });\n        return recordJudge(\"judge.factuality\", score, rationale);\n      },\n      async closedQA(criteria) {\n        const { score, rationale } = await judgeClosedQA({\n          input: lastInput,\n          output: reply,\n          criteria,\n        });\n        return recordJudge(\"judge.closedQA\", score, rationale);\n      },\n      async custom(spec) {\n        const { score, rationale } = await judgeCustom(spec, {\n          input: lastInput,\n          output: reply,\n        });\n        return recordJudge(`judge.${spec.name}`, score, rationale);\n      },\n    },\n    skip(reason) {\n      throw new SkipSignal(reason);\n    },\n  };\n\n  // `def.timeoutMs` (per-eval) wins over the runner default; when both are\n  // unset the eval runs unbounded (undefined = no timeout).\n  const timeoutMs = def.timeoutMs ?? options.timeoutMs;\n  let timer: ReturnType<typeof setTimeout> | undefined;\n\n  try {\n    if (timeoutMs === undefined) {\n      await def.test(t);\n    } else {\n      // Race the test against the timeout; on elapse, abort the turn and settle\n      // non-passing. Only the driver turn cancels — non-driver hangs (judge, sleep) run on.\n      const timeout = new Promise<never>((_, reject) => {\n        timer = setTimeout(() => {\n          controller.abort();\n          reject(new Error(`eval timed out after ${timeoutMs}ms`));\n        }, timeoutMs);\n      });\n      await Promise.race([Promise.resolve(def.test(t)), timeout]);\n    }\n  } catch (err) {\n    if (err instanceof SkipSignal) {\n      return {\n        id: options.id,\n        description: def.description,\n        skipped: { reason: err.reason },\n        assertions,\n        passed: true,\n        traceId: lastTraceId,\n      };\n    }\n    return {\n      id: options.id,\n      description: def.description,\n      assertions,\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n      traceId: lastTraceId,\n    };\n  } finally {\n    if (timer) clearTimeout(timer);\n  }\n\n  const passed = assertions.every(\n    (a) => a.pass || (a.severity === \"soft\" && !options.strict),\n  );\n\n  return {\n    id: options.id,\n    description: def.description,\n    assertions,\n    passed,\n    traceId: lastTraceId,\n    // Only a *failing* eval whose turn broke at the transport/agent level is a\n    // retryable infra flake; a passing eval (or a pure assertion mismatch, where\n    // the turn itself succeeded) is real signal and must not be retried.\n    infraFailure: (turnFailed && !passed) || undefined,\n  };\n}\n"],"mappings":";;;;AAcA,MAAM,0BAA0B;;AAGhC,IAAM,aAAN,cAAyB,MAAM;CAC7B,YAAY,AAAO,QAAiB;AAClC,QAAM,eAAe;EADJ;AAEjB,OAAK,OAAO;;;;;;;;AAShB,SAAS,aAAa,QAAiB,UAA4B;AACjE,KAAI,MAAM,QAAQ,SAAS,CACzB,QACE,MAAM,QAAQ,OAAO,IACrB,OAAO,WAAW,SAAS,UAC3B,SAAS,OAAO,MAAM,MAAM,aAAa,OAAO,IAAI,KAAK,CAAC;AAG9D,KAAI,cAAc,SAAS,EAAE;AAC3B,MAAI,CAAC,cAAc,OAAO,CAAE,QAAO;AAEnC,SAAO,OAAO,KAAK,SAAS,CAAC,OAC1B,QACC,OAAO,OAAO,QAAQ,IAAI,IAAI,aAAa,OAAO,MAAM,SAAS,KAAK,CACzE;;AAEH,QAAO,WAAW;;AAGpB,SAAS,cAAc,OAAkD;AACvE,QAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,MAAM;;;;;;;AAwB7E,eAAsB,QACpB,KACA,SACqB;CACrB,MAAM,aAAgC,EAAE;CACxC,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,YAAsB,EAAE;CAC5B,IAAI,kBAAkD,EAAE;CACxD,IAAI;CACJ,IAAI;CACJ,IAAI,gBAAgB;CAEpB,IAAI,aAAa;CAEjB,MAAM,aAAa,IAAI,iBAAiB;CAExC,MAAM,UACJ,OACA,MACA,OACA,WACoB;EACpB,MAAM,SAA0B;GAC9B;GACA,UAAU;GACV;GACA;GACA;GACD;AACD,aAAW,KAAK,OAAO;EACvB,MAAM,SAA0B;GAC9B,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,QAAQ,WAAmB;AAGzB,WAAO,QAAQ,OAAO,UAAU,OAAO,OAAO,IAAI,OAAO;AACzD,WAAO;;GAEV;AACD,SAAO;;CAMT,MAAM,eACJ,OACA,OACA,cAEA,OAAO,OAAO,SAAS,yBAAyB,OAAO,UAAU;CAEnE,MAAM,IAAiB;EACrB,MAAM,KAAK,SAAS;AAClB,eAAY;GACZ,MAAM,IAAI,MAAM,QAAQ,OAAO,KAAK,SAAS,EAC3C,QAAQ,WAAW,QACpB,CAAC;AACF,WAAQ,EAAE;AACV,eAAY,EAAE;AACd,qBAAkB,EAAE;AACpB,eAAY,EAAE;AACd,mBAAgB,EAAE;AAClB,OAAI,CAAC,EAAE,UAAW,cAAa;AAC/B,OAAI,EAAE,QAAS,eAAc,EAAE;;EAEjC,QAAQ;AACN,WAAQ,OAAO,SAAS;;EAE1B,IAAI,QAAQ;AACV,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,QAAQ;AACV,UAAO,QAAQ,KAAK,UAAU,EAAE;;EAElC,IAAI,WAAW;AACb,UAAO,QAAQ,KAAK;;EAEtB,YAAY;AACV,UAAO,OACL,aACA,eACA,QACA,gBAAgB,SAAY,2CAC7B;;EAEH,WAAW,MAAM;AACf,UAAO,OACL,cAAc,KAAK,IACnB,UAAU,SAAS,KAAK,EACxB,QACA,kBAAkB,KAAK,0BACrB,UAAU,SAAS,UAAU,KAAK,KAAK,GAAG,OAC3C,GACF;;EAEH,eAAe,MAAM,UAAU;GAC7B,MAAM,WAAW,gBAAgB,QAAQ,MAAM,EAAE,SAAS,KAAK;GAC/D,MAAM,OAAO,SAAS,MAAM,MAAM,aAAa,EAAE,MAAM,SAAS,CAAC;GAEjE,MAAM,OAAO,SAAS,SAClB,SACG,KAAK,MAAM,IAAI,OAAO,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,KAAK,CAAC,GAAG,CACxD,KAAK,KAAK,GACb;AACJ,UAAO,OACL,kBAAkB,KAAK,IACvB,MACA,QACA,kBAAkB,KAAK,sBAAsB,KAAK,UAChD,SACD,CAAC,mBAAmB,KAAK,GAC3B;;EAEH,MAAM,OAAe,SAAkB;GACrC,MAAM,IAAI,QAAQ,MAAM;AACxB,UAAO,OAAO,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;;EAEnD,OAAO;GACL,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,OAAO,cAAc,MAAM,gBAAgB;KACjD,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,oBAAoB,OAAO,UAAU;;GAE1D,MAAM,SAAS,UAAU;IACvB,MAAM,EAAE,OAAO,cAAc,MAAM,cAAc;KAC/C,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,kBAAkB,OAAO,UAAU;;GAExD,MAAM,OAAO,MAAM;IACjB,MAAM,EAAE,OAAO,cAAc,MAAM,YAAY,MAAM;KACnD,OAAO;KACP,QAAQ;KACT,CAAC;AACF,WAAO,YAAY,SAAS,KAAK,QAAQ,OAAO,UAAU;;GAE7D;EACD,KAAK,QAAQ;AACX,SAAM,IAAI,WAAW,OAAO;;EAE/B;CAID,MAAM,YAAY,IAAI,aAAa,QAAQ;CAC3C,IAAI;AAEJ,KAAI;AACF,MAAI,cAAc,OAChB,OAAM,IAAI,KAAK,EAAE;OACZ;GAGL,MAAM,UAAU,IAAI,SAAgB,GAAG,WAAW;AAChD,YAAQ,iBAAiB;AACvB,gBAAW,OAAO;AAClB,4BAAO,IAAI,MAAM,wBAAwB,UAAU,IAAI,CAAC;OACvD,UAAU;KACb;AACF,SAAM,QAAQ,KAAK,CAAC,QAAQ,QAAQ,IAAI,KAAK,EAAE,CAAC,EAAE,QAAQ,CAAC;;UAEtD,KAAK;AACZ,MAAI,eAAe,WACjB,QAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB,SAAS,EAAE,QAAQ,IAAI,QAAQ;GAC/B;GACA,QAAQ;GACR,SAAS;GACV;AAEH,SAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB;GACA,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACvD,SAAS;GACV;WACO;AACR,MAAI,MAAO,cAAa,MAAM;;CAGhC,MAAM,SAAS,WAAW,OACvB,MAAM,EAAE,QAAS,EAAE,aAAa,UAAU,CAAC,QAAQ,OACrD;AAED,QAAO;EACL,IAAI,QAAQ;EACZ,aAAa,IAAI;EACjB;EACA;EACA,SAAS;EAIT,cAAe,cAAc,CAAC,UAAW;EAC1C"}

package/dist/evals/run-evals.d.ts

@@ -12,12 +12,15 @@ interface RunEvalsOptions {
   baseUrl: string;
   /** Substring filter on `<agent>/<id>` (or an exact agent id). */
   filter?: string;
+  /**
+   * Only run evals whose `tags` intersect this list. Empty/undefined runs all.
+   * Tags live on the eval def, so filtering happens after each file is loaded.
+   */
+  tags?: string[];
   /** Soft assertion failures also fail the eval. */
   strict?: boolean;
   /** Extra request headers for the driver (e.g. auth for a deployed app). */
   headers?: Record<string, string>;
-  /** Per-turn wall-clock timeout (ms) before a turn is failed. Defaults to 120s. */
-  timeoutMs?: number;
   /**
    * Max evals to drive concurrently. Each eval opens one stream to the app as
    * the same user, so keep this at or below the app's
@@ -54,6 +57,18 @@ interface RunEvalsOptions {
   warehouseId?: string;
   /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */
   now?: number;
+  /**
+   * Default per-eval timeout (ms): `runEval` races the whole test against it and
+   * it also caps each driver turn. A per-eval `def.timeoutMs` overrides it, and
+   * it wins over an agent's `evals.config.ts` `timeoutMs`. Unbounded when unset.
+   */
+  timeoutMs?: number;
+  /**
+   * Re-run an eval up to this many extra times when it fails on infrastructure —
+   * a thrown error/timeout (`result.error`) or a transport/agent turn failure
+   * (`result.infraFailure`). Assertion failures are never retried. Defaults to `0`.
+   */
+  retries?: number;
   /** Progress callback, invoked as evals are discovered, started, and finished. */
   onEvent?: (event: EvalProgress) => void;
 }
@@ -83,6 +98,20 @@ interface EvalRunSummary {
     finish: FinishOutcome;
   };
 }
+/**
+ * Run `attempt` up to `1 + retries` times, stopping as soon as it returns a
+ * result that is neither a thrown error / per-eval timeout (`error`) nor a
+ * transport/agent turn failure (`infraFailure`). Assertion failures set
+ * neither, so a failed-but-completed eval is returned on the first try and
+ * never retried. Returns the last result when every attempt failed on infra.
+ *
+ * Between attempts it waits a full-jittered exponential backoff (infra flakes
+ * are overload-correlated). `retries` is coerced to a finite non-negative
+ * integer; `baseDelayMs: 0` disables the wait (tests).
+ */
+declare function runWithRetries(retries: number, attempt: (attemptNumber: number) => Promise<EvalResult>, options?: {
+  baseDelayMs?: number;
+}): Promise<EvalResult>;
 /**
  * Discover, load, and run every eval under each agent's `evals/` dir, driving
  * the agents on a running app. Never throws for an individual eval — load/run
@@ -90,5 +119,5 @@ interface EvalRunSummary {
  */
 declare function runEvalsInDir(options: RunEvalsOptions): Promise<EvalRunSummary>;
 //#endregion
-export { EvalProgress, EvalRunSummary, RunEvalsOptions, runEvalsInDir };
+export { EvalProgress, EvalRunSummary, RunEvalsOptions, runEvalsInDir, runWithRetries };
 //# sourceMappingURL=run-evals.d.ts.map

package/dist/evals/run-evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-evals.d.ts","names":[],"sources":["../../src/evals/run-evals.ts"],"mappings":";;;;;;;UAciB,eAAA;;EAEf,OAAA;EAF8B;EAI9B,OAAA;EAMU;EAJV,MAAA;EAyCkB;EAvClB,MAAA;EAuC8B;EArC9B,OAAA,GAAU,MAAA;EANV;EAQA,SAAA;EAJA;;;;;;EAWA,WAAA;EAQE;;;;;EAFF,MAAA;IACE,IAAA;IACA,KAAA;IACA,YAAA,UAeF;IAbE,cAAA;EAAA;EAiBgB;;;;EAXlB,KAAA;IAAU,IAAA;IAAc,KAAA;IAAe,KAAA;EAAA;EAef;;;;EAVxB,eAAA,GAAkB,eAAA;EAYa;EAV/B,WAAA;EAWI;EATJ,GAAA;EAS4B;EAP5B,OAAA,IAAW,KAAA,EAAO,YAAA;AAAA;AAAA,KAGR,YAAA;EACN,IAAA;EAAoB,KAAA;AAAA;EACpB,IAAA;EAAqB,KAAA;AAAA;EACrB,IAAA;EAAe,EAAA;EAAY,KAAA;EAAe,KAAA;AAAA;EAC1C,IAAA;EAAgB,MAAA,EAAQ,UAAA;EAAY,KAAA;EAAe,KAAA;AAAA;AAAA,UAExC,cAAA;EACf,OAAA,EAAS,UAAA;EAE6D;EAAtE,MAAA;IAAW,KAAA;IAAe,MAAA,EAAQ,aAAA;IAAe,MAAA,EAAQ,aAAA;EAAA;AAAA;;;;;;iBAoOrC,aAAA,CACpB,OAAA,EAAS,eAAA,GACR,OAAA,CAAQ,cAAA"}
+{"version":3,"file":"run-evals.d.ts","names":[],"sources":["../../src/evals/run-evals.ts"],"mappings":";;;;;;;UAmBiB,eAAA;;EAEf,OAAA;EAF8B;EAI9B,OAAA;EAWU;EATV,MAAA;EAwDkB;;;;EAnDlB,IAAA;EALA;EAOA,MAAA;EAAA;EAEA,OAAA,GAAU,MAAA;EAAA;;;;;;EAOV,WAAA;EAiBA;;;;;EAXA,MAAA;IACE,IAAA;IACA,KAAA;IACA,YAAA,UA6BF;IA3BE,cAAA;EAAA;EA6BS;;;AAGb;EA1BE,KAAA;IAAU,IAAA;IAAc,KAAA;IAAe,KAAA;EAAA;EA4BnC;;;;EAvBJ,eAAA,GAAkB,eAAA;EAwB4B;EAtB9C,WAAA;EAuBoB;EArBpB,GAAA;EAqBwC;;;;AAE1C;EAjBE,SAAA;;;;;;EAMA,OAAA;EAYA;EAVA,OAAA,IAAW,KAAA,EAAO,YAAA;AAAA;AAAA,KAGR,YAAA;EACN,IAAA;EAAoB,KAAA;AAAA;EACpB,IAAA;EAAqB,KAAA;AAAA;EACrB,IAAA;EAAe,EAAA;EAAY,KAAA;EAAe,KAAA;AAAA;EAC1C,IAAA;EAAgB,MAAA,EAAQ,UAAA;EAAY,KAAA;EAAe,KAAA;AAAA;AAAA,UAExC,cAAA;EACf,OAAA,EAAS,UAAA;EA2OmC;EAzO5C,MAAA;IAAW,KAAA;IAAe,MAAA,EAAQ,aAAA;IAAe,MAAA,EAAQ,aAAA;EAAA;AAAA;;;;;;;;;;;;iBAuOrC,cAAA,CACpB,OAAA,UACA,OAAA,GAAU,aAAA,aAA0B,OAAA,CAAQ,UAAA,GAC5C,OAAA;EAAW,WAAA;AAAA,IACV,OAAA,CAAQ,UAAA;;;;;;iBA8GW,aAAA,CACpB,OAAA,EAAS,eAAA,GACR,OAAA,CAAQ,cAAA"}