@databricks/appkit 0.72.0 → 0.74.0

package/dist/evals/define-eval.js

@@ -22,7 +22,11 @@ function defineEval(def) {
 	if (typeof def.test !== "function") throw new Error("defineEval: `test` must be a function");
 	return def;
 }
+/** Define per-directory eval config. Default-export from `evals.config.ts`. */
+function defineEvalConfig(config) {
+	return config;
+}
 
 //#endregion
-export { defineEval };
+export { defineEval, defineEvalConfig };
 //# sourceMappingURL=define-eval.js.map

package/dist/evals/define-eval.js.map

@@ -1 +1 @@
-{"version":3,"file":"define-eval.js","names":[],"sources":["../../src/evals/define-eval.ts"],"sourcesContent":["import type { EvalDefinition } from \"./types\";\n\n/**\n * Define an agent eval. Default-export the result from a\n * `server/agents/<id>/evals/*.eval.ts` file.\n *\n * @example\n * ```ts\n * import { defineEval, includes } from \"@databricks/appkit/beta\";\n *\n * export default defineEval({\n *   description: \"Weather agent basic coverage\",\n *   async test(t) {\n *     await t.send(\"What's the weather in Brooklyn?\");\n *     t.succeeded();\n *     t.calledTool(\"get_weather\");\n *     t.check(t.reply, includes(\"Sunny\"));\n *   },\n * });\n * ```\n */\nexport function defineEval(def: EvalDefinition): EvalDefinition {\n  if (typeof def.test !== \"function\") {\n    throw new Error(\"defineEval: `test` must be a function\");\n  }\n  return def;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,WAAW,KAAqC;AAC9D,KAAI,OAAO,IAAI,SAAS,WACtB,OAAM,IAAI,MAAM,wCAAwC;AAE1D,QAAO"}
+{"version":3,"file":"define-eval.js","names":[],"sources":["../../src/evals/define-eval.ts"],"sourcesContent":["import type { EvalConfig, EvalDefinition } from \"./types\";\n\n/**\n * Define an agent eval. Default-export the result from a\n * `server/agents/<id>/evals/*.eval.ts` file.\n *\n * @example\n * ```ts\n * import { defineEval, includes } from \"@databricks/appkit/beta\";\n *\n * export default defineEval({\n *   description: \"Weather agent basic coverage\",\n *   async test(t) {\n *     await t.send(\"What's the weather in Brooklyn?\");\n *     t.succeeded();\n *     t.calledTool(\"get_weather\");\n *     t.check(t.reply, includes(\"Sunny\"));\n *   },\n * });\n * ```\n */\nexport function defineEval(def: EvalDefinition): EvalDefinition {\n  if (typeof def.test !== \"function\") {\n    throw new Error(\"defineEval: `test` must be a function\");\n  }\n  return def;\n}\n\n/** Define per-directory eval config. Default-export from `evals.config.ts`. */\nexport function defineEvalConfig(config: EvalConfig): EvalConfig {\n  return config;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,WAAW,KAAqC;AAC9D,KAAI,OAAO,IAAI,SAAS,WACtB,OAAM,IAAI,MAAM,wCAAwC;AAE1D,QAAO;;;AAIT,SAAgB,iBAAiB,QAAgC;AAC/D,QAAO"}

package/dist/evals/discover.d.ts

@@ -8,6 +8,13 @@ interface DiscoveredEval {
   /** The agent id (the `server/agents/<agent>` directory name). */
   agent: string;
 }
+/** A per-agent `evals.config.ts` found under `server/agents/<agent>/evals/`. */
+interface DiscoveredEvalConfig {
+  /** Absolute path to the `evals.config.ts` file. */
+  file: string;
+  /** The agent id whose evals this config applies to. */
+  agent: string;
+}
 /**
  * Discover evals under `<rootDir>/server/agents/<agent>/evals/` — co-located
  * with each agent's `agent.{md,ts}` (same folder-per-agent layout the agents
@@ -15,6 +22,13 @@ interface DiscoveredEval {
  * path relative to that evals dir with `.eval.ts` stripped. Sorted + stable.
  */
 declare function discoverEvalFiles(rootDir: string): DiscoveredEval[];
+/**
+ * Discover the per-agent `evals.config.ts` (from {@link defineEvalConfig}) at
+ * `<rootDir>/server/agents/<agent>/evals/evals.config.ts`. Config is per-agent:
+ * each agent's config applies only to that agent's evals. Agents without a
+ * config file are omitted. Returns a stable, sorted list.
+ */
+declare function discoverEvalConfigs(rootDir: string): DiscoveredEvalConfig[];
 //#endregion
-export { DiscoveredEval, discoverEvalFiles };
+export { DiscoveredEval, DiscoveredEvalConfig, discoverEvalConfigs, discoverEvalFiles };
 //# sourceMappingURL=discover.d.ts.map

package/dist/evals/discover.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"discover.d.ts","names":[],"sources":["../../src/evals/discover.ts"],"mappings":";;UAOiB,cAAA;EAAc;EAE7B,IAAA;EAF6B;EAI7B,EAAA;EAAA;EAEA,KAAA;AAAA;;AAoBF;;;;;iBAAgB,iBAAA,CAAkB,OAAA,WAAkB,cAAA"}
+{"version":3,"file":"discover.d.ts","names":[],"sources":["../../src/evals/discover.ts"],"mappings":";;UAOiB,cAAA;EAAc;EAE7B,IAAA;EAF6B;EAI7B,EAAA;EAAA;EAEA,KAAA;AAAA;;UAIe,oBAAA;EAAoB;EAEnC,IAAA;EAAA;EAEA,KAAA;AAAA;;;;;AAsDF;;iBAlCgB,iBAAA,CAAkB,OAAA,WAAkB,cAAA;;;;;;;iBAkCpC,mBAAA,CAAoB,OAAA,WAAkB,oBAAA"}

package/dist/evals/discover.js

@@ -1,7 +1,7 @@
 import { agentDirNames } from "../core/agent/agent-dirs.js";
 import { CODE_AGENTS_SOURCE_DIR } from "../core/agent/load-code-agents.js";
 import path from "node:path";
-import { readdirSync } from "node:fs";
+import { existsSync, readdirSync } from "node:fs";
 
 //#region src/evals/discover.ts
 /** Recursively collect `*.eval.ts` files under `dir`. Empty when `dir` is absent. */
@@ -43,7 +43,31 @@ function discoverEvalFiles(rootDir) {
 	}
 	return out.sort((a, b) => a.agent.localeCompare(b.agent) || a.id.localeCompare(b.id));
 }
+/**
+* Discover the per-agent `evals.config.ts` (from {@link defineEvalConfig}) at
+* `<rootDir>/server/agents/<agent>/evals/evals.config.ts`. Config is per-agent:
+* each agent's config applies only to that agent's evals. Agents without a
+* config file are omitted. Returns a stable, sorted list.
+*/
+function discoverEvalConfigs(rootDir) {
+	const agentsDir = path.join(rootDir, CODE_AGENTS_SOURCE_DIR);
+	const out = [];
+	let entries;
+	try {
+		entries = readdirSync(agentsDir, { withFileTypes: true });
+	} catch {
+		return out;
+	}
+	for (const agent of agentDirNames(entries)) {
+		const file = path.join(agentsDir, agent, "evals", "evals.config.ts");
+		if (existsSync(file)) out.push({
+			file,
+			agent
+		});
+	}
+	return out.sort((a, b) => a.agent.localeCompare(b.agent));
+}
 
 //#endregion
-export { discoverEvalFiles };
+export { discoverEvalConfigs, discoverEvalFiles };
 //# sourceMappingURL=discover.js.map

package/dist/evals/discover.js.map

@@ -1 +1 @@
-{"version":3,"file":"discover.js","names":[],"sources":["../../src/evals/discover.ts"],"sourcesContent":["import { type Dirent, readdirSync } from \"node:fs\";\nimport path from \"node:path\";\n\nimport { agentDirNames } from \"../core/agent/agent-dirs\";\nimport { CODE_AGENTS_SOURCE_DIR } from \"../core/agent/load-code-agents\";\n\n/** An eval file found under `server/agents/<agent>/evals/`. */\nexport interface DiscoveredEval {\n  /** Absolute path to the `*.eval.ts` file. */\n  file: string;\n  /** Id relative to the agent's evals dir, without `.eval.ts` (e.g. `weather/basic`). */\n  id: string;\n  /** The agent id (the `server/agents/<agent>` directory name). */\n  agent: string;\n}\n\n/** Recursively collect `*.eval.ts` files under `dir`. Empty when `dir` is absent. */\nfunction evalFilesIn(dir: string): string[] {\n  try {\n    return readdirSync(dir, { recursive: true, withFileTypes: true })\n      .filter((e) => e.isFile() && e.name.endsWith(\".eval.ts\"))\n      .map((e) => path.join(e.parentPath, e.name));\n  } catch {\n    return [];\n  }\n}\n\n/**\n * Discover evals under `<rootDir>/server/agents/<agent>/evals/` — co-located\n * with each agent's `agent.{md,ts}` (same folder-per-agent layout the agents\n * plugin discovers). The agent id is the folder name; the eval id is the file\n * path relative to that evals dir with `.eval.ts` stripped. Sorted + stable.\n */\nexport function discoverEvalFiles(rootDir: string): DiscoveredEval[] {\n  const agentsDir = path.join(rootDir, CODE_AGENTS_SOURCE_DIR);\n  const out: DiscoveredEval[] = [];\n\n  let entries: Dirent[];\n  try {\n    entries = readdirSync(agentsDir, { withFileTypes: true });\n  } catch {\n    return out;\n  }\n\n  for (const agent of agentDirNames(entries)) {\n    const evalsDir = path.join(agentsDir, agent, \"evals\");\n    for (const file of evalFilesIn(evalsDir)) {\n      const id = path\n        .relative(evalsDir, file)\n        .replace(/\\.eval\\.ts$/, \"\")\n        .split(path.sep)\n        .join(\"/\");\n      out.push({ file, id, agent });\n    }\n  }\n\n  return out.sort(\n    (a, b) => a.agent.localeCompare(b.agent) || a.id.localeCompare(b.id),\n  );\n}\n"],"mappings":";;;;;;;AAiBA,SAAS,YAAY,KAAuB;AAC1C,KAAI;AACF,SAAO,YAAY,KAAK;GAAE,WAAW;GAAM,eAAe;GAAM,CAAC,CAC9D,QAAQ,MAAM,EAAE,QAAQ,IAAI,EAAE,KAAK,SAAS,WAAW,CAAC,CACxD,KAAK,MAAM,KAAK,KAAK,EAAE,YAAY,EAAE,KAAK,CAAC;SACxC;AACN,SAAO,EAAE;;;;;;;;;AAUb,SAAgB,kBAAkB,SAAmC;CACnE,MAAM,YAAY,KAAK,KAAK,SAAS,uBAAuB;CAC5D,MAAM,MAAwB,EAAE;CAEhC,IAAI;AACJ,KAAI;AACF,YAAU,YAAY,WAAW,EAAE,eAAe,MAAM,CAAC;SACnD;AACN,SAAO;;AAGT,MAAK,MAAM,SAAS,cAAc,QAAQ,EAAE;EAC1C,MAAM,WAAW,KAAK,KAAK,WAAW,OAAO,QAAQ;AACrD,OAAK,MAAM,QAAQ,YAAY,SAAS,EAAE;GACxC,MAAM,KAAK,KACR,SAAS,UAAU,KAAK,CACxB,QAAQ,eAAe,GAAG,CAC1B,MAAM,KAAK,IAAI,CACf,KAAK,IAAI;AACZ,OAAI,KAAK;IAAE;IAAM;IAAI;IAAO,CAAC;;;AAIjC,QAAO,IAAI,MACR,GAAG,MAAM,EAAE,MAAM,cAAc,EAAE,MAAM,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,CACrE"}
+{"version":3,"file":"discover.js","names":[],"sources":["../../src/evals/discover.ts"],"sourcesContent":["import { type Dirent, existsSync, readdirSync } from \"node:fs\";\nimport path from \"node:path\";\n\nimport { agentDirNames } from \"../core/agent/agent-dirs\";\nimport { CODE_AGENTS_SOURCE_DIR } from \"../core/agent/load-code-agents\";\n\n/** An eval file found under `server/agents/<agent>/evals/`. */\nexport interface DiscoveredEval {\n  /** Absolute path to the `*.eval.ts` file. */\n  file: string;\n  /** Id relative to the agent's evals dir, without `.eval.ts` (e.g. `weather/basic`). */\n  id: string;\n  /** The agent id (the `server/agents/<agent>` directory name). */\n  agent: string;\n}\n\n/** A per-agent `evals.config.ts` found under `server/agents/<agent>/evals/`. */\nexport interface DiscoveredEvalConfig {\n  /** Absolute path to the `evals.config.ts` file. */\n  file: string;\n  /** The agent id whose evals this config applies to. */\n  agent: string;\n}\n\n/** Recursively collect `*.eval.ts` files under `dir`. Empty when `dir` is absent. */\nfunction evalFilesIn(dir: string): string[] {\n  try {\n    return readdirSync(dir, { recursive: true, withFileTypes: true })\n      .filter((e) => e.isFile() && e.name.endsWith(\".eval.ts\"))\n      .map((e) => path.join(e.parentPath, e.name));\n  } catch {\n    return [];\n  }\n}\n\n/**\n * Discover evals under `<rootDir>/server/agents/<agent>/evals/` — co-located\n * with each agent's `agent.{md,ts}` (same folder-per-agent layout the agents\n * plugin discovers). The agent id is the folder name; the eval id is the file\n * path relative to that evals dir with `.eval.ts` stripped. Sorted + stable.\n */\nexport function discoverEvalFiles(rootDir: string): DiscoveredEval[] {\n  const agentsDir = path.join(rootDir, CODE_AGENTS_SOURCE_DIR);\n  const out: DiscoveredEval[] = [];\n\n  let entries: Dirent[];\n  try {\n    entries = readdirSync(agentsDir, { withFileTypes: true });\n  } catch {\n    return out;\n  }\n\n  for (const agent of agentDirNames(entries)) {\n    const evalsDir = path.join(agentsDir, agent, \"evals\");\n    for (const file of evalFilesIn(evalsDir)) {\n      const id = path\n        .relative(evalsDir, file)\n        .replace(/\\.eval\\.ts$/, \"\")\n        .split(path.sep)\n        .join(\"/\");\n      out.push({ file, id, agent });\n    }\n  }\n\n  return out.sort(\n    (a, b) => a.agent.localeCompare(b.agent) || a.id.localeCompare(b.id),\n  );\n}\n\n/**\n * Discover the per-agent `evals.config.ts` (from {@link defineEvalConfig}) at\n * `<rootDir>/server/agents/<agent>/evals/evals.config.ts`. Config is per-agent:\n * each agent's config applies only to that agent's evals. Agents without a\n * config file are omitted. Returns a stable, sorted list.\n */\nexport function discoverEvalConfigs(rootDir: string): DiscoveredEvalConfig[] {\n  const agentsDir = path.join(rootDir, CODE_AGENTS_SOURCE_DIR);\n  const out: DiscoveredEvalConfig[] = [];\n\n  let entries: Dirent[];\n  try {\n    entries = readdirSync(agentsDir, { withFileTypes: true });\n  } catch {\n    return out;\n  }\n\n  for (const agent of agentDirNames(entries)) {\n    const file = path.join(agentsDir, agent, \"evals\", \"evals.config.ts\");\n    if (existsSync(file)) out.push({ file, agent });\n  }\n\n  return out.sort((a, b) => a.agent.localeCompare(b.agent));\n}\n"],"mappings":";;;;;;;AAyBA,SAAS,YAAY,KAAuB;AAC1C,KAAI;AACF,SAAO,YAAY,KAAK;GAAE,WAAW;GAAM,eAAe;GAAM,CAAC,CAC9D,QAAQ,MAAM,EAAE,QAAQ,IAAI,EAAE,KAAK,SAAS,WAAW,CAAC,CACxD,KAAK,MAAM,KAAK,KAAK,EAAE,YAAY,EAAE,KAAK,CAAC;SACxC;AACN,SAAO,EAAE;;;;;;;;;AAUb,SAAgB,kBAAkB,SAAmC;CACnE,MAAM,YAAY,KAAK,KAAK,SAAS,uBAAuB;CAC5D,MAAM,MAAwB,EAAE;CAEhC,IAAI;AACJ,KAAI;AACF,YAAU,YAAY,WAAW,EAAE,eAAe,MAAM,CAAC;SACnD;AACN,SAAO;;AAGT,MAAK,MAAM,SAAS,cAAc,QAAQ,EAAE;EAC1C,MAAM,WAAW,KAAK,KAAK,WAAW,OAAO,QAAQ;AACrD,OAAK,MAAM,QAAQ,YAAY,SAAS,EAAE;GACxC,MAAM,KAAK,KACR,SAAS,UAAU,KAAK,CACxB,QAAQ,eAAe,GAAG,CAC1B,MAAM,KAAK,IAAI,CACf,KAAK,IAAI;AACZ,OAAI,KAAK;IAAE;IAAM;IAAI;IAAO,CAAC;;;AAIjC,QAAO,IAAI,MACR,GAAG,MAAM,EAAE,MAAM,cAAc,EAAE,MAAM,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,CACrE;;;;;;;;AASH,SAAgB,oBAAoB,SAAyC;CAC3E,MAAM,YAAY,KAAK,KAAK,SAAS,uBAAuB;CAC5D,MAAM,MAA8B,EAAE;CAEtC,IAAI;AACJ,KAAI;AACF,YAAU,YAAY,WAAW,EAAE,eAAe,MAAM,CAAC;SACnD;AACN,SAAO;;AAGT,MAAK,MAAM,SAAS,cAAc,QAAQ,EAAE;EAC1C,MAAM,OAAO,KAAK,KAAK,WAAW,OAAO,SAAS,kBAAkB;AACpE,MAAI,WAAW,KAAK,CAAE,KAAI,KAAK;GAAE;GAAM;GAAO,CAAC;;AAGjD,QAAO,IAAI,MAAM,GAAG,MAAM,EAAE,MAAM,cAAc,EAAE,MAAM,CAAC"}

package/dist/evals/http-driver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"http-driver.d.ts","names":[],"sources":["../../src/evals/http-driver.ts"],"mappings":";;;UAGiB,iBAAA;;EAEf,OAAA;EAFgC;EAIhC,KAAA;EAEgB;EAAhB,OAAA,GAAU,MAAA;EAFV;EAIA,IAAA;EAFU;EAIV,WAAA;EAAA;;;;AA6GF;;;EApGE,SAAA;AAAA;;;;;;;iBAoGc,gBAAA,CAAiB,OAAA,EAAS,iBAAA,GAAoB,UAAA"}
+{"version":3,"file":"http-driver.d.ts","names":[],"sources":["../../src/evals/http-driver.ts"],"mappings":";;;UAGiB,iBAAA;;EAEf,OAAA;EAFgC;EAIhC,KAAA;EAEgB;EAAhB,OAAA,GAAU,MAAA;EAFV;EAIA,IAAA;EAFU;EAIV,WAAA;EAAA;;;;AA4IF;;;EAnIE,SAAA;AAAA;;;;;;;iBAmIc,gBAAA,CAAiB,OAAA,EAAS,iBAAA,GAAoB,UAAA"}

package/dist/evals/http-driver.js

@@ -1,19 +1,37 @@
 import { readSseEvents } from "../stream/sse-reader.js";
 
 //#region src/evals/http-driver.ts
-/** Record a `function_call` output item once per call id (deduped). */
+/**
+* Record a `function_call` output item, keyed by `call_id ?? name`, capturing
+* its parsed arguments. A later `done` event's fuller args win over the initial
+* `added` event's (often empty) args.
+*/
 function recordToolCall(item, state) {
 	if (item?.type !== "function_call" || !item.name) return;
 	const key = item.call_id ?? item.name;
-	if (state.seen.has(key)) return;
-	state.seen.add(key);
-	state.toolCalls.push(item.name);
+	const args = parseArgs(item.arguments);
+	const existing = state.toolCalls.get(key);
+	if (!existing) state.toolCalls.set(key, {
+		name: item.name,
+		args
+	});
+	else if (Object.keys(args).length > 0) existing.args = args;
 }
 /** Apply an `appkit.metadata` event's thread/trace ids. */
 function applyMetadata(data, state, setThread) {
 	if (data?.threadId) setThread(data.threadId);
 	if (data?.mlflowTraceId) state.traceId = data.mlflowTraceId;
 }
+/** Parse a function-call `arguments` JSON string; `{}` on missing/invalid. */
+function parseArgs(raw) {
+	if (typeof raw !== "string" || raw.trim() === "") return {};
+	try {
+		const parsed = JSON.parse(raw);
+		return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
+	} catch {
+		return {};
+	}
+}
 /** Parse a single Responses-API SSE `data:` payload into the running totals. */
 function applyEvent(event, state, setThread) {
 	const type = event.type;
@@ -55,62 +73,71 @@ function createHttpDriver(options) {
 	const chatPath = options.path ?? "/api/agents/chat";
 	const timeoutMs = options.timeoutMs ?? 12e4;
 	let threadId;
-	return { async send(message) {
-		const signal = AbortSignal.timeout(timeoutMs);
-		let res;
-		try {
-			res = await fetch(`${options.baseUrl}${chatPath}`, {
-				method: "POST",
-				headers: {
-					"content-type": "application/json",
-					...options.headers
-				},
-				body: buildRequestBody(message, options, threadId),
-				redirect: "manual",
-				signal
-			});
-		} catch {
-			return {
+	return {
+		reset() {
+			threadId = void 0;
+		},
+		async send(message, opts) {
+			const timeout = AbortSignal.timeout(timeoutMs);
+			const signal = opts?.signal ? AbortSignal.any([timeout, opts.signal]) : timeout;
+			let res;
+			try {
+				res = await fetch(`${options.baseUrl}${chatPath}`, {
+					method: "POST",
+					headers: {
+						"content-type": "application/json",
+						...options.headers
+					},
+					body: buildRequestBody(message, options, threadId),
+					redirect: "manual",
+					signal
+				});
+			} catch {
+				return {
+					reply: "",
+					toolCalls: [],
+					toolCallDetails: [],
+					succeeded: false
+				};
+			}
+			if (!res.ok || !res.body) return {
 				reply: "",
 				toolCalls: [],
-				succeeded: false
+				toolCallDetails: [],
+				succeeded: false,
+				sessionId: threadId
 			};
-		}
-		if (!res.ok || !res.body) return {
-			reply: "",
-			toolCalls: [],
-			succeeded: false,
-			sessionId: threadId
-		};
-		const state = {
-			reply: "",
-			toolCalls: [],
-			seen: /* @__PURE__ */ new Set(),
-			ok: true
-		};
-		const setThread = (id) => {
-			threadId = id;
-		};
-		try {
-			for await (const { event, data } of readSseEvents(res.body, signal)) {
-				if (data === "[DONE]") continue;
-				if (event === "error") state.ok = false;
-				try {
-					applyEvent(JSON.parse(data), state, setThread);
-				} catch {}
+			const state = {
+				reply: "",
+				toolCalls: /* @__PURE__ */ new Map(),
+				ok: true
+			};
+			const setThread = (id) => {
+				threadId = id;
+			};
+			try {
+				for await (const { event, data } of readSseEvents(res.body, signal)) {
+					if (data === "[DONE]") continue;
+					if (event === "error") state.ok = false;
+					try {
+						applyEvent(JSON.parse(data), state, setThread);
+					} catch {}
+				}
+			} catch {
+				state.ok = false;
 			}
-		} catch {
-			state.ok = false;
+			if (signal.aborted) state.ok = false;
+			const toolCallDetails = [...state.toolCalls.values()];
+			return {
+				reply: state.reply,
+				toolCalls: toolCallDetails.map((c) => c.name),
+				toolCallDetails,
+				succeeded: state.ok,
+				sessionId: threadId,
+				traceId: state.traceId
+			};
 		}
-		if (signal.aborted) state.ok = false;
-		return {
-			reply: state.reply,
-			toolCalls: state.toolCalls,
-			succeeded: state.ok,
-			sessionId: threadId,
-			traceId: state.traceId
-		};
-	} };
+	};
 }
 
 //#endregion

package/dist/evals/http-driver.js.map

@@ -1 +1 @@
-{"version":3,"file":"http-driver.js","names":[],"sources":["../../src/evals/http-driver.ts"],"sourcesContent":["import { readSseEvents } from \"../stream/sse-reader\";\nimport type { DriveResult, EvalDriver } from \"./types\";\n\nexport interface HttpDriverOptions {\n  /** Base URL of the running app, e.g. `http://localhost:3000`. */\n  baseUrl: string;\n  /** Agent alias to target. Omit to use the app's default agent. */\n  agent?: string;\n  /** Extra request headers (e.g. auth for a deployed app). */\n  headers?: Record<string, string>;\n  /** Chat endpoint path. Defaults to `/api/agents/chat`. */\n  path?: string;\n  /** MLflow run id to link each turn's trace to (for evaluation runs). */\n  mlflowRunId?: string;\n  /**\n   * Max wall-clock time for a single turn before it is abandoned as a failed\n   * turn (`succeeded: false`). Without this a hung agent — a blocked tool, a\n   * stalled model — never ends the SSE stream (heartbeats keep it alive), so\n   * the read loop spins forever and wedges the whole sequential suite.\n   * Defaults to 120s.\n   */\n  // ponytail: total per-turn cap; switch to an idle timeout if long legit turns get killed.\n  timeoutMs?: number;\n}\n\n/** Mutable running totals accumulated while draining one turn's SSE stream. */\ninterface DriveState {\n  reply: string;\n  toolCalls: string[];\n  seen: Set<string>;\n  ok: boolean;\n  traceId?: string;\n}\n\n/** Record a `function_call` output item once per call id (deduped). */\nfunction recordToolCall(\n  item: { type?: string; name?: string; call_id?: string } | undefined,\n  state: DriveState,\n): void {\n  if (item?.type !== \"function_call\" || !item.name) return;\n  const key = item.call_id ?? item.name;\n  if (state.seen.has(key)) return;\n  state.seen.add(key);\n  state.toolCalls.push(item.name);\n}\n\n/** Apply an `appkit.metadata` event's thread/trace ids. */\nfunction applyMetadata(\n  data: { threadId?: string; mlflowTraceId?: string } | undefined,\n  state: DriveState,\n  setThread: (id: string) => void,\n): void {\n  if (data?.threadId) setThread(data.threadId);\n  if (data?.mlflowTraceId) state.traceId = data.mlflowTraceId;\n}\n\n/** Parse a single Responses-API SSE `data:` payload into the running totals. */\nfunction applyEvent(\n  event: Record<string, unknown>,\n  state: DriveState,\n  setThread: (id: string) => void,\n): void {\n  const type = event.type;\n  if (type === \"response.output_text.delta\") {\n    if (typeof event.delta === \"string\") state.reply += event.delta;\n    return;\n  }\n  if (\n    type === \"response.output_item.added\" ||\n    type === \"response.output_item.done\"\n  ) {\n    const item = event.item as {\n      type?: string;\n      name?: string;\n      call_id?: string;\n      content?: Array<{ text?: string }>;\n    };\n    recordToolCall(item, state);\n    // A terminal `message` item carries the full reply text and *replaces*\n    // the accumulated deltas — the server emits no delta for a full message\n    // (event-translator `handleFullMessage`), so a non-streaming adapter's\n    // whole reply arrives only here. Guard on non-empty so a streaming turn's\n    // trailing done event can't clobber the deltas with \"\".\n    if (type === \"response.output_item.done\" && item.type === \"message\") {\n      const text = (item.content ?? []).map((c) => c.text ?? \"\").join(\"\");\n      if (text) state.reply = text;\n    }\n    return;\n  }\n  if (type === \"error\" || type === \"response.failed\") {\n    state.ok = false;\n    return;\n  }\n  if (type === \"appkit.metadata\") {\n    applyMetadata(\n      event.data as { threadId?: string; mlflowTraceId?: string },\n      state,\n      setThread,\n    );\n  }\n}\n\n/** Build the chat request payload, including only the optional fields that are set. */\nfunction buildRequestBody(\n  message: string,\n  options: HttpDriverOptions,\n  threadId: string | undefined,\n): string {\n  return JSON.stringify({\n    message,\n    ...(options.agent ? { agent: options.agent } : {}),\n    ...(threadId ? { threadId } : {}),\n    ...(options.mlflowRunId ? { mlflowRunId: options.mlflowRunId } : {}),\n  });\n}\n\n/**\n * Drives an agent by POSTing to a running app's chat endpoint and parsing the\n * SSE response. Keeps the thread id across `send`s so multi-turn evals share a\n * conversation. Agent/stream errors surface as `succeeded: false` rather than\n * throwing, so `t.succeeded()` can assert on them.\n */\nexport function createHttpDriver(options: HttpDriverOptions): EvalDriver {\n  const chatPath = options.path ?? \"/api/agents/chat\";\n  const timeoutMs = options.timeoutMs ?? 120_000;\n  let threadId: string | undefined;\n\n  return {\n    async send(message: string): Promise<DriveResult> {\n      // Bounds connect + the entire read below. Passed to both the fetch and\n      // the SSE reader: on expiry the reader is cancelled and the turn fails.\n      const signal = AbortSignal.timeout(timeoutMs);\n      let res: Response;\n      try {\n        res = await fetch(`${options.baseUrl}${chatPath}`, {\n          method: \"POST\",\n          headers: { \"content-type\": \"application/json\", ...options.headers },\n          body: buildRequestBody(message, options, threadId),\n          // A chat endpoint never needs a redirect; following one would replay\n          // custom auth headers (`options.headers`) to the redirect target.\n          // A 3xx becomes an opaque response (res.ok === false), handled as a\n          // failed turn by the !res.ok guard below.\n          redirect: \"manual\",\n          signal,\n        });\n      } catch {\n        return { reply: \"\", toolCalls: [], succeeded: false };\n      }\n\n      if (!res.ok || !res.body) {\n        return {\n          reply: \"\",\n          toolCalls: [],\n          succeeded: false,\n          sessionId: threadId,\n        };\n      }\n\n      const state: DriveState = {\n        reply: \"\",\n        toolCalls: [],\n        seen: new Set<string>(),\n        ok: true,\n      };\n      const setThread = (id: string) => {\n        threadId = id;\n      };\n      try {\n        for await (const { event, data } of readSseEvents(res.body, signal)) {\n          if (data === \"[DONE]\") continue;\n          // A stream-level error frame (a thrown exception in the generator)\n          // is framed as `event: error` with a payload that carries no `type`,\n          // so the SSE event name — not the payload — is the real signal.\n          if (event === \"error\") state.ok = false;\n          try {\n            applyEvent(\n              JSON.parse(data) as Record<string, unknown>,\n              state,\n              setThread,\n            );\n          } catch {\n            // skip malformed event payloads\n          }\n        }\n      } catch {\n        // mid-stream transport error or DoS-guard throw: a broken turn.\n        state.ok = false;\n      }\n      // A timeout ends the iterator cleanly (reader cancel) rather than\n      // throwing, so mark an aborted turn failed explicitly.\n      if (signal.aborted) state.ok = false;\n\n      return {\n        reply: state.reply,\n        toolCalls: state.toolCalls,\n        succeeded: state.ok,\n        sessionId: threadId,\n        traceId: state.traceId,\n      };\n    },\n  };\n}\n"],"mappings":";;;;AAmCA,SAAS,eACP,MACA,OACM;AACN,KAAI,MAAM,SAAS,mBAAmB,CAAC,KAAK,KAAM;CAClD,MAAM,MAAM,KAAK,WAAW,KAAK;AACjC,KAAI,MAAM,KAAK,IAAI,IAAI,CAAE;AACzB,OAAM,KAAK,IAAI,IAAI;AACnB,OAAM,UAAU,KAAK,KAAK,KAAK;;;AAIjC,SAAS,cACP,MACA,OACA,WACM;AACN,KAAI,MAAM,SAAU,WAAU,KAAK,SAAS;AAC5C,KAAI,MAAM,cAAe,OAAM,UAAU,KAAK;;;AAIhD,SAAS,WACP,OACA,OACA,WACM;CACN,MAAM,OAAO,MAAM;AACnB,KAAI,SAAS,8BAA8B;AACzC,MAAI,OAAO,MAAM,UAAU,SAAU,OAAM,SAAS,MAAM;AAC1D;;AAEF,KACE,SAAS,gCACT,SAAS,6BACT;EACA,MAAM,OAAO,MAAM;AAMnB,iBAAe,MAAM,MAAM;AAM3B,MAAI,SAAS,+BAA+B,KAAK,SAAS,WAAW;GACnE,MAAM,QAAQ,KAAK,WAAW,EAAE,EAAE,KAAK,MAAM,EAAE,QAAQ,GAAG,CAAC,KAAK,GAAG;AACnE,OAAI,KAAM,OAAM,QAAQ;;AAE1B;;AAEF,KAAI,SAAS,WAAW,SAAS,mBAAmB;AAClD,QAAM,KAAK;AACX;;AAEF,KAAI,SAAS,kBACX,eACE,MAAM,MACN,OACA,UACD;;;AAKL,SAAS,iBACP,SACA,SACA,UACQ;AACR,QAAO,KAAK,UAAU;EACpB;EACA,GAAI,QAAQ,QAAQ,EAAE,OAAO,QAAQ,OAAO,GAAG,EAAE;EACjD,GAAI,WAAW,EAAE,UAAU,GAAG,EAAE;EAChC,GAAI,QAAQ,cAAc,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;EACpE,CAAC;;;;;;;;AASJ,SAAgB,iBAAiB,SAAwC;CACvE,MAAM,WAAW,QAAQ,QAAQ;CACjC,MAAM,YAAY,QAAQ,aAAa;CACvC,IAAI;AAEJ,QAAO,EACL,MAAM,KAAK,SAAuC;EAGhD,MAAM,SAAS,YAAY,QAAQ,UAAU;EAC7C,IAAI;AACJ,MAAI;AACF,SAAM,MAAM,MAAM,GAAG,QAAQ,UAAU,YAAY;IACjD,QAAQ;IACR,SAAS;KAAE,gBAAgB;KAAoB,GAAG,QAAQ;KAAS;IACnE,MAAM,iBAAiB,SAAS,SAAS,SAAS;IAKlD,UAAU;IACV;IACD,CAAC;UACI;AACN,UAAO;IAAE,OAAO;IAAI,WAAW,EAAE;IAAE,WAAW;IAAO;;AAGvD,MAAI,CAAC,IAAI,MAAM,CAAC,IAAI,KAClB,QAAO;GACL,OAAO;GACP,WAAW,EAAE;GACb,WAAW;GACX,WAAW;GACZ;EAGH,MAAM,QAAoB;GACxB,OAAO;GACP,WAAW,EAAE;GACb,sBAAM,IAAI,KAAa;GACvB,IAAI;GACL;EACD,MAAM,aAAa,OAAe;AAChC,cAAW;;AAEb,MAAI;AACF,cAAW,MAAM,EAAE,OAAO,UAAU,cAAc,IAAI,MAAM,OAAO,EAAE;AACnE,QAAI,SAAS,SAAU;AAIvB,QAAI,UAAU,QAAS,OAAM,KAAK;AAClC,QAAI;AACF,gBACE,KAAK,MAAM,KAAK,EAChB,OACA,UACD;YACK;;UAIJ;AAEN,SAAM,KAAK;;AAIb,MAAI,OAAO,QAAS,OAAM,KAAK;AAE/B,SAAO;GACL,OAAO,MAAM;GACb,WAAW,MAAM;GACjB,WAAW,MAAM;GACjB,WAAW;GACX,SAAS,MAAM;GAChB;IAEJ"}
+{"version":3,"file":"http-driver.js","names":[],"sources":["../../src/evals/http-driver.ts"],"sourcesContent":["import { readSseEvents } from \"../stream/sse-reader\";\nimport type { DriveResult, EvalDriver } from \"./types\";\n\nexport interface HttpDriverOptions {\n  /** Base URL of the running app, e.g. `http://localhost:3000`. */\n  baseUrl: string;\n  /** Agent alias to target. Omit to use the app's default agent. */\n  agent?: string;\n  /** Extra request headers (e.g. auth for a deployed app). */\n  headers?: Record<string, string>;\n  /** Chat endpoint path. Defaults to `/api/agents/chat`. */\n  path?: string;\n  /** MLflow run id to link each turn's trace to (for evaluation runs). */\n  mlflowRunId?: string;\n  /**\n   * Max wall-clock time for a single turn before it is abandoned as a failed\n   * turn (`succeeded: false`). Without this a hung agent — a blocked tool, a\n   * stalled model — never ends the SSE stream (heartbeats keep it alive), so\n   * the read loop spins forever and wedges the whole sequential suite.\n   * Defaults to 120s.\n   */\n  // ponytail: total per-turn cap; switch to an idle timeout if long legit turns get killed.\n  timeoutMs?: number;\n}\n\n/** Mutable running totals accumulated while draining one turn's SSE stream. */\ninterface DriveState {\n  reply: string;\n  /** Captured tool calls, keyed by `call_id ?? name` (dedupe). */\n  toolCalls: Map<string, ToolCall>;\n  ok: boolean;\n  traceId?: string;\n}\n\n/**\n * Record a `function_call` output item, keyed by `call_id ?? name`, capturing\n * its parsed arguments. A later `done` event's fuller args win over the initial\n * `added` event's (often empty) args.\n */\nfunction recordToolCall(\n  item:\n    | { type?: string; name?: string; call_id?: string; arguments?: string }\n    | undefined,\n  state: DriveState,\n): void {\n  if (item?.type !== \"function_call\" || !item.name) return;\n  const key = item.call_id ?? item.name;\n  const args = parseArgs(item.arguments);\n  const existing = state.toolCalls.get(key);\n  if (!existing) {\n    state.toolCalls.set(key, { name: item.name, args });\n  } else if (Object.keys(args).length > 0) {\n    // The initial `added` event may carry empty args while the later `done`\n    // carries the full arguments — keep the fuller set.\n    existing.args = args;\n  }\n}\n\n/** Apply an `appkit.metadata` event's thread/trace ids. */\nfunction applyMetadata(\n  data: { threadId?: string; mlflowTraceId?: string } | undefined,\n  state: DriveState,\n  setThread: (id: string) => void,\n): void {\n  if (data?.threadId) setThread(data.threadId);\n  if (data?.mlflowTraceId) state.traceId = data.mlflowTraceId;\n}\n\n/** A single captured tool call, deduped by `call_id ?? name`. */\ntype ToolCall = { name: string; args: Record<string, unknown> };\n\n/** Parse a function-call `arguments` JSON string; `{}` on missing/invalid. */\nfunction parseArgs(raw: unknown): Record<string, unknown> {\n  if (typeof raw !== \"string\" || raw.trim() === \"\") return {};\n  try {\n    const parsed = JSON.parse(raw);\n    return parsed !== null &&\n      typeof parsed === \"object\" &&\n      !Array.isArray(parsed)\n      ? (parsed as Record<string, unknown>)\n      : {};\n  } catch {\n    return {};\n  }\n}\n\n/** Parse a single Responses-API SSE `data:` payload into the running totals. */\nfunction applyEvent(\n  event: Record<string, unknown>,\n  state: DriveState,\n  setThread: (id: string) => void,\n): void {\n  const type = event.type;\n  if (type === \"response.output_text.delta\") {\n    if (typeof event.delta === \"string\") state.reply += event.delta;\n    return;\n  }\n  if (\n    type === \"response.output_item.added\" ||\n    type === \"response.output_item.done\"\n  ) {\n    const item = event.item as {\n      type?: string;\n      name?: string;\n      call_id?: string;\n      arguments?: string;\n      content?: Array<{ text?: string }>;\n    };\n    recordToolCall(item, state);\n    // A terminal `message` item carries the full reply text and *replaces*\n    // the accumulated deltas — the server emits no delta for a full message\n    // (event-translator `handleFullMessage`), so a non-streaming adapter's\n    // whole reply arrives only here. Guard on non-empty so a streaming turn's\n    // trailing done event can't clobber the deltas with \"\".\n    if (type === \"response.output_item.done\" && item.type === \"message\") {\n      const text = (item.content ?? []).map((c) => c.text ?? \"\").join(\"\");\n      if (text) state.reply = text;\n    }\n    return;\n  }\n  if (type === \"error\" || type === \"response.failed\") {\n    state.ok = false;\n    return;\n  }\n  if (type === \"appkit.metadata\") {\n    applyMetadata(\n      event.data as { threadId?: string; mlflowTraceId?: string },\n      state,\n      setThread,\n    );\n  }\n}\n\n/** Build the chat request payload, including only the optional fields that are set. */\nfunction buildRequestBody(\n  message: string,\n  options: HttpDriverOptions,\n  threadId: string | undefined,\n): string {\n  return JSON.stringify({\n    message,\n    ...(options.agent ? { agent: options.agent } : {}),\n    ...(threadId ? { threadId } : {}),\n    ...(options.mlflowRunId ? { mlflowRunId: options.mlflowRunId } : {}),\n  });\n}\n\n/**\n * Drives an agent by POSTing to a running app's chat endpoint and parsing the\n * SSE response. Keeps the thread id across `send`s so multi-turn evals share a\n * conversation. Agent/stream errors surface as `succeeded: false` rather than\n * throwing, so `t.succeeded()` can assert on them.\n */\nexport function createHttpDriver(options: HttpDriverOptions): EvalDriver {\n  const chatPath = options.path ?? \"/api/agents/chat\";\n  const timeoutMs = options.timeoutMs ?? 120_000;\n  let threadId: string | undefined;\n\n  return {\n    reset(): void {\n      threadId = undefined;\n    },\n    async send(\n      message: string,\n      opts?: { signal?: AbortSignal },\n    ): Promise<DriveResult> {\n      // Bounds connect + the entire read below. Passed to both the fetch and\n      // the SSE reader: on expiry the reader is cancelled and the turn fails.\n      // Composed with the caller's signal so a per-eval timeout also aborts this turn.\n      const timeout = AbortSignal.timeout(timeoutMs);\n      const signal = opts?.signal\n        ? AbortSignal.any([timeout, opts.signal])\n        : timeout;\n      let res: Response;\n      try {\n        res = await fetch(`${options.baseUrl}${chatPath}`, {\n          method: \"POST\",\n          headers: { \"content-type\": \"application/json\", ...options.headers },\n          body: buildRequestBody(message, options, threadId),\n          // A chat endpoint never needs a redirect; following one would replay\n          // custom auth headers (`options.headers`) to the redirect target.\n          // A 3xx becomes an opaque response (res.ok === false), handled as a\n          // failed turn by the !res.ok guard below.\n          redirect: \"manual\",\n          signal,\n        });\n      } catch {\n        return {\n          reply: \"\",\n          toolCalls: [],\n          toolCallDetails: [],\n          succeeded: false,\n        };\n      }\n\n      if (!res.ok || !res.body) {\n        return {\n          reply: \"\",\n          toolCalls: [],\n          toolCallDetails: [],\n          succeeded: false,\n          sessionId: threadId,\n        };\n      }\n\n      const state: DriveState = {\n        reply: \"\",\n        toolCalls: new Map<string, ToolCall>(),\n        ok: true,\n      };\n      const setThread = (id: string) => {\n        threadId = id;\n      };\n      try {\n        for await (const { event, data } of readSseEvents(res.body, signal)) {\n          if (data === \"[DONE]\") continue;\n          // A stream-level error frame (a thrown exception in the generator)\n          // is framed as `event: error` with a payload that carries no `type`,\n          // so the SSE event name — not the payload — is the real signal.\n          if (event === \"error\") state.ok = false;\n          try {\n            applyEvent(\n              JSON.parse(data) as Record<string, unknown>,\n              state,\n              setThread,\n            );\n          } catch {\n            // skip malformed event payloads\n          }\n        }\n      } catch {\n        // mid-stream transport error or DoS-guard throw: a broken turn.\n        state.ok = false;\n      }\n      // A timeout ends the iterator cleanly (reader cancel) rather than\n      // throwing, so mark an aborted turn failed explicitly.\n      if (signal.aborted) state.ok = false;\n\n      const toolCallDetails = [...state.toolCalls.values()];\n      return {\n        reply: state.reply,\n        toolCalls: toolCallDetails.map((c) => c.name),\n        toolCallDetails,\n        succeeded: state.ok,\n        sessionId: threadId,\n        traceId: state.traceId,\n      };\n    },\n  };\n}\n"],"mappings":";;;;;;;;AAuCA,SAAS,eACP,MAGA,OACM;AACN,KAAI,MAAM,SAAS,mBAAmB,CAAC,KAAK,KAAM;CAClD,MAAM,MAAM,KAAK,WAAW,KAAK;CACjC,MAAM,OAAO,UAAU,KAAK,UAAU;CACtC,MAAM,WAAW,MAAM,UAAU,IAAI,IAAI;AACzC,KAAI,CAAC,SACH,OAAM,UAAU,IAAI,KAAK;EAAE,MAAM,KAAK;EAAM;EAAM,CAAC;UAC1C,OAAO,KAAK,KAAK,CAAC,SAAS,EAGpC,UAAS,OAAO;;;AAKpB,SAAS,cACP,MACA,OACA,WACM;AACN,KAAI,MAAM,SAAU,WAAU,KAAK,SAAS;AAC5C,KAAI,MAAM,cAAe,OAAM,UAAU,KAAK;;;AAOhD,SAAS,UAAU,KAAuC;AACxD,KAAI,OAAO,QAAQ,YAAY,IAAI,MAAM,KAAK,GAAI,QAAO,EAAE;AAC3D,KAAI;EACF,MAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,SAAO,WAAW,QAChB,OAAO,WAAW,YAClB,CAAC,MAAM,QAAQ,OAAO,GACnB,SACD,EAAE;SACA;AACN,SAAO,EAAE;;;;AAKb,SAAS,WACP,OACA,OACA,WACM;CACN,MAAM,OAAO,MAAM;AACnB,KAAI,SAAS,8BAA8B;AACzC,MAAI,OAAO,MAAM,UAAU,SAAU,OAAM,SAAS,MAAM;AAC1D;;AAEF,KACE,SAAS,gCACT,SAAS,6BACT;EACA,MAAM,OAAO,MAAM;AAOnB,iBAAe,MAAM,MAAM;AAM3B,MAAI,SAAS,+BAA+B,KAAK,SAAS,WAAW;GACnE,MAAM,QAAQ,KAAK,WAAW,EAAE,EAAE,KAAK,MAAM,EAAE,QAAQ,GAAG,CAAC,KAAK,GAAG;AACnE,OAAI,KAAM,OAAM,QAAQ;;AAE1B;;AAEF,KAAI,SAAS,WAAW,SAAS,mBAAmB;AAClD,QAAM,KAAK;AACX;;AAEF,KAAI,SAAS,kBACX,eACE,MAAM,MACN,OACA,UACD;;;AAKL,SAAS,iBACP,SACA,SACA,UACQ;AACR,QAAO,KAAK,UAAU;EACpB;EACA,GAAI,QAAQ,QAAQ,EAAE,OAAO,QAAQ,OAAO,GAAG,EAAE;EACjD,GAAI,WAAW,EAAE,UAAU,GAAG,EAAE;EAChC,GAAI,QAAQ,cAAc,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;EACpE,CAAC;;;;;;;;AASJ,SAAgB,iBAAiB,SAAwC;CACvE,MAAM,WAAW,QAAQ,QAAQ;CACjC,MAAM,YAAY,QAAQ,aAAa;CACvC,IAAI;AAEJ,QAAO;EACL,QAAc;AACZ,cAAW;;EAEb,MAAM,KACJ,SACA,MACsB;GAItB,MAAM,UAAU,YAAY,QAAQ,UAAU;GAC9C,MAAM,SAAS,MAAM,SACjB,YAAY,IAAI,CAAC,SAAS,KAAK,OAAO,CAAC,GACvC;GACJ,IAAI;AACJ,OAAI;AACF,UAAM,MAAM,MAAM,GAAG,QAAQ,UAAU,YAAY;KACjD,QAAQ;KACR,SAAS;MAAE,gBAAgB;MAAoB,GAAG,QAAQ;MAAS;KACnE,MAAM,iBAAiB,SAAS,SAAS,SAAS;KAKlD,UAAU;KACV;KACD,CAAC;WACI;AACN,WAAO;KACL,OAAO;KACP,WAAW,EAAE;KACb,iBAAiB,EAAE;KACnB,WAAW;KACZ;;AAGH,OAAI,CAAC,IAAI,MAAM,CAAC,IAAI,KAClB,QAAO;IACL,OAAO;IACP,WAAW,EAAE;IACb,iBAAiB,EAAE;IACnB,WAAW;IACX,WAAW;IACZ;GAGH,MAAM,QAAoB;IACxB,OAAO;IACP,2BAAW,IAAI,KAAuB;IACtC,IAAI;IACL;GACD,MAAM,aAAa,OAAe;AAChC,eAAW;;AAEb,OAAI;AACF,eAAW,MAAM,EAAE,OAAO,UAAU,cAAc,IAAI,MAAM,OAAO,EAAE;AACnE,SAAI,SAAS,SAAU;AAIvB,SAAI,UAAU,QAAS,OAAM,KAAK;AAClC,SAAI;AACF,iBACE,KAAK,MAAM,KAAK,EAChB,OACA,UACD;aACK;;WAIJ;AAEN,UAAM,KAAK;;AAIb,OAAI,OAAO,QAAS,OAAM,KAAK;GAE/B,MAAM,kBAAkB,CAAC,GAAG,MAAM,UAAU,QAAQ,CAAC;AACrD,UAAO;IACL,OAAO,MAAM;IACb,WAAW,gBAAgB,KAAK,MAAM,EAAE,KAAK;IAC7C;IACA,WAAW,MAAM;IACjB,WAAW;IACX,SAAS,MAAM;IAChB;;EAEJ"}

package/dist/evals/index.d.ts

@@ -0,0 +1,14 @@
+import { DatabricksAuth, ResolveDatabricksAuthOptions, resolveDatabricksAuth, resolveWorkspaceClient } from "../connectors/mlflow/auth.js";
+import { MlflowClient, PostResult, normalizeHost } from "../connectors/mlflow/client.js";
+import "../connectors/mlflow/index.js";
+import { DatasetRow, ReadEvalDatasetOptions, readEvalDataset, userTurns } from "./dataset.js";
+import { AssertionHandle, AssertionResult, CustomJudgeSpec, DriveResult, EvalDefinition, EvalDriver, EvalResult, MatchResult, Matcher, Severity, TestContext } from "./types.js";
+import { defineEval, defineEvalConfig } from "./define-eval.js";
+import { DiscoveredEval, DiscoveredEvalConfig, discoverEvalConfigs, discoverEvalFiles } from "./discover.js";
+import { HttpDriverOptions, createHttpDriver } from "./http-driver.js";
+import { JudgeConfig, JudgeScore, configureJudge, isJudgeConfigured } from "./judge.js";
+import { equals, includes, matches } from "./matchers.js";
+import { Assessment, ReportOutcome, buildAssessments, reportToMlflow } from "./mlflow-report.js";
+import { EvalSummary, evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, summarize } from "./report.js";
+import { RunEvalOptions, runEval } from "./run-eval.js";
+import { EvalProgress, EvalRunSummary, RunEvalsOptions, runEvalsInDir, runWithRetries } from "./run-evals.js";

package/dist/evals/index.js

@@ -1,13 +1,14 @@
-import { resolveDatabricksAuth } from "../connectors/mlflow/auth.js";
+import { resolveDatabricksAuth, resolveWorkspaceClient } from "../connectors/mlflow/auth.js";
 import { MlflowClient, normalizeHost } from "../connectors/mlflow/client.js";
-import { defineEval } from "./define-eval.js";
-import { discoverEvalFiles } from "./discover.js";
+import { readEvalDataset, userTurns } from "./dataset.js";
+import { defineEval, defineEvalConfig } from "./define-eval.js";
+import { discoverEvalConfigs, discoverEvalFiles } from "./discover.js";
 import { createHttpDriver } from "./http-driver.js";
 import { configureJudge, isJudgeConfigured } from "./judge.js";
 import { equals, includes, matches } from "./matchers.js";
 import { buildAssessments, reportToMlflow } from "./mlflow-report.js";
-import { evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, summarize } from "./report.js";
+import { evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, summarize } from "./report.js";
 import { runEval } from "./run-eval.js";
-import { runEvalsInDir } from "./run-evals.js";
+import { runEvalsInDir, runWithRetries } from "./run-evals.js";
 
 export {  };

package/dist/evals/judge.d.ts

@@ -1,4 +1,5 @@
 import { MlflowClient } from "../connectors/mlflow/client.js";
+import "../connectors/mlflow/index.js";
 
 //#region src/evals/judge.d.ts
 interface JudgeConfig {

package/dist/evals/judge.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"judge.d.ts","names":[],"sources":["../../src/evals/judge.ts"],"mappings":";;;UAuBiB,WAAA;;EAEf,MAAA,EAAQ,YAAA;EAFkB;EAI1B,KAAA;EAFoB;EAIpB,KAAA;AAAA;;UAIe,UAAA;EACf,KAAA;EACA,SAAA;AAAA;;;;;AAQF;iBAAsB,cAAA,CAAe,MAAA,EAAQ,WAAA,GAAc,OAAA;AAAA,iBAe3C,iBAAA,CAAA"}
+{"version":3,"file":"judge.d.ts","names":[],"sources":["../../src/evals/judge.ts"],"mappings":";;;;UAuBiB,WAAA;;EAEf,MAAA,EAAQ,YAAA;EAFO;EAIf,KAAA;;EAEA,KAAA;AAAA;;UAIe,UAAA;EACf,KAAA;EACA,SAAA;AAAA;AAFF;;;;;AAAA,iBAUsB,cAAA,CAAe,MAAA,EAAQ,WAAA,GAAc,OAAA;AAAA,iBAe3C,iBAAA,CAAA"}

package/dist/evals/mlflow-report.d.ts

@@ -1,4 +1,5 @@
 import { MlflowClient } from "../connectors/mlflow/client.js";
+import "../connectors/mlflow/index.js";
 import { EvalResult } from "./types.js";
 
 //#region src/evals/mlflow-report.d.ts

package/dist/evals/mlflow-report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mlflow-report.d.ts","names":[],"sources":["../../src/evals/mlflow-report.ts"],"mappings":";;;;;UAqBiB,UAAA;EACf,QAAA;EACA,eAAA;EACA,MAAA;IAAU,WAAA;IAA6C,SAAA;EAAA;EACvD,QAAA;IAAY,KAAA;EAAA;EACZ,SAAA;EACA,QAAA,GAAW,MAAA;AAAA;AAAA,UAGI,aAAA;EACf,OAAA;EACA,OAAA;EACA,QAAA,EAAU,KAAA;IAAQ,OAAA;IAAiB,MAAA;IAAiB,KAAA;EAAA;AAAA;AAAA,iBA2DtC,gBAAA,CAAiB,MAAA,EAAQ,UAAA,GAAa,UAAA;;;;;iBA6FhC,cAAA,CACpB,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,UAAA,IACT,cAAA,YACC,OAAA,CAAQ,aAAA"}
+{"version":3,"file":"mlflow-report.d.ts","names":[],"sources":["../../src/evals/mlflow-report.ts"],"mappings":";;;;;;UAqBiB,UAAA;EACf,QAAA;EACA,eAAA;EACA,MAAA;IAAU,WAAA;IAA6C,SAAA;EAAA;EACvD,QAAA;IAAY,KAAA;EAAA;EACZ,SAAA;EACA,QAAA,GAAW,MAAA;AAAA;AAAA,UAGI,aAAA;EACf,OAAA;EACA,OAAA;EACA,QAAA,EAAU,KAAA;IAAQ,OAAA;IAAiB,MAAA;IAAiB,KAAA;EAAA;AAAA;AAAA,iBA2DtC,gBAAA,CAAiB,MAAA,EAAQ,UAAA,GAAa,UAAA;;;;;iBA6FhC,cAAA,CACpB,MAAA,EAAQ,YAAA,EACR,OAAA,EAAS,UAAA,IACT,cAAA,YACC,OAAA,CAAQ,aAAA"}

package/dist/evals/mlflow-run.d.ts

@@ -1,3 +1,5 @@
+import "../connectors/mlflow/index.js";
+
 //#region src/evals/mlflow-run.d.ts
 interface FinishOutcome {
   finished: boolean;

package/dist/evals/mlflow-run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mlflow-run.d.ts","names":[],"sources":["../../src/evals/mlflow-run.ts"],"mappings":";UA6EiB,aAAA;EACf,QAAA;;EAEA,YAAA;;EAEA,WAAA;AAAA"}
+{"version":3,"file":"mlflow-run.d.ts","names":[],"sources":["../../src/evals/mlflow-run.ts"],"mappings":";;;UA6EiB,aAAA;EACf,QAAA;;EAEA,YAAA;;EAEA,WAAA;AAAA"}

package/dist/evals/report.d.ts

@@ -8,6 +8,8 @@ interface EvalSummary {
   skipped: number;
   /** True when no eval failed (skips don't count as failures). */
   allPassed: boolean;
+  /** Fraction of scored (non-skipped) evals that passed, 0..1 (1 when none scored). */
+  passRate: number;
 }
 declare function summarize(results: EvalResult[]): EvalSummary;
 /** Status glyph for a single eval result. */
@@ -20,6 +22,19 @@ declare function formatEvalDetail(result: EvalResult): string[];
 declare function formatSummaryLine(results: EvalResult[]): string;
 /** Render all results as a human-readable console report (non-streaming). */
 declare function formatEvalResults(results: EvalResult[]): string;
+/**
+ * Render results as a machine-readable JSON report (2-space indented):
+ * `{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types —
+ * every field present on a result round-trips.
+ */
+declare function formatResultsJson(results: EvalResult[]): string;
+/**
+ * Render results as JUnit XML for standard CI test reporters: a single
+ * `<testsuite name="appkit-agent-evals">` with one `<testcase>` per result.
+ * Failures carry a `<failure>` (error or failing-gate summary); skips a
+ * `<skipped>`. All attribute/text values are XML-escaped.
+ */
+declare function formatResultsJUnit(results: EvalResult[]): string;
 //#endregion
-export { EvalSummary, evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, summarize };
+export { EvalSummary, evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, summarize };
 //# sourceMappingURL=report.d.ts.map

package/dist/evals/report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report.d.ts","names":[],"sources":["../../src/evals/report.ts"],"mappings":";;;UAEiB,WAAA;EACf,KAAA;EACA,MAAA;EACA,MAAA;EACA,OAAA;EAJ0B;EAM1B,SAAA;AAAA;AAAA,iBAGc,SAAA,CAAU,OAAA,EAAS,UAAA,KAAe,WAAA;;iBAmBlC,SAAA,CAAU,MAAA,EAAQ,UAAA;;iBAMlB,kBAAA,CAAmB,MAAA,EAAQ,UAAA;AAzB3C;AAAA,iBAqCgB,gBAAA,CAAiB,MAAA,EAAQ,UAAA;;iBAYzB,iBAAA,CAAkB,OAAA,EAAS,UAAA;;iBAM3B,iBAAA,CAAkB,OAAA,EAAS,UAAA"}
+{"version":3,"file":"report.d.ts","names":[],"sources":["../../src/evals/report.ts"],"mappings":";;;UAEiB,WAAA;EACf,KAAA;EACA,MAAA;EACA,MAAA;EACA,OAAA;EAJ0B;EAM1B,SAAA;EAJA;EAMA,QAAA;AAAA;AAAA,iBAGc,SAAA,CAAU,OAAA,EAAS,UAAA,KAAe,WAAA;;iBAqBlC,SAAA,CAAU,MAAA,EAAQ,UAAA;;iBAMlB,kBAAA,CAAmB,MAAA,EAAQ,UAAA;;iBAY3B,gBAAA,CAAiB,MAAA,EAAQ,UAAA;;iBAYzB,iBAAA,CAAkB,OAAA,EAAS,UAAA;;iBAM3B,iBAAA,CAAkB,OAAA,EAAS,UAAA;;;AApC3C;;;iBAoDgB,iBAAA,CAAkB,OAAA,EAAS,UAAA;;AA9C3C;;;;;iBA2FgB,kBAAA,CAAmB,OAAA,EAAS,UAAA"}

package/dist/evals/report.js

@@ -6,12 +6,14 @@ function summarize(results) {
 	for (const r of results) if (r.skipped) skipped++;
 	else if (r.passed) passed++;
 	else failed++;
+	const scored = passed + failed;
 	return {
 		total: results.length,
 		passed,
 		failed,
 		skipped,
-		allPassed: failed === 0
+		allPassed: failed === 0,
+		passRate: scored === 0 ? 1 : passed / scored
 	};
 }
 /** Status glyph for a single eval result. */
@@ -51,7 +53,67 @@ function formatEvalResults(results) {
 	lines.push(formatSummaryLine(results));
 	return lines.join("\n");
 }
+/**
+* Render results as a machine-readable JSON report (2-space indented):
+* `{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types —
+* every field present on a result round-trips.
+*/
+function formatResultsJson(results) {
+	return JSON.stringify({
+		summary: summarize(results),
+		results
+	}, null, 2);
+}
+/**
+* Drop the C0 control chars XML 1.0 forbids even when escaped (except tab, LF,
+* CR) — a raw NUL or ANSI escape would otherwise make the JUnit doc reject on parse.
+*/
+function stripXmlControlChars(value) {
+	let out = "";
+	for (const ch of value) {
+		const code = ch.codePointAt(0);
+		if (code >= 32 || code === 9 || code === 10 || code === 13) out += ch;
+	}
+	return out;
+}
+/** Escape a value for use in XML text/attribute content (control chars dropped). */
+function escapeXml(value) {
+	return stripXmlControlChars(value).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+}
+/** One-line reason a result failed: its error, else its failing gate labels. */
+function failureMessage(result) {
+	if (result.error) return result.error;
+	const gates = result.assertions.filter((a) => !a.pass && a.severity === "gate").map((a) => a.detail ? `${a.label} — ${a.detail}` : a.label);
+	return gates.length ? gates.join("; ") : "eval failed";
+}
+/**
+* Render results as JUnit XML for standard CI test reporters: a single
+* `<testsuite name="appkit-agent-evals">` with one `<testcase>` per result.
+* Failures carry a `<failure>` (error or failing-gate summary); skips a
+* `<skipped>`. All attribute/text values are XML-escaped.
+*/
+function formatResultsJUnit(results) {
+	const s = summarize(results);
+	const lines = [];
+	lines.push("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+	lines.push(`<testsuite name="appkit-agent-evals" tests="${s.total}" failures="${s.failed}" skipped="${s.skipped}">`);
+	for (const r of results) {
+		const open = `  <testcase name="${escapeXml(r.id)}" classname="agent-eval"`;
+		if (r.skipped) {
+			lines.push(`${open}>`);
+			lines.push(r.skipped.reason ? `    <skipped message="${escapeXml(r.skipped.reason)}"/>` : "    <skipped/>");
+			lines.push("  </testcase>");
+		} else if (!r.passed) {
+			const message = failureMessage(r);
+			lines.push(`${open}>`);
+			lines.push(`    <failure message="${escapeXml(message)}"/>`);
+			lines.push("  </testcase>");
+		} else lines.push(`${open}/>`);
+	}
+	lines.push("</testsuite>");
+	return lines.join("\n");
+}
 
 //#endregion
-export { evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, summarize };
+export { evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, summarize };
 //# sourceMappingURL=report.js.map

package/dist/evals/report.js.map

@@ -1 +1 @@
-{"version":3,"file":"report.js","names":[],"sources":["../../src/evals/report.ts"],"sourcesContent":["import type { EvalResult } from \"./types\";\n\nexport interface EvalSummary {\n  total: number;\n  passed: number;\n  failed: number;\n  skipped: number;\n  /** True when no eval failed (skips don't count as failures). */\n  allPassed: boolean;\n}\n\nexport function summarize(results: EvalResult[]): EvalSummary {\n  let passed = 0;\n  let failed = 0;\n  let skipped = 0;\n  for (const r of results) {\n    if (r.skipped) skipped++;\n    else if (r.passed) passed++;\n    else failed++;\n  }\n  return {\n    total: results.length,\n    passed,\n    failed,\n    skipped,\n    allPassed: failed === 0,\n  };\n}\n\n/** Status glyph for a single eval result. */\nexport function evalGlyph(result: EvalResult): string {\n  if (result.skipped) return \"−\";\n  return result.passed ? \"✓\" : \"✗\";\n}\n\n/** The one-line header for a single eval result (no failure detail). */\nexport function formatEvalHeadline(result: EvalResult): string {\n  if (result.skipped) {\n    return `− ${result.id} (skipped${\n      result.skipped.reason ? `: ${result.skipped.reason}` : \"\"\n    })`;\n  }\n  return `${evalGlyph(result)} ${result.id}${\n    result.description ? ` — ${result.description}` : \"\"\n  }`;\n}\n\n/** Indented detail lines for a failing eval (error + failing assertions). */\nexport function formatEvalDetail(result: EvalResult): string[] {\n  const lines: string[] = [];\n  if (result.error) lines.push(`    error: ${result.error}`);\n  for (const a of result.assertions) {\n    if (a.pass) continue;\n    const tag = a.severity === \"soft\" ? \"soft\" : \"gate\";\n    lines.push(`    ✗ [${tag}] ${a.label}${a.detail ? ` — ${a.detail}` : \"\"}`);\n  }\n  return lines;\n}\n\n/** The final PASS/FAIL summary line. */\nexport function formatSummaryLine(results: EvalResult[]): string {\n  const s = summarize(results);\n  return `${s.allPassed ? \"PASS\" : \"FAIL\"} — ${s.passed} passed, ${s.failed} failed, ${s.skipped} skipped (${s.total} total)`;\n}\n\n/** Render all results as a human-readable console report (non-streaming). */\nexport function formatEvalResults(results: EvalResult[]): string {\n  const lines: string[] = [];\n  for (const r of results) {\n    lines.push(formatEvalHeadline(r));\n    lines.push(...formatEvalDetail(r));\n  }\n  lines.push(\"\");\n  lines.push(formatSummaryLine(results));\n  return lines.join(\"\\n\");\n}\n"],"mappings":";AAWA,SAAgB,UAAU,SAAoC;CAC5D,IAAI,SAAS;CACb,IAAI,SAAS;CACb,IAAI,UAAU;AACd,MAAK,MAAM,KAAK,QACd,KAAI,EAAE,QAAS;UACN,EAAE,OAAQ;KACd;AAEP,QAAO;EACL,OAAO,QAAQ;EACf;EACA;EACA;EACA,WAAW,WAAW;EACvB;;;AAIH,SAAgB,UAAU,QAA4B;AACpD,KAAI,OAAO,QAAS,QAAO;AAC3B,QAAO,OAAO,SAAS,MAAM;;;AAI/B,SAAgB,mBAAmB,QAA4B;AAC7D,KAAI,OAAO,QACT,QAAO,KAAK,OAAO,GAAG,WACpB,OAAO,QAAQ,SAAS,KAAK,OAAO,QAAQ,WAAW,GACxD;AAEH,QAAO,GAAG,UAAU,OAAO,CAAC,GAAG,OAAO,KACpC,OAAO,cAAc,MAAM,OAAO,gBAAgB;;;AAKtD,SAAgB,iBAAiB,QAA8B;CAC7D,MAAM,QAAkB,EAAE;AAC1B,KAAI,OAAO,MAAO,OAAM,KAAK,cAAc,OAAO,QAAQ;AAC1D,MAAK,MAAM,KAAK,OAAO,YAAY;AACjC,MAAI,EAAE,KAAM;EACZ,MAAM,MAAM,EAAE,aAAa,SAAS,SAAS;AAC7C,QAAM,KAAK,UAAU,IAAI,IAAI,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,WAAW,KAAK;;AAE5E,QAAO;;;AAIT,SAAgB,kBAAkB,SAA+B;CAC/D,MAAM,IAAI,UAAU,QAAQ;AAC5B,QAAO,GAAG,EAAE,YAAY,SAAS,OAAO,KAAK,EAAE,OAAO,WAAW,EAAE,OAAO,WAAW,EAAE,QAAQ,YAAY,EAAE,MAAM;;;AAIrH,SAAgB,kBAAkB,SAA+B;CAC/D,MAAM,QAAkB,EAAE;AAC1B,MAAK,MAAM,KAAK,SAAS;AACvB,QAAM,KAAK,mBAAmB,EAAE,CAAC;AACjC,QAAM,KAAK,GAAG,iBAAiB,EAAE,CAAC;;AAEpC,OAAM,KAAK,GAAG;AACd,OAAM,KAAK,kBAAkB,QAAQ,CAAC;AACtC,QAAO,MAAM,KAAK,KAAK"}
+{"version":3,"file":"report.js","names":[],"sources":["../../src/evals/report.ts"],"sourcesContent":["import type { EvalResult } from \"./types\";\n\nexport interface EvalSummary {\n  total: number;\n  passed: number;\n  failed: number;\n  skipped: number;\n  /** True when no eval failed (skips don't count as failures). */\n  allPassed: boolean;\n  /** Fraction of scored (non-skipped) evals that passed, 0..1 (1 when none scored). */\n  passRate: number;\n}\n\nexport function summarize(results: EvalResult[]): EvalSummary {\n  let passed = 0;\n  let failed = 0;\n  let skipped = 0;\n  for (const r of results) {\n    if (r.skipped) skipped++;\n    else if (r.passed) passed++;\n    else failed++;\n  }\n  const scored = passed + failed;\n  return {\n    total: results.length,\n    passed,\n    failed,\n    skipped,\n    allPassed: failed === 0,\n    passRate: scored === 0 ? 1 : passed / scored,\n  };\n}\n\n/** Status glyph for a single eval result. */\nexport function evalGlyph(result: EvalResult): string {\n  if (result.skipped) return \"−\";\n  return result.passed ? \"✓\" : \"✗\";\n}\n\n/** The one-line header for a single eval result (no failure detail). */\nexport function formatEvalHeadline(result: EvalResult): string {\n  if (result.skipped) {\n    return `− ${result.id} (skipped${\n      result.skipped.reason ? `: ${result.skipped.reason}` : \"\"\n    })`;\n  }\n  return `${evalGlyph(result)} ${result.id}${\n    result.description ? ` — ${result.description}` : \"\"\n  }`;\n}\n\n/** Indented detail lines for a failing eval (error + failing assertions). */\nexport function formatEvalDetail(result: EvalResult): string[] {\n  const lines: string[] = [];\n  if (result.error) lines.push(`    error: ${result.error}`);\n  for (const a of result.assertions) {\n    if (a.pass) continue;\n    const tag = a.severity === \"soft\" ? \"soft\" : \"gate\";\n    lines.push(`    ✗ [${tag}] ${a.label}${a.detail ? ` — ${a.detail}` : \"\"}`);\n  }\n  return lines;\n}\n\n/** The final PASS/FAIL summary line. */\nexport function formatSummaryLine(results: EvalResult[]): string {\n  const s = summarize(results);\n  return `${s.allPassed ? \"PASS\" : \"FAIL\"} — ${s.passed} passed, ${s.failed} failed, ${s.skipped} skipped (${s.total} total)`;\n}\n\n/** Render all results as a human-readable console report (non-streaming). */\nexport function formatEvalResults(results: EvalResult[]): string {\n  const lines: string[] = [];\n  for (const r of results) {\n    lines.push(formatEvalHeadline(r));\n    lines.push(...formatEvalDetail(r));\n  }\n  lines.push(\"\");\n  lines.push(formatSummaryLine(results));\n  return lines.join(\"\\n\");\n}\n\n/**\n * Render results as a machine-readable JSON report (2-space indented):\n * `{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types —\n * every field present on a result round-trips.\n */\nexport function formatResultsJson(results: EvalResult[]): string {\n  return JSON.stringify({ summary: summarize(results), results }, null, 2);\n}\n\n/**\n * Drop the C0 control chars XML 1.0 forbids even when escaped (except tab, LF,\n * CR) — a raw NUL or ANSI escape would otherwise make the JUnit doc reject on parse.\n */\nfunction stripXmlControlChars(value: string): string {\n  // Code-point filter, not a regex: oxlint `no-control-regex` rejects the class.\n  let out = \"\";\n  for (const ch of value) {\n    const code = ch.codePointAt(0) as number;\n    if (code >= 0x20 || code === 0x09 || code === 0x0a || code === 0x0d) {\n      out += ch;\n    }\n  }\n  return out;\n}\n\n/** Escape a value for use in XML text/attribute content (control chars dropped). */\nfunction escapeXml(value: string): string {\n  return stripXmlControlChars(value)\n    .replace(/&/g, \"&amp;\")\n    .replace(/</g, \"&lt;\")\n    .replace(/>/g, \"&gt;\")\n    .replace(/\"/g, \"&quot;\")\n    .replace(/'/g, \"&apos;\");\n}\n\n/** One-line reason a result failed: its error, else its failing gate labels. */\nfunction failureMessage(result: EvalResult): string {\n  if (result.error) return result.error;\n  const gates = result.assertions\n    .filter((a) => !a.pass && a.severity === \"gate\")\n    .map((a) => (a.detail ? `${a.label} — ${a.detail}` : a.label));\n  return gates.length ? gates.join(\"; \") : \"eval failed\";\n}\n\n/**\n * Render results as JUnit XML for standard CI test reporters: a single\n * `<testsuite name=\"appkit-agent-evals\">` with one `<testcase>` per result.\n * Failures carry a `<failure>` (error or failing-gate summary); skips a\n * `<skipped>`. All attribute/text values are XML-escaped.\n */\nexport function formatResultsJUnit(results: EvalResult[]): string {\n  const s = summarize(results);\n  const lines: string[] = [];\n  lines.push('<?xml version=\"1.0\" encoding=\"UTF-8\"?>');\n  lines.push(\n    `<testsuite name=\"appkit-agent-evals\" tests=\"${s.total}\" failures=\"${s.failed}\" skipped=\"${s.skipped}\">`,\n  );\n  for (const r of results) {\n    const open = `  <testcase name=\"${escapeXml(r.id)}\" classname=\"agent-eval\"`;\n    if (r.skipped) {\n      lines.push(`${open}>`);\n      lines.push(\n        r.skipped.reason\n          ? `    <skipped message=\"${escapeXml(r.skipped.reason)}\"/>`\n          : \"    <skipped/>\",\n      );\n      lines.push(\"  </testcase>\");\n    } else if (!r.passed) {\n      const message = failureMessage(r);\n      lines.push(`${open}>`);\n      lines.push(`    <failure message=\"${escapeXml(message)}\"/>`);\n      lines.push(\"  </testcase>\");\n    } else {\n      lines.push(`${open}/>`);\n    }\n  }\n  lines.push(\"</testsuite>\");\n  return lines.join(\"\\n\");\n}\n"],"mappings":";AAaA,SAAgB,UAAU,SAAoC;CAC5D,IAAI,SAAS;CACb,IAAI,SAAS;CACb,IAAI,UAAU;AACd,MAAK,MAAM,KAAK,QACd,KAAI,EAAE,QAAS;UACN,EAAE,OAAQ;KACd;CAEP,MAAM,SAAS,SAAS;AACxB,QAAO;EACL,OAAO,QAAQ;EACf;EACA;EACA;EACA,WAAW,WAAW;EACtB,UAAU,WAAW,IAAI,IAAI,SAAS;EACvC;;;AAIH,SAAgB,UAAU,QAA4B;AACpD,KAAI,OAAO,QAAS,QAAO;AAC3B,QAAO,OAAO,SAAS,MAAM;;;AAI/B,SAAgB,mBAAmB,QAA4B;AAC7D,KAAI,OAAO,QACT,QAAO,KAAK,OAAO,GAAG,WACpB,OAAO,QAAQ,SAAS,KAAK,OAAO,QAAQ,WAAW,GACxD;AAEH,QAAO,GAAG,UAAU,OAAO,CAAC,GAAG,OAAO,KACpC,OAAO,cAAc,MAAM,OAAO,gBAAgB;;;AAKtD,SAAgB,iBAAiB,QAA8B;CAC7D,MAAM,QAAkB,EAAE;AAC1B,KAAI,OAAO,MAAO,OAAM,KAAK,cAAc,OAAO,QAAQ;AAC1D,MAAK,MAAM,KAAK,OAAO,YAAY;AACjC,MAAI,EAAE,KAAM;EACZ,MAAM,MAAM,EAAE,aAAa,SAAS,SAAS;AAC7C,QAAM,KAAK,UAAU,IAAI,IAAI,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,WAAW,KAAK;;AAE5E,QAAO;;;AAIT,SAAgB,kBAAkB,SAA+B;CAC/D,MAAM,IAAI,UAAU,QAAQ;AAC5B,QAAO,GAAG,EAAE,YAAY,SAAS,OAAO,KAAK,EAAE,OAAO,WAAW,EAAE,OAAO,WAAW,EAAE,QAAQ,YAAY,EAAE,MAAM;;;AAIrH,SAAgB,kBAAkB,SAA+B;CAC/D,MAAM,QAAkB,EAAE;AAC1B,MAAK,MAAM,KAAK,SAAS;AACvB,QAAM,KAAK,mBAAmB,EAAE,CAAC;AACjC,QAAM,KAAK,GAAG,iBAAiB,EAAE,CAAC;;AAEpC,OAAM,KAAK,GAAG;AACd,OAAM,KAAK,kBAAkB,QAAQ,CAAC;AACtC,QAAO,MAAM,KAAK,KAAK;;;;;;;AAQzB,SAAgB,kBAAkB,SAA+B;AAC/D,QAAO,KAAK,UAAU;EAAE,SAAS,UAAU,QAAQ;EAAE;EAAS,EAAE,MAAM,EAAE;;;;;;AAO1E,SAAS,qBAAqB,OAAuB;CAEnD,IAAI,MAAM;AACV,MAAK,MAAM,MAAM,OAAO;EACtB,MAAM,OAAO,GAAG,YAAY,EAAE;AAC9B,MAAI,QAAQ,MAAQ,SAAS,KAAQ,SAAS,MAAQ,SAAS,GAC7D,QAAO;;AAGX,QAAO;;;AAIT,SAAS,UAAU,OAAuB;AACxC,QAAO,qBAAqB,MAAM,CAC/B,QAAQ,MAAM,QAAQ,CACtB,QAAQ,MAAM,OAAO,CACrB,QAAQ,MAAM,OAAO,CACrB,QAAQ,MAAM,SAAS,CACvB,QAAQ,MAAM,SAAS;;;AAI5B,SAAS,eAAe,QAA4B;AAClD,KAAI,OAAO,MAAO,QAAO,OAAO;CAChC,MAAM,QAAQ,OAAO,WAClB,QAAQ,MAAM,CAAC,EAAE,QAAQ,EAAE,aAAa,OAAO,CAC/C,KAAK,MAAO,EAAE,SAAS,GAAG,EAAE,MAAM,KAAK,EAAE,WAAW,EAAE,MAAO;AAChE,QAAO,MAAM,SAAS,MAAM,KAAK,KAAK,GAAG;;;;;;;;AAS3C,SAAgB,mBAAmB,SAA+B;CAChE,MAAM,IAAI,UAAU,QAAQ;CAC5B,MAAM,QAAkB,EAAE;AAC1B,OAAM,KAAK,6CAAyC;AACpD,OAAM,KACJ,+CAA+C,EAAE,MAAM,cAAc,EAAE,OAAO,aAAa,EAAE,QAAQ,IACtG;AACD,MAAK,MAAM,KAAK,SAAS;EACvB,MAAM,OAAO,qBAAqB,UAAU,EAAE,GAAG,CAAC;AAClD,MAAI,EAAE,SAAS;AACb,SAAM,KAAK,GAAG,KAAK,GAAG;AACtB,SAAM,KACJ,EAAE,QAAQ,SACN,yBAAyB,UAAU,EAAE,QAAQ,OAAO,CAAC,OACrD,iBACL;AACD,SAAM,KAAK,gBAAgB;aAClB,CAAC,EAAE,QAAQ;GACpB,MAAM,UAAU,eAAe,EAAE;AACjC,SAAM,KAAK,GAAG,KAAK,GAAG;AACtB,SAAM,KAAK,yBAAyB,UAAU,QAAQ,CAAC,KAAK;AAC5D,SAAM,KAAK,gBAAgB;QAE3B,OAAM,KAAK,GAAG,KAAK,IAAI;;AAG3B,OAAM,KAAK,eAAe;AAC1B,QAAO,MAAM,KAAK,KAAK"}

package/dist/evals/run-eval.d.ts

@@ -1,3 +1,4 @@
+import { DatasetRow } from "./dataset.js";
 import { EvalDefinition, EvalDriver, EvalResult } from "./types.js";
 
 //#region src/evals/run-eval.d.ts
@@ -8,6 +9,13 @@ interface RunEvalOptions {
   driver: EvalDriver;
   /** When true, soft assertion failures also fail the eval. */
   strict?: boolean;
+  /** Dataset row bound to `t.input`/`t.expected` for dataset-driven evals. */
+  row?: DatasetRow;
+  /**
+   * Runner-level default per-eval timeout (ms). `def.timeoutMs` wins over this;
+   * when both are unset the eval runs unbounded (current behavior).
+   */
+  timeoutMs?: number;
 }
 /**
  * Run a single eval against a driver. Never throws for assertion or agent

package/dist/evals/run-eval.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-eval.d.ts","names":[],"sources":["../../src/evals/run-eval.ts"],"mappings":";;;UAsBiB,cAAA;;EAEf,EAAA;EAF6B;EAI7B,MAAA,EAAQ,UAAA;EAAU;EAElB,MAAA;AAAA;;;;;AAQF;iBAAsB,OAAA,CACpB,GAAA,EAAK,cAAA,EACL,OAAA,EAAS,cAAA,GACR,OAAA,CAAQ,UAAA"}
+{"version":3,"file":"run-eval.d.ts","names":[],"sources":["../../src/evals/run-eval.ts"],"mappings":";;;;UAoDiB,cAAA;;EAEf,EAAA;EAF6B;EAI7B,MAAA,EAAQ,UAAA;EAIQ;EAFhB,MAAA;EAFA;EAIA,GAAA,GAAM,UAAA;EAFN;;;;EAOA,SAAA;AAAA;AAQF;;;;;AAAA,iBAAsB,OAAA,CACpB,GAAA,EAAK,cAAA,EACL,OAAA,EAAS,cAAA,GACR,OAAA,CAAQ,UAAA"}