@databricks/appkit 0.73.0 → 0.74.0

package/CLAUDE.md

@@ -99,9 +99,11 @@ npx @databricks/appkit docs <query>
 - [Function: createWorkspaceClient()](./docs/api/appkit/Function.createWorkspaceClient.md): Construct an AppKit workspace client.
 - [Function: database()](./docs/api/appkit/Function.database.md): Create a typed database plugin registration for a finalized schema.
 - [Function: defineEval()](./docs/api/appkit/Function.defineEval.md): Define an agent eval. Default-export the result from a
+- [Function: defineEvalConfig()](./docs/api/appkit/Function.defineEvalConfig.md): Define per-directory eval config. Default-export from evals.config.ts.
 - [Function: defineManifest()](./docs/api/appkit/Function.defineManifest.md): Validates a raw manifest (typically a manifest.json import) against the
 - [Function: defineSchema()](./docs/api/appkit/Function.defineSchema.md): Compile one declared schema. The returned type keeps the table names the
 - [Function: defineTool()](./docs/api/appkit/Function.defineTool.md): Defines a single tool entry for a plugin's internal registry.
+- [Function: discoverEvalConfigs()](./docs/api/appkit/Function.discoverEvalConfigs.md): Discover the per-agent evals.config.ts (from defineEvalConfig) at
 - [Function: discoverEvalFiles()](./docs/api/appkit/Function.discoverEvalFiles.md): Discover evals under /server/agents//evals/ — co-located
 - [Function: enumColumn()](./docs/api/appkit/Function.enumColumn.md): Parameters
 - [Function: equals()](./docs/api/appkit/Function.equals.md): Passes when the value equals expected exactly.
@@ -113,6 +115,8 @@ npx @databricks/appkit docs <query>
 - [Function: formatEvalDetail()](./docs/api/appkit/Function.formatEvalDetail.md): Indented detail lines for a failing eval (error + failing assertions).
 - [Function: formatEvalHeadline()](./docs/api/appkit/Function.formatEvalHeadline.md): The one-line header for a single eval result (no failure detail).
 - [Function: formatEvalResults()](./docs/api/appkit/Function.formatEvalResults.md): Render all results as a human-readable console report (non-streaming).
+- [Function: formatResultsJson()](./docs/api/appkit/Function.formatResultsJson.md): Render results as a machine-readable JSON report (2-space indented):
+- [Function: formatResultsJUnit()](./docs/api/appkit/Function.formatResultsJUnit.md): Render results as JUnit XML for standard CI test reporters: a single
 - [Function: formatSummaryLine()](./docs/api/appkit/Function.formatSummaryLine.md): The final PASS/FAIL summary line.
 - [Function: fromSupervisorApi()](./docs/api/appkit/Function.fromSupervisorApi.md): Creates an AgentAdapter backed by the Databricks AI Gateway
 - [Function: functionToolToDefinition()](./docs/api/appkit/Function.functionToolToDefinition.md): Parameters
@@ -148,11 +152,13 @@ npx @databricks/appkit docs <query>
 - [Function: runAgent()](./docs/api/appkit/Function.runAgent.md): Standalone agent execution without createApp. Resolves the adapter, binds
 - [Function: runEval()](./docs/api/appkit/Function.runEval.md): Run a single eval against a driver. Never throws for assertion or agent
 - [Function: runEvalsInDir()](./docs/api/appkit/Function.runEvalsInDir.md): Discover, load, and run every eval under each agent's evals/ dir, driving
+- [Function: runWithRetries()](./docs/api/appkit/Function.runWithRetries.md): Run attempt up to 1 + retries times, stopping as soon as it returns a
 - [Function: summarize()](./docs/api/appkit/Function.summarize.md): Parameters
 - [Function: text()](./docs/api/appkit/Function.text.md): Returns
 - [Function: timestamp()](./docs/api/appkit/Function.timestamp.md): Parameters
 - [Function: tool()](./docs/api/appkit/Function.tool.md): Factory for defining function tools with Zod schemas.
 - [Function: toolsFromRegistry()](./docs/api/appkit/Function.toolsFromRegistry.md): Produces the AgentToolDefinition[] a ToolProvider exposes to the LLM,
+- [Function: userTurns()](./docs/api/appkit/Function.userTurns.md): Extract every user-message content, in order, from an MLflow
 - [Function: uuid()](./docs/api/appkit/Function.uuid.md): Returns
 - [Function: varchar()](./docs/api/appkit/Function.varchar.md): Parameters
 - [Interface: AgentAdapter](./docs/api/appkit/Interface.AgentAdapter.md): Properties
@@ -174,6 +180,7 @@ npx @databricks/appkit docs <query>
 - [Interface: DatabricksAuth](./docs/api/appkit/Interface.DatabricksAuth.md): Resolved Databricks host + bearer token for the eval runner's REST calls.
 - [Interface: DatasetRow](./docs/api/appkit/Interface.DatasetRow.md): One row of a managed evaluation dataset. inputs are the kwargs passed to the
 - [Interface: DiscoveredEval](./docs/api/appkit/Interface.DiscoveredEval.md): An eval file found under server/agents//evals/.
+- [Interface: DiscoveredEvalConfig](./docs/api/appkit/Interface.DiscoveredEvalConfig.md): A per-agent evals.config.ts found under server/agents//evals/.
 - [Interface: DriveResult](./docs/api/appkit/Interface.DriveResult.md): What a driver returns for a single t.send.
 - [Interface: EndpointConfig](./docs/api/appkit/Interface.EndpointConfig.md): Properties
 - [Interface: EntityMutationHooks<TTable>](./docs/api/appkit/Interface.EntityMutationHooks.md): Mutation lifecycle for one entity. A before hook may return a replacement

package/dist/appkit/package.js

@@ -1,6 +1,6 @@
 //#region package.json
 var name = "@databricks/appkit";
-var version = "0.73.0";
+var version = "0.74.0";
 
 //#endregion
 export { name, version };

package/dist/beta.d.ts

@@ -17,17 +17,17 @@ import { defineSchema } from "./database/schema-builder/define-schema.js";
 import { fk } from "./database/schema-builder/fk.js";
 import { DatabricksAuth, ResolveDatabricksAuthOptions, resolveDatabricksAuth, resolveWorkspaceClient } from "./connectors/mlflow/auth.js";
 import { MlflowClient, PostResult, normalizeHost } from "./connectors/mlflow/client.js";
-import { DatasetRow, ReadEvalDatasetOptions, readEvalDataset } from "./evals/dataset.js";
+import { DatasetRow, ReadEvalDatasetOptions, readEvalDataset, userTurns } from "./evals/dataset.js";
 import { AssertionHandle, AssertionResult, CustomJudgeSpec, DriveResult, EvalDefinition, EvalDriver, EvalResult, MatchResult, Matcher, Severity, TestContext } from "./evals/types.js";
-import { defineEval } from "./evals/define-eval.js";
-import { DiscoveredEval, discoverEvalFiles } from "./evals/discover.js";
+import { defineEval, defineEvalConfig } from "./evals/define-eval.js";
+import { DiscoveredEval, DiscoveredEvalConfig, discoverEvalConfigs, discoverEvalFiles } from "./evals/discover.js";
 import { HttpDriverOptions, createHttpDriver } from "./evals/http-driver.js";
 import { JudgeConfig, JudgeScore, configureJudge, isJudgeConfigured } from "./evals/judge.js";
 import { equals, includes, matches } from "./evals/matchers.js";
 import { Assessment, ReportOutcome, buildAssessments, reportToMlflow } from "./evals/mlflow-report.js";
-import { EvalSummary, evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, summarize } from "./evals/report.js";
+import { EvalSummary, evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, summarize } from "./evals/report.js";
 import { RunEvalOptions, runEval } from "./evals/run-eval.js";
-import { EvalProgress, EvalRunSummary, RunEvalsOptions, runEvalsInDir } from "./evals/run-evals.js";
+import { EvalProgress, EvalRunSummary, RunEvalsOptions, runEvalsInDir, runWithRetries } from "./evals/run-evals.js";
 import "./evals/index.js";
 import { agentIdFromMarkdownPath, loadAgentFromFile, loadAgentsFromDir } from "./core/agent/load-agents.js";
 import { agents } from "./plugins/agents/agents.js";
@@ -40,4 +40,4 @@ import { DatabaseApiConfig, DatabaseApiWriteOperation, DatabaseApiWritesConfig,
 import { database } from "./plugins/database/database.js";
 import "./plugins/database/index.js";
 import "./plugins/beta-exports.generated.js";
-export { type AgentAdapter, type AgentDefinition, type AgentEvent, type AgentInput, type AgentRunContext, type AgentTool, type AgentToolDefinition, type AgentTools, type AgentToolsFn, type AgentsPluginConfig, AppKitMcpClient, AssertionHandle, AssertionResult, Assessment, type AutoInheritToolsConfig, type BaseSystemPromptOption, CustomJudgeSpec, type DatabaseApiConfig, type DatabaseApiWriteOperation, type DatabaseApiWritesConfig, type DatabaseExports, DatabricksAdapter, DatabricksAuth, DatasetRow, DiscoveredEval, DriveResult, type EntityHooks, type EntityMutationHooks, EvalDefinition, EvalDriver, EvalProgress, EvalResult, EvalRunSummary, EvalSummary, type FunctionTool, type GenerationParams, type HookApp, type HookContext, type HostedSupervisorTool, type HostedTool, HttpDriverOptions, type IAiSearchConfig, type IDatabaseConfig, type IndexConfig, JudgeConfig, JudgeScore, MatchResult, Matcher, type McpConnectAllResult, type Message, MlflowClient, type PluginToolkitProvider, type Plugins, PostResult, type PromptContext, ReadEvalDatasetOptions, type ReadSerializer, type ReadSerializerContext, type RegisteredAgent, ReportOutcome, type RerankerConfig, ResolveDatabricksAuthOptions, type ResolvedToolEntry, type RunAgentInput, type RunAgentResult, RunEvalOptions, RunEvalsOptions, SUPERVISOR_EXTENSION_KEY, type Schema, type SearchFilters, type SearchRequest, type SearchResponse, type SearchResult, Severity, SupervisorApiAdapter, type SupervisorApiAdapterOptions, type SupervisorExtension, type SupervisorTool, TestContext, type Thread, type ThreadStore, type ToolAnnotations, type ToolConfig, type ToolEntry, type ToolProvider, type ToolRegistry, type ToolkitEntry, type ToolkitOptions, type TransactionClient, type WorkspaceClientLike, agentIdFromMarkdownPath, agents, aiSearch, bigid, bigint, boolean, buildAssessments, configureJudge, createAgent, createHttpDriver, database, defineEval, defineSchema, defineTool, discoverEvalFiles, enumColumn, equals, evalGlyph, executeFromRegistry, fk, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, fromSupervisorApi, functionToolToDefinition, id, includes, integer, isFunctionTool, isHostedTool, isJudgeConfigured, isSupervisorTool, isToolkitEntry, jsonb, loadAgentFromFile, loadAgentsFromDir, matches, mcpServer, normalizeHost, parseTextToolCalls, readEvalDataset, reportToMlflow, resolveDatabricksAuth, resolveHostedTools, resolveWorkspaceClient, runAgent, runEval, runEvalsInDir, summarize, supervisorTools, text, timestamp, tool, toolsFromRegistry, uuid, varchar };
+export { type AgentAdapter, type AgentDefinition, type AgentEvent, type AgentInput, type AgentRunContext, type AgentTool, type AgentToolDefinition, type AgentTools, type AgentToolsFn, type AgentsPluginConfig, AppKitMcpClient, AssertionHandle, AssertionResult, Assessment, type AutoInheritToolsConfig, type BaseSystemPromptOption, CustomJudgeSpec, type DatabaseApiConfig, type DatabaseApiWriteOperation, type DatabaseApiWritesConfig, type DatabaseExports, DatabricksAdapter, DatabricksAuth, DatasetRow, DiscoveredEval, DiscoveredEvalConfig, DriveResult, type EntityHooks, type EntityMutationHooks, EvalDefinition, EvalDriver, EvalProgress, EvalResult, EvalRunSummary, EvalSummary, type FunctionTool, type GenerationParams, type HookApp, type HookContext, type HostedSupervisorTool, type HostedTool, HttpDriverOptions, type IAiSearchConfig, type IDatabaseConfig, type IndexConfig, JudgeConfig, JudgeScore, MatchResult, Matcher, type McpConnectAllResult, type Message, MlflowClient, type PluginToolkitProvider, type Plugins, PostResult, type PromptContext, ReadEvalDatasetOptions, type ReadSerializer, type ReadSerializerContext, type RegisteredAgent, ReportOutcome, type RerankerConfig, ResolveDatabricksAuthOptions, type ResolvedToolEntry, type RunAgentInput, type RunAgentResult, RunEvalOptions, RunEvalsOptions, SUPERVISOR_EXTENSION_KEY, type Schema, type SearchFilters, type SearchRequest, type SearchResponse, type SearchResult, Severity, SupervisorApiAdapter, type SupervisorApiAdapterOptions, type SupervisorExtension, type SupervisorTool, TestContext, type Thread, type ThreadStore, type ToolAnnotations, type ToolConfig, type ToolEntry, type ToolProvider, type ToolRegistry, type ToolkitEntry, type ToolkitOptions, type TransactionClient, type WorkspaceClientLike, agentIdFromMarkdownPath, agents, aiSearch, bigid, bigint, boolean, buildAssessments, configureJudge, createAgent, createHttpDriver, database, defineEval, defineEvalConfig, defineSchema, defineTool, discoverEvalConfigs, discoverEvalFiles, enumColumn, equals, evalGlyph, executeFromRegistry, fk, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, fromSupervisorApi, functionToolToDefinition, id, includes, integer, isFunctionTool, isHostedTool, isJudgeConfigured, isSupervisorTool, isToolkitEntry, jsonb, loadAgentFromFile, loadAgentsFromDir, matches, mcpServer, normalizeHost, parseTextToolCalls, readEvalDataset, reportToMlflow, resolveDatabricksAuth, resolveHostedTools, resolveWorkspaceClient, runAgent, runEval, runEvalsInDir, runWithRetries, summarize, supervisorTools, text, timestamp, tool, toolsFromRegistry, userTurns, uuid, varchar };

package/dist/beta.js

@@ -15,16 +15,16 @@ import { isToolkitEntry } from "./core/agent/types.js";
 import { runAgent } from "./core/agent/run-agent.js";
 import "./core/agent/tools/index.js";
 import "./database/schema-builder/index.js";
-import { readEvalDataset } from "./evals/dataset.js";
-import { defineEval } from "./evals/define-eval.js";
-import { discoverEvalFiles } from "./evals/discover.js";
+import { readEvalDataset, userTurns } from "./evals/dataset.js";
+import { defineEval, defineEvalConfig } from "./evals/define-eval.js";
+import { discoverEvalConfigs, discoverEvalFiles } from "./evals/discover.js";
 import { createHttpDriver } from "./evals/http-driver.js";
 import { configureJudge, isJudgeConfigured } from "./evals/judge.js";
 import { equals, includes, matches } from "./evals/matchers.js";
 import { buildAssessments, reportToMlflow } from "./evals/mlflow-report.js";
-import { evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, summarize } from "./evals/report.js";
+import { evalGlyph, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, summarize } from "./evals/report.js";
 import { runEval } from "./evals/run-eval.js";
-import { runEvalsInDir } from "./evals/run-evals.js";
+import { runEvalsInDir, runWithRetries } from "./evals/run-evals.js";
 import "./evals/index.js";
 import { agentIdFromMarkdownPath, loadAgentFromFile, loadAgentsFromDir } from "./core/agent/load-agents.js";
 import { agents } from "./plugins/agents/agents.js";
@@ -33,4 +33,4 @@ import { aiSearch } from "./plugins/ai-search/ai-search.js";
 import { database } from "./plugins/database/database.js";
 import "./plugins/beta-exports.generated.js";
 
-export { AppKitMcpClient, DatabricksAdapter, MlflowClient, SUPERVISOR_EXTENSION_KEY, SupervisorApiAdapter, agentIdFromMarkdownPath, agents, aiSearch, bigid, bigint, boolean, buildAssessments, configureJudge, createAgent, createHttpDriver, database, defineEval, defineSchema, defineTool, discoverEvalFiles, enumColumn, equals, evalGlyph, executeFromRegistry, fk, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatSummaryLine, fromSupervisorApi, functionToolToDefinition, id, includes, integer, isFunctionTool, isHostedTool, isJudgeConfigured, isSupervisorTool, isToolkitEntry, jsonb, loadAgentFromFile, loadAgentsFromDir, matches, mcpServer, normalizeHost, parseTextToolCalls, readEvalDataset, reportToMlflow, resolveDatabricksAuth, resolveHostedTools, resolveWorkspaceClient, runAgent, runEval, runEvalsInDir, summarize, supervisorTools, text, timestamp, tool, toolsFromRegistry, uuid, varchar };
+export { AppKitMcpClient, DatabricksAdapter, MlflowClient, SUPERVISOR_EXTENSION_KEY, SupervisorApiAdapter, agentIdFromMarkdownPath, agents, aiSearch, bigid, bigint, boolean, buildAssessments, configureJudge, createAgent, createHttpDriver, database, defineEval, defineEvalConfig, defineSchema, defineTool, discoverEvalConfigs, discoverEvalFiles, enumColumn, equals, evalGlyph, executeFromRegistry, fk, formatEvalDetail, formatEvalHeadline, formatEvalResults, formatResultsJUnit, formatResultsJson, formatSummaryLine, fromSupervisorApi, functionToolToDefinition, id, includes, integer, isFunctionTool, isHostedTool, isJudgeConfigured, isSupervisorTool, isToolkitEntry, jsonb, loadAgentFromFile, loadAgentsFromDir, matches, mcpServer, normalizeHost, parseTextToolCalls, readEvalDataset, reportToMlflow, resolveDatabricksAuth, resolveHostedTools, resolveWorkspaceClient, runAgent, runEval, runEvalsInDir, runWithRetries, summarize, supervisorTools, text, timestamp, tool, toolsFromRegistry, userTurns, uuid, varchar };

package/dist/cli/commands/agent/eval.js

@@ -1,4 +1,5 @@
-import { Command } from "commander";
+import fs from "node:fs";
+import { Command, Option } from "commander";
 
 //#region src/cli/commands/agent/eval.ts
 /**
@@ -24,6 +25,17 @@ function parseHeaders(values) {
 	return headers;
 }
 /**
+* Parse `--min-pass-rate`: a finite number in `[0, 1]`, or `undefined` when
+* unset. Throws on blank, out-of-range, or non-numeric input so a bad gate
+* value fails fast instead of silently disabling or inverting the CI gate.
+*/
+function parsePassRate(raw) {
+	if (raw === void 0) return void 0;
+	const n = Number(raw);
+	if (raw.trim() === "" || !Number.isFinite(n) || n < 0 || n > 1) throw new Error(`Invalid --min-pass-rate "${raw}" — expected a number in [0, 1]`);
+	return n;
+}
+/**
 * Native MLflow "Evaluation run" config — only when creds + an experiment are
 * all present (traces live in the app; the run + scores are driven from here).
 */
@@ -47,17 +59,22 @@ function resolveJudge(opts, auth) {
 		model
 	} : void 0;
 }
-/** Progress reporter: stream each eval as it runs instead of going silent. */
-function makeProgressReporter(runner, url) {
+/**
+* Progress reporter: stream each eval as it runs instead of going silent. In a
+* machine reporter (json/junit) the live per-eval streaming is suppressed and
+* banners go to stderr (via `info`), keeping stdout clean for the report.
+*/
+function makeProgressReporter(runner, url, machine, info) {
 	return (event) => {
 		switch (event.type) {
 			case "discovered":
-				console.log(`Running ${event.total} eval${event.total === 1 ? "" : "s"} against ${url}\n`);
+				info(`Running ${event.total} eval${event.total === 1 ? "" : "s"} against ${url}\n`);
 				break;
 			case "run-created":
-				console.log(`MLflow evaluation run: ${event.runId}\n`);
+				info(`MLflow evaluation run: ${event.runId}\n`);
 				break;
 			case "result":
+				if (machine) break;
 				console.log(`[${event.index + 1}/${event.total}] ${runner.formatEvalHeadline(event.result)}`);
 				for (const line of runner.formatEvalDetail(event.result)) console.log(line);
 				break;
@@ -67,33 +84,53 @@ function makeProgressReporter(runner, url) {
 function formatFailureLine(f) {
 	return `  ✗ trace ${f.traceId}: ${f.status ?? ""} ${f.error ?? ""}`.trim();
 }
-/** Print the MLflow assessment/finish outcome after a run that created one. */
-function printMlflowOutcome(mlflow) {
+/**
+* Print the MLflow assessment/finish outcome after a run that created one. The
+* summary line goes through `info` (stderr under a machine reporter); per-trace
+* failures and finish errors always go to stderr.
+*/
+function printMlflowOutcome(mlflow, info) {
 	const { report, finish } = mlflow;
-	console.log(`MLflow: ${report.written} assessment(s) written` + (report.skipped ? `, ${report.skipped} skipped` : "") + (report.failures.length ? `, ${report.failures.length} failed` : ""));
+	info(`MLflow: ${report.written} assessment(s) written` + (report.skipped ? `, ${report.skipped} skipped` : "") + (report.failures.length ? `, ${report.failures.length} failed` : ""));
 	for (const f of report.failures) console.error(formatFailureLine(f));
 	if (finish.metricsError) console.error(`  ⚠ metrics not logged: ${finish.metricsError}`);
 	if (!finish.finished) console.error(`  ✗ run left RUNNING — failed to finish: ${finish.finishError ?? "unknown"}`);
 }
 async function runAgentEval(filter, opts) {
 	const runner = await loadRunner();
-	const auth = await runner.resolveDatabricksAuth({
+	const credentials = {
 		profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE,
 		host: opts.databricksHost ?? process.env.DATABRICKS_HOST,
 		token: opts.databricksToken ?? process.env.DATABRICKS_TOKEN
-	}) ?? {};
+	};
+	const auth = await runner.resolveDatabricksAuth(credentials) ?? {};
 	const warehouseId = opts.warehouseId ?? process.env.DATABRICKS_WAREHOUSE_ID;
-	const workspaceClient = runner.resolveWorkspaceClient({
-		profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE,
-		host: opts.databricksHost ?? process.env.DATABRICKS_HOST,
-		token: opts.databricksToken ?? process.env.DATABRICKS_TOKEN
-	});
+	const workspaceClient = runner.resolveWorkspaceClient(credentials);
+	const parsedTimeout = opts.timeout ? Number.parseInt(opts.timeout, 10) : void 0;
+	const timeoutMs = parsedTimeout && parsedTimeout > 0 ? parsedTimeout : void 0;
+	const parsedRetries = opts.retries ? Number.parseInt(opts.retries, 10) : void 0;
+	const retries = parsedRetries && parsedRetries > 0 ? parsedRetries : void 0;
+	let minPassRate;
+	try {
+		minPassRate = parsePassRate(opts.minPassRate);
+	} catch (err) {
+		console.error(err instanceof Error ? err.message : String(err));
+		process.exitCode = 1;
+		return;
+	}
+	const reporter = opts.reporter ?? "text";
+	const machine = reporter !== "text";
+	const info = (msg) => {
+		if (machine) console.error(msg);
+		else console.log(msg);
+	};
 	let summary;
 	try {
 		summary = await runner.runEvalsInDir({
 			rootDir: opts.root,
 			baseUrl: opts.url,
 			filter,
+			tags: opts.tag,
 			strict: opts.strict,
 			headers: opts.header ? parseHeaders(opts.header) : void 0,
 			concurrency: opts.concurrency,
@@ -101,19 +138,43 @@ async function runAgentEval(filter, opts) {
 			judge: resolveJudge(opts, auth),
 			workspaceClient,
 			warehouseId,
-			onEvent: makeProgressReporter(runner, opts.url)
+			timeoutMs,
+			retries,
+			onEvent: makeProgressReporter(runner, opts.url, machine, info)
 		});
 	} catch (err) {
 		console.error(`\nEval run failed: ${err instanceof Error ? err.message : String(err)}`);
 		process.exitCode = 1;
 		return;
 	}
-	console.log(`\n${runner.formatSummaryLine(summary.results)}`);
-	if (summary.mlflow) printMlflowOutcome(summary.mlflow);
-	else console.log("\nMLflow evaluation run skipped — pass --experiment (or set MLFLOW_EXPERIMENT_ID) plus --profile/--databricks-host to create one.");
-	if (!runner.summarize(summary.results).allPassed) process.exitCode = 1;
+	info(`\n${runner.formatSummaryLine(summary.results)}`);
+	if (summary.mlflow) printMlflowOutcome(summary.mlflow, info);
+	else info("\nMLflow evaluation run skipped — pass --experiment (or set MLFLOW_EXPERIMENT_ID) plus --profile/--databricks-host to create one.");
+	if (machine) {
+		const report = reporter === "json" ? runner.formatResultsJson(summary.results) : runner.formatResultsJUnit(summary.results);
+		if (opts.output) {
+			try {
+				fs.writeFileSync(opts.output, `${report}\n`);
+			} catch (err) {
+				console.error(`Failed to write ${reporter} report to ${opts.output}: ${err instanceof Error ? err.message : String(err)}`);
+				process.exitCode = 1;
+				return;
+			}
+			info(`Wrote ${reporter} report to ${opts.output}`);
+		} else process.stdout.write(`${report}\n`);
+	}
+	const stats = runner.summarize(summary.results);
+	if (minPassRate !== void 0) {
+		const ok = stats.passRate >= minPassRate;
+		info(`Pass rate ${(stats.passRate * 100).toFixed(0)}% (threshold ${(minPassRate * 100).toFixed(0)}%) — ${ok ? "OK" : "below threshold"}`);
+		if (!ok) process.exitCode = 1;
+	} else if (!stats.allPassed) process.exitCode = 1;
 }
-const agentEvalCommand = new Command("eval").description("Run agent evals (server/agents/<id>/evals/*.eval.ts) against a running app").argument("[filter]", "Only run evals whose <agent>/<id> contains this substring (or an exact agent id)").option("--url <url>", "Base URL of the running app", "http://localhost:3000").option("--strict", "Fail on soft-assertion misses too", false).option("--concurrency <n>", "Max evals to run concurrently (default 4; keep at or below the app's max concurrent streams per user)", (v) => Number.parseInt(v, 10)).option("--root <dir>", "Project root containing server/agents/ (default: cwd)").option("--header <header...>", "Extra request header as 'Key: value' (repeatable)").option("--profile <name>", "Databricks CLI profile to authenticate with via OAuth (default: DATABRICKS_CONFIG_PROFILE)").option("--databricks-host <host>", "Databricks host for writing MLflow assessments (default: DATABRICKS_HOST)").option("--databricks-token <token>", "Databricks token for writing MLflow assessments (default: DATABRICKS_TOKEN)").option("--experiment <id>", "MLflow experiment id for the evaluation run (default: MLFLOW_EXPERIMENT_ID)").option("--warehouse-id <id>", "SQL warehouse id for reading managed eval datasets and writing assessments to UC-backed experiments (default: DATABRICKS_WAREHOUSE_ID, or MLFLOW_TRACING_SQL_WAREHOUSE_ID for assessments)").option("--judge-model <endpoint>", "Databricks serving endpoint to use as the LLM judge for t.judge.* (default: APPKIT_JUDGE_MODEL)").action(runAgentEval);
+const agentEvalCommand = new Command("eval").description("Run agent evals (server/agents/<id>/evals/*.eval.ts) against a running app").argument("[filter]", "Only run evals whose <agent>/<id> contains this substring (or an exact agent id)").option("--url <url>", "Base URL of the running app", "http://localhost:3000").option("--strict", "Fail on soft-assertion misses too", false).option("--concurrency <n>", "Max evals to run concurrently (default 4; keep at or below the app's max concurrent streams per user)", (v) => Number.parseInt(v, 10)).option("--root <dir>", "Project root containing server/agents/ (default: cwd)").option("--header <header...>", "Extra request header as 'Key: value' (repeatable)").option("--tag <tag...>", "Only run evals tagged with one of these tags (repeatable)").option("--profile <name>", "Databricks CLI profile to authenticate with via OAuth (default: DATABRICKS_CONFIG_PROFILE)").option("--databricks-host <host>", "Databricks host for writing MLflow assessments (default: DATABRICKS_HOST)").option("--databricks-token <token>", "Databricks token for writing MLflow assessments (default: DATABRICKS_TOKEN)").option("--experiment <id>", "MLflow experiment id for the evaluation run (default: MLFLOW_EXPERIMENT_ID)").option("--warehouse-id <id>", "SQL warehouse id for reading managed eval datasets and writing assessments to UC-backed experiments (default: DATABRICKS_WAREHOUSE_ID, or MLFLOW_TRACING_SQL_WAREHOUSE_ID for assessments)").option("--judge-model <endpoint>", "Databricks serving endpoint to use as the LLM judge for t.judge.* (default: APPKIT_JUDGE_MODEL)").option("--timeout <ms>", "Default per-eval timeout in ms (a per-eval timeoutMs overrides it)").option("--retries <n>", "Re-run an eval up to N times when it fails on an infra error (turn/timeout); assertion failures are not retried").option("--min-pass-rate <rate>", "Gate on aggregate pass rate (0..1) instead of requiring every eval to pass; exit 1 when below").addOption(new Option("--reporter <format>", "Report format: text (live console), json (dashboards), or junit (CI test reporters)").choices([
+	"text",
+	"json",
+	"junit"
+]).default("text")).option("--output <file>", "Write the json/junit report to this file instead of stdout (ignored for text)").action(runAgentEval);
 
 //#endregion
 export { agentEvalCommand };

package/dist/cli/commands/agent/eval.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval.js","names":[],"sources":["../../../../src/cli/commands/agent/eval.ts"],"sourcesContent":["import { Command } from \"commander\";\n\ninterface EvalRunSummary {\n  results: unknown[];\n  mlflow?: {\n    runId: string;\n    report: {\n      written: number;\n      skipped: number;\n      failures: Array<{ traceId: string; status?: number; error?: string }>;\n    };\n    finish: { finished: boolean; metricsError?: string; finishError?: string };\n  };\n}\n\ntype EvalProgress =\n  | { type: \"discovered\"; total: number }\n  | { type: \"run-created\"; runId: string }\n  | { type: \"start\"; id: string; index: number; total: number }\n  | { type: \"result\"; result: unknown; index: number; total: number };\n\n/** Subset of `@databricks/appkit/beta`'s eval runner used by this command. */\ninterface EvalRunner {\n  runEvalsInDir(opts: {\n    rootDir?: string;\n    baseUrl: string;\n    filter?: string;\n    strict?: boolean;\n    headers?: Record<string, string>;\n    concurrency?: number;\n    mlflow?: {\n      host: string;\n      token: string;\n      experimentId: string;\n      sqlWarehouseId?: string;\n    };\n    judge?: { host: string; token: string; model: string };\n    workspaceClient?: unknown;\n    warehouseId?: string;\n    onEvent?: (event: EvalProgress) => void;\n  }): Promise<EvalRunSummary>;\n  resolveDatabricksAuth(opts: {\n    profile?: string;\n    host?: string;\n    token?: string;\n  }): Promise<{ host: string; token: string } | undefined>;\n  resolveWorkspaceClient(opts: {\n    profile?: string;\n    host?: string;\n    token?: string;\n  }): unknown;\n  formatEvalHeadline(result: unknown): string;\n  evalGlyph(result: unknown): string;\n  formatEvalDetail(result: unknown): string[];\n  formatSummaryLine(results: unknown[]): string;\n  summarize(results: unknown[]): { allPassed: boolean };\n}\n\n/**\n * Loaded at runtime from the consuming project so this command (which ships in\n * `@databricks/shared`) doesn't take a build-time dependency on appkit. The\n * specifier is a variable so the type checker treats it as `any`.\n */\nasync function loadRunner(): Promise<EvalRunner> {\n  const spec = \"@databricks/appkit/beta\";\n  try {\n    return (await import(spec)) as unknown as EvalRunner;\n  } catch (err) {\n    throw new Error(\n      \"Could not load @databricks/appkit. Run `appkit agent eval` from a \" +\n        \"project with @databricks/appkit installed. \" +\n        `Cause: ${err instanceof Error ? err.message : String(err)}`,\n    );\n  }\n}\n\nfunction parseHeaders(values: string[]): Record<string, string> {\n  const headers: Record<string, string> = {};\n  for (const v of values) {\n    const i = v.indexOf(\":\");\n    if (i === -1) continue;\n    headers[v.slice(0, i).trim()] = v.slice(i + 1).trim();\n  }\n  return headers;\n}\n\ninterface EvalOptions {\n  url: string;\n  strict?: boolean;\n  root?: string;\n  header?: string[];\n  profile?: string;\n  databricksHost?: string;\n  databricksToken?: string;\n  experiment?: string;\n  judgeModel?: string;\n  concurrency?: number;\n  warehouseId?: string;\n}\n\n/** Resolved Databricks host + bearer (either field may be absent). */\ntype Auth = { host?: string; token?: string };\n\n/**\n * Native MLflow \"Evaluation run\" config — only when creds + an experiment are\n * all present (traces live in the app; the run + scores are driven from here).\n */\nfunction resolveMlflow(opts: EvalOptions, auth: Auth) {\n  const experimentId = opts.experiment ?? process.env.MLFLOW_EXPERIMENT_ID;\n  if (!(auth.host && auth.token && experimentId)) return undefined;\n  // UC-backed experiments need a SQL warehouse to write assessments to their\n  // V4 traces. Mirror mlflow's env var, and accept the common DATABRICKS one.\n  const sqlWarehouseId =\n    opts.warehouseId ??\n    process.env.MLFLOW_TRACING_SQL_WAREHOUSE_ID ??\n    process.env.DATABRICKS_WAREHOUSE_ID;\n  return {\n    host: auth.host,\n    token: auth.token,\n    experimentId,\n    ...(sqlWarehouseId ? { sqlWarehouseId } : {}),\n  };\n}\n\n/** LLM-as-judge config — reuses the Databricks creds + a judge serving endpoint. */\nfunction resolveJudge(opts: EvalOptions, auth: Auth) {\n  const model = opts.judgeModel ?? process.env.APPKIT_JUDGE_MODEL;\n  return model && auth.host && auth.token\n    ? { host: auth.host, token: auth.token, model }\n    : undefined;\n}\n\n/** Progress reporter: stream each eval as it runs instead of going silent. */\nfunction makeProgressReporter(\n  runner: EvalRunner,\n  url: string,\n): (event: EvalProgress) => void {\n  return (event) => {\n    switch (event.type) {\n      case \"discovered\":\n        console.log(\n          `Running ${event.total} eval${event.total === 1 ? \"\" : \"s\"} against ${url}\\n`,\n        );\n        break;\n      case \"run-created\":\n        console.log(`MLflow evaluation run: ${event.runId}\\n`);\n        break;\n      case \"result\": {\n        // One full line per completion — evals run concurrently, so a split\n        // \"start … glyph\" prefix would interleave into garbage.\n        console.log(\n          `[${event.index + 1}/${event.total}] ${runner.formatEvalHeadline(event.result)}`,\n        );\n        for (const line of runner.formatEvalDetail(event.result)) {\n          console.log(line);\n        }\n        break;\n      }\n    }\n  };\n}\n\nfunction formatFailureLine(f: {\n  traceId: string;\n  status?: number;\n  error?: string;\n}): string {\n  return `  ✗ trace ${f.traceId}: ${f.status ?? \"\"} ${f.error ?? \"\"}`.trim();\n}\n\n/** Print the MLflow assessment/finish outcome after a run that created one. */\nfunction printMlflowOutcome(\n  mlflow: NonNullable<EvalRunSummary[\"mlflow\"]>,\n): void {\n  const { report, finish } = mlflow;\n  console.log(\n    `MLflow: ${report.written} assessment(s) written` +\n      (report.skipped ? `, ${report.skipped} skipped` : \"\") +\n      (report.failures.length ? `, ${report.failures.length} failed` : \"\"),\n  );\n  for (const f of report.failures) {\n    console.error(formatFailureLine(f));\n  }\n  if (finish.metricsError) {\n    console.error(`  ⚠ metrics not logged: ${finish.metricsError}`);\n  }\n  if (!finish.finished) {\n    console.error(\n      `  ✗ run left RUNNING — failed to finish: ${finish.finishError ?? \"unknown\"}`,\n    );\n  }\n}\n\nasync function runAgentEval(\n  filter: string | undefined,\n  opts: EvalOptions,\n): Promise<void> {\n  const runner = await loadRunner();\n\n  // Resolve Databricks host + bearer the AppKit-native way: an explicit\n  // host/token (or DATABRICKS_* env) wins; otherwise the SDK mints an OAuth\n  // token from the CLI profile — so no hand-set PAT is required.\n  const auth: Auth =\n    (await runner.resolveDatabricksAuth({\n      profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE,\n      host: opts.databricksHost ?? process.env.DATABRICKS_HOST,\n      token: opts.databricksToken ?? process.env.DATABRICKS_TOKEN,\n    })) ?? {};\n\n  // Managed-dataset reads: a workspace client (same profile/host/token) + a SQL\n  // warehouse. Only needed by evals that declare `dataset`.\n  const warehouseId = opts.warehouseId ?? process.env.DATABRICKS_WAREHOUSE_ID;\n  const workspaceClient = runner.resolveWorkspaceClient({\n    profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE,\n    host: opts.databricksHost ?? process.env.DATABRICKS_HOST,\n    token: opts.databricksToken ?? process.env.DATABRICKS_TOKEN,\n  });\n\n  let summary: EvalRunSummary;\n  try {\n    summary = await runner.runEvalsInDir({\n      rootDir: opts.root,\n      baseUrl: opts.url,\n      filter,\n      strict: opts.strict,\n      headers: opts.header ? parseHeaders(opts.header) : undefined,\n      concurrency: opts.concurrency,\n      mlflow: resolveMlflow(opts, auth),\n      judge: resolveJudge(opts, auth),\n      workspaceClient,\n      warehouseId,\n      onEvent: makeProgressReporter(runner, opts.url),\n    });\n  } catch (err) {\n    // Setup failures (e.g. a bad --experiment for the MLflow run) reject before\n    // any eval runs; surface a clean message + non-zero exit rather than an\n    // unhandled promise rejection with a raw stack.\n    console.error(\n      `\\nEval run failed: ${err instanceof Error ? err.message : String(err)}`,\n    );\n    process.exitCode = 1;\n    return;\n  }\n  console.log(`\\n${runner.formatSummaryLine(summary.results)}`);\n\n  if (summary.mlflow) {\n    printMlflowOutcome(summary.mlflow);\n  } else {\n    console.log(\n      \"\\nMLflow evaluation run skipped — pass --experiment (or set\" +\n        \" MLFLOW_EXPERIMENT_ID) plus --profile/--databricks-host to create one.\",\n    );\n  }\n\n  if (!runner.summarize(summary.results).allPassed) {\n    process.exitCode = 1;\n  }\n}\n\nexport const agentEvalCommand = new Command(\"eval\")\n  .description(\n    \"Run agent evals (server/agents/<id>/evals/*.eval.ts) against a running app\",\n  )\n  .argument(\n    \"[filter]\",\n    \"Only run evals whose <agent>/<id> contains this substring (or an exact agent id)\",\n  )\n  .option(\"--url <url>\", \"Base URL of the running app\", \"http://localhost:3000\")\n  .option(\"--strict\", \"Fail on soft-assertion misses too\", false)\n  .option(\n    \"--concurrency <n>\",\n    \"Max evals to run concurrently (default 4; keep at or below the app's max concurrent streams per user)\",\n    (v) => Number.parseInt(v, 10),\n  )\n  .option(\n    \"--root <dir>\",\n    \"Project root containing server/agents/ (default: cwd)\",\n  )\n  .option(\n    \"--header <header...>\",\n    \"Extra request header as 'Key: value' (repeatable)\",\n  )\n  .option(\n    \"--profile <name>\",\n    \"Databricks CLI profile to authenticate with via OAuth (default: DATABRICKS_CONFIG_PROFILE)\",\n  )\n  .option(\n    \"--databricks-host <host>\",\n    \"Databricks host for writing MLflow assessments (default: DATABRICKS_HOST)\",\n  )\n  .option(\n    \"--databricks-token <token>\",\n    \"Databricks token for writing MLflow assessments (default: DATABRICKS_TOKEN)\",\n  )\n  .option(\n    \"--experiment <id>\",\n    \"MLflow experiment id for the evaluation run (default: MLFLOW_EXPERIMENT_ID)\",\n  )\n  .option(\n    \"--warehouse-id <id>\",\n    \"SQL warehouse id for reading managed eval datasets and writing assessments to UC-backed experiments (default: DATABRICKS_WAREHOUSE_ID, or MLFLOW_TRACING_SQL_WAREHOUSE_ID for assessments)\",\n  )\n  .option(\n    \"--judge-model <endpoint>\",\n    \"Databricks serving endpoint to use as the LLM judge for t.judge.* (default: APPKIT_JUDGE_MODEL)\",\n  )\n  .action(runAgentEval);\n"],"mappings":";;;;;;;;AA+DA,eAAe,aAAkC;CAC/C,MAAM,OAAO;AACb,KAAI;AACF,SAAQ,MAAM,OAAO;UACd,KAAK;AACZ,QAAM,IAAI,MACR,yHAEY,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GAC7D;;;AAIL,SAAS,aAAa,QAA0C;CAC9D,MAAM,UAAkC,EAAE;AAC1C,MAAK,MAAM,KAAK,QAAQ;EACtB,MAAM,IAAI,EAAE,QAAQ,IAAI;AACxB,MAAI,MAAM,GAAI;AACd,UAAQ,EAAE,MAAM,GAAG,EAAE,CAAC,MAAM,IAAI,EAAE,MAAM,IAAI,EAAE,CAAC,MAAM;;AAEvD,QAAO;;;;;;AAwBT,SAAS,cAAc,MAAmB,MAAY;CACpD,MAAM,eAAe,KAAK,cAAc,QAAQ,IAAI;AACpD,KAAI,EAAE,KAAK,QAAQ,KAAK,SAAS,cAAe,QAAO;CAGvD,MAAM,iBACJ,KAAK,eACL,QAAQ,IAAI,mCACZ,QAAQ,IAAI;AACd,QAAO;EACL,MAAM,KAAK;EACX,OAAO,KAAK;EACZ;EACA,GAAI,iBAAiB,EAAE,gBAAgB,GAAG,EAAE;EAC7C;;;AAIH,SAAS,aAAa,MAAmB,MAAY;CACnD,MAAM,QAAQ,KAAK,cAAc,QAAQ,IAAI;AAC7C,QAAO,SAAS,KAAK,QAAQ,KAAK,QAC9B;EAAE,MAAM,KAAK;EAAM,OAAO,KAAK;EAAO;EAAO,GAC7C;;;AAIN,SAAS,qBACP,QACA,KAC+B;AAC/B,SAAQ,UAAU;AAChB,UAAQ,MAAM,MAAd;GACE,KAAK;AACH,YAAQ,IACN,WAAW,MAAM,MAAM,OAAO,MAAM,UAAU,IAAI,KAAK,IAAI,WAAW,IAAI,IAC3E;AACD;GACF,KAAK;AACH,YAAQ,IAAI,0BAA0B,MAAM,MAAM,IAAI;AACtD;GACF,KAAK;AAGH,YAAQ,IACN,IAAI,MAAM,QAAQ,EAAE,GAAG,MAAM,MAAM,IAAI,OAAO,mBAAmB,MAAM,OAAO,GAC/E;AACD,SAAK,MAAM,QAAQ,OAAO,iBAAiB,MAAM,OAAO,CACtD,SAAQ,IAAI,KAAK;AAEnB;;;;AAMR,SAAS,kBAAkB,GAIhB;AACT,QAAO,aAAa,EAAE,QAAQ,IAAI,EAAE,UAAU,GAAG,GAAG,EAAE,SAAS,KAAK,MAAM;;;AAI5E,SAAS,mBACP,QACM;CACN,MAAM,EAAE,QAAQ,WAAW;AAC3B,SAAQ,IACN,WAAW,OAAO,QAAQ,2BACvB,OAAO,UAAU,KAAK,OAAO,QAAQ,YAAY,OACjD,OAAO,SAAS,SAAS,KAAK,OAAO,SAAS,OAAO,WAAW,IACpE;AACD,MAAK,MAAM,KAAK,OAAO,SACrB,SAAQ,MAAM,kBAAkB,EAAE,CAAC;AAErC,KAAI,OAAO,aACT,SAAQ,MAAM,2BAA2B,OAAO,eAAe;AAEjE,KAAI,CAAC,OAAO,SACV,SAAQ,MACN,4CAA4C,OAAO,eAAe,YACnE;;AAIL,eAAe,aACb,QACA,MACe;CACf,MAAM,SAAS,MAAM,YAAY;CAKjC,MAAM,OACH,MAAM,OAAO,sBAAsB;EAClC,SAAS,KAAK,WAAW,QAAQ,IAAI;EACrC,MAAM,KAAK,kBAAkB,QAAQ,IAAI;EACzC,OAAO,KAAK,mBAAmB,QAAQ,IAAI;EAC5C,CAAC,IAAK,EAAE;CAIX,MAAM,cAAc,KAAK,eAAe,QAAQ,IAAI;CACpD,MAAM,kBAAkB,OAAO,uBAAuB;EACpD,SAAS,KAAK,WAAW,QAAQ,IAAI;EACrC,MAAM,KAAK,kBAAkB,QAAQ,IAAI;EACzC,OAAO,KAAK,mBAAmB,QAAQ,IAAI;EAC5C,CAAC;CAEF,IAAI;AACJ,KAAI;AACF,YAAU,MAAM,OAAO,cAAc;GACnC,SAAS,KAAK;GACd,SAAS,KAAK;GACd;GACA,QAAQ,KAAK;GACb,SAAS,KAAK,SAAS,aAAa,KAAK,OAAO,GAAG;GACnD,aAAa,KAAK;GAClB,QAAQ,cAAc,MAAM,KAAK;GACjC,OAAO,aAAa,MAAM,KAAK;GAC/B;GACA;GACA,SAAS,qBAAqB,QAAQ,KAAK,IAAI;GAChD,CAAC;UACK,KAAK;AAIZ,UAAQ,MACN,sBAAsB,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GACvE;AACD,UAAQ,WAAW;AACnB;;AAEF,SAAQ,IAAI,KAAK,OAAO,kBAAkB,QAAQ,QAAQ,GAAG;AAE7D,KAAI,QAAQ,OACV,oBAAmB,QAAQ,OAAO;KAElC,SAAQ,IACN,oIAED;AAGH,KAAI,CAAC,OAAO,UAAU,QAAQ,QAAQ,CAAC,UACrC,SAAQ,WAAW;;AAIvB,MAAa,mBAAmB,IAAI,QAAQ,OAAO,CAChD,YACC,6EACD,CACA,SACC,YACA,mFACD,CACA,OAAO,eAAe,+BAA+B,wBAAwB,CAC7E,OAAO,YAAY,qCAAqC,MAAM,CAC9D,OACC,qBACA,0GACC,MAAM,OAAO,SAAS,GAAG,GAAG,CAC9B,CACA,OACC,gBACA,wDACD,CACA,OACC,wBACA,oDACD,CACA,OACC,oBACA,6FACD,CACA,OACC,4BACA,4EACD,CACA,OACC,8BACA,8EACD,CACA,OACC,qBACA,8EACD,CACA,OACC,uBACA,6LACD,CACA,OACC,4BACA,kGACD,CACA,OAAO,aAAa"}
+{"version":3,"file":"eval.js","names":[],"sources":["../../../../src/cli/commands/agent/eval.ts"],"sourcesContent":["import fs from \"node:fs\";\n\nimport { Command, Option } from \"commander\";\n\ninterface EvalRunSummary {\n  results: unknown[];\n  mlflow?: {\n    runId: string;\n    report: {\n      written: number;\n      skipped: number;\n      failures: Array<{ traceId: string; status?: number; error?: string }>;\n    };\n    finish: { finished: boolean; metricsError?: string; finishError?: string };\n  };\n}\n\ntype EvalProgress =\n  | { type: \"discovered\"; total: number }\n  | { type: \"run-created\"; runId: string }\n  | { type: \"start\"; id: string; index: number; total: number }\n  | { type: \"result\"; result: unknown; index: number; total: number };\n\n/** Subset of `@databricks/appkit/beta`'s eval runner used by this command. */\ninterface EvalRunner {\n  runEvalsInDir(opts: {\n    rootDir?: string;\n    baseUrl: string;\n    filter?: string;\n    tags?: string[];\n    strict?: boolean;\n    headers?: Record<string, string>;\n    concurrency?: number;\n    mlflow?: {\n      host: string;\n      token: string;\n      experimentId: string;\n      sqlWarehouseId?: string;\n    };\n    judge?: { host: string; token: string; model: string };\n    workspaceClient?: unknown;\n    warehouseId?: string;\n    timeoutMs?: number;\n    retries?: number;\n    onEvent?: (event: EvalProgress) => void;\n  }): Promise<EvalRunSummary>;\n  resolveDatabricksAuth(opts: {\n    profile?: string;\n    host?: string;\n    token?: string;\n  }): Promise<{ host: string; token: string } | undefined>;\n  resolveWorkspaceClient(opts: {\n    profile?: string;\n    host?: string;\n    token?: string;\n  }): unknown;\n  formatEvalHeadline(result: unknown): string;\n  evalGlyph(result: unknown): string;\n  formatEvalDetail(result: unknown): string[];\n  formatSummaryLine(results: unknown[]): string;\n  formatResultsJson(results: unknown[]): string;\n  formatResultsJUnit(results: unknown[]): string;\n  summarize(results: unknown[]): { allPassed: boolean; passRate: number };\n}\n\n/**\n * Loaded at runtime from the consuming project so this command (which ships in\n * `@databricks/shared`) doesn't take a build-time dependency on appkit. The\n * specifier is a variable so the type checker treats it as `any`.\n */\nasync function loadRunner(): Promise<EvalRunner> {\n  const spec = \"@databricks/appkit/beta\";\n  try {\n    return (await import(spec)) as unknown as EvalRunner;\n  } catch (err) {\n    throw new Error(\n      \"Could not load @databricks/appkit. Run `appkit agent eval` from a \" +\n        \"project with @databricks/appkit installed. \" +\n        `Cause: ${err instanceof Error ? err.message : String(err)}`,\n    );\n  }\n}\n\nfunction parseHeaders(values: string[]): Record<string, string> {\n  const headers: Record<string, string> = {};\n  for (const v of values) {\n    const i = v.indexOf(\":\");\n    if (i === -1) continue;\n    headers[v.slice(0, i).trim()] = v.slice(i + 1).trim();\n  }\n  return headers;\n}\n\n/**\n * Parse `--min-pass-rate`: a finite number in `[0, 1]`, or `undefined` when\n * unset. Throws on blank, out-of-range, or non-numeric input so a bad gate\n * value fails fast instead of silently disabling or inverting the CI gate.\n */\nexport function parsePassRate(raw: string | undefined): number | undefined {\n  if (raw === undefined) return undefined;\n  const n = Number(raw);\n  // Reject blank too: `Number(\"\")` is 0, which would silently disable the gate.\n  if (raw.trim() === \"\" || !Number.isFinite(n) || n < 0 || n > 1) {\n    throw new Error(\n      `Invalid --min-pass-rate \"${raw}\" — expected a number in [0, 1]`,\n    );\n  }\n  return n;\n}\n\ninterface EvalOptions {\n  url: string;\n  strict?: boolean;\n  root?: string;\n  header?: string[];\n  tag?: string[];\n  profile?: string;\n  databricksHost?: string;\n  databricksToken?: string;\n  experiment?: string;\n  judgeModel?: string;\n  concurrency?: number;\n  warehouseId?: string;\n  timeout?: string;\n  retries?: string;\n  minPassRate?: string;\n  reporter?: \"text\" | \"json\" | \"junit\";\n  output?: string;\n}\n\n/** Resolved Databricks host + bearer (either field may be absent). */\ntype Auth = { host?: string; token?: string };\n\n/**\n * Native MLflow \"Evaluation run\" config — only when creds + an experiment are\n * all present (traces live in the app; the run + scores are driven from here).\n */\nfunction resolveMlflow(opts: EvalOptions, auth: Auth) {\n  const experimentId = opts.experiment ?? process.env.MLFLOW_EXPERIMENT_ID;\n  if (!(auth.host && auth.token && experimentId)) return undefined;\n  // UC-backed experiments need a SQL warehouse to write assessments to their\n  // V4 traces. Mirror mlflow's env var, and accept the common DATABRICKS one.\n  const sqlWarehouseId =\n    opts.warehouseId ??\n    process.env.MLFLOW_TRACING_SQL_WAREHOUSE_ID ??\n    process.env.DATABRICKS_WAREHOUSE_ID;\n  return {\n    host: auth.host,\n    token: auth.token,\n    experimentId,\n    ...(sqlWarehouseId ? { sqlWarehouseId } : {}),\n  };\n}\n\n/** LLM-as-judge config — reuses the Databricks creds + a judge serving endpoint. */\nfunction resolveJudge(opts: EvalOptions, auth: Auth) {\n  const model = opts.judgeModel ?? process.env.APPKIT_JUDGE_MODEL;\n  return model && auth.host && auth.token\n    ? { host: auth.host, token: auth.token, model }\n    : undefined;\n}\n\n/**\n * Progress reporter: stream each eval as it runs instead of going silent. In a\n * machine reporter (json/junit) the live per-eval streaming is suppressed and\n * banners go to stderr (via `info`), keeping stdout clean for the report.\n */\nfunction makeProgressReporter(\n  runner: EvalRunner,\n  url: string,\n  machine: boolean,\n  info: (msg: string) => void,\n): (event: EvalProgress) => void {\n  return (event) => {\n    switch (event.type) {\n      case \"discovered\":\n        info(\n          `Running ${event.total} eval${event.total === 1 ? \"\" : \"s\"} against ${url}\\n`,\n        );\n        break;\n      case \"run-created\":\n        info(`MLflow evaluation run: ${event.runId}\\n`);\n        break;\n      case \"result\": {\n        if (machine) break;\n        // One full line per completion — evals run concurrently, so a split\n        // \"start … glyph\" prefix would interleave into garbage.\n        console.log(\n          `[${event.index + 1}/${event.total}] ${runner.formatEvalHeadline(event.result)}`,\n        );\n        for (const line of runner.formatEvalDetail(event.result)) {\n          console.log(line);\n        }\n        break;\n      }\n    }\n  };\n}\n\nfunction formatFailureLine(f: {\n  traceId: string;\n  status?: number;\n  error?: string;\n}): string {\n  return `  ✗ trace ${f.traceId}: ${f.status ?? \"\"} ${f.error ?? \"\"}`.trim();\n}\n\n/**\n * Print the MLflow assessment/finish outcome after a run that created one. The\n * summary line goes through `info` (stderr under a machine reporter); per-trace\n * failures and finish errors always go to stderr.\n */\nfunction printMlflowOutcome(\n  mlflow: NonNullable<EvalRunSummary[\"mlflow\"]>,\n  info: (msg: string) => void,\n): void {\n  const { report, finish } = mlflow;\n  info(\n    `MLflow: ${report.written} assessment(s) written` +\n      (report.skipped ? `, ${report.skipped} skipped` : \"\") +\n      (report.failures.length ? `, ${report.failures.length} failed` : \"\"),\n  );\n  for (const f of report.failures) {\n    console.error(formatFailureLine(f));\n  }\n  if (finish.metricsError) {\n    console.error(`  ⚠ metrics not logged: ${finish.metricsError}`);\n  }\n  if (!finish.finished) {\n    console.error(\n      `  ✗ run left RUNNING — failed to finish: ${finish.finishError ?? \"unknown\"}`,\n    );\n  }\n}\n\nasync function runAgentEval(\n  filter: string | undefined,\n  opts: EvalOptions,\n): Promise<void> {\n  const runner = await loadRunner();\n\n  // Databricks credentials shared by auth resolution and the workspace client:\n  // an explicit flag/DATABRICKS_* env wins, else the SDK resolves from the CLI\n  // profile.\n  const credentials = {\n    profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE,\n    host: opts.databricksHost ?? process.env.DATABRICKS_HOST,\n    token: opts.databricksToken ?? process.env.DATABRICKS_TOKEN,\n  };\n\n  // Resolve Databricks host + bearer the AppKit-native way: an explicit\n  // host/token wins; otherwise the SDK mints an OAuth token from the CLI\n  // profile — so no hand-set PAT is required.\n  const auth: Auth = (await runner.resolveDatabricksAuth(credentials)) ?? {};\n\n  // Managed-dataset reads: a workspace client (same profile/host/token) + a SQL\n  // warehouse. Only needed by evals that declare `dataset`.\n  const warehouseId = opts.warehouseId ?? process.env.DATABRICKS_WAREHOUSE_ID;\n  const workspaceClient = runner.resolveWorkspaceClient(credentials);\n\n  // Runner-level default per-eval timeout (ms). A per-eval `timeoutMs` wins.\n  const parsedTimeout = opts.timeout\n    ? Number.parseInt(opts.timeout, 10)\n    : undefined;\n  const timeoutMs =\n    parsedTimeout && parsedTimeout > 0 ? parsedTimeout : undefined;\n\n  // Extra attempts for evals that fail on an infra error (turn/timeout). Junk\n  // or negative input falls back to no retries.\n  const parsedRetries = opts.retries\n    ? Number.parseInt(opts.retries, 10)\n    : undefined;\n  const retries =\n    parsedRetries && parsedRetries > 0 ? parsedRetries : undefined;\n\n  // Validate up front so a bad gate value fails before the run, not after.\n  let minPassRate: number | undefined;\n  try {\n    minPassRate = parsePassRate(opts.minPassRate);\n  } catch (err) {\n    console.error(err instanceof Error ? err.message : String(err));\n    process.exitCode = 1;\n    return;\n  }\n\n  // In a machine reporter (json/junit), stdout is reserved for the report (it\n  // may be piped), so human-facing lines go to stderr and the per-eval live\n  // streaming is suppressed. Text mode keeps its current stdout behavior.\n  const reporter = opts.reporter ?? \"text\";\n  const machine = reporter !== \"text\";\n  const info = (msg: string): void => {\n    if (machine) console.error(msg);\n    else console.log(msg);\n  };\n\n  let summary: EvalRunSummary;\n  try {\n    summary = await runner.runEvalsInDir({\n      rootDir: opts.root,\n      baseUrl: opts.url,\n      filter,\n      tags: opts.tag,\n      strict: opts.strict,\n      headers: opts.header ? parseHeaders(opts.header) : undefined,\n      concurrency: opts.concurrency,\n      mlflow: resolveMlflow(opts, auth),\n      judge: resolveJudge(opts, auth),\n      workspaceClient,\n      warehouseId,\n      timeoutMs,\n      retries,\n      onEvent: makeProgressReporter(runner, opts.url, machine, info),\n    });\n  } catch (err) {\n    // Setup failures (e.g. a bad --experiment for the MLflow run) reject before\n    // any eval runs; surface a clean message + non-zero exit rather than an\n    // unhandled promise rejection with a raw stack.\n    console.error(\n      `\\nEval run failed: ${err instanceof Error ? err.message : String(err)}`,\n    );\n    process.exitCode = 1;\n    return;\n  }\n\n  // The final human summary always shows (stderr for machine reporters so it\n  // never pollutes the report on stdout/file).\n  info(`\\n${runner.formatSummaryLine(summary.results)}`);\n\n  if (summary.mlflow) {\n    printMlflowOutcome(summary.mlflow, info);\n  } else {\n    info(\n      \"\\nMLflow evaluation run skipped — pass --experiment (or set\" +\n        \" MLFLOW_EXPERIMENT_ID) plus --profile/--databricks-host to create one.\",\n    );\n  }\n\n  // Machine-readable report: build the string with a pure formatter, then emit\n  // it to --output <file> or stdout (kept clean of the human noise above).\n  if (machine) {\n    const report =\n      reporter === \"json\"\n        ? runner.formatResultsJson(summary.results)\n        : runner.formatResultsJUnit(summary.results);\n    if (opts.output) {\n      try {\n        fs.writeFileSync(opts.output, `${report}\\n`);\n      } catch (err) {\n        console.error(\n          `Failed to write ${reporter} report to ${opts.output}: ${\n            err instanceof Error ? err.message : String(err)\n          }`,\n        );\n        process.exitCode = 1;\n        return;\n      }\n      info(`Wrote ${reporter} report to ${opts.output}`);\n    } else {\n      process.stdout.write(`${report}\\n`);\n    }\n  }\n\n  const stats = runner.summarize(summary.results);\n  if (minPassRate !== undefined) {\n    // Threshold mode: gate on the aggregate pass rate rather than requiring\n    // every eval to pass.\n    const ok = stats.passRate >= minPassRate;\n    info(\n      `Pass rate ${(stats.passRate * 100).toFixed(0)}% (threshold ${(\n        minPassRate * 100\n      ).toFixed(0)}%) — ${ok ? \"OK\" : \"below threshold\"}`,\n    );\n    if (!ok) process.exitCode = 1;\n  } else if (!stats.allPassed) {\n    process.exitCode = 1;\n  }\n}\n\nexport const agentEvalCommand = new Command(\"eval\")\n  .description(\n    \"Run agent evals (server/agents/<id>/evals/*.eval.ts) against a running app\",\n  )\n  .argument(\n    \"[filter]\",\n    \"Only run evals whose <agent>/<id> contains this substring (or an exact agent id)\",\n  )\n  .option(\"--url <url>\", \"Base URL of the running app\", \"http://localhost:3000\")\n  .option(\"--strict\", \"Fail on soft-assertion misses too\", false)\n  .option(\n    \"--concurrency <n>\",\n    \"Max evals to run concurrently (default 4; keep at or below the app's max concurrent streams per user)\",\n    (v) => Number.parseInt(v, 10),\n  )\n  .option(\n    \"--root <dir>\",\n    \"Project root containing server/agents/ (default: cwd)\",\n  )\n  .option(\n    \"--header <header...>\",\n    \"Extra request header as 'Key: value' (repeatable)\",\n  )\n  .option(\n    \"--tag <tag...>\",\n    \"Only run evals tagged with one of these tags (repeatable)\",\n  )\n  .option(\n    \"--profile <name>\",\n    \"Databricks CLI profile to authenticate with via OAuth (default: DATABRICKS_CONFIG_PROFILE)\",\n  )\n  .option(\n    \"--databricks-host <host>\",\n    \"Databricks host for writing MLflow assessments (default: DATABRICKS_HOST)\",\n  )\n  .option(\n    \"--databricks-token <token>\",\n    \"Databricks token for writing MLflow assessments (default: DATABRICKS_TOKEN)\",\n  )\n  .option(\n    \"--experiment <id>\",\n    \"MLflow experiment id for the evaluation run (default: MLFLOW_EXPERIMENT_ID)\",\n  )\n  .option(\n    \"--warehouse-id <id>\",\n    \"SQL warehouse id for reading managed eval datasets and writing assessments to UC-backed experiments (default: DATABRICKS_WAREHOUSE_ID, or MLFLOW_TRACING_SQL_WAREHOUSE_ID for assessments)\",\n  )\n  .option(\n    \"--judge-model <endpoint>\",\n    \"Databricks serving endpoint to use as the LLM judge for t.judge.* (default: APPKIT_JUDGE_MODEL)\",\n  )\n  .option(\n    \"--timeout <ms>\",\n    \"Default per-eval timeout in ms (a per-eval timeoutMs overrides it)\",\n  )\n  .option(\n    \"--retries <n>\",\n    \"Re-run an eval up to N times when it fails on an infra error (turn/timeout); assertion failures are not retried\",\n  )\n  .option(\n    \"--min-pass-rate <rate>\",\n    \"Gate on aggregate pass rate (0..1) instead of requiring every eval to pass; exit 1 when below\",\n  )\n  .addOption(\n    new Option(\n      \"--reporter <format>\",\n      \"Report format: text (live console), json (dashboards), or junit (CI test reporters)\",\n    )\n      .choices([\"text\", \"json\", \"junit\"])\n      .default(\"text\"),\n  )\n  .option(\n    \"--output <file>\",\n    \"Write the json/junit report to this file instead of stdout (ignored for text)\",\n  )\n  .action(runAgentEval);\n"],"mappings":";;;;;;;;;AAsEA,eAAe,aAAkC;CAC/C,MAAM,OAAO;AACb,KAAI;AACF,SAAQ,MAAM,OAAO;UACd,KAAK;AACZ,QAAM,IAAI,MACR,yHAEY,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GAC7D;;;AAIL,SAAS,aAAa,QAA0C;CAC9D,MAAM,UAAkC,EAAE;AAC1C,MAAK,MAAM,KAAK,QAAQ;EACtB,MAAM,IAAI,EAAE,QAAQ,IAAI;AACxB,MAAI,MAAM,GAAI;AACd,UAAQ,EAAE,MAAM,GAAG,EAAE,CAAC,MAAM,IAAI,EAAE,MAAM,IAAI,EAAE,CAAC,MAAM;;AAEvD,QAAO;;;;;;;AAQT,SAAgB,cAAc,KAA6C;AACzE,KAAI,QAAQ,OAAW,QAAO;CAC9B,MAAM,IAAI,OAAO,IAAI;AAErB,KAAI,IAAI,MAAM,KAAK,MAAM,CAAC,OAAO,SAAS,EAAE,IAAI,IAAI,KAAK,IAAI,EAC3D,OAAM,IAAI,MACR,4BAA4B,IAAI,iCACjC;AAEH,QAAO;;;;;;AA8BT,SAAS,cAAc,MAAmB,MAAY;CACpD,MAAM,eAAe,KAAK,cAAc,QAAQ,IAAI;AACpD,KAAI,EAAE,KAAK,QAAQ,KAAK,SAAS,cAAe,QAAO;CAGvD,MAAM,iBACJ,KAAK,eACL,QAAQ,IAAI,mCACZ,QAAQ,IAAI;AACd,QAAO;EACL,MAAM,KAAK;EACX,OAAO,KAAK;EACZ;EACA,GAAI,iBAAiB,EAAE,gBAAgB,GAAG,EAAE;EAC7C;;;AAIH,SAAS,aAAa,MAAmB,MAAY;CACnD,MAAM,QAAQ,KAAK,cAAc,QAAQ,IAAI;AAC7C,QAAO,SAAS,KAAK,QAAQ,KAAK,QAC9B;EAAE,MAAM,KAAK;EAAM,OAAO,KAAK;EAAO;EAAO,GAC7C;;;;;;;AAQN,SAAS,qBACP,QACA,KACA,SACA,MAC+B;AAC/B,SAAQ,UAAU;AAChB,UAAQ,MAAM,MAAd;GACE,KAAK;AACH,SACE,WAAW,MAAM,MAAM,OAAO,MAAM,UAAU,IAAI,KAAK,IAAI,WAAW,IAAI,IAC3E;AACD;GACF,KAAK;AACH,SAAK,0BAA0B,MAAM,MAAM,IAAI;AAC/C;GACF,KAAK;AACH,QAAI,QAAS;AAGb,YAAQ,IACN,IAAI,MAAM,QAAQ,EAAE,GAAG,MAAM,MAAM,IAAI,OAAO,mBAAmB,MAAM,OAAO,GAC/E;AACD,SAAK,MAAM,QAAQ,OAAO,iBAAiB,MAAM,OAAO,CACtD,SAAQ,IAAI,KAAK;AAEnB;;;;AAMR,SAAS,kBAAkB,GAIhB;AACT,QAAO,aAAa,EAAE,QAAQ,IAAI,EAAE,UAAU,GAAG,GAAG,EAAE,SAAS,KAAK,MAAM;;;;;;;AAQ5E,SAAS,mBACP,QACA,MACM;CACN,MAAM,EAAE,QAAQ,WAAW;AAC3B,MACE,WAAW,OAAO,QAAQ,2BACvB,OAAO,UAAU,KAAK,OAAO,QAAQ,YAAY,OACjD,OAAO,SAAS,SAAS,KAAK,OAAO,SAAS,OAAO,WAAW,IACpE;AACD,MAAK,MAAM,KAAK,OAAO,SACrB,SAAQ,MAAM,kBAAkB,EAAE,CAAC;AAErC,KAAI,OAAO,aACT,SAAQ,MAAM,2BAA2B,OAAO,eAAe;AAEjE,KAAI,CAAC,OAAO,SACV,SAAQ,MACN,4CAA4C,OAAO,eAAe,YACnE;;AAIL,eAAe,aACb,QACA,MACe;CACf,MAAM,SAAS,MAAM,YAAY;CAKjC,MAAM,cAAc;EAClB,SAAS,KAAK,WAAW,QAAQ,IAAI;EACrC,MAAM,KAAK,kBAAkB,QAAQ,IAAI;EACzC,OAAO,KAAK,mBAAmB,QAAQ,IAAI;EAC5C;CAKD,MAAM,OAAc,MAAM,OAAO,sBAAsB,YAAY,IAAK,EAAE;CAI1E,MAAM,cAAc,KAAK,eAAe,QAAQ,IAAI;CACpD,MAAM,kBAAkB,OAAO,uBAAuB,YAAY;CAGlE,MAAM,gBAAgB,KAAK,UACvB,OAAO,SAAS,KAAK,SAAS,GAAG,GACjC;CACJ,MAAM,YACJ,iBAAiB,gBAAgB,IAAI,gBAAgB;CAIvD,MAAM,gBAAgB,KAAK,UACvB,OAAO,SAAS,KAAK,SAAS,GAAG,GACjC;CACJ,MAAM,UACJ,iBAAiB,gBAAgB,IAAI,gBAAgB;CAGvD,IAAI;AACJ,KAAI;AACF,gBAAc,cAAc,KAAK,YAAY;UACtC,KAAK;AACZ,UAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,CAAC;AAC/D,UAAQ,WAAW;AACnB;;CAMF,MAAM,WAAW,KAAK,YAAY;CAClC,MAAM,UAAU,aAAa;CAC7B,MAAM,QAAQ,QAAsB;AAClC,MAAI,QAAS,SAAQ,MAAM,IAAI;MAC1B,SAAQ,IAAI,IAAI;;CAGvB,IAAI;AACJ,KAAI;AACF,YAAU,MAAM,OAAO,cAAc;GACnC,SAAS,KAAK;GACd,SAAS,KAAK;GACd;GACA,MAAM,KAAK;GACX,QAAQ,KAAK;GACb,SAAS,KAAK,SAAS,aAAa,KAAK,OAAO,GAAG;GACnD,aAAa,KAAK;GAClB,QAAQ,cAAc,MAAM,KAAK;GACjC,OAAO,aAAa,MAAM,KAAK;GAC/B;GACA;GACA;GACA;GACA,SAAS,qBAAqB,QAAQ,KAAK,KAAK,SAAS,KAAK;GAC/D,CAAC;UACK,KAAK;AAIZ,UAAQ,MACN,sBAAsB,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GACvE;AACD,UAAQ,WAAW;AACnB;;AAKF,MAAK,KAAK,OAAO,kBAAkB,QAAQ,QAAQ,GAAG;AAEtD,KAAI,QAAQ,OACV,oBAAmB,QAAQ,QAAQ,KAAK;KAExC,MACE,oIAED;AAKH,KAAI,SAAS;EACX,MAAM,SACJ,aAAa,SACT,OAAO,kBAAkB,QAAQ,QAAQ,GACzC,OAAO,mBAAmB,QAAQ,QAAQ;AAChD,MAAI,KAAK,QAAQ;AACf,OAAI;AACF,OAAG,cAAc,KAAK,QAAQ,GAAG,OAAO,IAAI;YACrC,KAAK;AACZ,YAAQ,MACN,mBAAmB,SAAS,aAAa,KAAK,OAAO,IACnD,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GAEnD;AACD,YAAQ,WAAW;AACnB;;AAEF,QAAK,SAAS,SAAS,aAAa,KAAK,SAAS;QAElD,SAAQ,OAAO,MAAM,GAAG,OAAO,IAAI;;CAIvC,MAAM,QAAQ,OAAO,UAAU,QAAQ,QAAQ;AAC/C,KAAI,gBAAgB,QAAW;EAG7B,MAAM,KAAK,MAAM,YAAY;AAC7B,OACE,cAAc,MAAM,WAAW,KAAK,QAAQ,EAAE,CAAC,gBAC7C,cAAc,KACd,QAAQ,EAAE,CAAC,OAAO,KAAK,OAAO,oBACjC;AACD,MAAI,CAAC,GAAI,SAAQ,WAAW;YACnB,CAAC,MAAM,UAChB,SAAQ,WAAW;;AAIvB,MAAa,mBAAmB,IAAI,QAAQ,OAAO,CAChD,YACC,6EACD,CACA,SACC,YACA,mFACD,CACA,OAAO,eAAe,+BAA+B,wBAAwB,CAC7E,OAAO,YAAY,qCAAqC,MAAM,CAC9D,OACC,qBACA,0GACC,MAAM,OAAO,SAAS,GAAG,GAAG,CAC9B,CACA,OACC,gBACA,wDACD,CACA,OACC,wBACA,oDACD,CACA,OACC,kBACA,4DACD,CACA,OACC,oBACA,6FACD,CACA,OACC,4BACA,4EACD,CACA,OACC,8BACA,8EACD,CACA,OACC,qBACA,8EACD,CACA,OACC,uBACA,6LACD,CACA,OACC,4BACA,kGACD,CACA,OACC,kBACA,qEACD,CACA,OACC,iBACA,kHACD,CACA,OACC,0BACA,gGACD,CACA,UACC,IAAI,OACF,uBACA,sFACD,CACE,QAAQ;CAAC;CAAQ;CAAQ;CAAQ,CAAC,CAClC,QAAQ,OAAO,CACnB,CACA,OACC,mBACA,gFACD,CACA,OAAO,aAAa"}

package/dist/evals/dataset.d.ts

@@ -19,6 +19,19 @@ interface ReadEvalDatasetOptions {
   /** Optional row cap. */
   limit?: number;
 }
+/**
+ * Extract every user-message content, in order, from an MLflow
+ * `{"messages":[{"role":"user","content":"..."}]}` input. A dataset row can
+ * carry a full multi-turn conversation; replaying these against one thread (one
+ * `t.send` per returned string) lets the agent see the accumulating history.
+ *
+ * Only `role === "user"` turns are returned — any interleaved `assistant`/
+ * `system` messages in the row are ignored, since the agent generates its own
+ * responses; you never inject the dataset's assistant turns. A single-user-turn
+ * row yields a one-element array (backward compatible); a row with no `messages`
+ * yields `[]`.
+ */
+declare function userTurns(input: Record<string, unknown>): string[];
 /**
  * Read a Databricks managed evaluation dataset (a Unity Catalog table with
  * `inputs`/`expectations` columns) into rows, over the public SQL Statement
@@ -32,5 +45,5 @@ interface ReadEvalDatasetOptions {
  */
 declare function readEvalDataset(client: WorkspaceClient, options: ReadEvalDatasetOptions): Promise<DatasetRow[]>;
 //#endregion
-export { DatasetRow, ReadEvalDatasetOptions, readEvalDataset };
+export { DatasetRow, ReadEvalDatasetOptions, readEvalDataset, userTurns };
 //# sourceMappingURL=dataset.d.ts.map

package/dist/evals/dataset.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dataset.d.ts","names":[],"sources":["../../src/evals/dataset.ts"],"mappings":";;;;;AASA;;;;UAAiB,UAAA;EACf,MAAA,EAAQ,MAAA;EACR,YAAA,GAAe,MAAA;AAAA;AAAA,UAGA,sBAAA;EAHM;EAKrB,KAAA;EAFqC;EAIrC,WAAA;EAJqC;EAMrC,KAAA;AAAA;;;;AAyBF;;;;;;;;iBAAsB,eAAA,CACpB,MAAA,EAAQ,eAAA,EACR,OAAA,EAAS,sBAAA,GACR,OAAA,CAAQ,UAAA"}
+{"version":3,"file":"dataset.d.ts","names":[],"sources":["../../src/evals/dataset.ts"],"mappings":";;;;;AASA;;;;UAAiB,UAAA;EACf,MAAA,EAAQ,MAAA;EACR,YAAA,GAAe,MAAA;AAAA;AAAA,UAGA,sBAAA;EAHM;EAKrB,KAAA;EAFqC;EAIrC,WAAA;EAJqC;EAMrC,KAAA;AAAA;;;;AAeF;;;;;AAkCA;;;;iBAlCgB,SAAA,CAAU,KAAA,EAAO,MAAA;;;;;;;;;;;;iBAkCX,eAAA,CACpB,MAAA,EAAQ,eAAA,EACR,OAAA,EAAS,sBAAA,GACR,OAAA,CAAQ,UAAA"}

package/dist/evals/dataset.js

@@ -2,6 +2,21 @@ import { SQLWarehouseConnector } from "../connectors/sql-warehouse/client.js";
 import "../connectors/index.js";
 
 //#region src/evals/dataset.ts
+/**
+* Extract every user-message content, in order, from an MLflow
+* `{"messages":[{"role":"user","content":"..."}]}` input. A dataset row can
+* carry a full multi-turn conversation; replaying these against one thread (one
+* `t.send` per returned string) lets the agent see the accumulating history.
+*
+* Only `role === "user"` turns are returned — any interleaved `assistant`/
+* `system` messages in the row are ignored, since the agent generates its own
+* responses; you never inject the dataset's assistant turns. A single-user-turn
+* row yields a one-element array (backward compatible); a row with no `messages`
+* yields `[]`.
+*/
+function userTurns(input) {
+	return (Array.isArray(input.messages) ? input.messages : []).filter((m) => typeof m === "object" && m !== null).filter((m) => m.role === "user").map((m) => typeof m.content === "string" ? m.content : "");
+}
 /** A managed eval dataset is a UC table; only 3-level names are valid. */
 const UC_TABLE = /^[A-Za-z0-9_]+\.[A-Za-z0-9_]+\.[A-Za-z0-9_]+$/;
 /** Coerce a cell (already JSON-parsed by the connector for JSON columns) to a record. */
@@ -32,5 +47,5 @@ async function readEvalDataset(client, options) {
 }
 
 //#endregion
-export { readEvalDataset };
+export { readEvalDataset, userTurns };
 //# sourceMappingURL=dataset.js.map

package/dist/evals/dataset.js.map

@@ -1 +1 @@
-{"version":3,"file":"dataset.js","names":[],"sources":["../../src/evals/dataset.ts"],"sourcesContent":["import { SQLWarehouseConnector } from \"../connectors\";\nimport type { WorkspaceClient } from \"../workspace-client\";\n\n/**\n * One row of a managed evaluation dataset. `inputs` are the kwargs passed to the\n * agent for the turn; `expectations` (when present) is the row's ground truth /\n * guidelines. Mirrors the `{inputs, expectations}` shape of `mlflow.genai`\n * datasets and of the Unity Catalog table backing a managed eval dataset.\n */\nexport interface DatasetRow {\n  inputs: Record<string, unknown>;\n  expectations?: Record<string, unknown>;\n}\n\nexport interface ReadEvalDatasetOptions {\n  /** Fully-qualified UC table: `catalog.schema.table`. */\n  table: string;\n  /** SQL warehouse id to run the read against. */\n  warehouseId: string;\n  /** Optional row cap. */\n  limit?: number;\n}\n\n/** A managed eval dataset is a UC table; only 3-level names are valid. */\nconst UC_TABLE = /^[A-Za-z0-9_]+\\.[A-Za-z0-9_]+\\.[A-Za-z0-9_]+$/;\n\n/** Coerce a cell (already JSON-parsed by the connector for JSON columns) to a record. */\nfunction toRecord(value: unknown): Record<string, unknown> | undefined {\n  if (value && typeof value === \"object\" && !Array.isArray(value)) {\n    return value as Record<string, unknown>;\n  }\n  return undefined;\n}\n\n/**\n * Read a Databricks managed evaluation dataset (a Unity Catalog table with\n * `inputs`/`expectations` columns) into rows, over the public SQL Statement\n * Execution API. Reuses {@link SQLWarehouseConnector} for submit/poll/transform\n * — its result transform already JSON-parses string columns into objects, so\n * `inputs`/`expectations` come back as records whether the table stores them as\n * JSON strings or structs.\n *\n * The Python `mlflow.genai.datasets` API needs a Spark session (no TS\n * equivalent), so we read the backing table directly.\n */\nexport async function readEvalDataset(\n  client: WorkspaceClient,\n  options: ReadEvalDatasetOptions,\n): Promise<DatasetRow[]> {\n  if (!UC_TABLE.test(options.table)) {\n    throw new Error(\n      `Invalid dataset table \"${options.table}\" — expected catalog.schema.table`,\n    );\n  }\n\n  const limit =\n    typeof options.limit === \"number\" && options.limit > 0\n      ? ` LIMIT ${Math.floor(options.limit)}`\n      : \"\";\n  const connector = new SQLWarehouseConnector({});\n  const response = await connector.executeStatement(client, {\n    warehouse_id: options.warehouseId,\n    statement: `SELECT inputs, expectations FROM ${options.table}${limit}`,\n  });\n\n  const rows =\n    (response.result as { data?: Array<Record<string, unknown>> } | undefined)\n      ?.data ?? [];\n\n  return rows.map((row) => ({\n    inputs: toRecord(row.inputs) ?? {},\n    expectations: toRecord(row.expectations),\n  }));\n}\n"],"mappings":";;;;;AAwBA,MAAM,WAAW;;AAGjB,SAAS,SAAS,OAAqD;AACrE,KAAI,SAAS,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,MAAM,CAC7D,QAAO;;;;;;;;;;;;;AAgBX,eAAsB,gBACpB,QACA,SACuB;AACvB,KAAI,CAAC,SAAS,KAAK,QAAQ,MAAM,CAC/B,OAAM,IAAI,MACR,0BAA0B,QAAQ,MAAM,mCACzC;CAGH,MAAM,QACJ,OAAO,QAAQ,UAAU,YAAY,QAAQ,QAAQ,IACjD,UAAU,KAAK,MAAM,QAAQ,MAAM,KACnC;AAWN,UATiB,MADC,IAAI,sBAAsB,EAAE,CAAC,CACd,iBAAiB,QAAQ;EACxD,cAAc,QAAQ;EACtB,WAAW,oCAAoC,QAAQ,QAAQ;EAChE,CAAC,EAGU,QACN,QAAQ,EAAE,EAEJ,KAAK,SAAS;EACxB,QAAQ,SAAS,IAAI,OAAO,IAAI,EAAE;EAClC,cAAc,SAAS,IAAI,aAAa;EACzC,EAAE"}
+{"version":3,"file":"dataset.js","names":[],"sources":["../../src/evals/dataset.ts"],"sourcesContent":["import { SQLWarehouseConnector } from \"../connectors\";\nimport type { WorkspaceClient } from \"../workspace-client\";\n\n/**\n * One row of a managed evaluation dataset. `inputs` are the kwargs passed to the\n * agent for the turn; `expectations` (when present) is the row's ground truth /\n * guidelines. Mirrors the `{inputs, expectations}` shape of `mlflow.genai`\n * datasets and of the Unity Catalog table backing a managed eval dataset.\n */\nexport interface DatasetRow {\n  inputs: Record<string, unknown>;\n  expectations?: Record<string, unknown>;\n}\n\nexport interface ReadEvalDatasetOptions {\n  /** Fully-qualified UC table: `catalog.schema.table`. */\n  table: string;\n  /** SQL warehouse id to run the read against. */\n  warehouseId: string;\n  /** Optional row cap. */\n  limit?: number;\n}\n\n/**\n * Extract every user-message content, in order, from an MLflow\n * `{\"messages\":[{\"role\":\"user\",\"content\":\"...\"}]}` input. A dataset row can\n * carry a full multi-turn conversation; replaying these against one thread (one\n * `t.send` per returned string) lets the agent see the accumulating history.\n *\n * Only `role === \"user\"` turns are returned — any interleaved `assistant`/\n * `system` messages in the row are ignored, since the agent generates its own\n * responses; you never inject the dataset's assistant turns. A single-user-turn\n * row yields a one-element array (backward compatible); a row with no `messages`\n * yields `[]`.\n */\nexport function userTurns(input: Record<string, unknown>): string[] {\n  const messages = Array.isArray(input.messages) ? input.messages : [];\n  // Dataset rows are external data: skip null/non-object entries; coerce non-string content to \"\".\n  return messages\n    .filter(\n      (m): m is { role?: unknown; content?: unknown } =>\n        typeof m === \"object\" && m !== null,\n    )\n    .filter((m) => m.role === \"user\")\n    .map((m) => (typeof m.content === \"string\" ? m.content : \"\"));\n}\n\n/** A managed eval dataset is a UC table; only 3-level names are valid. */\nconst UC_TABLE = /^[A-Za-z0-9_]+\\.[A-Za-z0-9_]+\\.[A-Za-z0-9_]+$/;\n\n/** Coerce a cell (already JSON-parsed by the connector for JSON columns) to a record. */\nfunction toRecord(value: unknown): Record<string, unknown> | undefined {\n  if (value && typeof value === \"object\" && !Array.isArray(value)) {\n    return value as Record<string, unknown>;\n  }\n  return undefined;\n}\n\n/**\n * Read a Databricks managed evaluation dataset (a Unity Catalog table with\n * `inputs`/`expectations` columns) into rows, over the public SQL Statement\n * Execution API. Reuses {@link SQLWarehouseConnector} for submit/poll/transform\n * — its result transform already JSON-parses string columns into objects, so\n * `inputs`/`expectations` come back as records whether the table stores them as\n * JSON strings or structs.\n *\n * The Python `mlflow.genai.datasets` API needs a Spark session (no TS\n * equivalent), so we read the backing table directly.\n */\nexport async function readEvalDataset(\n  client: WorkspaceClient,\n  options: ReadEvalDatasetOptions,\n): Promise<DatasetRow[]> {\n  if (!UC_TABLE.test(options.table)) {\n    throw new Error(\n      `Invalid dataset table \"${options.table}\" — expected catalog.schema.table`,\n    );\n  }\n\n  const limit =\n    typeof options.limit === \"number\" && options.limit > 0\n      ? ` LIMIT ${Math.floor(options.limit)}`\n      : \"\";\n  const connector = new SQLWarehouseConnector({});\n  const response = await connector.executeStatement(client, {\n    warehouse_id: options.warehouseId,\n    statement: `SELECT inputs, expectations FROM ${options.table}${limit}`,\n  });\n\n  const rows =\n    (response.result as { data?: Array<Record<string, unknown>> } | undefined)\n      ?.data ?? [];\n\n  return rows.map((row) => ({\n    inputs: toRecord(row.inputs) ?? {},\n    expectations: toRecord(row.expectations),\n  }));\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAmCA,SAAgB,UAAU,OAA0C;AAGlE,SAFiB,MAAM,QAAQ,MAAM,SAAS,GAAG,MAAM,WAAW,EAAE,EAGjE,QACE,MACC,OAAO,MAAM,YAAY,MAAM,KAClC,CACA,QAAQ,MAAM,EAAE,SAAS,OAAO,CAChC,KAAK,MAAO,OAAO,EAAE,YAAY,WAAW,EAAE,UAAU,GAAI;;;AAIjE,MAAM,WAAW;;AAGjB,SAAS,SAAS,OAAqD;AACrE,KAAI,SAAS,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,MAAM,CAC7D,QAAO;;;;;;;;;;;;;AAgBX,eAAsB,gBACpB,QACA,SACuB;AACvB,KAAI,CAAC,SAAS,KAAK,QAAQ,MAAM,CAC/B,OAAM,IAAI,MACR,0BAA0B,QAAQ,MAAM,mCACzC;CAGH,MAAM,QACJ,OAAO,QAAQ,UAAU,YAAY,QAAQ,QAAQ,IACjD,UAAU,KAAK,MAAM,QAAQ,MAAM,KACnC;AAWN,UATiB,MADC,IAAI,sBAAsB,EAAE,CAAC,CACd,iBAAiB,QAAQ;EACxD,cAAc,QAAQ;EACtB,WAAW,oCAAoC,QAAQ,QAAQ;EAChE,CAAC,EAGU,QACN,QAAQ,EAAE,EAEJ,KAAK,SAAS;EACxB,QAAQ,SAAS,IAAI,OAAO,IAAI,EAAE;EAClC,cAAc,SAAS,IAAI,aAAa;EACzC,EAAE"}

package/dist/evals/define-eval.d.ts

@@ -1,4 +1,4 @@
-import { EvalDefinition } from "./types.js";
+import { EvalConfig, EvalDefinition } from "./types.js";
 
 //#region src/evals/define-eval.d.ts
 /**
@@ -21,6 +21,8 @@ import { EvalDefinition } from "./types.js";
  * ```
  */
 declare function defineEval(def: EvalDefinition): EvalDefinition;
+/** Define per-directory eval config. Default-export from `evals.config.ts`. */
+declare function defineEvalConfig(config: EvalConfig): EvalConfig;
 //#endregion
-export { defineEval };
+export { defineEval, defineEvalConfig };
 //# sourceMappingURL=define-eval.d.ts.map

package/dist/evals/define-eval.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"define-eval.d.ts","names":[],"sources":["../../src/evals/define-eval.ts"],"mappings":";;;;;AAqBA;;;;;;;;;;;;;;;;;iBAAgB,UAAA,CAAW,GAAA,EAAK,cAAA,GAAiB,cAAA"}
+{"version":3,"file":"define-eval.d.ts","names":[],"sources":["../../src/evals/define-eval.ts"],"mappings":";;;;;AAqBA;;;;;;;;;AAQA;;;;;;;;iBARgB,UAAA,CAAW,GAAA,EAAK,cAAA,GAAiB,cAAA;;iBAQjC,gBAAA,CAAiB,MAAA,EAAQ,UAAA,GAAa,UAAA"}

package/dist/evals/define-eval.js

@@ -22,7 +22,11 @@ function defineEval(def) {
 	if (typeof def.test !== "function") throw new Error("defineEval: `test` must be a function");
 	return def;
 }
+/** Define per-directory eval config. Default-export from `evals.config.ts`. */
+function defineEvalConfig(config) {
+	return config;
+}
 
 //#endregion
-export { defineEval };
+export { defineEval, defineEvalConfig };
 //# sourceMappingURL=define-eval.js.map

package/dist/evals/define-eval.js.map

@@ -1 +1 @@
-{"version":3,"file":"define-eval.js","names":[],"sources":["../../src/evals/define-eval.ts"],"sourcesContent":["import type { EvalDefinition } from \"./types\";\n\n/**\n * Define an agent eval. Default-export the result from a\n * `server/agents/<id>/evals/*.eval.ts` file.\n *\n * @example\n * ```ts\n * import { defineEval, includes } from \"@databricks/appkit/beta\";\n *\n * export default defineEval({\n *   description: \"Weather agent basic coverage\",\n *   async test(t) {\n *     await t.send(\"What's the weather in Brooklyn?\");\n *     t.succeeded();\n *     t.calledTool(\"get_weather\");\n *     t.check(t.reply, includes(\"Sunny\"));\n *   },\n * });\n * ```\n */\nexport function defineEval(def: EvalDefinition): EvalDefinition {\n  if (typeof def.test !== \"function\") {\n    throw new Error(\"defineEval: `test` must be a function\");\n  }\n  return def;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,WAAW,KAAqC;AAC9D,KAAI,OAAO,IAAI,SAAS,WACtB,OAAM,IAAI,MAAM,wCAAwC;AAE1D,QAAO"}
+{"version":3,"file":"define-eval.js","names":[],"sources":["../../src/evals/define-eval.ts"],"sourcesContent":["import type { EvalConfig, EvalDefinition } from \"./types\";\n\n/**\n * Define an agent eval. Default-export the result from a\n * `server/agents/<id>/evals/*.eval.ts` file.\n *\n * @example\n * ```ts\n * import { defineEval, includes } from \"@databricks/appkit/beta\";\n *\n * export default defineEval({\n *   description: \"Weather agent basic coverage\",\n *   async test(t) {\n *     await t.send(\"What's the weather in Brooklyn?\");\n *     t.succeeded();\n *     t.calledTool(\"get_weather\");\n *     t.check(t.reply, includes(\"Sunny\"));\n *   },\n * });\n * ```\n */\nexport function defineEval(def: EvalDefinition): EvalDefinition {\n  if (typeof def.test !== \"function\") {\n    throw new Error(\"defineEval: `test` must be a function\");\n  }\n  return def;\n}\n\n/** Define per-directory eval config. Default-export from `evals.config.ts`. */\nexport function defineEvalConfig(config: EvalConfig): EvalConfig {\n  return config;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,WAAW,KAAqC;AAC9D,KAAI,OAAO,IAAI,SAAS,WACtB,OAAM,IAAI,MAAM,wCAAwC;AAE1D,QAAO;;;AAIT,SAAgB,iBAAiB,QAAgC;AAC/D,QAAO"}

package/dist/evals/discover.d.ts

@@ -8,6 +8,13 @@ interface DiscoveredEval {
   /** The agent id (the `server/agents/<agent>` directory name). */
   agent: string;
 }
+/** A per-agent `evals.config.ts` found under `server/agents/<agent>/evals/`. */
+interface DiscoveredEvalConfig {
+  /** Absolute path to the `evals.config.ts` file. */
+  file: string;
+  /** The agent id whose evals this config applies to. */
+  agent: string;
+}
 /**
  * Discover evals under `<rootDir>/server/agents/<agent>/evals/` — co-located
  * with each agent's `agent.{md,ts}` (same folder-per-agent layout the agents
@@ -15,6 +22,13 @@ interface DiscoveredEval {
  * path relative to that evals dir with `.eval.ts` stripped. Sorted + stable.
  */
 declare function discoverEvalFiles(rootDir: string): DiscoveredEval[];
+/**
+ * Discover the per-agent `evals.config.ts` (from {@link defineEvalConfig}) at
+ * `<rootDir>/server/agents/<agent>/evals/evals.config.ts`. Config is per-agent:
+ * each agent's config applies only to that agent's evals. Agents without a
+ * config file are omitted. Returns a stable, sorted list.
+ */
+declare function discoverEvalConfigs(rootDir: string): DiscoveredEvalConfig[];
 //#endregion
-export { DiscoveredEval, discoverEvalFiles };
+export { DiscoveredEval, DiscoveredEvalConfig, discoverEvalConfigs, discoverEvalFiles };
 //# sourceMappingURL=discover.d.ts.map

package/dist/evals/discover.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"discover.d.ts","names":[],"sources":["../../src/evals/discover.ts"],"mappings":";;UAOiB,cAAA;EAAc;EAE7B,IAAA;EAF6B;EAI7B,EAAA;EAAA;EAEA,KAAA;AAAA;;AAoBF;;;;;iBAAgB,iBAAA,CAAkB,OAAA,WAAkB,cAAA"}
+{"version":3,"file":"discover.d.ts","names":[],"sources":["../../src/evals/discover.ts"],"mappings":";;UAOiB,cAAA;EAAc;EAE7B,IAAA;EAF6B;EAI7B,EAAA;EAAA;EAEA,KAAA;AAAA;;UAIe,oBAAA;EAAoB;EAEnC,IAAA;EAAA;EAEA,KAAA;AAAA;;;;;AAsDF;;iBAlCgB,iBAAA,CAAkB,OAAA,WAAkB,cAAA;;;;;;;iBAkCpC,mBAAA,CAAoB,OAAA,WAAkB,oBAAA"}

package/dist/evals/discover.js

@@ -1,7 +1,7 @@
 import { agentDirNames } from "../core/agent/agent-dirs.js";
 import { CODE_AGENTS_SOURCE_DIR } from "../core/agent/load-code-agents.js";
 import path from "node:path";
-import { readdirSync } from "node:fs";
+import { existsSync, readdirSync } from "node:fs";
 
 //#region src/evals/discover.ts
 /** Recursively collect `*.eval.ts` files under `dir`. Empty when `dir` is absent. */
@@ -43,7 +43,31 @@ function discoverEvalFiles(rootDir) {
 	}
 	return out.sort((a, b) => a.agent.localeCompare(b.agent) || a.id.localeCompare(b.id));
 }
+/**
+* Discover the per-agent `evals.config.ts` (from {@link defineEvalConfig}) at
+* `<rootDir>/server/agents/<agent>/evals/evals.config.ts`. Config is per-agent:
+* each agent's config applies only to that agent's evals. Agents without a
+* config file are omitted. Returns a stable, sorted list.
+*/
+function discoverEvalConfigs(rootDir) {
+	const agentsDir = path.join(rootDir, CODE_AGENTS_SOURCE_DIR);
+	const out = [];
+	let entries;
+	try {
+		entries = readdirSync(agentsDir, { withFileTypes: true });
+	} catch {
+		return out;
+	}
+	for (const agent of agentDirNames(entries)) {
+		const file = path.join(agentsDir, agent, "evals", "evals.config.ts");
+		if (existsSync(file)) out.push({
+			file,
+			agent
+		});
+	}
+	return out.sort((a, b) => a.agent.localeCompare(b.agent));
+}
 
 //#endregion
-export { discoverEvalFiles };
+export { discoverEvalConfigs, discoverEvalFiles };
 //# sourceMappingURL=discover.js.map