@databricks/appkit-ui 0.74.1 → 0.75.0

package/CLAUDE.md

@@ -110,6 +110,7 @@ npx @databricks/appkit docs <query>
 - [Function: evalGlyph()](./docs/api/appkit/Function.evalGlyph.md): Status glyph for a single eval result.
 - [Function: executeFromRegistry()](./docs/api/appkit/Function.executeFromRegistry.md): Validates tool-call arguments against the entry's schema and invokes its
 - [Function: extractServingEndpoints()](./docs/api/appkit/Function.extractServingEndpoints.md): Extract serving endpoint config from a server file by AST-parsing it.
+- [Function: findRootEvalConfig()](./docs/api/appkit/Function.findRootEvalConfig.md): Path to the root evals.config.ts (from defineEvalConfig) at
 - [Function: findServerFile()](./docs/api/appkit/Function.findServerFile.md): Find the server entry file by checking candidate paths in order.
 - [Function: fk()](./docs/api/appkit/Function.fk.md): Declare foreign-key to another column.
 - [Function: formatEvalDetail()](./docs/api/appkit/Function.formatEvalDetail.md): Indented detail lines for a failing eval (error + failing assertions).
@@ -140,6 +141,7 @@ npx @databricks/appkit docs <query>
 - [Function: jsonb()](./docs/api/appkit/Function.jsonb.md): Returns
 - [Function: loadAgentFromFile()](./docs/api/appkit/Function.loadAgentFromFile.md): Loads a single markdown agent file and resolves its frontmatter against
 - [Function: loadAgentsFromDir()](./docs/api/appkit/Function.loadAgentsFromDir.md): Scans a directory for one subdirectory per agent, each containing
+- [Function: loadRootEvalConfig()](./docs/api/appkit/Function.loadRootEvalConfig.md): Load the root evals.config.ts under rootDir (the project root), or return
 - [Function: matches()](./docs/api/appkit/Function.matches.md): Passes when the value matches pattern.
 - [Function: mcpServer()](./docs/api/appkit/Function.mcpServer.md): Factory for declaring a custom MCP server tool.
 - [Function: normalizeHost()](./docs/api/appkit/Function.normalizeHost.md): Ensure the host has a scheme (Databricks env often lacks https://).
@@ -189,6 +191,7 @@ npx @databricks/appkit docs <query>
 - [Interface: EvalResult](./docs/api/appkit/Interface.EvalResult.md): The outcome of running one eval.
 - [Interface: EvalRunSummary](./docs/api/appkit/Interface.EvalRunSummary.md): Properties
 - [Interface: EvalSummary](./docs/api/appkit/Interface.EvalSummary.md): Properties
+- [Interface: EvalWebServer](./docs/api/appkit/Interface.EvalWebServer.md): Auto-start config for the app under test, à la Playwright's webServer. When
 - [Interface: FilePolicyUser](./docs/api/appkit/Interface.FilePolicyUser.md): Minimal user identity passed to the policy function.
 - [Interface: FileResource](./docs/api/appkit/Interface.FileResource.md): Describes the file or directory being acted upon.
 - [Interface: FunctionTool](./docs/api/appkit/Interface.FunctionTool.md): Properties

package/dist/cli/commands/agent/eval.js

@@ -1,5 +1,6 @@
 import fs from "node:fs";
 import { Command, Option } from "commander";
+import { spawn } from "node:child_process";
 
 //#region src/cli/commands/agent/eval.ts
 /**
@@ -40,6 +41,69 @@ function positiveInt(raw) {
 	const n = raw ? Number.parseInt(raw, 10) : NaN;
 	return n > 0 ? n : void 0;
 }
+/** True when `url` answers with any HTTP response (a 404 still proves it's up). */
+async function isServerUp(url) {
+	try {
+		await fetch(url, { signal: AbortSignal.timeout(2e3) });
+		return true;
+	} catch {
+		return false;
+	}
+}
+/**
+* Start the app under test per a root config's `webServer`, à la Playwright:
+* reuse a server already answering at `url` (unless `reuseExisting: false`),
+* else spawn `command`, poll `url` until it answers or `timeoutMs` elapses.
+* Returns a `stop()` that kills the spawned process group (a no-op when the
+* server was reused). Under a machine reporter (`machine`) our logs and the
+* child's stdout are both routed to stderr, so the report stream on stdout
+* stays clean. Throws if the server never comes up.
+*/
+async function startWebServer(webServer, baseUrl, machine) {
+	const url = webServer.url ?? baseUrl;
+	const reuse = webServer.reuseExisting !== false;
+	const noop = { stop: () => {} };
+	if (reuse && await isServerUp(url)) {
+		console.error(`Reusing server already running at ${url}`);
+		return noop;
+	}
+	console.error(`Starting web server: ${webServer.command}`);
+	const child = spawn(webServer.command, {
+		shell: true,
+		detached: true,
+		stdio: machine ? [
+			"ignore",
+			2,
+			"inherit"
+		] : "inherit"
+	});
+	let exited = false;
+	child.on("exit", () => {
+		exited = true;
+	});
+	const stop = () => {
+		if (exited || child.pid === void 0) return;
+		try {
+			process.kill(-child.pid, "SIGTERM");
+		} catch {}
+	};
+	const deadline = Date.now() + (webServer.timeoutMs ?? 6e4);
+	try {
+		while (Date.now() < deadline) {
+			if (exited) throw new Error("web server exited before becoming ready");
+			if (await isServerUp(url)) {
+				console.error(`Web server ready at ${url}`);
+				return { stop };
+			}
+			await new Promise((r) => setTimeout(r, 500));
+		}
+	} catch (err) {
+		stop();
+		throw err;
+	}
+	stop();
+	throw new Error(`web server did not respond at ${url} within ${webServer.timeoutMs ?? 6e4}ms`);
+}
 /**
 * Native MLflow "Evaluation run" config — only when creds + an experiment are
 * all present (traces live in the app; the run + scores are driven from here).
@@ -103,6 +167,9 @@ function printMlflowOutcome(mlflow, info) {
 }
 async function runAgentEval(filter, opts) {
 	const runner = await loadRunner();
+	const rootDir = opts.root ?? process.cwd();
+	const config = await runner.loadRootEvalConfig(rootDir) ?? {};
+	const baseUrl = opts.url ?? config.baseUrl ?? "http://localhost:8000";
 	const credentials = {
 		profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE,
 		host: opts.databricksHost ?? process.env.DATABRICKS_HOST,
@@ -111,7 +178,8 @@ async function runAgentEval(filter, opts) {
 	const auth = await runner.resolveDatabricksAuth(credentials) ?? {};
 	const warehouseId = opts.warehouseId ?? process.env.DATABRICKS_WAREHOUSE_ID;
 	const workspaceClient = runner.resolveWorkspaceClient(credentials);
-	const timeoutMs = positiveInt(opts.timeout);
+	const concurrency = positiveInt(opts.concurrency) ?? config.maxConcurrency;
+	const timeoutMs = positiveInt(opts.timeout) ?? config.timeoutMs;
 	const retries = positiveInt(opts.retries);
 	let minPassRate;
 	try {
@@ -127,28 +195,32 @@ async function runAgentEval(filter, opts) {
 		if (machine) console.error(msg);
 		else console.log(msg);
 	};
+	let server;
 	let summary;
 	try {
+		server = config.webServer ? await startWebServer(config.webServer, baseUrl, machine) : void 0;
 		summary = await runner.runEvalsInDir({
-			rootDir: opts.root,
-			baseUrl: opts.url,
+			rootDir,
+			baseUrl,
 			filter,
 			tags: opts.tag,
 			strict: opts.strict,
 			headers: opts.header ? parseHeaders(opts.header) : void 0,
-			concurrency: opts.concurrency,
+			concurrency,
 			mlflow: resolveMlflow(opts, auth),
 			judge: resolveJudge(opts, auth),
 			workspaceClient,
 			warehouseId,
 			timeoutMs,
 			retries,
-			onEvent: makeProgressReporter(runner, opts.url, machine, info)
+			onEvent: makeProgressReporter(runner, baseUrl, machine, info)
 		});
 	} catch (err) {
 		console.error(`\nEval run failed: ${err instanceof Error ? err.message : String(err)}`);
 		process.exitCode = 1;
 		return;
+	} finally {
+		server?.stop();
 	}
 	info(`\n${runner.formatSummaryLine(summary.results)}`);
 	if (summary.mlflow) printMlflowOutcome(summary.mlflow, info);
@@ -173,7 +245,7 @@ async function runAgentEval(filter, opts) {
 		if (!ok) process.exitCode = 1;
 	} else if (!stats.allPassed) process.exitCode = 1;
 }
-const agentEvalCommand = new Command("eval").description("Run agent evals (server/agents/<id>/evals/*.eval.ts) against a running app").argument("[filter]", "Only run evals whose <agent>/<id> contains this substring (or an exact agent id)").option("--url <url>", "Base URL of the running app", "http://localhost:3000").option("--strict", "Fail on soft-assertion misses too", false).option("--concurrency <n>", "Max evals to run concurrently (default 4; keep at or below the app's max concurrent streams per user)", (v) => Number.parseInt(v, 10)).option("--root <dir>", "Project root containing server/agents/ (default: cwd)").option("--header <header...>", "Extra request header as 'Key: value' (repeatable)").option("--tag <tag...>", "Only run evals tagged with one of these tags (repeatable)").option("--profile <name>", "Databricks CLI profile to authenticate with via OAuth (default: DATABRICKS_CONFIG_PROFILE)").option("--databricks-host <host>", "Databricks host for writing MLflow assessments (default: DATABRICKS_HOST)").option("--databricks-token <token>", "Databricks token for writing MLflow assessments (default: DATABRICKS_TOKEN)").option("--experiment <id>", "MLflow experiment id for the evaluation run (default: MLFLOW_EXPERIMENT_ID)").option("--warehouse-id <id>", "SQL warehouse id for reading managed eval datasets and writing assessments to UC-backed experiments (default: DATABRICKS_WAREHOUSE_ID, or MLFLOW_TRACING_SQL_WAREHOUSE_ID for assessments)").option("--judge-model <endpoint>", "Databricks serving endpoint to use as the LLM judge for t.judge.* (default: APPKIT_JUDGE_MODEL)").option("--timeout <ms>", "Default per-eval timeout in ms (a per-eval timeoutMs overrides it)").option("--retries <n>", "Re-run an eval up to N times when it fails on an infra error (turn/timeout); assertion failures are not retried").option("--min-pass-rate <rate>", "Gate on aggregate pass rate (0..1) instead of requiring every eval to pass; exit 1 when below").addOption(new Option("--reporter <format>", "Report format: text (live console), json (dashboards), or junit (CI test reporters)").choices([
+const agentEvalCommand = new Command("eval").description("Run agent evals (server/agents/<id>/evals/*.eval.ts) against a running app").argument("[filter]", "Only run evals whose <agent>/<id> contains this substring (or an exact agent id)").option("--url <url>", "Base URL of the app to drive (default: evals.config.ts baseUrl, else http://localhost:8000)").option("--strict", "Fail on soft-assertion misses too", false).option("--concurrency <n>", "Max evals to run concurrently (default 4; keep at or below the app's max concurrent streams per user)").option("--root <dir>", "Project root containing server/agents/ (default: cwd)").option("--header <header...>", "Extra request header as 'Key: value' (repeatable)").option("--tag <tag...>", "Only run evals tagged with one of these tags (repeatable)").option("--profile <name>", "Databricks CLI profile to authenticate with via OAuth (default: DATABRICKS_CONFIG_PROFILE)").option("--databricks-host <host>", "Databricks host for writing MLflow assessments (default: DATABRICKS_HOST)").option("--databricks-token <token>", "Databricks token for writing MLflow assessments (default: DATABRICKS_TOKEN)").option("--experiment <id>", "MLflow experiment id for the evaluation run (default: MLFLOW_EXPERIMENT_ID)").option("--warehouse-id <id>", "SQL warehouse id for reading managed eval datasets and writing assessments to UC-backed experiments (default: DATABRICKS_WAREHOUSE_ID, or MLFLOW_TRACING_SQL_WAREHOUSE_ID for assessments)").option("--judge-model <endpoint>", "Databricks serving endpoint to use as the LLM judge for t.judge.* (default: APPKIT_JUDGE_MODEL)").option("--timeout <ms>", "Default per-eval timeout in ms (a per-eval timeoutMs overrides it)").option("--retries <n>", "Re-run an eval up to N times when it fails on an infra error (turn/timeout); assertion failures are not retried").option("--min-pass-rate <rate>", "Gate on aggregate pass rate (0..1) instead of requiring every eval to pass; exit 1 when below").addOption(new Option("--reporter <format>", "Report format: text (live console), json (dashboards), or junit (CI test reporters)").choices([
 	"text",
 	"json",
 	"junit"

package/dist/cli/commands/agent/eval.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval.js","names":[],"sources":["../../../../src/cli/commands/agent/eval.ts"],"sourcesContent":["import fs from \"node:fs\";\n\nimport { Command, Option } from \"commander\";\n\ninterface EvalRunSummary {\n  results: unknown[];\n  mlflow?: {\n    runId: string;\n    report: {\n      written: number;\n      skipped: number;\n      failures: Array<{ traceId: string; status?: number; error?: string }>;\n    };\n    finish: { finished: boolean; metricsError?: string; finishError?: string };\n  };\n}\n\ntype EvalProgress =\n  | { type: \"discovered\"; total: number }\n  | { type: \"run-created\"; runId: string }\n  | { type: \"start\"; id: string; index: number; total: number }\n  | { type: \"result\"; result: unknown; index: number; total: number };\n\n/** Subset of `@databricks/appkit/beta`'s eval runner used by this command. */\ninterface EvalRunner {\n  runEvalsInDir(opts: {\n    rootDir?: string;\n    baseUrl: string;\n    filter?: string;\n    tags?: string[];\n    strict?: boolean;\n    headers?: Record<string, string>;\n    concurrency?: number;\n    mlflow?: {\n      host: string;\n      token: string;\n      experimentId: string;\n      sqlWarehouseId?: string;\n    };\n    judge?: { host: string; token: string; model: string };\n    workspaceClient?: unknown;\n    warehouseId?: string;\n    timeoutMs?: number;\n    retries?: number;\n    onEvent?: (event: EvalProgress) => void;\n  }): Promise<EvalRunSummary>;\n  resolveDatabricksAuth(opts: {\n    profile?: string;\n    host?: string;\n    token?: string;\n  }): Promise<{ host: string; token: string } | undefined>;\n  resolveWorkspaceClient(opts: {\n    profile?: string;\n    host?: string;\n    token?: string;\n  }): unknown;\n  formatEvalHeadline(result: unknown): string;\n  evalGlyph(result: unknown): string;\n  formatEvalDetail(result: unknown): string[];\n  formatSummaryLine(results: unknown[]): string;\n  formatResultsJson(results: unknown[]): string;\n  formatResultsJUnit(results: unknown[]): string;\n  summarize(results: unknown[]): { allPassed: boolean; passRate: number };\n}\n\n/**\n * Loaded at runtime from the consuming project so this command (which ships in\n * `@databricks/shared`) doesn't take a build-time dependency on appkit. The\n * specifier is a variable so the type checker treats it as `any`.\n */\nasync function loadRunner(): Promise<EvalRunner> {\n  const spec = \"@databricks/appkit/beta\";\n  try {\n    return (await import(spec)) as unknown as EvalRunner;\n  } catch (err) {\n    throw new Error(\n      \"Could not load @databricks/appkit. Run `appkit agent eval` from a \" +\n        \"project with @databricks/appkit installed. \" +\n        `Cause: ${err instanceof Error ? err.message : String(err)}`,\n    );\n  }\n}\n\nfunction parseHeaders(values: string[]): Record<string, string> {\n  const headers: Record<string, string> = {};\n  for (const v of values) {\n    const i = v.indexOf(\":\");\n    if (i === -1) continue;\n    headers[v.slice(0, i).trim()] = v.slice(i + 1).trim();\n  }\n  return headers;\n}\n\n/**\n * Parse `--min-pass-rate`: a finite number in `[0, 1]`, or `undefined` when\n * unset. Throws on blank, out-of-range, or non-numeric input so a bad gate\n * value fails fast instead of silently disabling or inverting the CI gate.\n */\nexport function parsePassRate(raw: string | undefined): number | undefined {\n  if (raw === undefined) return undefined;\n  const n = Number(raw);\n  // Reject blank too: `Number(\"\")` is 0, which would silently disable the gate.\n  if (raw.trim() === \"\" || !Number.isFinite(n) || n < 0 || n > 1) {\n    throw new Error(\n      `Invalid --min-pass-rate \"${raw}\" — expected a number in [0, 1]`,\n    );\n  }\n  return n;\n}\n\n/** Parse a positive-integer CLI option; junk, zero, or negative → undefined. */\nfunction positiveInt(raw: string | undefined): number | undefined {\n  const n = raw ? Number.parseInt(raw, 10) : Number.NaN;\n  return n > 0 ? n : undefined;\n}\n\ninterface EvalOptions {\n  url: string;\n  strict?: boolean;\n  root?: string;\n  header?: string[];\n  tag?: string[];\n  profile?: string;\n  databricksHost?: string;\n  databricksToken?: string;\n  experiment?: string;\n  judgeModel?: string;\n  concurrency?: number;\n  warehouseId?: string;\n  timeout?: string;\n  retries?: string;\n  minPassRate?: string;\n  reporter?: \"text\" | \"json\" | \"junit\";\n  output?: string;\n}\n\n/** Resolved Databricks host + bearer (either field may be absent). */\ntype Auth = { host?: string; token?: string };\n\n/**\n * Native MLflow \"Evaluation run\" config — only when creds + an experiment are\n * all present (traces live in the app; the run + scores are driven from here).\n */\nfunction resolveMlflow(opts: EvalOptions, auth: Auth) {\n  const experimentId = opts.experiment ?? process.env.MLFLOW_EXPERIMENT_ID;\n  if (!(auth.host && auth.token && experimentId)) return undefined;\n  // UC-backed experiments need a SQL warehouse to write assessments to their\n  // V4 traces. Mirror mlflow's env var, and accept the common DATABRICKS one.\n  const sqlWarehouseId =\n    opts.warehouseId ??\n    process.env.MLFLOW_TRACING_SQL_WAREHOUSE_ID ??\n    process.env.DATABRICKS_WAREHOUSE_ID;\n  return {\n    host: auth.host,\n    token: auth.token,\n    experimentId,\n    ...(sqlWarehouseId ? { sqlWarehouseId } : {}),\n  };\n}\n\n/** LLM-as-judge config — reuses the Databricks creds + a judge serving endpoint. */\nfunction resolveJudge(opts: EvalOptions, auth: Auth) {\n  const model = opts.judgeModel ?? process.env.APPKIT_JUDGE_MODEL;\n  return model && auth.host && auth.token\n    ? { host: auth.host, token: auth.token, model }\n    : undefined;\n}\n\n/**\n * Progress reporter: stream each eval as it runs instead of going silent. In a\n * machine reporter (json/junit) the live per-eval streaming is suppressed and\n * banners go to stderr (via `info`), keeping stdout clean for the report.\n */\nfunction makeProgressReporter(\n  runner: EvalRunner,\n  url: string,\n  machine: boolean,\n  info: (msg: string) => void,\n): (event: EvalProgress) => void {\n  return (event) => {\n    switch (event.type) {\n      case \"discovered\":\n        info(\n          `Running ${event.total} eval${event.total === 1 ? \"\" : \"s\"} against ${url}\\n`,\n        );\n        break;\n      case \"run-created\":\n        info(`MLflow evaluation run: ${event.runId}\\n`);\n        break;\n      case \"result\": {\n        if (machine) break;\n        // One full line per completion — evals run concurrently, so a split\n        // \"start … glyph\" prefix would interleave into garbage.\n        console.log(\n          `[${event.index + 1}/${event.total}] ${runner.formatEvalHeadline(event.result)}`,\n        );\n        for (const line of runner.formatEvalDetail(event.result)) {\n          console.log(line);\n        }\n        break;\n      }\n    }\n  };\n}\n\nfunction formatFailureLine(f: {\n  traceId: string;\n  status?: number;\n  error?: string;\n}): string {\n  return `  ✗ trace ${f.traceId}: ${f.status ?? \"\"} ${f.error ?? \"\"}`.trim();\n}\n\n/**\n * Print the MLflow assessment/finish outcome after a run that created one. The\n * summary line goes through `info` (stderr under a machine reporter); per-trace\n * failures and finish errors always go to stderr.\n */\nfunction printMlflowOutcome(\n  mlflow: NonNullable<EvalRunSummary[\"mlflow\"]>,\n  info: (msg: string) => void,\n): void {\n  const { report, finish } = mlflow;\n  info(\n    `MLflow: ${report.written} assessment(s) written` +\n      (report.skipped ? `, ${report.skipped} skipped` : \"\") +\n      (report.failures.length ? `, ${report.failures.length} failed` : \"\"),\n  );\n  for (const f of report.failures) {\n    console.error(formatFailureLine(f));\n  }\n  if (finish.metricsError) {\n    console.error(`  ⚠ metrics not logged: ${finish.metricsError}`);\n  }\n  if (!finish.finished) {\n    console.error(\n      `  ✗ run left RUNNING — failed to finish: ${finish.finishError ?? \"unknown\"}`,\n    );\n  }\n}\n\nasync function runAgentEval(\n  filter: string | undefined,\n  opts: EvalOptions,\n): Promise<void> {\n  const runner = await loadRunner();\n\n  // Databricks credentials shared by auth resolution and the workspace client:\n  // an explicit flag/DATABRICKS_* env wins, else the SDK resolves from the CLI\n  // profile.\n  const credentials = {\n    profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE,\n    host: opts.databricksHost ?? process.env.DATABRICKS_HOST,\n    token: opts.databricksToken ?? process.env.DATABRICKS_TOKEN,\n  };\n\n  // Resolve Databricks host + bearer the AppKit-native way: an explicit\n  // host/token wins; otherwise the SDK mints an OAuth token from the CLI\n  // profile — so no hand-set PAT is required.\n  const auth: Auth = (await runner.resolveDatabricksAuth(credentials)) ?? {};\n\n  // Managed-dataset reads: a workspace client (same profile/host/token) + a SQL\n  // warehouse. Only needed by evals that declare `dataset`.\n  const warehouseId = opts.warehouseId ?? process.env.DATABRICKS_WAREHOUSE_ID;\n  const workspaceClient = runner.resolveWorkspaceClient(credentials);\n\n  // Runner-level default per-eval timeout (ms). A per-eval `timeoutMs` wins.\n  const timeoutMs = positiveInt(opts.timeout);\n\n  // Extra attempts for evals that fail on an infra error (turn/timeout). Junk\n  // or negative input falls back to no retries.\n  const retries = positiveInt(opts.retries);\n\n  // Validate up front so a bad gate value fails before the run, not after.\n  let minPassRate: number | undefined;\n  try {\n    minPassRate = parsePassRate(opts.minPassRate);\n  } catch (err) {\n    console.error(err instanceof Error ? err.message : String(err));\n    process.exitCode = 1;\n    return;\n  }\n\n  // In a machine reporter (json/junit), stdout is reserved for the report (it\n  // may be piped), so human-facing lines go to stderr and the per-eval live\n  // streaming is suppressed. Text mode keeps its current stdout behavior.\n  const reporter = opts.reporter ?? \"text\";\n  const machine = reporter !== \"text\";\n  const info = (msg: string): void => {\n    if (machine) console.error(msg);\n    else console.log(msg);\n  };\n\n  let summary: EvalRunSummary;\n  try {\n    summary = await runner.runEvalsInDir({\n      rootDir: opts.root,\n      baseUrl: opts.url,\n      filter,\n      tags: opts.tag,\n      strict: opts.strict,\n      headers: opts.header ? parseHeaders(opts.header) : undefined,\n      concurrency: opts.concurrency,\n      mlflow: resolveMlflow(opts, auth),\n      judge: resolveJudge(opts, auth),\n      workspaceClient,\n      warehouseId,\n      timeoutMs,\n      retries,\n      onEvent: makeProgressReporter(runner, opts.url, machine, info),\n    });\n  } catch (err) {\n    // Setup failures (e.g. a bad --experiment for the MLflow run) reject before\n    // any eval runs; surface a clean message + non-zero exit rather than an\n    // unhandled promise rejection with a raw stack.\n    console.error(\n      `\\nEval run failed: ${err instanceof Error ? err.message : String(err)}`,\n    );\n    process.exitCode = 1;\n    return;\n  }\n\n  // The final human summary always shows (stderr for machine reporters so it\n  // never pollutes the report on stdout/file).\n  info(`\\n${runner.formatSummaryLine(summary.results)}`);\n\n  if (summary.mlflow) {\n    printMlflowOutcome(summary.mlflow, info);\n  } else {\n    info(\n      \"\\nMLflow evaluation run skipped — pass --experiment (or set\" +\n        \" MLFLOW_EXPERIMENT_ID) plus --profile/--databricks-host to create one.\",\n    );\n  }\n\n  // Machine-readable report: build the string with a pure formatter, then emit\n  // it to --output <file> or stdout (kept clean of the human noise above).\n  if (machine) {\n    const report =\n      reporter === \"json\"\n        ? runner.formatResultsJson(summary.results)\n        : runner.formatResultsJUnit(summary.results);\n    if (opts.output) {\n      try {\n        fs.writeFileSync(opts.output, `${report}\\n`);\n      } catch (err) {\n        console.error(\n          `Failed to write ${reporter} report to ${opts.output}: ${\n            err instanceof Error ? err.message : String(err)\n          }`,\n        );\n        process.exitCode = 1;\n        return;\n      }\n      info(`Wrote ${reporter} report to ${opts.output}`);\n    } else {\n      process.stdout.write(`${report}\\n`);\n    }\n  }\n\n  const stats = runner.summarize(summary.results);\n  if (minPassRate !== undefined) {\n    // Threshold mode: gate on the aggregate pass rate rather than requiring\n    // every eval to pass.\n    const ok = stats.passRate >= minPassRate;\n    info(\n      `Pass rate ${(stats.passRate * 100).toFixed(0)}% (threshold ${(\n        minPassRate * 100\n      ).toFixed(0)}%) — ${ok ? \"OK\" : \"below threshold\"}`,\n    );\n    if (!ok) process.exitCode = 1;\n  } else if (!stats.allPassed) {\n    process.exitCode = 1;\n  }\n}\n\nexport const agentEvalCommand = new Command(\"eval\")\n  .description(\n    \"Run agent evals (server/agents/<id>/evals/*.eval.ts) against a running app\",\n  )\n  .argument(\n    \"[filter]\",\n    \"Only run evals whose <agent>/<id> contains this substring (or an exact agent id)\",\n  )\n  .option(\"--url <url>\", \"Base URL of the running app\", \"http://localhost:3000\")\n  .option(\"--strict\", \"Fail on soft-assertion misses too\", false)\n  .option(\n    \"--concurrency <n>\",\n    \"Max evals to run concurrently (default 4; keep at or below the app's max concurrent streams per user)\",\n    (v) => Number.parseInt(v, 10),\n  )\n  .option(\n    \"--root <dir>\",\n    \"Project root containing server/agents/ (default: cwd)\",\n  )\n  .option(\n    \"--header <header...>\",\n    \"Extra request header as 'Key: value' (repeatable)\",\n  )\n  .option(\n    \"--tag <tag...>\",\n    \"Only run evals tagged with one of these tags (repeatable)\",\n  )\n  .option(\n    \"--profile <name>\",\n    \"Databricks CLI profile to authenticate with via OAuth (default: DATABRICKS_CONFIG_PROFILE)\",\n  )\n  .option(\n    \"--databricks-host <host>\",\n    \"Databricks host for writing MLflow assessments (default: DATABRICKS_HOST)\",\n  )\n  .option(\n    \"--databricks-token <token>\",\n    \"Databricks token for writing MLflow assessments (default: DATABRICKS_TOKEN)\",\n  )\n  .option(\n    \"--experiment <id>\",\n    \"MLflow experiment id for the evaluation run (default: MLFLOW_EXPERIMENT_ID)\",\n  )\n  .option(\n    \"--warehouse-id <id>\",\n    \"SQL warehouse id for reading managed eval datasets and writing assessments to UC-backed experiments (default: DATABRICKS_WAREHOUSE_ID, or MLFLOW_TRACING_SQL_WAREHOUSE_ID for assessments)\",\n  )\n  .option(\n    \"--judge-model <endpoint>\",\n    \"Databricks serving endpoint to use as the LLM judge for t.judge.* (default: APPKIT_JUDGE_MODEL)\",\n  )\n  .option(\n    \"--timeout <ms>\",\n    \"Default per-eval timeout in ms (a per-eval timeoutMs overrides it)\",\n  )\n  .option(\n    \"--retries <n>\",\n    \"Re-run an eval up to N times when it fails on an infra error (turn/timeout); assertion failures are not retried\",\n  )\n  .option(\n    \"--min-pass-rate <rate>\",\n    \"Gate on aggregate pass rate (0..1) instead of requiring every eval to pass; exit 1 when below\",\n  )\n  .addOption(\n    new Option(\n      \"--reporter <format>\",\n      \"Report format: text (live console), json (dashboards), or junit (CI test reporters)\",\n    )\n      .choices([\"text\", \"json\", \"junit\"])\n      .default(\"text\"),\n  )\n  .option(\n    \"--output <file>\",\n    \"Write the json/junit report to this file instead of stdout (ignored for text)\",\n  )\n  .action(runAgentEval);\n"],"mappings":";;;;;;;;;AAsEA,eAAe,aAAkC;CAC/C,MAAM,OAAO;AACb,KAAI;AACF,SAAQ,MAAM,OAAO;UACd,KAAK;AACZ,QAAM,IAAI,MACR,yHAEY,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GAC7D;;;AAIL,SAAS,aAAa,QAA0C;CAC9D,MAAM,UAAkC,EAAE;AAC1C,MAAK,MAAM,KAAK,QAAQ;EACtB,MAAM,IAAI,EAAE,QAAQ,IAAI;AACxB,MAAI,MAAM,GAAI;AACd,UAAQ,EAAE,MAAM,GAAG,EAAE,CAAC,MAAM,IAAI,EAAE,MAAM,IAAI,EAAE,CAAC,MAAM;;AAEvD,QAAO;;;;;;;AAQT,SAAgB,cAAc,KAA6C;AACzE,KAAI,QAAQ,OAAW,QAAO;CAC9B,MAAM,IAAI,OAAO,IAAI;AAErB,KAAI,IAAI,MAAM,KAAK,MAAM,CAAC,OAAO,SAAS,EAAE,IAAI,IAAI,KAAK,IAAI,EAC3D,OAAM,IAAI,MACR,4BAA4B,IAAI,iCACjC;AAEH,QAAO;;;AAIT,SAAS,YAAY,KAA6C;CAChE,MAAM,IAAI,MAAM,OAAO,SAAS,KAAK,GAAG,GAAG;AAC3C,QAAO,IAAI,IAAI,IAAI;;;;;;AA8BrB,SAAS,cAAc,MAAmB,MAAY;CACpD,MAAM,eAAe,KAAK,cAAc,QAAQ,IAAI;AACpD,KAAI,EAAE,KAAK,QAAQ,KAAK,SAAS,cAAe,QAAO;CAGvD,MAAM,iBACJ,KAAK,eACL,QAAQ,IAAI,mCACZ,QAAQ,IAAI;AACd,QAAO;EACL,MAAM,KAAK;EACX,OAAO,KAAK;EACZ;EACA,GAAI,iBAAiB,EAAE,gBAAgB,GAAG,EAAE;EAC7C;;;AAIH,SAAS,aAAa,MAAmB,MAAY;CACnD,MAAM,QAAQ,KAAK,cAAc,QAAQ,IAAI;AAC7C,QAAO,SAAS,KAAK,QAAQ,KAAK,QAC9B;EAAE,MAAM,KAAK;EAAM,OAAO,KAAK;EAAO;EAAO,GAC7C;;;;;;;AAQN,SAAS,qBACP,QACA,KACA,SACA,MAC+B;AAC/B,SAAQ,UAAU;AAChB,UAAQ,MAAM,MAAd;GACE,KAAK;AACH,SACE,WAAW,MAAM,MAAM,OAAO,MAAM,UAAU,IAAI,KAAK,IAAI,WAAW,IAAI,IAC3E;AACD;GACF,KAAK;AACH,SAAK,0BAA0B,MAAM,MAAM,IAAI;AAC/C;GACF,KAAK;AACH,QAAI,QAAS;AAGb,YAAQ,IACN,IAAI,MAAM,QAAQ,EAAE,GAAG,MAAM,MAAM,IAAI,OAAO,mBAAmB,MAAM,OAAO,GAC/E;AACD,SAAK,MAAM,QAAQ,OAAO,iBAAiB,MAAM,OAAO,CACtD,SAAQ,IAAI,KAAK;AAEnB;;;;AAMR,SAAS,kBAAkB,GAIhB;AACT,QAAO,aAAa,EAAE,QAAQ,IAAI,EAAE,UAAU,GAAG,GAAG,EAAE,SAAS,KAAK,MAAM;;;;;;;AAQ5E,SAAS,mBACP,QACA,MACM;CACN,MAAM,EAAE,QAAQ,WAAW;AAC3B,MACE,WAAW,OAAO,QAAQ,2BACvB,OAAO,UAAU,KAAK,OAAO,QAAQ,YAAY,OACjD,OAAO,SAAS,SAAS,KAAK,OAAO,SAAS,OAAO,WAAW,IACpE;AACD,MAAK,MAAM,KAAK,OAAO,SACrB,SAAQ,MAAM,kBAAkB,EAAE,CAAC;AAErC,KAAI,OAAO,aACT,SAAQ,MAAM,2BAA2B,OAAO,eAAe;AAEjE,KAAI,CAAC,OAAO,SACV,SAAQ,MACN,4CAA4C,OAAO,eAAe,YACnE;;AAIL,eAAe,aACb,QACA,MACe;CACf,MAAM,SAAS,MAAM,YAAY;CAKjC,MAAM,cAAc;EAClB,SAAS,KAAK,WAAW,QAAQ,IAAI;EACrC,MAAM,KAAK,kBAAkB,QAAQ,IAAI;EACzC,OAAO,KAAK,mBAAmB,QAAQ,IAAI;EAC5C;CAKD,MAAM,OAAc,MAAM,OAAO,sBAAsB,YAAY,IAAK,EAAE;CAI1E,MAAM,cAAc,KAAK,eAAe,QAAQ,IAAI;CACpD,MAAM,kBAAkB,OAAO,uBAAuB,YAAY;CAGlE,MAAM,YAAY,YAAY,KAAK,QAAQ;CAI3C,MAAM,UAAU,YAAY,KAAK,QAAQ;CAGzC,IAAI;AACJ,KAAI;AACF,gBAAc,cAAc,KAAK,YAAY;UACtC,KAAK;AACZ,UAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,CAAC;AAC/D,UAAQ,WAAW;AACnB;;CAMF,MAAM,WAAW,KAAK,YAAY;CAClC,MAAM,UAAU,aAAa;CAC7B,MAAM,QAAQ,QAAsB;AAClC,MAAI,QAAS,SAAQ,MAAM,IAAI;MAC1B,SAAQ,IAAI,IAAI;;CAGvB,IAAI;AACJ,KAAI;AACF,YAAU,MAAM,OAAO,cAAc;GACnC,SAAS,KAAK;GACd,SAAS,KAAK;GACd;GACA,MAAM,KAAK;GACX,QAAQ,KAAK;GACb,SAAS,KAAK,SAAS,aAAa,KAAK,OAAO,GAAG;GACnD,aAAa,KAAK;GAClB,QAAQ,cAAc,MAAM,KAAK;GACjC,OAAO,aAAa,MAAM,KAAK;GAC/B;GACA;GACA;GACA;GACA,SAAS,qBAAqB,QAAQ,KAAK,KAAK,SAAS,KAAK;GAC/D,CAAC;UACK,KAAK;AAIZ,UAAQ,MACN,sBAAsB,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GACvE;AACD,UAAQ,WAAW;AACnB;;AAKF,MAAK,KAAK,OAAO,kBAAkB,QAAQ,QAAQ,GAAG;AAEtD,KAAI,QAAQ,OACV,oBAAmB,QAAQ,QAAQ,KAAK;KAExC,MACE,oIAED;AAKH,KAAI,SAAS;EACX,MAAM,SACJ,aAAa,SACT,OAAO,kBAAkB,QAAQ,QAAQ,GACzC,OAAO,mBAAmB,QAAQ,QAAQ;AAChD,MAAI,KAAK,QAAQ;AACf,OAAI;AACF,OAAG,cAAc,KAAK,QAAQ,GAAG,OAAO,IAAI;YACrC,KAAK;AACZ,YAAQ,MACN,mBAAmB,SAAS,aAAa,KAAK,OAAO,IACnD,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GAEnD;AACD,YAAQ,WAAW;AACnB;;AAEF,QAAK,SAAS,SAAS,aAAa,KAAK,SAAS;QAElD,SAAQ,OAAO,MAAM,GAAG,OAAO,IAAI;;CAIvC,MAAM,QAAQ,OAAO,UAAU,QAAQ,QAAQ;AAC/C,KAAI,gBAAgB,QAAW;EAG7B,MAAM,KAAK,MAAM,YAAY;AAC7B,OACE,cAAc,MAAM,WAAW,KAAK,QAAQ,EAAE,CAAC,gBAC7C,cAAc,KACd,QAAQ,EAAE,CAAC,OAAO,KAAK,OAAO,oBACjC;AACD,MAAI,CAAC,GAAI,SAAQ,WAAW;YACnB,CAAC,MAAM,UAChB,SAAQ,WAAW;;AAIvB,MAAa,mBAAmB,IAAI,QAAQ,OAAO,CAChD,YACC,6EACD,CACA,SACC,YACA,mFACD,CACA,OAAO,eAAe,+BAA+B,wBAAwB,CAC7E,OAAO,YAAY,qCAAqC,MAAM,CAC9D,OACC,qBACA,0GACC,MAAM,OAAO,SAAS,GAAG,GAAG,CAC9B,CACA,OACC,gBACA,wDACD,CACA,OACC,wBACA,oDACD,CACA,OACC,kBACA,4DACD,CACA,OACC,oBACA,6FACD,CACA,OACC,4BACA,4EACD,CACA,OACC,8BACA,8EACD,CACA,OACC,qBACA,8EACD,CACA,OACC,uBACA,6LACD,CACA,OACC,4BACA,kGACD,CACA,OACC,kBACA,qEACD,CACA,OACC,iBACA,kHACD,CACA,OACC,0BACA,gGACD,CACA,UACC,IAAI,OACF,uBACA,sFACD,CACE,QAAQ;CAAC;CAAQ;CAAQ;CAAQ,CAAC,CAClC,QAAQ,OAAO,CACnB,CACA,OACC,mBACA,gFACD,CACA,OAAO,aAAa"}
+{"version":3,"file":"eval.js","names":[],"sources":["../../../../src/cli/commands/agent/eval.ts"],"sourcesContent":["import { type ChildProcess, spawn } from \"node:child_process\";\nimport fs from \"node:fs\";\n\nimport { Command, Option } from \"commander\";\n\ninterface EvalRunSummary {\n  results: unknown[];\n  mlflow?: {\n    runId: string;\n    report: {\n      written: number;\n      skipped: number;\n      failures: Array<{ traceId: string; status?: number; error?: string }>;\n    };\n    finish: { finished: boolean; metricsError?: string; finishError?: string };\n  };\n}\n\ntype EvalProgress =\n  | { type: \"discovered\"; total: number }\n  | { type: \"run-created\"; runId: string }\n  | { type: \"start\"; id: string; index: number; total: number }\n  | { type: \"result\"; result: unknown; index: number; total: number };\n\n/** Subset of `@databricks/appkit/beta`'s eval runner used by this command. */\ninterface EvalRunner {\n  runEvalsInDir(opts: {\n    rootDir?: string;\n    baseUrl: string;\n    filter?: string;\n    tags?: string[];\n    strict?: boolean;\n    headers?: Record<string, string>;\n    concurrency?: number;\n    mlflow?: {\n      host: string;\n      token: string;\n      experimentId: string;\n      sqlWarehouseId?: string;\n    };\n    judge?: { host: string; token: string; model: string };\n    workspaceClient?: unknown;\n    warehouseId?: string;\n    timeoutMs?: number;\n    retries?: number;\n    onEvent?: (event: EvalProgress) => void;\n  }): Promise<EvalRunSummary>;\n  resolveDatabricksAuth(opts: {\n    profile?: string;\n    host?: string;\n    token?: string;\n  }): Promise<{ host: string; token: string } | undefined>;\n  resolveWorkspaceClient(opts: {\n    profile?: string;\n    host?: string;\n    token?: string;\n  }): unknown;\n  formatEvalHeadline(result: unknown): string;\n  loadRootEvalConfig(rootDir: string): Promise<EvalConfig | undefined>;\n  evalGlyph(result: unknown): string;\n  formatEvalDetail(result: unknown): string[];\n  formatSummaryLine(results: unknown[]): string;\n  formatResultsJson(results: unknown[]): string;\n  formatResultsJUnit(results: unknown[]): string;\n  summarize(results: unknown[]): { allPassed: boolean; passRate: number };\n}\n\n/** Subset of `@databricks/appkit/beta`'s `EvalConfig` the CLI reads. */\ninterface EvalConfig {\n  maxConcurrency?: number;\n  timeoutMs?: number;\n  baseUrl?: string;\n  webServer?: {\n    command: string;\n    url?: string;\n    timeoutMs?: number;\n    reuseExisting?: boolean;\n  };\n}\n\n/**\n * Loaded at runtime from the consuming project so this command (which ships in\n * `@databricks/shared`) doesn't take a build-time dependency on appkit. The\n * specifier is a variable so the type checker treats it as `any`.\n */\nasync function loadRunner(): Promise<EvalRunner> {\n  const spec = \"@databricks/appkit/beta\";\n  try {\n    return (await import(spec)) as unknown as EvalRunner;\n  } catch (err) {\n    throw new Error(\n      \"Could not load @databricks/appkit. Run `appkit agent eval` from a \" +\n        \"project with @databricks/appkit installed. \" +\n        `Cause: ${err instanceof Error ? err.message : String(err)}`,\n    );\n  }\n}\n\nfunction parseHeaders(values: string[]): Record<string, string> {\n  const headers: Record<string, string> = {};\n  for (const v of values) {\n    const i = v.indexOf(\":\");\n    if (i === -1) continue;\n    headers[v.slice(0, i).trim()] = v.slice(i + 1).trim();\n  }\n  return headers;\n}\n\n/**\n * Parse `--min-pass-rate`: a finite number in `[0, 1]`, or `undefined` when\n * unset. Throws on blank, out-of-range, or non-numeric input so a bad gate\n * value fails fast instead of silently disabling or inverting the CI gate.\n */\nexport function parsePassRate(raw: string | undefined): number | undefined {\n  if (raw === undefined) return undefined;\n  const n = Number(raw);\n  // Reject blank too: `Number(\"\")` is 0, which would silently disable the gate.\n  if (raw.trim() === \"\" || !Number.isFinite(n) || n < 0 || n > 1) {\n    throw new Error(\n      `Invalid --min-pass-rate \"${raw}\" — expected a number in [0, 1]`,\n    );\n  }\n  return n;\n}\n\n/** Parse a positive-integer CLI option; junk, zero, or negative → undefined. */\nfunction positiveInt(raw: string | undefined): number | undefined {\n  const n = raw ? Number.parseInt(raw, 10) : Number.NaN;\n  return n > 0 ? n : undefined;\n}\n\n/** True when `url` answers with any HTTP response (a 404 still proves it's up). */\nasync function isServerUp(url: string): Promise<boolean> {\n  try {\n    await fetch(url, { signal: AbortSignal.timeout(2000) });\n    return true;\n  } catch {\n    return false;\n  }\n}\n\n/**\n * Start the app under test per a root config's `webServer`, à la Playwright:\n * reuse a server already answering at `url` (unless `reuseExisting: false`),\n * else spawn `command`, poll `url` until it answers or `timeoutMs` elapses.\n * Returns a `stop()` that kills the spawned process group (a no-op when the\n * server was reused). Under a machine reporter (`machine`) our logs and the\n * child's stdout are both routed to stderr, so the report stream on stdout\n * stays clean. Throws if the server never comes up.\n */\nasync function startWebServer(\n  webServer: NonNullable<EvalConfig[\"webServer\"]>,\n  baseUrl: string,\n  machine: boolean,\n): Promise<{ stop: () => void }> {\n  const url = webServer.url ?? baseUrl;\n  const reuse = webServer.reuseExisting !== false;\n  const noop = { stop: () => {} };\n\n  if (reuse && (await isServerUp(url))) {\n    console.error(`Reusing server already running at ${url}`);\n    return noop;\n  }\n\n  console.error(`Starting web server: ${webServer.command}`);\n  // `detached` + a negative-PID kill lets us tear down the whole process group\n  // (dev servers spawn child processes). Under a machine reporter the child's\n  // stdout is sent to our stderr (fd 2) so it can't corrupt the report we later\n  // write to stdout; in text mode it inherits so the user sees build output.\n  const child: ChildProcess = spawn(webServer.command, {\n    shell: true,\n    detached: true,\n    stdio: machine ? [\"ignore\", 2, \"inherit\"] : \"inherit\",\n  });\n\n  let exited = false;\n  child.on(\"exit\", () => {\n    exited = true;\n  });\n\n  const stop = (): void => {\n    if (exited || child.pid === undefined) return;\n    try {\n      process.kill(-child.pid, \"SIGTERM\");\n    } catch {\n      // Group already gone, or never became a leader — best-effort.\n    }\n  };\n\n  const deadline = Date.now() + (webServer.timeoutMs ?? 60_000);\n  try {\n    while (Date.now() < deadline) {\n      if (exited) throw new Error(\"web server exited before becoming ready\");\n      if (await isServerUp(url)) {\n        console.error(`Web server ready at ${url}`);\n        return { stop };\n      }\n      await new Promise((r) => setTimeout(r, 500));\n    }\n  } catch (err) {\n    stop();\n    throw err;\n  }\n  stop();\n  throw new Error(\n    `web server did not respond at ${url} within ${webServer.timeoutMs ?? 60_000}ms`,\n  );\n}\n\ninterface EvalOptions {\n  url?: string;\n  strict?: boolean;\n  root?: string;\n  header?: string[];\n  tag?: string[];\n  profile?: string;\n  databricksHost?: string;\n  databricksToken?: string;\n  experiment?: string;\n  judgeModel?: string;\n  concurrency?: string;\n  warehouseId?: string;\n  timeout?: string;\n  retries?: string;\n  minPassRate?: string;\n  reporter?: \"text\" | \"json\" | \"junit\";\n  output?: string;\n}\n\n/** Resolved Databricks host + bearer (either field may be absent). */\ntype Auth = { host?: string; token?: string };\n\n/**\n * Native MLflow \"Evaluation run\" config — only when creds + an experiment are\n * all present (traces live in the app; the run + scores are driven from here).\n */\nfunction resolveMlflow(opts: EvalOptions, auth: Auth) {\n  const experimentId = opts.experiment ?? process.env.MLFLOW_EXPERIMENT_ID;\n  if (!(auth.host && auth.token && experimentId)) return undefined;\n  // UC-backed experiments need a SQL warehouse to write assessments to their\n  // V4 traces. Mirror mlflow's env var, and accept the common DATABRICKS one.\n  const sqlWarehouseId =\n    opts.warehouseId ??\n    process.env.MLFLOW_TRACING_SQL_WAREHOUSE_ID ??\n    process.env.DATABRICKS_WAREHOUSE_ID;\n  return {\n    host: auth.host,\n    token: auth.token,\n    experimentId,\n    ...(sqlWarehouseId ? { sqlWarehouseId } : {}),\n  };\n}\n\n/** LLM-as-judge config — reuses the Databricks creds + a judge serving endpoint. */\nfunction resolveJudge(opts: EvalOptions, auth: Auth) {\n  const model = opts.judgeModel ?? process.env.APPKIT_JUDGE_MODEL;\n  return model && auth.host && auth.token\n    ? { host: auth.host, token: auth.token, model }\n    : undefined;\n}\n\n/**\n * Progress reporter: stream each eval as it runs instead of going silent. In a\n * machine reporter (json/junit) the live per-eval streaming is suppressed and\n * banners go to stderr (via `info`), keeping stdout clean for the report.\n */\nfunction makeProgressReporter(\n  runner: EvalRunner,\n  url: string,\n  machine: boolean,\n  info: (msg: string) => void,\n): (event: EvalProgress) => void {\n  return (event) => {\n    switch (event.type) {\n      case \"discovered\":\n        info(\n          `Running ${event.total} eval${event.total === 1 ? \"\" : \"s\"} against ${url}\\n`,\n        );\n        break;\n      case \"run-created\":\n        info(`MLflow evaluation run: ${event.runId}\\n`);\n        break;\n      case \"result\": {\n        if (machine) break;\n        // One full line per completion — evals run concurrently, so a split\n        // \"start … glyph\" prefix would interleave into garbage.\n        console.log(\n          `[${event.index + 1}/${event.total}] ${runner.formatEvalHeadline(event.result)}`,\n        );\n        for (const line of runner.formatEvalDetail(event.result)) {\n          console.log(line);\n        }\n        break;\n      }\n    }\n  };\n}\n\nfunction formatFailureLine(f: {\n  traceId: string;\n  status?: number;\n  error?: string;\n}): string {\n  return `  ✗ trace ${f.traceId}: ${f.status ?? \"\"} ${f.error ?? \"\"}`.trim();\n}\n\n/**\n * Print the MLflow assessment/finish outcome after a run that created one. The\n * summary line goes through `info` (stderr under a machine reporter); per-trace\n * failures and finish errors always go to stderr.\n */\nfunction printMlflowOutcome(\n  mlflow: NonNullable<EvalRunSummary[\"mlflow\"]>,\n  info: (msg: string) => void,\n): void {\n  const { report, finish } = mlflow;\n  info(\n    `MLflow: ${report.written} assessment(s) written` +\n      (report.skipped ? `, ${report.skipped} skipped` : \"\") +\n      (report.failures.length ? `, ${report.failures.length} failed` : \"\"),\n  );\n  for (const f of report.failures) {\n    console.error(formatFailureLine(f));\n  }\n  if (finish.metricsError) {\n    console.error(`  ⚠ metrics not logged: ${finish.metricsError}`);\n  }\n  if (!finish.finished) {\n    console.error(\n      `  ✗ run left RUNNING — failed to finish: ${finish.finishError ?? \"unknown\"}`,\n    );\n  }\n}\n\nasync function runAgentEval(\n  filter: string | undefined,\n  opts: EvalOptions,\n): Promise<void> {\n  const runner = await loadRunner();\n\n  // Root `evals.config.ts` (project root) carries run-wide settings — baseUrl,\n  // webServer, and defaults for concurrency/timeout. A CLI flag always wins.\n  const rootDir = opts.root ?? process.cwd();\n  const config = (await runner.loadRootEvalConfig(rootDir)) ?? {};\n\n  // Base URL: --url flag > config.baseUrl > built-in default. `--url` has no\n  // commander default so an unset flag is undefined and lets config win.\n  const baseUrl = opts.url ?? config.baseUrl ?? \"http://localhost:8000\";\n\n  // Databricks credentials shared by auth resolution and the workspace client:\n  // an explicit flag/DATABRICKS_* env wins, else the SDK resolves from the CLI\n  // profile.\n  const credentials = {\n    profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE,\n    host: opts.databricksHost ?? process.env.DATABRICKS_HOST,\n    token: opts.databricksToken ?? process.env.DATABRICKS_TOKEN,\n  };\n\n  // Resolve Databricks host + bearer the AppKit-native way: an explicit\n  // host/token wins; otherwise the SDK mints an OAuth token from the CLI\n  // profile — so no hand-set PAT is required.\n  const auth: Auth = (await runner.resolveDatabricksAuth(credentials)) ?? {};\n\n  // Managed-dataset reads: a workspace client (same profile/host/token) + a SQL\n  // warehouse. Only needed by evals that declare `dataset`.\n  const warehouseId = opts.warehouseId ?? process.env.DATABRICKS_WAREHOUSE_ID;\n  const workspaceClient = runner.resolveWorkspaceClient(credentials);\n\n  // Max concurrency: the `--concurrency` flag wins over the root config's value\n  // (junk/zero/negative → undefined, so it falls back); else the runner default.\n  const concurrency = positiveInt(opts.concurrency) ?? config.maxConcurrency;\n\n  // Runner-level default per-eval timeout (ms). --timeout flag wins over the\n  // root config; a per-eval `timeoutMs` overrides both (applied in the runner).\n  const timeoutMs = positiveInt(opts.timeout) ?? config.timeoutMs;\n\n  // Extra attempts for evals that fail on an infra error (turn/timeout). Junk\n  // or negative input falls back to no retries.\n  const retries = positiveInt(opts.retries);\n\n  // Validate up front so a bad gate value fails before the run, not after.\n  let minPassRate: number | undefined;\n  try {\n    minPassRate = parsePassRate(opts.minPassRate);\n  } catch (err) {\n    console.error(err instanceof Error ? err.message : String(err));\n    process.exitCode = 1;\n    return;\n  }\n\n  // In a machine reporter (json/junit), stdout is reserved for the report (it\n  // may be piped), so human-facing lines go to stderr and the per-eval live\n  // streaming is suppressed. Text mode keeps its current stdout behavior.\n  const reporter = opts.reporter ?? \"text\";\n  const machine = reporter !== \"text\";\n  const info = (msg: string): void => {\n    if (machine) console.error(msg);\n    else console.log(msg);\n  };\n\n  // Boot the app under test if the root config declares a webServer (reuses an\n  // already-running server unless told otherwise); always torn down after. The\n  // boot runs inside the try so a server that never comes up takes the clean\n  // error path below instead of escaping as an unhandled rejection.\n  let server: { stop: () => void } | undefined;\n  let summary: EvalRunSummary;\n  try {\n    server = config.webServer\n      ? await startWebServer(config.webServer, baseUrl, machine)\n      : undefined;\n    summary = await runner.runEvalsInDir({\n      rootDir,\n      baseUrl,\n      filter,\n      tags: opts.tag,\n      strict: opts.strict,\n      headers: opts.header ? parseHeaders(opts.header) : undefined,\n      concurrency,\n      mlflow: resolveMlflow(opts, auth),\n      judge: resolveJudge(opts, auth),\n      workspaceClient,\n      warehouseId,\n      timeoutMs,\n      retries,\n      onEvent: makeProgressReporter(runner, baseUrl, machine, info),\n    });\n  } catch (err) {\n    // Setup failures — a web server that never comes up, or e.g. a bad\n    // --experiment for the MLflow run — reject before any eval runs; surface a\n    // clean message + non-zero exit rather than an unhandled promise rejection\n    // with a raw stack.\n    console.error(\n      `\\nEval run failed: ${err instanceof Error ? err.message : String(err)}`,\n    );\n    process.exitCode = 1;\n    return;\n  } finally {\n    server?.stop();\n  }\n\n  // The final human summary always shows (stderr for machine reporters so it\n  // never pollutes the report on stdout/file).\n  info(`\\n${runner.formatSummaryLine(summary.results)}`);\n\n  if (summary.mlflow) {\n    printMlflowOutcome(summary.mlflow, info);\n  } else {\n    info(\n      \"\\nMLflow evaluation run skipped — pass --experiment (or set\" +\n        \" MLFLOW_EXPERIMENT_ID) plus --profile/--databricks-host to create one.\",\n    );\n  }\n\n  // Machine-readable report: build the string with a pure formatter, then emit\n  // it to --output <file> or stdout (kept clean of the human noise above).\n  if (machine) {\n    const report =\n      reporter === \"json\"\n        ? runner.formatResultsJson(summary.results)\n        : runner.formatResultsJUnit(summary.results);\n    if (opts.output) {\n      try {\n        fs.writeFileSync(opts.output, `${report}\\n`);\n      } catch (err) {\n        console.error(\n          `Failed to write ${reporter} report to ${opts.output}: ${\n            err instanceof Error ? err.message : String(err)\n          }`,\n        );\n        process.exitCode = 1;\n        return;\n      }\n      info(`Wrote ${reporter} report to ${opts.output}`);\n    } else {\n      process.stdout.write(`${report}\\n`);\n    }\n  }\n\n  const stats = runner.summarize(summary.results);\n  if (minPassRate !== undefined) {\n    // Threshold mode: gate on the aggregate pass rate rather than requiring\n    // every eval to pass.\n    const ok = stats.passRate >= minPassRate;\n    info(\n      `Pass rate ${(stats.passRate * 100).toFixed(0)}% (threshold ${(\n        minPassRate * 100\n      ).toFixed(0)}%) — ${ok ? \"OK\" : \"below threshold\"}`,\n    );\n    if (!ok) process.exitCode = 1;\n  } else if (!stats.allPassed) {\n    process.exitCode = 1;\n  }\n}\n\nexport const agentEvalCommand = new Command(\"eval\")\n  .description(\n    \"Run agent evals (server/agents/<id>/evals/*.eval.ts) against a running app\",\n  )\n  .argument(\n    \"[filter]\",\n    \"Only run evals whose <agent>/<id> contains this substring (or an exact agent id)\",\n  )\n  .option(\n    \"--url <url>\",\n    \"Base URL of the app to drive (default: evals.config.ts baseUrl, else http://localhost:8000)\",\n  )\n  .option(\"--strict\", \"Fail on soft-assertion misses too\", false)\n  .option(\n    \"--concurrency <n>\",\n    \"Max evals to run concurrently (default 4; keep at or below the app's max concurrent streams per user)\",\n  )\n  .option(\n    \"--root <dir>\",\n    \"Project root containing server/agents/ (default: cwd)\",\n  )\n  .option(\n    \"--header <header...>\",\n    \"Extra request header as 'Key: value' (repeatable)\",\n  )\n  .option(\n    \"--tag <tag...>\",\n    \"Only run evals tagged with one of these tags (repeatable)\",\n  )\n  .option(\n    \"--profile <name>\",\n    \"Databricks CLI profile to authenticate with via OAuth (default: DATABRICKS_CONFIG_PROFILE)\",\n  )\n  .option(\n    \"--databricks-host <host>\",\n    \"Databricks host for writing MLflow assessments (default: DATABRICKS_HOST)\",\n  )\n  .option(\n    \"--databricks-token <token>\",\n    \"Databricks token for writing MLflow assessments (default: DATABRICKS_TOKEN)\",\n  )\n  .option(\n    \"--experiment <id>\",\n    \"MLflow experiment id for the evaluation run (default: MLFLOW_EXPERIMENT_ID)\",\n  )\n  .option(\n    \"--warehouse-id <id>\",\n    \"SQL warehouse id for reading managed eval datasets and writing assessments to UC-backed experiments (default: DATABRICKS_WAREHOUSE_ID, or MLFLOW_TRACING_SQL_WAREHOUSE_ID for assessments)\",\n  )\n  .option(\n    \"--judge-model <endpoint>\",\n    \"Databricks serving endpoint to use as the LLM judge for t.judge.* (default: APPKIT_JUDGE_MODEL)\",\n  )\n  .option(\n    \"--timeout <ms>\",\n    \"Default per-eval timeout in ms (a per-eval timeoutMs overrides it)\",\n  )\n  .option(\n    \"--retries <n>\",\n    \"Re-run an eval up to N times when it fails on an infra error (turn/timeout); assertion failures are not retried\",\n  )\n  .option(\n    \"--min-pass-rate <rate>\",\n    \"Gate on aggregate pass rate (0..1) instead of requiring every eval to pass; exit 1 when below\",\n  )\n  .addOption(\n    new Option(\n      \"--reporter <format>\",\n      \"Report format: text (live console), json (dashboards), or junit (CI test reporters)\",\n    )\n      .choices([\"text\", \"json\", \"junit\"])\n      .default(\"text\"),\n  )\n  .option(\n    \"--output <file>\",\n    \"Write the json/junit report to this file instead of stdout (ignored for text)\",\n  )\n  .action(runAgentEval);\n"],"mappings":";;;;;;;;;;AAqFA,eAAe,aAAkC;CAC/C,MAAM,OAAO;AACb,KAAI;AACF,SAAQ,MAAM,OAAO;UACd,KAAK;AACZ,QAAM,IAAI,MACR,yHAEY,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GAC7D;;;AAIL,SAAS,aAAa,QAA0C;CAC9D,MAAM,UAAkC,EAAE;AAC1C,MAAK,MAAM,KAAK,QAAQ;EACtB,MAAM,IAAI,EAAE,QAAQ,IAAI;AACxB,MAAI,MAAM,GAAI;AACd,UAAQ,EAAE,MAAM,GAAG,EAAE,CAAC,MAAM,IAAI,EAAE,MAAM,IAAI,EAAE,CAAC,MAAM;;AAEvD,QAAO;;;;;;;AAQT,SAAgB,cAAc,KAA6C;AACzE,KAAI,QAAQ,OAAW,QAAO;CAC9B,MAAM,IAAI,OAAO,IAAI;AAErB,KAAI,IAAI,MAAM,KAAK,MAAM,CAAC,OAAO,SAAS,EAAE,IAAI,IAAI,KAAK,IAAI,EAC3D,OAAM,IAAI,MACR,4BAA4B,IAAI,iCACjC;AAEH,QAAO;;;AAIT,SAAS,YAAY,KAA6C;CAChE,MAAM,IAAI,MAAM,OAAO,SAAS,KAAK,GAAG,GAAG;AAC3C,QAAO,IAAI,IAAI,IAAI;;;AAIrB,eAAe,WAAW,KAA+B;AACvD,KAAI;AACF,QAAM,MAAM,KAAK,EAAE,QAAQ,YAAY,QAAQ,IAAK,EAAE,CAAC;AACvD,SAAO;SACD;AACN,SAAO;;;;;;;;;;;;AAaX,eAAe,eACb,WACA,SACA,SAC+B;CAC/B,MAAM,MAAM,UAAU,OAAO;CAC7B,MAAM,QAAQ,UAAU,kBAAkB;CAC1C,MAAM,OAAO,EAAE,YAAY,IAAI;AAE/B,KAAI,SAAU,MAAM,WAAW,IAAI,EAAG;AACpC,UAAQ,MAAM,qCAAqC,MAAM;AACzD,SAAO;;AAGT,SAAQ,MAAM,wBAAwB,UAAU,UAAU;CAK1D,MAAM,QAAsB,MAAM,UAAU,SAAS;EACnD,OAAO;EACP,UAAU;EACV,OAAO,UAAU;GAAC;GAAU;GAAG;GAAU,GAAG;EAC7C,CAAC;CAEF,IAAI,SAAS;AACb,OAAM,GAAG,cAAc;AACrB,WAAS;GACT;CAEF,MAAM,aAAmB;AACvB,MAAI,UAAU,MAAM,QAAQ,OAAW;AACvC,MAAI;AACF,WAAQ,KAAK,CAAC,MAAM,KAAK,UAAU;UAC7B;;CAKV,MAAM,WAAW,KAAK,KAAK,IAAI,UAAU,aAAa;AACtD,KAAI;AACF,SAAO,KAAK,KAAK,GAAG,UAAU;AAC5B,OAAI,OAAQ,OAAM,IAAI,MAAM,0CAA0C;AACtE,OAAI,MAAM,WAAW,IAAI,EAAE;AACzB,YAAQ,MAAM,uBAAuB,MAAM;AAC3C,WAAO,EAAE,MAAM;;AAEjB,SAAM,IAAI,SAAS,MAAM,WAAW,GAAG,IAAI,CAAC;;UAEvC,KAAK;AACZ,QAAM;AACN,QAAM;;AAER,OAAM;AACN,OAAM,IAAI,MACR,iCAAiC,IAAI,UAAU,UAAU,aAAa,IAAO,IAC9E;;;;;;AA8BH,SAAS,cAAc,MAAmB,MAAY;CACpD,MAAM,eAAe,KAAK,cAAc,QAAQ,IAAI;AACpD,KAAI,EAAE,KAAK,QAAQ,KAAK,SAAS,cAAe,QAAO;CAGvD,MAAM,iBACJ,KAAK,eACL,QAAQ,IAAI,mCACZ,QAAQ,IAAI;AACd,QAAO;EACL,MAAM,KAAK;EACX,OAAO,KAAK;EACZ;EACA,GAAI,iBAAiB,EAAE,gBAAgB,GAAG,EAAE;EAC7C;;;AAIH,SAAS,aAAa,MAAmB,MAAY;CACnD,MAAM,QAAQ,KAAK,cAAc,QAAQ,IAAI;AAC7C,QAAO,SAAS,KAAK,QAAQ,KAAK,QAC9B;EAAE,MAAM,KAAK;EAAM,OAAO,KAAK;EAAO;EAAO,GAC7C;;;;;;;AAQN,SAAS,qBACP,QACA,KACA,SACA,MAC+B;AAC/B,SAAQ,UAAU;AAChB,UAAQ,MAAM,MAAd;GACE,KAAK;AACH,SACE,WAAW,MAAM,MAAM,OAAO,MAAM,UAAU,IAAI,KAAK,IAAI,WAAW,IAAI,IAC3E;AACD;GACF,KAAK;AACH,SAAK,0BAA0B,MAAM,MAAM,IAAI;AAC/C;GACF,KAAK;AACH,QAAI,QAAS;AAGb,YAAQ,IACN,IAAI,MAAM,QAAQ,EAAE,GAAG,MAAM,MAAM,IAAI,OAAO,mBAAmB,MAAM,OAAO,GAC/E;AACD,SAAK,MAAM,QAAQ,OAAO,iBAAiB,MAAM,OAAO,CACtD,SAAQ,IAAI,KAAK;AAEnB;;;;AAMR,SAAS,kBAAkB,GAIhB;AACT,QAAO,aAAa,EAAE,QAAQ,IAAI,EAAE,UAAU,GAAG,GAAG,EAAE,SAAS,KAAK,MAAM;;;;;;;AAQ5E,SAAS,mBACP,QACA,MACM;CACN,MAAM,EAAE,QAAQ,WAAW;AAC3B,MACE,WAAW,OAAO,QAAQ,2BACvB,OAAO,UAAU,KAAK,OAAO,QAAQ,YAAY,OACjD,OAAO,SAAS,SAAS,KAAK,OAAO,SAAS,OAAO,WAAW,IACpE;AACD,MAAK,MAAM,KAAK,OAAO,SACrB,SAAQ,MAAM,kBAAkB,EAAE,CAAC;AAErC,KAAI,OAAO,aACT,SAAQ,MAAM,2BAA2B,OAAO,eAAe;AAEjE,KAAI,CAAC,OAAO,SACV,SAAQ,MACN,4CAA4C,OAAO,eAAe,YACnE;;AAIL,eAAe,aACb,QACA,MACe;CACf,MAAM,SAAS,MAAM,YAAY;CAIjC,MAAM,UAAU,KAAK,QAAQ,QAAQ,KAAK;CAC1C,MAAM,SAAU,MAAM,OAAO,mBAAmB,QAAQ,IAAK,EAAE;CAI/D,MAAM,UAAU,KAAK,OAAO,OAAO,WAAW;CAK9C,MAAM,cAAc;EAClB,SAAS,KAAK,WAAW,QAAQ,IAAI;EACrC,MAAM,KAAK,kBAAkB,QAAQ,IAAI;EACzC,OAAO,KAAK,mBAAmB,QAAQ,IAAI;EAC5C;CAKD,MAAM,OAAc,MAAM,OAAO,sBAAsB,YAAY,IAAK,EAAE;CAI1E,MAAM,cAAc,KAAK,eAAe,QAAQ,IAAI;CACpD,MAAM,kBAAkB,OAAO,uBAAuB,YAAY;CAIlE,MAAM,cAAc,YAAY,KAAK,YAAY,IAAI,OAAO;CAI5D,MAAM,YAAY,YAAY,KAAK,QAAQ,IAAI,OAAO;CAItD,MAAM,UAAU,YAAY,KAAK,QAAQ;CAGzC,IAAI;AACJ,KAAI;AACF,gBAAc,cAAc,KAAK,YAAY;UACtC,KAAK;AACZ,UAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,CAAC;AAC/D,UAAQ,WAAW;AACnB;;CAMF,MAAM,WAAW,KAAK,YAAY;CAClC,MAAM,UAAU,aAAa;CAC7B,MAAM,QAAQ,QAAsB;AAClC,MAAI,QAAS,SAAQ,MAAM,IAAI;MAC1B,SAAQ,IAAI,IAAI;;CAOvB,IAAI;CACJ,IAAI;AACJ,KAAI;AACF,WAAS,OAAO,YACZ,MAAM,eAAe,OAAO,WAAW,SAAS,QAAQ,GACxD;AACJ,YAAU,MAAM,OAAO,cAAc;GACnC;GACA;GACA;GACA,MAAM,KAAK;GACX,QAAQ,KAAK;GACb,SAAS,KAAK,SAAS,aAAa,KAAK,OAAO,GAAG;GACnD;GACA,QAAQ,cAAc,MAAM,KAAK;GACjC,OAAO,aAAa,MAAM,KAAK;GAC/B;GACA;GACA;GACA;GACA,SAAS,qBAAqB,QAAQ,SAAS,SAAS,KAAK;GAC9D,CAAC;UACK,KAAK;AAKZ,UAAQ,MACN,sBAAsB,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GACvE;AACD,UAAQ,WAAW;AACnB;WACQ;AACR,UAAQ,MAAM;;AAKhB,MAAK,KAAK,OAAO,kBAAkB,QAAQ,QAAQ,GAAG;AAEtD,KAAI,QAAQ,OACV,oBAAmB,QAAQ,QAAQ,KAAK;KAExC,MACE,oIAED;AAKH,KAAI,SAAS;EACX,MAAM,SACJ,aAAa,SACT,OAAO,kBAAkB,QAAQ,QAAQ,GACzC,OAAO,mBAAmB,QAAQ,QAAQ;AAChD,MAAI,KAAK,QAAQ;AACf,OAAI;AACF,OAAG,cAAc,KAAK,QAAQ,GAAG,OAAO,IAAI;YACrC,KAAK;AACZ,YAAQ,MACN,mBAAmB,SAAS,aAAa,KAAK,OAAO,IACnD,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GAEnD;AACD,YAAQ,WAAW;AACnB;;AAEF,QAAK,SAAS,SAAS,aAAa,KAAK,SAAS;QAElD,SAAQ,OAAO,MAAM,GAAG,OAAO,IAAI;;CAIvC,MAAM,QAAQ,OAAO,UAAU,QAAQ,QAAQ;AAC/C,KAAI,gBAAgB,QAAW;EAG7B,MAAM,KAAK,MAAM,YAAY;AAC7B,OACE,cAAc,MAAM,WAAW,KAAK,QAAQ,EAAE,CAAC,gBAC7C,cAAc,KACd,QAAQ,EAAE,CAAC,OAAO,KAAK,OAAO,oBACjC;AACD,MAAI,CAAC,GAAI,SAAQ,WAAW;YACnB,CAAC,MAAM,UAChB,SAAQ,WAAW;;AAIvB,MAAa,mBAAmB,IAAI,QAAQ,OAAO,CAChD,YACC,6EACD,CACA,SACC,YACA,mFACD,CACA,OACC,eACA,8FACD,CACA,OAAO,YAAY,qCAAqC,MAAM,CAC9D,OACC,qBACA,wGACD,CACA,OACC,gBACA,wDACD,CACA,OACC,wBACA,oDACD,CACA,OACC,kBACA,4DACD,CACA,OACC,oBACA,6FACD,CACA,OACC,4BACA,4EACD,CACA,OACC,8BACA,8EACD,CACA,OACC,qBACA,8EACD,CACA,OACC,uBACA,6LACD,CACA,OACC,4BACA,kGACD,CACA,OACC,kBACA,qEACD,CACA,OACC,iBACA,kHACD,CACA,OACC,0BACA,gGACD,CACA,UACC,IAAI,OACF,uBACA,sFACD,CACE,QAAQ;CAAC;CAAQ;CAAQ;CAAQ,CAAC,CAClC,QAAQ,OAAO,CACnB,CACA,OACC,mBACA,gFACD,CACA,OAAO,aAAa"}

package/dist/cli/commands/registry/add.js

@@ -7,9 +7,9 @@ import { parseEnv } from "./env-reconcile.js";
 import fs from "node:fs";
 import path from "node:path";
 import { Command } from "commander";
+import { spawnSync } from "node:child_process";
 import process from "node:process";
 import pc from "picocolors";
-import { spawnSync } from "node:child_process";
 
 //#region src/cli/commands/registry/add.ts
 /** Subdirectories that commonly hold the frontend / server in an AppKit app. */

package/dist/cli/commands/registry/config-writer.js

@@ -2,8 +2,8 @@ import { APP_YAML_FILE, DATABRICKS_YML_FILE, bindingToNode } from "../../deploy-
 import { planHasContent } from "./config-plan.js";
 import fs from "node:fs";
 import path from "node:path";
-import pc from "picocolors";
 import { spawnSync } from "node:child_process";
+import pc from "picocolors";
 import { parseDocument } from "yaml";
 
 //#region src/cli/commands/registry/config-writer.ts

package/dist/cli/index.js

@@ -30,7 +30,10 @@ cmd.addCommand(doctorCommand);
 cmd.addCommand(registryCommand, { hidden: true });
 cmd.addCommand(addCommand, { hidden: true });
 cmd.addCommand(agentCommand);
-await cmd.parseAsync();
+await cmd.parseAsync().catch((err) => {
+	console.error(`\n${err instanceof Error ? err.message : String(err)}`);
+	process.exitCode = 1;
+});
 
 //#endregion
 export {  };

package/dist/cli/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":[],"sources":["../../src/cli/index.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { readFileSync } from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\n\nimport { Command } from \"commander\";\n\nimport { agentCommand } from \"./commands/agent/index.js\";\nimport { codemodCommand } from \"./commands/codemod/index.js\";\nimport { docsCommand } from \"./commands/docs.js\";\nimport { doctorCommand } from \"./commands/doctor/index.js\";\nimport { generateTypesCommand } from \"./commands/generate-types.js\";\nimport { lintCommand } from \"./commands/lint.js\";\nimport { pluginCommand } from \"./commands/plugin/index.js\";\nimport { addCommand } from \"./commands/registry/add.js\";\nimport { registryCommand } from \"./commands/registry/index.js\";\nimport { setupCommand } from \"./commands/setup.js\";\n\nconst __dirname = dirname(fileURLToPath(import.meta.url));\nconst pkgPath = join(__dirname, \"../../package.json\");\nconst pkg = JSON.parse(readFileSync(pkgPath, \"utf-8\"));\n\nconst cmd = new Command();\n\ncmd\n  .name(\"appkit\")\n  .description(\"CLI tools for Databricks AppKit\")\n  .version(pkg.version);\n\ncmd.addCommand(setupCommand);\ncmd.addCommand(generateTypesCommand);\ncmd.addCommand(lintCommand);\ncmd.addCommand(docsCommand);\ncmd.addCommand(pluginCommand);\ncmd.addCommand(codemodCommand);\ncmd.addCommand(doctorCommand);\n// Registry commands are executable but hidden from --help while the feature\n// is still in development (registry + add work end-to-end but aren't announced).\ncmd.addCommand(registryCommand, { hidden: true });\ncmd.addCommand(addCommand, { hidden: true });\ncmd.addCommand(agentCommand);\n\nawait cmd.parseAsync();\n"],"mappings":";;;;;;;;;;;;;;;;;;AAoBA,MAAM,UAAU,KADE,QAAQ,cAAc,OAAO,KAAK,IAAI,CAAC,EACzB,qBAAqB;AACrD,MAAM,MAAM,KAAK,MAAM,aAAa,SAAS,QAAQ,CAAC;AAEtD,MAAM,MAAM,IAAI,SAAS;AAEzB,IACG,KAAK,SAAS,CACd,YAAY,kCAAkC,CAC9C,QAAQ,IAAI,QAAQ;AAEvB,IAAI,WAAW,aAAa;AAC5B,IAAI,WAAW,qBAAqB;AACpC,IAAI,WAAW,YAAY;AAC3B,IAAI,WAAW,YAAY;AAC3B,IAAI,WAAW,cAAc;AAC7B,IAAI,WAAW,eAAe;AAC9B,IAAI,WAAW,cAAc;AAG7B,IAAI,WAAW,iBAAiB,EAAE,QAAQ,MAAM,CAAC;AACjD,IAAI,WAAW,YAAY,EAAE,QAAQ,MAAM,CAAC;AAC5C,IAAI,WAAW,aAAa;AAE5B,MAAM,IAAI,YAAY"}
+{"version":3,"file":"index.js","names":[],"sources":["../../src/cli/index.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { readFileSync } from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\n\nimport { Command } from \"commander\";\n\nimport { agentCommand } from \"./commands/agent/index.js\";\nimport { codemodCommand } from \"./commands/codemod/index.js\";\nimport { docsCommand } from \"./commands/docs.js\";\nimport { doctorCommand } from \"./commands/doctor/index.js\";\nimport { generateTypesCommand } from \"./commands/generate-types.js\";\nimport { lintCommand } from \"./commands/lint.js\";\nimport { pluginCommand } from \"./commands/plugin/index.js\";\nimport { addCommand } from \"./commands/registry/add.js\";\nimport { registryCommand } from \"./commands/registry/index.js\";\nimport { setupCommand } from \"./commands/setup.js\";\n\nconst __dirname = dirname(fileURLToPath(import.meta.url));\nconst pkgPath = join(__dirname, \"../../package.json\");\nconst pkg = JSON.parse(readFileSync(pkgPath, \"utf-8\"));\n\nconst cmd = new Command();\n\ncmd\n  .name(\"appkit\")\n  .description(\"CLI tools for Databricks AppKit\")\n  .version(pkg.version);\n\ncmd.addCommand(setupCommand);\ncmd.addCommand(generateTypesCommand);\ncmd.addCommand(lintCommand);\ncmd.addCommand(docsCommand);\ncmd.addCommand(pluginCommand);\ncmd.addCommand(codemodCommand);\ncmd.addCommand(doctorCommand);\n// Registry commands are executable but hidden from --help while the feature\n// is still in development (registry + add work end-to-end but aren't announced).\ncmd.addCommand(registryCommand, { hidden: true });\ncmd.addCommand(addCommand, { hidden: true });\ncmd.addCommand(agentCommand);\n\n// Surface a clean message for any uncaught setup failure (e.g. a root\n// evals.config.ts that throws on import) instead of a raw unhandled-rejection\n// stack; a non-zero exit still gates CI.\nawait cmd.parseAsync().catch((err: unknown) => {\n  console.error(`\\n${err instanceof Error ? err.message : String(err)}`);\n  process.exitCode = 1;\n});\n"],"mappings":";;;;;;;;;;;;;;;;;;AAoBA,MAAM,UAAU,KADE,QAAQ,cAAc,OAAO,KAAK,IAAI,CAAC,EACzB,qBAAqB;AACrD,MAAM,MAAM,KAAK,MAAM,aAAa,SAAS,QAAQ,CAAC;AAEtD,MAAM,MAAM,IAAI,SAAS;AAEzB,IACG,KAAK,SAAS,CACd,YAAY,kCAAkC,CAC9C,QAAQ,IAAI,QAAQ;AAEvB,IAAI,WAAW,aAAa;AAC5B,IAAI,WAAW,qBAAqB;AACpC,IAAI,WAAW,YAAY;AAC3B,IAAI,WAAW,YAAY;AAC3B,IAAI,WAAW,cAAc;AAC7B,IAAI,WAAW,eAAe;AAC9B,IAAI,WAAW,cAAc;AAG7B,IAAI,WAAW,iBAAiB,EAAE,QAAQ,MAAM,CAAC;AACjD,IAAI,WAAW,YAAY,EAAE,QAAQ,MAAM,CAAC;AAC5C,IAAI,WAAW,aAAa;AAK5B,MAAM,IAAI,YAAY,CAAC,OAAO,QAAiB;AAC7C,SAAQ,MAAM,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GAAG;AACtE,SAAQ,WAAW;EACnB"}

package/docs/api/appkit/Function.findRootEvalConfig.md

@@ -0,0 +1,18 @@
+# Function: findRootEvalConfig()
+
+```ts
+function findRootEvalConfig(rootDir: string): string | undefined;
+
+```
+
+Path to the root `evals.config.ts` (from [defineEvalConfig](./docs/api/appkit/Function.defineEvalConfig.md)) at `<rootDir>/evals.config.ts`, or `undefined` when absent. The root config holds run-wide settings (`baseUrl`, `webServer`); it's distinct from the per-agent configs found by [discoverEvalConfigs](./docs/api/appkit/Function.discoverEvalConfigs.md).
+
+## Parameters[​](#parameters "Direct link to Parameters")
+
+| Parameter | Type     |
+| --------- | -------- |
+| `rootDir` | `string` |
+
+## Returns[​](#returns "Direct link to Returns")
+
+`string` | `undefined`

package/docs/api/appkit/Function.loadRootEvalConfig.md

@@ -0,0 +1,18 @@
+# Function: loadRootEvalConfig()
+
+```ts
+function loadRootEvalConfig(rootDir: string): Promise<EvalConfig | undefined>;
+
+```
+
+Load the root `evals.config.ts` under `rootDir` (the project root), or return `undefined` when there is none. This is the run-wide config carrying `baseUrl`/`webServer`; the CLI reads it to resolve options and manage the app-under-test lifecycle before calling [runEvalsInDir](./docs/api/appkit/Function.runEvalsInDir.md).
+
+## Parameters[​](#parameters "Direct link to Parameters")
+
+| Parameter | Type     |
+| --------- | -------- |
+| `rootDir` | `string` |
+
+## Returns[​](#returns "Direct link to Returns")
+
+`Promise`<`EvalConfig` | `undefined`>

package/docs/api/appkit/Interface.EvalWebServer.md

@@ -0,0 +1,47 @@
+# Interface: EvalWebServer
+
+Auto-start config for the app under test, à la Playwright's `webServer`. When set in a root `evals.config.ts`, the CLI boots the app before running evals and tears it down after — so you don't have to start the server by hand.
+
+## Properties[​](#properties "Direct link to Properties")
+
+### command[​](#command "Direct link to command")
+
+```ts
+command: string;
+
+```
+
+Shell command that starts the app, e.g. `"npm run dev"`.
+
+***
+
+### reuseExisting?[​](#reuseexisting "Direct link to reuseExisting?")
+
+```ts
+optional reuseExisting: boolean;
+
+```
+
+When `true` (default), reuse a server already answering at `url` instead of spawning one — so a running `dev` server is used as-is. Set `false` to always spawn a fresh server.
+
+***
+
+### timeoutMs?[​](#timeoutms "Direct link to timeoutMs?")
+
+```ts
+optional timeoutMs: number;
+
+```
+
+How long to wait for `url` to answer before giving up. Defaults to 60s.
+
+***
+
+### url?[​](#url "Direct link to url?")
+
+```ts
+optional url: string;
+
+```
+
+URL polled until it answers before evals start. Defaults to the run's `baseUrl` (`--url`). Readiness = any HTTP response (a 404 still proves the server is up).

package/docs/api/appkit.md

@@ -63,6 +63,7 @@ Documentation merge entry for Typedoc — combines the stable `@databricks/appki
 | [EvalResult](./docs/api/appkit/Interface.EvalResult.md)                                               | The outcome of running one eval.                                                                                                                                                                                                                                                                                                                                                                                                                              |
 | [EvalRunSummary](./docs/api/appkit/Interface.EvalRunSummary.md)                                       | -                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
 | [EvalSummary](./docs/api/appkit/Interface.EvalSummary.md)                                             | -                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| [EvalWebServer](./docs/api/appkit/Interface.EvalWebServer.md)                                         | Auto-start config for the app under test, à la Playwright's `webServer`. When set in a root `evals.config.ts`, the CLI boots the app before running evals and tears it down after — so you don't have to start the server by hand.                                                                                                                                                                                                                            |
 | [FilePolicyUser](./docs/api/appkit/Interface.FilePolicyUser.md)                                       | Minimal user identity passed to the policy function.                                                                                                                                                                                                                                                                                                                                                                                                          |
 | [FileResource](./docs/api/appkit/Interface.FileResource.md)                                           | Describes the file or directory being acted upon.                                                                                                                                                                                                                                                                                                                                                                                                             |
 | [FunctionTool](./docs/api/appkit/Interface.FunctionTool.md)                                           | -                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
@@ -210,6 +211,7 @@ Documentation merge entry for Typedoc — combines the stable `@databricks/appki
 | [evalGlyph](./docs/api/appkit/Function.evalGlyph.md)                                   | Status glyph for a single eval result.                                                                                                                                                                                                                                                                                                                                                                     |
 | [executeFromRegistry](./docs/api/appkit/Function.executeFromRegistry.md)               | Validates tool-call arguments against the entry's schema and invokes its handler. On validation failure, returns an LLM-friendly error string (matching the behavior of `tool()`) rather than throwing, so the model can self-correct on its next turn.                                                                                                                                                    |
 | [extractServingEndpoints](./docs/api/appkit/Function.extractServingEndpoints.md)       | Extract serving endpoint config from a server file by AST-parsing it. Looks for `serving({ endpoints: { alias: { env: "..." }, ... } })` calls and extracts the endpoint alias names and their environment variable mappings.                                                                                                                                                                              |
+| [findRootEvalConfig](./docs/api/appkit/Function.findRootEvalConfig.md)                 | Path to the root `evals.config.ts` (from [defineEvalConfig](./docs/api/appkit/Function.defineEvalConfig.md)) at `<rootDir>/evals.config.ts`, or `undefined` when absent. The root config holds run-wide settings (`baseUrl`, `webServer`); it's distinct from the per-agent configs found by [discoverEvalConfigs](./docs/api/appkit/Function.discoverEvalConfigs.md).                         |
 | [findServerFile](./docs/api/appkit/Function.findServerFile.md)                         | Find the server entry file by checking candidate paths in order.                                                                                                                                                                                                                                                                                                                                           |
 | [fk](./docs/api/appkit/Function.fk.md)                                                 | Declare foreign-key to another column.                                                                                                                                                                                                                                                                                                                                                                     |
 | [formatEvalDetail](./docs/api/appkit/Function.formatEvalDetail.md)                     | Indented detail lines for a failing eval (error + failing assertions).                                                                                                                                                                                                                                                                                                                                     |
@@ -240,6 +242,7 @@ Documentation merge entry for Typedoc — combines the stable `@databricks/appki
 | [jsonb](./docs/api/appkit/Function.jsonb.md)                                           | -                                                                                                                                                                                                                                                                                                                                                                                                          |
 | [loadAgentFromFile](./docs/api/appkit/Function.loadAgentFromFile.md)                   | Loads a single markdown agent file and resolves its frontmatter against registered plugin toolkits + ambient tool library.                                                                                                                                                                                                                                                                                 |
 | [loadAgentsFromDir](./docs/api/appkit/Function.loadAgentsFromDir.md)                   | Scans a directory for one subdirectory per agent, each containing `agent.md` (frontmatter + body). Produces an `AgentDefinition` record keyed by agent id (folder name). Throws on frontmatter errors or unresolved references. Returns an empty map if the directory does not exist.                                                                                                                      |
+| [loadRootEvalConfig](./docs/api/appkit/Function.loadRootEvalConfig.md)                 | Load the root `evals.config.ts` under `rootDir` (the project root), or return `undefined` when there is none. This is the run-wide config carrying `baseUrl`/`webServer`; the CLI reads it to resolve options and manage the app-under-test lifecycle before calling [runEvalsInDir](./docs/api/appkit/Function.runEvalsInDir.md).                                                                   |
 | [matches](./docs/api/appkit/Function.matches.md)                                       | Passes when the value matches `pattern`.                                                                                                                                                                                                                                                                                                                                                                   |
 | [mcpServer](./docs/api/appkit/Function.mcpServer.md)                                   | Factory for declaring a custom MCP server tool.                                                                                                                                                                                                                                                                                                                                                            |
 | [normalizeHost](./docs/api/appkit/Function.normalizeHost.md)                           | Ensure the host has a scheme (Databricks env often lacks `https://`).                                                                                                                                                                                                                                                                                                                                      |

package/llms.txt

@@ -110,6 +110,7 @@ npx @databricks/appkit docs <query>
 - [Function: evalGlyph()](./docs/api/appkit/Function.evalGlyph.md): Status glyph for a single eval result.
 - [Function: executeFromRegistry()](./docs/api/appkit/Function.executeFromRegistry.md): Validates tool-call arguments against the entry's schema and invokes its
 - [Function: extractServingEndpoints()](./docs/api/appkit/Function.extractServingEndpoints.md): Extract serving endpoint config from a server file by AST-parsing it.
+- [Function: findRootEvalConfig()](./docs/api/appkit/Function.findRootEvalConfig.md): Path to the root evals.config.ts (from defineEvalConfig) at
 - [Function: findServerFile()](./docs/api/appkit/Function.findServerFile.md): Find the server entry file by checking candidate paths in order.
 - [Function: fk()](./docs/api/appkit/Function.fk.md): Declare foreign-key to another column.
 - [Function: formatEvalDetail()](./docs/api/appkit/Function.formatEvalDetail.md): Indented detail lines for a failing eval (error + failing assertions).
@@ -140,6 +141,7 @@ npx @databricks/appkit docs <query>
 - [Function: jsonb()](./docs/api/appkit/Function.jsonb.md): Returns
 - [Function: loadAgentFromFile()](./docs/api/appkit/Function.loadAgentFromFile.md): Loads a single markdown agent file and resolves its frontmatter against
 - [Function: loadAgentsFromDir()](./docs/api/appkit/Function.loadAgentsFromDir.md): Scans a directory for one subdirectory per agent, each containing
+- [Function: loadRootEvalConfig()](./docs/api/appkit/Function.loadRootEvalConfig.md): Load the root evals.config.ts under rootDir (the project root), or return
 - [Function: matches()](./docs/api/appkit/Function.matches.md): Passes when the value matches pattern.
 - [Function: mcpServer()](./docs/api/appkit/Function.mcpServer.md): Factory for declaring a custom MCP server tool.
 - [Function: normalizeHost()](./docs/api/appkit/Function.normalizeHost.md): Ensure the host has a scheme (Databricks env often lacks https://).
@@ -189,6 +191,7 @@ npx @databricks/appkit docs <query>
 - [Interface: EvalResult](./docs/api/appkit/Interface.EvalResult.md): The outcome of running one eval.
 - [Interface: EvalRunSummary](./docs/api/appkit/Interface.EvalRunSummary.md): Properties
 - [Interface: EvalSummary](./docs/api/appkit/Interface.EvalSummary.md): Properties
+- [Interface: EvalWebServer](./docs/api/appkit/Interface.EvalWebServer.md): Auto-start config for the app under test, à la Playwright's webServer. When
 - [Interface: FilePolicyUser](./docs/api/appkit/Interface.FilePolicyUser.md): Minimal user identity passed to the policy function.
 - [Interface: FileResource](./docs/api/appkit/Interface.FileResource.md): Describes the file or directory being acted upon.
 - [Interface: FunctionTool](./docs/api/appkit/Interface.FunctionTool.md): Properties

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@databricks/appkit-ui",
   "type": "module",
-  "version": "0.74.1",
+  "version": "0.75.0",
   "license": "Apache-2.0",
   "sideEffects": [
     "**/*.css"