@databricks/appkit 0.71.0 → 0.72.0

package/dist/evals/run-eval.js

@@ -0,0 +1,144 @@
+import { judgeClosedQA, judgeCustom, judgeFactuality } from "./judge.js";
+
+//#region src/evals/run-eval.ts
+/** Default pass threshold for an LLM-judge score (0..1) before `.atLeast()`. */
+const DEFAULT_JUDGE_THRESHOLD = .5;
+/** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */
+var SkipSignal = class extends Error {
+	constructor(reason) {
+		super("eval skipped");
+		this.reason = reason;
+		this.name = "SkipSignal";
+	}
+};
+/**
+* Run a single eval against a driver. Never throws for assertion or agent
+* failures — those become a non-passing {@link EvalResult}. Only a malformed
+* eval definition surfaces as `result.error`.
+*/
+async function runEval(def, options) {
+	const assertions = [];
+	let reply = "";
+	let lastInput = "";
+	let toolCalls = [];
+	let sessionId;
+	let lastTraceId;
+	let lastSucceeded = false;
+	const record = (label, pass, score, detail) => {
+		const result = {
+			label,
+			severity: "gate",
+			pass,
+			score,
+			detail
+		};
+		assertions.push(result);
+		const handle = {
+			gate() {
+				result.severity = "gate";
+				return handle;
+			},
+			soft() {
+				result.severity = "soft";
+				return handle;
+			},
+			atLeast(threshold) {
+				result.severity = "soft";
+				result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold;
+				return handle;
+			}
+		};
+		return handle;
+	};
+	const recordJudge = (label, score, rationale) => record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale).soft();
+	const t = {
+		async send(message) {
+			lastInput = message;
+			const r = await options.driver.send(message);
+			reply = r.reply;
+			toolCalls = r.toolCalls;
+			sessionId = r.sessionId;
+			lastSucceeded = r.succeeded;
+			if (r.traceId) lastTraceId = r.traceId;
+		},
+		get reply() {
+			return reply;
+		},
+		get toolCalls() {
+			return toolCalls;
+		},
+		get sessionId() {
+			return sessionId;
+		},
+		succeeded() {
+			return record("succeeded", lastSucceeded, void 0, lastSucceeded ? void 0 : "agent turn did not complete successfully");
+		},
+		calledTool(name) {
+			return record(`calledTool(${name})`, toolCalls.includes(name), void 0, `expected tool "${name}" to be called (called: ${toolCalls.length ? toolCalls.join(", ") : "none"})`);
+		},
+		check(value, matcher) {
+			const m = matcher(value);
+			return record("check", m.pass, m.score, m.detail);
+		},
+		judge: {
+			async factuality(expected) {
+				const { score, rationale } = await judgeFactuality({
+					input: lastInput,
+					output: reply,
+					expected
+				});
+				return recordJudge("judge.factuality", score, rationale);
+			},
+			async closedQA(criteria) {
+				const { score, rationale } = await judgeClosedQA({
+					input: lastInput,
+					output: reply,
+					criteria
+				});
+				return recordJudge("judge.closedQA", score, rationale);
+			},
+			async custom(spec) {
+				const { score, rationale } = await judgeCustom(spec, {
+					input: lastInput,
+					output: reply
+				});
+				return recordJudge(`judge.${spec.name}`, score, rationale);
+			}
+		},
+		skip(reason) {
+			throw new SkipSignal(reason);
+		}
+	};
+	try {
+		await def.test(t);
+	} catch (err) {
+		if (err instanceof SkipSignal) return {
+			id: options.id,
+			description: def.description,
+			skipped: { reason: err.reason },
+			assertions,
+			passed: true,
+			traceId: lastTraceId
+		};
+		return {
+			id: options.id,
+			description: def.description,
+			assertions,
+			passed: false,
+			error: err instanceof Error ? err.message : String(err),
+			traceId: lastTraceId
+		};
+	}
+	const passed = assertions.every((a) => a.pass || a.severity === "soft" && !options.strict);
+	return {
+		id: options.id,
+		description: def.description,
+		assertions,
+		passed,
+		traceId: lastTraceId
+	};
+}
+
+//#endregion
+export { runEval };
+//# sourceMappingURL=run-eval.js.map

package/dist/evals/run-eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-eval.js","names":[],"sources":["../../src/evals/run-eval.ts"],"sourcesContent":["import { judgeClosedQA, judgeCustom, judgeFactuality } from \"./judge\";\nimport type {\n  AssertionHandle,\n  AssertionResult,\n  EvalDefinition,\n  EvalDriver,\n  EvalResult,\n  Matcher,\n  TestContext,\n} from \"./types\";\n\n/** Default pass threshold for an LLM-judge score (0..1) before `.atLeast()`. */\nconst DEFAULT_JUDGE_THRESHOLD = 0.5;\n\n/** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */\nclass SkipSignal extends Error {\n  constructor(public reason?: string) {\n    super(\"eval skipped\");\n    this.name = \"SkipSignal\";\n  }\n}\n\nexport interface RunEvalOptions {\n  /** Stable id for the eval (e.g. its file path relative to the evals dir). */\n  id: string;\n  /** Drives the agent and returns reply/tool-calls/success per `send`. */\n  driver: EvalDriver;\n  /** When true, soft assertion failures also fail the eval. */\n  strict?: boolean;\n}\n\n/**\n * Run a single eval against a driver. Never throws for assertion or agent\n * failures — those become a non-passing {@link EvalResult}. Only a malformed\n * eval definition surfaces as `result.error`.\n */\nexport async function runEval(\n  def: EvalDefinition,\n  options: RunEvalOptions,\n): Promise<EvalResult> {\n  const assertions: AssertionResult[] = [];\n  let reply = \"\";\n  let lastInput = \"\";\n  let toolCalls: string[] = [];\n  let sessionId: string | undefined;\n  let lastTraceId: string | undefined;\n  let lastSucceeded = false;\n\n  const record = (\n    label: string,\n    pass: boolean,\n    score?: number,\n    detail?: string,\n  ): AssertionHandle => {\n    const result: AssertionResult = {\n      label,\n      severity: \"gate\",\n      pass,\n      score,\n      detail,\n    };\n    assertions.push(result);\n    const handle: AssertionHandle = {\n      gate() {\n        result.severity = \"gate\";\n        return handle;\n      },\n      soft() {\n        result.severity = \"soft\";\n        return handle;\n      },\n      atLeast(threshold: number) {\n        result.severity = \"soft\";\n        result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold;\n        return handle;\n      },\n    };\n    return handle;\n  };\n\n  // LLM-judge assertions are scored and soft by default; the caller chains\n  // `.atLeast(n)` to set the pass threshold or `.gate()` to promote.\n  const recordJudge = (\n    label: string,\n    score: number,\n    rationale?: string,\n  ): AssertionHandle =>\n    record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale).soft();\n\n  const t: TestContext = {\n    async send(message) {\n      lastInput = message;\n      const r = await options.driver.send(message);\n      reply = r.reply;\n      toolCalls = r.toolCalls;\n      sessionId = r.sessionId;\n      lastSucceeded = r.succeeded;\n      if (r.traceId) lastTraceId = r.traceId;\n    },\n    get reply() {\n      return reply;\n    },\n    get toolCalls() {\n      return toolCalls;\n    },\n    get sessionId() {\n      return sessionId;\n    },\n    succeeded() {\n      return record(\n        \"succeeded\",\n        lastSucceeded,\n        undefined,\n        lastSucceeded ? undefined : \"agent turn did not complete successfully\",\n      );\n    },\n    calledTool(name) {\n      return record(\n        `calledTool(${name})`,\n        toolCalls.includes(name),\n        undefined,\n        `expected tool \"${name}\" to be called (called: ${\n          toolCalls.length ? toolCalls.join(\", \") : \"none\"\n        })`,\n      );\n    },\n    check(value: string, matcher: Matcher) {\n      const m = matcher(value);\n      return record(\"check\", m.pass, m.score, m.detail);\n    },\n    judge: {\n      async factuality(expected) {\n        const { score, rationale } = await judgeFactuality({\n          input: lastInput,\n          output: reply,\n          expected,\n        });\n        return recordJudge(\"judge.factuality\", score, rationale);\n      },\n      async closedQA(criteria) {\n        const { score, rationale } = await judgeClosedQA({\n          input: lastInput,\n          output: reply,\n          criteria,\n        });\n        return recordJudge(\"judge.closedQA\", score, rationale);\n      },\n      async custom(spec) {\n        const { score, rationale } = await judgeCustom(spec, {\n          input: lastInput,\n          output: reply,\n        });\n        return recordJudge(`judge.${spec.name}`, score, rationale);\n      },\n    },\n    skip(reason) {\n      throw new SkipSignal(reason);\n    },\n  };\n\n  try {\n    await def.test(t);\n  } catch (err) {\n    if (err instanceof SkipSignal) {\n      return {\n        id: options.id,\n        description: def.description,\n        skipped: { reason: err.reason },\n        assertions,\n        passed: true,\n        traceId: lastTraceId,\n      };\n    }\n    return {\n      id: options.id,\n      description: def.description,\n      assertions,\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n      traceId: lastTraceId,\n    };\n  }\n\n  const passed = assertions.every(\n    (a) => a.pass || (a.severity === \"soft\" && !options.strict),\n  );\n\n  return {\n    id: options.id,\n    description: def.description,\n    assertions,\n    passed,\n    traceId: lastTraceId,\n  };\n}\n"],"mappings":";;;;AAYA,MAAM,0BAA0B;;AAGhC,IAAM,aAAN,cAAyB,MAAM;CAC7B,YAAY,AAAO,QAAiB;AAClC,QAAM,eAAe;EADJ;AAEjB,OAAK,OAAO;;;;;;;;AAkBhB,eAAsB,QACpB,KACA,SACqB;CACrB,MAAM,aAAgC,EAAE;CACxC,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,YAAsB,EAAE;CAC5B,IAAI;CACJ,IAAI;CACJ,IAAI,gBAAgB;CAEpB,MAAM,UACJ,OACA,MACA,OACA,WACoB;EACpB,MAAM,SAA0B;GAC9B;GACA,UAAU;GACV;GACA;GACA;GACD;AACD,aAAW,KAAK,OAAO;EACvB,MAAM,SAA0B;GAC9B,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,OAAO;AACL,WAAO,WAAW;AAClB,WAAO;;GAET,QAAQ,WAAmB;AACzB,WAAO,WAAW;AAClB,WAAO,QAAQ,OAAO,UAAU,OAAO,OAAO,IAAI,OAAO;AACzD,WAAO;;GAEV;AACD,SAAO;;CAKT,MAAM,eACJ,OACA,OACA,cAEA,OAAO,OAAO,SAAS,yBAAyB,OAAO,UAAU,CAAC,MAAM;CAE1E,MAAM,IAAiB;EACrB,MAAM,KAAK,SAAS;AAClB,eAAY;GACZ,MAAM,IAAI,MAAM,QAAQ,OAAO,KAAK,QAAQ;AAC5C,WAAQ,EAAE;AACV,eAAY,EAAE;AACd,eAAY,EAAE;AACd,mBAAgB,EAAE;AAClB,OAAI,EAAE,QAAS,eAAc,EAAE;;EAEjC,IAAI,QAAQ;AACV,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,IAAI,YAAY;AACd,UAAO;;EAET,YAAY;AACV,UAAO,OACL,aACA,eACA,QACA,gBAAgB,SAAY,2CAC7B;;EAEH,WAAW,MAAM;AACf,UAAO,OACL,cAAc,KAAK,IACnB,UAAU,SAAS,KAAK,EACxB,QACA,kBAAkB,KAAK,0BACrB,UAAU,SAAS,UAAU,KAAK,KAAK,GAAG,OAC3C,GACF;;EAEH,MAAM,OAAe,SAAkB;GACrC,MAAM,IAAI,QAAQ,MAAM;AACxB,UAAO,OAAO,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;;EAEnD,OAAO;GACL,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,OAAO,cAAc,MAAM,gBAAgB;KACjD,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,oBAAoB,OAAO,UAAU;;GAE1D,MAAM,SAAS,UAAU;IACvB,MAAM,EAAE,OAAO,cAAc,MAAM,cAAc;KAC/C,OAAO;KACP,QAAQ;KACR;KACD,CAAC;AACF,WAAO,YAAY,kBAAkB,OAAO,UAAU;;GAExD,MAAM,OAAO,MAAM;IACjB,MAAM,EAAE,OAAO,cAAc,MAAM,YAAY,MAAM;KACnD,OAAO;KACP,QAAQ;KACT,CAAC;AACF,WAAO,YAAY,SAAS,KAAK,QAAQ,OAAO,UAAU;;GAE7D;EACD,KAAK,QAAQ;AACX,SAAM,IAAI,WAAW,OAAO;;EAE/B;AAED,KAAI;AACF,QAAM,IAAI,KAAK,EAAE;UACV,KAAK;AACZ,MAAI,eAAe,WACjB,QAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB,SAAS,EAAE,QAAQ,IAAI,QAAQ;GAC/B;GACA,QAAQ;GACR,SAAS;GACV;AAEH,SAAO;GACL,IAAI,QAAQ;GACZ,aAAa,IAAI;GACjB;GACA,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACvD,SAAS;GACV;;CAGH,MAAM,SAAS,WAAW,OACvB,MAAM,EAAE,QAAS,EAAE,aAAa,UAAU,CAAC,QAAQ,OACrD;AAED,QAAO;EACL,IAAI,QAAQ;EACZ,aAAa,IAAI;EACjB;EACA;EACA,SAAS;EACV"}

package/dist/evals/run-evals.d.ts

@@ -0,0 +1,85 @@
+import { EvalResult } from "./types.js";
+import { ReportOutcome } from "./mlflow-report.js";
+import { FinishOutcome } from "./mlflow-run.js";
+
+//#region src/evals/run-evals.d.ts
+interface RunEvalsOptions {
+  /** Project root containing `server/agents/`. Defaults to `process.cwd()`. */
+  rootDir?: string;
+  /** Base URL of the running app to drive, e.g. `http://localhost:3000`. */
+  baseUrl: string;
+  /** Substring filter on `<agent>/<id>` (or an exact agent id). */
+  filter?: string;
+  /** Soft assertion failures also fail the eval. */
+  strict?: boolean;
+  /** Extra request headers for the driver (e.g. auth for a deployed app). */
+  headers?: Record<string, string>;
+  /** Per-turn wall-clock timeout (ms) before a turn is failed. Defaults to 120s. */
+  timeoutMs?: number;
+  /**
+   * Max evals to drive concurrently. Each eval opens one stream to the app as
+   * the same user, so keep this at or below the app's
+   * `maxConcurrentStreamsPerUser` (default 5) or the surplus streams hit the
+   * 429 guard. Defaults to 4; clamped to `[1, total]`.
+   */
+  concurrency?: number;
+  /**
+   * When set, create a native MLflow "Evaluation run": each eval's trace is
+   * linked to the run, pass/fail is written as feedback, and aggregate metrics
+   * are logged. Requires Databricks creds + the target experiment.
+   */
+  mlflow?: {
+    host: string;
+    token: string;
+    experimentId: string; /** SQL warehouse id for writing assessments to UC-backed (V4) traces. */
+    sqlWarehouseId?: string;
+  };
+  /**
+   * When set, enable `t.judge.*` LLM-as-judge scoring via autoevals against a
+   * Databricks serving endpoint (`model`).
+   */
+  judge?: {
+    host: string;
+    token: string;
+    model: string;
+  };
+  /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */
+  now?: number;
+  /** Progress callback, invoked as evals are discovered, started, and finished. */
+  onEvent?: (event: EvalProgress) => void;
+}
+type EvalProgress = {
+  type: "discovered";
+  total: number;
+} | {
+  type: "run-created";
+  runId: string;
+} | {
+  type: "start";
+  id: string;
+  index: number;
+  total: number;
+} | {
+  type: "result";
+  result: EvalResult;
+  index: number;
+  total: number;
+};
+interface EvalRunSummary {
+  results: EvalResult[];
+  /** Present when an MLflow evaluation run was created. */
+  mlflow?: {
+    runId: string;
+    report: ReportOutcome;
+    finish: FinishOutcome;
+  };
+}
+/**
+ * Discover, load, and run every eval under each agent's `evals/` dir, driving
+ * the agents on a running app. Never throws for an individual eval — load/run
+ * failures become non-passing {@link EvalResult}s.
+ */
+declare function runEvalsInDir(options: RunEvalsOptions): Promise<EvalRunSummary>;
+//#endregion
+export { EvalProgress, EvalRunSummary, RunEvalsOptions, runEvalsInDir };
+//# sourceMappingURL=run-evals.d.ts.map

package/dist/evals/run-evals.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-evals.d.ts","names":[],"sources":["../../src/evals/run-evals.ts"],"mappings":";;;;;UAYiB,eAAA;;EAEf,OAAA;EAF8B;EAI9B,OAAA;EAoC8B;EAlC9B,MAAA;EAFA;EAIA,MAAA;EAAA;EAEA,OAAA,GAAU,MAAA;EAAA;EAEV,SAAA;EAOA;;;;;;EAAA,WAAA;EAiBU;;;;;EAXV,MAAA;IACE,IAAA;IACA,KAAA;IACA,YAAA,UAeQ;IAbR,cAAA;EAAA;EAiBoC;;;;EAXtC,KAAA;IAAU,IAAA;IAAc,KAAA;IAAe,KAAA;EAAA;EAWnC;EATJ,GAAA;EAS4B;EAP5B,OAAA,IAAW,KAAA,EAAO,YAAA;AAAA;AAAA,KAGR,YAAA;EACN,IAAA;EAAoB,KAAA;AAAA;EACpB,IAAA;EAAqB,KAAA;AAAA;EACrB,IAAA;EAAe,EAAA;EAAY,KAAA;EAAe,KAAA;AAAA;EAC1C,IAAA;EAAgB,MAAA,EAAQ,UAAA;EAAY,KAAA;EAAe,KAAA;AAAA;AAAA,UAExC,cAAA;EACf,OAAA,EAAS,UAAA;EAE6D;EAAtE,MAAA;IAAW,KAAA;IAAe,MAAA,EAAQ,aAAA;IAAe,MAAA,EAAQ,aAAA;EAAA;AAAA;;;;;;iBAqIrC,aAAA,CACpB,OAAA,EAAS,eAAA,GACR,OAAA,CAAQ,cAAA"}

package/dist/evals/run-evals.js

@@ -0,0 +1,178 @@
+import { MlflowClient } from "../connectors/mlflow/client.js";
+import { discoverEvalFiles } from "./discover.js";
+import { createHttpDriver } from "./http-driver.js";
+import { configureJudge, teardownJudge } from "./judge.js";
+import { mapPool } from "./pool.js";
+import { reportToMlflow } from "./mlflow-report.js";
+import { runEval } from "./run-eval.js";
+import { createEvalRun, finishEvalRun } from "./mlflow-run.js";
+import { pathToFileURL } from "node:url";
+
+//#region src/evals/run-evals.ts
+/**
+* Load a `*.eval.ts` file and return its default-exported {@link EvalDefinition}.
+* Uses tsx's programmatic loader so TypeScript eval files run without a build
+* step. The specifier is indirected so the type checker doesn't try to resolve
+* tsx's internal entry.
+*/
+async function loadEval(file) {
+	const tsxApi = "tsx/esm/api";
+	let tsImport;
+	try {
+		({tsImport} = await import(tsxApi));
+	} catch {
+		throw new Error("Running .eval.ts files requires `tsx`. Install it as a dev dependency (`pnpm add -D tsx`).");
+	}
+	const def = resolveEvalDefault(await tsImport(pathToFileURL(file).href, import.meta.url));
+	if (!def) throw new Error(`${file}: must default-export defineEval({ test })`);
+	return def;
+}
+/**
+* Unwrap the eval default export across module-interop shapes. Depending on
+* whether the eval file is treated as ESM or CJS, the value lands at
+* `mod.default` (ESM), `mod.default.default` (CJS `__esModule` double-wrap), or
+* `mod` itself. Returns the first candidate that looks like an eval.
+*/
+function resolveEvalDefault(mod) {
+	let candidate = mod;
+	for (let i = 0; i < 4 && candidate; i++) {
+		if (typeof candidate.test === "function") return candidate;
+		candidate = candidate.default;
+	}
+}
+/**
+* Load and run a single discovered eval. Never throws — a load/run failure
+* becomes a non-passing {@link EvalResult} so one bad eval can't abort the run.
+*/
+async function runOne(d, id, runId, options) {
+	try {
+		const def = await loadEval(d.file);
+		return await runEval(def, {
+			id,
+			driver: createHttpDriver({
+				baseUrl: options.baseUrl,
+				agent: def.agent ?? d.agent,
+				headers: options.headers,
+				mlflowRunId: runId,
+				timeoutMs: options.timeoutMs
+			}),
+			strict: options.strict
+		});
+	} catch (err) {
+		return {
+			id,
+			assertions: [],
+			passed: false,
+			error: err instanceof Error ? err.message : String(err)
+		};
+	}
+}
+/** Configure the LLM judge when judge creds were supplied; otherwise a no-op. */
+async function maybeConfigureJudge(options) {
+	if (!options.judge) return;
+	await configureJudge({
+		client: new MlflowClient(options.judge.host, options.judge.token),
+		token: options.judge.token,
+		model: options.judge.model
+	});
+}
+/**
+* Report per-eval assessments and finish the MLflow run, when one was created.
+* Returns the run summary, or `undefined` when there was no run to finalize.
+*/
+async function finalizeMlflow(client, runId, results, options) {
+	if (!client || !runId) return void 0;
+	let report = {
+		written: 0,
+		skipped: 0,
+		failures: []
+	};
+	try {
+		report = await reportToMlflow(client, results, options.mlflow?.sqlWarehouseId);
+	} catch (err) {
+		report.failures.push({
+			traceId: "(report)",
+			error: err instanceof Error ? err.message : String(err)
+		});
+	}
+	const finish = await finishEvalRun(client, {
+		runId,
+		results,
+		endTime: options.now ?? Date.now()
+	});
+	return {
+		runId,
+		report,
+		finish
+	};
+}
+/**
+* Default max evals in flight. Each eval opens one stream to the app as the
+* same user; the server caps concurrent streams per user at 5 by default
+* (`maxConcurrentStreamsPerUser`), so 4 leaves headroom under that limit.
+*/
+const DEFAULT_CONCURRENCY = 4;
+/**
+* Discover, load, and run every eval under each agent's `evals/` dir, driving
+* the agents on a running app. Never throws for an individual eval — load/run
+* failures become non-passing {@link EvalResult}s.
+*/
+async function runEvalsInDir(options) {
+	const root = options.rootDir ?? process.cwd();
+	const now = options.now ?? Date.now();
+	let discovered = discoverEvalFiles(root);
+	if (options.filter) {
+		const f = options.filter;
+		discovered = discovered.filter((d) => d.agent === f || `${d.agent}/${d.id}`.includes(f));
+	}
+	const emit = options.onEvent ?? (() => {});
+	const total = discovered.length;
+	emit({
+		type: "discovered",
+		total
+	});
+	await maybeConfigureJudge(options);
+	try {
+		let runId;
+		let mlflowClient;
+		if (options.mlflow) {
+			mlflowClient = new MlflowClient(options.mlflow.host, options.mlflow.token);
+			runId = await createEvalRun(mlflowClient, {
+				experimentId: options.mlflow.experimentId,
+				runName: `appkit-eval ${new Date(now).toISOString()}`,
+				startTime: now
+			});
+			emit({
+				type: "run-created",
+				runId
+			});
+		}
+		const results = await mapPool(discovered, options.concurrency ?? DEFAULT_CONCURRENCY, async (d, index) => {
+			const id = `${d.agent}/${d.id}`;
+			emit({
+				type: "start",
+				id,
+				index,
+				total
+			});
+			const result = await runOne(d, id, runId, options);
+			emit({
+				type: "result",
+				result,
+				index,
+				total
+			});
+			return result;
+		});
+		const summary = { results };
+		const mlflow = await finalizeMlflow(mlflowClient, runId, results, options);
+		if (mlflow) summary.mlflow = mlflow;
+		return summary;
+	} finally {
+		teardownJudge();
+	}
+}
+
+//#endregion
+export { runEvalsInDir };
+//# sourceMappingURL=run-evals.js.map

package/dist/evals/run-evals.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-evals.js","names":[],"sources":["../../src/evals/run-evals.ts"],"sourcesContent":["import { pathToFileURL } from \"node:url\";\n\nimport { MlflowClient } from \"../connectors/mlflow\";\nimport { type DiscoveredEval, discoverEvalFiles } from \"./discover\";\nimport { createHttpDriver } from \"./http-driver\";\nimport { configureJudge, teardownJudge } from \"./judge\";\nimport { type ReportOutcome, reportToMlflow } from \"./mlflow-report\";\nimport { createEvalRun, type FinishOutcome, finishEvalRun } from \"./mlflow-run\";\nimport { mapPool } from \"./pool\";\nimport { runEval } from \"./run-eval\";\nimport type { EvalDefinition, EvalResult } from \"./types\";\n\nexport interface RunEvalsOptions {\n  /** Project root containing `server/agents/`. Defaults to `process.cwd()`. */\n  rootDir?: string;\n  /** Base URL of the running app to drive, e.g. `http://localhost:3000`. */\n  baseUrl: string;\n  /** Substring filter on `<agent>/<id>` (or an exact agent id). */\n  filter?: string;\n  /** Soft assertion failures also fail the eval. */\n  strict?: boolean;\n  /** Extra request headers for the driver (e.g. auth for a deployed app). */\n  headers?: Record<string, string>;\n  /** Per-turn wall-clock timeout (ms) before a turn is failed. Defaults to 120s. */\n  timeoutMs?: number;\n  /**\n   * Max evals to drive concurrently. Each eval opens one stream to the app as\n   * the same user, so keep this at or below the app's\n   * `maxConcurrentStreamsPerUser` (default 5) or the surplus streams hit the\n   * 429 guard. Defaults to 4; clamped to `[1, total]`.\n   */\n  concurrency?: number;\n  /**\n   * When set, create a native MLflow \"Evaluation run\": each eval's trace is\n   * linked to the run, pass/fail is written as feedback, and aggregate metrics\n   * are logged. Requires Databricks creds + the target experiment.\n   */\n  mlflow?: {\n    host: string;\n    token: string;\n    experimentId: string;\n    /** SQL warehouse id for writing assessments to UC-backed (V4) traces. */\n    sqlWarehouseId?: string;\n  };\n  /**\n   * When set, enable `t.judge.*` LLM-as-judge scoring via autoevals against a\n   * Databricks serving endpoint (`model`).\n   */\n  judge?: { host: string; token: string; model: string };\n  /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */\n  now?: number;\n  /** Progress callback, invoked as evals are discovered, started, and finished. */\n  onEvent?: (event: EvalProgress) => void;\n}\n\nexport type EvalProgress =\n  | { type: \"discovered\"; total: number }\n  | { type: \"run-created\"; runId: string }\n  | { type: \"start\"; id: string; index: number; total: number }\n  | { type: \"result\"; result: EvalResult; index: number; total: number };\n\nexport interface EvalRunSummary {\n  results: EvalResult[];\n  /** Present when an MLflow evaluation run was created. */\n  mlflow?: { runId: string; report: ReportOutcome; finish: FinishOutcome };\n}\n\n/**\n * Load a `*.eval.ts` file and return its default-exported {@link EvalDefinition}.\n * Uses tsx's programmatic loader so TypeScript eval files run without a build\n * step. The specifier is indirected so the type checker doesn't try to resolve\n * tsx's internal entry.\n */\nasync function loadEval(file: string): Promise<EvalDefinition> {\n  const tsxApi = \"tsx/esm/api\";\n  let tsImport: (specifier: string, parentURL: string) => Promise<unknown>;\n  try {\n    ({ tsImport } = (await import(tsxApi)) as {\n      tsImport: (specifier: string, parentURL: string) => Promise<unknown>;\n    });\n  } catch {\n    throw new Error(\n      \"Running .eval.ts files requires `tsx`. Install it as a dev dependency (`pnpm add -D tsx`).\",\n    );\n  }\n\n  const mod = await tsImport(pathToFileURL(file).href, import.meta.url);\n  const def = resolveEvalDefault(mod);\n  if (!def) {\n    throw new Error(`${file}: must default-export defineEval({ test })`);\n  }\n  return def;\n}\n\n/**\n * Unwrap the eval default export across module-interop shapes. Depending on\n * whether the eval file is treated as ESM or CJS, the value lands at\n * `mod.default` (ESM), `mod.default.default` (CJS `__esModule` double-wrap), or\n * `mod` itself. Returns the first candidate that looks like an eval.\n */\nexport function resolveEvalDefault(mod: unknown): EvalDefinition | undefined {\n  let candidate: unknown = mod;\n  for (let i = 0; i < 4 && candidate; i++) {\n    if (typeof (candidate as EvalDefinition).test === \"function\") {\n      return candidate as EvalDefinition;\n    }\n    candidate = (candidate as { default?: unknown }).default;\n  }\n  return undefined;\n}\n\n/**\n * Load and run a single discovered eval. Never throws — a load/run failure\n * becomes a non-passing {@link EvalResult} so one bad eval can't abort the run.\n */\nasync function runOne(\n  d: DiscoveredEval,\n  id: string,\n  runId: string | undefined,\n  options: RunEvalsOptions,\n): Promise<EvalResult> {\n  try {\n    const def = await loadEval(d.file);\n    const driver = createHttpDriver({\n      baseUrl: options.baseUrl,\n      agent: def.agent ?? d.agent,\n      headers: options.headers,\n      mlflowRunId: runId,\n      timeoutMs: options.timeoutMs,\n    });\n    return await runEval(def, { id, driver, strict: options.strict });\n  } catch (err) {\n    return {\n      id,\n      assertions: [],\n      passed: false,\n      error: err instanceof Error ? err.message : String(err),\n    };\n  }\n}\n\n/** Configure the LLM judge when judge creds were supplied; otherwise a no-op. */\nasync function maybeConfigureJudge(options: RunEvalsOptions): Promise<void> {\n  if (!options.judge) return;\n  await configureJudge({\n    client: new MlflowClient(options.judge.host, options.judge.token),\n    token: options.judge.token,\n    model: options.judge.model,\n  });\n}\n\n/**\n * Report per-eval assessments and finish the MLflow run, when one was created.\n * Returns the run summary, or `undefined` when there was no run to finalize.\n */\nasync function finalizeMlflow(\n  client: MlflowClient | undefined,\n  runId: string | undefined,\n  results: EvalResult[],\n  options: RunEvalsOptions,\n): Promise<EvalRunSummary[\"mlflow\"]> {\n  if (!client || !runId) return undefined;\n  // reportToMlflow is not supposed to throw, but if it ever does the run must\n  // still be finished — otherwise it hangs in RUNNING forever.\n  let report: ReportOutcome = { written: 0, skipped: 0, failures: [] };\n  try {\n    report = await reportToMlflow(\n      client,\n      results,\n      options.mlflow?.sqlWarehouseId,\n    );\n  } catch (err) {\n    report.failures.push({\n      traceId: \"(report)\",\n      error: err instanceof Error ? err.message : String(err),\n    });\n  }\n  const finish = await finishEvalRun(client, {\n    runId,\n    results,\n    endTime: options.now ?? Date.now(),\n  });\n  return { runId, report, finish };\n}\n\n/**\n * Default max evals in flight. Each eval opens one stream to the app as the\n * same user; the server caps concurrent streams per user at 5 by default\n * (`maxConcurrentStreamsPerUser`), so 4 leaves headroom under that limit.\n */\nconst DEFAULT_CONCURRENCY = 4;\n\n/**\n * Discover, load, and run every eval under each agent's `evals/` dir, driving\n * the agents on a running app. Never throws for an individual eval — load/run\n * failures become non-passing {@link EvalResult}s.\n */\nexport async function runEvalsInDir(\n  options: RunEvalsOptions,\n): Promise<EvalRunSummary> {\n  const root = options.rootDir ?? process.cwd();\n  const now = options.now ?? Date.now();\n  let discovered = discoverEvalFiles(root);\n\n  if (options.filter) {\n    const f = options.filter;\n    discovered = discovered.filter(\n      (d) => d.agent === f || `${d.agent}/${d.id}`.includes(f),\n    );\n  }\n\n  const emit = options.onEvent ?? (() => {});\n  const total = discovered.length;\n  emit({ type: \"discovered\", total });\n\n  // The judge sets OPENAI_* env vars globally (autoevals reads them per call),\n  // so tear them down in `finally` once the run is over — pass or throw — so\n  // the bearer doesn't linger in process.env.\n  await maybeConfigureJudge(options);\n  try {\n    // Create the MLflow evaluation run up front so each eval's trace can be\n    // linked to it as it runs. One client is shared by run create/finish and\n    // the per-trace assessment writes.\n    let runId: string | undefined;\n    let mlflowClient: MlflowClient | undefined;\n    if (options.mlflow) {\n      mlflowClient = new MlflowClient(\n        options.mlflow.host,\n        options.mlflow.token,\n      );\n      runId = await createEvalRun(mlflowClient, {\n        experimentId: options.mlflow.experimentId,\n        runName: `appkit-eval ${new Date(now).toISOString()}`,\n        startTime: now,\n      });\n      emit({ type: \"run-created\", runId });\n    }\n\n    // Run evals through a bounded pool so independent turns overlap instead of\n    // summing their latencies. runOne never throws, so a pool worker never\n    // rejects; results preserve discovery order (mapPool writes by index).\n    const results = await mapPool(\n      discovered,\n      options.concurrency ?? DEFAULT_CONCURRENCY,\n      async (d, index) => {\n        const id = `${d.agent}/${d.id}`;\n        emit({ type: \"start\", id, index, total });\n        const result = await runOne(d, id, runId, options);\n        emit({ type: \"result\", result, index, total });\n        return result;\n      },\n    );\n\n    const summary: EvalRunSummary = { results };\n    const mlflow = await finalizeMlflow(mlflowClient, runId, results, options);\n    if (mlflow) summary.mlflow = mlflow;\n    return summary;\n  } finally {\n    teardownJudge();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAyEA,eAAe,SAAS,MAAuC;CAC7D,MAAM,SAAS;CACf,IAAI;AACJ,KAAI;AACF,GAAC,CAAE,YAAc,MAAM,OAAO;SAGxB;AACN,QAAM,IAAI,MACR,6FACD;;CAIH,MAAM,MAAM,mBADA,MAAM,SAAS,cAAc,KAAK,CAAC,MAAM,OAAO,KAAK,IAAI,CAClC;AACnC,KAAI,CAAC,IACH,OAAM,IAAI,MAAM,GAAG,KAAK,4CAA4C;AAEtE,QAAO;;;;;;;;AAST,SAAgB,mBAAmB,KAA0C;CAC3E,IAAI,YAAqB;AACzB,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,WAAW,KAAK;AACvC,MAAI,OAAQ,UAA6B,SAAS,WAChD,QAAO;AAET,cAAa,UAAoC;;;;;;;AASrD,eAAe,OACb,GACA,IACA,OACA,SACqB;AACrB,KAAI;EACF,MAAM,MAAM,MAAM,SAAS,EAAE,KAAK;AAQlC,SAAO,MAAM,QAAQ,KAAK;GAAE;GAAI,QAPjB,iBAAiB;IAC9B,SAAS,QAAQ;IACjB,OAAO,IAAI,SAAS,EAAE;IACtB,SAAS,QAAQ;IACjB,aAAa;IACb,WAAW,QAAQ;IACpB,CAAC;GACsC,QAAQ,QAAQ;GAAQ,CAAC;UAC1D,KAAK;AACZ,SAAO;GACL;GACA,YAAY,EAAE;GACd,QAAQ;GACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACxD;;;;AAKL,eAAe,oBAAoB,SAAyC;AAC1E,KAAI,CAAC,QAAQ,MAAO;AACpB,OAAM,eAAe;EACnB,QAAQ,IAAI,aAAa,QAAQ,MAAM,MAAM,QAAQ,MAAM,MAAM;EACjE,OAAO,QAAQ,MAAM;EACrB,OAAO,QAAQ,MAAM;EACtB,CAAC;;;;;;AAOJ,eAAe,eACb,QACA,OACA,SACA,SACmC;AACnC,KAAI,CAAC,UAAU,CAAC,MAAO,QAAO;CAG9B,IAAI,SAAwB;EAAE,SAAS;EAAG,SAAS;EAAG,UAAU,EAAE;EAAE;AACpE,KAAI;AACF,WAAS,MAAM,eACb,QACA,SACA,QAAQ,QAAQ,eACjB;UACM,KAAK;AACZ,SAAO,SAAS,KAAK;GACnB,SAAS;GACT,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GACxD,CAAC;;CAEJ,MAAM,SAAS,MAAM,cAAc,QAAQ;EACzC;EACA;EACA,SAAS,QAAQ,OAAO,KAAK,KAAK;EACnC,CAAC;AACF,QAAO;EAAE;EAAO;EAAQ;EAAQ;;;;;;;AAQlC,MAAM,sBAAsB;;;;;;AAO5B,eAAsB,cACpB,SACyB;CACzB,MAAM,OAAO,QAAQ,WAAW,QAAQ,KAAK;CAC7C,MAAM,MAAM,QAAQ,OAAO,KAAK,KAAK;CACrC,IAAI,aAAa,kBAAkB,KAAK;AAExC,KAAI,QAAQ,QAAQ;EAClB,MAAM,IAAI,QAAQ;AAClB,eAAa,WAAW,QACrB,MAAM,EAAE,UAAU,KAAK,GAAG,EAAE,MAAM,GAAG,EAAE,KAAK,SAAS,EAAE,CACzD;;CAGH,MAAM,OAAO,QAAQ,kBAAkB;CACvC,MAAM,QAAQ,WAAW;AACzB,MAAK;EAAE,MAAM;EAAc;EAAO,CAAC;AAKnC,OAAM,oBAAoB,QAAQ;AAClC,KAAI;EAIF,IAAI;EACJ,IAAI;AACJ,MAAI,QAAQ,QAAQ;AAClB,kBAAe,IAAI,aACjB,QAAQ,OAAO,MACf,QAAQ,OAAO,MAChB;AACD,WAAQ,MAAM,cAAc,cAAc;IACxC,cAAc,QAAQ,OAAO;IAC7B,SAAS,eAAe,IAAI,KAAK,IAAI,CAAC,aAAa;IACnD,WAAW;IACZ,CAAC;AACF,QAAK;IAAE,MAAM;IAAe;IAAO,CAAC;;EAMtC,MAAM,UAAU,MAAM,QACpB,YACA,QAAQ,eAAe,qBACvB,OAAO,GAAG,UAAU;GAClB,MAAM,KAAK,GAAG,EAAE,MAAM,GAAG,EAAE;AAC3B,QAAK;IAAE,MAAM;IAAS;IAAI;IAAO;IAAO,CAAC;GACzC,MAAM,SAAS,MAAM,OAAO,GAAG,IAAI,OAAO,QAAQ;AAClD,QAAK;IAAE,MAAM;IAAU;IAAQ;IAAO;IAAO,CAAC;AAC9C,UAAO;IAEV;EAED,MAAM,UAA0B,EAAE,SAAS;EAC3C,MAAM,SAAS,MAAM,eAAe,cAAc,OAAO,SAAS,QAAQ;AAC1E,MAAI,OAAQ,SAAQ,SAAS;AAC7B,SAAO;WACC;AACR,iBAAe"}

package/dist/evals/types.d.ts

@@ -0,0 +1,127 @@
+//#region src/evals/types.d.ts
+/**
+ * Agent evaluation primitives — an eve-style authoring API that runs against
+ * AppKit agents and reports to Databricks MLflow.
+ *
+ * Evals live in `server/agents/<id>/evals/*.eval.ts`, each default-exporting a
+ * {@link EvalDefinition} via {@link defineEval}. A runner drives the agent
+ * (today over HTTP against a running app), and the `test` function asserts on
+ * the reply and tool usage with deterministic matchers (and, later, LLM judges).
+ */
+/** Result of a deterministic matcher run against a value. */
+interface MatchResult {
+  pass: boolean;
+  /** Optional 0..1 score for scored matchers (similarity, judges). */
+  score?: number;
+  /** Human-readable explanation, shown on failure. */
+  detail?: string;
+}
+/** A deterministic matcher: inspects a string value and returns a result. */
+type Matcher = (value: string) => MatchResult;
+/** Whether an assertion fails the eval (`gate`) or is tracked only (`soft`). */
+type Severity = "gate" | "soft";
+/** A single recorded assertion outcome. */
+interface AssertionResult {
+  label: string;
+  severity: Severity;
+  pass: boolean;
+  score?: number;
+  detail?: string;
+}
+/**
+ * Chainable handle returned by every assertion to control its severity.
+ * Mirrors eve: assertions are gates by default; `.soft()` demotes to a tracked
+ * metric; `.atLeast(n)` is a soft, score-thresholded assertion.
+ */
+interface AssertionHandle {
+  /** Promote to a hard gate — failure fails the eval (non-zero exit). */
+  gate(): AssertionHandle;
+  /** Demote to a tracked metric — doesn't fail unless running with `strict`. */
+  soft(): AssertionHandle;
+  /** Soft assertion that passes only when the score is at least `threshold`. */
+  atLeast(threshold: number): AssertionHandle;
+}
+/** What a driver returns for a single `t.send`. */
+interface DriveResult {
+  /** The final assistant message text. */
+  reply: string;
+  /** Names of tools the agent called during the turn. */
+  toolCalls: string[];
+  /** Whether the turn completed without an agent/stream error. */
+  succeeded: boolean;
+  /** Thread/session id, when the driver exposes one. */
+  sessionId?: string;
+  /** MLflow trace id for the turn, when tracing is enabled on the app. */
+  traceId?: string;
+}
+/**
+ * Abstraction over how the agent is driven. The HTTP driver posts to a running
+ * app's agents endpoint; future drivers (in-process) implement the same shape.
+ */
+interface EvalDriver {
+  send(message: string): Promise<DriveResult>;
+}
+/** The `t` context passed to an eval's `test` function. */
+interface TestContext {
+  /** Send a user message to the agent and capture its response. */
+  send(message: string): Promise<void>;
+  /** The last assistant reply. */
+  readonly reply: string;
+  /** Tools called during the last turn. */
+  readonly toolCalls: string[];
+  /** The current session/thread id, if any. */
+  readonly sessionId: string | undefined;
+  /** Assert the last turn completed successfully (gate by default). */
+  succeeded(): AssertionHandle;
+  /** Assert a tool was called during the run (gate by default). */
+  calledTool(name: string): AssertionHandle;
+  /** Assert a value against a matcher, e.g. `t.check(t.reply, includes("Sunny"))`. */
+  check(value: string, matcher: Matcher): AssertionHandle;
+  /**
+   * LLM-as-judge scoring of the last reply (via autoevals → a Databricks judge
+   * model). Each returns a scored, soft-by-default assertion; chain `.atLeast(n)`
+   * to set the pass threshold or `.gate()` to make it a hard gate. Requires the
+   * judge to be configured (`--judge-model`).
+   */
+  judge: {
+    /** Score factuality of the reply against an expected reference. */factuality(expected: string): Promise<AssertionHandle>; /** Score whether the reply answers the question, per optional `criteria`. */
+    closedQA(criteria: string): Promise<AssertionHandle>; /** A custom prompt-template judge (the TS analog of MLflow's `@scorer`). */
+    custom(spec: CustomJudgeSpec): Promise<AssertionHandle>;
+  };
+  /** Skip this eval with an optional reason. */
+  skip(reason?: string): never;
+}
+/** A custom LLM-judge definition: a prompt template and choice→score mapping. */
+interface CustomJudgeSpec {
+  name: string;
+  promptTemplate: string;
+  choiceScores: Record<string, number>;
+}
+/** A single eval, default-exported from a `*.eval.ts` file. */
+interface EvalDefinition {
+  /** Short human description, shown in reports. */
+  description?: string;
+  /** Target agent id. Defaults to the eval's parent `server/agents/<id>` dir. */
+  agent?: string;
+  /** The eval body: drive the agent and assert on its behavior. */
+  test(t: TestContext): Promise<void> | void;
+}
+/** The outcome of running one eval. */
+interface EvalResult {
+  id: string;
+  description?: string;
+  /** Set when the eval called `t.skip`. */
+  skipped?: {
+    reason?: string;
+  };
+  assertions: AssertionResult[];
+  /** True when all gates passed (and, under strict, all soft assertions too). */
+  passed: boolean;
+  /** Set when the eval threw before completing. */
+  error?: string;
+  /** MLflow trace id of the eval's last turn, for attaching assessments. */
+  traceId?: string;
+}
+//#endregion
+export { AssertionHandle, AssertionResult, CustomJudgeSpec, DriveResult, EvalDefinition, EvalDriver, EvalResult, MatchResult, Matcher, Severity, TestContext };
+//# sourceMappingURL=types.d.ts.map

package/dist/evals/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","names":[],"sources":["../../src/evals/types.ts"],"mappings":";;AAWA;;;;;;;;;UAAiB,WAAA;EACf,IAAA;;EAEA,KAAA;EAMkD;EAJlD,MAAA;AAAA;;KAIU,OAAA,IAAW,KAAA,aAAkB,WAAA;;KAG7B,QAAA;;UAGK,eAAA;EACf,KAAA;EACA,QAAA,EAAU,QAAA;EACV,IAAA;EACA,KAAA;EACA,MAAA;AAAA;;;;AAQF;;UAAiB,eAAA;EAEP;EAAR,IAAA,IAAQ,eAAA;EAIoB;EAF5B,IAAA,IAAQ,eAAA;EAEmC;EAA3C,OAAA,CAAQ,SAAA,WAAoB,eAAA;AAAA;;UAIb,WAAA;EAJf;EAMA,KAAA;EAN4B;EAQ5B,SAAA;EAR2C;EAU3C,SAAA;EAN0B;EAQ1B,SAAA;EAR0B;EAU1B,OAAA;AAAA;;;;;UAOe,UAAA;EACf,IAAA,CAAK,OAAA,WAAkB,OAAA,CAAQ,WAAA;AAAA;;UAIhB,WAAA;EAJf;EAMA,IAAA,CAAK,OAAA,WAAkB,OAAA;EANA;EAAA,SAQd,KAAA;EARiC;EAAA,SAUjC,SAAA;EANM;EAAA,SAQN,SAAA;;EAET,SAAA,IAAa,eAAA;EAAA;EAEb,UAAA,CAAW,IAAA,WAAe,eAAA;EAEI;EAA9B,KAAA,CAAM,KAAA,UAAe,OAAA,EAAS,OAAA,GAAU,eAAA;EASA;;;;;;EAFxC,KAAA;IAMwC,mEAJtC,UAAA,CAAW,QAAA,WAAmB,OAAA,CAAQ,eAAA,GArBxC;IAuBE,QAAA,CAAS,QAAA,WAAmB,OAAA,CAAQ,eAAA,GAvBf;IAyBrB,MAAA,CAAO,IAAA,EAAM,eAAA,GAAkB,OAAA,CAAQ,eAAA;EAAA;EAnBhC;EAsBT,IAAA,CAAK,MAAA;AAAA;;UAIU,eAAA;EACf,IAAA;EACA,cAAA;EACA,YAAA,EAAc,MAAA;AAAA;;UAIC,cAAA;EApBf;EAsBA,WAAA;EApBa;EAsBb,KAAA;EAtBwC;EAwBxC,IAAA,CAAK,CAAA,EAAG,WAAA,GAAc,OAAA;AAAA;;UAIP,UAAA;EACf,EAAA;EACA,WAAA;EA1BS;EA4BT,OAAA;IAAY,MAAA;EAAA;EACZ,UAAA,EAAY,eAAA;EA1BQ;EA4BpB,MAAA;EAxBe;EA0Bf,KAAA;;EAEA,OAAA;AAAA"}

package/dist/plugins/agents/agents.js

@@ -17,8 +17,8 @@ import { functionToolToDefinition, isFunctionTool } from "../../core/agent/tools
 import { isHostedTool, resolveHostedTools } from "../../core/agent/tools/hosted-tools.js";
 import { isToolkitEntry } from "../../core/agent/types.js";
 import "../../core/agent/tools/index.js";
-import { loadAgentsFromDir } from "../../core/agent/load-agents.js";
 import { CODE_AGENTS_SOURCE_DIR } from "../../core/agent/load-code-agents.js";
+import { loadAgentsFromDir } from "../../core/agent/load-agents.js";
 import { ActiveStreamTracker } from "./active-stream-tracker.js";
 import { buildAdapterExtensions, supervisorToolDescription, warnOnCapabilityMismatch } from "./adapter-extensions.js";
 import { requiresApproval } from "./approval.js";