@alexkroman1/aai-cli 7.0.0 → 8.1.0

package/dist/{deploy-DvYeD8ia.mjs → deploy-DRC8jWr8.mjs}

@@ -3,7 +3,7 @@ import { a as ok } from "./_output-CKkmWs7i.mjs";
 import { n as log, r as notify, t as fmtUrl } from "./_ui-u7T4YooX.mjs";
 import { o as errorMessage$1 } from "./_utils-B8QmtFhK.mjs";
 import { s as updateProjectConfig } from "./_config-CmJOFsAP.mjs";
-import { i as evalWorkerConfig, t as buildAgentBundle } from "./_bundler-B4RqNF5Z.mjs";
+import { i as evalWorkerConfig, t as buildAgentBundle } from "./_bundler-CDuYl5Gb.mjs";
 import { t as resolveServerEnv } from "./_server-common-6e0QI4mq.mjs";
 import { a as resolveDeployTarget } from "./_agent-BTwZJu4P.mjs";
 import { assertTypechecks } from "./_typecheck-gate-BXvUNnfs.mjs";

package/dist/{dev-B59JIfWO.mjs → dev-QuuOOUtv.mjs}

@@ -12,7 +12,7 @@ import { styleText } from "node:util";
 async function executeDev(opts) {
 	const port = parsePort(opts.port);
 	const agentName = path.basename(path.resolve(opts.cwd));
-	const { startDevServer } = await import("./_dev-server-CiKFH9qw.mjs");
+	const { startDevServer } = await import("./_dev-server-Dt3TCjhj.mjs");
 	let cleanup;
 	let shuttingDown = false;
 	const onSignal = () => {

package/dist/eval-DfQ8cm5p.mjs

@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+import { a as ok, n as fail } from "./_output-CKkmWs7i.mjs";
+import { n as log } from "./_ui-u7T4YooX.mjs";
+import { t as resolveServerEnv } from "./_server-common-6e0QI4mq.mjs";
+import { classifyVitestError, runVitest } from "./test-BvyT6sAc.mjs";
+//#region eval.ts
+/**
+* `aai eval` — run the agent's behaviour evals via vitest.
+*
+* An eval is not a test, and this is a separate command rather than part of
+* `aai test` for the three reasons that follow from that:
+*
+* - **It spends money.** Every case drives a real LLM on a real key, so it
+*   cannot run on every save, in `aai build`, or in a pre-commit hook the way
+*   the unit tests do.
+* - **It is a noisy instrument.** A behaviour eval measures a probabilistic
+*   system: identical code has scored 0.56 and 0.60 on the same task set with
+*   9 of 25 tasks flipping outcome. A green run is weaker evidence than a green
+*   test run, and a red one is not automatically a regression. Mixing the two
+*   into one verdict devalues the reliable half.
+* - **It is slow by construction** — one model turn per utterance. Hence the
+*   per-case budget below rather than vitest's 5s default, which no case that
+*   waits on a live reply can meet.
+*
+* What it runs is an ordinary vitest file, `agent.eval.test.ts`, which is why
+* there is no eval DSL here: the project already has a runner, and the harness
+* that makes an agent drivable from text is published
+* (`@alexkroman1/aai-runtime/eval`).
+*/
+/** The files `aai eval` runs, in preference order. */
+const EVAL_FILES = ["agent.eval.test.ts", "agent.eval.test.js"];
+/** Execute the agent's evals and return a structured result. */
+async function executeEval(cwd) {
+	log.step("Running agent evals");
+	const env = await resolveServerEnv(cwd);
+	try {
+		if (!runVitest(cwd, {
+			candidates: EVAL_FILES,
+			extraArgs: ["--testTimeout", String(3e5)],
+			env
+		})) {
+			log.info("No eval file found. Create agent.eval.test.ts to measure what the agent does — see `openEvalSession` in @alexkroman1/aai-runtime/eval.");
+			return ok({
+				passed: true,
+				skipped: true
+			});
+		}
+		log.success("Evals passed");
+		return ok({ passed: true });
+	} catch (err) {
+		const { code, message } = classifyVitestError(err, "Evals");
+		return fail(code, message);
+	}
+}
+//#endregion
+export { executeEval };

package/dist/eval.d.ts

@@ -0,0 +1,43 @@
+/**
+ * `aai eval` — run the agent's behaviour evals via vitest.
+ *
+ * An eval is not a test, and this is a separate command rather than part of
+ * `aai test` for the three reasons that follow from that:
+ *
+ * - **It spends money.** Every case drives a real LLM on a real key, so it
+ *   cannot run on every save, in `aai build`, or in a pre-commit hook the way
+ *   the unit tests do.
+ * - **It is a noisy instrument.** A behaviour eval measures a probabilistic
+ *   system: identical code has scored 0.56 and 0.60 on the same task set with
+ *   9 of 25 tasks flipping outcome. A green run is weaker evidence than a green
+ *   test run, and a red one is not automatically a regression. Mixing the two
+ *   into one verdict devalues the reliable half.
+ * - **It is slow by construction** — one model turn per utterance. Hence the
+ *   per-case budget below rather than vitest's 5s default, which no case that
+ *   waits on a live reply can meet.
+ *
+ * What it runs is an ordinary vitest file, `agent.eval.test.ts`, which is why
+ * there is no eval DSL here: the project already has a runner, and the harness
+ * that makes an agent drivable from text is published
+ * (`@alexkroman1/aai-runtime/eval`).
+ */
+import { type CommandResult } from "./_output.ts";
+type EvalData = {
+    passed: boolean;
+    skipped?: boolean;
+};
+/** The files `aai eval` runs, in preference order. */
+export declare const EVAL_FILES: readonly ["agent.eval.test.ts", "agent.eval.test.js"];
+/**
+ * How long one eval case may run.
+ *
+ * Vitest's default is 5s, which is shorter than a single live model turn on a
+ * slow day — so without this every case fails as a timeout and the report says
+ * nothing about the agent. Generous rather than tuned: the harness has its own
+ * 90s per-turn timeout with a message naming the events it saw, and that is the
+ * diagnostic worth reaching. This ceiling only exists so a wedged run ends.
+ */
+export declare const EVAL_TEST_TIMEOUT_MS = 300000;
+/** Execute the agent's evals and return a structured result. */
+export declare function executeEval(cwd: string): Promise<CommandResult<EvalData>>;
+export {};

package/dist/{init-CpJVjEg0.mjs → init-CUo87gcI.mjs}

@@ -105,7 +105,7 @@ function collectWarnings() {
 }
 /** Publish after init and return deploy metadata if successful. */
 async function tryPublish(cwd, server, warn) {
-	const { executePublish } = await import("./studio--MUV0cid.mjs");
+	const { executePublish } = await import("./studio-CzTzPqe1.mjs");
 	try {
 		const result = await executePublish({
 			cwd,
@@ -126,7 +126,7 @@ async function tryPublish(cwd, server, warn) {
 }
 /** Scaffold the project, optionally showing a spinner. */
 async function scaffoldProject(dir, cwd, template, silent) {
-	const { runInit } = await import("./_init-CE4HKdgV.mjs");
+	const { runInit } = await import("./_init-BhEfQ6Yi.mjs");
 	await withSpinner(silent, {
 		start: `Creating ${dir}`,
 		done: "Project created",

package/dist/{logs-9q-psa4q.mjs → logs-CQ2T46dr.mjs}

@@ -2,7 +2,7 @@
 import { a as ok } from "./_output-CKkmWs7i.mjs";
 import { n as log } from "./_ui-u7T4YooX.mjs";
 import { f as checkedResponse } from "./_studio-DR5yzZ1T.mjs";
-import { n as slugRequest } from "./_slug-api-nRFaBEPJ.mjs";
+import { n as slugRequest } from "./_slug-api-z-USxPtL.mjs";
 import { isRecord, omitUndefined } from "@alexkroman1/aai/utils";
 import { sleep } from "@alexkroman1/aai/internal";
 function isLogsPage(value) {