anpord 0.1.12 → 0.1.13

package/dist/bin.mjs

@@ -1,15 +1,16 @@
 #!/usr/bin/env node
-import { a as ChannelName, o as PromptId, s as VersionNumber, t as AnpordApi } from "./client-D22IiHGL.mjs";
+import { a as ChannelName, o as PromptId, s as VersionNumber, t as AnpordApi } from "./client-BfdjFtlV.mjs";
+import { C as EvalTrigger, s as EvalRun } from "./evals-Dcc_x6jW.mjs";
 import { i as tokenMatcher, r as asAnpordError } from "./errors-B0YknR5V.mjs";
 import { ClientLayer, webUrlConfig } from "./config.mjs";
-import { n as compileEvalEffect } from "./compiler-CBMPXIzV.mjs";
+import { n as compileEvalEffect } from "./compiler-D96wI7UP.mjs";
 import { Args, Command, Options } from "@effect/cli";
 import { FetchHttpClient, FileSystem, HttpClient, HttpClientError, HttpClientRequest } from "@effect/platform";
 import { NodeContext, NodeRuntime } from "@effect/platform-node";
 import { Cause, Clock, Config, Console, Data, Duration, Effect, Layer, Option, ParseResult, Redacted, Ref, Schema } from "effect";
 import { parseDocument } from "yaml";
 //#region package.json
-var version$1 = "0.1.12";
+var version$1 = "0.1.13";
 //#endregion
 //#region ../template/src/extract.ts
 /** Reads escapes too, so text the renderer leaves literal is never reported as
@@ -58,10 +59,11 @@ const evalFilesIn = (directory) => Effect.gen(function* () {
 }).pipe(Effect.withSpan("Cli.evalFilesIn"));
 //#endregion
 //#region src/cli/eval-gate.ts
+const EvalGate = Schema.Literal("strict", "never", "regressed", "unscored");
 const regressions = (run) => run.cells.filter((cell) => cell.comparison?.verdict === "regressed");
 const unscored = (run) => run.cells.filter((cell) => (cell.distribution?.scored ?? 0) === 0);
 const rate = (value) => `${Math.round(value * 100) / 100}`;
-const versionClause$1 = (run, cell, found) => found.baselineHarnessVersion === found.candidateHarnessVersion ? "" : `${run.tasks[cell.taskIndex]?.harness ?? "harness"} ${found.baselineHarnessVersion} → ${found.candidateHarnessVersion}, `;
+const versionClause = (run, cell, found) => found.baselineHarnessVersion === found.candidateHarnessVersion ? "" : `${run.tasks[cell.taskIndex]?.harness ?? "harness"} ${found.baselineHarnessVersion} → ${found.candidateHarnessVersion}, `;
 const profileClause = (run, cell, found) => {
 	const { baselineProfileVersion, candidateProfileVersion } = found;
 	if (baselineProfileVersion === null || candidateProfileVersion === null || baselineProfileVersion === candidateProfileVersion) return "";
@@ -70,10 +72,20 @@ const profileClause = (run, cell, found) => {
 const regressionSentence = (run, cell) => {
 	const found = cell.comparison;
 	if (found === null) return `${cell.caseName} regressed against its baseline.`;
-	return `${cell.caseName} regressed against its baseline: ${versionClause$1(run, cell, found)}${profileClause(run, cell, found)}pass rate ${rate(found.baselinePassRate)} → ${rate(found.candidatePassRate)}.`;
+	return `${cell.caseName} regressed against its baseline: ${versionClause(run, cell, found)}${profileClause(run, cell, found)}pass rate ${rate(found.baselinePassRate)} → ${rate(found.candidatePassRate)}.`;
 };
-const problemsWith = (run, failOn) => {
+const problemsWith = (run, failOn, expected) => {
 	if (run.status === "failed") return [run.failure ?? "The run failed."];
+	if (run.status !== "finished") return ["The run has not finished."];
+	if (run.cells.length === 0) return ["The run recorded no cells."];
+	if (failOn === "strict") {
+		if (expected && run.cells.length !== expected.cells) return [`Expected ${expected.cells} cells, received ${run.cells.length}.`];
+		return run.cells.flatMap((cell) => {
+			if (cell.status !== "finished" || cell.trials.length === 0) return [`${cell.caseName} has no complete trial results.`];
+			if (expected && cell.trials.length !== expected.trials) return [`${cell.caseName}: expected ${expected.trials} trials, received ${cell.trials.length}.`];
+			return cell.trials.flatMap((trial) => trial.status === "passed" && trial.passed ? [] : [`${cell.caseName}, trial ${trial.ordinal}: ${trial.status}.`]);
+		});
+	}
 	if (failOn === "never") return [];
 	const found = regressions(run).map((cell) => regressionSentence(run, cell));
 	return failOn === "unscored" ? [...found, ...unscored(run).map((cell) => `${cell.caseName} produced no scored trials.`)] : found;
@@ -115,43 +127,43 @@ const DONE = "●";
 const RUNNING = "◐";
 const FILLED = "▰";
 const HOLLOW = "▱";
-const PERCENT$1 = 100;
+const PERCENT = 100;
 const SECONDS = 1e3;
 const MINUTE = 60;
 const paint = (colour, text) => `${colour}${text}${RESET}`;
-const elapsedOf = (ms) => {
+const formatElapsed = (ms) => {
 	const total = Math.floor(ms / SECONDS);
 	const minutes = Math.floor(total / MINUTE);
 	return minutes === 0 ? `${total}s` : `${minutes}m${String(total % MINUTE).padStart(2, "0")}s`;
 };
-const markOf = (cell) => {
+const formatStatus = (cell) => {
 	if (cell.status === "finished") return paint(GREEN, DONE);
 	return cell.status === "failed" ? paint(RED, DONE) : paint(YELLOW, RUNNING);
 };
-const trialsOf = (cell, trials) => {
+const formatTrialProgress = (cell, trials) => {
 	const settled = cell.trials.filter((trial) => trial.status !== "queued" && trial.status !== "running").length;
 	return `${FILLED.repeat(settled)}${paint(DIM, HOLLOW.repeat(Math.max(0, trials - settled)))}`;
 };
-const rateOf$1 = (cell) => {
+const formatPassRate = (cell) => {
 	const rate = cell.distribution?.passRate;
 	if (rate === void 0 || cell.distribution?.scored === 0) return paint(DIM, "—");
-	const shown = `${Math.round(rate * PERCENT$1)}%`;
+	const shown = `${Math.round(rate * PERCENT)}%`;
 	return paint(rate === 1 ? GREEN : RED, shown);
 };
-const variantOf = (run, cell) => {
+const formatVariant = (run, cell) => {
 	const task = run.tasks[cell.taskIndex];
 	return task === void 0 ? "?" : `${task.harness}/${task.model}`;
 };
-const widest = (run) => run.cells.reduce((width, cell) => Math.max(width, variantOf(run, cell).length), 0);
-const gridOf = (run, trials, elapsedMs) => {
+const widest = (run) => run.cells.reduce((width, cell) => Math.max(width, formatVariant(run, cell).length), 0);
+const formatGrid = (run, trials, elapsedMs) => {
 	const width = widest(run);
 	const lines = [];
 	for (const caseName of run.cases) {
 		lines.push(`  ${BOLD}${caseName}${RESET}`);
-		for (const cell of run.cells.filter((one) => one.caseName === caseName)) lines.push(`    ${markOf(cell)} ${variantOf(run, cell).padEnd(width)}  ${trialsOf(cell, trials)}  ${rateOf$1(cell)}`);
+		for (const cell of run.cells.filter((one) => one.caseName === caseName)) lines.push(`    ${formatStatus(cell)} ${formatVariant(run, cell).padEnd(width)}  ${formatTrialProgress(cell, trials)}  ${formatPassRate(cell)}`);
 		lines.push("");
 	}
-	lines.push(paint(DIM, `  ${elapsedOf(elapsedMs)} elapsed`));
+	lines.push(paint(DIM, `  ${formatElapsed(elapsedMs)} elapsed`));
 	return lines;
 };
 const up = (rows) => `[${rows}A`;
@@ -160,12 +172,12 @@ const liveGrid = (trials, interactive) => Effect.gen(function* () {
 	return (run, elapsedMs) => Effect.gen(function* () {
 		if (!interactive) return;
 		const rows = yield* Ref.getAndSet(drawn, 0);
-		const lines = gridOf(run, trials, elapsedMs);
+		const lines = formatGrid(run, trials, elapsedMs);
 		yield* note(`${rows === 0 ? "" : up(rows)}${lines.join("\n")}`);
 		yield* Ref.set(drawn, lines.length);
 	});
 });
-const summaryOf$1 = (run, trials, drawn) => drawn ? "" : gridOf(run, trials, 0).join("\n");
+const formatGridSummary = (run, trials, drawn) => drawn ? "" : formatGrid(run, trials, 0).join("\n");
 //#endregion
 //#region src/imports/evals-json-errors.ts
 var CaseFileUnreadable = class extends Data.TaggedError("CaseFileUnreadable") {
@@ -282,7 +294,7 @@ const renderEvalSuite = (file) => {
 		"  ],",
 		"  tasks: [",
 		"    /* Name the harness, model and sandbox this suite runs on. */",
-		"    { harness: \"codex\", model: \"gpt-5.6-sol\", provider: \"daytona\" },",
+		"    { harness: \"codex\", model: \"gpt-5.6-sol\" },",
 		"  ],",
 		"});"
 	].join("\n")}\n`;
@@ -415,7 +427,7 @@ const renderYamlSuite = (files) => `${[
 	"  ],",
 	"  tasks: [",
 	"    /* Name the harness, model and sandbox this suite runs on. */",
-	"    { harness: \"codex\", model: \"gpt-5.6-sol\", provider: \"daytona\" },",
+	"    { harness: \"codex\", model: \"gpt-5.6-sol\" },",
 	"  ],",
 	"});"
 ].join("\n")}\n`;
@@ -515,12 +527,102 @@ const importEval = Command.make("import", {
 	return yield* note(summaryOf(suite.tally));
 }).pipe(Effect.withSpan("Cli.evalImport"))).pipe(Command.withDescription("Turn a case file a team already wrote into a suite"));
 //#endregion
+//#region src/cli/eval-outcome.ts
+const EvalOutcome = Schema.Struct({
+	file: Schema.String,
+	problems: Schema.Array(Schema.String),
+	run: Schema.NullOr(EvalRun),
+	runId: Schema.NullOr(Schema.String)
+});
+Schema.Struct({
+	conclusion: Schema.Literal("failure", "neutral", "success"),
+	details_url: Schema.optional(Schema.String),
+	name: Schema.Literal("anpord"),
+	output: Schema.Struct({
+		summary: Schema.String,
+		title: Schema.String
+	})
+});
+const TRUNCATED = "\n\n… truncated";
+const TRAILING_SLASH = /\/$/;
+const TITLES = {
+	failure: "Eval gate failed",
+	success: "Eval gate passed",
+	neutral: "Evals still running"
+};
+const percent = (rate) => rate === void 0 ? "-" : `${Math.round(rate * 100)}%`;
+const escaped = (text) => text.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll("|", "\\|").replaceAll(/[\r\n]/g, " ");
+const formatComparison = (run, cell) => {
+	const comparison = cell.comparison;
+	if (comparison === null) return "-";
+	const { baselineHarnessVersion: before, candidateHarnessVersion: after } = comparison;
+	const changed = before === after ? "" : ` (${run.tasks[cell.taskIndex]?.harness} ${before} → ${after})`;
+	return `${comparison.verdict}${changed}`;
+};
+const formatCellRow = (run, cell) => {
+	const rate = cell.distribution?.scored ? cell.distribution.passRate : void 0;
+	return `| ${escaped(cell.caseName)} | ${escaped(formatVariant(run, cell))} | ${percent(rate)} | ${percent(cell.comparison?.baselinePassRate)} | ${escaped(formatComparison(run, cell))} |`;
+};
+const runUrl = (webUrl, id) => `${webUrl.replace(TRAILING_SLASH, "")}/evals/${encodeURIComponent(id)}`;
+const formatOutcome = ({ file, problems, run, runId }, webUrl) => [
+	`### ${escaped(file)}`,
+	"",
+	...runId === null ? [] : [`[View run](${runUrl(webUrl, runId)})`, ""],
+	...problems.map((problem) => `- ${escaped(problem)}`),
+	...run === null ? [] : [
+		"",
+		"| Case | Variant | Pass rate | Baseline | Verdict |",
+		"| --- | --- | --- | --- | --- |",
+		...run.cells.map((cell) => formatCellRow(run, cell))
+	]
+].join("\n");
+const buildGithubCheck = (outcomes, webUrl) => {
+	const failed = outcomes.some((outcome) => outcome.problems.length > 0);
+	const completed = outcomes.length > 0 && outcomes.every((outcome) => outcome.run !== null);
+	const first = outcomes.find((outcome) => outcome.runId !== null);
+	const conclusion = failed ? "failure" : completed ? "success" : "neutral";
+	const summary = outcomes.map((outcome) => formatOutcome(outcome, webUrl)).join("\n\n");
+	return {
+		conclusion,
+		details_url: first?.runId ? runUrl(webUrl, first.runId) : void 0,
+		name: "anpord",
+		output: {
+			title: TITLES[conclusion],
+			summary: summary.length < 65535 ? summary : `${summary.slice(0, 65521)}${TRUNCATED}`
+		}
+	};
+};
+//#endregion
+//#region src/cli/eval-report.ts
+const reportJson = Schema.encodeSync(Schema.parseJson(Schema.Array(EvalOutcome)));
+const appendSummary = (text) => Effect.gen(function* () {
+	const path = yield* Config.string("GITHUB_STEP_SUMMARY").pipe(Config.option);
+	if (Option.isSome(path) && path.value !== "") yield* (yield* FileSystem.FileSystem).writeFileString(path.value, `${text}\n\n`, { flag: "a" });
+});
+const reportStarted = (file, id) => Effect.gen(function* () {
+	const url = runUrl(yield* webUrlConfig, id);
+	yield* note(`${file}: ${url}`);
+	yield* appendSummary(`[Run ${id}](${url}) started.`);
+});
+const writeReport = (outcomes, path) => Effect.gen(function* () {
+	if (Option.isSome(path)) yield* (yield* FileSystem.FileSystem).writeFileString(path.value, reportJson(outcomes));
+});
+const reportFinished = (outcomes) => Effect.gen(function* () {
+	const report = buildGithubCheck(outcomes, yield* webUrlConfig);
+	yield* appendSummary(`## ${report.output.title}\n\n${report.output.summary}`);
+});
+//#endregion
 //#region src/cli/eval-run.ts
 const FIRST_POLL = 2e3;
 const SLOWEST_POLL = 1e4;
 const WIDENING = 1.5;
 const running = (run) => run.status === "running";
-const waitForRun = (id, onProgress) => Effect.gen(function* () {
+var EvalWaitTimeout = class extends Data.TaggedError("EvalWaitTimeout") {
+	get message() {
+		return `Timed out after ${this.seconds}s waiting for ${this.runId}. The remote run was not cancelled.`;
+	}
+};
+const waitForRun = (id, onProgress, timeoutSeconds) => Effect.gen(function* () {
 	const api = yield* AnpordApi;
 	const startedAt = yield* Clock.currentTimeMillis;
 	const gap = yield* Ref.make(FIRST_POLL);
@@ -538,73 +640,28 @@ const waitForRun = (id, onProgress) => Effect.gen(function* () {
 		body: () => waitThenPoll,
 		while: running
 	});
-}).pipe(Effect.withSpan("Cli.waitForRun", { attributes: { runId: id } }));
-Schema.Struct({
-	conclusion: Schema.Literal("failure", "neutral", "success"),
-	details_url: Schema.optional(Schema.String),
-	name: Schema.Literal("anpord"),
-	output: Schema.Struct({
-		summary: Schema.String,
-		title: Schema.String
+}).pipe(Effect.timeoutFail({
+	duration: Duration.seconds(timeoutSeconds),
+	onTimeout: () => new EvalWaitTimeout({
+		runId: id,
+		seconds: timeoutSeconds
 	})
+}), Effect.withSpan("Cli.waitForRun", { attributes: { runId: id } }));
+//#endregion
+//#region src/cli/eval-trigger.ts
+const evalTrigger = Effect.gen(function* () {
+	const github = yield* Config.boolean("GITHUB_ACTIONS").pipe(Config.withDefault(false));
+	const ci = yield* Config.boolean("CI").pipe(Config.withDefault(false));
+	if (!github) return { source: ci ? "ci" : "cli" };
+	const repository = yield* Config.string("GITHUB_REPOSITORY");
+	const runId = yield* Config.string("GITHUB_RUN_ID");
+	const attempt = yield* Config.string("GITHUB_RUN_ATTEMPT").pipe(Config.withDefault("1"));
+	const server = yield* Config.string("GITHUB_SERVER_URL").pipe(Config.withDefault("https://github.com"));
+	return yield* Schema.decodeUnknown(EvalTrigger)({
+		source: "ci",
+		url: `${server}/${repository}/actions/runs/${runId}/attempts/${attempt}`
+	});
 });
-const TRUNCATED = "\n\n… truncated";
-const PERCENT = 100;
-const ABSENT = "—";
-const percent = (rate) => rate === void 0 ? ABSENT : `${Math.round(rate * PERCENT)}%`;
-const rateOf = (cell) => cell.distribution === null || cell.distribution.scored === 0 ? ABSENT : percent(cell.distribution.passRate);
-const versionsOf = (comparison) => ({
-	baseline: "baselineHarnessVersion" in comparison && typeof comparison.baselineHarnessVersion === "string" ? comparison.baselineHarnessVersion : void 0,
-	candidate: "candidateHarnessVersion" in comparison && typeof comparison.candidateHarnessVersion === "string" ? comparison.candidateHarnessVersion : void 0
-});
-const versionClause = (run, cell) => {
-	if (cell.comparison === null) return "";
-	const { baseline, candidate } = versionsOf(cell.comparison);
-	const harness = run.tasks[cell.taskIndex]?.harness ?? "harness";
-	return baseline === void 0 || candidate === void 0 || baseline === candidate ? "" : ` (${harness} ${baseline} → ${candidate})`;
-};
-const verdictOf = (run, cell) => cell.comparison === null ? ABSENT : `${cell.comparison.verdict}${versionClause(run, cell)}`;
-const escaped = (text) => text.replaceAll("|", "\\|");
-const rowOf = (run, cell) => `| ${escaped(cell.caseName)} | ${escaped(variantOf(run, cell))} | ${rateOf(cell)} | ${percent(cell.comparison?.baselinePassRate)} | ${verdictOf(run, cell)} |`;
-const tableOf = (file, run) => [
-	`### ${escaped(file)}`,
-	"",
-	...run.failure === null ? [] : [`Run failed: ${run.failure}`, ""],
-	"| Case | Variant | Pass rate | Baseline | Verdict |",
-	"| --- | --- | --- | --- | --- |",
-	...run.cells.map((cell) => rowOf(run, cell))
-].join("\n");
-const truncated = (summary) => summary.length < 65535 ? summary : `${summary.slice(0, 65521)}${TRUNCATED}`;
-const conclusionOf = (cells) => {
-	const verdicts = cells.flatMap((cell) => cell.comparison === null ? [] : [cell.comparison.verdict]);
-	if (verdicts.includes("regressed")) return "failure";
-	return verdicts.every((verdict) => verdict === "incomparable") ? "neutral" : "success";
-};
-const TITLES = {
-	failure: "A cell regressed against its baseline",
-	neutral: "Nothing to compare against a baseline",
-	success: "No cell regressed against its baseline"
-};
-const checkRunOf = (outcomes, webUrl) => {
-	const finished = outcomes.flatMap((outcome) => Option.match(outcome.run, {
-		onNone: () => [],
-		onSome: (run) => [{
-			file: outcome.file,
-			run
-		}]
-	}));
-	const conclusion = conclusionOf(finished.flatMap(({ run }) => run.cells));
-	const first = finished[0];
-	return {
-		conclusion,
-		details_url: first === void 0 ? void 0 : `${webUrl}/evals/${first.run.id}`,
-		name: "anpord",
-		output: {
-			summary: truncated(finished.map(({ file, run }) => tableOf(file, run)).join("\n\n")),
-			title: TITLES[conclusion]
-		}
-	};
-};
 //#endregion
 //#region src/cli/github-check-client.ts
 const API = "https://api.github.com";
@@ -665,64 +722,90 @@ const githubContext = Effect.gen(function* () {
 });
 //#endregion
 //#region src/cli/eval-command.ts
-const asJson$1 = Options.boolean("json").pipe(Options.withDescription("Print the finished run as JSON"));
-const evalFile = Args.text({ name: "file" }).pipe(Args.withDescription("A TypeScript file that default exports defineEval(...); every *.eval.ts is run when omitted"), Args.optional);
-const noWait = Options.boolean("no-wait").pipe(Options.withDescription("Start the run and print its id, without waiting"));
-const failOn = Options.choice("fail-on", [
-	"never",
-	"regressed",
-	"unscored"
-]).pipe(Options.withDescription("What makes the command exit nonzero"), Options.withDefault("regressed"));
-const runOneEval = (file, options) => Effect.gen(function* () {
-	const api = yield* AnpordApi;
-	const payload = yield* compileEvalEffect(file);
-	const started = yield* api.evals.start({ payload });
-	if (options.skipWait) {
-		yield* json(started);
-		return {
+const asJson$1 = Options.boolean("json").pipe(Options.withDescription("Print each finished run as JSON"));
+const evalFile = Args.text({ name: "file" }).pipe(Args.withDescription("A TypeScript eval file; discovers *.eval.ts when omitted"), Args.optional);
+const noWait = Options.boolean("no-wait").pipe(Options.withDescription("Start runs without waiting"));
+const failOn = Options.choice("fail-on", EvalGate.literals).pipe(Options.withDescription("strict requires every trial to pass"), Options.withDefault("strict"));
+const timeout = Options.integer("timeout").pipe(Options.withDescription("Maximum seconds to wait per run"), Options.withSchema(Schema.Int.pipe(Schema.positive())), Options.withDefault(1200));
+const output = Options.text("output").pipe(Options.withDescription("Write a JSON report to this file"), Options.optional);
+const describe$1 = (error) => error instanceof Error ? error.message : String(error);
+const runOneEval = (file, options, save) => Effect.gen(function* () {
+	let runId = null;
+	return yield* Effect.gen(function* () {
+		const api = yield* AnpordApi;
+		const payload = yield* compileEvalEffect(file);
+		const trigger = yield* evalTrigger;
+		const started = yield* api.evals.start({ payload: {
+			...payload,
+			trigger
+		} });
+		runId = started.id;
+		yield* reportStarted(file, runId);
+		const pending = {
 			file,
+			runId,
 			problems: [],
-			run: Option.none()
+			run: null
 		};
-	}
-	const live = !options.wantsJson && (yield* attended);
-	if (live || options.label) yield* note(`${file} · run ${started.id}`);
-	const draw = yield* liveGrid(payload.trials, live);
-	const run = yield* waitForRun(started.id, draw);
-	yield* options.wantsJson ? json(run) : note(summaryOf$1(run, payload.trials, live));
-	return {
+		yield* save(pending);
+		if (options.skipWait) {
+			yield* json(started);
+			return pending;
+		}
+		const live = !options.wantsJson && (yield* attended);
+		const draw = yield* liveGrid(payload.trials, live);
+		const run = yield* waitForRun(runId, draw, options.timeoutSeconds);
+		yield* options.wantsJson ? json(run) : note(formatGridSummary(run, payload.trials, live));
+		return {
+			file,
+			runId,
+			run,
+			problems: problemsWith(run, options.gate, {
+				cells: payload.cases.length * payload.tasks.length,
+				trials: payload.trials
+			})
+		};
+	}).pipe(Effect.catchAll((error) => Effect.succeed({
 		file,
-		problems: problemsWith(run, options.gate),
-		run: Option.some(run)
-	};
+		runId,
+		run: null,
+		problems: [describe$1(error)]
+	})));
 });
-const describe$1 = (error) => error instanceof Error ? error.message : String(error);
 const reportToGithub = (outcomes) => Effect.gen(function* () {
-	if (outcomes.every((outcome) => Option.isNone(outcome.run))) return;
 	const context = yield* githubContext;
 	if (Option.isNone(context)) return;
 	const webUrl = yield* webUrlConfig;
-	yield* postCheckRun(context.value, checkRunOf(outcomes, webUrl));
-	yield* note(`Posted the anpord check on ${context.value.sha}`);
+	yield* postCheckRun(context.value, buildGithubCheck(outcomes, webUrl));
 }).pipe(Effect.catchAll((error) => note(`The GitHub check was not posted. ${describe$1(error)}`)));
 const runEval = Command.make("eval", {
 	asJson: asJson$1,
 	evalFile,
 	failOn,
-	noWait
-}, ({ asJson: wantsJson, evalFile: file, failOn: gate, noWait: skipWait }) => Effect.gen(function* () {
+	noWait,
+	output,
+	timeout
+}, ({ asJson: wantsJson, evalFile: file, failOn: gate, noWait: skipWait, output: path, timeout: timeoutSeconds }) => Effect.gen(function* () {
 	const files = yield* Option.match(file, {
 		onNone: () => evalFilesIn("."),
 		onSome: (one) => Effect.succeed([one])
 	});
 	if (files.length === 0) return yield* Effect.fail(new NoEvalFiles());
-	const outcomes = yield* Effect.forEach(files, (one) => runOneEval(one, {
-		gate,
-		label: files.length > 1,
-		skipWait,
-		wantsJson
+	const outcomes = [];
+	yield* Effect.forEach(files, (one, index) => Effect.gen(function* () {
+		const save = (outcome) => Effect.gen(function* () {
+			outcomes[index] = outcome;
+			yield* writeReport(outcomes, path);
+		});
+		yield* save(yield* runOneEval(one, {
+			gate,
+			skipWait,
+			timeoutSeconds,
+			wantsJson
+		}, save));
 	}));
-	yield* reportToGithub(outcomes);
+	yield* reportFinished(outcomes);
+	if (!skipWait) yield* reportToGithub(outcomes);
 	return yield* failWhen(outcomes.flatMap((outcome) => outcome.problems));
 }).pipe(Effect.provide(ClientLayer))).pipe(Command.withDescription("Compile and run an eval from TypeScript"), Command.withSubcommands([importEval]));
 //#endregion

package/dist/cli-runtime.cjs

@@ -118,8 +118,8 @@ const executeCli = (definition, args, journal = JOURNAL) => effect.Effect.gen(fu
 		stdout: ""
 	}))));
 }).pipe(effect.Effect.withSpan("CliMock.execute"), effect.Effect.runPromise);
-const runCli = async (definition) => {
-	const result = await executeCli(definition, process.argv.slice(2));
+const runCli = async (definition, journal) => {
+	const result = await executeCli(definition, process.argv.slice(2), journal);
 	process.stdout.write(result.stdout);
 	process.stderr.write(result.stderr);
 	process.exitCode = result.exitCode;

package/dist/cli-runtime.d.cts

@@ -6,6 +6,6 @@ interface CliExecution {
   readonly stdout: string;
 }
 declare const executeCli: (definition: CliDefinition, args: readonly string[], journal?: string) => Promise<CliExecution>;
-declare const runCli: (definition: CliDefinition) => Promise<void>;
+declare const runCli: (definition: CliDefinition, journal?: string) => Promise<void>;
 //#endregion
 export { CliExecution, executeCli, runCli };

package/dist/cli-runtime.d.mts

@@ -6,6 +6,6 @@ interface CliExecution {
   readonly stdout: string;
 }
 declare const executeCli: (definition: CliDefinition, args: readonly string[], journal?: string) => Promise<CliExecution>;
-declare const runCli: (definition: CliDefinition) => Promise<void>;
+declare const runCli: (definition: CliDefinition, journal?: string) => Promise<void>;
 //#endregion
 export { CliExecution, executeCli, runCli };

package/dist/cli-runtime.mjs

@@ -117,8 +117,8 @@ const executeCli = (definition, args, journal = JOURNAL) => Effect.gen(function*
 		stdout: ""
 	}))));
 }).pipe(Effect.withSpan("CliMock.execute"), Effect.runPromise);
-const runCli = async (definition) => {
-	const result = await executeCli(definition, process.argv.slice(2));
+const runCli = async (definition, journal) => {
+	const result = await executeCli(definition, process.argv.slice(2), journal);
 	process.stdout.write(result.stdout);
 	process.stderr.write(result.stderr);
 	process.exitCode = result.exitCode;

package/dist/{client-D22IiHGL.mjs → client-BfdjFtlV.mjs}

@@ -1,4 +1,4 @@
-import { C as EvalPrompt, S as EvalCaseName, _ as HarnessProfile, a as EvalPageCursor, c as EvalRun, d as EvalValidator, f as EvalVariables, h as StartedEval, i as EvalName, l as EvalRunPage, m as RerunCellRequest, n as EvalCellHistoryEntry, o as EvalPrepare, p as ModelCatalogue, r as EvalHarness, s as EvalProvider, t as CaseCache, u as EvalSource, v as PROFILE_HARNESS_RULE, w as EvalVerify, x as profileFitsHarness } from "./evals-CvOTyvCX.mjs";
+import { C as EvalTrigger, E as EvalVerify, T as EvalPrompt, _ as HarnessProfile, a as EvalPageCursor, c as EvalRunPage, d as EvalValidator, f as EvalVariables, h as StartedEval, i as EvalName, l as EvalSandbox, m as RerunCellRequest, n as EvalCellHistoryEntry, o as EvalPrepare, p as ModelCatalogue, r as EvalHarness, s as EvalRun, t as CaseCache, u as EvalSource, v as PROFILE_HARNESS_RULE, w as EvalCaseName, x as profileFitsHarness } from "./evals-Dcc_x6jW.mjs";
 import { FetchHttpClient, HttpApi, HttpApiClient, HttpApiEndpoint, HttpApiGroup, HttpApiMiddleware, HttpApiSchema, HttpApiSecurity, HttpClient, HttpClientRequest, OpenApi } from "@effect/platform";
 import { Context, Effect, Layer, Redacted, Schema } from "effect";
 //#region ../schema/src/domain/prompts.ts
@@ -135,9 +135,9 @@ const EvalModelsRequest = Schema.Struct({
 	description: "Select a harness whose available models should be listed.",
 	identifier: "EvalModelsRequest"
 });
-const PublicEvalProvider = EvalProvider.annotations({
-	description: "A hosted sandbox provider.",
-	identifier: "PublicEvalProvider"
+const PublicEvalSandbox = EvalSandbox.annotations({
+	description: "The hosted sandbox a task runs in.",
+	identifier: "PublicEvalSandbox"
 });
 const ListEvalsRequest = Schema.Struct({
 	cursor: Schema.optional(Schema.NullOr(EvalPageCursor)),
@@ -162,12 +162,13 @@ const PublicEvalTask = Schema.Struct({
 	harness: EvalHarness,
 	model: Schema.String.pipe(Schema.minLength(1)),
 	profile: Schema.optional(HarnessProfile),
-	provider: PublicEvalProvider
+	sandbox: Schema.optional(PublicEvalSandbox)
 }).pipe(Schema.filter(profileFitsHarness, { message: () => PROFILE_HARNESS_RULE })).annotations({
-	description: `A harness, model, and hosted sandbox combination, with an optional profile layered on the harness. ${PROFILE_HARNESS_RULE}`,
+	description: `A harness and model, with an optional sandbox and an optional profile layered on the harness. Omit the sandbox to use the default. ${PROFILE_HARNESS_RULE}`,
 	identifier: "StartEvalTask"
 });
 const PublicStartEvalRequest = Schema.Struct({
+	trigger: Schema.optional(EvalTrigger),
 	cases: Schema.Array(PublicEvalCase).pipe(Schema.minItems(1), Schema.maxItems(100)),
 	name: Schema.optional(EvalName),
 	prompt: EvalPrompt,