anpord 0.1.11 → 0.1.13

package/dist/bin.cjs

@@ -1,23 +1,20 @@
 #!/usr/bin/env node
-const require_client = require("./client-Ci0woWLW.cjs");
-const require_errors = require("./errors-Boum34f5.cjs");
+const require_client = require("./client-C-P2I00z.cjs");
+const require_evals = require("./evals-DouMsrsM.cjs");
+const require_errors = require("./errors-BX1wry8K.cjs");
 const require_config = require("./config.cjs");
-const require_compiler = require("./compiler-Ap8DM-JU.cjs");
+const require_compiler = require("./compiler-uwb1pkdK.cjs");
 let _effect_cli = require("@effect/cli");
 let _effect_platform = require("@effect/platform");
 let _effect_platform_node = require("@effect/platform-node");
 let effect = require("effect");
 let yaml = require("yaml");
 //#region package.json
-var version$1 = "0.1.11";
+var version$1 = "0.1.13";
 //#endregion
 //#region ../template/src/extract.ts
-/**
-* Distinct variable names in source order, so callers can count or list them.
-*
-* Escaped braces are read here too, so a name the renderer will treat as
-* literal text is never reported as a variable the editor should draw.
-*/
+/** Reads escapes too, so text the renderer leaves literal is never reported as
+* a variable. */
 function extractVariables(template) {
 	const names = [];
 	for (const [, open, close, name] of template.matchAll(require_errors.tokenMatcher())) if (open === void 0 && close === void 0 && name !== void 0) names.push(name);
@@ -25,13 +22,6 @@ function extractVariables(template) {
 }
 //#endregion
 //#region src/cli/declarations.ts
-/**
-* The generated file has to open by importing the module it augments.
-*
-* Without it `declare module` shadows the module rather than adding to it, and
-* every import of the SDK stops resolving with an error naming the missing
-* export rather than the file that hid it.
-*/
 const PREAMBLE = [
 	"// Generated by `anpord generate`. Do not edit.",
 	"// Run it again after changing which variables a prompt uses.",
@@ -40,7 +30,6 @@ const PREAMBLE = [
 	"declare module \"anpord\" {",
 	"  interface AnpordPromptVariables {"
 ].join("\n");
-/** Quoted, because an id may carry characters an identifier cannot. */
 const entry = (id, names) => {
 	return `    "${id}": ${names.length === 0 ? "Record<string, never>" : `{ ${names.map((name) => `"${name}": string`).join("; ")} }`};`;
 };
@@ -70,10 +59,11 @@ const evalFilesIn = (directory) => effect.Effect.gen(function* () {
 }).pipe(effect.Effect.withSpan("Cli.evalFilesIn"));
 //#endregion
 //#region src/cli/eval-gate.ts
+const EvalGate = effect.Schema.Literal("strict", "never", "regressed", "unscored");
 const regressions = (run) => run.cells.filter((cell) => cell.comparison?.verdict === "regressed");
 const unscored = (run) => run.cells.filter((cell) => (cell.distribution?.scored ?? 0) === 0);
 const rate = (value) => `${Math.round(value * 100) / 100}`;
-const versionClause$1 = (run, cell, found) => found.baselineHarnessVersion === found.candidateHarnessVersion ? "" : `${run.tasks[cell.taskIndex]?.harness ?? "harness"} ${found.baselineHarnessVersion} → ${found.candidateHarnessVersion}, `;
+const versionClause = (run, cell, found) => found.baselineHarnessVersion === found.candidateHarnessVersion ? "" : `${run.tasks[cell.taskIndex]?.harness ?? "harness"} ${found.baselineHarnessVersion} → ${found.candidateHarnessVersion}, `;
 const profileClause = (run, cell, found) => {
 	const { baselineProfileVersion, candidateProfileVersion } = found;
 	if (baselineProfileVersion === null || candidateProfileVersion === null || baselineProfileVersion === candidateProfileVersion) return "";
@@ -82,10 +72,20 @@ const profileClause = (run, cell, found) => {
 const regressionSentence = (run, cell) => {
 	const found = cell.comparison;
 	if (found === null) return `${cell.caseName} regressed against its baseline.`;
-	return `${cell.caseName} regressed against its baseline: ${versionClause$1(run, cell, found)}${profileClause(run, cell, found)}pass rate ${rate(found.baselinePassRate)} → ${rate(found.candidatePassRate)}.`;
+	return `${cell.caseName} regressed against its baseline: ${versionClause(run, cell, found)}${profileClause(run, cell, found)}pass rate ${rate(found.baselinePassRate)} → ${rate(found.candidatePassRate)}.`;
 };
-const problemsWith = (run, failOn) => {
+const problemsWith = (run, failOn, expected) => {
 	if (run.status === "failed") return [run.failure ?? "The run failed."];
+	if (run.status !== "finished") return ["The run has not finished."];
+	if (run.cells.length === 0) return ["The run recorded no cells."];
+	if (failOn === "strict") {
+		if (expected && run.cells.length !== expected.cells) return [`Expected ${expected.cells} cells, received ${run.cells.length}.`];
+		return run.cells.flatMap((cell) => {
+			if (cell.status !== "finished" || cell.trials.length === 0) return [`${cell.caseName} has no complete trial results.`];
+			if (expected && cell.trials.length !== expected.trials) return [`${cell.caseName}: expected ${expected.trials} trials, received ${cell.trials.length}.`];
+			return cell.trials.flatMap((trial) => trial.status === "passed" && trial.passed ? [] : [`${cell.caseName}, trial ${trial.ordinal}: ${trial.status}.`]);
+		});
+	}
 	if (failOn === "never") return [];
 	const found = regressions(run).map((cell) => regressionSentence(run, cell));
 	return failOn === "unscored" ? [...found, ...unscored(run).map((cell) => `${cell.caseName} produced no scored trials.`)] : found;
@@ -108,8 +108,6 @@ const promptContent = (prompt) => effect.Effect.sync(() => {
 	process.stdout.write(prompt.content);
 	if (!prompt.content.endsWith("\n")) process.stdout.write("\n");
 });
-/** A result the caller may pipe into another tool, so it belongs on stdout
-* alongside the prompt content rather than beside the status messages. */
 const row = (line) => effect.Effect.sync(() => {
 	process.stdout.write(`${line}\n`);
 });
@@ -129,43 +127,43 @@ const DONE = "●";
 const RUNNING = "◐";
 const FILLED = "▰";
 const HOLLOW = "▱";
-const PERCENT$1 = 100;
+const PERCENT = 100;
 const SECONDS = 1e3;
 const MINUTE = 60;
 const paint = (colour, text) => `${colour}${text}${RESET}`;
-const elapsedOf = (ms) => {
+const formatElapsed = (ms) => {
 	const total = Math.floor(ms / SECONDS);
 	const minutes = Math.floor(total / MINUTE);
 	return minutes === 0 ? `${total}s` : `${minutes}m${String(total % MINUTE).padStart(2, "0")}s`;
 };
-const markOf = (cell) => {
+const formatStatus = (cell) => {
 	if (cell.status === "finished") return paint(GREEN, DONE);
 	return cell.status === "failed" ? paint(RED, DONE) : paint(YELLOW, RUNNING);
 };
-const trialsOf = (cell, trials) => {
+const formatTrialProgress = (cell, trials) => {
 	const settled = cell.trials.filter((trial) => trial.status !== "queued" && trial.status !== "running").length;
 	return `${FILLED.repeat(settled)}${paint(DIM, HOLLOW.repeat(Math.max(0, trials - settled)))}`;
 };
-const rateOf$1 = (cell) => {
+const formatPassRate = (cell) => {
 	const rate = cell.distribution?.passRate;
 	if (rate === void 0 || cell.distribution?.scored === 0) return paint(DIM, "—");
-	const shown = `${Math.round(rate * PERCENT$1)}%`;
+	const shown = `${Math.round(rate * PERCENT)}%`;
 	return paint(rate === 1 ? GREEN : RED, shown);
 };
-const variantOf = (run, cell) => {
+const formatVariant = (run, cell) => {
 	const task = run.tasks[cell.taskIndex];
 	return task === void 0 ? "?" : `${task.harness}/${task.model}`;
 };
-const widest = (run) => run.cells.reduce((width, cell) => Math.max(width, variantOf(run, cell).length), 0);
-const gridOf = (run, trials, elapsedMs) => {
+const widest = (run) => run.cells.reduce((width, cell) => Math.max(width, formatVariant(run, cell).length), 0);
+const formatGrid = (run, trials, elapsedMs) => {
 	const width = widest(run);
 	const lines = [];
 	for (const caseName of run.cases) {
 		lines.push(`  ${BOLD}${caseName}${RESET}`);
-		for (const cell of run.cells.filter((one) => one.caseName === caseName)) lines.push(`    ${markOf(cell)} ${variantOf(run, cell).padEnd(width)}  ${trialsOf(cell, trials)}  ${rateOf$1(cell)}`);
+		for (const cell of run.cells.filter((one) => one.caseName === caseName)) lines.push(`    ${formatStatus(cell)} ${formatVariant(run, cell).padEnd(width)}  ${formatTrialProgress(cell, trials)}  ${formatPassRate(cell)}`);
 		lines.push("");
 	}
-	lines.push(paint(DIM, `  ${elapsedOf(elapsedMs)} elapsed`));
+	lines.push(paint(DIM, `  ${formatElapsed(elapsedMs)} elapsed`));
 	return lines;
 };
 const up = (rows) => `[${rows}A`;
@@ -174,12 +172,12 @@ const liveGrid = (trials, interactive) => effect.Effect.gen(function* () {
 	return (run, elapsedMs) => effect.Effect.gen(function* () {
 		if (!interactive) return;
 		const rows = yield* effect.Ref.getAndSet(drawn, 0);
-		const lines = gridOf(run, trials, elapsedMs);
+		const lines = formatGrid(run, trials, elapsedMs);
 		yield* note(`${rows === 0 ? "" : up(rows)}${lines.join("\n")}`);
 		yield* effect.Ref.set(drawn, lines.length);
 	});
 });
-const summaryOf$1 = (run, trials, drawn) => drawn ? "" : gridOf(run, trials, 0).join("\n");
+const formatGridSummary = (run, trials, drawn) => drawn ? "" : formatGrid(run, trials, 0).join("\n");
 //#endregion
 //#region src/imports/evals-json-errors.ts
 var CaseFileUnreadable = class extends effect.Data.TaggedError("CaseFileUnreadable") {
@@ -192,8 +190,6 @@ var CaseFileNotJson = class extends effect.Data.TaggedError("CaseFileNotJson") {
 		return `${this.path} is not JSON: ${this.reason}`;
 	}
 };
-/** Names the case rather than the byte offset, because the author fixing it
-* reads the file by case, not by position. */
 var CaseFileNotEvalsJson = class extends effect.Data.TaggedError("CaseFileNotEvalsJson") {
 	get message() {
 		return `${this.path} is not an evals-json file: ${this.reason}`;
@@ -206,29 +202,17 @@ var CaseFileEmpty = class extends effect.Data.TaggedError("CaseFileEmpty") {
 };
 //#endregion
 //#region src/imports/typescript-literal.ts
-/** A backslash-escaped double-quoted literal. Backslash first, so the escapes
-* added after it are not themselves escaped. */
 const quoted = (value) => `"${value.replaceAll("\\", "\\\\").replaceAll("\"", "\\\"").replaceAll("\n", "\\n").replaceAll("\r", "\\r").replaceAll("	", "\\t")}"`;
-/** A template literal, which keeps a multi-line prompt readable in the
-* generated file. A backtick ends the literal and `${` opens a substitution,
-* so both are escaped; a lone `$` is left alone because only the pair means
-* anything. A trailing backslash would escape the closing backtick. */
 const templated = (value) => `\`${value.replaceAll("\\", "\\\\").replaceAll("`", "\\`").replaceAll("${", "\\${")}\``;
-/** A comment cannot carry the sequence that closes it, and a newline would end
-* a line comment and let the rest of the text run as code. */
 const commentSafe = (value) => value.replaceAll("*/", "*\\/").replaceAll(/\r?\n/g, " ");
 //#endregion
 //#region src/imports/unwritten-check.ts
 const PLACEHOLDER = "unwritten";
-/** The author's own words, kept whole, because they are the specification for
-* the check that replaces the line beneath them. */
 const proseLine = (text) => [
 	"          /* Write this check, then delete the line under it: */",
 	`          /* ${commentSafe(text)} */`,
 	`          ${PLACEHOLDER}(${quoted(text)}),`
 ].join("\n");
-/** Local rather than imported, so the generated file carries its own proof
-* that an unconverted assertion fails. Deleting the last call deletes it. */
 const placeholderBlock = [
 	"/* A check nobody has written yet. It is false, so the case stays red until",
 	"   the sentence above it becomes a real check. The argument is that",
@@ -259,8 +243,6 @@ const nameOf = (subject) => `${slug$1(subject.name ?? "case", "case")}-${subject
 const scorerLine = (assertion) => [`          /* ${commentSafe(assertion.text)} */`, `          ${SCORER_OF[assertion.kind]}(answer, [${assertion.needles.map(quoted).join(", ")}]),`].join("\n");
 const scorerLines = (subject) => subject.assertions.map((assertion) => isStructured(assertion) ? scorerLine(assertion) : proseLine(assertion)).join("\n");
 const expectationComment = (subject) => subject.expected_output === void 0 ? [] : [`      /* The file's expected output: ${commentSafe(subject.expected_output)} */`];
-/** The JSON names fixture directories by convention and carries none of their
-* contents, so the author supplies the files the name stood for. */
 const sourceComment = (subject) => subject.files.length === 0 ? "      /* This case named no fixture directory. Add the files it starts from. */" : `      /* This case named ${subject.files.map(commentSafe).join(", ")}. The JSON carries the directory names, not their contents, so add the files here. */`;
 const caseBlock$1 = (subject) => [
 	"    {",
@@ -312,23 +294,19 @@ const renderEvalSuite = (file) => {
 		"  ],",
 		"  tasks: [",
 		"    /* Name the harness, model and sandbox this suite runs on. */",
-		"    { harness: \"codex\", model: \"gpt-5.6-sol\", provider: \"daytona\" },",
+		"    { harness: \"codex\", model: \"gpt-5.6-sol\" },",
 		"  ],",
 		"});"
 	].join("\n")}\n`;
 };
 //#endregion
 //#region src/imports/evals-json-schema.ts
-/** The three checks that convert mechanically, because each is a needle list
-* and a rule over it rather than a description of intent. */
 const AssertionKind = effect.Schema.Literal("content_contains_any", "content_contains_all", "content_contains_none");
 const StructuredAssertion = effect.Schema.Struct({
 	kind: AssertionKind,
 	needles: effect.Schema.Array(effect.Schema.String),
 	text: effect.Schema.String
 });
-/** Both dialects occur, sometimes in sibling files: a bare string is prose one
-* person wrote for another, an object is a mechanical check. */
 const Assertion = effect.Schema.Union(effect.Schema.String, StructuredAssertion);
 const EvalsJsonCase = effect.Schema.Struct({
 	assertions: effect.Schema.Array(Assertion),
@@ -356,8 +334,6 @@ const idAt = (parsed, index) => {
 	const found = parsed?.evals?.[index];
 	return found?.id === void 0 ? `${index}` : String(found.id);
 };
-/** The case as its author sees it. A path into the decoded value tells them
-* nothing; the id is what they search the file for. */
 const located = (parsed, path) => {
 	const [head, index, ...rest] = path;
 	if (head !== "evals" || typeof index !== "number") return path.map(String).join(".");
@@ -393,8 +369,6 @@ var CaseFileNotYaml = class extends effect.Data.TaggedError("CaseFileNotYaml") {
 		return `${this.path} is not YAML: ${this.reason}`;
 	}
 };
-/** Names the file rather than a path into the decoded value, because one case
-* is one file and the file name is what the author searches for. */
 var CaseFileNotYamlCase = class extends effect.Data.TaggedError("CaseFileNotYamlCase") {
 	get message() {
 		return `${this.path} is not a yaml case: ${this.reason}`;
@@ -411,20 +385,12 @@ const slug = (value, fallback) => {
 	const cleaned = value.toLowerCase().replaceAll(/[^a-z0-9]+/g, "-").replaceAll(/^-+|-+$/g, "");
 	return cleaned === "" ? fallback : cleaned;
 };
-/** Every line is prose a person wrote for a model judge, so none of it
-* converts and the whole list is what a human still owes. A case with no
-* lines owes one too: something has to say what a good answer is. */
 const tallyOf = (files) => ({
 	cases: files.length,
 	converted: 0,
 	needsAuthor: files.reduce((total, file) => total + Math.max(file.subject.judge_context.length, 1), 0)
 });
-/** Anpord has no step budget, so the number is kept as a note rather than
-* dropped: the suite it came from ran under it, and a case that needed ten
-* steps is a different case from one that needed a hundred. */
 const budgetComment = (subject) => `      /* The file allowed ${subject.max_steps} steps. Anpord does not cap steps, so this is a note, not a limit. */`;
-/** A file with no judge context says nothing about what a good answer is, so
-* it is owed a check like any other rather than passing by default. */
 const UNJUDGED = "This case named no judge context. Write what a good answer is.";
 const judgeLines = (subject) => (subject.judge_context.length === 0 ? [UNJUDGED] : subject.judge_context).map((line) => proseLine(line)).join("\n");
 const caseBlock = (file) => [
@@ -446,8 +412,6 @@ const caseBlock = (file) => [
 	"      },",
 	"    },"
 ].join("\n");
-/** A directory has no name of its own in the files, so a suite built from
-* several is named generically and the author renames it. */
 const suiteName = (files) => files.length === 1 ? slug(files[0]?.subject.name ?? "", "imported-suite") : "imported-suite";
 const renderYamlSuite = (files) => `${[
 	"import { defineEval, files } from \"anpord\";",
@@ -463,18 +427,13 @@ const renderYamlSuite = (files) => `${[
 	"  ],",
 	"  tasks: [",
 	"    /* Name the harness, model and sandbox this suite runs on. */",
-	"    { harness: \"codex\", model: \"gpt-5.6-sol\", provider: \"daytona\" },",
+	"    { harness: \"codex\", model: \"gpt-5.6-sol\" },",
 	"  ],",
 	"});"
 ].join("\n")}\n`;
 //#endregion
 //#region src/imports/yaml-cases-schema.ts
-/** The runner's own default when a file omits the field, kept here so an
-* imported case carries the budget it actually ran under. */
 const DEFAULT_MAX_STEPS = 15;
-/** A missing `judge_context` is a case whose author wrote nothing about how to
-* judge it, which the runner reads as one generic line. It is not an error, so
-* it decodes to an empty list and the generated case says so. */
 const YamlCase = effect.Schema.Struct({
 	judge_context: effect.Schema.optionalWith(effect.Schema.Array(effect.Schema.String), { default: () => [] }),
 	max_steps: effect.Schema.optionalWith(effect.Schema.Int, { default: () => DEFAULT_MAX_STEPS }),
@@ -484,10 +443,6 @@ const YamlCase = effect.Schema.Struct({
 const decodeYamlCase = effect.Schema.decodeUnknown(YamlCase, { errors: "all" });
 //#endregion
 //#region src/imports/yaml-document.ts
-/** A YAML document can parse into a value while still carrying errors the
-* author needs to see, so the errors are read rather than the throw relied on.
-* The value is returned as `unknown` because the schema, not the parser,
-* decides what shape it has. */
 const parseYamlDocument = (body) => effect.Effect.try({
 	catch: (cause) => cause instanceof Error ? cause.message : String(cause),
 	try: () => {
@@ -519,8 +474,6 @@ const readCase = (path) => effect.Effect.gen(function* () {
 		subject: yield* decode(path, parsed)
 	};
 });
-/** Sorted, so the same directory always produces the same suite and a diff of
-* two imports shows what changed rather than what moved. */
 const caseFilesIn = (directory) => effect.Effect.gen(function* () {
 	return (yield* (yield* _effect_platform.FileSystem.FileSystem).readDirectory(directory).pipe(effect.Effect.mapError((cause) => new CaseDirectoryUnreadable({
 		cause,
@@ -547,20 +500,13 @@ const importYamlCases = (path) => effect.Effect.gen(function* () {
 }).pipe(effect.Effect.withSpan("Imports.yamlCases"));
 //#endregion
 //#region src/cli/import-formats.ts
-/** A format is an entry here, so adding one adds an entry rather than editing
-* the command. Each name carries the importer it names, so the parsed option
-* is the importer itself and no lookup can miss. */
 const FORMATS = [["evals-json", importEvalsJson], ["yaml", importYamlCases]];
 //#endregion
 //#region src/cli/eval-import.ts
-/** A path rather than a file, because a format may keep one case per file and
-* import a directory of them as one suite. */
 const caseFile = _effect_cli.Args.path({ name: "path" }).pipe(_effect_cli.Args.withDescription("The case file, or a directory of them, to read"));
 const format = _effect_cli.Options.choiceWithValue("format", FORMATS).pipe(_effect_cli.Options.withDescription("The case file's format"));
 const out$1 = _effect_cli.Options.file("out").pipe(_effect_cli.Options.withDescription("Where to write the suite; it goes to stdout when omitted"), _effect_cli.Options.optional);
 const plural = (count, one) => `${count} ${count === 1 ? one : `${one}s`}`;
-/** The count of unwritten checks leads, because a suite that reports a pass
-* for a check nobody wrote is the failure this product exists to prevent. */
 const summaryOf = (tally) => {
 	const read = `Read ${plural(tally.cases, "case")}, converted ${plural(tally.converted, "assertion")}.`;
 	return tally.needsAuthor === 0 ? read : `${read} ${plural(tally.needsAuthor, "assertion")} could not be converted and ${tally.needsAuthor === 1 ? "needs" : "need"} a human: each is written as prose, and the suite fails until you write the check it describes.`;
@@ -581,12 +527,102 @@ const importEval = _effect_cli.Command.make("import", {
 	return yield* note(summaryOf(suite.tally));
 }).pipe(effect.Effect.withSpan("Cli.evalImport"))).pipe(_effect_cli.Command.withDescription("Turn a case file a team already wrote into a suite"));
 //#endregion
+//#region src/cli/eval-outcome.ts
+const EvalOutcome = effect.Schema.Struct({
+	file: effect.Schema.String,
+	problems: effect.Schema.Array(effect.Schema.String),
+	run: effect.Schema.NullOr(require_evals.EvalRun),
+	runId: effect.Schema.NullOr(effect.Schema.String)
+});
+effect.Schema.Struct({
+	conclusion: effect.Schema.Literal("failure", "neutral", "success"),
+	details_url: effect.Schema.optional(effect.Schema.String),
+	name: effect.Schema.Literal("anpord"),
+	output: effect.Schema.Struct({
+		summary: effect.Schema.String,
+		title: effect.Schema.String
+	})
+});
+const TRUNCATED = "\n\n… truncated";
+const TRAILING_SLASH = /\/$/;
+const TITLES = {
+	failure: "Eval gate failed",
+	success: "Eval gate passed",
+	neutral: "Evals still running"
+};
+const percent = (rate) => rate === void 0 ? "-" : `${Math.round(rate * 100)}%`;
+const escaped = (text) => text.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll("|", "\\|").replaceAll(/[\r\n]/g, " ");
+const formatComparison = (run, cell) => {
+	const comparison = cell.comparison;
+	if (comparison === null) return "-";
+	const { baselineHarnessVersion: before, candidateHarnessVersion: after } = comparison;
+	const changed = before === after ? "" : ` (${run.tasks[cell.taskIndex]?.harness} ${before} → ${after})`;
+	return `${comparison.verdict}${changed}`;
+};
+const formatCellRow = (run, cell) => {
+	const rate = cell.distribution?.scored ? cell.distribution.passRate : void 0;
+	return `| ${escaped(cell.caseName)} | ${escaped(formatVariant(run, cell))} | ${percent(rate)} | ${percent(cell.comparison?.baselinePassRate)} | ${escaped(formatComparison(run, cell))} |`;
+};
+const runUrl = (webUrl, id) => `${webUrl.replace(TRAILING_SLASH, "")}/evals/${encodeURIComponent(id)}`;
+const formatOutcome = ({ file, problems, run, runId }, webUrl) => [
+	`### ${escaped(file)}`,
+	"",
+	...runId === null ? [] : [`[View run](${runUrl(webUrl, runId)})`, ""],
+	...problems.map((problem) => `- ${escaped(problem)}`),
+	...run === null ? [] : [
+		"",
+		"| Case | Variant | Pass rate | Baseline | Verdict |",
+		"| --- | --- | --- | --- | --- |",
+		...run.cells.map((cell) => formatCellRow(run, cell))
+	]
+].join("\n");
+const buildGithubCheck = (outcomes, webUrl) => {
+	const failed = outcomes.some((outcome) => outcome.problems.length > 0);
+	const completed = outcomes.length > 0 && outcomes.every((outcome) => outcome.run !== null);
+	const first = outcomes.find((outcome) => outcome.runId !== null);
+	const conclusion = failed ? "failure" : completed ? "success" : "neutral";
+	const summary = outcomes.map((outcome) => formatOutcome(outcome, webUrl)).join("\n\n");
+	return {
+		conclusion,
+		details_url: first?.runId ? runUrl(webUrl, first.runId) : void 0,
+		name: "anpord",
+		output: {
+			title: TITLES[conclusion],
+			summary: summary.length < 65535 ? summary : `${summary.slice(0, 65521)}${TRUNCATED}`
+		}
+	};
+};
+//#endregion
+//#region src/cli/eval-report.ts
+const reportJson = effect.Schema.encodeSync(effect.Schema.parseJson(effect.Schema.Array(EvalOutcome)));
+const appendSummary = (text) => effect.Effect.gen(function* () {
+	const path = yield* effect.Config.string("GITHUB_STEP_SUMMARY").pipe(effect.Config.option);
+	if (effect.Option.isSome(path) && path.value !== "") yield* (yield* _effect_platform.FileSystem.FileSystem).writeFileString(path.value, `${text}\n\n`, { flag: "a" });
+});
+const reportStarted = (file, id) => effect.Effect.gen(function* () {
+	const url = runUrl(yield* require_config.webUrlConfig, id);
+	yield* note(`${file}: ${url}`);
+	yield* appendSummary(`[Run ${id}](${url}) started.`);
+});
+const writeReport = (outcomes, path) => effect.Effect.gen(function* () {
+	if (effect.Option.isSome(path)) yield* (yield* _effect_platform.FileSystem.FileSystem).writeFileString(path.value, reportJson(outcomes));
+});
+const reportFinished = (outcomes) => effect.Effect.gen(function* () {
+	const report = buildGithubCheck(outcomes, yield* require_config.webUrlConfig);
+	yield* appendSummary(`## ${report.output.title}\n\n${report.output.summary}`);
+});
+//#endregion
 //#region src/cli/eval-run.ts
 const FIRST_POLL = 2e3;
 const SLOWEST_POLL = 1e4;
 const WIDENING = 1.5;
 const running = (run) => run.status === "running";
-const waitForRun = (id, onProgress) => effect.Effect.gen(function* () {
+var EvalWaitTimeout = class extends effect.Data.TaggedError("EvalWaitTimeout") {
+	get message() {
+		return `Timed out after ${this.seconds}s waiting for ${this.runId}. The remote run was not cancelled.`;
+	}
+};
+const waitForRun = (id, onProgress, timeoutSeconds) => effect.Effect.gen(function* () {
 	const api = yield* require_client.AnpordApi;
 	const startedAt = yield* effect.Clock.currentTimeMillis;
 	const gap = yield* effect.Ref.make(FIRST_POLL);
@@ -604,73 +640,28 @@ const waitForRun = (id, onProgress) => effect.Effect.gen(function* () {
 		body: () => waitThenPoll,
 		while: running
 	});
-}).pipe(effect.Effect.withSpan("Cli.waitForRun", { attributes: { runId: id } }));
-effect.Schema.Struct({
-	conclusion: effect.Schema.Literal("failure", "neutral", "success"),
-	details_url: effect.Schema.optional(effect.Schema.String),
-	name: effect.Schema.Literal("anpord"),
-	output: effect.Schema.Struct({
-		summary: effect.Schema.String,
-		title: effect.Schema.String
+}).pipe(effect.Effect.timeoutFail({
+	duration: effect.Duration.seconds(timeoutSeconds),
+	onTimeout: () => new EvalWaitTimeout({
+		runId: id,
+		seconds: timeoutSeconds
 	})
+}), effect.Effect.withSpan("Cli.waitForRun", { attributes: { runId: id } }));
+//#endregion
+//#region src/cli/eval-trigger.ts
+const evalTrigger = effect.Effect.gen(function* () {
+	const github = yield* effect.Config.boolean("GITHUB_ACTIONS").pipe(effect.Config.withDefault(false));
+	const ci = yield* effect.Config.boolean("CI").pipe(effect.Config.withDefault(false));
+	if (!github) return { source: ci ? "ci" : "cli" };
+	const repository = yield* effect.Config.string("GITHUB_REPOSITORY");
+	const runId = yield* effect.Config.string("GITHUB_RUN_ID");
+	const attempt = yield* effect.Config.string("GITHUB_RUN_ATTEMPT").pipe(effect.Config.withDefault("1"));
+	const server = yield* effect.Config.string("GITHUB_SERVER_URL").pipe(effect.Config.withDefault("https://github.com"));
+	return yield* effect.Schema.decodeUnknown(require_evals.EvalTrigger)({
+		source: "ci",
+		url: `${server}/${repository}/actions/runs/${runId}/attempts/${attempt}`
+	});
 });
-const TRUNCATED = "\n\n… truncated";
-const PERCENT = 100;
-const ABSENT = "—";
-const percent = (rate) => rate === void 0 ? ABSENT : `${Math.round(rate * PERCENT)}%`;
-const rateOf = (cell) => cell.distribution === null || cell.distribution.scored === 0 ? ABSENT : percent(cell.distribution.passRate);
-const versionsOf = (comparison) => ({
-	baseline: "baselineHarnessVersion" in comparison && typeof comparison.baselineHarnessVersion === "string" ? comparison.baselineHarnessVersion : void 0,
-	candidate: "candidateHarnessVersion" in comparison && typeof comparison.candidateHarnessVersion === "string" ? comparison.candidateHarnessVersion : void 0
-});
-const versionClause = (run, cell) => {
-	if (cell.comparison === null) return "";
-	const { baseline, candidate } = versionsOf(cell.comparison);
-	const harness = run.tasks[cell.taskIndex]?.harness ?? "harness";
-	return baseline === void 0 || candidate === void 0 || baseline === candidate ? "" : ` (${harness} ${baseline} → ${candidate})`;
-};
-const verdictOf = (run, cell) => cell.comparison === null ? ABSENT : `${cell.comparison.verdict}${versionClause(run, cell)}`;
-const escaped = (text) => text.replaceAll("|", "\\|");
-const rowOf = (run, cell) => `| ${escaped(cell.caseName)} | ${escaped(variantOf(run, cell))} | ${rateOf(cell)} | ${percent(cell.comparison?.baselinePassRate)} | ${verdictOf(run, cell)} |`;
-const tableOf = (file, run) => [
-	`### ${escaped(file)}`,
-	"",
-	...run.failure === null ? [] : [`Run failed: ${run.failure}`, ""],
-	"| Case | Variant | Pass rate | Baseline | Verdict |",
-	"| --- | --- | --- | --- | --- |",
-	...run.cells.map((cell) => rowOf(run, cell))
-].join("\n");
-const truncated = (summary) => summary.length < 65535 ? summary : `${summary.slice(0, 65521)}${TRUNCATED}`;
-const conclusionOf = (cells) => {
-	const verdicts = cells.flatMap((cell) => cell.comparison === null ? [] : [cell.comparison.verdict]);
-	if (verdicts.includes("regressed")) return "failure";
-	return verdicts.every((verdict) => verdict === "incomparable") ? "neutral" : "success";
-};
-const TITLES = {
-	failure: "A cell regressed against its baseline",
-	neutral: "Nothing to compare against a baseline",
-	success: "No cell regressed against its baseline"
-};
-const checkRunOf = (outcomes, webUrl) => {
-	const finished = outcomes.flatMap((outcome) => effect.Option.match(outcome.run, {
-		onNone: () => [],
-		onSome: (run) => [{
-			file: outcome.file,
-			run
-		}]
-	}));
-	const conclusion = conclusionOf(finished.flatMap(({ run }) => run.cells));
-	const first = finished[0];
-	return {
-		conclusion,
-		details_url: first === void 0 ? void 0 : `${webUrl}/evals/${first.run.id}`,
-		name: "anpord",
-		output: {
-			summary: truncated(finished.map(({ file, run }) => tableOf(file, run)).join("\n\n")),
-			title: TITLES[conclusion]
-		}
-	};
-};
 //#endregion
 //#region src/cli/github-check-client.ts
 const API = "https://api.github.com";
@@ -731,64 +722,90 @@ const githubContext = effect.Effect.gen(function* () {
 });
 //#endregion
 //#region src/cli/eval-command.ts
-const asJson$1 = _effect_cli.Options.boolean("json").pipe(_effect_cli.Options.withDescription("Print the finished run as JSON"));
-const evalFile = _effect_cli.Args.text({ name: "file" }).pipe(_effect_cli.Args.withDescription("A TypeScript file that default exports defineEval(...); every *.eval.ts is run when omitted"), _effect_cli.Args.optional);
-const noWait = _effect_cli.Options.boolean("no-wait").pipe(_effect_cli.Options.withDescription("Start the run and print its id, without waiting"));
-const failOn = _effect_cli.Options.choice("fail-on", [
-	"never",
-	"regressed",
-	"unscored"
-]).pipe(_effect_cli.Options.withDescription("What makes the command exit nonzero"), _effect_cli.Options.withDefault("regressed"));
-const runOneEval = (file, options) => effect.Effect.gen(function* () {
-	const api = yield* require_client.AnpordApi;
-	const payload = yield* require_compiler.compileEvalEffect(file);
-	const started = yield* api.evals.start({ payload });
-	if (options.skipWait) {
-		yield* json(started);
-		return {
+const asJson$1 = _effect_cli.Options.boolean("json").pipe(_effect_cli.Options.withDescription("Print each finished run as JSON"));
+const evalFile = _effect_cli.Args.text({ name: "file" }).pipe(_effect_cli.Args.withDescription("A TypeScript eval file; discovers *.eval.ts when omitted"), _effect_cli.Args.optional);
+const noWait = _effect_cli.Options.boolean("no-wait").pipe(_effect_cli.Options.withDescription("Start runs without waiting"));
+const failOn = _effect_cli.Options.choice("fail-on", EvalGate.literals).pipe(_effect_cli.Options.withDescription("strict requires every trial to pass"), _effect_cli.Options.withDefault("strict"));
+const timeout = _effect_cli.Options.integer("timeout").pipe(_effect_cli.Options.withDescription("Maximum seconds to wait per run"), _effect_cli.Options.withSchema(effect.Schema.Int.pipe(effect.Schema.positive())), _effect_cli.Options.withDefault(1200));
+const output = _effect_cli.Options.text("output").pipe(_effect_cli.Options.withDescription("Write a JSON report to this file"), _effect_cli.Options.optional);
+const describe$1 = (error) => error instanceof Error ? error.message : String(error);
+const runOneEval = (file, options, save) => effect.Effect.gen(function* () {
+	let runId = null;
+	return yield* effect.Effect.gen(function* () {
+		const api = yield* require_client.AnpordApi;
+		const payload = yield* require_compiler.compileEvalEffect(file);
+		const trigger = yield* evalTrigger;
+		const started = yield* api.evals.start({ payload: {
+			...payload,
+			trigger
+		} });
+		runId = started.id;
+		yield* reportStarted(file, runId);
+		const pending = {
 			file,
+			runId,
 			problems: [],
-			run: effect.Option.none()
+			run: null
 		};
-	}
-	const live = !options.wantsJson && (yield* attended);
-	if (live || options.label) yield* note(`${file} · run ${started.id}`);
-	const draw = yield* liveGrid(payload.trials, live);
-	const run = yield* waitForRun(started.id, draw);
-	yield* options.wantsJson ? json(run) : note(summaryOf$1(run, payload.trials, live));
-	return {
+		yield* save(pending);
+		if (options.skipWait) {
+			yield* json(started);
+			return pending;
+		}
+		const live = !options.wantsJson && (yield* attended);
+		const draw = yield* liveGrid(payload.trials, live);
+		const run = yield* waitForRun(runId, draw, options.timeoutSeconds);
+		yield* options.wantsJson ? json(run) : note(formatGridSummary(run, payload.trials, live));
+		return {
+			file,
+			runId,
+			run,
+			problems: problemsWith(run, options.gate, {
+				cells: payload.cases.length * payload.tasks.length,
+				trials: payload.trials
+			})
+		};
+	}).pipe(effect.Effect.catchAll((error) => effect.Effect.succeed({
 		file,
-		problems: problemsWith(run, options.gate),
-		run: effect.Option.some(run)
-	};
+		runId,
+		run: null,
+		problems: [describe$1(error)]
+	})));
 });
-const describe$1 = (error) => error instanceof Error ? error.message : String(error);
 const reportToGithub = (outcomes) => effect.Effect.gen(function* () {
-	if (outcomes.every((outcome) => effect.Option.isNone(outcome.run))) return;
 	const context = yield* githubContext;
 	if (effect.Option.isNone(context)) return;
 	const webUrl = yield* require_config.webUrlConfig;
-	yield* postCheckRun(context.value, checkRunOf(outcomes, webUrl));
-	yield* note(`Posted the anpord check on ${context.value.sha}`);
+	yield* postCheckRun(context.value, buildGithubCheck(outcomes, webUrl));
 }).pipe(effect.Effect.catchAll((error) => note(`The GitHub check was not posted. ${describe$1(error)}`)));
 const runEval = _effect_cli.Command.make("eval", {
 	asJson: asJson$1,
 	evalFile,
 	failOn,
-	noWait
-}, ({ asJson: wantsJson, evalFile: file, failOn: gate, noWait: skipWait }) => effect.Effect.gen(function* () {
+	noWait,
+	output,
+	timeout
+}, ({ asJson: wantsJson, evalFile: file, failOn: gate, noWait: skipWait, output: path, timeout: timeoutSeconds }) => effect.Effect.gen(function* () {
 	const files = yield* effect.Option.match(file, {
 		onNone: () => evalFilesIn("."),
 		onSome: (one) => effect.Effect.succeed([one])
 	});
 	if (files.length === 0) return yield* effect.Effect.fail(new NoEvalFiles());
-	const outcomes = yield* effect.Effect.forEach(files, (one) => runOneEval(one, {
-		gate,
-		label: files.length > 1,
-		skipWait,
-		wantsJson
+	const outcomes = [];
+	yield* effect.Effect.forEach(files, (one, index) => effect.Effect.gen(function* () {
+		const save = (outcome) => effect.Effect.gen(function* () {
+			outcomes[index] = outcome;
+			yield* writeReport(outcomes, path);
+		});
+		yield* save(yield* runOneEval(one, {
+			gate,
+			skipWait,
+			timeoutSeconds,
+			wantsJson
+		}, save));
 	}));
-	yield* reportToGithub(outcomes);
+	yield* reportFinished(outcomes);
+	if (!skipWait) yield* reportToGithub(outcomes);
 	return yield* failWhen(outcomes.flatMap((outcome) => outcome.problems));
 }).pipe(effect.Effect.provide(require_config.ClientLayer))).pipe(_effect_cli.Command.withDescription("Compile and run an eval from TypeScript"), _effect_cli.Command.withSubcommands([importEval]));
 //#endregion
@@ -835,12 +852,6 @@ const promote = _effect_cli.Command.make("promote", {
 	} });
 	return yield* note(`${id} v${pin} is now ${to}`);
 })).pipe(_effect_cli.Command.withDescription("Point a channel at a version"));
-/**
-* Read from the stream rather than by opening `/dev/stdin` as a file. The path
-* only names the pipe, so reading it races whoever is writing: a body arriving
-* in more than one chunk, which is what a pipe does under load, was read as
-* whatever had landed by then.
-*/
 const readStdin = effect.Effect.async((resume) => {
 	let body = "";
 	process.stdin.setEncoding("utf8");
@@ -865,8 +876,6 @@ const push = _effect_cli.Command.make("push", {
 	return yield* note(`${id} is now v${prompt.version}`);
 })).pipe(_effect_cli.Command.withDescription("Add a version to a prompt"));
 const out = _effect_cli.Options.file("out").pipe(_effect_cli.Options.withDescription("Where to write the declarations"), _effect_cli.Options.withDefault("anpord-env.d.ts"));
-/** Reading one prompt at a time because the list carries no content, bounded
-* so a large organisation does not open a connection per prompt. */
 const READ_AT_ONCE = 8;
 const writeDeclarations = ({ out: path }) => effect.Effect.gen(function* () {
 	const api = yield* require_client.AnpordApi;
@@ -878,8 +887,6 @@ const writeDeclarations = ({ out: path }) => effect.Effect.gen(function* () {
 });
 const DESCRIPTION = "Write TypeScript declarations for prompt variables";
 const generate = _effect_cli.Command.make("generate", { out }, writeDeclarations).pipe(_effect_cli.Command.withDescription(DESCRIPTION));
-/** A second command rather than an alias, because a command carries one name
-* and the shorter one is what anybody types twice. */
 const gen = _effect_cli.Command.make("gen", { out }, writeDeclarations).pipe(_effect_cli.Command.withDescription(DESCRIPTION));
 const withClient = (command) => _effect_cli.Command.provide(command, require_config.ClientLayer);
 const commands = [