anpord 0.1.12 → 0.1.13

package/dist/{client-BnimnYiJ.cjs → client-C-P2I00z.cjs}

@@ -1,4 +1,4 @@
-const require_evals = require("./evals-B04M0apj.cjs");
+const require_evals = require("./evals-DouMsrsM.cjs");
 let _effect_platform = require("@effect/platform");
 let effect = require("effect");
 //#region ../schema/src/domain/prompts.ts
@@ -135,9 +135,9 @@ const EvalModelsRequest = effect.Schema.Struct({
 	description: "Select a harness whose available models should be listed.",
 	identifier: "EvalModelsRequest"
 });
-const PublicEvalProvider = require_evals.EvalProvider.annotations({
-	description: "A hosted sandbox provider.",
-	identifier: "PublicEvalProvider"
+const PublicEvalSandbox = require_evals.EvalSandbox.annotations({
+	description: "The hosted sandbox a task runs in.",
+	identifier: "PublicEvalSandbox"
 });
 const ListEvalsRequest = effect.Schema.Struct({
 	cursor: effect.Schema.optional(effect.Schema.NullOr(require_evals.EvalPageCursor)),
@@ -162,12 +162,13 @@ const PublicEvalTask = effect.Schema.Struct({
 	harness: require_evals.EvalHarness,
 	model: effect.Schema.String.pipe(effect.Schema.minLength(1)),
 	profile: effect.Schema.optional(require_evals.HarnessProfile),
-	provider: PublicEvalProvider
+	sandbox: effect.Schema.optional(PublicEvalSandbox)
 }).pipe(effect.Schema.filter(require_evals.profileFitsHarness, { message: () => require_evals.PROFILE_HARNESS_RULE })).annotations({
-	description: `A harness, model, and hosted sandbox combination, with an optional profile layered on the harness. ${require_evals.PROFILE_HARNESS_RULE}`,
+	description: `A harness and model, with an optional sandbox and an optional profile layered on the harness. Omit the sandbox to use the default. ${require_evals.PROFILE_HARNESS_RULE}`,
 	identifier: "StartEvalTask"
 });
 const PublicStartEvalRequest = effect.Schema.Struct({
+	trigger: effect.Schema.optional(require_evals.EvalTrigger),
 	cases: effect.Schema.Array(PublicEvalCase).pipe(effect.Schema.minItems(1), effect.Schema.maxItems(100)),
 	name: effect.Schema.optional(require_evals.EvalName),
 	prompt: require_evals.EvalPrompt,

package/dist/{compiler-CBMPXIzV.mjs → compiler-D96wI7UP.mjs}

@@ -1,8 +1,8 @@
-import { _ as HarnessProfile, b as ProfilePath, d as EvalValidator, g as EnvName, x as profileFitsHarness, y as PROFILE_LIMITS } from "./evals-CvOTyvCX.mjs";
+import { S as EvalSourceFiles, _ as HarnessProfile, b as ProfilePath, d as EvalValidator, g as EnvName, x as profileFitsHarness, y as PROFILE_LIMITS } from "./evals-Dcc_x6jW.mjs";
 import { t as EvalJudge } from "./eval-judges-CwTvTq7N.mjs";
 import { a as parseRepo, i as cloneUrlOf, r as repo } from "./source-Fg1h2VNn.mjs";
-import { Data, Effect, Option, Schema } from "effect";
-import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { Config, Data, Effect, Option, Schema } from "effect";
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { existsSync, realpathSync } from "node:fs";
 import { mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
 import { fileURLToPath, pathToFileURL } from "node:url";
@@ -12,12 +12,52 @@ import { tmpdir } from "node:os";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 import { parse, stringify } from "smol-toml";
+//#region src/evals/source-files.ts
+const CODE_FILE = /\.(?:[cm]?[jt]s|[jt]sx)$/;
+const sdkRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const SourceMap = Schema.parseJson(Schema.Struct({
+	version: Schema.Literal(3),
+	sources: Schema.Array(Schema.String),
+	sourcesContent: Schema.Array(Schema.NullOr(Schema.String))
+}));
+const projectRoot = (entry) => {
+	let directory = dirname(entry);
+	while (!existsSync(resolve(directory, "package.json"))) {
+		const parent = dirname(directory);
+		if (parent === directory) return dirname(entry);
+		directory = parent;
+	}
+	return directory;
+};
+const sourceFiles = (map, entry) => {
+	const decoded = Schema.decodeUnknownSync(SourceMap)(map);
+	const root = realpathSync(projectRoot(entry));
+	const entryPath = relative(root, realpathSync(entry)).split(sep).join("/");
+	const files = decoded.sources.flatMap((source, index) => {
+		const absolute = resolve(dirname(entry), source);
+		const path = relative(root, absolute).split(sep).join("/");
+		const content = decoded.sourcesContent[index];
+		if (content == null || path.startsWith("../") || path.split("/").includes("node_modules") || absolute.startsWith(`${sdkRoot}${sep}`) || !CODE_FILE.test(path) || source.includes(":") || basename(path) === "anpord-eval-entry.ts") return [];
+		return [{
+			path,
+			content
+		}];
+	});
+	return Schema.decodeUnknownSync(EvalSourceFiles)(files.toSorted((a, b) => Number(b.path === entryPath) - Number(a.path === entryPath) || a.path.localeCompare(b.path)));
+};
+//#endregion
 //#region src/evals/eval-bundle.ts
 const authoringExports = ["export const defineEval = value => value;", `export { empty, files, repo } from "./source";`].join("\n");
 const authoringDir = dirname(fileURLToPath(import.meta.url));
 const ANPORD_MODULE = /^anpord$/;
 const ANY_MODULE = /.*/;
 const localModules = [
+	{
+		built: "validator-runtime.mjs",
+		filter: /^anpord\/validators\/runtime$/,
+		namespace: "anpord-validator-runtime",
+		source: "validator-runtime.ts"
+	},
 	{
 		built: "validators.mjs",
 		filter: /^anpord\/validators$/,
@@ -91,6 +131,8 @@ const bundle = (contents, entry, options = {}) => Effect.tryPromise({
 		format: "esm",
 		metafile: true,
 		minify: options.minify,
+		outfile: resolve(dirname(entry), "anpord-bundle.mjs"),
+		sourcemap: options.captureSource ? "external" : false,
 		platform: "node",
 		resolveExtensions: [
 			".ts",
@@ -110,7 +152,8 @@ const bundle = (contents, entry, options = {}) => Effect.tryPromise({
 		write: false
 	}).then((result) => ({
 		inputs: Object.keys(result.metafile?.inputs ?? {}).map((path) => resolve(path)),
-		source: result.outputFiles[0]?.text ?? ""
+		source: result.outputFiles.find((file) => file.path.endsWith(".mjs"))?.text ?? "",
+		...options.captureSource ? { sourceFiles: sourceFiles(result.outputFiles.find((file) => file.path.endsWith(".map"))?.text ?? "", entry) } : {}
 	})),
 	catch: (cause) => new Error(`Could not compile ${entry}`, { cause })
 });
@@ -159,65 +202,12 @@ const packageProgram = (root, entry, source) => {
 };
 //#endregion
 //#region src/evals/runner-source.ts
-const runtime = `
-import { access, readFile } from "node:fs/promises";
-import { spawn } from "node:child_process";
-
-const exec = (command) => new Promise((resolve, reject) => {
-  const child = spawn("/bin/sh", ["-lc", command], { cwd: process.cwd() });
-  let stderr = "";
-  let stdout = "";
-  child.stderr.on("data", chunk => { stderr += chunk; });
-  child.stdout.on("data", chunk => { stdout += chunk; });
-  child.on("error", reject);
-  child.on("close", code => resolve({ exitCode: code ?? 1, stderr, stdout }));
-});
-
-const readOrEmpty = (path) =>
-  path === undefined ? Promise.resolve("") : readFile(path, "utf8").then(text => text, () => "");
-
-const calls = async (path, key, value) => {
-  const text = await readOrEmpty(path);
-  const calls = text.trim() === "" ? [] : text.trim().split("\\n").map((line) => JSON.parse(line));
-  return value === undefined ? calls : calls.filter((call) => call[key] === value);
-};
-
-const context = {
-  answer: () => readOrEmpty(process.env.ANPORD_ANSWER_FILE),
-  cli: { calls: cli => calls(".anpord/cli-calls.jsonl", "cli", cli) },
-  exec,
-  exists: path => access(path).then(() => true, () => false),
-  mcp: { calls: server => calls(".anpord/mcp-calls.jsonl", "server", server) },
-  readText: path => readFile(path, "utf8"),
-  prepared: JSON.parse(process.env.ANPORD_PREPARE_VALUE ?? "{}"),
-  transcript: () => readOrEmpty(process.env.ANPORD_TRANSCRIPT_FILE),
-};
-
-try {
-  const raw = await validate(context);
-  const result = typeof raw === "boolean" ? { passed: raw } : raw;
-  if (typeof result?.passed !== "boolean") throw new Error("A validator must return a boolean or { passed, message? }");
-  console.log("ANPORD_VALIDATOR_RESULT=" + JSON.stringify(result));
-} catch (error) {
-  console.error(error instanceof Error ? error.stack : String(error));
-  process.exitCode = 1;
-}
-`;
-const validatorCaseEntry = (entry, index) => `import definition from ${JSON.stringify(entry)};
+const validatorCaseEntry = (entry, index, names = []) => `import definition from ${JSON.stringify(entry)};
+import { runValidators } from "anpord/validators/runtime";
 const selected = definition.cases[${index}].validate;
-const checks = (Array.isArray(selected) ? selected : [selected]).filter(value => typeof value === "function");
-const validate = async context => {
-  const messages = [];
-  for (const check of checks) {
-    const raw = await check(context);
-    const result = typeof raw === "boolean" ? { passed: raw } : raw;
-    if (typeof result?.passed !== "boolean") throw new Error("Invalid validator result");
-    if (result.message) messages.push(result.message);
-    if (!result.passed) return { passed: false, message: messages.join("; ") };
-  }
-  return { passed: true, message: messages.join("; ") || "Code validators passed" };
-};
-${runtime}`;
+const names = ${JSON.stringify(names)};
+const checks = (Array.isArray(selected) ? selected : [selected]).flatMap((validate, index) => typeof validate === "function" ? [{ index, name: names.find(check => check.index === index)?.name ?? "Validator " + (index + 1), validate }] : []);
+await runValidators(checks, definition.captureValidation !== false);`;
 const prepareRuntime = `
 import { access, readFile } from "node:fs/promises";
 import { spawn } from "node:child_process";
@@ -268,11 +258,12 @@ import { runMcpServer } from "anpord/mcp/runtime";
 const server = definition.mcp?.[${index}];
 if (server === undefined) throw new Error("MCP server ${index} is missing");
 runMcpServer(server);`;
-const cliEntry = (entry, index) => `import definition from ${JSON.stringify(entry)};
+const cliEntry = (entry, index) => `import { fileURLToPath } from "node:url";
+import definition from ${JSON.stringify(entry)};
 import { runCli } from "anpord/cli/runtime";
 const cli = definition.cli?.[${index}];
 if (cli === undefined) throw new Error("CLI ${index} is missing");
-await runCli(cli);`;
+await runCli(cli, fileURLToPath(new URL("../../cli-calls.jsonl", import.meta.url)));`;
 //#endregion
 //#region src/evals/cli-profile.ts
 const compileClis = (entry, definitions) => Effect.gen(function* () {
@@ -309,22 +300,37 @@ const withClis = (task, clis) => {
 };
 //#endregion
 //#region src/evals/compile-validator.ts
-const compileValidator = (entry, subject, caseIndex) => Effect.gen(function* () {
+const compileValidator = (entry, subject, caseIndex, captureSource, captureValidation = true) => Effect.gen(function* () {
 	if (subject.validate === void 0) return null;
 	const validations = Array.isArray(subject.validate) ? subject.validate : [subject.validate];
 	if (validations.length === 0 || validations.length > 20) return yield* Effect.fail(/* @__PURE__ */ new Error("Use between 1 and 20 validators"));
 	const judges = yield* Effect.forEach(validations.filter((value) => typeof value !== "function"), (value) => Schema.decodeUnknown(EvalJudge)(value, { onExcessProperty: "error" }));
 	if (new Set(judges.map(({ name }) => name)).size !== judges.length) return yield* Effect.fail(/* @__PURE__ */ new Error("Judge names must be unique within a case"));
-	const checks = validations.some((value) => typeof value === "function") ? [{
+	const manifest = validations.flatMap((value, index) => typeof value === "function" ? [{
+		index,
+		name: value.name || `Validator ${index + 1}`
+	}] : []);
+	const hasCode = manifest.length > 0;
+	const compiled = yield* bundle(hasCode ? validatorCaseEntry(entry, caseIndex, manifest) : definitionEntry(entry), entry, {
+		minify: true,
+		captureSource
+	});
+	const checks = hasCode ? [{
 		name: typeof subject.validate === "function" ? subject.validate.name || subject.name : subject.name,
-		source: (yield* bundle(validatorCaseEntry(entry, caseIndex), entry, { minify: true })).source
+		source: compiled.source,
+		manifest,
+		capture: captureValidation
 	}] : [];
-	return yield* Schema.decodeUnknown(EvalValidator)(judges.length > 0 ? {
-		kind: "judged",
-		name: subject.name,
-		checks,
-		judges
-	} : checks[0]);
+	return yield* Schema.decodeUnknown(EvalValidator)({
+		...judges.length > 0 ? {
+			kind: "judged",
+			name: subject.name,
+			checks,
+			judges
+		} : checks[0],
+		sourceFiles: compiled.sourceFiles,
+		capture: captureValidation
+	});
 });
 //#endregion
 //#region src/evals/definition-loader.ts
@@ -355,26 +361,25 @@ const isDefinition = (value) => Option.isSome(decodeShape(value));
 //#region src/evals/local-repo.ts
 const run = promisify(execFile);
 const git = (args, cwd) => Effect.tryPromise(() => run("git", [...args], { cwd })).pipe(Effect.map(({ stdout }) => stdout.trim()), Effect.orElseSucceed(() => ""));
-const environment = () => globalThis.process?.env ?? {};
 const remoteUrl = (cwd) => Effect.gen(function* () {
-	const { GITHUB_REPOSITORY } = environment();
+	const repository = yield* Config.string("GITHUB_REPOSITORY").pipe(Config.withDefault(""));
 	const origin = yield* git([
 		"remote",
 		"get-url",
 		"origin"
 	], cwd);
-	return origin === "" && GITHUB_REPOSITORY !== void 0 ? GITHUB_REPOSITORY : origin;
+	return origin === "" ? repository : origin;
 });
 const fetchableRef = (cwd) => Effect.gen(function* () {
-	const { GITHUB_HEAD_REF } = environment();
-	if (GITHUB_HEAD_REF !== void 0 && GITHUB_HEAD_REF !== "") return GITHUB_HEAD_REF;
+	const ci = yield* Config.boolean("GITHUB_ACTIONS").pipe(Config.withDefault(false));
 	const [head, onRemote] = yield* Effect.all([git(["rev-parse", "HEAD"], cwd), git([
 		"branch",
 		"--remotes",
 		"--contains",
 		"HEAD"
 	], cwd)], { concurrency: 2 });
-	return head === "" || onRemote === "" ? null : head;
+	if (ci && head === "") return yield* Effect.fail(/* @__PURE__ */ new Error("Check out the commit before running evals in GitHub Actions."));
+	return head === "" || !ci && onRemote === "" ? null : head;
 });
 const localRepo = (cwd) => Effect.gen(function* () {
 	const [remote, ref] = yield* Effect.all([remoteUrl(cwd), fetchableRef(cwd)], { concurrency: 2 });
@@ -698,7 +703,7 @@ const profileTask = (entry, task) => Effect.gen(function* () {
 		harness: task.harness.base,
 		model: task.model,
 		profile,
-		provider: task.provider
+		sandbox: task.sandbox
 	};
 }).pipe(Effect.withSpan("Eval.profileTask", { attributes: { profile: task.harness.profile.name } }));
 //#endregion
@@ -707,7 +712,7 @@ const taskOf = (entry, task, clis, mcp) => Effect.gen(function* () {
 	const compiled = typeof task.harness === "string" ? {
 		harness: task.harness,
 		model: task.model,
-		provider: task.provider
+		sandbox: task.sandbox
 	} : yield* profileTask(entry, {
 		...task,
 		harness: task.harness
@@ -732,7 +737,7 @@ const compileEvalEffect = (path) => Effect.gen(function* () {
 	const fallback = definition.source === void 0 && definition.cases.some((subject) => subject.source === void 0) ? yield* localRepo(dirname(entry)) : Option.none();
 	const cases = yield* Effect.forEach(definition.cases, (subject, caseIndex) => Effect.gen(function* () {
 		if (subject.validate !== void 0 === (typeof subject.verify === "string")) return yield* Effect.fail(/* @__PURE__ */ new Error(`${subject.name} must have exactly one of validate or verify`));
-		const validator = yield* compileValidator(entry, subject, caseIndex);
+		const validator = yield* compileValidator(entry, subject, caseIndex, definition.captureSource !== false, definition.captureValidation !== false);
 		const prepare = typeof subject.prepare === "function" ? yield* bundledCaseModule(entry, loaded.inputs, subject.prepare.name || `${subject.name}-prepare`, prepareEntry) : null;
 		return {
 			...subject.cache === void 0 ? {} : { cache: subject.cache },

package/dist/{compiler-BFxukiJy.cjs → compiler-uwb1pkdK.cjs}

@@ -1,4 +1,4 @@
-const require_evals = require("./evals-B04M0apj.cjs");
+const require_evals = require("./evals-DouMsrsM.cjs");
 const require_eval_judges = require("./eval-judges-BkmiVLLq.cjs");
 const require_source = require("./source-DvlGawEw.cjs");
 let effect = require("effect");
@@ -12,12 +12,52 @@ let node_os = require("node:os");
 let node_child_process = require("node:child_process");
 let node_util = require("node:util");
 let smol_toml = require("smol-toml");
+//#region src/evals/source-files.ts
+const CODE_FILE = /\.(?:[cm]?[jt]s|[jt]sx)$/;
+const sdkRoot = (0, node_path.resolve)((0, node_path.dirname)((0, node_url.fileURLToPath)(require("url").pathToFileURL(__filename).href)), "..");
+const SourceMap = effect.Schema.parseJson(effect.Schema.Struct({
+	version: effect.Schema.Literal(3),
+	sources: effect.Schema.Array(effect.Schema.String),
+	sourcesContent: effect.Schema.Array(effect.Schema.NullOr(effect.Schema.String))
+}));
+const projectRoot = (entry) => {
+	let directory = (0, node_path.dirname)(entry);
+	while (!(0, node_fs.existsSync)((0, node_path.resolve)(directory, "package.json"))) {
+		const parent = (0, node_path.dirname)(directory);
+		if (parent === directory) return (0, node_path.dirname)(entry);
+		directory = parent;
+	}
+	return directory;
+};
+const sourceFiles = (map, entry) => {
+	const decoded = effect.Schema.decodeUnknownSync(SourceMap)(map);
+	const root = (0, node_fs.realpathSync)(projectRoot(entry));
+	const entryPath = (0, node_path.relative)(root, (0, node_fs.realpathSync)(entry)).split(node_path.sep).join("/");
+	const files = decoded.sources.flatMap((source, index) => {
+		const absolute = (0, node_path.resolve)((0, node_path.dirname)(entry), source);
+		const path = (0, node_path.relative)(root, absolute).split(node_path.sep).join("/");
+		const content = decoded.sourcesContent[index];
+		if (content == null || path.startsWith("../") || path.split("/").includes("node_modules") || absolute.startsWith(`${sdkRoot}${node_path.sep}`) || !CODE_FILE.test(path) || source.includes(":") || (0, node_path.basename)(path) === "anpord-eval-entry.ts") return [];
+		return [{
+			path,
+			content
+		}];
+	});
+	return effect.Schema.decodeUnknownSync(require_evals.EvalSourceFiles)(files.toSorted((a, b) => Number(b.path === entryPath) - Number(a.path === entryPath) || a.path.localeCompare(b.path)));
+};
+//#endregion
 //#region src/evals/eval-bundle.ts
 const authoringExports = ["export const defineEval = value => value;", `export { empty, files, repo } from "./source";`].join("\n");
 const authoringDir = (0, node_path.dirname)((0, node_url.fileURLToPath)(require("url").pathToFileURL(__filename).href));
 const ANPORD_MODULE = /^anpord$/;
 const ANY_MODULE = /.*/;
 const localModules = [
+	{
+		built: "validator-runtime.mjs",
+		filter: /^anpord\/validators\/runtime$/,
+		namespace: "anpord-validator-runtime",
+		source: "validator-runtime.ts"
+	},
 	{
 		built: "validators.mjs",
 		filter: /^anpord\/validators$/,
@@ -91,6 +131,8 @@ const bundle = (contents, entry, options = {}) => effect.Effect.tryPromise({
 		format: "esm",
 		metafile: true,
 		minify: options.minify,
+		outfile: (0, node_path.resolve)((0, node_path.dirname)(entry), "anpord-bundle.mjs"),
+		sourcemap: options.captureSource ? "external" : false,
 		platform: "node",
 		resolveExtensions: [
 			".ts",
@@ -110,7 +152,8 @@ const bundle = (contents, entry, options = {}) => effect.Effect.tryPromise({
 		write: false
 	}).then((result) => ({
 		inputs: Object.keys(result.metafile?.inputs ?? {}).map((path) => (0, node_path.resolve)(path)),
-		source: result.outputFiles[0]?.text ?? ""
+		source: result.outputFiles.find((file) => file.path.endsWith(".mjs"))?.text ?? "",
+		...options.captureSource ? { sourceFiles: sourceFiles(result.outputFiles.find((file) => file.path.endsWith(".map"))?.text ?? "", entry) } : {}
 	})),
 	catch: (cause) => new Error(`Could not compile ${entry}`, { cause })
 });
@@ -159,65 +202,12 @@ const packageProgram = (root, entry, source) => {
 };
 //#endregion
 //#region src/evals/runner-source.ts
-const runtime = `
-import { access, readFile } from "node:fs/promises";
-import { spawn } from "node:child_process";
-
-const exec = (command) => new Promise((resolve, reject) => {
-  const child = spawn("/bin/sh", ["-lc", command], { cwd: process.cwd() });
-  let stderr = "";
-  let stdout = "";
-  child.stderr.on("data", chunk => { stderr += chunk; });
-  child.stdout.on("data", chunk => { stdout += chunk; });
-  child.on("error", reject);
-  child.on("close", code => resolve({ exitCode: code ?? 1, stderr, stdout }));
-});
-
-const readOrEmpty = (path) =>
-  path === undefined ? Promise.resolve("") : readFile(path, "utf8").then(text => text, () => "");
-
-const calls = async (path, key, value) => {
-  const text = await readOrEmpty(path);
-  const calls = text.trim() === "" ? [] : text.trim().split("\\n").map((line) => JSON.parse(line));
-  return value === undefined ? calls : calls.filter((call) => call[key] === value);
-};
-
-const context = {
-  answer: () => readOrEmpty(process.env.ANPORD_ANSWER_FILE),
-  cli: { calls: cli => calls(".anpord/cli-calls.jsonl", "cli", cli) },
-  exec,
-  exists: path => access(path).then(() => true, () => false),
-  mcp: { calls: server => calls(".anpord/mcp-calls.jsonl", "server", server) },
-  readText: path => readFile(path, "utf8"),
-  prepared: JSON.parse(process.env.ANPORD_PREPARE_VALUE ?? "{}"),
-  transcript: () => readOrEmpty(process.env.ANPORD_TRANSCRIPT_FILE),
-};
-
-try {
-  const raw = await validate(context);
-  const result = typeof raw === "boolean" ? { passed: raw } : raw;
-  if (typeof result?.passed !== "boolean") throw new Error("A validator must return a boolean or { passed, message? }");
-  console.log("ANPORD_VALIDATOR_RESULT=" + JSON.stringify(result));
-} catch (error) {
-  console.error(error instanceof Error ? error.stack : String(error));
-  process.exitCode = 1;
-}
-`;
-const validatorCaseEntry = (entry, index) => `import definition from ${JSON.stringify(entry)};
+const validatorCaseEntry = (entry, index, names = []) => `import definition from ${JSON.stringify(entry)};
+import { runValidators } from "anpord/validators/runtime";
 const selected = definition.cases[${index}].validate;
-const checks = (Array.isArray(selected) ? selected : [selected]).filter(value => typeof value === "function");
-const validate = async context => {
-  const messages = [];
-  for (const check of checks) {
-    const raw = await check(context);
-    const result = typeof raw === "boolean" ? { passed: raw } : raw;
-    if (typeof result?.passed !== "boolean") throw new Error("Invalid validator result");
-    if (result.message) messages.push(result.message);
-    if (!result.passed) return { passed: false, message: messages.join("; ") };
-  }
-  return { passed: true, message: messages.join("; ") || "Code validators passed" };
-};
-${runtime}`;
+const names = ${JSON.stringify(names)};
+const checks = (Array.isArray(selected) ? selected : [selected]).flatMap((validate, index) => typeof validate === "function" ? [{ index, name: names.find(check => check.index === index)?.name ?? "Validator " + (index + 1), validate }] : []);
+await runValidators(checks, definition.captureValidation !== false);`;
 const prepareRuntime = `
 import { access, readFile } from "node:fs/promises";
 import { spawn } from "node:child_process";
@@ -268,11 +258,12 @@ import { runMcpServer } from "anpord/mcp/runtime";
 const server = definition.mcp?.[${index}];
 if (server === undefined) throw new Error("MCP server ${index} is missing");
 runMcpServer(server);`;
-const cliEntry = (entry, index) => `import definition from ${JSON.stringify(entry)};
+const cliEntry = (entry, index) => `import { fileURLToPath } from "node:url";
+import definition from ${JSON.stringify(entry)};
 import { runCli } from "anpord/cli/runtime";
 const cli = definition.cli?.[${index}];
 if (cli === undefined) throw new Error("CLI ${index} is missing");
-await runCli(cli);`;
+await runCli(cli, fileURLToPath(new URL("../../cli-calls.jsonl", import.meta.url)));`;
 //#endregion
 //#region src/evals/cli-profile.ts
 const compileClis = (entry, definitions) => effect.Effect.gen(function* () {
@@ -309,22 +300,37 @@ const withClis = (task, clis) => {
 };
 //#endregion
 //#region src/evals/compile-validator.ts
-const compileValidator = (entry, subject, caseIndex) => effect.Effect.gen(function* () {
+const compileValidator = (entry, subject, caseIndex, captureSource, captureValidation = true) => effect.Effect.gen(function* () {
 	if (subject.validate === void 0) return null;
 	const validations = Array.isArray(subject.validate) ? subject.validate : [subject.validate];
 	if (validations.length === 0 || validations.length > 20) return yield* effect.Effect.fail(/* @__PURE__ */ new Error("Use between 1 and 20 validators"));
 	const judges = yield* effect.Effect.forEach(validations.filter((value) => typeof value !== "function"), (value) => effect.Schema.decodeUnknown(require_eval_judges.EvalJudge)(value, { onExcessProperty: "error" }));
 	if (new Set(judges.map(({ name }) => name)).size !== judges.length) return yield* effect.Effect.fail(/* @__PURE__ */ new Error("Judge names must be unique within a case"));
-	const checks = validations.some((value) => typeof value === "function") ? [{
+	const manifest = validations.flatMap((value, index) => typeof value === "function" ? [{
+		index,
+		name: value.name || `Validator ${index + 1}`
+	}] : []);
+	const hasCode = manifest.length > 0;
+	const compiled = yield* bundle(hasCode ? validatorCaseEntry(entry, caseIndex, manifest) : definitionEntry(entry), entry, {
+		minify: true,
+		captureSource
+	});
+	const checks = hasCode ? [{
 		name: typeof subject.validate === "function" ? subject.validate.name || subject.name : subject.name,
-		source: (yield* bundle(validatorCaseEntry(entry, caseIndex), entry, { minify: true })).source
+		source: compiled.source,
+		manifest,
+		capture: captureValidation
 	}] : [];
-	return yield* effect.Schema.decodeUnknown(require_evals.EvalValidator)(judges.length > 0 ? {
-		kind: "judged",
-		name: subject.name,
-		checks,
-		judges
-	} : checks[0]);
+	return yield* effect.Schema.decodeUnknown(require_evals.EvalValidator)({
+		...judges.length > 0 ? {
+			kind: "judged",
+			name: subject.name,
+			checks,
+			judges
+		} : checks[0],
+		sourceFiles: compiled.sourceFiles,
+		capture: captureValidation
+	});
 });
 //#endregion
 //#region src/evals/definition-loader.ts
@@ -355,26 +361,25 @@ const isDefinition = (value) => effect.Option.isSome(decodeShape(value));
 //#region src/evals/local-repo.ts
 const run = (0, node_util.promisify)(node_child_process.execFile);
 const git = (args, cwd) => effect.Effect.tryPromise(() => run("git", [...args], { cwd })).pipe(effect.Effect.map(({ stdout }) => stdout.trim()), effect.Effect.orElseSucceed(() => ""));
-const environment = () => globalThis.process?.env ?? {};
 const remoteUrl = (cwd) => effect.Effect.gen(function* () {
-	const { GITHUB_REPOSITORY } = environment();
+	const repository = yield* effect.Config.string("GITHUB_REPOSITORY").pipe(effect.Config.withDefault(""));
 	const origin = yield* git([
 		"remote",
 		"get-url",
 		"origin"
 	], cwd);
-	return origin === "" && GITHUB_REPOSITORY !== void 0 ? GITHUB_REPOSITORY : origin;
+	return origin === "" ? repository : origin;
 });
 const fetchableRef = (cwd) => effect.Effect.gen(function* () {
-	const { GITHUB_HEAD_REF } = environment();
-	if (GITHUB_HEAD_REF !== void 0 && GITHUB_HEAD_REF !== "") return GITHUB_HEAD_REF;
+	const ci = yield* effect.Config.boolean("GITHUB_ACTIONS").pipe(effect.Config.withDefault(false));
 	const [head, onRemote] = yield* effect.Effect.all([git(["rev-parse", "HEAD"], cwd), git([
 		"branch",
 		"--remotes",
 		"--contains",
 		"HEAD"
 	], cwd)], { concurrency: 2 });
-	return head === "" || onRemote === "" ? null : head;
+	if (ci && head === "") return yield* effect.Effect.fail(/* @__PURE__ */ new Error("Check out the commit before running evals in GitHub Actions."));
+	return head === "" || !ci && onRemote === "" ? null : head;
 });
 const localRepo = (cwd) => effect.Effect.gen(function* () {
 	const [remote, ref] = yield* effect.Effect.all([remoteUrl(cwd), fetchableRef(cwd)], { concurrency: 2 });
@@ -698,7 +703,7 @@ const profileTask = (entry, task) => effect.Effect.gen(function* () {
 		harness: task.harness.base,
 		model: task.model,
 		profile,
-		provider: task.provider
+		sandbox: task.sandbox
 	};
 }).pipe(effect.Effect.withSpan("Eval.profileTask", { attributes: { profile: task.harness.profile.name } }));
 //#endregion
@@ -707,7 +712,7 @@ const taskOf = (entry, task, clis, mcp) => effect.Effect.gen(function* () {
 	const compiled = typeof task.harness === "string" ? {
 		harness: task.harness,
 		model: task.model,
-		provider: task.provider
+		sandbox: task.sandbox
 	} : yield* profileTask(entry, {
 		...task,
 		harness: task.harness
@@ -732,7 +737,7 @@ const compileEvalEffect = (path) => effect.Effect.gen(function* () {
 	const fallback = definition.source === void 0 && definition.cases.some((subject) => subject.source === void 0) ? yield* localRepo((0, node_path.dirname)(entry)) : effect.Option.none();
 	const cases = yield* effect.Effect.forEach(definition.cases, (subject, caseIndex) => effect.Effect.gen(function* () {
 		if (subject.validate !== void 0 === (typeof subject.verify === "string")) return yield* effect.Effect.fail(/* @__PURE__ */ new Error(`${subject.name} must have exactly one of validate or verify`));
-		const validator = yield* compileValidator(entry, subject, caseIndex);
+		const validator = yield* compileValidator(entry, subject, caseIndex, definition.captureSource !== false, definition.captureValidation !== false);
 		const prepare = typeof subject.prepare === "function" ? yield* bundledCaseModule(entry, loaded.inputs, subject.prepare.name || `${subject.name}-prepare`, prepareEntry) : null;
 		return {
 			...subject.cache === void 0 ? {} : { cache: subject.cache },

package/dist/config.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_client = require("./client-BnimnYiJ.cjs");
+const require_client = require("./client-C-P2I00z.cjs");
 let effect = require("effect");
 //#region src/client/config.ts
 const apiKeyConfig = effect.Config.redacted("ANPORD_API_KEY");

package/dist/config.d.cts

@@ -1,4 +1,4 @@
-import { t as AnpordApi } from "./client-CFTj9ad0.cjs";
+import { t as AnpordApi } from "./client-BirFO3jz.cjs";
 import { Config, Layer } from "effect";
 //#region src/client/config.d.ts
 declare const apiKeyConfig: Config.Config<import("effect/Redacted").Redacted<string>>;

package/dist/config.d.mts

@@ -1,4 +1,4 @@
-import { t as AnpordApi } from "./client-Byoa41pX.mjs";
+import { t as AnpordApi } from "./client-BgRs8JvW.mjs";
 import { Config, Layer } from "effect";
 //#region src/client/config.d.ts
 declare const apiKeyConfig: Config.Config<import("effect/Redacted").Redacted<string>>;

package/dist/config.mjs

@@ -1,4 +1,4 @@
-import { n as DEFAULT_BASE_URL, r as layer } from "./client-D22IiHGL.mjs";
+import { n as DEFAULT_BASE_URL, r as layer } from "./client-BfdjFtlV.mjs";
 import { Config, Effect, Layer } from "effect";
 //#region src/client/config.ts
 const apiKeyConfig = Config.redacted("ANPORD_API_KEY");