anpord 0.1.11 → 0.1.13

package/dist/client-C-P2I00z.cjs

@@ -0,0 +1,346 @@
+const require_evals = require("./evals-DouMsrsM.cjs");
+let _effect_platform = require("@effect/platform");
+let effect = require("effect");
+//#region ../schema/src/domain/prompts.ts
+const ChannelName = effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(36), effect.Schema.pattern(/^[a-z0-9][a-z0-9_-]*$/, { message: () => "Channel must be lowercase alphanumeric, optionally with - or _" }), effect.Schema.brand("ChannelName")).annotations({ description: "Addresses a version. `latest` is derived from the highest version rather than stored, so it cannot drift from the version table." });
+ChannelName.make("production");
+ChannelName.make("latest");
+const PromptId = effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(255), effect.Schema.pattern(/^[a-z0-9][a-z0-9/_-]*$/, { message: () => "Prompt id must be lowercase alphanumeric, optionally with / _ or -" }), effect.Schema.brand("PromptId"));
+const PromptName = effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(255), effect.Schema.brand("PromptName"));
+const VersionNumber = effect.Schema.Int.pipe(effect.Schema.positive(), effect.Schema.brand("VersionNumber"));
+effect.Schema.NumberFromString.pipe(effect.Schema.int(), effect.Schema.positive(), effect.Schema.brand("VersionNumber"));
+effect.Schema.NumberFromString.pipe(effect.Schema.int(), effect.Schema.positive(), effect.Schema.lessThanOrEqualTo(100));
+const Timestamp = effect.Schema.Union(effect.Schema.DateFromSelf, effect.Schema.Date);
+const CommitMessage = effect.Schema.String.pipe(effect.Schema.maxLength(500));
+const PromptContent = effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(262144, { message: () => "Prompt content is longer than 256 KB" }));
+const PromptDescription = effect.Schema.String.pipe(effect.Schema.maxLength(2e3));
+const PromptConfig = effect.Schema.Record({
+	key: effect.Schema.String,
+	value: effect.Schema.Unknown
+});
+const Author = effect.Schema.Struct({
+	image: effect.Schema.NullOr(effect.Schema.String),
+	name: effect.Schema.String
+});
+effect.Schema.Struct({
+	author: effect.Schema.NullOr(Author),
+	channel: effect.Schema.NullOr(ChannelName),
+	commitMessage: effect.Schema.NullOr(CommitMessage),
+	config: PromptConfig,
+	content: effect.Schema.String,
+	createdAt: Timestamp,
+	id: PromptId,
+	name: PromptName,
+	version: VersionNumber,
+	versionId: effect.Schema.String
+});
+effect.Schema.Struct({
+	channel: ChannelName,
+	updatedAt: Timestamp,
+	updatedBy: effect.Schema.NullOr(Author),
+	version: VersionNumber
+});
+const PromptSummary = effect.Schema.Struct({
+	author: effect.Schema.NullOr(Author),
+	description: effect.Schema.NullOr(effect.Schema.String),
+	id: PromptId,
+	latestVersion: effect.Schema.NullOr(VersionNumber),
+	name: PromptName,
+	productionVersion: effect.Schema.NullOr(VersionNumber),
+	updatedAt: Timestamp
+});
+effect.Schema.Struct({
+	id: PromptId,
+	updatedAt: Timestamp
+});
+effect.Schema.Literal("all", "draft", "live");
+effect.Schema.Literal("name", "updated");
+effect.Schema.Struct({
+	items: effect.Schema.Array(PromptSummary),
+	nextCursor: effect.Schema.NullOr(effect.Schema.String)
+});
+effect.Schema.Struct({
+	commitMessage: effect.Schema.optional(CommitMessage),
+	config: effect.Schema.optional(PromptConfig),
+	content: PromptContent,
+	description: effect.Schema.optional(PromptDescription),
+	id: PromptId,
+	name: PromptName,
+	publish: effect.Schema.optional(effect.Schema.Boolean)
+});
+effect.Schema.Struct({
+	commitMessage: effect.Schema.optional(CommitMessage),
+	config: effect.Schema.optional(PromptConfig),
+	content: PromptContent,
+	publish: effect.Schema.optional(effect.Schema.Boolean)
+});
+effect.Schema.Struct({
+	commitMessage: effect.Schema.optional(CommitMessage),
+	config: effect.Schema.optional(PromptConfig),
+	content: PromptContent
+});
+effect.Schema.Struct({
+	description: effect.Schema.optional(PromptDescription),
+	id: effect.Schema.optional(PromptId),
+	name: effect.Schema.optional(PromptName)
+});
+effect.Schema.Struct({
+	channel: ChannelName,
+	version: VersionNumber
+});
+effect.Schema.Struct({
+	channel: effect.Schema.optional(ChannelName),
+	version: effect.Schema.optional(VersionNumber)
+});
+//#endregion
+//#region ../schema/src/domain/errors.ts
+var NotFound = class extends effect.Schema.TaggedError()("NotFound", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 404 })) {};
+var Conflict = class extends effect.Schema.TaggedError()("Conflict", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 409 })) {};
+var BadRequest = class extends effect.Schema.TaggedError()("BadRequest", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 400 })) {};
+var Unauthorized = class extends effect.Schema.TaggedError()("Unauthorized", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 401 })) {};
+var Forbidden = class extends effect.Schema.TaggedError()("Forbidden", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 403 })) {};
+effect.Schema.TaggedError()("InternalError", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 500 }));
+//#endregion
+//#region ../schema/src/internal/authentication.ts
+var CurrentActor = class extends effect.Context.Tag("@anpord/schema/CurrentActor")() {};
+_effect_platform.HttpApiMiddleware.Tag()("@anpord/schema/Authentication", {
+	failure: Unauthorized,
+	provides: CurrentActor,
+	security: { session: _effect_platform.HttpApiSecurity.apiKey({
+		in: "cookie",
+		key: "anpord.session_token"
+	}) }
+});
+//#endregion
+//#region ../schema/src/public/authentication.ts
+var ApiKeyAuthentication = class extends _effect_platform.HttpApiMiddleware.Tag()("@anpord/schema/ApiKeyAuthentication", {
+	failure: Unauthorized,
+	provides: CurrentActor,
+	security: { bearer: _effect_platform.HttpApiSecurity.bearer }
+}) {};
+//#endregion
+//#region ../schema/src/public/evals-api.ts
+const EvalRunRequest = effect.Schema.Struct({ id: effect.Schema.String }).annotations({
+	description: "Select an eval run by id.",
+	identifier: "EvalRunRequest"
+});
+const EvalCellRequest = effect.Schema.Struct({ cellKey: effect.Schema.String }).annotations({
+	description: "Select an eval cell by its stable key.",
+	identifier: "EvalCellRequest"
+});
+const EvalModelsRequest = effect.Schema.Struct({
+	harness: require_evals.EvalHarness,
+	q: effect.Schema.optional(effect.Schema.String)
+}).annotations({
+	description: "Select a harness whose available models should be listed.",
+	identifier: "EvalModelsRequest"
+});
+const PublicEvalSandbox = require_evals.EvalSandbox.annotations({
+	description: "The hosted sandbox a task runs in.",
+	identifier: "PublicEvalSandbox"
+});
+const ListEvalsRequest = effect.Schema.Struct({
+	cursor: effect.Schema.optional(effect.Schema.NullOr(require_evals.EvalPageCursor)),
+	limit: effect.Schema.optional(effect.Schema.Int)
+}).annotations({
+	description: "Where to read from, and how much.",
+	identifier: "ListEvalsRequest"
+});
+const PublicEvalCase = effect.Schema.Struct({
+	cache: effect.Schema.optional(require_evals.CaseCache),
+	name: require_evals.EvalCaseName,
+	prepare: effect.Schema.optional(effect.Schema.NullOr(require_evals.EvalPrepare)),
+	source: effect.Schema.optional(require_evals.EvalSource),
+	validator: effect.Schema.optional(effect.Schema.NullOr(require_evals.EvalValidator)),
+	variables: effect.Schema.optional(require_evals.EvalVariables),
+	verify: effect.Schema.NullOr(require_evals.EvalVerify)
+}).pipe(effect.Schema.filter(({ validator, verify }) => !(validator !== void 0 && validator !== null && verify !== null), { message: () => "Use either validator or verify, not both." })).annotations({
+	description: "A task, workspace source, setup command, and verifier.",
+	identifier: "StartEvalCase"
+});
+const PublicEvalTask = effect.Schema.Struct({
+	harness: require_evals.EvalHarness,
+	model: effect.Schema.String.pipe(effect.Schema.minLength(1)),
+	profile: effect.Schema.optional(require_evals.HarnessProfile),
+	sandbox: effect.Schema.optional(PublicEvalSandbox)
+}).pipe(effect.Schema.filter(require_evals.profileFitsHarness, { message: () => require_evals.PROFILE_HARNESS_RULE })).annotations({
+	description: `A harness and model, with an optional sandbox and an optional profile layered on the harness. Omit the sandbox to use the default. ${require_evals.PROFILE_HARNESS_RULE}`,
+	identifier: "StartEvalTask"
+});
+const PublicStartEvalRequest = effect.Schema.Struct({
+	trigger: effect.Schema.optional(require_evals.EvalTrigger),
+	cases: effect.Schema.Array(PublicEvalCase).pipe(effect.Schema.minItems(1), effect.Schema.maxItems(100)),
+	name: effect.Schema.optional(require_evals.EvalName),
+	prompt: require_evals.EvalPrompt,
+	tasks: effect.Schema.Array(PublicEvalTask).pipe(effect.Schema.minItems(1), effect.Schema.maxItems(20)),
+	trials: effect.Schema.Int.pipe(effect.Schema.between(1, 10))
+}).annotations({
+	description: `Start a grid with at most 100 total case, task, and trial combinations.`,
+	identifier: "StartEvalRequest"
+});
+var PublicEvalsGroup = class extends _effect_platform.HttpApiGroup.make("evals").add(_effect_platform.HttpApiEndpoint.post("list", "/evals.list").setPayload(ListEvalsRequest).addSuccess(require_evals.EvalRunPage).annotate(_effect_platform.OpenApi.Summary, "List eval runs").annotate(_effect_platform.OpenApi.Description, "Newest first. Pass the `next` cursor from a response to read the page after it; a null `next` means there are no more.")).add(_effect_platform.HttpApiEndpoint.post("start", "/evals.start").setPayload(PublicStartEvalRequest).addSuccess(require_evals.StartedEval).annotate(_effect_platform.OpenApi.Summary, "Start an eval run").annotate(_effect_platform.OpenApi.Description, `Starts the grid and returns its id while trials continue in the background. ${require_evals.PROFILE_HARNESS_RULE}`)).add(_effect_platform.HttpApiEndpoint.post("get", "/evals.get").setPayload(EvalRunRequest).addSuccess(require_evals.EvalRun).annotate(_effect_platform.OpenApi.Summary, "Get an eval run")).add(_effect_platform.HttpApiEndpoint.post("cellHistory", "/evals.cellHistory").setPayload(EvalCellRequest).addSuccess(effect.Schema.Array(require_evals.EvalCellHistoryEntry)).annotate(_effect_platform.OpenApi.Summary, "List a cell's history").annotate(_effect_platform.OpenApi.Description, "Returns the 20 most recent results.")).add(_effect_platform.HttpApiEndpoint.post("rerunCell", "/evals.rerunCell").setPayload(effect.Schema.extend(EvalRunRequest, effect.Schema.extend(EvalCellRequest, require_evals.RerunCellRequest))).addSuccess(require_evals.StartedEval).annotate(_effect_platform.OpenApi.Summary, "Rerun one cell")).add(_effect_platform.HttpApiEndpoint.post("models", "/evals.models").setPayload(EvalModelsRequest).addSuccess(require_evals.ModelCatalogue).annotate(_effect_platform.OpenApi.Summary, "List models available to the harness").annotate(_effect_platform.OpenApi.Description, "The command harness has no catalogue of its own, so its list is empty: the model is whatever the profile's run command reads from ANPORD_MODEL.")).addError(BadRequest).addError(Conflict).addError(Forbidden).addError(NotFound).middleware(ApiKeyAuthentication).annotate(_effect_platform.OpenApi.Title, "Evals").annotate(_effect_platform.OpenApi.Description, "Run cases across harness, model, and sandbox combinations and compare the results with their baselines.") {};
+//#endregion
+//#region ../schema/src/public/requests.ts
+const GetPromptRequest = effect.Schema.Struct({
+	channel: effect.Schema.optional(ChannelName),
+	id: PromptId,
+	includeVersions: effect.Schema.optional(effect.Schema.Boolean),
+	version: effect.Schema.optional(VersionNumber)
+}).annotations({
+	description: "Resolve a prompt. Give a version to pin one, a channel to follow one, or neither for the organization's default channel.",
+	identifier: "GetPromptRequest"
+});
+effect.Schema.Struct({ id: PromptId }).annotations({
+	description: "Show a prompt's version history.",
+	identifier: "ListVersionsRequest"
+});
+const ListPromptsRequest = effect.Schema.Record({
+	key: effect.Schema.String,
+	value: effect.Schema.Never
+}).annotations({
+	description: "No parameters; returns up to 100 prompts in the organization.",
+	identifier: "ListPromptsRequest"
+});
+const CreatePromptRequest = effect.Schema.Struct({
+	config: effect.Schema.optional(PromptConfig),
+	content: PromptContent,
+	description: effect.Schema.optional(effect.Schema.String),
+	id: PromptId,
+	message: effect.Schema.optional(CommitMessage),
+	name: PromptName
+}).annotations({
+	description: "Create a prompt and publish its first version to the default channel.",
+	identifier: "CreatePromptRequest"
+});
+const UpdatePromptRequest = effect.Schema.Struct({
+	config: effect.Schema.optional(PromptConfig),
+	content: PromptContent,
+	id: PromptId,
+	message: effect.Schema.optional(CommitMessage)
+}).annotations({
+	description: "Add a version to a prompt. The new version becomes latest, and omitted config is stored as an empty object.",
+	identifier: "UpdatePromptRequest"
+});
+const PromotePromptRequest = effect.Schema.Struct({
+	channel: ChannelName,
+	id: PromptId,
+	version: VersionNumber
+}).annotations({
+	description: "Point a channel at a version.",
+	identifier: "PromotePromptRequest"
+});
+effect.Schema.Struct({ id: PromptId }).annotations({
+	description: "Archive a prompt. Existing selectors stay readable.",
+	identifier: "ArchivePromptRequest"
+});
+const Ok = effect.Schema.Struct({ ok: effect.Schema.Literal(true) }).annotations({
+	description: "The operation succeeded.",
+	identifier: "Ok"
+});
+//#endregion
+//#region ../schema/src/public/shapes.ts
+const Instant = effect.Schema.DateTimeUtc.annotations({
+	description: "An ISO-8601 timestamp in UTC.",
+	identifier: "Instant",
+	jsonSchema: { format: "date-time" }
+});
+const PublicPrompt = effect.Schema.Struct({
+	channel: effect.Schema.NullOr(ChannelName),
+	config: PromptConfig,
+	content: effect.Schema.String,
+	createdAt: Instant,
+	id: PromptId,
+	message: effect.Schema.NullOr(CommitMessage),
+	name: PromptName,
+	version: VersionNumber
+}).annotations({
+	description: "A prompt resolved at a specific version.",
+	identifier: "ResolvedPrompt"
+});
+const PublicVersion = effect.Schema.Struct({
+	createdAt: Instant,
+	message: effect.Schema.NullOr(CommitMessage),
+	version: VersionNumber
+}).annotations({
+	description: "One entry in a prompt's history.",
+	identifier: "Version"
+});
+const PublicPromptWithVersions = effect.Schema.extend(PublicPrompt, effect.Schema.Struct({ versions: effect.Schema.optional(effect.Schema.Array(PublicVersion)) })).annotations({
+	description: "A prompt, with its version history when `includeVersions` is set.",
+	identifier: "Prompt"
+});
+const PublicPromptSummary = effect.Schema.Struct({
+	id: PromptId,
+	latestVersion: effect.Schema.NullOr(VersionNumber),
+	name: PromptName,
+	productionVersion: effect.Schema.NullOr(VersionNumber),
+	updatedAt: Instant
+}).annotations({
+	description: "A prompt without its content.",
+	identifier: "PromptSummary"
+});
+const PromptList = effect.Schema.Struct({ data: effect.Schema.Array(PublicPromptSummary) }).annotations({
+	description: "Every prompt in the organization.",
+	identifier: "PromptList"
+});
+//#endregion
+//#region ../schema/src/public/prompts-api.ts
+var PublicPromptsGroup = class extends _effect_platform.HttpApiGroup.make("prompts").add(_effect_platform.HttpApiEndpoint.post("get", "/prompts.get").setPayload(GetPromptRequest).addSuccess(PublicPromptWithVersions).annotate(_effect_platform.OpenApi.Summary, "Resolve a prompt").annotate(_effect_platform.OpenApi.Description, "Returns the content a caller should send to a model. With no selector this follows the organization's default channel.")).add(_effect_platform.HttpApiEndpoint.post("list", "/prompts.list").setPayload(ListPromptsRequest).addSuccess(PromptList).annotate(_effect_platform.OpenApi.Summary, "List prompts").annotate(_effect_platform.OpenApi.Description, "Up to 100 prompts in the organization, without content.")).add(_effect_platform.HttpApiEndpoint.post("create", "/prompts.create").setPayload(CreatePromptRequest).addSuccess(PublicPromptWithVersions).annotate(_effect_platform.OpenApi.Summary, "Create a prompt").annotate(_effect_platform.OpenApi.Description, "Creates the prompt and its first version in one call.")).add(_effect_platform.HttpApiEndpoint.post("update", "/prompts.update").setPayload(UpdatePromptRequest).addSuccess(PublicPromptWithVersions).annotate(_effect_platform.OpenApi.Summary, "Add a version").annotate(_effect_platform.OpenApi.Description, "Content is versioned, so updating a prompt appends a version rather than overwriting one. Earlier versions stay readable.")).add(_effect_platform.HttpApiEndpoint.post("promote", "/prompts.promote").setPayload(PromotePromptRequest).addSuccess(Ok).annotate(_effect_platform.OpenApi.Summary, "Promote a version to a channel").annotate(_effect_platform.OpenApi.Description, "Points a channel, such as production, at a version. This is how a version goes live without callers changing anything.")).addError(BadRequest).addError(Forbidden).addError(Conflict).addError(NotFound).middleware(ApiKeyAuthentication).annotate(_effect_platform.OpenApi.Title, "Prompts").annotate(_effect_platform.OpenApi.Description, "Read and write prompts. Content is versioned, so writes append rather than overwrite, and channels decide which version callers receive.") {};
+//#endregion
+//#region ../schema/src/public/api.ts
+var PublicApi = class extends _effect_platform.HttpApi.make("anpord-public").add(PublicEvalsGroup).add(PublicPromptsGroup).prefix("/v1").annotate(_effect_platform.OpenApi.Title, "Anpord API").annotate(_effect_platform.OpenApi.Version, "1.0.0").annotate(_effect_platform.OpenApi.Description, "Run agent evals and manage prompts. Every endpoint takes a JSON body over POST and authenticates with a bearer API key.").annotate(_effect_platform.OpenApi.Servers, [{
+	description: "Production",
+	url: "https://api.anpord.com"
+}]) {};
+//#endregion
+//#region ../schema/src/public/client.ts
+const DEFAULT_BASE_URL = "https://api.anpord.com";
+const make = ({ apiKey, baseUrl = DEFAULT_BASE_URL }) => _effect_platform.HttpApiClient.make(PublicApi, {
+	baseUrl,
+	transformClient: _effect_platform.HttpClient.mapRequest(_effect_platform.HttpClientRequest.bearerToken(effect.Redacted.value(apiKey)))
+});
+var AnpordApi = class extends effect.Effect.Tag("@anpord/sdk/AnpordApi")() {};
+const layer = (options) => effect.Layer.effect(AnpordApi, make(options)).pipe(effect.Layer.provide(_effect_platform.FetchHttpClient.layer));
+//#endregion
+Object.defineProperty(exports, "AnpordApi", {
+	enumerable: true,
+	get: function() {
+		return AnpordApi;
+	}
+});
+Object.defineProperty(exports, "ChannelName", {
+	enumerable: true,
+	get: function() {
+		return ChannelName;
+	}
+});
+Object.defineProperty(exports, "DEFAULT_BASE_URL", {
+	enumerable: true,
+	get: function() {
+		return DEFAULT_BASE_URL;
+	}
+});
+Object.defineProperty(exports, "PromptId", {
+	enumerable: true,
+	get: function() {
+		return PromptId;
+	}
+});
+Object.defineProperty(exports, "VersionNumber", {
+	enumerable: true,
+	get: function() {
+		return VersionNumber;
+	}
+});
+Object.defineProperty(exports, "layer", {
+	enumerable: true,
+	get: function() {
+		return layer;
+	}
+});
+Object.defineProperty(exports, "make", {
+	enumerable: true,
+	get: function() {
+		return make;
+	}
+});

package/dist/{compiler-DXlpU_In.mjs → compiler-D96wI7UP.mjs}

@@ -1,7 +1,8 @@
-import { a as ProfilePath, i as PROFILE_LIMITS, n as HarnessProfile, o as profileFitsHarness, t as EnvName } from "./harness-profile-BMjsN460.mjs";
+import { S as EvalSourceFiles, _ as HarnessProfile, b as ProfilePath, d as EvalValidator, g as EnvName, x as profileFitsHarness, y as PROFILE_LIMITS } from "./evals-Dcc_x6jW.mjs";
+import { t as EvalJudge } from "./eval-judges-CwTvTq7N.mjs";
 import { a as parseRepo, i as cloneUrlOf, r as repo } from "./source-Fg1h2VNn.mjs";
-import { Data, Effect, Option, Schema } from "effect";
-import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { Config, Data, Effect, Option, Schema } from "effect";
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { existsSync, realpathSync } from "node:fs";
 import { mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
 import { fileURLToPath, pathToFileURL } from "node:url";
@@ -11,12 +12,58 @@ import { tmpdir } from "node:os";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 import { parse, stringify } from "smol-toml";
+//#region src/evals/source-files.ts
+const CODE_FILE = /\.(?:[cm]?[jt]s|[jt]sx)$/;
+const sdkRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const SourceMap = Schema.parseJson(Schema.Struct({
+	version: Schema.Literal(3),
+	sources: Schema.Array(Schema.String),
+	sourcesContent: Schema.Array(Schema.NullOr(Schema.String))
+}));
+const projectRoot = (entry) => {
+	let directory = dirname(entry);
+	while (!existsSync(resolve(directory, "package.json"))) {
+		const parent = dirname(directory);
+		if (parent === directory) return dirname(entry);
+		directory = parent;
+	}
+	return directory;
+};
+const sourceFiles = (map, entry) => {
+	const decoded = Schema.decodeUnknownSync(SourceMap)(map);
+	const root = realpathSync(projectRoot(entry));
+	const entryPath = relative(root, realpathSync(entry)).split(sep).join("/");
+	const files = decoded.sources.flatMap((source, index) => {
+		const absolute = resolve(dirname(entry), source);
+		const path = relative(root, absolute).split(sep).join("/");
+		const content = decoded.sourcesContent[index];
+		if (content == null || path.startsWith("../") || path.split("/").includes("node_modules") || absolute.startsWith(`${sdkRoot}${sep}`) || !CODE_FILE.test(path) || source.includes(":") || basename(path) === "anpord-eval-entry.ts") return [];
+		return [{
+			path,
+			content
+		}];
+	});
+	return Schema.decodeUnknownSync(EvalSourceFiles)(files.toSorted((a, b) => Number(b.path === entryPath) - Number(a.path === entryPath) || a.path.localeCompare(b.path)));
+};
+//#endregion
 //#region src/evals/eval-bundle.ts
 const authoringExports = ["export const defineEval = value => value;", `export { empty, files, repo } from "./source";`].join("\n");
 const authoringDir = dirname(fileURLToPath(import.meta.url));
 const ANPORD_MODULE = /^anpord$/;
 const ANY_MODULE = /.*/;
 const localModules = [
+	{
+		built: "validator-runtime.mjs",
+		filter: /^anpord\/validators\/runtime$/,
+		namespace: "anpord-validator-runtime",
+		source: "validator-runtime.ts"
+	},
+	{
+		built: "validators.mjs",
+		filter: /^anpord\/validators$/,
+		namespace: "anpord-validators",
+		source: "../validators.ts"
+	},
 	{
 		built: "cli.mjs",
 		filter: /^anpord\/cli$/,
@@ -84,6 +131,8 @@ const bundle = (contents, entry, options = {}) => Effect.tryPromise({
 		format: "esm",
 		metafile: true,
 		minify: options.minify,
+		outfile: resolve(dirname(entry), "anpord-bundle.mjs"),
+		sourcemap: options.captureSource ? "external" : false,
 		platform: "node",
 		resolveExtensions: [
 			".ts",
@@ -103,7 +152,8 @@ const bundle = (contents, entry, options = {}) => Effect.tryPromise({
 		write: false
 	}).then((result) => ({
 		inputs: Object.keys(result.metafile?.inputs ?? {}).map((path) => resolve(path)),
-		source: result.outputFiles[0]?.text ?? ""
+		source: result.outputFiles.find((file) => file.path.endsWith(".mjs"))?.text ?? "",
+		...options.captureSource ? { sourceFiles: sourceFiles(result.outputFiles.find((file) => file.path.endsWith(".map"))?.text ?? "", entry) } : {}
 	})),
 	catch: (cause) => new Error(`Could not compile ${entry}`, { cause })
 });
@@ -152,51 +202,12 @@ const packageProgram = (root, entry, source) => {
 };
 //#endregion
 //#region src/evals/runner-source.ts
-const runtime = `
-import { access, readFile } from "node:fs/promises";
-import { spawn } from "node:child_process";
-
-const exec = (command) => new Promise((resolve, reject) => {
-  const child = spawn("/bin/sh", ["-lc", command], { cwd: process.cwd() });
-  let stderr = "";
-  let stdout = "";
-  child.stderr.on("data", chunk => { stderr += chunk; });
-  child.stdout.on("data", chunk => { stdout += chunk; });
-  child.on("error", reject);
-  child.on("close", code => resolve({ exitCode: code ?? 1, stderr, stdout }));
-});
-
-const readOrEmpty = (path) =>
-  path === undefined ? Promise.resolve("") : readFile(path, "utf8").then(text => text, () => "");
-
-const calls = async (path, key, value) => {
-  const text = await readOrEmpty(path);
-  const calls = text.trim() === "" ? [] : text.trim().split("\\n").map((line) => JSON.parse(line));
-  return value === undefined ? calls : calls.filter((call) => call[key] === value);
-};
-
-const context = {
-  answer: () => readOrEmpty(process.env.ANPORD_ANSWER_FILE),
-  cli: { calls: cli => calls(".anpord/cli-calls.jsonl", "cli", cli) },
-  exec,
-  exists: path => access(path).then(() => true, () => false),
-  mcp: { calls: server => calls(".anpord/mcp-calls.jsonl", "server", server) },
-  readText: path => readFile(path, "utf8"),
-  prepared: JSON.parse(process.env.ANPORD_PREPARE_VALUE ?? "{}"),
-  transcript: () => readOrEmpty(process.env.ANPORD_TRANSCRIPT_FILE),
-};
-
-try {
-  const raw = await validate(context);
-  const result = typeof raw === "boolean" ? { passed: raw } : raw;
-  if (typeof result?.passed !== "boolean") throw new Error("A validator must return a boolean or { passed, message? }");
-  console.log("ANPORD_VALIDATOR_RESULT=" + JSON.stringify(result));
-} catch (error) {
-  console.error(error instanceof Error ? error.stack : String(error));
-  process.exitCode = 1;
-}
-`;
-const validatorEntry = (module, name) => `import { ${name} as validate } from ${JSON.stringify(module)};\n${runtime}`;
+const validatorCaseEntry = (entry, index, names = []) => `import definition from ${JSON.stringify(entry)};
+import { runValidators } from "anpord/validators/runtime";
+const selected = definition.cases[${index}].validate;
+const names = ${JSON.stringify(names)};
+const checks = (Array.isArray(selected) ? selected : [selected]).flatMap((validate, index) => typeof validate === "function" ? [{ index, name: names.find(check => check.index === index)?.name ?? "Validator " + (index + 1), validate }] : []);
+await runValidators(checks, definition.captureValidation !== false);`;
 const prepareRuntime = `
 import { access, readFile } from "node:fs/promises";
 import { spawn } from "node:child_process";
@@ -247,11 +258,12 @@ import { runMcpServer } from "anpord/mcp/runtime";
 const server = definition.mcp?.[${index}];
 if (server === undefined) throw new Error("MCP server ${index} is missing");
 runMcpServer(server);`;
-const cliEntry = (entry, index) => `import definition from ${JSON.stringify(entry)};
+const cliEntry = (entry, index) => `import { fileURLToPath } from "node:url";
+import definition from ${JSON.stringify(entry)};
 import { runCli } from "anpord/cli/runtime";
 const cli = definition.cli?.[${index}];
 if (cli === undefined) throw new Error("CLI ${index} is missing");
-await runCli(cli);`;
+await runCli(cli, fileURLToPath(new URL("../../cli-calls.jsonl", import.meta.url)));`;
 //#endregion
 //#region src/evals/cli-profile.ts
 const compileClis = (entry, definitions) => Effect.gen(function* () {
@@ -287,6 +299,40 @@ const withClis = (task, clis) => {
 	};
 };
 //#endregion
+//#region src/evals/compile-validator.ts
+const compileValidator = (entry, subject, caseIndex, captureSource, captureValidation = true) => Effect.gen(function* () {
+	if (subject.validate === void 0) return null;
+	const validations = Array.isArray(subject.validate) ? subject.validate : [subject.validate];
+	if (validations.length === 0 || validations.length > 20) return yield* Effect.fail(/* @__PURE__ */ new Error("Use between 1 and 20 validators"));
+	const judges = yield* Effect.forEach(validations.filter((value) => typeof value !== "function"), (value) => Schema.decodeUnknown(EvalJudge)(value, { onExcessProperty: "error" }));
+	if (new Set(judges.map(({ name }) => name)).size !== judges.length) return yield* Effect.fail(/* @__PURE__ */ new Error("Judge names must be unique within a case"));
+	const manifest = validations.flatMap((value, index) => typeof value === "function" ? [{
+		index,
+		name: value.name || `Validator ${index + 1}`
+	}] : []);
+	const hasCode = manifest.length > 0;
+	const compiled = yield* bundle(hasCode ? validatorCaseEntry(entry, caseIndex, manifest) : definitionEntry(entry), entry, {
+		minify: true,
+		captureSource
+	});
+	const checks = hasCode ? [{
+		name: typeof subject.validate === "function" ? subject.validate.name || subject.name : subject.name,
+		source: compiled.source,
+		manifest,
+		capture: captureValidation
+	}] : [];
+	return yield* Schema.decodeUnknown(EvalValidator)({
+		...judges.length > 0 ? {
+			kind: "judged",
+			name: subject.name,
+			checks,
+			judges
+		} : checks[0],
+		sourceFiles: compiled.sourceFiles,
+		capture: captureValidation
+	});
+});
+//#endregion
 //#region src/evals/definition-loader.ts
 const loadDefinition = (entry) => bundle(definitionEntry(entry), entry).pipe(Effect.flatMap(({ inputs, source }) => Effect.acquireUseRelease(Effect.tryPromise(() => mkdtemp(join(tmpdir(), "anpord-eval-"))), (directory) => Effect.tryPromise({
 	try: async () => {
@@ -315,26 +361,25 @@ const isDefinition = (value) => Option.isSome(decodeShape(value));
 //#region src/evals/local-repo.ts
 const run = promisify(execFile);
 const git = (args, cwd) => Effect.tryPromise(() => run("git", [...args], { cwd })).pipe(Effect.map(({ stdout }) => stdout.trim()), Effect.orElseSucceed(() => ""));
-const environment = () => globalThis.process?.env ?? {};
 const remoteUrl = (cwd) => Effect.gen(function* () {
-	const { GITHUB_REPOSITORY } = environment();
+	const repository = yield* Config.string("GITHUB_REPOSITORY").pipe(Config.withDefault(""));
 	const origin = yield* git([
 		"remote",
 		"get-url",
 		"origin"
 	], cwd);
-	return origin === "" && GITHUB_REPOSITORY !== void 0 ? GITHUB_REPOSITORY : origin;
+	return origin === "" ? repository : origin;
 });
 const fetchableRef = (cwd) => Effect.gen(function* () {
-	const { GITHUB_HEAD_REF } = environment();
-	if (GITHUB_HEAD_REF !== void 0 && GITHUB_HEAD_REF !== "") return GITHUB_HEAD_REF;
+	const ci = yield* Config.boolean("GITHUB_ACTIONS").pipe(Config.withDefault(false));
 	const [head, onRemote] = yield* Effect.all([git(["rev-parse", "HEAD"], cwd), git([
 		"branch",
 		"--remotes",
 		"--contains",
 		"HEAD"
 	], cwd)], { concurrency: 2 });
-	return head === "" || onRemote === "" ? null : head;
+	if (ci && head === "") return yield* Effect.fail(/* @__PURE__ */ new Error("Check out the commit before running evals in GitHub Actions."));
+	return head === "" || !ci && onRemote === "" ? null : head;
 });
 const localRepo = (cwd) => Effect.gen(function* () {
 	const [remote, ref] = yield* Effect.all([remoteUrl(cwd), fetchableRef(cwd)], { concurrency: 2 });
@@ -542,14 +587,6 @@ const decodeManifest = (dir, text) => Schema.decodeUnknown(ManifestJson)(text).p
 	dir,
 	reason: issue.message
 })));
-/**
-* A manifest value that names a regular file is replaced by that file's
-* content; anything else is kept as written. Env values are always literal.
-*
-* A file outside the profile directory is refused rather than read: the
-* manifest is shipped with the profile, and a prompt that only exists on one
-* machine is not part of it.
-*/
 const inlined = (dir, value) => Effect.gen(function* () {
 	const target = resolve(dir, value);
 	const position = relative(dir, target);
@@ -589,7 +626,6 @@ const readProfileManifest = (dir) => Effect.gen(function* () {
 const SHIPPED_ROOTS = ["home", "workspace"];
 const SKIPPED_DIRECTORIES = /* @__PURE__ */ new Set(["node_modules", ".git"]);
 const isMissing = (cause) => cause instanceof Error && "code" in cause && cause.code === "ENOENT";
-/** Every regular file below `current`, as a posix path relative to `dir`. */
 const walk = async (dir, current) => {
 	const entries = await readdir(current, { withFileTypes: true });
 	return (await Promise.all(entries.map((entry) => {
@@ -612,7 +648,7 @@ const shippedPaths = (dir) => Effect.tryPromise({
 	}
 });
 const validPath = (path) => Schema.decodeUnknown(ProfilePath)(path).pipe(Effect.mapError(() => new ProfilePathInvalid({ path })));
-/** The file's text, or nothing when it holds a NUL byte and is not text. */
+/** Nothing when the file holds a NUL byte and so is not text. */
 const readShipped = (dir, path) => Effect.gen(function* () {
 	const bytes = yield* Effect.tryPromise({
 		catch: (cause) => new ProfileDirectoryUnreadable({
@@ -648,8 +684,6 @@ const fittingHarness = (base, profile) => {
 		step: "run"
 	});
 };
-/** The task a definition's `{ base, profile }` harness compiles to, with the
-* profile directory read relative to the eval file. */
 const profileTask = (entry, task) => Effect.gen(function* () {
 	const dir = resolve(dirname(entry), task.harness.profile.dir);
 	yield* Effect.tryPromise({
@@ -669,7 +703,7 @@ const profileTask = (entry, task) => Effect.gen(function* () {
 		harness: task.harness.base,
 		model: task.model,
 		profile,
-		provider: task.provider
+		sandbox: task.sandbox
 	};
 }).pipe(Effect.withSpan("Eval.profileTask", { attributes: { profile: task.harness.profile.name } }));
 //#endregion
@@ -678,7 +712,7 @@ const taskOf = (entry, task, clis, mcp) => Effect.gen(function* () {
 	const compiled = typeof task.harness === "string" ? {
 		harness: task.harness,
 		model: task.model,
-		provider: task.provider
+		sandbox: task.sandbox
 	} : yield* profileTask(entry, {
 		...task,
 		harness: task.harness
@@ -701,10 +735,9 @@ const compileEvalEffect = (path) => Effect.gen(function* () {
 	const clis = yield* compileClis(entry, definition.cli ?? []);
 	const mcp = yield* compileMcpServers(entry, definition.mcp ?? []);
 	const fallback = definition.source === void 0 && definition.cases.some((subject) => subject.source === void 0) ? yield* localRepo(dirname(entry)) : Option.none();
-	const cases = yield* Effect.forEach(definition.cases, (subject) => Effect.gen(function* () {
-		const hasValidator = typeof subject.validate === "function";
-		if (hasValidator === (typeof subject.verify === "string")) return yield* Effect.fail(/* @__PURE__ */ new Error(`${subject.name} must have exactly one of validate or verify`));
-		const validator = hasValidator ? yield* bundledCaseModule(entry, loaded.inputs, subject.validate?.name || subject.name, validatorEntry) : null;
+	const cases = yield* Effect.forEach(definition.cases, (subject, caseIndex) => Effect.gen(function* () {
+		if (subject.validate !== void 0 === (typeof subject.verify === "string")) return yield* Effect.fail(/* @__PURE__ */ new Error(`${subject.name} must have exactly one of validate or verify`));
+		const validator = yield* compileValidator(entry, subject, caseIndex, definition.captureSource !== false, definition.captureValidation !== false);
 		const prepare = typeof subject.prepare === "function" ? yield* bundledCaseModule(entry, loaded.inputs, subject.prepare.name || `${subject.name}-prepare`, prepareEntry) : null;
 		return {
 			...subject.cache === void 0 ? {} : { cache: subject.cache },