anpord 0.1.13 → 0.1.16

package/dist/{compiler-uwb1pkdK.cjs → compiler-4FKsNxZ-.cjs}

@@ -1,10 +1,11 @@
-const require_evals = require("./evals-DouMsrsM.cjs");
 const require_eval_judges = require("./eval-judges-BkmiVLLq.cjs");
+const require_api_mocks = require("./api-mocks-B3qngoqS.cjs");
+const require_evals_api = require("./evals-api-kOuyIqvB.cjs");
 const require_source = require("./source-DvlGawEw.cjs");
 let effect = require("effect");
+let node_fs_promises = require("node:fs/promises");
 let node_path = require("node:path");
 let node_fs = require("node:fs");
-let node_fs_promises = require("node:fs/promises");
 let node_url = require("node:url");
 let esbuild = require("esbuild");
 let node_zlib = require("node:zlib");
@@ -43,7 +44,7 @@ const sourceFiles = (map, entry) => {
 			content
 		}];
 	});
-	return effect.Schema.decodeUnknownSync(require_evals.EvalSourceFiles)(files.toSorted((a, b) => Number(b.path === entryPath) - Number(a.path === entryPath) || a.path.localeCompare(b.path)));
+	return effect.Schema.decodeUnknownSync(require_evals_api.EvalSourceFiles)(files.toSorted((a, b) => Number(b.path === entryPath) - Number(a.path === entryPath) || a.path.localeCompare(b.path)));
 };
 //#endregion
 //#region src/evals/eval-bundle.ts
@@ -52,6 +53,24 @@ const authoringDir = (0, node_path.dirname)((0, node_url.fileURLToPath)(require(
 const ANPORD_MODULE = /^anpord$/;
 const ANY_MODULE = /.*/;
 const localModules = [
+	{
+		built: "api.mjs",
+		filter: /^anpord\/api$/,
+		namespace: "anpord-api-authoring",
+		source: "../mock-api/index.ts"
+	},
+	{
+		built: "api-runtime.mjs",
+		filter: /^anpord\/api\/runtime$/,
+		namespace: "anpord-api-runtime",
+		source: "../mock-api/runtime.ts"
+	},
+	{
+		built: "api-context.mjs",
+		filter: /^anpord\/api\/context$/,
+		namespace: "anpord-api-context",
+		source: "../mock-api/context.ts"
+	},
 	{
 		built: "validator-runtime.mjs",
 		filter: /^anpord\/validators\/runtime$/,
@@ -128,6 +147,7 @@ const bundle = (contents, entry, options = {}) => effect.Effect.tryPromise({
 	try: () => (0, esbuild.build)({
 		absWorkingDir: process.cwd(),
 		bundle: true,
+		banner: { js: "import { createRequire as anpordCreateRequire } from \"node:module\"; const require = anpordCreateRequire(import.meta.url);" },
 		format: "esm",
 		metafile: true,
 		minify: options.minify,
@@ -158,6 +178,52 @@ const bundle = (contents, entry, options = {}) => effect.Effect.tryPromise({
 	catch: (cause) => new Error(`Could not compile ${entry}`, { cause })
 });
 //#endregion
+//#region src/evals/program-package.ts
+const CHUNK_SIZE = 9e4;
+const packageProgram = (root, entry, source) => {
+	const archive = (0, node_zlib.gzipSync)(source).toString("base64");
+	const chunks = Array.from({ length: Math.ceil(archive.length / CHUNK_SIZE) }, (_, index) => archive.slice(index * CHUNK_SIZE, (index + 1) * CHUNK_SIZE));
+	const names = chunks.map((_, index) => `${index}.txt`);
+	const loader = `#!/usr/bin/env node\nimport{readFile,writeFile}from"node:fs/promises";import{gunzipSync}from"node:zlib";const root=new URL("./",import.meta.url),names=${JSON.stringify(names)},data=(await Promise.all(names.map(name=>readFile(new URL(name,root),"utf8")))).join(""),runtime=new URL("runtime.mjs",root);await writeFile(runtime,gunzipSync(Buffer.from(data,"base64")));await import(runtime.href);`;
+	return {
+		entry: `${root}/${entry}`,
+		files: Object.fromEntries([[`${root}/${entry}`, loader], ...chunks.map((chunk, index) => [`${root}/${index}.txt`, chunk])])
+	};
+};
+//#endregion
+//#region src/evals/api-profile.ts
+const compileApis = (entry, definitions) => effect.Effect.gen(function* () {
+	if (definitions.length === 0) return {};
+	const names = definitions.map(({ name }) => name);
+	if (new Set(names).size !== names.length) return yield* effect.Effect.fail(/* @__PURE__ */ new Error("Duplicate API mock name"));
+	const { source } = yield* bundle(`import definition from ${JSON.stringify(entry)};
+import { runApiServers } from "anpord/api/runtime";
+runApiServers(definition.api ?? []);`, entry, { minify: true });
+	const program = packageProgram("workspace/.anpord/api", "server.mjs", source);
+	return {
+		...program.files,
+		[require_api_mocks.API_PROGRAM]: JSON.stringify({ entry: program.entry.slice(10) })
+	};
+});
+const withApis = (task, files) => {
+	if (Object.keys(files).length === 0) return task;
+	const profile = task.profile ?? {
+		name: "anpord-api",
+		files: {}
+	};
+	for (const path of Object.keys(files)) if (profile.files[path] !== void 0) throw new Error(`Profile file ${path} is reserved for API mocks`);
+	return {
+		...task,
+		profile: {
+			...profile,
+			files: {
+				...profile.files,
+				...files
+			}
+		}
+	};
+};
+//#endregion
 //#region src/evals/case-modules.ts
 const realOrGiven = (path) => {
 	try {
@@ -175,7 +241,7 @@ const exportingModule = (entry, inputs, name) => effect.Effect.gen(function* ()
 });
 const bundledCaseModule = (entry, inputs, name, wrap) => effect.Effect.gen(function* () {
 	const module = yield* exportingModule(entry, inputs, name);
-	const { source } = yield* bundle(wrap(module, name), entry);
+	const { source } = yield* bundle(wrap(module, name), entry, { minify: true });
 	return {
 		name,
 		source
@@ -188,19 +254,6 @@ const appendInstall = (profile, command) => ({
 	install: profile.install ? `${profile.install} && ${command}` : command
 });
 //#endregion
-//#region src/evals/program-package.ts
-const CHUNK_SIZE = 9e4;
-const packageProgram = (root, entry, source) => {
-	const archive = (0, node_zlib.gzipSync)(source).toString("base64");
-	const chunks = Array.from({ length: Math.ceil(archive.length / CHUNK_SIZE) }, (_, index) => archive.slice(index * CHUNK_SIZE, (index + 1) * CHUNK_SIZE));
-	const names = chunks.map((_, index) => `${index}.txt`);
-	const loader = `#!/usr/bin/env node\nimport{readFile,writeFile}from"node:fs/promises";import{gunzipSync}from"node:zlib";const root=new URL("./",import.meta.url),names=${JSON.stringify(names)},data=(await Promise.all(names.map(name=>readFile(new URL(name,root),"utf8")))).join(""),runtime=new URL("runtime.mjs",root);await writeFile(runtime,gunzipSync(Buffer.from(data,"base64")));await import(runtime.href);`;
-	return {
-		entry: `${root}/${entry}`,
-		files: Object.fromEntries([[`${root}/${entry}`, loader], ...names.map((name, index) => [`${root}/${name}`, chunks[index]])])
-	};
-};
-//#endregion
 //#region src/evals/runner-source.ts
 const validatorCaseEntry = (entry, index, names = []) => `import definition from ${JSON.stringify(entry)};
 import { runValidators } from "anpord/validators/runtime";
@@ -209,6 +262,7 @@ const names = ${JSON.stringify(names)};
 const checks = (Array.isArray(selected) ? selected : [selected]).flatMap((validate, index) => typeof validate === "function" ? [{ index, name: names.find(check => check.index === index)?.name ?? "Validator " + (index + 1), validate }] : []);
 await runValidators(checks, definition.captureValidation !== false);`;
 const prepareRuntime = `
+import { apiContext } from "anpord/api/context";
 import { access, readFile } from "node:fs/promises";
 import { spawn } from "node:child_process";
 
@@ -236,6 +290,7 @@ const exec = (file, args = [], options = {}) => new Promise((resolve, reject) =>
 });
 
 const context = {
+  api: apiContext,
   cached: process.env.ANPORD_CACHE_RESTORED === "1",
   exec,
   exists: path => access(path).then(() => true, () => false),
@@ -321,7 +376,7 @@ const compileValidator = (entry, subject, caseIndex, captureSource, captureValid
 		manifest,
 		capture: captureValidation
 	}] : [];
-	return yield* effect.Schema.decodeUnknown(require_evals.EvalValidator)({
+	return yield* effect.Schema.decodeUnknown(require_evals_api.EvalValidator)({
 		...judges.length > 0 ? {
 			kind: "judged",
 			name: subject.name,
@@ -487,7 +542,7 @@ const adapters = {
 const applyMcpHarness = (harness, profile, servers) => {
 	const adapter = adapters[harness];
 	if (adapter === null) throw new Error(`MCP mock servers are not supported by ${harness}`);
-	return effect.Schema.decodeUnknownSync(require_evals.HarnessProfile)(adapter.apply(profile, servers));
+	return effect.Schema.decodeUnknownSync(require_evals_api.HarnessProfile)(adapter.apply(profile, servers));
 };
 //#endregion
 //#region src/evals/mcp-profile.ts
@@ -533,17 +588,17 @@ var ProfilePathInvalid = class extends effect.Data.TaggedError("ProfilePathInval
 };
 var ProfileFileTooLarge = class extends effect.Data.TaggedError("ProfileFileTooLarge") {
 	get message() {
-		return `${this.path} is ${this.chars} characters; a profile file holds at most ${require_evals.PROFILE_LIMITS.fileChars}`;
+		return `${this.path} is ${this.chars} characters; a profile file holds at most ${require_evals_api.PROFILE_LIMITS.fileChars}`;
 	}
 };
 var ProfileTooLarge = class extends effect.Data.TaggedError("ProfileTooLarge") {
 	get message() {
-		return `The profile's files hold ${this.chars} characters; at most ${require_evals.PROFILE_LIMITS.totalChars} fit`;
+		return `The profile's files hold ${this.chars} characters; at most ${require_evals_api.PROFILE_LIMITS.totalChars} fit`;
 	}
 };
 var ProfileTooManyFiles = class extends effect.Data.TaggedError("ProfileTooManyFiles") {
 	get message() {
-		return `The profile has ${this.count} files; at most ${require_evals.PROFILE_LIMITS.files} fit`;
+		return `The profile has ${this.count} files; at most ${require_evals_api.PROFILE_LIMITS.files} fit`;
 	}
 };
 var ProfileManifestInvalid = class extends effect.Data.TaggedError("ProfileManifestInvalid") {
@@ -570,7 +625,7 @@ var ProfileStepNotSupported = class extends effect.Data.TaggedError("ProfileStep
 //#region src/evals/profile-manifest.ts
 const Manifest = effect.Schema.Struct({
 	env: effect.Schema.optional(effect.Schema.Record({
-		key: require_evals.EnvName,
+		key: require_evals_api.EnvName,
 		value: effect.Schema.String
 	})),
 	install: effect.Schema.optional(effect.Schema.String),
@@ -647,7 +702,7 @@ const shippedPaths = (dir) => effect.Effect.tryPromise({
 		return (await Promise.all(SHIPPED_ROOTS.map((root) => walkRoot(dir, root)))).flat().sort();
 	}
 });
-const validPath = (path) => effect.Schema.decodeUnknown(require_evals.ProfilePath)(path).pipe(effect.Effect.mapError(() => new ProfilePathInvalid({ path })));
+const validPath = (path) => effect.Schema.decodeUnknown(require_evals_api.ProfilePath)(path).pipe(effect.Effect.mapError(() => new ProfilePathInvalid({ path })));
 /** Nothing when the file holds a NUL byte and so is not text. */
 const readShipped = (dir, path) => effect.Effect.gen(function* () {
 	const bytes = yield* effect.Effect.tryPromise({
@@ -659,7 +714,7 @@ const readShipped = (dir, path) => effect.Effect.gen(function* () {
 	});
 	if (bytes.includes(0)) return [];
 	const content = bytes.toString("utf8");
-	if (content.length > require_evals.PROFILE_LIMITS.fileChars) return yield* new ProfileFileTooLarge({
+	if (content.length > require_evals_api.PROFILE_LIMITS.fileChars) return yield* new ProfileFileTooLarge({
 		chars: content.length,
 		path
 	});
@@ -667,14 +722,14 @@ const readShipped = (dir, path) => effect.Effect.gen(function* () {
 });
 const shippedFiles = (dir) => effect.Effect.gen(function* () {
 	const paths = yield* shippedPaths(dir);
-	if (paths.length > require_evals.PROFILE_LIMITS.files) return yield* new ProfileTooManyFiles({ count: paths.length });
+	if (paths.length > require_evals_api.PROFILE_LIMITS.files) return yield* new ProfileTooManyFiles({ count: paths.length });
 	const entries = yield* effect.Effect.forEach(paths, (path) => readShipped(dir, path), { concurrency: 8 }).pipe(effect.Effect.map((read) => read.flat()));
 	const chars = entries.reduce((sum, [, content]) => sum + content.length, 0);
-	if (chars > require_evals.PROFILE_LIMITS.totalChars) return yield* new ProfileTooLarge({ chars });
+	if (chars > require_evals_api.PROFILE_LIMITS.totalChars) return yield* new ProfileTooLarge({ chars });
 	return Object.fromEntries(entries);
 });
 const fittingHarness = (base, profile) => {
-	if (require_evals.profileFitsHarness({
+	if (require_evals_api.profileFitsHarness({
 		harness: base,
 		profile
 	})) return effect.Effect.succeed(profile);
@@ -708,7 +763,7 @@ const profileTask = (entry, task) => effect.Effect.gen(function* () {
 }).pipe(effect.Effect.withSpan("Eval.profileTask", { attributes: { profile: task.harness.profile.name } }));
 //#endregion
 //#region src/evals/compiler.ts
-const taskOf = (entry, task, clis, mcp) => effect.Effect.gen(function* () {
+const taskOf = (entry, task, clis, mcp, apis) => effect.Effect.gen(function* () {
 	const compiled = typeof task.harness === "string" ? {
 		harness: task.harness,
 		model: task.model,
@@ -717,7 +772,7 @@ const taskOf = (entry, task, clis, mcp) => effect.Effect.gen(function* () {
 		...task,
 		harness: task.harness
 	});
-	return yield* effect.Effect.try(() => withClis(withMcpServers(compiled, mcp), clis));
+	return yield* effect.Effect.try(() => withApis(withClis(withMcpServers(compiled, mcp), clis), apis));
 });
 const sourceFor = (definition, subject, fallback) => {
 	const source = subject.source ?? definition.source;
@@ -734,6 +789,7 @@ const compileEvalEffect = (path) => effect.Effect.gen(function* () {
 	if (!isDefinition(definition)) return yield* effect.Effect.fail(/* @__PURE__ */ new Error(`${entry} must default export defineEval({ ... })`));
 	const clis = yield* compileClis(entry, definition.cli ?? []);
 	const mcp = yield* compileMcpServers(entry, definition.mcp ?? []);
+	const apis = yield* compileApis(entry, definition.api ?? []);
 	const fallback = definition.source === void 0 && definition.cases.some((subject) => subject.source === void 0) ? yield* localRepo((0, node_path.dirname)(entry)) : effect.Option.none();
 	const cases = yield* effect.Effect.forEach(definition.cases, (subject, caseIndex) => effect.Effect.gen(function* () {
 		if (subject.validate !== void 0 === (typeof subject.verify === "string")) return yield* effect.Effect.fail(/* @__PURE__ */ new Error(`${subject.name} must have exactly one of validate or verify`));
@@ -749,14 +805,14 @@ const compileEvalEffect = (path) => effect.Effect.gen(function* () {
 			verify: subject.verify ?? null
 		};
 	}), { concurrency: 4 });
-	const tasks = yield* effect.Effect.forEach(definition.tasks, (task) => taskOf(entry, task, clis, mcp), { concurrency: 4 });
-	return {
+	const tasks = yield* effect.Effect.forEach(definition.tasks, (task) => taskOf(entry, task, clis, mcp, apis), { concurrency: 4 });
+	return yield* effect.Schema.decodeUnknown(require_evals_api.PublicStartEvalRequest)({
 		cases,
 		name: definition.name,
 		prompt: definition.prompt,
 		tasks,
 		trials: definition.trials
-	};
+	});
 }).pipe(effect.Effect.withSpan("Eval.compile"));
 const compileEval = (path) => effect.Effect.runPromise(compileEvalEffect(path));
 //#endregion

package/dist/{compiler-D96wI7UP.mjs → compiler-Dsltkkj5.mjs}

@@ -1,10 +1,11 @@
-import { S as EvalSourceFiles, _ as HarnessProfile, b as ProfilePath, d as EvalValidator, g as EnvName, x as profileFitsHarness, y as PROFILE_LIMITS } from "./evals-Dcc_x6jW.mjs";
 import { t as EvalJudge } from "./eval-judges-CwTvTq7N.mjs";
+import { r as API_PROGRAM } from "./api-mocks-BfKw_hXc.mjs";
+import { a as EvalValidator, c as PROFILE_LIMITS, d as EvalSourceFiles, l as ProfilePath, n as PublicStartEvalRequest, o as EnvName, s as HarnessProfile, u as profileFitsHarness } from "./evals-api-DDv4gXU1.mjs";
 import { a as parseRepo, i as cloneUrlOf, r as repo } from "./source-Fg1h2VNn.mjs";
 import { Config, Data, Effect, Option, Schema } from "effect";
+import { mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
 import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { existsSync, realpathSync } from "node:fs";
-import { mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { build } from "esbuild";
 import { gzipSync } from "node:zlib";
@@ -52,6 +53,24 @@ const authoringDir = dirname(fileURLToPath(import.meta.url));
 const ANPORD_MODULE = /^anpord$/;
 const ANY_MODULE = /.*/;
 const localModules = [
+	{
+		built: "api.mjs",
+		filter: /^anpord\/api$/,
+		namespace: "anpord-api-authoring",
+		source: "../mock-api/index.ts"
+	},
+	{
+		built: "api-runtime.mjs",
+		filter: /^anpord\/api\/runtime$/,
+		namespace: "anpord-api-runtime",
+		source: "../mock-api/runtime.ts"
+	},
+	{
+		built: "api-context.mjs",
+		filter: /^anpord\/api\/context$/,
+		namespace: "anpord-api-context",
+		source: "../mock-api/context.ts"
+	},
 	{
 		built: "validator-runtime.mjs",
 		filter: /^anpord\/validators\/runtime$/,
@@ -128,6 +147,7 @@ const bundle = (contents, entry, options = {}) => Effect.tryPromise({
 	try: () => build({
 		absWorkingDir: process.cwd(),
 		bundle: true,
+		banner: { js: "import { createRequire as anpordCreateRequire } from \"node:module\"; const require = anpordCreateRequire(import.meta.url);" },
 		format: "esm",
 		metafile: true,
 		minify: options.minify,
@@ -158,6 +178,52 @@ const bundle = (contents, entry, options = {}) => Effect.tryPromise({
 	catch: (cause) => new Error(`Could not compile ${entry}`, { cause })
 });
 //#endregion
+//#region src/evals/program-package.ts
+const CHUNK_SIZE = 9e4;
+const packageProgram = (root, entry, source) => {
+	const archive = gzipSync(source).toString("base64");
+	const chunks = Array.from({ length: Math.ceil(archive.length / CHUNK_SIZE) }, (_, index) => archive.slice(index * CHUNK_SIZE, (index + 1) * CHUNK_SIZE));
+	const names = chunks.map((_, index) => `${index}.txt`);
+	const loader = `#!/usr/bin/env node\nimport{readFile,writeFile}from"node:fs/promises";import{gunzipSync}from"node:zlib";const root=new URL("./",import.meta.url),names=${JSON.stringify(names)},data=(await Promise.all(names.map(name=>readFile(new URL(name,root),"utf8")))).join(""),runtime=new URL("runtime.mjs",root);await writeFile(runtime,gunzipSync(Buffer.from(data,"base64")));await import(runtime.href);`;
+	return {
+		entry: `${root}/${entry}`,
+		files: Object.fromEntries([[`${root}/${entry}`, loader], ...chunks.map((chunk, index) => [`${root}/${index}.txt`, chunk])])
+	};
+};
+//#endregion
+//#region src/evals/api-profile.ts
+const compileApis = (entry, definitions) => Effect.gen(function* () {
+	if (definitions.length === 0) return {};
+	const names = definitions.map(({ name }) => name);
+	if (new Set(names).size !== names.length) return yield* Effect.fail(/* @__PURE__ */ new Error("Duplicate API mock name"));
+	const { source } = yield* bundle(`import definition from ${JSON.stringify(entry)};
+import { runApiServers } from "anpord/api/runtime";
+runApiServers(definition.api ?? []);`, entry, { minify: true });
+	const program = packageProgram("workspace/.anpord/api", "server.mjs", source);
+	return {
+		...program.files,
+		[API_PROGRAM]: JSON.stringify({ entry: program.entry.slice(10) })
+	};
+});
+const withApis = (task, files) => {
+	if (Object.keys(files).length === 0) return task;
+	const profile = task.profile ?? {
+		name: "anpord-api",
+		files: {}
+	};
+	for (const path of Object.keys(files)) if (profile.files[path] !== void 0) throw new Error(`Profile file ${path} is reserved for API mocks`);
+	return {
+		...task,
+		profile: {
+			...profile,
+			files: {
+				...profile.files,
+				...files
+			}
+		}
+	};
+};
+//#endregion
 //#region src/evals/case-modules.ts
 const realOrGiven = (path) => {
 	try {
@@ -175,7 +241,7 @@ const exportingModule = (entry, inputs, name) => Effect.gen(function* () {
 });
 const bundledCaseModule = (entry, inputs, name, wrap) => Effect.gen(function* () {
 	const module = yield* exportingModule(entry, inputs, name);
-	const { source } = yield* bundle(wrap(module, name), entry);
+	const { source } = yield* bundle(wrap(module, name), entry, { minify: true });
 	return {
 		name,
 		source
@@ -188,19 +254,6 @@ const appendInstall = (profile, command) => ({
 	install: profile.install ? `${profile.install} && ${command}` : command
 });
 //#endregion
-//#region src/evals/program-package.ts
-const CHUNK_SIZE = 9e4;
-const packageProgram = (root, entry, source) => {
-	const archive = gzipSync(source).toString("base64");
-	const chunks = Array.from({ length: Math.ceil(archive.length / CHUNK_SIZE) }, (_, index) => archive.slice(index * CHUNK_SIZE, (index + 1) * CHUNK_SIZE));
-	const names = chunks.map((_, index) => `${index}.txt`);
-	const loader = `#!/usr/bin/env node\nimport{readFile,writeFile}from"node:fs/promises";import{gunzipSync}from"node:zlib";const root=new URL("./",import.meta.url),names=${JSON.stringify(names)},data=(await Promise.all(names.map(name=>readFile(new URL(name,root),"utf8")))).join(""),runtime=new URL("runtime.mjs",root);await writeFile(runtime,gunzipSync(Buffer.from(data,"base64")));await import(runtime.href);`;
-	return {
-		entry: `${root}/${entry}`,
-		files: Object.fromEntries([[`${root}/${entry}`, loader], ...names.map((name, index) => [`${root}/${name}`, chunks[index]])])
-	};
-};
-//#endregion
 //#region src/evals/runner-source.ts
 const validatorCaseEntry = (entry, index, names = []) => `import definition from ${JSON.stringify(entry)};
 import { runValidators } from "anpord/validators/runtime";
@@ -209,6 +262,7 @@ const names = ${JSON.stringify(names)};
 const checks = (Array.isArray(selected) ? selected : [selected]).flatMap((validate, index) => typeof validate === "function" ? [{ index, name: names.find(check => check.index === index)?.name ?? "Validator " + (index + 1), validate }] : []);
 await runValidators(checks, definition.captureValidation !== false);`;
 const prepareRuntime = `
+import { apiContext } from "anpord/api/context";
 import { access, readFile } from "node:fs/promises";
 import { spawn } from "node:child_process";
 
@@ -236,6 +290,7 @@ const exec = (file, args = [], options = {}) => new Promise((resolve, reject) =>
 });
 
 const context = {
+  api: apiContext,
   cached: process.env.ANPORD_CACHE_RESTORED === "1",
   exec,
   exists: path => access(path).then(() => true, () => false),
@@ -708,7 +763,7 @@ const profileTask = (entry, task) => Effect.gen(function* () {
 }).pipe(Effect.withSpan("Eval.profileTask", { attributes: { profile: task.harness.profile.name } }));
 //#endregion
 //#region src/evals/compiler.ts
-const taskOf = (entry, task, clis, mcp) => Effect.gen(function* () {
+const taskOf = (entry, task, clis, mcp, apis) => Effect.gen(function* () {
 	const compiled = typeof task.harness === "string" ? {
 		harness: task.harness,
 		model: task.model,
@@ -717,7 +772,7 @@ const taskOf = (entry, task, clis, mcp) => Effect.gen(function* () {
 		...task,
 		harness: task.harness
 	});
-	return yield* Effect.try(() => withClis(withMcpServers(compiled, mcp), clis));
+	return yield* Effect.try(() => withApis(withClis(withMcpServers(compiled, mcp), clis), apis));
 });
 const sourceFor = (definition, subject, fallback) => {
 	const source = subject.source ?? definition.source;
@@ -734,6 +789,7 @@ const compileEvalEffect = (path) => Effect.gen(function* () {
 	if (!isDefinition(definition)) return yield* Effect.fail(/* @__PURE__ */ new Error(`${entry} must default export defineEval({ ... })`));
 	const clis = yield* compileClis(entry, definition.cli ?? []);
 	const mcp = yield* compileMcpServers(entry, definition.mcp ?? []);
+	const apis = yield* compileApis(entry, definition.api ?? []);
 	const fallback = definition.source === void 0 && definition.cases.some((subject) => subject.source === void 0) ? yield* localRepo(dirname(entry)) : Option.none();
 	const cases = yield* Effect.forEach(definition.cases, (subject, caseIndex) => Effect.gen(function* () {
 		if (subject.validate !== void 0 === (typeof subject.verify === "string")) return yield* Effect.fail(/* @__PURE__ */ new Error(`${subject.name} must have exactly one of validate or verify`));
@@ -749,14 +805,14 @@ const compileEvalEffect = (path) => Effect.gen(function* () {
 			verify: subject.verify ?? null
 		};
 	}), { concurrency: 4 });
-	const tasks = yield* Effect.forEach(definition.tasks, (task) => taskOf(entry, task, clis, mcp), { concurrency: 4 });
-	return {
+	const tasks = yield* Effect.forEach(definition.tasks, (task) => taskOf(entry, task, clis, mcp, apis), { concurrency: 4 });
+	return yield* Schema.decodeUnknown(PublicStartEvalRequest)({
 		cases,
 		name: definition.name,
 		prompt: definition.prompt,
 		tasks,
 		trials: definition.trials
-	};
+	});
 }).pipe(Effect.withSpan("Eval.compile"));
 const compileEval = (path) => Effect.runPromise(compileEvalEffect(path));
 //#endregion

package/dist/config.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_client = require("./client-C-P2I00z.cjs");
+const require_client = require("./client-CfYNoI3L.cjs");
 let effect = require("effect");
 //#region src/client/config.ts
 const apiKeyConfig = effect.Config.redacted("ANPORD_API_KEY");

package/dist/config.d.cts

@@ -1,4 +1,4 @@
-import { t as AnpordApi } from "./client-BirFO3jz.cjs";
+import { t as AnpordApi } from "./client-CMkx-LKC.cjs";
 import { Config, Layer } from "effect";
 //#region src/client/config.d.ts
 declare const apiKeyConfig: Config.Config<import("effect/Redacted").Redacted<string>>;

package/dist/config.d.mts

@@ -1,4 +1,4 @@
-import { t as AnpordApi } from "./client-BgRs8JvW.mjs";
+import { t as AnpordApi } from "./client-C8tBcLYs.mjs";
 import { Config, Layer } from "effect";
 //#region src/client/config.d.ts
 declare const apiKeyConfig: Config.Config<import("effect/Redacted").Redacted<string>>;

package/dist/config.mjs

@@ -1,4 +1,4 @@
-import { n as DEFAULT_BASE_URL, r as layer } from "./client-BfdjFtlV.mjs";
+import { n as DEFAULT_BASE_URL, r as layer } from "./client-YWKJd3f-.mjs";
 import { Config, Effect, Layer } from "effect";
 //#region src/client/config.ts
 const apiKeyConfig = Config.redacted("ANPORD_API_KEY");

package/dist/define-NZvmQuIv.d.cts

@@ -0,0 +1,34 @@
+import { StandardSchemaV1 } from "@standard-schema/spec";
+//#region src/mock-api/define.d.ts
+type Responses = Readonly<Record<number, StandardSchemaV1>>;
+type ResponseOf<R extends Responses> = { [Status in keyof R & number]: {
+  readonly status: Status;
+  readonly body: StandardSchemaV1.InferInput<R[Status]>;
+  readonly headers?: Readonly<Record<string, string>>;
+}; }[keyof R & number];
+interface ApiHandlerContext {
+  readonly log: (value: unknown) => void;
+  readonly signal: AbortSignal;
+}
+interface EndpointDefinition {
+  readonly description?: string;
+  handler(input: unknown, context: ApiHandlerContext): ResponseOf<Responses> | Promise<ResponseOf<Responses>>;
+  readonly inputSchema: StandardSchemaV1;
+  readonly method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS";
+  readonly path: `/${string}`;
+  readonly responses: Responses;
+}
+type EndpointOptions<Input extends StandardSchemaV1, Output extends Responses, Result> = Omit<EndpointDefinition, "inputSchema" | "responses" | "handler"> & {
+  readonly inputSchema: Input;
+  readonly responses: Output;
+  handler(input: StandardSchemaV1.InferOutput<Input>, context: ApiHandlerContext): Result | Promise<Result>;
+};
+declare const endpoint: <Input extends StandardSchemaV1, const Output extends Responses, const Result extends ResponseOf<Output>>(definition: EndpointOptions<Input, Output, Result>) => EndpointOptions<Input, Output, Result>;
+interface ApiDefinition {
+  readonly endpoints: readonly EndpointDefinition[];
+  readonly name: string;
+  readonly redact?: readonly string[];
+}
+declare const api: (definition: ApiDefinition) => ApiDefinition;
+//#endregion
+export { endpoint as a, api as i, ApiHandlerContext as n, EndpointDefinition as r, ApiDefinition as t };

package/dist/define-NZvmQuIv.d.mts

@@ -0,0 +1,34 @@
+import { StandardSchemaV1 } from "@standard-schema/spec";
+//#region src/mock-api/define.d.ts
+type Responses = Readonly<Record<number, StandardSchemaV1>>;
+type ResponseOf<R extends Responses> = { [Status in keyof R & number]: {
+  readonly status: Status;
+  readonly body: StandardSchemaV1.InferInput<R[Status]>;
+  readonly headers?: Readonly<Record<string, string>>;
+}; }[keyof R & number];
+interface ApiHandlerContext {
+  readonly log: (value: unknown) => void;
+  readonly signal: AbortSignal;
+}
+interface EndpointDefinition {
+  readonly description?: string;
+  handler(input: unknown, context: ApiHandlerContext): ResponseOf<Responses> | Promise<ResponseOf<Responses>>;
+  readonly inputSchema: StandardSchemaV1;
+  readonly method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS";
+  readonly path: `/${string}`;
+  readonly responses: Responses;
+}
+type EndpointOptions<Input extends StandardSchemaV1, Output extends Responses, Result> = Omit<EndpointDefinition, "inputSchema" | "responses" | "handler"> & {
+  readonly inputSchema: Input;
+  readonly responses: Output;
+  handler(input: StandardSchemaV1.InferOutput<Input>, context: ApiHandlerContext): Result | Promise<Result>;
+};
+declare const endpoint: <Input extends StandardSchemaV1, const Output extends Responses, const Result extends ResponseOf<Output>>(definition: EndpointOptions<Input, Output, Result>) => EndpointOptions<Input, Output, Result>;
+interface ApiDefinition {
+  readonly endpoints: readonly EndpointDefinition[];
+  readonly name: string;
+  readonly redact?: readonly string[];
+}
+declare const api: (definition: ApiDefinition) => ApiDefinition;
+//#endregion
+export { endpoint as a, api as i, ApiHandlerContext as n, EndpointDefinition as r, ApiDefinition as t };

package/dist/{eval-judges-DPPUftbh.d.cts → eval-judges-JiC8cUCE.d.cts}

@@ -17,7 +17,7 @@ declare const EvalJudge: Schema.Union<[Schema.Struct<{
   }>;
 }>, Schema.Struct<{
   provider: Schema.optional<typeof Schema.Never>;
-  harness: Schema.SchemaClass<"codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor", "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor", never>;
+  harness: Schema.SchemaClass<"cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen", "cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen", never>;
   kind: Schema.Literal<["judge"]>;
   name: Schema.filter<Schema.filter<typeof Schema.String>>;
   model: Schema.filter<Schema.filter<typeof Schema.String>>;

package/dist/{eval-judges-DPPUftbh.d.mts → eval-judges-JiC8cUCE.d.mts}

@@ -17,7 +17,7 @@ declare const EvalJudge: Schema.Union<[Schema.Struct<{
   }>;
 }>, Schema.Struct<{
   provider: Schema.optional<typeof Schema.Never>;
-  harness: Schema.SchemaClass<"codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor", "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor", never>;
+  harness: Schema.SchemaClass<"cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen", "cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen", never>;
   kind: Schema.Literal<["judge"]>;
   name: Schema.filter<Schema.filter<typeof Schema.String>>;
   model: Schema.filter<Schema.filter<typeof Schema.String>>;

package/dist/{eval-validations-BRdoZrTQ.mjs → eval-validations-Bmagd2y7.mjs}

@@ -12,7 +12,7 @@ const ValidationValue = Schema.Struct({
 }).annotations({ identifier: "ValidationValue" });
 const ValidationCall = Schema.Struct({
 	index: Schema.NonNegativeInt,
-	method: Schema.Literal("answer", "transcript", "readText", "exists", "exec", "cli.calls", "mcp.calls"),
+	method: Schema.Literal("answer", "transcript", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"),
 	startedAt: Schema.Number,
 	durationMs: Schema.NullOr(Schema.NonNegativeInt),
 	input: ValidationValue,
@@ -110,4 +110,4 @@ const validationExecution = (identity, startedAt) => ({
 	logs: []
 });
 //#endregion
-export { validationExecution as a, validationCapture as i, VALIDATION_FRAME as n, validationSnapshot as o, unavailableValue as r, EvalValidations as t };
+export { validationCapture as a, unavailableValue as i, VALIDATION_FRAME as n, validationExecution as o, ValidationValue as r, validationSnapshot as s, EvalValidations as t };

package/dist/{eval-validations-_MlnQ_uv.cjs → eval-validations-CqMUt5ra.cjs}

@@ -12,7 +12,7 @@ const ValidationValue = effect.Schema.Struct({
 }).annotations({ identifier: "ValidationValue" });
 const ValidationCall = effect.Schema.Struct({
 	index: effect.Schema.NonNegativeInt,
-	method: effect.Schema.Literal("answer", "transcript", "readText", "exists", "exec", "cli.calls", "mcp.calls"),
+	method: effect.Schema.Literal("answer", "transcript", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"),
 	startedAt: effect.Schema.Number,
 	durationMs: effect.Schema.NullOr(effect.Schema.NonNegativeInt),
 	input: ValidationValue,
@@ -122,6 +122,12 @@ Object.defineProperty(exports, "VALIDATION_FRAME", {
 		return VALIDATION_FRAME;
 	}
 });
+Object.defineProperty(exports, "ValidationValue", {
+	enumerable: true,
+	get: function() {
+		return ValidationValue;
+	}
+});
 Object.defineProperty(exports, "unavailableValue", {
 	enumerable: true,
 	get: function() {

package/dist/eval.cjs

@@ -1,4 +1,4 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_compiler = require("./compiler-uwb1pkdK.cjs");
+const require_compiler = require("./compiler-4FKsNxZ-.cjs");
 exports.compileEval = require_compiler.compileEval;
 exports.compileEvalEffect = require_compiler.compileEvalEffect;