anpord 0.1.13 → 0.1.17

package/dist/{compiler-uwb1pkdK.cjs → compiler-CYwJNlT4.cjs}

@@ -1,10 +1,11 @@
-const require_evals = require("./evals-DouMsrsM.cjs");
-const require_eval_judges = require("./eval-judges-BkmiVLLq.cjs");
+const require_eval_judges = require("./eval-judges-1zF4PVqV.cjs");
+const require_api_mocks = require("./api-mocks-CuVw86Qw.cjs");
+const require_evals_api = require("./evals-api-CqhPZCr5.cjs");
 const require_source = require("./source-DvlGawEw.cjs");
 let effect = require("effect");
+let node_fs_promises = require("node:fs/promises");
 let node_path = require("node:path");
 let node_fs = require("node:fs");
-let node_fs_promises = require("node:fs/promises");
 let node_url = require("node:url");
 let esbuild = require("esbuild");
 let node_zlib = require("node:zlib");
@@ -43,15 +44,33 @@ const sourceFiles = (map, entry) => {
 			content
 		}];
 	});
-	return effect.Schema.decodeUnknownSync(require_evals.EvalSourceFiles)(files.toSorted((a, b) => Number(b.path === entryPath) - Number(a.path === entryPath) || a.path.localeCompare(b.path)));
+	return effect.Schema.decodeUnknownSync(require_evals_api.EvalSourceFiles)(files.toSorted((a, b) => Number(b.path === entryPath) - Number(a.path === entryPath) || a.path.localeCompare(b.path)));
 };
 //#endregion
 //#region src/evals/eval-bundle.ts
-const authoringExports = ["export const defineEval = value => value;", `export { empty, files, repo } from "./source";`].join("\n");
+const authoringExports = ["export const defineEval = (first, second) => typeof first === 'string' ? second : first;", `export { empty, files, repo } from "./source";`].join("\n");
 const authoringDir = (0, node_path.dirname)((0, node_url.fileURLToPath)(require("url").pathToFileURL(__filename).href));
 const ANPORD_MODULE = /^anpord$/;
 const ANY_MODULE = /.*/;
 const localModules = [
+	{
+		built: "api.mjs",
+		filter: /^anpord\/api$/,
+		namespace: "anpord-api-authoring",
+		source: "../mock-api/index.ts"
+	},
+	{
+		built: "api-runtime.mjs",
+		filter: /^anpord\/api\/runtime$/,
+		namespace: "anpord-api-runtime",
+		source: "../mock-api/runtime.ts"
+	},
+	{
+		built: "api-context.mjs",
+		filter: /^anpord\/api\/context$/,
+		namespace: "anpord-api-context",
+		source: "../mock-api/context.ts"
+	},
 	{
 		built: "validator-runtime.mjs",
 		filter: /^anpord\/validators\/runtime$/,
@@ -128,6 +147,7 @@ const bundle = (contents, entry, options = {}) => effect.Effect.tryPromise({
 	try: () => (0, esbuild.build)({
 		absWorkingDir: process.cwd(),
 		bundle: true,
+		banner: { js: "import { createRequire as anpordCreateRequire } from \"node:module\"; const require = anpordCreateRequire(import.meta.url);" },
 		format: "esm",
 		metafile: true,
 		minify: options.minify,
@@ -158,36 +178,6 @@ const bundle = (contents, entry, options = {}) => effect.Effect.tryPromise({
 	catch: (cause) => new Error(`Could not compile ${entry}`, { cause })
 });
 //#endregion
-//#region src/evals/case-modules.ts
-const realOrGiven = (path) => {
-	try {
-		return node_fs.realpathSync.native(path);
-	} catch {
-		return path;
-	}
-};
-const sameFile = (one, other) => one === other || realOrGiven(one) === realOrGiven(other);
-const exportingModule = (entry, inputs, name) => effect.Effect.gen(function* () {
-	const pattern = new RegExp(`\\bexport\\s+(?:async\\s+)?(?:const|function)\\s+${name}\\b`);
-	const matches = yield* effect.Effect.filter(inputs.filter((path) => !(sameFile(path, entry) || path.includes(`${node_path.sep}node_modules${node_path.sep}`))), (path) => effect.Effect.tryPromise(() => (0, node_fs_promises.readFile)(path, "utf8")).pipe(effect.Effect.map((source) => pattern.test(source)), effect.Effect.orElseSucceed(() => false)), { concurrency: 8 });
-	if (matches.length !== 1) return yield* effect.Effect.fail(/* @__PURE__ */ new Error(`${name} must be one named function or const export from a separate TypeScript file`));
-	return matches[0];
-});
-const bundledCaseModule = (entry, inputs, name, wrap) => effect.Effect.gen(function* () {
-	const module = yield* exportingModule(entry, inputs, name);
-	const { source } = yield* bundle(wrap(module, name), entry);
-	return {
-		name,
-		source
-	};
-});
-//#endregion
-//#region src/evals/profile-compose.ts
-const appendInstall = (profile, command) => ({
-	...profile,
-	install: profile.install ? `${profile.install} && ${command}` : command
-});
-//#endregion
 //#region src/evals/program-package.ts
 const CHUNK_SIZE = 9e4;
 const packageProgram = (root, entry, source) => {
@@ -197,18 +187,20 @@ const packageProgram = (root, entry, source) => {
 	const loader = `#!/usr/bin/env node\nimport{readFile,writeFile}from"node:fs/promises";import{gunzipSync}from"node:zlib";const root=new URL("./",import.meta.url),names=${JSON.stringify(names)},data=(await Promise.all(names.map(name=>readFile(new URL(name,root),"utf8")))).join(""),runtime=new URL("runtime.mjs",root);await writeFile(runtime,gunzipSync(Buffer.from(data,"base64")));await import(runtime.href);`;
 	return {
 		entry: `${root}/${entry}`,
-		files: Object.fromEntries([[`${root}/${entry}`, loader], ...names.map((name, index) => [`${root}/${name}`, chunks[index]])])
+		files: Object.fromEntries([[`${root}/${entry}`, loader], ...chunks.map((chunk, index) => [`${root}/${index}.txt`, chunk])])
 	};
 };
 //#endregion
 //#region src/evals/runner-source.ts
-const validatorCaseEntry = (entry, index, names = []) => `import definition from ${JSON.stringify(entry)};
+const importsDefinition = ({ entry, exportName }) => exportName === null ? `import definition from ${JSON.stringify(entry)};` : `import { ${exportName} as definition } from ${JSON.stringify(entry)};`;
+const validatorCaseEntry = (ref, index, names = []) => `${importsDefinition(ref)}
 import { runValidators } from "anpord/validators/runtime";
 const selected = definition.cases[${index}].validate;
 const names = ${JSON.stringify(names)};
 const checks = (Array.isArray(selected) ? selected : [selected]).flatMap((validate, index) => typeof validate === "function" ? [{ index, name: names.find(check => check.index === index)?.name ?? "Validator " + (index + 1), validate }] : []);
 await runValidators(checks, definition.captureValidation !== false);`;
 const prepareRuntime = `
+import { apiContext } from "anpord/api/context";
 import { access, readFile } from "node:fs/promises";
 import { spawn } from "node:child_process";
 
@@ -236,6 +228,7 @@ const exec = (file, args = [], options = {}) => new Promise((resolve, reject) =>
 });
 
 const context = {
+  api: apiContext,
   cached: process.env.ANPORD_CACHE_RESTORED === "1",
   exec,
   exists: path => access(path).then(() => true, () => false),
@@ -252,25 +245,88 @@ try {
 }
 `;
 const prepareEntry = (module, name) => `import { ${name} as setup } from ${JSON.stringify(module)};\n${prepareRuntime}`;
-const definitionEntry = (entry) => `import definition from ${JSON.stringify(entry)}; export default definition;`;
-const mcpEntry = (entry, index) => `import definition from ${JSON.stringify(entry)};
+const definitionEntry = (ref) => `${importsDefinition(ref)} export default definition;`;
+const mcpEntry = (ref, index) => `${importsDefinition(ref)}
 import { runMcpServer } from "anpord/mcp/runtime";
 const server = definition.mcp?.[${index}];
 if (server === undefined) throw new Error("MCP server ${index} is missing");
 runMcpServer(server);`;
-const cliEntry = (entry, index) => `import { fileURLToPath } from "node:url";
-import definition from ${JSON.stringify(entry)};
+const cliEntry = (ref, index) => `import { fileURLToPath } from "node:url";
+${importsDefinition(ref)}
 import { runCli } from "anpord/cli/runtime";
 const cli = definition.cli?.[${index}];
 if (cli === undefined) throw new Error("CLI ${index} is missing");
 await runCli(cli, fileURLToPath(new URL("../../cli-calls.jsonl", import.meta.url)));`;
 //#endregion
+//#region src/evals/api-profile.ts
+const compileApis = (ref, definitions) => effect.Effect.gen(function* () {
+	if (definitions.length === 0) return {};
+	const names = definitions.map(({ name }) => name);
+	if (new Set(names).size !== names.length) return yield* effect.Effect.fail(/* @__PURE__ */ new Error("Duplicate API mock name"));
+	const { source } = yield* bundle(`${importsDefinition(ref)}
+import { runApiServers } from "anpord/api/runtime";
+runApiServers(definition.api ?? []);`, ref.entry, { minify: true });
+	const program = packageProgram("workspace/.anpord/api", "server.mjs", source);
+	return {
+		...program.files,
+		[require_api_mocks.API_PROGRAM]: JSON.stringify({ entry: program.entry.slice(10) })
+	};
+});
+const withApis = (task, files) => {
+	if (Object.keys(files).length === 0) return task;
+	const profile = task.profile ?? {
+		name: "anpord-api",
+		files: {}
+	};
+	for (const path of Object.keys(files)) if (profile.files[path] !== void 0) throw new Error(`Profile file ${path} is reserved for API mocks`);
+	return {
+		...task,
+		profile: {
+			...profile,
+			files: {
+				...profile.files,
+				...files
+			}
+		}
+	};
+};
+//#endregion
+//#region src/evals/case-modules.ts
+const realOrGiven = (path) => {
+	try {
+		return node_fs.realpathSync.native(path);
+	} catch {
+		return path;
+	}
+};
+const sameFile = (one, other) => one === other || realOrGiven(one) === realOrGiven(other);
+const exportingModule = (entry, inputs, name) => effect.Effect.gen(function* () {
+	const pattern = new RegExp(`\\bexport\\s+(?:async\\s+)?(?:const|function)\\s+${name}\\b`);
+	const matches = yield* effect.Effect.filter(inputs.filter((path) => !(sameFile(path, entry) || path.includes(`${node_path.sep}node_modules${node_path.sep}`))), (path) => effect.Effect.tryPromise(() => (0, node_fs_promises.readFile)(path, "utf8")).pipe(effect.Effect.map((source) => pattern.test(source)), effect.Effect.orElseSucceed(() => false)), { concurrency: 8 });
+	if (matches.length !== 1) return yield* effect.Effect.fail(/* @__PURE__ */ new Error(`${name} must be one named function or const export from a separate TypeScript file`));
+	return matches[0];
+});
+const bundledCaseModule = (entry, inputs, name, wrap) => effect.Effect.gen(function* () {
+	const module = yield* exportingModule(entry, inputs, name);
+	const { source } = yield* bundle(wrap(module, name), entry, { minify: true });
+	return {
+		name,
+		source
+	};
+});
+//#endregion
+//#region src/evals/profile-compose.ts
+const appendInstall = (profile, command) => ({
+	...profile,
+	install: profile.install ? `${profile.install} && ${command}` : command
+});
+//#endregion
 //#region src/evals/cli-profile.ts
-const compileClis = (entry, definitions) => effect.Effect.gen(function* () {
+const compileClis = (ref, definitions) => effect.Effect.gen(function* () {
 	const paths = definitions.map(({ path }) => path);
 	const duplicate = paths.find((path, index) => paths.indexOf(path) !== index);
 	if (duplicate !== void 0) return yield* effect.Effect.fail(/* @__PURE__ */ new Error(`Duplicate CLI: ${duplicate}`));
-	return yield* effect.Effect.forEach(definitions, (definition, index) => bundle(cliEntry(entry, index), entry, { minify: true }).pipe(effect.Effect.map(({ source }) => ({
+	return yield* effect.Effect.forEach(definitions, (definition, index) => bundle(cliEntry(ref, index), ref.entry, { minify: true }).pipe(effect.Effect.map(({ source }) => ({
 		...packageProgram(`workspace/.anpord/cli/${index}`, "cli.mjs", source),
 		path: definition.path
 	}))), { concurrency: 4 });
@@ -300,7 +356,7 @@ const withClis = (task, clis) => {
 };
 //#endregion
 //#region src/evals/compile-validator.ts
-const compileValidator = (entry, subject, caseIndex, captureSource, captureValidation = true) => effect.Effect.gen(function* () {
+const compileValidator = (ref, subject, caseIndex, captureSource, captureValidation = true) => effect.Effect.gen(function* () {
 	if (subject.validate === void 0) return null;
 	const validations = Array.isArray(subject.validate) ? subject.validate : [subject.validate];
 	if (validations.length === 0 || validations.length > 20) return yield* effect.Effect.fail(/* @__PURE__ */ new Error("Use between 1 and 20 validators"));
@@ -311,7 +367,7 @@ const compileValidator = (entry, subject, caseIndex, captureSource, captureValid
 		name: value.name || `Validator ${index + 1}`
 	}] : []);
 	const hasCode = manifest.length > 0;
-	const compiled = yield* bundle(hasCode ? validatorCaseEntry(entry, caseIndex, manifest) : definitionEntry(entry), entry, {
+	const compiled = yield* bundle(hasCode ? validatorCaseEntry(ref, caseIndex, manifest) : definitionEntry(ref), ref.entry, {
 		minify: true,
 		captureSource
 	});
@@ -321,7 +377,7 @@ const compileValidator = (entry, subject, caseIndex, captureSource, captureValid
 		manifest,
 		capture: captureValidation
 	}] : [];
-	return yield* effect.Schema.decodeUnknown(require_evals.EvalValidator)({
+	return yield* effect.Schema.decodeUnknown(require_evals_api.EvalValidator)({
 		...judges.length > 0 ? {
 			kind: "judged",
 			name: subject.name,
@@ -334,7 +390,7 @@ const compileValidator = (entry, subject, caseIndex, captureSource, captureValid
 });
 //#endregion
 //#region src/evals/definition-loader.ts
-const loadDefinition = (entry) => bundle(definitionEntry(entry), entry).pipe(effect.Effect.flatMap(({ inputs, source }) => effect.Effect.acquireUseRelease(effect.Effect.tryPromise(() => (0, node_fs_promises.mkdtemp)((0, node_path.join)((0, node_os.tmpdir)(), "anpord-eval-"))), (directory) => effect.Effect.tryPromise({
+const loadDefinition = (ref) => bundle(definitionEntry(ref), ref.entry).pipe(effect.Effect.flatMap(({ inputs, source }) => effect.Effect.acquireUseRelease(effect.Effect.tryPromise(() => (0, node_fs_promises.mkdtemp)((0, node_path.join)((0, node_os.tmpdir)(), "anpord-eval-"))), (directory) => effect.Effect.tryPromise({
 	try: async () => {
 		const output = (0, node_path.join)(directory, "definition.mjs");
 		await (0, node_fs_promises.writeFile)(output, source);
@@ -343,7 +399,7 @@ const loadDefinition = (entry) => bundle(definitionEntry(entry), entry).pipe(eff
 			inputs
 		};
 	},
-	catch: (cause) => new Error(`Could not load ${entry}: ${cause instanceof Error ? cause.message : String(cause)}`, { cause })
+	catch: (cause) => new Error(`Could not load ${ref.entry}: ${cause instanceof Error ? cause.message : String(cause)}`, { cause })
 }), (directory) => effect.Effect.promise(() => (0, node_fs_promises.rm)(directory, {
 	force: true,
 	recursive: true
@@ -391,6 +447,39 @@ const localRepo = (cwd) => effect.Effect.gen(function* () {
 	});
 }).pipe(effect.Effect.withSpan("Eval.localRepo"));
 //#endregion
+//#region src/evals/define.ts
+const SOURCE_URL = Symbol.for("anpord.sourceUrl");
+function defineEval(first, second) {
+	if (typeof first !== "string") return first;
+	return Object.defineProperty(second, SOURCE_URL, {
+		enumerable: false,
+		value: first
+	});
+}
+const sourceUrlOf = (definition) => definition[SOURCE_URL];
+//#endregion
+//#region src/evals/locate.ts
+const HINT = "Pass import.meta.url as the first argument to defineEval, or compile by path.";
+const exportHolding = async (entry, definition) => {
+	const module = await import(entry);
+	if (module.default === definition) return null;
+	const named = Object.keys(module).find((key) => module[key] === definition);
+	if (named === void 0) throw new Error(`The definition passed is not exported from ${entry}. ${HINT}`);
+	return named;
+};
+const locate = (definition) => effect.Effect.gen(function* () {
+	const url = sourceUrlOf(definition);
+	if (url === void 0) return yield* effect.Effect.fail(/* @__PURE__ */ new Error(`This eval does not know where it was written. ${HINT}`));
+	const exportName = yield* effect.Effect.tryPromise({
+		catch: (cause) => cause instanceof Error ? cause : new Error(String(cause)),
+		try: () => exportHolding(url, definition)
+	});
+	return {
+		entry: (0, node_url.fileURLToPath)(url),
+		exportName
+	};
+});
+//#endregion
 //#region src/evals/mcp-harness.ts
 const Entries = effect.Schema.Record({
 	key: effect.Schema.String,
@@ -487,15 +576,15 @@ const adapters = {
 const applyMcpHarness = (harness, profile, servers) => {
 	const adapter = adapters[harness];
 	if (adapter === null) throw new Error(`MCP mock servers are not supported by ${harness}`);
-	return effect.Schema.decodeUnknownSync(require_evals.HarnessProfile)(adapter.apply(profile, servers));
+	return effect.Schema.decodeUnknownSync(require_evals_api.HarnessProfile)(adapter.apply(profile, servers));
 };
 //#endregion
 //#region src/evals/mcp-profile.ts
-const compileMcpServers = (entry, definitions) => effect.Effect.gen(function* () {
+const compileMcpServers = (ref, definitions) => effect.Effect.gen(function* () {
 	const names = definitions.map(({ name }) => name);
 	const duplicate = names.find((name, index) => names.indexOf(name) !== index);
 	if (duplicate !== void 0) return yield* effect.Effect.fail(/* @__PURE__ */ new Error(`Duplicate MCP server: ${duplicate}`));
-	return yield* effect.Effect.forEach(definitions, (definition, index) => bundle(mcpEntry(entry, index), entry, { minify: true }).pipe(effect.Effect.map(({ source }) => ({
+	return yield* effect.Effect.forEach(definitions, (definition, index) => bundle(mcpEntry(ref, index), ref.entry, { minify: true }).pipe(effect.Effect.map(({ source }) => ({
 		...packageProgram(`workspace/.anpord/mcp/${index}`, "server.mjs", source),
 		name: definition.name
 	}))), { concurrency: 4 });
@@ -533,17 +622,17 @@ var ProfilePathInvalid = class extends effect.Data.TaggedError("ProfilePathInval
 };
 var ProfileFileTooLarge = class extends effect.Data.TaggedError("ProfileFileTooLarge") {
 	get message() {
-		return `${this.path} is ${this.chars} characters; a profile file holds at most ${require_evals.PROFILE_LIMITS.fileChars}`;
+		return `${this.path} is ${this.chars} characters; a profile file holds at most ${require_evals_api.PROFILE_LIMITS.fileChars}`;
 	}
 };
 var ProfileTooLarge = class extends effect.Data.TaggedError("ProfileTooLarge") {
 	get message() {
-		return `The profile's files hold ${this.chars} characters; at most ${require_evals.PROFILE_LIMITS.totalChars} fit`;
+		return `The profile's files hold ${this.chars} characters; at most ${require_evals_api.PROFILE_LIMITS.totalChars} fit`;
 	}
 };
 var ProfileTooManyFiles = class extends effect.Data.TaggedError("ProfileTooManyFiles") {
 	get message() {
-		return `The profile has ${this.count} files; at most ${require_evals.PROFILE_LIMITS.files} fit`;
+		return `The profile has ${this.count} files; at most ${require_evals_api.PROFILE_LIMITS.files} fit`;
 	}
 };
 var ProfileManifestInvalid = class extends effect.Data.TaggedError("ProfileManifestInvalid") {
@@ -570,7 +659,7 @@ var ProfileStepNotSupported = class extends effect.Data.TaggedError("ProfileStep
 //#region src/evals/profile-manifest.ts
 const Manifest = effect.Schema.Struct({
 	env: effect.Schema.optional(effect.Schema.Record({
-		key: require_evals.EnvName,
+		key: require_evals_api.EnvName,
 		value: effect.Schema.String
 	})),
 	install: effect.Schema.optional(effect.Schema.String),
@@ -647,7 +736,7 @@ const shippedPaths = (dir) => effect.Effect.tryPromise({
 		return (await Promise.all(SHIPPED_ROOTS.map((root) => walkRoot(dir, root)))).flat().sort();
 	}
 });
-const validPath = (path) => effect.Schema.decodeUnknown(require_evals.ProfilePath)(path).pipe(effect.Effect.mapError(() => new ProfilePathInvalid({ path })));
+const validPath = (path) => effect.Schema.decodeUnknown(require_evals_api.ProfilePath)(path).pipe(effect.Effect.mapError(() => new ProfilePathInvalid({ path })));
 /** Nothing when the file holds a NUL byte and so is not text. */
 const readShipped = (dir, path) => effect.Effect.gen(function* () {
 	const bytes = yield* effect.Effect.tryPromise({
@@ -659,7 +748,7 @@ const readShipped = (dir, path) => effect.Effect.gen(function* () {
 	});
 	if (bytes.includes(0)) return [];
 	const content = bytes.toString("utf8");
-	if (content.length > require_evals.PROFILE_LIMITS.fileChars) return yield* new ProfileFileTooLarge({
+	if (content.length > require_evals_api.PROFILE_LIMITS.fileChars) return yield* new ProfileFileTooLarge({
 		chars: content.length,
 		path
 	});
@@ -667,14 +756,14 @@ const readShipped = (dir, path) => effect.Effect.gen(function* () {
 });
 const shippedFiles = (dir) => effect.Effect.gen(function* () {
 	const paths = yield* shippedPaths(dir);
-	if (paths.length > require_evals.PROFILE_LIMITS.files) return yield* new ProfileTooManyFiles({ count: paths.length });
+	if (paths.length > require_evals_api.PROFILE_LIMITS.files) return yield* new ProfileTooManyFiles({ count: paths.length });
 	const entries = yield* effect.Effect.forEach(paths, (path) => readShipped(dir, path), { concurrency: 8 }).pipe(effect.Effect.map((read) => read.flat()));
 	const chars = entries.reduce((sum, [, content]) => sum + content.length, 0);
-	if (chars > require_evals.PROFILE_LIMITS.totalChars) return yield* new ProfileTooLarge({ chars });
+	if (chars > require_evals_api.PROFILE_LIMITS.totalChars) return yield* new ProfileTooLarge({ chars });
 	return Object.fromEntries(entries);
 });
 const fittingHarness = (base, profile) => {
-	if (require_evals.profileFitsHarness({
+	if (require_evals_api.profileFitsHarness({
 		harness: base,
 		profile
 	})) return effect.Effect.succeed(profile);
@@ -708,7 +797,7 @@ const profileTask = (entry, task) => effect.Effect.gen(function* () {
 }).pipe(effect.Effect.withSpan("Eval.profileTask", { attributes: { profile: task.harness.profile.name } }));
 //#endregion
 //#region src/evals/compiler.ts
-const taskOf = (entry, task, clis, mcp) => effect.Effect.gen(function* () {
+const taskOf = (entry, task, clis, mcp, apis) => effect.Effect.gen(function* () {
 	const compiled = typeof task.harness === "string" ? {
 		harness: task.harness,
 		model: task.model,
@@ -717,7 +806,7 @@ const taskOf = (entry, task, clis, mcp) => effect.Effect.gen(function* () {
 		...task,
 		harness: task.harness
 	});
-	return yield* effect.Effect.try(() => withClis(withMcpServers(compiled, mcp), clis));
+	return yield* effect.Effect.try(() => withApis(withClis(withMcpServers(compiled, mcp), clis), apis));
 });
 const sourceFor = (definition, subject, fallback) => {
 	const source = subject.source ?? definition.source;
@@ -727,17 +816,21 @@ const sourceFor = (definition, subject, fallback) => {
 		onSome: (source) => ({ source })
 	});
 };
-const compileEvalEffect = (path) => effect.Effect.gen(function* () {
-	const entry = (0, node_path.resolve)(path);
-	const loaded = yield* loadDefinition(entry);
+const compileRefEffect = (ref) => effect.Effect.gen(function* () {
+	const entry = ref.entry;
+	const loaded = yield* loadDefinition(ref);
 	const definition = loaded.definition;
-	if (!isDefinition(definition)) return yield* effect.Effect.fail(/* @__PURE__ */ new Error(`${entry} must default export defineEval({ ... })`));
-	const clis = yield* compileClis(entry, definition.cli ?? []);
-	const mcp = yield* compileMcpServers(entry, definition.mcp ?? []);
+	if (!isDefinition(definition)) {
+		const named = ref.exportName === null ? "default export" : `export ${ref.exportName}`;
+		return yield* effect.Effect.fail(/* @__PURE__ */ new Error(`${entry} must ${named} defineEval({ ... })`));
+	}
+	const clis = yield* compileClis(ref, definition.cli ?? []);
+	const mcp = yield* compileMcpServers(ref, definition.mcp ?? []);
+	const apis = yield* compileApis(ref, definition.api ?? []);
 	const fallback = definition.source === void 0 && definition.cases.some((subject) => subject.source === void 0) ? yield* localRepo((0, node_path.dirname)(entry)) : effect.Option.none();
 	const cases = yield* effect.Effect.forEach(definition.cases, (subject, caseIndex) => effect.Effect.gen(function* () {
 		if (subject.validate !== void 0 === (typeof subject.verify === "string")) return yield* effect.Effect.fail(/* @__PURE__ */ new Error(`${subject.name} must have exactly one of validate or verify`));
-		const validator = yield* compileValidator(entry, subject, caseIndex, definition.captureSource !== false, definition.captureValidation !== false);
+		const validator = yield* compileValidator(ref, subject, caseIndex, definition.captureSource !== false, definition.captureValidation !== false);
 		const prepare = typeof subject.prepare === "function" ? yield* bundledCaseModule(entry, loaded.inputs, subject.prepare.name || `${subject.name}-prepare`, prepareEntry) : null;
 		return {
 			...subject.cache === void 0 ? {} : { cache: subject.cache },
@@ -749,17 +842,36 @@ const compileEvalEffect = (path) => effect.Effect.gen(function* () {
 			verify: subject.verify ?? null
 		};
 	}), { concurrency: 4 });
-	const tasks = yield* effect.Effect.forEach(definition.tasks, (task) => taskOf(entry, task, clis, mcp), { concurrency: 4 });
-	return {
+	const tasks = yield* effect.Effect.forEach(definition.tasks, (task) => taskOf(entry, task, clis, mcp, apis), { concurrency: 4 });
+	return yield* effect.Schema.decodeUnknown(require_evals_api.PublicStartEvalRequest)({
 		cases,
 		name: definition.name,
 		prompt: definition.prompt,
 		tasks,
 		trials: definition.trials
-	};
+	});
 }).pipe(effect.Effect.withSpan("Eval.compile"));
+const compileDefinitionEffect = (definition) => effect.Effect.flatMap(locate(definition), compileRefEffect);
+const compileDefinition = (definition) => effect.Effect.runPromise(compileDefinitionEffect(definition));
+const refOfPath = (path) => ({
+	entry: (0, node_path.resolve)(path),
+	exportName: null
+});
+const compileEvalEffect = (path) => compileRefEffect(refOfPath(path));
 const compileEval = (path) => effect.Effect.runPromise(compileEvalEffect(path));
 //#endregion
+Object.defineProperty(exports, "compileDefinition", {
+	enumerable: true,
+	get: function() {
+		return compileDefinition;
+	}
+});
+Object.defineProperty(exports, "compileDefinitionEffect", {
+	enumerable: true,
+	get: function() {
+		return compileDefinitionEffect;
+	}
+});
 Object.defineProperty(exports, "compileEval", {
 	enumerable: true,
 	get: function() {
@@ -772,3 +884,21 @@ Object.defineProperty(exports, "compileEvalEffect", {
 		return compileEvalEffect;
 	}
 });
+Object.defineProperty(exports, "compileRefEffect", {
+	enumerable: true,
+	get: function() {
+		return compileRefEffect;
+	}
+});
+Object.defineProperty(exports, "defineEval", {
+	enumerable: true,
+	get: function() {
+		return defineEval;
+	}
+});
+Object.defineProperty(exports, "sourceUrlOf", {
+	enumerable: true,
+	get: function() {
+		return sourceUrlOf;
+	}
+});

package/dist/config.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_client = require("./client-C-P2I00z.cjs");
+const require_client = require("./client-DziJjMFh.cjs");
 let effect = require("effect");
 //#region src/client/config.ts
 const apiKeyConfig = effect.Config.redacted("ANPORD_API_KEY");

package/dist/config.d.cts

@@ -1,4 +1,4 @@
-import { t as AnpordApi } from "./client-BirFO3jz.cjs";
+import { t as AnpordApi } from "./client-BVoIRPot.cjs";
 import { Config, Layer } from "effect";
 //#region src/client/config.d.ts
 declare const apiKeyConfig: Config.Config<import("effect/Redacted").Redacted<string>>;

package/dist/config.d.mts

@@ -1,4 +1,4 @@
-import { t as AnpordApi } from "./client-BgRs8JvW.mjs";
+import { t as AnpordApi } from "./client-_OgJhHpJ.mjs";
 import { Config, Layer } from "effect";
 //#region src/client/config.d.ts
 declare const apiKeyConfig: Config.Config<import("effect/Redacted").Redacted<string>>;

package/dist/config.mjs

@@ -1,4 +1,4 @@
-import { n as DEFAULT_BASE_URL, r as layer } from "./client-BfdjFtlV.mjs";
+import { n as DEFAULT_BASE_URL, r as layer } from "./client-DE3YNtWc.mjs";
 import { Config, Effect, Layer } from "effect";
 //#region src/client/config.ts
 const apiKeyConfig = Config.redacted("ANPORD_API_KEY");

package/dist/define-NZvmQuIv.d.cts

@@ -0,0 +1,34 @@
+import { StandardSchemaV1 } from "@standard-schema/spec";
+//#region src/mock-api/define.d.ts
+type Responses = Readonly<Record<number, StandardSchemaV1>>;
+type ResponseOf<R extends Responses> = { [Status in keyof R & number]: {
+  readonly status: Status;
+  readonly body: StandardSchemaV1.InferInput<R[Status]>;
+  readonly headers?: Readonly<Record<string, string>>;
+}; }[keyof R & number];
+interface ApiHandlerContext {
+  readonly log: (value: unknown) => void;
+  readonly signal: AbortSignal;
+}
+interface EndpointDefinition {
+  readonly description?: string;
+  handler(input: unknown, context: ApiHandlerContext): ResponseOf<Responses> | Promise<ResponseOf<Responses>>;
+  readonly inputSchema: StandardSchemaV1;
+  readonly method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS";
+  readonly path: `/${string}`;
+  readonly responses: Responses;
+}
+type EndpointOptions<Input extends StandardSchemaV1, Output extends Responses, Result> = Omit<EndpointDefinition, "inputSchema" | "responses" | "handler"> & {
+  readonly inputSchema: Input;
+  readonly responses: Output;
+  handler(input: StandardSchemaV1.InferOutput<Input>, context: ApiHandlerContext): Result | Promise<Result>;
+};
+declare const endpoint: <Input extends StandardSchemaV1, const Output extends Responses, const Result extends ResponseOf<Output>>(definition: EndpointOptions<Input, Output, Result>) => EndpointOptions<Input, Output, Result>;
+interface ApiDefinition {
+  readonly endpoints: readonly EndpointDefinition[];
+  readonly name: string;
+  readonly redact?: readonly string[];
+}
+declare const api: (definition: ApiDefinition) => ApiDefinition;
+//#endregion
+export { endpoint as a, api as i, ApiHandlerContext as n, EndpointDefinition as r, ApiDefinition as t };

package/dist/define-NZvmQuIv.d.mts

@@ -0,0 +1,34 @@
+import { StandardSchemaV1 } from "@standard-schema/spec";
+//#region src/mock-api/define.d.ts
+type Responses = Readonly<Record<number, StandardSchemaV1>>;
+type ResponseOf<R extends Responses> = { [Status in keyof R & number]: {
+  readonly status: Status;
+  readonly body: StandardSchemaV1.InferInput<R[Status]>;
+  readonly headers?: Readonly<Record<string, string>>;
+}; }[keyof R & number];
+interface ApiHandlerContext {
+  readonly log: (value: unknown) => void;
+  readonly signal: AbortSignal;
+}
+interface EndpointDefinition {
+  readonly description?: string;
+  handler(input: unknown, context: ApiHandlerContext): ResponseOf<Responses> | Promise<ResponseOf<Responses>>;
+  readonly inputSchema: StandardSchemaV1;
+  readonly method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS";
+  readonly path: `/${string}`;
+  readonly responses: Responses;
+}
+type EndpointOptions<Input extends StandardSchemaV1, Output extends Responses, Result> = Omit<EndpointDefinition, "inputSchema" | "responses" | "handler"> & {
+  readonly inputSchema: Input;
+  readonly responses: Output;
+  handler(input: StandardSchemaV1.InferOutput<Input>, context: ApiHandlerContext): Result | Promise<Result>;
+};
+declare const endpoint: <Input extends StandardSchemaV1, const Output extends Responses, const Result extends ResponseOf<Output>>(definition: EndpointOptions<Input, Output, Result>) => EndpointOptions<Input, Output, Result>;
+interface ApiDefinition {
+  readonly endpoints: readonly EndpointDefinition[];
+  readonly name: string;
+  readonly redact?: readonly string[];
+}
+declare const api: (definition: ApiDefinition) => ApiDefinition;
+//#endregion
+export { endpoint as a, api as i, ApiHandlerContext as n, EndpointDefinition as r, ApiDefinition as t };

package/dist/{eval-judges-DPPUftbh.d.cts → eval-judges-0FkhUOIB.d.cts}

@@ -6,7 +6,7 @@ declare const EvalJudge: Schema.Union<[Schema.Struct<{
   kind: Schema.Literal<["judge"]>;
   name: Schema.filter<Schema.filter<typeof Schema.String>>;
   model: Schema.filter<Schema.filter<typeof Schema.String>>;
-  rubric: Schema.filter<Schema.filter<typeof Schema.String>>;
+  prompt: Schema.filter<Schema.filter<typeof Schema.String>>;
   expected: Schema.optional<Schema.filter<typeof Schema.String>>;
   choices: Schema.filter<Schema.Record$<Schema.filter<typeof Schema.String>, Schema.filter<typeof Schema.Number>>>;
   threshold: Schema.optionalWith<Schema.filter<typeof Schema.Number>, {
@@ -21,7 +21,7 @@ declare const EvalJudge: Schema.Union<[Schema.Struct<{
   kind: Schema.Literal<["judge"]>;
   name: Schema.filter<Schema.filter<typeof Schema.String>>;
   model: Schema.filter<Schema.filter<typeof Schema.String>>;
-  rubric: Schema.filter<Schema.filter<typeof Schema.String>>;
+  prompt: Schema.filter<Schema.filter<typeof Schema.String>>;
   expected: Schema.optional<Schema.filter<typeof Schema.String>>;
   choices: Schema.filter<Schema.Record$<Schema.filter<typeof Schema.String>, Schema.filter<typeof Schema.Number>>>;
   threshold: Schema.optionalWith<Schema.filter<typeof Schema.Number>, {

package/dist/{eval-judges-DPPUftbh.d.mts → eval-judges-0FkhUOIB.d.mts}

@@ -6,7 +6,7 @@ declare const EvalJudge: Schema.Union<[Schema.Struct<{
   kind: Schema.Literal<["judge"]>;
   name: Schema.filter<Schema.filter<typeof Schema.String>>;
   model: Schema.filter<Schema.filter<typeof Schema.String>>;
-  rubric: Schema.filter<Schema.filter<typeof Schema.String>>;
+  prompt: Schema.filter<Schema.filter<typeof Schema.String>>;
   expected: Schema.optional<Schema.filter<typeof Schema.String>>;
   choices: Schema.filter<Schema.Record$<Schema.filter<typeof Schema.String>, Schema.filter<typeof Schema.Number>>>;
   threshold: Schema.optionalWith<Schema.filter<typeof Schema.Number>, {
@@ -21,7 +21,7 @@ declare const EvalJudge: Schema.Union<[Schema.Struct<{
   kind: Schema.Literal<["judge"]>;
   name: Schema.filter<Schema.filter<typeof Schema.String>>;
   model: Schema.filter<Schema.filter<typeof Schema.String>>;
-  rubric: Schema.filter<Schema.filter<typeof Schema.String>>;
+  prompt: Schema.filter<Schema.filter<typeof Schema.String>>;
   expected: Schema.optional<Schema.filter<typeof Schema.String>>;
   choices: Schema.filter<Schema.Record$<Schema.filter<typeof Schema.String>, Schema.filter<typeof Schema.Number>>>;
   threshold: Schema.optionalWith<Schema.filter<typeof Schema.Number>, {

package/dist/{eval-judges-BkmiVLLq.cjs → eval-judges-1zF4PVqV.cjs}

@@ -9,7 +9,7 @@ const fields = {
 	kind: effect.Schema.Literal("judge"),
 	name: text.pipe(effect.Schema.maxLength(100)),
 	model: text.pipe(effect.Schema.maxLength(200)),
-	rubric: text.pipe(effect.Schema.maxLength(32e3)),
+	prompt: text.pipe(effect.Schema.maxLength(32e3)),
 	expected: effect.Schema.optional(effect.Schema.String.pipe(effect.Schema.maxLength(32e3))),
 	choices: effect.Schema.Record({
 		key: text,

package/dist/{eval-judges-CwTvTq7N.mjs → eval-judges-Bee_ABJt.mjs}

@@ -9,7 +9,7 @@ const fields = {
 	kind: Schema.Literal("judge"),
 	name: text.pipe(Schema.maxLength(100)),
 	model: text.pipe(Schema.maxLength(200)),
-	rubric: text.pipe(Schema.maxLength(32e3)),
+	prompt: text.pipe(Schema.maxLength(32e3)),
 	expected: Schema.optional(Schema.String.pipe(Schema.maxLength(32e3))),
 	choices: Schema.Record({
 		key: text,