anpord 0.1.13 → 0.1.17

package/dist/{compiler-D96wI7UP.mjs → compiler-C62ss3a4.mjs}

@@ -1,10 +1,11 @@
-import { S as EvalSourceFiles, _ as HarnessProfile, b as ProfilePath, d as EvalValidator, g as EnvName, x as profileFitsHarness, y as PROFILE_LIMITS } from "./evals-Dcc_x6jW.mjs";
-import { t as EvalJudge } from "./eval-judges-CwTvTq7N.mjs";
+import { t as EvalJudge } from "./eval-judges-Bee_ABJt.mjs";
+import { r as API_PROGRAM } from "./api-mocks-wNUnpzJf.mjs";
+import { a as EvalValidator, c as PROFILE_LIMITS, d as EvalSourceFiles, l as ProfilePath, n as PublicStartEvalRequest, o as EnvName, s as HarnessProfile, u as profileFitsHarness } from "./evals-api-CsdU741J.mjs";
 import { a as parseRepo, i as cloneUrlOf, r as repo } from "./source-Fg1h2VNn.mjs";
 import { Config, Data, Effect, Option, Schema } from "effect";
+import { mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
 import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { existsSync, realpathSync } from "node:fs";
-import { mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { build } from "esbuild";
 import { gzipSync } from "node:zlib";
@@ -47,11 +48,29 @@ const sourceFiles = (map, entry) => {
 };
 //#endregion
 //#region src/evals/eval-bundle.ts
-const authoringExports = ["export const defineEval = value => value;", `export { empty, files, repo } from "./source";`].join("\n");
+const authoringExports = ["export const defineEval = (first, second) => typeof first === 'string' ? second : first;", `export { empty, files, repo } from "./source";`].join("\n");
 const authoringDir = dirname(fileURLToPath(import.meta.url));
 const ANPORD_MODULE = /^anpord$/;
 const ANY_MODULE = /.*/;
 const localModules = [
+	{
+		built: "api.mjs",
+		filter: /^anpord\/api$/,
+		namespace: "anpord-api-authoring",
+		source: "../mock-api/index.ts"
+	},
+	{
+		built: "api-runtime.mjs",
+		filter: /^anpord\/api\/runtime$/,
+		namespace: "anpord-api-runtime",
+		source: "../mock-api/runtime.ts"
+	},
+	{
+		built: "api-context.mjs",
+		filter: /^anpord\/api\/context$/,
+		namespace: "anpord-api-context",
+		source: "../mock-api/context.ts"
+	},
 	{
 		built: "validator-runtime.mjs",
 		filter: /^anpord\/validators\/runtime$/,
@@ -128,6 +147,7 @@ const bundle = (contents, entry, options = {}) => Effect.tryPromise({
 	try: () => build({
 		absWorkingDir: process.cwd(),
 		bundle: true,
+		banner: { js: "import { createRequire as anpordCreateRequire } from \"node:module\"; const require = anpordCreateRequire(import.meta.url);" },
 		format: "esm",
 		metafile: true,
 		minify: options.minify,
@@ -158,36 +178,6 @@ const bundle = (contents, entry, options = {}) => Effect.tryPromise({
 	catch: (cause) => new Error(`Could not compile ${entry}`, { cause })
 });
 //#endregion
-//#region src/evals/case-modules.ts
-const realOrGiven = (path) => {
-	try {
-		return realpathSync.native(path);
-	} catch {
-		return path;
-	}
-};
-const sameFile = (one, other) => one === other || realOrGiven(one) === realOrGiven(other);
-const exportingModule = (entry, inputs, name) => Effect.gen(function* () {
-	const pattern = new RegExp(`\\bexport\\s+(?:async\\s+)?(?:const|function)\\s+${name}\\b`);
-	const matches = yield* Effect.filter(inputs.filter((path) => !(sameFile(path, entry) || path.includes(`${sep}node_modules${sep}`))), (path) => Effect.tryPromise(() => readFile(path, "utf8")).pipe(Effect.map((source) => pattern.test(source)), Effect.orElseSucceed(() => false)), { concurrency: 8 });
-	if (matches.length !== 1) return yield* Effect.fail(/* @__PURE__ */ new Error(`${name} must be one named function or const export from a separate TypeScript file`));
-	return matches[0];
-});
-const bundledCaseModule = (entry, inputs, name, wrap) => Effect.gen(function* () {
-	const module = yield* exportingModule(entry, inputs, name);
-	const { source } = yield* bundle(wrap(module, name), entry);
-	return {
-		name,
-		source
-	};
-});
-//#endregion
-//#region src/evals/profile-compose.ts
-const appendInstall = (profile, command) => ({
-	...profile,
-	install: profile.install ? `${profile.install} && ${command}` : command
-});
-//#endregion
 //#region src/evals/program-package.ts
 const CHUNK_SIZE = 9e4;
 const packageProgram = (root, entry, source) => {
@@ -197,18 +187,20 @@ const packageProgram = (root, entry, source) => {
 	const loader = `#!/usr/bin/env node\nimport{readFile,writeFile}from"node:fs/promises";import{gunzipSync}from"node:zlib";const root=new URL("./",import.meta.url),names=${JSON.stringify(names)},data=(await Promise.all(names.map(name=>readFile(new URL(name,root),"utf8")))).join(""),runtime=new URL("runtime.mjs",root);await writeFile(runtime,gunzipSync(Buffer.from(data,"base64")));await import(runtime.href);`;
 	return {
 		entry: `${root}/${entry}`,
-		files: Object.fromEntries([[`${root}/${entry}`, loader], ...names.map((name, index) => [`${root}/${name}`, chunks[index]])])
+		files: Object.fromEntries([[`${root}/${entry}`, loader], ...chunks.map((chunk, index) => [`${root}/${index}.txt`, chunk])])
 	};
 };
 //#endregion
 //#region src/evals/runner-source.ts
-const validatorCaseEntry = (entry, index, names = []) => `import definition from ${JSON.stringify(entry)};
+const importsDefinition = ({ entry, exportName }) => exportName === null ? `import definition from ${JSON.stringify(entry)};` : `import { ${exportName} as definition } from ${JSON.stringify(entry)};`;
+const validatorCaseEntry = (ref, index, names = []) => `${importsDefinition(ref)}
 import { runValidators } from "anpord/validators/runtime";
 const selected = definition.cases[${index}].validate;
 const names = ${JSON.stringify(names)};
 const checks = (Array.isArray(selected) ? selected : [selected]).flatMap((validate, index) => typeof validate === "function" ? [{ index, name: names.find(check => check.index === index)?.name ?? "Validator " + (index + 1), validate }] : []);
 await runValidators(checks, definition.captureValidation !== false);`;
 const prepareRuntime = `
+import { apiContext } from "anpord/api/context";
 import { access, readFile } from "node:fs/promises";
 import { spawn } from "node:child_process";
 
@@ -236,6 +228,7 @@ const exec = (file, args = [], options = {}) => new Promise((resolve, reject) =>
 });
 
 const context = {
+  api: apiContext,
   cached: process.env.ANPORD_CACHE_RESTORED === "1",
   exec,
   exists: path => access(path).then(() => true, () => false),
@@ -252,25 +245,88 @@ try {
 }
 `;
 const prepareEntry = (module, name) => `import { ${name} as setup } from ${JSON.stringify(module)};\n${prepareRuntime}`;
-const definitionEntry = (entry) => `import definition from ${JSON.stringify(entry)}; export default definition;`;
-const mcpEntry = (entry, index) => `import definition from ${JSON.stringify(entry)};
+const definitionEntry = (ref) => `${importsDefinition(ref)} export default definition;`;
+const mcpEntry = (ref, index) => `${importsDefinition(ref)}
 import { runMcpServer } from "anpord/mcp/runtime";
 const server = definition.mcp?.[${index}];
 if (server === undefined) throw new Error("MCP server ${index} is missing");
 runMcpServer(server);`;
-const cliEntry = (entry, index) => `import { fileURLToPath } from "node:url";
-import definition from ${JSON.stringify(entry)};
+const cliEntry = (ref, index) => `import { fileURLToPath } from "node:url";
+${importsDefinition(ref)}
 import { runCli } from "anpord/cli/runtime";
 const cli = definition.cli?.[${index}];
 if (cli === undefined) throw new Error("CLI ${index} is missing");
 await runCli(cli, fileURLToPath(new URL("../../cli-calls.jsonl", import.meta.url)));`;
 //#endregion
+//#region src/evals/api-profile.ts
+const compileApis = (ref, definitions) => Effect.gen(function* () {
+	if (definitions.length === 0) return {};
+	const names = definitions.map(({ name }) => name);
+	if (new Set(names).size !== names.length) return yield* Effect.fail(/* @__PURE__ */ new Error("Duplicate API mock name"));
+	const { source } = yield* bundle(`${importsDefinition(ref)}
+import { runApiServers } from "anpord/api/runtime";
+runApiServers(definition.api ?? []);`, ref.entry, { minify: true });
+	const program = packageProgram("workspace/.anpord/api", "server.mjs", source);
+	return {
+		...program.files,
+		[API_PROGRAM]: JSON.stringify({ entry: program.entry.slice(10) })
+	};
+});
+const withApis = (task, files) => {
+	if (Object.keys(files).length === 0) return task;
+	const profile = task.profile ?? {
+		name: "anpord-api",
+		files: {}
+	};
+	for (const path of Object.keys(files)) if (profile.files[path] !== void 0) throw new Error(`Profile file ${path} is reserved for API mocks`);
+	return {
+		...task,
+		profile: {
+			...profile,
+			files: {
+				...profile.files,
+				...files
+			}
+		}
+	};
+};
+//#endregion
+//#region src/evals/case-modules.ts
+const realOrGiven = (path) => {
+	try {
+		return realpathSync.native(path);
+	} catch {
+		return path;
+	}
+};
+const sameFile = (one, other) => one === other || realOrGiven(one) === realOrGiven(other);
+const exportingModule = (entry, inputs, name) => Effect.gen(function* () {
+	const pattern = new RegExp(`\\bexport\\s+(?:async\\s+)?(?:const|function)\\s+${name}\\b`);
+	const matches = yield* Effect.filter(inputs.filter((path) => !(sameFile(path, entry) || path.includes(`${sep}node_modules${sep}`))), (path) => Effect.tryPromise(() => readFile(path, "utf8")).pipe(Effect.map((source) => pattern.test(source)), Effect.orElseSucceed(() => false)), { concurrency: 8 });
+	if (matches.length !== 1) return yield* Effect.fail(/* @__PURE__ */ new Error(`${name} must be one named function or const export from a separate TypeScript file`));
+	return matches[0];
+});
+const bundledCaseModule = (entry, inputs, name, wrap) => Effect.gen(function* () {
+	const module = yield* exportingModule(entry, inputs, name);
+	const { source } = yield* bundle(wrap(module, name), entry, { minify: true });
+	return {
+		name,
+		source
+	};
+});
+//#endregion
+//#region src/evals/profile-compose.ts
+const appendInstall = (profile, command) => ({
+	...profile,
+	install: profile.install ? `${profile.install} && ${command}` : command
+});
+//#endregion
 //#region src/evals/cli-profile.ts
-const compileClis = (entry, definitions) => Effect.gen(function* () {
+const compileClis = (ref, definitions) => Effect.gen(function* () {
 	const paths = definitions.map(({ path }) => path);
 	const duplicate = paths.find((path, index) => paths.indexOf(path) !== index);
 	if (duplicate !== void 0) return yield* Effect.fail(/* @__PURE__ */ new Error(`Duplicate CLI: ${duplicate}`));
-	return yield* Effect.forEach(definitions, (definition, index) => bundle(cliEntry(entry, index), entry, { minify: true }).pipe(Effect.map(({ source }) => ({
+	return yield* Effect.forEach(definitions, (definition, index) => bundle(cliEntry(ref, index), ref.entry, { minify: true }).pipe(Effect.map(({ source }) => ({
 		...packageProgram(`workspace/.anpord/cli/${index}`, "cli.mjs", source),
 		path: definition.path
 	}))), { concurrency: 4 });
@@ -300,7 +356,7 @@ const withClis = (task, clis) => {
 };
 //#endregion
 //#region src/evals/compile-validator.ts
-const compileValidator = (entry, subject, caseIndex, captureSource, captureValidation = true) => Effect.gen(function* () {
+const compileValidator = (ref, subject, caseIndex, captureSource, captureValidation = true) => Effect.gen(function* () {
 	if (subject.validate === void 0) return null;
 	const validations = Array.isArray(subject.validate) ? subject.validate : [subject.validate];
 	if (validations.length === 0 || validations.length > 20) return yield* Effect.fail(/* @__PURE__ */ new Error("Use between 1 and 20 validators"));
@@ -311,7 +367,7 @@ const compileValidator = (entry, subject, caseIndex, captureSource, captureValid
 		name: value.name || `Validator ${index + 1}`
 	}] : []);
 	const hasCode = manifest.length > 0;
-	const compiled = yield* bundle(hasCode ? validatorCaseEntry(entry, caseIndex, manifest) : definitionEntry(entry), entry, {
+	const compiled = yield* bundle(hasCode ? validatorCaseEntry(ref, caseIndex, manifest) : definitionEntry(ref), ref.entry, {
 		minify: true,
 		captureSource
 	});
@@ -334,7 +390,7 @@ const compileValidator = (entry, subject, caseIndex, captureSource, captureValid
 });
 //#endregion
 //#region src/evals/definition-loader.ts
-const loadDefinition = (entry) => bundle(definitionEntry(entry), entry).pipe(Effect.flatMap(({ inputs, source }) => Effect.acquireUseRelease(Effect.tryPromise(() => mkdtemp(join(tmpdir(), "anpord-eval-"))), (directory) => Effect.tryPromise({
+const loadDefinition = (ref) => bundle(definitionEntry(ref), ref.entry).pipe(Effect.flatMap(({ inputs, source }) => Effect.acquireUseRelease(Effect.tryPromise(() => mkdtemp(join(tmpdir(), "anpord-eval-"))), (directory) => Effect.tryPromise({
 	try: async () => {
 		const output = join(directory, "definition.mjs");
 		await writeFile(output, source);
@@ -343,7 +399,7 @@ const loadDefinition = (entry) => bundle(definitionEntry(entry), entry).pipe(Eff
 			inputs
 		};
 	},
-	catch: (cause) => new Error(`Could not load ${entry}: ${cause instanceof Error ? cause.message : String(cause)}`, { cause })
+	catch: (cause) => new Error(`Could not load ${ref.entry}: ${cause instanceof Error ? cause.message : String(cause)}`, { cause })
 }), (directory) => Effect.promise(() => rm(directory, {
 	force: true,
 	recursive: true
@@ -391,6 +447,39 @@ const localRepo = (cwd) => Effect.gen(function* () {
 	});
 }).pipe(Effect.withSpan("Eval.localRepo"));
 //#endregion
+//#region src/evals/define.ts
+const SOURCE_URL = Symbol.for("anpord.sourceUrl");
+function defineEval(first, second) {
+	if (typeof first !== "string") return first;
+	return Object.defineProperty(second, SOURCE_URL, {
+		enumerable: false,
+		value: first
+	});
+}
+const sourceUrlOf = (definition) => definition[SOURCE_URL];
+//#endregion
+//#region src/evals/locate.ts
+const HINT = "Pass import.meta.url as the first argument to defineEval, or compile by path.";
+const exportHolding = async (entry, definition) => {
+	const module = await import(entry);
+	if (module.default === definition) return null;
+	const named = Object.keys(module).find((key) => module[key] === definition);
+	if (named === void 0) throw new Error(`The definition passed is not exported from ${entry}. ${HINT}`);
+	return named;
+};
+const locate = (definition) => Effect.gen(function* () {
+	const url = sourceUrlOf(definition);
+	if (url === void 0) return yield* Effect.fail(/* @__PURE__ */ new Error(`This eval does not know where it was written. ${HINT}`));
+	const exportName = yield* Effect.tryPromise({
+		catch: (cause) => cause instanceof Error ? cause : new Error(String(cause)),
+		try: () => exportHolding(url, definition)
+	});
+	return {
+		entry: fileURLToPath(url),
+		exportName
+	};
+});
+//#endregion
 //#region src/evals/mcp-harness.ts
 const Entries = Schema.Record({
 	key: Schema.String,
@@ -491,11 +580,11 @@ const applyMcpHarness = (harness, profile, servers) => {
 };
 //#endregion
 //#region src/evals/mcp-profile.ts
-const compileMcpServers = (entry, definitions) => Effect.gen(function* () {
+const compileMcpServers = (ref, definitions) => Effect.gen(function* () {
 	const names = definitions.map(({ name }) => name);
 	const duplicate = names.find((name, index) => names.indexOf(name) !== index);
 	if (duplicate !== void 0) return yield* Effect.fail(/* @__PURE__ */ new Error(`Duplicate MCP server: ${duplicate}`));
-	return yield* Effect.forEach(definitions, (definition, index) => bundle(mcpEntry(entry, index), entry, { minify: true }).pipe(Effect.map(({ source }) => ({
+	return yield* Effect.forEach(definitions, (definition, index) => bundle(mcpEntry(ref, index), ref.entry, { minify: true }).pipe(Effect.map(({ source }) => ({
 		...packageProgram(`workspace/.anpord/mcp/${index}`, "server.mjs", source),
 		name: definition.name
 	}))), { concurrency: 4 });
@@ -708,7 +797,7 @@ const profileTask = (entry, task) => Effect.gen(function* () {
 }).pipe(Effect.withSpan("Eval.profileTask", { attributes: { profile: task.harness.profile.name } }));
 //#endregion
 //#region src/evals/compiler.ts
-const taskOf = (entry, task, clis, mcp) => Effect.gen(function* () {
+const taskOf = (entry, task, clis, mcp, apis) => Effect.gen(function* () {
 	const compiled = typeof task.harness === "string" ? {
 		harness: task.harness,
 		model: task.model,
@@ -717,7 +806,7 @@ const taskOf = (entry, task, clis, mcp) => Effect.gen(function* () {
 		...task,
 		harness: task.harness
 	});
-	return yield* Effect.try(() => withClis(withMcpServers(compiled, mcp), clis));
+	return yield* Effect.try(() => withApis(withClis(withMcpServers(compiled, mcp), clis), apis));
 });
 const sourceFor = (definition, subject, fallback) => {
 	const source = subject.source ?? definition.source;
@@ -727,17 +816,21 @@ const sourceFor = (definition, subject, fallback) => {
 		onSome: (source) => ({ source })
 	});
 };
-const compileEvalEffect = (path) => Effect.gen(function* () {
-	const entry = resolve(path);
-	const loaded = yield* loadDefinition(entry);
+const compileRefEffect = (ref) => Effect.gen(function* () {
+	const entry = ref.entry;
+	const loaded = yield* loadDefinition(ref);
 	const definition = loaded.definition;
-	if (!isDefinition(definition)) return yield* Effect.fail(/* @__PURE__ */ new Error(`${entry} must default export defineEval({ ... })`));
-	const clis = yield* compileClis(entry, definition.cli ?? []);
-	const mcp = yield* compileMcpServers(entry, definition.mcp ?? []);
+	if (!isDefinition(definition)) {
+		const named = ref.exportName === null ? "default export" : `export ${ref.exportName}`;
+		return yield* Effect.fail(/* @__PURE__ */ new Error(`${entry} must ${named} defineEval({ ... })`));
+	}
+	const clis = yield* compileClis(ref, definition.cli ?? []);
+	const mcp = yield* compileMcpServers(ref, definition.mcp ?? []);
+	const apis = yield* compileApis(ref, definition.api ?? []);
 	const fallback = definition.source === void 0 && definition.cases.some((subject) => subject.source === void 0) ? yield* localRepo(dirname(entry)) : Option.none();
 	const cases = yield* Effect.forEach(definition.cases, (subject, caseIndex) => Effect.gen(function* () {
 		if (subject.validate !== void 0 === (typeof subject.verify === "string")) return yield* Effect.fail(/* @__PURE__ */ new Error(`${subject.name} must have exactly one of validate or verify`));
-		const validator = yield* compileValidator(entry, subject, caseIndex, definition.captureSource !== false, definition.captureValidation !== false);
+		const validator = yield* compileValidator(ref, subject, caseIndex, definition.captureSource !== false, definition.captureValidation !== false);
 		const prepare = typeof subject.prepare === "function" ? yield* bundledCaseModule(entry, loaded.inputs, subject.prepare.name || `${subject.name}-prepare`, prepareEntry) : null;
 		return {
 			...subject.cache === void 0 ? {} : { cache: subject.cache },
@@ -749,15 +842,22 @@ const compileEvalEffect = (path) => Effect.gen(function* () {
 			verify: subject.verify ?? null
 		};
 	}), { concurrency: 4 });
-	const tasks = yield* Effect.forEach(definition.tasks, (task) => taskOf(entry, task, clis, mcp), { concurrency: 4 });
-	return {
+	const tasks = yield* Effect.forEach(definition.tasks, (task) => taskOf(entry, task, clis, mcp, apis), { concurrency: 4 });
+	return yield* Schema.decodeUnknown(PublicStartEvalRequest)({
 		cases,
 		name: definition.name,
 		prompt: definition.prompt,
 		tasks,
 		trials: definition.trials
-	};
+	});
 }).pipe(Effect.withSpan("Eval.compile"));
+const compileDefinitionEffect = (definition) => Effect.flatMap(locate(definition), compileRefEffect);
+const compileDefinition = (definition) => Effect.runPromise(compileDefinitionEffect(definition));
+const refOfPath = (path) => ({
+	entry: resolve(path),
+	exportName: null
+});
+const compileEvalEffect = (path) => compileRefEffect(refOfPath(path));
 const compileEval = (path) => Effect.runPromise(compileEvalEffect(path));
 //#endregion
-export { compileEvalEffect as n, compileEval as t };
+export { compileRefEffect as a, compileEvalEffect as i, compileDefinitionEffect as n, defineEval as o, compileEval as r, sourceUrlOf as s, compileDefinition as t };