anpord 0.1.3 → 0.1.5

package/README.md

@@ -1,9 +1,6 @@
 # anpord
 
-TypeScript SDK for [Anpord](https://anpord.com) — customer configuration for AI products.
-
-> **Placeholder release.** This version reserves the package name. There is no
-> client yet — the API surface lands in a future release. Watch this space.
+TypeScript SDK for [Anpord](https://anpord.com). Run coding agent evals across harnesses, models, and sandboxes, and manage versioned prompts from the same client.
 
 ## Install
 
@@ -11,6 +8,74 @@ TypeScript SDK for [Anpord](https://anpord.com) — customer configuration for A
 npm install anpord
 ```
 
+## Run an eval
+
+```ts
+import { Anpord } from "anpord";
+
+const anpord = new Anpord({ apiKey: process.env.ANPORD_API_KEY });
+
+const run = await anpord.evals.startAndWait({
+  cases: [
+    {
+      name: "writes the requested file",
+      variables: { task: "Create hello.txt containing exactly hello" },
+      verify: "test \"$(cat hello.txt)\" = hello",
+    },
+  ],
+  prompt: "{{task}}",
+  tasks: [{ harness: "codex", model: "gpt-5.6-sol", provider: "daytona" }],
+  trials: 3,
+});
+
+for (const cell of run.cells) {
+  console.log(cell.caseName, cell.distribution?.passRate);
+}
+```
+
+For a TypeScript validator, export the function from its own file:
+
+```ts
+import type { Validator } from "anpord";
+
+export const hasGreeting: Validator = async ({ readText }) =>
+  (await readText("hello.txt")) === "hello";
+```
+
+Reference it directly from `anpord.eval.ts`:
+
+```ts
+import { defineEval } from "anpord";
+import { hasGreeting } from "./validators/greeting";
+
+export default defineEval({
+  name: "greeting",
+  cases: [
+    {
+      name: "greeting",
+      variables: { task: "Write hello.txt" },
+      validate: hasGreeting,
+    },
+  ],
+  prompt: "{{task}}",
+  tasks: [{ harness: "codex", model: "gpt-5.6-sol", provider: "daytona" }],
+  trials: 3,
+});
+```
+
+```sh
+npx anpord eval
+```
+
+## Resolve a prompt
+
+```ts
+const prompt = await anpord.prompts.get({ id: "support-reply" });
+console.log(prompt.content, prompt.version);
+```
+
+See the [documentation](https://docs.anpord.com) for eval concepts, prompt releases, SDK methods, and the API reference.
+
 ## License
 
 MIT

package/dist/cli.cjs

@@ -1,20 +1,200 @@
 #!/usr/bin/env node
-const require_client = require("./client-Cno4LwRq.cjs");
+const require_client = require("./client-BCZEcfVE.cjs");
 const require_config = require("./config.cjs");
-const require_errors = require("./errors-C0E5A8fN.cjs");
+const require_errors = require("./errors-Boum34f5.cjs");
+const require_compiler = require("./compiler-CZoX4jkx.cjs");
 let _effect_platform = require("@effect/platform");
 let effect = require("effect");
 let _effect_cli = require("@effect/cli");
 let _effect_platform_node = require("@effect/platform-node");
+//#region package.json
+var version$1 = "0.1.5";
+//#endregion
+//#region ../template/src/extract.ts
+/**
+* Distinct variable names in source order, so callers can count or list them.
+*
+* Escaped braces are read here too, so a name the renderer will treat as
+* literal text is never reported as a variable the editor should draw.
+*/
+function extractVariables(template) {
+	const names = [];
+	for (const [, open, close, name] of template.matchAll(require_errors.tokenMatcher())) if (open === void 0 && close === void 0 && name !== void 0) names.push(name);
+	return [...new Set(names)];
+}
+//#endregion
+//#region src/cli/declarations.ts
+/**
+* The generated file has to open by importing the module it augments.
+*
+* Without it `declare module` shadows the module rather than adding to it, and
+* every import of the SDK stops resolving with an error naming the missing
+* export rather than the file that hid it.
+*/
+const PREAMBLE = [
+	"// Generated by `anpord generate`. Do not edit.",
+	"// Run it again after changing which variables a prompt uses.",
+	"import \"anpord\";",
+	"",
+	"declare module \"anpord\" {",
+	"  interface AnpordPromptVariables {"
+].join("\n");
+/** Quoted, because an id may carry characters an identifier cannot. */
+const entry = (id, names) => {
+	return `    "${id}": ${names.length === 0 ? "Record<string, never>" : `{ ${names.map((name) => `"${name}": string`).join("; ")} }`};`;
+};
+const declarationFile = (prompts) => [
+	PREAMBLE,
+	...[...prompts].sort(([left], [right]) => left.localeCompare(right)).map(([id, names]) => entry(id, names)),
+	"  }",
+	"}",
+	""
+].join("\n");
+//#endregion
+//#region src/cli/eval-files.ts
+const SUFFIX = ".eval.ts";
+const SKIPPED = /* @__PURE__ */ new Set([
+	"node_modules",
+	"dist",
+	"build",
+	".git"
+]);
+const hidden = (segment) => segment.startsWith(".");
+const wanted = (path) => {
+	const segments = path.split("/");
+	return path.endsWith(SUFFIX) && segments.every((segment) => !(SKIPPED.has(segment) || hidden(segment)));
+};
+const evalFilesIn = (directory) => effect.Effect.gen(function* () {
+	return (yield* (yield* _effect_platform.FileSystem.FileSystem).readDirectory(directory, { recursive: true })).filter(wanted).sort();
+}).pipe(effect.Effect.withSpan("Cli.evalFilesIn"));
+//#endregion
+//#region src/cli/eval-gate.ts
+const regressions = (run) => run.cells.filter((cell) => cell.comparison?.verdict === "regressed");
+const unscored = (run) => run.cells.filter((cell) => (cell.distribution?.scored ?? 0) === 0);
+const problemsWith = (run, failOn) => {
+	if (run.status === "failed") return [run.failure ?? "The run failed."];
+	if (failOn === "never") return [];
+	const found = regressions(run).map((cell) => `${cell.caseName} regressed against its baseline.`);
+	return failOn === "unscored" ? [...found, ...unscored(run).map((cell) => `${cell.caseName} produced no scored trials.`)] : found;
+};
+const failWhen = (problems) => problems.length === 0 ? effect.Effect.void : effect.Effect.fail(new EvalGateFailed({ problems }));
+var EvalGateFailed = class extends effect.Data.TaggedError("EvalGateFailed") {
+	get message() {
+		return this.problems.join("\n");
+	}
+};
+var NoEvalFiles = class extends effect.Data.TaggedError("NoEvalFiles") {
+	get message() {
+		return "No *.eval.ts file here. Name one, or pass a file to run.";
+	}
+};
+//#endregion
 //#region src/cli/render.ts
 const json = (value) => effect.Console.log(JSON.stringify(value, null, 2));
 const promptContent = (prompt) => effect.Effect.sync(() => {
 	process.stdout.write(prompt.content);
 	if (!prompt.content.endsWith("\n")) process.stdout.write("\n");
 });
+/** A result the caller may pipe into another tool, so it belongs on stdout
+* alongside the prompt content rather than beside the status messages. */
+const row = (line) => effect.Effect.sync(() => {
+	process.stdout.write(`${line}\n`);
+});
 const note = (message) => effect.Effect.sync(() => {
 	process.stderr.write(`${message}\n`);
 });
+const attended = effect.Effect.sync(() => globalThis.process?.stdout?.isTTY === true);
+//#endregion
+//#region src/cli/eval-grid.ts
+const DIM = "\x1B[2m";
+const BOLD = "\x1B[1m";
+const GREEN = "\x1B[32m";
+const RED = "\x1B[31m";
+const YELLOW = "\x1B[33m";
+const RESET = "\x1B[0m";
+const DONE = "●";
+const RUNNING = "◐";
+const FILLED = "▰";
+const HOLLOW = "▱";
+const PERCENT = 100;
+const SECONDS = 1e3;
+const MINUTE = 60;
+const paint = (colour, text) => `${colour}${text}${RESET}`;
+const elapsedOf = (ms) => {
+	const total = Math.floor(ms / SECONDS);
+	const minutes = Math.floor(total / MINUTE);
+	return minutes === 0 ? `${total}s` : `${minutes}m${String(total % MINUTE).padStart(2, "0")}s`;
+};
+const markOf = (cell) => {
+	if (cell.status === "finished") return paint(GREEN, DONE);
+	return cell.status === "failed" ? paint(RED, DONE) : paint(YELLOW, RUNNING);
+};
+const trialsOf = (cell, trials) => {
+	const settled = cell.trials.filter((trial) => trial.status !== "queued" && trial.status !== "running").length;
+	return `${FILLED.repeat(settled)}${paint(DIM, HOLLOW.repeat(Math.max(0, trials - settled)))}`;
+};
+const rateOf = (cell) => {
+	const rate = cell.distribution?.passRate;
+	if (rate === void 0 || cell.distribution?.scored === 0) return paint(DIM, "—");
+	const shown = `${Math.round(rate * PERCENT)}%`;
+	return paint(rate === 1 ? GREEN : RED, shown);
+};
+const variantOf = (run, cell) => {
+	const task = run.tasks[cell.taskIndex];
+	return task === void 0 ? "?" : `${task.harness}/${task.model}`;
+};
+const widest = (run) => run.cells.reduce((width, cell) => Math.max(width, variantOf(run, cell).length), 0);
+const gridOf = (run, trials, elapsedMs) => {
+	const width = widest(run);
+	const lines = [];
+	for (const caseName of run.cases) {
+		lines.push(`  ${BOLD}${caseName}${RESET}`);
+		for (const cell of run.cells.filter((one) => one.caseName === caseName)) lines.push(`    ${markOf(cell)} ${variantOf(run, cell).padEnd(width)}  ${trialsOf(cell, trials)}  ${rateOf(cell)}`);
+		lines.push("");
+	}
+	lines.push(paint(DIM, `  ${elapsedOf(elapsedMs)} elapsed`));
+	return lines;
+};
+const up = (rows) => `[${rows}A`;
+const liveGrid = (trials, interactive) => effect.Effect.gen(function* () {
+	const drawn = yield* effect.Ref.make(0);
+	return (run, elapsedMs) => effect.Effect.gen(function* () {
+		if (!interactive) return;
+		const rows = yield* effect.Ref.getAndSet(drawn, 0);
+		const lines = gridOf(run, trials, elapsedMs);
+		yield* note(`${rows === 0 ? "" : up(rows)}${lines.join("\n")}`);
+		yield* effect.Ref.set(drawn, lines.length);
+	});
+});
+const summaryOf = (run, trials, drawn) => drawn ? "" : gridOf(run, trials, 0).join("\n");
+//#endregion
+//#region src/cli/eval-run.ts
+const FIRST_POLL = 2e3;
+const SLOWEST_POLL = 1e4;
+const WIDENING = 1.5;
+const running = ({ run }) => run === null || run.status === "running";
+const widen = (gap) => Math.min(Math.round(gap * WIDENING), SLOWEST_POLL);
+const waitForRun = (id, onProgress) => effect.Effect.gen(function* () {
+	const api = yield* require_client.AnpordApi;
+	const startedAt = yield* effect.Clock.currentTimeMillis;
+	const step = ({ gap, run }) => effect.Effect.gen(function* () {
+		if (run !== null) yield* effect.Effect.sleep(effect.Duration.millis(gap));
+		const next = yield* api.evals.get({ payload: { id } });
+		yield* onProgress(next, (yield* effect.Clock.currentTimeMillis) - startedAt);
+		return {
+			gap: widen(gap),
+			run: next
+		};
+	});
+	const { run } = yield* effect.Effect.iterate({
+		gap: FIRST_POLL,
+		run: null
+	}, {
+		body: step,
+		while: running
+	});
+	return run ?? (yield* api.evals.get({ payload: { id } }));
+}).pipe(effect.Effect.withSpan("Cli.waitForRun", { attributes: { runId: id } }));
 //#endregion
 //#region src/cli/commands.ts
 const promptId = _effect_cli.Args.text({ name: "id" }).pipe(_effect_cli.Args.withDescription("The prompt's id, such as support-reply"), _effect_cli.Args.withSchema(require_client.PromptId));
@@ -38,7 +218,7 @@ const get = _effect_cli.Command.make("get", {
 const list = _effect_cli.Command.make("list", { asJson }, ({ asJson: wantsJson }) => effect.Effect.gen(function* () {
 	const { data } = yield* (yield* require_client.AnpordApi).prompts.list({ payload: {} });
 	if (wantsJson) return yield* json(data);
-	return yield* effect.Effect.forEach(data, (row) => note(`${row.id}\tv${row.latestVersion ?? "-"}\t${row.name}`));
+	return yield* effect.Effect.forEach(data, (summary) => row(`${summary.id}\tv${summary.latestVersion ?? "-"}\t${summary.name}`));
 })).pipe(_effect_cli.Command.withDescription("List every prompt"));
 const versions = _effect_cli.Command.make("versions", { promptId }, ({ promptId: id }) => effect.Effect.gen(function* () {
 	const prompt = yield* (yield* require_client.AnpordApi).prompts.get({ payload: {
@@ -59,25 +239,100 @@ const promote = _effect_cli.Command.make("promote", {
 	} });
 	return yield* note(`${id} v${pin} is now ${to}`);
 })).pipe(_effect_cli.Command.withDescription("Point a channel at a version"));
-const readStdin = effect.Effect.flatMap(_effect_platform.FileSystem.FileSystem, (fs) => fs.readFileString("/dev/stdin"));
+/**
+* Read from the stream rather than by opening `/dev/stdin` as a file. The path
+* only names the pipe, so reading it races whoever is writing: a body arriving
+* in more than one chunk, which is what a pipe does under load, was read as
+* whatever had landed by then.
+*/
+const readStdin = effect.Effect.async((resume) => {
+	let body = "";
+	process.stdin.setEncoding("utf8");
+	process.stdin.on("data", (chunk) => {
+		body += chunk;
+	});
+	process.stdin.on("end", () => resume(effect.Effect.succeed(body)));
+	process.stdin.on("error", (cause) => resume(effect.Effect.fail(new Error("Could not read the content from stdin", { cause }))));
+});
 const target = _effect_cli.Args.all([promptId, _effect_cli.Args.text({ name: "content" }).pipe(_effect_cli.Args.withDescription("The new content, or - to read stdin"))]);
+const push = _effect_cli.Command.make("push", {
+	message,
+	target
+}, ({ message: why, target: [id, content] }) => effect.Effect.gen(function* () {
+	const api = yield* require_client.AnpordApi;
+	const body = content === "-" ? yield* readStdin : content;
+	const prompt = yield* api.prompts.update({ payload: {
+		content: body,
+		id,
+		message: effect.Option.getOrUndefined(why)
+	} });
+	return yield* note(`${id} is now v${prompt.version}`);
+})).pipe(_effect_cli.Command.withDescription("Add a version to a prompt"));
+const out = _effect_cli.Options.file("out").pipe(_effect_cli.Options.withDescription("Where to write the declarations"), _effect_cli.Options.withDefault("anpord-env.d.ts"));
+/** Reading one prompt at a time because the list carries no content, bounded
+* so a large organisation does not open a connection per prompt. */
+const READ_AT_ONCE = 8;
+const writeDeclarations = ({ out: path }) => effect.Effect.gen(function* () {
+	const api = yield* require_client.AnpordApi;
+	const fs = yield* _effect_platform.FileSystem.FileSystem;
+	const { data } = yield* api.prompts.list({ payload: {} });
+	const prompts = yield* effect.Effect.forEach(data, (summary) => api.prompts.get({ payload: { id: summary.id } }).pipe(effect.Effect.map((prompt) => [prompt.id, extractVariables(prompt.content)])), { concurrency: READ_AT_ONCE });
+	yield* fs.writeFileString(path, declarationFile(prompts));
+	return yield* note(`Wrote ${prompts.length} ${prompts.length === 1 ? "prompt" : "prompts"} to ${path}`);
+});
+const DESCRIPTION = "Write TypeScript declarations for prompt variables";
+const generate = _effect_cli.Command.make("generate", { out }, writeDeclarations).pipe(_effect_cli.Command.withDescription(DESCRIPTION));
+/** A second command rather than an alias, because a command carries one name
+* and the shorter one is what anybody types twice. */
+const gen = _effect_cli.Command.make("gen", { out }, writeDeclarations).pipe(_effect_cli.Command.withDescription(DESCRIPTION));
+const evalFile = _effect_cli.Args.text({ name: "file" }).pipe(_effect_cli.Args.withDescription("A TypeScript file that default exports defineEval(...); every *.eval.ts is run when omitted"), _effect_cli.Args.optional);
+const noWait = _effect_cli.Options.boolean("no-wait").pipe(_effect_cli.Options.withDescription("Start the run and print its id, without waiting"));
+const failOn = _effect_cli.Options.choice("fail-on", [
+	"never",
+	"regressed",
+	"unscored"
+]).pipe(_effect_cli.Options.withDescription("What makes the command exit nonzero"), _effect_cli.Options.withDefault("regressed"));
+const runOneEval = (file, options) => effect.Effect.gen(function* () {
+	const api = yield* require_client.AnpordApi;
+	const payload = yield* require_compiler.compileEvalEffect(file);
+	const started = yield* api.evals.start({ payload });
+	if (options.skipWait) {
+		yield* json(started);
+		return [];
+	}
+	const live = !options.wantsJson && (yield* attended);
+	if (live || options.label) yield* note(`${file} · run ${started.id}`);
+	const draw = yield* liveGrid(payload.trials, live);
+	const run = yield* waitForRun(started.id, draw);
+	yield* options.wantsJson ? json(run) : note(summaryOf(run, payload.trials, live));
+	return problemsWith(run, options.gate);
+});
 const commands = [
+	_effect_cli.Command.make("eval", {
+		asJson,
+		evalFile,
+		failOn,
+		noWait
+	}, ({ asJson: wantsJson, evalFile: file, failOn: gate, noWait: skipWait }) => effect.Effect.gen(function* () {
+		const files = yield* effect.Option.match(file, {
+			onNone: () => evalFilesIn("."),
+			onSome: (one) => effect.Effect.succeed([one])
+		});
+		if (files.length === 0) return yield* effect.Effect.fail(new NoEvalFiles());
+		const found = yield* effect.Effect.forEach(files, (one) => runOneEval(one, {
+			gate,
+			label: files.length > 1,
+			skipWait,
+			wantsJson
+		}));
+		return yield* failWhen(found.flat());
+	})).pipe(_effect_cli.Command.withDescription("Compile and run an eval from TypeScript")),
+	gen,
+	generate,
 	get,
 	list,
 	promote,
-	_effect_cli.Command.make("push", {
-		message,
-		target
-	}, ({ message: why, target: [id, content] }) => effect.Effect.gen(function* () {
-		const api = yield* require_client.AnpordApi;
-		const body = content === "-" ? yield* readStdin : content;
-		const prompt = yield* api.prompts.update({ payload: {
-			content: body,
-			id,
-			message: effect.Option.getOrUndefined(why)
-		} });
-		return yield* note(`${id} is now v${prompt.version}`);
-	})).pipe(_effect_cli.Command.withDescription("Add a version to a prompt")),
+	push,
 	versions
 ];
 //#endregion
@@ -94,10 +349,10 @@ const reportFailure = (error) => effect.Effect.sync(() => {
 });
 //#endregion
 //#region src/cli/main.ts
-const anpord = _effect_cli.Command.make("anpord").pipe(_effect_cli.Command.withDescription("Read and publish prompts from the terminal"), _effect_cli.Command.withSubcommands(commands));
+const anpord = _effect_cli.Command.make("anpord").pipe(_effect_cli.Command.withDescription("Run evals and manage prompts from the terminal"), _effect_cli.Command.withSubcommands(commands));
 const ClientLayer = effect.Layer.unwrapEffect(effect.Effect.map(require_config.clientOptionsConfig, require_client.layer));
 _effect_cli.Command.run(anpord, {
 	name: "Anpord",
-	version: "0.1.0"
+	version: version$1
 })(process.argv).pipe(effect.Effect.provide(effect.Layer.mergeAll(ClientLayer, _effect_platform_node.NodeContext.layer)), effect.Effect.catchAllCause((cause) => effect.Cause.isInterruptedOnly(cause) ? effect.Effect.void : reportFailure(effect.Cause.failureOption(cause).pipe(effect.Option.getOrElse(() => effect.Cause.squash(cause))))), _effect_platform_node.NodeRuntime.runMain({ disableErrorReporting: true }));
 //#endregion