anpord 0.1.2 → 0.1.4

package/dist/cli.mjs

@@ -1,20 +1,200 @@
 #!/usr/bin/env node
-import { a as ChannelName, o as PromptId, r as layer, s as VersionNumber, t as AnpordApi } from "./client-bJBNwTKq.mjs";
+import { a as ChannelName, o as PromptId, r as layer, s as VersionNumber, t as AnpordApi } from "./client-BfhmGRQl.mjs";
 import { clientOptionsConfig } from "./config.mjs";
-import { r as asAnpordError } from "./errors-DEfS6oaQ.mjs";
+import { i as tokenMatcher, r as asAnpordError } from "./errors-C9bQUcA3.mjs";
+import { n as compileEvalEffect } from "./compiler-CKiewwUc.mjs";
 import { FileSystem } from "@effect/platform";
-import { Cause, Console, Effect, Layer, Option } from "effect";
+import { Cause, Clock, Console, Data, Duration, Effect, Layer, Option, Ref } from "effect";
 import { Args, Command, Options } from "@effect/cli";
 import { NodeContext, NodeRuntime } from "@effect/platform-node";
+//#region package.json
+var version$1 = "0.1.4";
+//#endregion
+//#region ../template/src/extract.ts
+/**
+* Distinct variable names in source order, so callers can count or list them.
+*
+* Escaped braces are read here too, so a name the renderer will treat as
+* literal text is never reported as a variable the editor should draw.
+*/
+function extractVariables(template) {
+	const names = [];
+	for (const [, open, close, name] of template.matchAll(tokenMatcher())) if (open === void 0 && close === void 0 && name !== void 0) names.push(name);
+	return [...new Set(names)];
+}
+//#endregion
+//#region src/cli/declarations.ts
+/**
+* The generated file has to open by importing the module it augments.
+*
+* Without it `declare module` shadows the module rather than adding to it, and
+* every import of the SDK stops resolving with an error naming the missing
+* export rather than the file that hid it.
+*/
+const PREAMBLE = [
+	"// Generated by `anpord generate`. Do not edit.",
+	"// Run it again after changing which variables a prompt uses.",
+	"import \"anpord\";",
+	"",
+	"declare module \"anpord\" {",
+	"  interface AnpordPromptVariables {"
+].join("\n");
+/** Quoted, because an id may carry characters an identifier cannot. */
+const entry = (id, names) => {
+	return `    "${id}": ${names.length === 0 ? "Record<string, never>" : `{ ${names.map((name) => `"${name}": string`).join("; ")} }`};`;
+};
+const declarationFile = (prompts) => [
+	PREAMBLE,
+	...[...prompts].sort(([left], [right]) => left.localeCompare(right)).map(([id, names]) => entry(id, names)),
+	"  }",
+	"}",
+	""
+].join("\n");
+//#endregion
+//#region src/cli/eval-files.ts
+const SUFFIX = ".eval.ts";
+const SKIPPED = /* @__PURE__ */ new Set([
+	"node_modules",
+	"dist",
+	"build",
+	".git"
+]);
+const hidden = (segment) => segment.startsWith(".");
+const wanted = (path) => {
+	const segments = path.split("/");
+	return path.endsWith(SUFFIX) && segments.every((segment) => !(SKIPPED.has(segment) || hidden(segment)));
+};
+const evalFilesIn = (directory) => Effect.gen(function* () {
+	return (yield* (yield* FileSystem.FileSystem).readDirectory(directory, { recursive: true })).filter(wanted).sort();
+}).pipe(Effect.withSpan("Cli.evalFilesIn"));
+//#endregion
+//#region src/cli/eval-gate.ts
+const regressions = (run) => run.cells.filter((cell) => cell.comparison?.verdict === "regressed");
+const unscored = (run) => run.cells.filter((cell) => (cell.distribution?.scored ?? 0) === 0);
+const problemsWith = (run, failOn) => {
+	if (run.status === "failed") return [run.failure ?? "The run failed."];
+	if (failOn === "never") return [];
+	const found = regressions(run).map((cell) => `${cell.caseName} regressed against its baseline.`);
+	return failOn === "unscored" ? [...found, ...unscored(run).map((cell) => `${cell.caseName} produced no scored trials.`)] : found;
+};
+const failWhen = (problems) => problems.length === 0 ? Effect.void : Effect.fail(new EvalGateFailed({ problems }));
+var EvalGateFailed = class extends Data.TaggedError("EvalGateFailed") {
+	get message() {
+		return this.problems.join("\n");
+	}
+};
+var NoEvalFiles = class extends Data.TaggedError("NoEvalFiles") {
+	get message() {
+		return "No *.eval.ts file here. Name one, or pass a file to run.";
+	}
+};
+//#endregion
 //#region src/cli/render.ts
 const json = (value) => Console.log(JSON.stringify(value, null, 2));
 const promptContent = (prompt) => Effect.sync(() => {
 	process.stdout.write(prompt.content);
 	if (!prompt.content.endsWith("\n")) process.stdout.write("\n");
 });
+/** A result the caller may pipe into another tool, so it belongs on stdout
+* alongside the prompt content rather than beside the status messages. */
+const row = (line) => Effect.sync(() => {
+	process.stdout.write(`${line}\n`);
+});
 const note = (message) => Effect.sync(() => {
 	process.stderr.write(`${message}\n`);
 });
+const attended = Effect.sync(() => globalThis.process?.stdout?.isTTY === true);
+//#endregion
+//#region src/cli/eval-grid.ts
+const DIM = "\x1B[2m";
+const BOLD = "\x1B[1m";
+const GREEN = "\x1B[32m";
+const RED = "\x1B[31m";
+const YELLOW = "\x1B[33m";
+const RESET = "\x1B[0m";
+const DONE = "●";
+const RUNNING = "◐";
+const FILLED = "▰";
+const HOLLOW = "▱";
+const PERCENT = 100;
+const SECONDS = 1e3;
+const MINUTE = 60;
+const paint = (colour, text) => `${colour}${text}${RESET}`;
+const elapsedOf = (ms) => {
+	const total = Math.floor(ms / SECONDS);
+	const minutes = Math.floor(total / MINUTE);
+	return minutes === 0 ? `${total}s` : `${minutes}m${String(total % MINUTE).padStart(2, "0")}s`;
+};
+const markOf = (cell) => {
+	if (cell.status === "finished") return paint(GREEN, DONE);
+	return cell.status === "failed" ? paint(RED, DONE) : paint(YELLOW, RUNNING);
+};
+const trialsOf = (cell, trials) => {
+	const settled = cell.trials.filter((trial) => trial.status !== "queued" && trial.status !== "running").length;
+	return `${FILLED.repeat(settled)}${paint(DIM, HOLLOW.repeat(Math.max(0, trials - settled)))}`;
+};
+const rateOf = (cell) => {
+	const rate = cell.distribution?.passRate;
+	if (rate === void 0 || cell.distribution?.scored === 0) return paint(DIM, "—");
+	const shown = `${Math.round(rate * PERCENT)}%`;
+	return paint(rate === 1 ? GREEN : RED, shown);
+};
+const variantOf = (run, cell) => {
+	const task = run.tasks[cell.taskIndex];
+	return task === void 0 ? "?" : `${task.harness}/${task.model}`;
+};
+const widest = (run) => run.cells.reduce((width, cell) => Math.max(width, variantOf(run, cell).length), 0);
+const gridOf = (run, trials, elapsedMs) => {
+	const width = widest(run);
+	const lines = [];
+	for (const caseName of run.cases) {
+		lines.push(`  ${BOLD}${caseName}${RESET}`);
+		for (const cell of run.cells.filter((one) => one.caseName === caseName)) lines.push(`    ${markOf(cell)} ${variantOf(run, cell).padEnd(width)}  ${trialsOf(cell, trials)}  ${rateOf(cell)}`);
+		lines.push("");
+	}
+	lines.push(paint(DIM, `  ${elapsedOf(elapsedMs)} elapsed`));
+	return lines;
+};
+const up = (rows) => `[${rows}A`;
+const liveGrid = (trials, interactive) => Effect.gen(function* () {
+	const drawn = yield* Ref.make(0);
+	return (run, elapsedMs) => Effect.gen(function* () {
+		if (!interactive) return;
+		const rows = yield* Ref.getAndSet(drawn, 0);
+		const lines = gridOf(run, trials, elapsedMs);
+		yield* note(`${rows === 0 ? "" : up(rows)}${lines.join("\n")}`);
+		yield* Ref.set(drawn, lines.length);
+	});
+});
+const summaryOf = (run, trials, drawn) => drawn ? "" : gridOf(run, trials, 0).join("\n");
+//#endregion
+//#region src/cli/eval-run.ts
+const FIRST_POLL = 2e3;
+const SLOWEST_POLL = 1e4;
+const WIDENING = 1.5;
+const running = ({ run }) => run === null || run.status === "running";
+const widen = (gap) => Math.min(Math.round(gap * WIDENING), SLOWEST_POLL);
+const waitForRun = (id, onProgress) => Effect.gen(function* () {
+	const api = yield* AnpordApi;
+	const startedAt = yield* Clock.currentTimeMillis;
+	const step = ({ gap, run }) => Effect.gen(function* () {
+		if (run !== null) yield* Effect.sleep(Duration.millis(gap));
+		const next = yield* api.evals.get({ payload: { id } });
+		yield* onProgress(next, (yield* Clock.currentTimeMillis) - startedAt);
+		return {
+			gap: widen(gap),
+			run: next
+		};
+	});
+	const { run } = yield* Effect.iterate({
+		gap: FIRST_POLL,
+		run: null
+	}, {
+		body: step,
+		while: running
+	});
+	return run ?? (yield* api.evals.get({ payload: { id } }));
+}).pipe(Effect.withSpan("Cli.waitForRun", { attributes: { runId: id } }));
 //#endregion
 //#region src/cli/commands.ts
 const promptId = Args.text({ name: "id" }).pipe(Args.withDescription("The prompt's id, such as support-reply"), Args.withSchema(PromptId));
@@ -38,7 +218,7 @@ const get = Command.make("get", {
 const list = Command.make("list", { asJson }, ({ asJson: wantsJson }) => Effect.gen(function* () {
 	const { data } = yield* (yield* AnpordApi).prompts.list({ payload: {} });
 	if (wantsJson) return yield* json(data);
-	return yield* Effect.forEach(data, (row) => note(`${row.id}\tv${row.latestVersion ?? "-"}\t${row.name}`));
+	return yield* Effect.forEach(data, (summary) => row(`${summary.id}\tv${summary.latestVersion ?? "-"}\t${summary.name}`));
 })).pipe(Command.withDescription("List every prompt"));
 const versions = Command.make("versions", { promptId }, ({ promptId: id }) => Effect.gen(function* () {
 	const prompt = yield* (yield* AnpordApi).prompts.get({ payload: {
@@ -59,25 +239,100 @@ const promote = Command.make("promote", {
 	} });
 	return yield* note(`${id} v${pin} is now ${to}`);
 })).pipe(Command.withDescription("Point a channel at a version"));
-const readStdin = Effect.flatMap(FileSystem.FileSystem, (fs) => fs.readFileString("/dev/stdin"));
+/**
+* Read from the stream rather than by opening `/dev/stdin` as a file. The path
+* only names the pipe, so reading it races whoever is writing: a body arriving
+* in more than one chunk, which is what a pipe does under load, was read as
+* whatever had landed by then.
+*/
+const readStdin = Effect.async((resume) => {
+	let body = "";
+	process.stdin.setEncoding("utf8");
+	process.stdin.on("data", (chunk) => {
+		body += chunk;
+	});
+	process.stdin.on("end", () => resume(Effect.succeed(body)));
+	process.stdin.on("error", (cause) => resume(Effect.fail(new Error("Could not read the content from stdin", { cause }))));
+});
 const target = Args.all([promptId, Args.text({ name: "content" }).pipe(Args.withDescription("The new content, or - to read stdin"))]);
+const push = Command.make("push", {
+	message,
+	target
+}, ({ message: why, target: [id, content] }) => Effect.gen(function* () {
+	const api = yield* AnpordApi;
+	const body = content === "-" ? yield* readStdin : content;
+	const prompt = yield* api.prompts.update({ payload: {
+		content: body,
+		id,
+		message: Option.getOrUndefined(why)
+	} });
+	return yield* note(`${id} is now v${prompt.version}`);
+})).pipe(Command.withDescription("Add a version to a prompt"));
+const out = Options.file("out").pipe(Options.withDescription("Where to write the declarations"), Options.withDefault("anpord-env.d.ts"));
+/** Reading one prompt at a time because the list carries no content, bounded
+* so a large organisation does not open a connection per prompt. */
+const READ_AT_ONCE = 8;
+const writeDeclarations = ({ out: path }) => Effect.gen(function* () {
+	const api = yield* AnpordApi;
+	const fs = yield* FileSystem.FileSystem;
+	const { data } = yield* api.prompts.list({ payload: {} });
+	const prompts = yield* Effect.forEach(data, (summary) => api.prompts.get({ payload: { id: summary.id } }).pipe(Effect.map((prompt) => [prompt.id, extractVariables(prompt.content)])), { concurrency: READ_AT_ONCE });
+	yield* fs.writeFileString(path, declarationFile(prompts));
+	return yield* note(`Wrote ${prompts.length} ${prompts.length === 1 ? "prompt" : "prompts"} to ${path}`);
+});
+const DESCRIPTION = "Write TypeScript declarations for prompt variables";
+const generate = Command.make("generate", { out }, writeDeclarations).pipe(Command.withDescription(DESCRIPTION));
+/** A second command rather than an alias, because a command carries one name
+* and the shorter one is what anybody types twice. */
+const gen = Command.make("gen", { out }, writeDeclarations).pipe(Command.withDescription(DESCRIPTION));
+const evalFile = Args.text({ name: "file" }).pipe(Args.withDescription("A TypeScript file that default exports defineEval(...); every *.eval.ts is run when omitted"), Args.optional);
+const noWait = Options.boolean("no-wait").pipe(Options.withDescription("Start the run and print its id, without waiting"));
+const failOn = Options.choice("fail-on", [
+	"never",
+	"regressed",
+	"unscored"
+]).pipe(Options.withDescription("What makes the command exit nonzero"), Options.withDefault("regressed"));
+const runOneEval = (file, options) => Effect.gen(function* () {
+	const api = yield* AnpordApi;
+	const payload = yield* compileEvalEffect(file);
+	const started = yield* api.evals.start({ payload });
+	if (options.skipWait) {
+		yield* json(started);
+		return [];
+	}
+	const live = !options.wantsJson && (yield* attended);
+	if (live || options.label) yield* note(`${file} · run ${started.id}`);
+	const draw = yield* liveGrid(payload.trials, live);
+	const run = yield* waitForRun(started.id, draw);
+	yield* options.wantsJson ? json(run) : note(summaryOf(run, payload.trials, live));
+	return problemsWith(run, options.gate);
+});
 const commands = [
+	Command.make("eval", {
+		asJson,
+		evalFile,
+		failOn,
+		noWait
+	}, ({ asJson: wantsJson, evalFile: file, failOn: gate, noWait: skipWait }) => Effect.gen(function* () {
+		const files = yield* Option.match(file, {
+			onNone: () => evalFilesIn("."),
+			onSome: (one) => Effect.succeed([one])
+		});
+		if (files.length === 0) return yield* Effect.fail(new NoEvalFiles());
+		const found = yield* Effect.forEach(files, (one) => runOneEval(one, {
+			gate,
+			label: files.length > 1,
+			skipWait,
+			wantsJson
+		}));
+		return yield* failWhen(found.flat());
+	})).pipe(Command.withDescription("Compile and run an eval from TypeScript")),
+	gen,
+	generate,
 	get,
 	list,
 	promote,
-	Command.make("push", {
-		message,
-		target
-	}, ({ message: why, target: [id, content] }) => Effect.gen(function* () {
-		const api = yield* AnpordApi;
-		const body = content === "-" ? yield* readStdin : content;
-		const prompt = yield* api.prompts.update({ payload: {
-			content: body,
-			id,
-			message: Option.getOrUndefined(why)
-		} });
-		return yield* note(`${id} is now v${prompt.version}`);
-	})).pipe(Command.withDescription("Add a version to a prompt")),
+	push,
 	versions
 ];
 //#endregion
@@ -94,11 +349,11 @@ const reportFailure = (error) => Effect.sync(() => {
 });
 //#endregion
 //#region src/cli/main.ts
-const anpord = Command.make("anpord").pipe(Command.withDescription("Read and publish prompts from the terminal"), Command.withSubcommands(commands));
+const anpord = Command.make("anpord").pipe(Command.withDescription("Run evals and manage prompts from the terminal"), Command.withSubcommands(commands));
 const ClientLayer = Layer.unwrapEffect(Effect.map(clientOptionsConfig, layer));
 Command.run(anpord, {
 	name: "Anpord",
-	version: "0.1.0"
+	version: version$1
 })(process.argv).pipe(Effect.provide(Layer.mergeAll(ClientLayer, NodeContext.layer)), Effect.catchAllCause((cause) => Cause.isInterruptedOnly(cause) ? Effect.void : reportFailure(Cause.failureOption(cause).pipe(Option.getOrElse(() => Cause.squash(cause))))), NodeRuntime.runMain({ disableErrorReporting: true }));
 //#endregion
 export {};