anpord 0.1.11 → 0.1.13

package/dist/{compiler-Ap8DM-JU.cjs → compiler-uwb1pkdK.cjs}

@@ -1,4 +1,5 @@
-const require_harness_profile = require("./harness-profile-CDQpPLPd.cjs");
+const require_evals = require("./evals-DouMsrsM.cjs");
+const require_eval_judges = require("./eval-judges-BkmiVLLq.cjs");
 const require_source = require("./source-DvlGawEw.cjs");
 let effect = require("effect");
 let node_path = require("node:path");
@@ -11,12 +12,58 @@ let node_os = require("node:os");
 let node_child_process = require("node:child_process");
 let node_util = require("node:util");
 let smol_toml = require("smol-toml");
+//#region src/evals/source-files.ts
+const CODE_FILE = /\.(?:[cm]?[jt]s|[jt]sx)$/;
+const sdkRoot = (0, node_path.resolve)((0, node_path.dirname)((0, node_url.fileURLToPath)(require("url").pathToFileURL(__filename).href)), "..");
+const SourceMap = effect.Schema.parseJson(effect.Schema.Struct({
+	version: effect.Schema.Literal(3),
+	sources: effect.Schema.Array(effect.Schema.String),
+	sourcesContent: effect.Schema.Array(effect.Schema.NullOr(effect.Schema.String))
+}));
+const projectRoot = (entry) => {
+	let directory = (0, node_path.dirname)(entry);
+	while (!(0, node_fs.existsSync)((0, node_path.resolve)(directory, "package.json"))) {
+		const parent = (0, node_path.dirname)(directory);
+		if (parent === directory) return (0, node_path.dirname)(entry);
+		directory = parent;
+	}
+	return directory;
+};
+const sourceFiles = (map, entry) => {
+	const decoded = effect.Schema.decodeUnknownSync(SourceMap)(map);
+	const root = (0, node_fs.realpathSync)(projectRoot(entry));
+	const entryPath = (0, node_path.relative)(root, (0, node_fs.realpathSync)(entry)).split(node_path.sep).join("/");
+	const files = decoded.sources.flatMap((source, index) => {
+		const absolute = (0, node_path.resolve)((0, node_path.dirname)(entry), source);
+		const path = (0, node_path.relative)(root, absolute).split(node_path.sep).join("/");
+		const content = decoded.sourcesContent[index];
+		if (content == null || path.startsWith("../") || path.split("/").includes("node_modules") || absolute.startsWith(`${sdkRoot}${node_path.sep}`) || !CODE_FILE.test(path) || source.includes(":") || (0, node_path.basename)(path) === "anpord-eval-entry.ts") return [];
+		return [{
+			path,
+			content
+		}];
+	});
+	return effect.Schema.decodeUnknownSync(require_evals.EvalSourceFiles)(files.toSorted((a, b) => Number(b.path === entryPath) - Number(a.path === entryPath) || a.path.localeCompare(b.path)));
+};
+//#endregion
 //#region src/evals/eval-bundle.ts
 const authoringExports = ["export const defineEval = value => value;", `export { empty, files, repo } from "./source";`].join("\n");
 const authoringDir = (0, node_path.dirname)((0, node_url.fileURLToPath)(require("url").pathToFileURL(__filename).href));
 const ANPORD_MODULE = /^anpord$/;
 const ANY_MODULE = /.*/;
 const localModules = [
+	{
+		built: "validator-runtime.mjs",
+		filter: /^anpord\/validators\/runtime$/,
+		namespace: "anpord-validator-runtime",
+		source: "validator-runtime.ts"
+	},
+	{
+		built: "validators.mjs",
+		filter: /^anpord\/validators$/,
+		namespace: "anpord-validators",
+		source: "../validators.ts"
+	},
 	{
 		built: "cli.mjs",
 		filter: /^anpord\/cli$/,
@@ -84,6 +131,8 @@ const bundle = (contents, entry, options = {}) => effect.Effect.tryPromise({
 		format: "esm",
 		metafile: true,
 		minify: options.minify,
+		outfile: (0, node_path.resolve)((0, node_path.dirname)(entry), "anpord-bundle.mjs"),
+		sourcemap: options.captureSource ? "external" : false,
 		platform: "node",
 		resolveExtensions: [
 			".ts",
@@ -103,7 +152,8 @@ const bundle = (contents, entry, options = {}) => effect.Effect.tryPromise({
 		write: false
 	}).then((result) => ({
 		inputs: Object.keys(result.metafile?.inputs ?? {}).map((path) => (0, node_path.resolve)(path)),
-		source: result.outputFiles[0]?.text ?? ""
+		source: result.outputFiles.find((file) => file.path.endsWith(".mjs"))?.text ?? "",
+		...options.captureSource ? { sourceFiles: sourceFiles(result.outputFiles.find((file) => file.path.endsWith(".map"))?.text ?? "", entry) } : {}
 	})),
 	catch: (cause) => new Error(`Could not compile ${entry}`, { cause })
 });
@@ -152,51 +202,12 @@ const packageProgram = (root, entry, source) => {
 };
 //#endregion
 //#region src/evals/runner-source.ts
-const runtime = `
-import { access, readFile } from "node:fs/promises";
-import { spawn } from "node:child_process";
-
-const exec = (command) => new Promise((resolve, reject) => {
-  const child = spawn("/bin/sh", ["-lc", command], { cwd: process.cwd() });
-  let stderr = "";
-  let stdout = "";
-  child.stderr.on("data", chunk => { stderr += chunk; });
-  child.stdout.on("data", chunk => { stdout += chunk; });
-  child.on("error", reject);
-  child.on("close", code => resolve({ exitCode: code ?? 1, stderr, stdout }));
-});
-
-const readOrEmpty = (path) =>
-  path === undefined ? Promise.resolve("") : readFile(path, "utf8").then(text => text, () => "");
-
-const calls = async (path, key, value) => {
-  const text = await readOrEmpty(path);
-  const calls = text.trim() === "" ? [] : text.trim().split("\\n").map((line) => JSON.parse(line));
-  return value === undefined ? calls : calls.filter((call) => call[key] === value);
-};
-
-const context = {
-  answer: () => readOrEmpty(process.env.ANPORD_ANSWER_FILE),
-  cli: { calls: cli => calls(".anpord/cli-calls.jsonl", "cli", cli) },
-  exec,
-  exists: path => access(path).then(() => true, () => false),
-  mcp: { calls: server => calls(".anpord/mcp-calls.jsonl", "server", server) },
-  readText: path => readFile(path, "utf8"),
-  prepared: JSON.parse(process.env.ANPORD_PREPARE_VALUE ?? "{}"),
-  transcript: () => readOrEmpty(process.env.ANPORD_TRANSCRIPT_FILE),
-};
-
-try {
-  const raw = await validate(context);
-  const result = typeof raw === "boolean" ? { passed: raw } : raw;
-  if (typeof result?.passed !== "boolean") throw new Error("A validator must return a boolean or { passed, message? }");
-  console.log("ANPORD_VALIDATOR_RESULT=" + JSON.stringify(result));
-} catch (error) {
-  console.error(error instanceof Error ? error.stack : String(error));
-  process.exitCode = 1;
-}
-`;
-const validatorEntry = (module, name) => `import { ${name} as validate } from ${JSON.stringify(module)};\n${runtime}`;
+const validatorCaseEntry = (entry, index, names = []) => `import definition from ${JSON.stringify(entry)};
+import { runValidators } from "anpord/validators/runtime";
+const selected = definition.cases[${index}].validate;
+const names = ${JSON.stringify(names)};
+const checks = (Array.isArray(selected) ? selected : [selected]).flatMap((validate, index) => typeof validate === "function" ? [{ index, name: names.find(check => check.index === index)?.name ?? "Validator " + (index + 1), validate }] : []);
+await runValidators(checks, definition.captureValidation !== false);`;
 const prepareRuntime = `
 import { access, readFile } from "node:fs/promises";
 import { spawn } from "node:child_process";
@@ -247,11 +258,12 @@ import { runMcpServer } from "anpord/mcp/runtime";
 const server = definition.mcp?.[${index}];
 if (server === undefined) throw new Error("MCP server ${index} is missing");
 runMcpServer(server);`;
-const cliEntry = (entry, index) => `import definition from ${JSON.stringify(entry)};
+const cliEntry = (entry, index) => `import { fileURLToPath } from "node:url";
+import definition from ${JSON.stringify(entry)};
 import { runCli } from "anpord/cli/runtime";
 const cli = definition.cli?.[${index}];
 if (cli === undefined) throw new Error("CLI ${index} is missing");
-await runCli(cli);`;
+await runCli(cli, fileURLToPath(new URL("../../cli-calls.jsonl", import.meta.url)));`;
 //#endregion
 //#region src/evals/cli-profile.ts
 const compileClis = (entry, definitions) => effect.Effect.gen(function* () {
@@ -287,6 +299,40 @@ const withClis = (task, clis) => {
 	};
 };
 //#endregion
+//#region src/evals/compile-validator.ts
+const compileValidator = (entry, subject, caseIndex, captureSource, captureValidation = true) => effect.Effect.gen(function* () {
+	if (subject.validate === void 0) return null;
+	const validations = Array.isArray(subject.validate) ? subject.validate : [subject.validate];
+	if (validations.length === 0 || validations.length > 20) return yield* effect.Effect.fail(/* @__PURE__ */ new Error("Use between 1 and 20 validators"));
+	const judges = yield* effect.Effect.forEach(validations.filter((value) => typeof value !== "function"), (value) => effect.Schema.decodeUnknown(require_eval_judges.EvalJudge)(value, { onExcessProperty: "error" }));
+	if (new Set(judges.map(({ name }) => name)).size !== judges.length) return yield* effect.Effect.fail(/* @__PURE__ */ new Error("Judge names must be unique within a case"));
+	const manifest = validations.flatMap((value, index) => typeof value === "function" ? [{
+		index,
+		name: value.name || `Validator ${index + 1}`
+	}] : []);
+	const hasCode = manifest.length > 0;
+	const compiled = yield* bundle(hasCode ? validatorCaseEntry(entry, caseIndex, manifest) : definitionEntry(entry), entry, {
+		minify: true,
+		captureSource
+	});
+	const checks = hasCode ? [{
+		name: typeof subject.validate === "function" ? subject.validate.name || subject.name : subject.name,
+		source: compiled.source,
+		manifest,
+		capture: captureValidation
+	}] : [];
+	return yield* effect.Schema.decodeUnknown(require_evals.EvalValidator)({
+		...judges.length > 0 ? {
+			kind: "judged",
+			name: subject.name,
+			checks,
+			judges
+		} : checks[0],
+		sourceFiles: compiled.sourceFiles,
+		capture: captureValidation
+	});
+});
+//#endregion
 //#region src/evals/definition-loader.ts
 const loadDefinition = (entry) => bundle(definitionEntry(entry), entry).pipe(effect.Effect.flatMap(({ inputs, source }) => effect.Effect.acquireUseRelease(effect.Effect.tryPromise(() => (0, node_fs_promises.mkdtemp)((0, node_path.join)((0, node_os.tmpdir)(), "anpord-eval-"))), (directory) => effect.Effect.tryPromise({
 	try: async () => {
@@ -315,26 +361,25 @@ const isDefinition = (value) => effect.Option.isSome(decodeShape(value));
 //#region src/evals/local-repo.ts
 const run = (0, node_util.promisify)(node_child_process.execFile);
 const git = (args, cwd) => effect.Effect.tryPromise(() => run("git", [...args], { cwd })).pipe(effect.Effect.map(({ stdout }) => stdout.trim()), effect.Effect.orElseSucceed(() => ""));
-const environment = () => globalThis.process?.env ?? {};
 const remoteUrl = (cwd) => effect.Effect.gen(function* () {
-	const { GITHUB_REPOSITORY } = environment();
+	const repository = yield* effect.Config.string("GITHUB_REPOSITORY").pipe(effect.Config.withDefault(""));
 	const origin = yield* git([
 		"remote",
 		"get-url",
 		"origin"
 	], cwd);
-	return origin === "" && GITHUB_REPOSITORY !== void 0 ? GITHUB_REPOSITORY : origin;
+	return origin === "" ? repository : origin;
 });
 const fetchableRef = (cwd) => effect.Effect.gen(function* () {
-	const { GITHUB_HEAD_REF } = environment();
-	if (GITHUB_HEAD_REF !== void 0 && GITHUB_HEAD_REF !== "") return GITHUB_HEAD_REF;
+	const ci = yield* effect.Config.boolean("GITHUB_ACTIONS").pipe(effect.Config.withDefault(false));
 	const [head, onRemote] = yield* effect.Effect.all([git(["rev-parse", "HEAD"], cwd), git([
 		"branch",
 		"--remotes",
 		"--contains",
 		"HEAD"
 	], cwd)], { concurrency: 2 });
-	return head === "" || onRemote === "" ? null : head;
+	if (ci && head === "") return yield* effect.Effect.fail(/* @__PURE__ */ new Error("Check out the commit before running evals in GitHub Actions."));
+	return head === "" || !ci && onRemote === "" ? null : head;
 });
 const localRepo = (cwd) => effect.Effect.gen(function* () {
 	const [remote, ref] = yield* effect.Effect.all([remoteUrl(cwd), fetchableRef(cwd)], { concurrency: 2 });
@@ -442,7 +487,7 @@ const adapters = {
 const applyMcpHarness = (harness, profile, servers) => {
 	const adapter = adapters[harness];
 	if (adapter === null) throw new Error(`MCP mock servers are not supported by ${harness}`);
-	return effect.Schema.decodeUnknownSync(require_harness_profile.HarnessProfile)(adapter.apply(profile, servers));
+	return effect.Schema.decodeUnknownSync(require_evals.HarnessProfile)(adapter.apply(profile, servers));
 };
 //#endregion
 //#region src/evals/mcp-profile.ts
@@ -488,17 +533,17 @@ var ProfilePathInvalid = class extends effect.Data.TaggedError("ProfilePathInval
 };
 var ProfileFileTooLarge = class extends effect.Data.TaggedError("ProfileFileTooLarge") {
 	get message() {
-		return `${this.path} is ${this.chars} characters; a profile file holds at most ${require_harness_profile.PROFILE_LIMITS.fileChars}`;
+		return `${this.path} is ${this.chars} characters; a profile file holds at most ${require_evals.PROFILE_LIMITS.fileChars}`;
 	}
 };
 var ProfileTooLarge = class extends effect.Data.TaggedError("ProfileTooLarge") {
 	get message() {
-		return `The profile's files hold ${this.chars} characters; at most ${require_harness_profile.PROFILE_LIMITS.totalChars} fit`;
+		return `The profile's files hold ${this.chars} characters; at most ${require_evals.PROFILE_LIMITS.totalChars} fit`;
 	}
 };
 var ProfileTooManyFiles = class extends effect.Data.TaggedError("ProfileTooManyFiles") {
 	get message() {
-		return `The profile has ${this.count} files; at most ${require_harness_profile.PROFILE_LIMITS.files} fit`;
+		return `The profile has ${this.count} files; at most ${require_evals.PROFILE_LIMITS.files} fit`;
 	}
 };
 var ProfileManifestInvalid = class extends effect.Data.TaggedError("ProfileManifestInvalid") {
@@ -525,7 +570,7 @@ var ProfileStepNotSupported = class extends effect.Data.TaggedError("ProfileStep
 //#region src/evals/profile-manifest.ts
 const Manifest = effect.Schema.Struct({
 	env: effect.Schema.optional(effect.Schema.Record({
-		key: require_harness_profile.EnvName,
+		key: require_evals.EnvName,
 		value: effect.Schema.String
 	})),
 	install: effect.Schema.optional(effect.Schema.String),
@@ -542,14 +587,6 @@ const decodeManifest = (dir, text) => effect.Schema.decodeUnknown(ManifestJson)(
 	dir,
 	reason: issue.message
 })));
-/**
-* A manifest value that names a regular file is replaced by that file's
-* content; anything else is kept as written. Env values are always literal.
-*
-* A file outside the profile directory is refused rather than read: the
-* manifest is shipped with the profile, and a prompt that only exists on one
-* machine is not part of it.
-*/
 const inlined = (dir, value) => effect.Effect.gen(function* () {
 	const target = (0, node_path.resolve)(dir, value);
 	const position = (0, node_path.relative)(dir, target);
@@ -589,7 +626,6 @@ const readProfileManifest = (dir) => effect.Effect.gen(function* () {
 const SHIPPED_ROOTS = ["home", "workspace"];
 const SKIPPED_DIRECTORIES = /* @__PURE__ */ new Set(["node_modules", ".git"]);
 const isMissing = (cause) => cause instanceof Error && "code" in cause && cause.code === "ENOENT";
-/** Every regular file below `current`, as a posix path relative to `dir`. */
 const walk = async (dir, current) => {
 	const entries = await (0, node_fs_promises.readdir)(current, { withFileTypes: true });
 	return (await Promise.all(entries.map((entry) => {
@@ -611,8 +647,8 @@ const shippedPaths = (dir) => effect.Effect.tryPromise({
 		return (await Promise.all(SHIPPED_ROOTS.map((root) => walkRoot(dir, root)))).flat().sort();
 	}
 });
-const validPath = (path) => effect.Schema.decodeUnknown(require_harness_profile.ProfilePath)(path).pipe(effect.Effect.mapError(() => new ProfilePathInvalid({ path })));
-/** The file's text, or nothing when it holds a NUL byte and is not text. */
+const validPath = (path) => effect.Schema.decodeUnknown(require_evals.ProfilePath)(path).pipe(effect.Effect.mapError(() => new ProfilePathInvalid({ path })));
+/** Nothing when the file holds a NUL byte and so is not text. */
 const readShipped = (dir, path) => effect.Effect.gen(function* () {
 	const bytes = yield* effect.Effect.tryPromise({
 		catch: (cause) => new ProfileDirectoryUnreadable({
@@ -623,7 +659,7 @@ const readShipped = (dir, path) => effect.Effect.gen(function* () {
 	});
 	if (bytes.includes(0)) return [];
 	const content = bytes.toString("utf8");
-	if (content.length > require_harness_profile.PROFILE_LIMITS.fileChars) return yield* new ProfileFileTooLarge({
+	if (content.length > require_evals.PROFILE_LIMITS.fileChars) return yield* new ProfileFileTooLarge({
 		chars: content.length,
 		path
 	});
@@ -631,14 +667,14 @@ const readShipped = (dir, path) => effect.Effect.gen(function* () {
 });
 const shippedFiles = (dir) => effect.Effect.gen(function* () {
 	const paths = yield* shippedPaths(dir);
-	if (paths.length > require_harness_profile.PROFILE_LIMITS.files) return yield* new ProfileTooManyFiles({ count: paths.length });
+	if (paths.length > require_evals.PROFILE_LIMITS.files) return yield* new ProfileTooManyFiles({ count: paths.length });
 	const entries = yield* effect.Effect.forEach(paths, (path) => readShipped(dir, path), { concurrency: 8 }).pipe(effect.Effect.map((read) => read.flat()));
 	const chars = entries.reduce((sum, [, content]) => sum + content.length, 0);
-	if (chars > require_harness_profile.PROFILE_LIMITS.totalChars) return yield* new ProfileTooLarge({ chars });
+	if (chars > require_evals.PROFILE_LIMITS.totalChars) return yield* new ProfileTooLarge({ chars });
 	return Object.fromEntries(entries);
 });
 const fittingHarness = (base, profile) => {
-	if (require_harness_profile.profileFitsHarness({
+	if (require_evals.profileFitsHarness({
 		harness: base,
 		profile
 	})) return effect.Effect.succeed(profile);
@@ -648,8 +684,6 @@ const fittingHarness = (base, profile) => {
 		step: "run"
 	});
 };
-/** The task a definition's `{ base, profile }` harness compiles to, with the
-* profile directory read relative to the eval file. */
 const profileTask = (entry, task) => effect.Effect.gen(function* () {
 	const dir = (0, node_path.resolve)((0, node_path.dirname)(entry), task.harness.profile.dir);
 	yield* effect.Effect.tryPromise({
@@ -669,7 +703,7 @@ const profileTask = (entry, task) => effect.Effect.gen(function* () {
 		harness: task.harness.base,
 		model: task.model,
 		profile,
-		provider: task.provider
+		sandbox: task.sandbox
 	};
 }).pipe(effect.Effect.withSpan("Eval.profileTask", { attributes: { profile: task.harness.profile.name } }));
 //#endregion
@@ -678,7 +712,7 @@ const taskOf = (entry, task, clis, mcp) => effect.Effect.gen(function* () {
 	const compiled = typeof task.harness === "string" ? {
 		harness: task.harness,
 		model: task.model,
-		provider: task.provider
+		sandbox: task.sandbox
 	} : yield* profileTask(entry, {
 		...task,
 		harness: task.harness
@@ -701,10 +735,9 @@ const compileEvalEffect = (path) => effect.Effect.gen(function* () {
 	const clis = yield* compileClis(entry, definition.cli ?? []);
 	const mcp = yield* compileMcpServers(entry, definition.mcp ?? []);
 	const fallback = definition.source === void 0 && definition.cases.some((subject) => subject.source === void 0) ? yield* localRepo((0, node_path.dirname)(entry)) : effect.Option.none();
-	const cases = yield* effect.Effect.forEach(definition.cases, (subject) => effect.Effect.gen(function* () {
-		const hasValidator = typeof subject.validate === "function";
-		if (hasValidator === (typeof subject.verify === "string")) return yield* effect.Effect.fail(/* @__PURE__ */ new Error(`${subject.name} must have exactly one of validate or verify`));
-		const validator = hasValidator ? yield* bundledCaseModule(entry, loaded.inputs, subject.validate?.name || subject.name, validatorEntry) : null;
+	const cases = yield* effect.Effect.forEach(definition.cases, (subject, caseIndex) => effect.Effect.gen(function* () {
+		if (subject.validate !== void 0 === (typeof subject.verify === "string")) return yield* effect.Effect.fail(/* @__PURE__ */ new Error(`${subject.name} must have exactly one of validate or verify`));
+		const validator = yield* compileValidator(entry, subject, caseIndex, definition.captureSource !== false, definition.captureValidation !== false);
 		const prepare = typeof subject.prepare === "function" ? yield* bundledCaseModule(entry, loaded.inputs, subject.prepare.name || `${subject.name}-prepare`, prepareEntry) : null;
 		return {
 			...subject.cache === void 0 ? {} : { cache: subject.cache },

package/dist/config.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_client = require("./client-Ci0woWLW.cjs");
+const require_client = require("./client-C-P2I00z.cjs");
 let effect = require("effect");
 //#region src/client/config.ts
 const apiKeyConfig = effect.Config.redacted("ANPORD_API_KEY");

package/dist/config.d.cts

@@ -1,4 +1,4 @@
-import { t as AnpordApi } from "./client-C9m0uLCz.cjs";
+import { t as AnpordApi } from "./client-BirFO3jz.cjs";
 import { Config, Layer } from "effect";
 //#region src/client/config.d.ts
 declare const apiKeyConfig: Config.Config<import("effect/Redacted").Redacted<string>>;

package/dist/config.d.mts

@@ -1,4 +1,4 @@
-import { t as AnpordApi } from "./client-TFCLuMv8.mjs";
+import { t as AnpordApi } from "./client-BgRs8JvW.mjs";
 import { Config, Layer } from "effect";
 //#region src/client/config.d.ts
 declare const apiKeyConfig: Config.Config<import("effect/Redacted").Redacted<string>>;

package/dist/config.mjs

@@ -1,4 +1,4 @@
-import { n as DEFAULT_BASE_URL, r as layer } from "./client-5K3dMI5Q.mjs";
+import { n as DEFAULT_BASE_URL, r as layer } from "./client-BfdjFtlV.mjs";
 import { Config, Effect, Layer } from "effect";
 //#region src/client/config.ts
 const apiKeyConfig = Config.redacted("ANPORD_API_KEY");

package/dist/{errors-C9bQUcA3.mjs → errors-B0YknR5V.mjs}

@@ -1,25 +1,9 @@
 import { ArrayFormatter } from "effect/ParseResult";
 //#region ../template/src/syntax.ts
-/**
-* What a variable looks like, defined once.
-*
-* The editor's atom node, the extractor and the renderer have to agree
-* exactly: a name the editor draws as a chip but the renderer does not fill
-* reaches the model as literal braces, and nobody notices until the output is
-* wrong.
-*/
 const VARIABLE_PATTERN = String.raw`\{\{\s*([\w.-]+)\s*\}\}`;
-/**
-* Escapes and variables together, escapes first.
-*
-* They cannot be two passes: `{{{{name}}}}` contains `{{name}}` starting at
-* its third brace, so a substitution pass run on its own would fill the very
-* text the escape exists to protect. Reading both in one sweep is what keeps
-* the extractor and the renderer agreeing on what counts as a variable.
-*
-* A fresh instance each call, because a global regular expression carries its
-* own `lastIndex` and a shared one silently skips matches on a second pass.
-*/
+/** One sweep, escapes first: `{{{{name}}}}` contains `{{name}}` from its third
+* brace, so a separate substitution pass would fill the escaped text. Fresh
+* instance each call because a global regex carries its own `lastIndex`. */
 const tokenMatcher = () => new RegExp(String.raw`(\{\{\{\{)|(\}\}\}\})|${VARIABLE_PATTERN}`, "g");
 //#endregion
 //#region src/client/errors.ts

package/dist/{errors-Boum34f5.cjs → errors-BX1wry8K.cjs}

@@ -1,25 +1,9 @@
 let effect_ParseResult = require("effect/ParseResult");
 //#region ../template/src/syntax.ts
-/**
-* What a variable looks like, defined once.
-*
-* The editor's atom node, the extractor and the renderer have to agree
-* exactly: a name the editor draws as a chip but the renderer does not fill
-* reaches the model as literal braces, and nobody notices until the output is
-* wrong.
-*/
 const VARIABLE_PATTERN = String.raw`\{\{\s*([\w.-]+)\s*\}\}`;
-/**
-* Escapes and variables together, escapes first.
-*
-* They cannot be two passes: `{{{{name}}}}` contains `{{name}}` starting at
-* its third brace, so a substitution pass run on its own would fill the very
-* text the escape exists to protect. Reading both in one sweep is what keeps
-* the extractor and the renderer agreeing on what counts as a variable.
-*
-* A fresh instance each call, because a global regular expression carries its
-* own `lastIndex` and a shared one silently skips matches on a second pass.
-*/
+/** One sweep, escapes first: `{{{{name}}}}` contains `{{name}}` from its third
+* brace, so a separate substitution pass would fill the escaped text. Fresh
+* instance each call because a global regex carries its own `lastIndex`. */
 const tokenMatcher = () => new RegExp(String.raw`(\{\{\{\{)|(\}\}\}\})|${VARIABLE_PATTERN}`, "g");
 //#endregion
 //#region src/client/errors.ts

package/dist/{errors-BK3f7Rdr.d.cts → errors-aGrd21jy.d.cts}

@@ -29,9 +29,6 @@ declare const Forbidden_base: Schema.TaggedErrorClass<Forbidden, "Forbidden", {
 } & {
   message: typeof Schema.String;
 }>;
-/** Distinct from Unauthorized: the caller is who they say they are, and the
- * answer is still no. Signing in again cannot help, so the client must not
- * treat this as an expired session. */
 declare class Forbidden extends Forbidden_base {}
 //#endregion
 export { Unauthorized as a, NotFound as i, Conflict as n, Forbidden as r, BadRequest as t };

package/dist/{errors-BK3f7Rdr.d.mts → errors-aGrd21jy.d.mts}

@@ -29,9 +29,6 @@ declare const Forbidden_base: Schema.TaggedErrorClass<Forbidden, "Forbidden", {
 } & {
   message: typeof Schema.String;
 }>;
-/** Distinct from Unauthorized: the caller is who they say they are, and the
- * answer is still no. Signing in again cannot help, so the client must not
- * treat this as an expired session. */
 declare class Forbidden extends Forbidden_base {}
 //#endregion
 export { Unauthorized as a, NotFound as i, Conflict as n, Forbidden as r, BadRequest as t };

package/dist/eval-judges-BkmiVLLq.cjs

@@ -0,0 +1,59 @@
+let effect = require("effect");
+//#region ../schema/src/domain/harness.ts
+const EvalHarness = effect.Schema.Literal("codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command");
+//#endregion
+//#region ../schema/src/domain/eval-judges.ts
+const text = effect.Schema.String.pipe(effect.Schema.minLength(1));
+const EvalScore = effect.Schema.Number.pipe(effect.Schema.between(0, 1));
+const fields = {
+	kind: effect.Schema.Literal("judge"),
+	name: text.pipe(effect.Schema.maxLength(100)),
+	model: text.pipe(effect.Schema.maxLength(200)),
+	rubric: text.pipe(effect.Schema.maxLength(32e3)),
+	expected: effect.Schema.optional(effect.Schema.String.pipe(effect.Schema.maxLength(32e3))),
+	choices: effect.Schema.Record({
+		key: text,
+		value: EvalScore
+	}).pipe(effect.Schema.filter((value) => Object.keys(value).length > 0 && Object.keys(value).length <= 20, { message: () => "A judge needs between 1 and 20 scored choices" })),
+	threshold: effect.Schema.optionalWith(EvalScore, { default: () => 1 }),
+	timeoutMs: effect.Schema.optionalWith(effect.Schema.Int.pipe(effect.Schema.between(1e3, 3e5)), { default: () => 12e4 })
+};
+const EvalJudge = effect.Schema.Union(effect.Schema.Struct({
+	...fields,
+	provider: effect.Schema.Literal("openai"),
+	harness: effect.Schema.optional(effect.Schema.Never)
+}), effect.Schema.Struct({
+	...fields,
+	provider: effect.Schema.optional(effect.Schema.Never),
+	harness: effect.Schema.Literal(...EvalHarness.literals.filter((harness) => harness !== "command"))
+}));
+const EvalJudgment = effect.Schema.Struct({
+	name: effect.Schema.String,
+	model: effect.Schema.String,
+	evaluator: effect.Schema.String,
+	score: effect.Schema.NullOr(EvalScore),
+	choice: effect.Schema.NullOr(effect.Schema.String),
+	reason: effect.Schema.String,
+	threshold: EvalScore,
+	durationMs: effect.Schema.NonNegativeInt,
+	error: effect.Schema.NullOr(effect.Schema.String)
+});
+//#endregion
+Object.defineProperty(exports, "EvalHarness", {
+	enumerable: true,
+	get: function() {
+		return EvalHarness;
+	}
+});
+Object.defineProperty(exports, "EvalJudge", {
+	enumerable: true,
+	get: function() {
+		return EvalJudge;
+	}
+});
+Object.defineProperty(exports, "EvalJudgment", {
+	enumerable: true,
+	get: function() {
+		return EvalJudgment;
+	}
+});

package/dist/eval-judges-CwTvTq7N.mjs

@@ -0,0 +1,42 @@
+import { Schema } from "effect";
+//#region ../schema/src/domain/harness.ts
+const EvalHarness = Schema.Literal("codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command");
+//#endregion
+//#region ../schema/src/domain/eval-judges.ts
+const text = Schema.String.pipe(Schema.minLength(1));
+const EvalScore = Schema.Number.pipe(Schema.between(0, 1));
+const fields = {
+	kind: Schema.Literal("judge"),
+	name: text.pipe(Schema.maxLength(100)),
+	model: text.pipe(Schema.maxLength(200)),
+	rubric: text.pipe(Schema.maxLength(32e3)),
+	expected: Schema.optional(Schema.String.pipe(Schema.maxLength(32e3))),
+	choices: Schema.Record({
+		key: text,
+		value: EvalScore
+	}).pipe(Schema.filter((value) => Object.keys(value).length > 0 && Object.keys(value).length <= 20, { message: () => "A judge needs between 1 and 20 scored choices" })),
+	threshold: Schema.optionalWith(EvalScore, { default: () => 1 }),
+	timeoutMs: Schema.optionalWith(Schema.Int.pipe(Schema.between(1e3, 3e5)), { default: () => 12e4 })
+};
+const EvalJudge = Schema.Union(Schema.Struct({
+	...fields,
+	provider: Schema.Literal("openai"),
+	harness: Schema.optional(Schema.Never)
+}), Schema.Struct({
+	...fields,
+	provider: Schema.optional(Schema.Never),
+	harness: Schema.Literal(...EvalHarness.literals.filter((harness) => harness !== "command"))
+}));
+const EvalJudgment = Schema.Struct({
+	name: Schema.String,
+	model: Schema.String,
+	evaluator: Schema.String,
+	score: Schema.NullOr(EvalScore),
+	choice: Schema.NullOr(Schema.String),
+	reason: Schema.String,
+	threshold: EvalScore,
+	durationMs: Schema.NonNegativeInt,
+	error: Schema.NullOr(Schema.String)
+});
+//#endregion
+export { EvalJudgment as n, EvalHarness as r, EvalJudge as t };

package/dist/eval-judges-DPPUftbh.d.cts

@@ -0,0 +1,48 @@
+import { Schema } from "effect";
+//#region ../schema/src/domain/eval-judges.d.ts
+declare const EvalJudge: Schema.Union<[Schema.Struct<{
+  provider: Schema.Literal<["openai"]>;
+  harness: Schema.optional<typeof Schema.Never>;
+  kind: Schema.Literal<["judge"]>;
+  name: Schema.filter<Schema.filter<typeof Schema.String>>;
+  model: Schema.filter<Schema.filter<typeof Schema.String>>;
+  rubric: Schema.filter<Schema.filter<typeof Schema.String>>;
+  expected: Schema.optional<Schema.filter<typeof Schema.String>>;
+  choices: Schema.filter<Schema.Record$<Schema.filter<typeof Schema.String>, Schema.filter<typeof Schema.Number>>>;
+  threshold: Schema.optionalWith<Schema.filter<typeof Schema.Number>, {
+    default: () => number;
+  }>;
+  timeoutMs: Schema.optionalWith<Schema.filter<typeof Schema.Int>, {
+    default: () => number;
+  }>;
+}>, Schema.Struct<{
+  provider: Schema.optional<typeof Schema.Never>;
+  harness: Schema.SchemaClass<"codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor", "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor", never>;
+  kind: Schema.Literal<["judge"]>;
+  name: Schema.filter<Schema.filter<typeof Schema.String>>;
+  model: Schema.filter<Schema.filter<typeof Schema.String>>;
+  rubric: Schema.filter<Schema.filter<typeof Schema.String>>;
+  expected: Schema.optional<Schema.filter<typeof Schema.String>>;
+  choices: Schema.filter<Schema.Record$<Schema.filter<typeof Schema.String>, Schema.filter<typeof Schema.Number>>>;
+  threshold: Schema.optionalWith<Schema.filter<typeof Schema.Number>, {
+    default: () => number;
+  }>;
+  timeoutMs: Schema.optionalWith<Schema.filter<typeof Schema.Int>, {
+    default: () => number;
+  }>;
+}>]>;
+type EvalJudge = typeof EvalJudge.Type;
+declare const EvalJudgment: Schema.Struct<{
+  name: typeof Schema.String;
+  model: typeof Schema.String;
+  evaluator: typeof Schema.String;
+  score: Schema.NullOr<Schema.filter<typeof Schema.Number>>;
+  choice: Schema.NullOr<typeof Schema.String>;
+  reason: typeof Schema.String;
+  threshold: Schema.filter<typeof Schema.Number>;
+  durationMs: Schema.refine<number, typeof Schema.NonNegative>;
+  error: Schema.NullOr<typeof Schema.String>;
+}>;
+type EvalJudgment = typeof EvalJudgment.Type;
+//#endregion
+export { EvalJudgment as n, EvalJudge as t };