anpord 0.1.4 → 0.1.6

package/dist/cli.cjs

@@ -1,14 +1,14 @@
 #!/usr/bin/env node
-const require_client = require("./client-BCZEcfVE.cjs");
+const require_client = require("./client-BdGNFO_D.cjs");
 const require_config = require("./config.cjs");
 const require_errors = require("./errors-Boum34f5.cjs");
-const require_compiler = require("./compiler-CZoX4jkx.cjs");
+const require_compiler = require("./compiler-7cyJPuPr.cjs");
 let _effect_platform = require("@effect/platform");
 let effect = require("effect");
 let _effect_cli = require("@effect/cli");
 let _effect_platform_node = require("@effect/platform-node");
 //#region package.json
-var version$1 = "0.1.4";
+var version$1 = "0.1.6";
 //#endregion
 //#region ../template/src/extract.ts
 /**

package/dist/cli.mjs

@@ -1,14 +1,14 @@
 #!/usr/bin/env node
-import { a as ChannelName, o as PromptId, r as layer, s as VersionNumber, t as AnpordApi } from "./client-BfhmGRQl.mjs";
+import { a as ChannelName, o as PromptId, r as layer, s as VersionNumber, t as AnpordApi } from "./client-POb2FNW1.mjs";
 import { clientOptionsConfig } from "./config.mjs";
 import { i as tokenMatcher, r as asAnpordError } from "./errors-C9bQUcA3.mjs";
-import { n as compileEvalEffect } from "./compiler-CKiewwUc.mjs";
+import { n as compileEvalEffect } from "./compiler-Dk5l0aof.mjs";
 import { FileSystem } from "@effect/platform";
 import { Cause, Clock, Console, Data, Duration, Effect, Layer, Option, Ref } from "effect";
 import { Args, Command, Options } from "@effect/cli";
 import { NodeContext, NodeRuntime } from "@effect/platform-node";
 //#region package.json
-var version$1 = "0.1.4";
+var version$1 = "0.1.6";
 //#endregion
 //#region ../template/src/extract.ts
 /**

package/dist/{client-BCZEcfVE.cjs → client-BdGNFO_D.cjs}

@@ -133,6 +133,20 @@ const EvalValidator = effect.Schema.Struct({
 	description: "A bundled TypeScript validator and its exported function name.",
 	identifier: "EvalValidator"
 });
+const EvalWorkspaceSetup = effect.Schema.Struct({
+	name: effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(100)),
+	source: effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(1e6))
+}).annotations({
+	description: "A bundled TypeScript workspace setup and its exported function name.",
+	identifier: "EvalWorkspaceSetup"
+});
+const EvalSetupValue = effect.Schema.Record({
+	key: effect.Schema.String,
+	value: effect.Schema.Unknown
+}).annotations({
+	description: "What a workspace setup returned, handed to the validator.",
+	identifier: "EvalSetupValue"
+});
 const EvalVariables = effect.Schema.Record({
 	key: effect.Schema.String,
 	value: effect.Schema.String
@@ -142,7 +156,7 @@ const EvalVariables = effect.Schema.Record({
 });
 const EvalCase = effect.Schema.Struct({
 	name: effect.Schema.String,
-	setup: effect.Schema.NullOr(effect.Schema.String),
+	setup: effect.Schema.NullOr(EvalWorkspaceSetup),
 	source: EvalSource,
 	variables: effect.Schema.optionalWith(EvalVariables, { default: () => ({}) }),
 	validator: effect.Schema.optionalWith(effect.Schema.NullOr(EvalValidator), { default: () => null }),
@@ -216,6 +230,7 @@ const EvalVerifyStep = effect.Schema.Struct({
 });
 const EvalTrial = effect.Schema.Struct({
 	commands: effect.Schema.Int,
+	setupValue: effect.Schema.NullOr(EvalSetupValue),
 	exitCode: effect.Schema.Int,
 	failedCommands: effect.Schema.Int,
 	filesChanged: effect.Schema.Array(effect.Schema.String),
@@ -265,7 +280,7 @@ const EvalSetup = effect.Schema.Struct({
 	prompt: effect.Schema.String,
 	repoRef: effect.Schema.NullOr(effect.Schema.String),
 	repoUrl: effect.Schema.NullOr(effect.Schema.String),
-	setupCommand: effect.Schema.NullOr(effect.Schema.String),
+	setupName: effect.Schema.NullOr(effect.Schema.String),
 	validatorName: effect.Schema.NullOr(effect.Schema.String),
 	verifyCommand: effect.Schema.NullOr(effect.Schema.String),
 	workspace: effect.Schema.String
@@ -472,7 +487,7 @@ const ListEvalsRequest = effect.Schema.Struct({
 });
 const PublicEvalCase = effect.Schema.Struct({
 	name: effect.Schema.String,
-	setup: effect.Schema.optional(effect.Schema.NullOr(effect.Schema.String)),
+	setup: effect.Schema.optional(effect.Schema.NullOr(EvalWorkspaceSetup)),
 	source: effect.Schema.optional(EvalSource),
 	validator: effect.Schema.optional(effect.Schema.NullOr(EvalValidator)),
 	variables: effect.Schema.optional(EvalVariables),

package/dist/{client-BfhmGRQl.mjs → client-POb2FNW1.mjs}

@@ -133,6 +133,20 @@ const EvalValidator = Schema.Struct({
 	description: "A bundled TypeScript validator and its exported function name.",
 	identifier: "EvalValidator"
 });
+const EvalWorkspaceSetup = Schema.Struct({
+	name: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(100)),
+	source: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(1e6))
+}).annotations({
+	description: "A bundled TypeScript workspace setup and its exported function name.",
+	identifier: "EvalWorkspaceSetup"
+});
+const EvalSetupValue = Schema.Record({
+	key: Schema.String,
+	value: Schema.Unknown
+}).annotations({
+	description: "What a workspace setup returned, handed to the validator.",
+	identifier: "EvalSetupValue"
+});
 const EvalVariables = Schema.Record({
 	key: Schema.String,
 	value: Schema.String
@@ -142,7 +156,7 @@ const EvalVariables = Schema.Record({
 });
 const EvalCase = Schema.Struct({
 	name: Schema.String,
-	setup: Schema.NullOr(Schema.String),
+	setup: Schema.NullOr(EvalWorkspaceSetup),
 	source: EvalSource,
 	variables: Schema.optionalWith(EvalVariables, { default: () => ({}) }),
 	validator: Schema.optionalWith(Schema.NullOr(EvalValidator), { default: () => null }),
@@ -216,6 +230,7 @@ const EvalVerifyStep = Schema.Struct({
 });
 const EvalTrial = Schema.Struct({
 	commands: Schema.Int,
+	setupValue: Schema.NullOr(EvalSetupValue),
 	exitCode: Schema.Int,
 	failedCommands: Schema.Int,
 	filesChanged: Schema.Array(Schema.String),
@@ -265,7 +280,7 @@ const EvalSetup = Schema.Struct({
 	prompt: Schema.String,
 	repoRef: Schema.NullOr(Schema.String),
 	repoUrl: Schema.NullOr(Schema.String),
-	setupCommand: Schema.NullOr(Schema.String),
+	setupName: Schema.NullOr(Schema.String),
 	validatorName: Schema.NullOr(Schema.String),
 	verifyCommand: Schema.NullOr(Schema.String),
 	workspace: Schema.String
@@ -472,7 +487,7 @@ const ListEvalsRequest = Schema.Struct({
 });
 const PublicEvalCase = Schema.Struct({
 	name: Schema.String,
-	setup: Schema.optional(Schema.NullOr(Schema.String)),
+	setup: Schema.optional(Schema.NullOr(EvalWorkspaceSetup)),
 	source: Schema.optional(EvalSource),
 	validator: Schema.optional(Schema.NullOr(EvalValidator)),
 	variables: Schema.optional(EvalVariables),

package/dist/{compiler-CZoX4jkx.cjs → compiler-7cyJPuPr.cjs}

@@ -59,6 +59,7 @@ const context = {
   exec,
   exists: path => access(path).then(() => true, () => false),
   readText: path => readFile(path, "utf8"),
+  setup: JSON.parse(process.env.ANPORD_SETUP_VALUE ?? "{}"),
 };
 
 try {
@@ -72,6 +73,49 @@ try {
 }
 `;
 const validatorEntry = (module, name) => `import { ${name} as validate } from ${JSON.stringify(module)};\n${runtime}`;
+const setupRuntime = `
+import { access, readFile } from "node:fs/promises";
+import { spawn } from "node:child_process";
+
+const exec = (file, args = [], options = {}) => new Promise((resolve, reject) => {
+  const child = spawn(file, args, {
+    cwd: options.cwd ?? process.cwd(),
+    env: { ...process.env, ...options.env },
+  });
+  let stderr = "";
+  let stdout = "";
+  let timer;
+  if (options.timeoutMs) {
+    timer = setTimeout(() => {
+      child.kill("SIGKILL");
+      reject(new Error(\`\${file} did not finish within \${options.timeoutMs}ms\`));
+    }, options.timeoutMs);
+  }
+  child.stderr.on("data", chunk => { stderr += chunk; });
+  child.stdout.on("data", chunk => { stdout += chunk; });
+  child.on("error", error => { clearTimeout(timer); reject(error); });
+  child.on("close", code => {
+    clearTimeout(timer);
+    resolve({ exitCode: code ?? 1, stderr, stdout });
+  });
+});
+
+const context = {
+  exec,
+  exists: path => access(path).then(() => true, () => false),
+  readText: path => readFile(path, "utf8"),
+  workspace: process.cwd(),
+};
+
+try {
+  const value = await setup(context) ?? {};
+  console.log("ANPORD_SETUP_RESULT=" + JSON.stringify(value));
+} catch (error) {
+  console.error(error instanceof Error ? error.stack : String(error));
+  process.exitCode = 1;
+}
+`;
+const setupEntry = (module, name) => `import { ${name} as setup } from ${JSON.stringify(module)};\n${setupRuntime}`;
 const definitionEntry = (entry) => `import definition from ${JSON.stringify(entry)}; export default definition;`;
 //#endregion
 //#region src/evals/compiler.ts
@@ -154,6 +198,14 @@ const sourceFor = (definition, subject, fallback) => {
 		onSome: (source) => ({ source })
 	});
 };
+const bundled = (entry, inputs, name, wrap) => effect.Effect.gen(function* () {
+	const module = yield* validatorModule(entry, inputs, name);
+	const { source } = yield* bundle(wrap(module, name), entry);
+	return {
+		name,
+		source
+	};
+});
 const compileEvalEffect = (path) => effect.Effect.gen(function* () {
 	const entry = (0, node_path.resolve)(path);
 	const loaded = yield* loadDefinition(entry);
@@ -164,13 +216,11 @@ const compileEvalEffect = (path) => effect.Effect.gen(function* () {
 		cases: yield* effect.Effect.forEach(definition.cases, (subject) => effect.Effect.gen(function* () {
 			const hasValidator = typeof subject.validate === "function";
 			if (hasValidator === (typeof subject.verify === "string")) return yield* effect.Effect.fail(/* @__PURE__ */ new Error(`${subject.name} must have exactly one of validate or verify`));
-			const validator = hasValidator ? {
-				name: subject.validate?.name || subject.name,
-				source: (yield* bundle(validatorEntry(yield* validatorModule(entry, loaded.inputs, subject.validate?.name || subject.name), subject.validate?.name || subject.name), entry)).source
-			} : null;
+			const validator = hasValidator ? yield* bundled(entry, loaded.inputs, subject.validate?.name || subject.name, validatorEntry) : null;
+			const setup = typeof subject.setup === "function" ? yield* bundled(entry, loaded.inputs, subject.setup.name || `${subject.name}-setup`, setupEntry) : null;
 			return {
 				name: subject.name,
-				setup: subject.setup ?? null,
+				setup,
 				...sourceFor(definition, subject, fallback),
 				validator,
 				variables: subject.variables ?? {},

package/dist/{compiler-CKiewwUc.mjs → compiler-Dk5l0aof.mjs}

@@ -60,6 +60,7 @@ const context = {
   exec,
   exists: path => access(path).then(() => true, () => false),
   readText: path => readFile(path, "utf8"),
+  setup: JSON.parse(process.env.ANPORD_SETUP_VALUE ?? "{}"),
 };
 
 try {
@@ -73,6 +74,49 @@ try {
 }
 `;
 const validatorEntry = (module, name) => `import { ${name} as validate } from ${JSON.stringify(module)};\n${runtime}`;
+const setupRuntime = `
+import { access, readFile } from "node:fs/promises";
+import { spawn } from "node:child_process";
+
+const exec = (file, args = [], options = {}) => new Promise((resolve, reject) => {
+  const child = spawn(file, args, {
+    cwd: options.cwd ?? process.cwd(),
+    env: { ...process.env, ...options.env },
+  });
+  let stderr = "";
+  let stdout = "";
+  let timer;
+  if (options.timeoutMs) {
+    timer = setTimeout(() => {
+      child.kill("SIGKILL");
+      reject(new Error(\`\${file} did not finish within \${options.timeoutMs}ms\`));
+    }, options.timeoutMs);
+  }
+  child.stderr.on("data", chunk => { stderr += chunk; });
+  child.stdout.on("data", chunk => { stdout += chunk; });
+  child.on("error", error => { clearTimeout(timer); reject(error); });
+  child.on("close", code => {
+    clearTimeout(timer);
+    resolve({ exitCode: code ?? 1, stderr, stdout });
+  });
+});
+
+const context = {
+  exec,
+  exists: path => access(path).then(() => true, () => false),
+  readText: path => readFile(path, "utf8"),
+  workspace: process.cwd(),
+};
+
+try {
+  const value = await setup(context) ?? {};
+  console.log("ANPORD_SETUP_RESULT=" + JSON.stringify(value));
+} catch (error) {
+  console.error(error instanceof Error ? error.stack : String(error));
+  process.exitCode = 1;
+}
+`;
+const setupEntry = (module, name) => `import { ${name} as setup } from ${JSON.stringify(module)};\n${setupRuntime}`;
 const definitionEntry = (entry) => `import definition from ${JSON.stringify(entry)}; export default definition;`;
 //#endregion
 //#region src/evals/compiler.ts
@@ -155,6 +199,14 @@ const sourceFor = (definition, subject, fallback) => {
 		onSome: (source) => ({ source })
 	});
 };
+const bundled = (entry, inputs, name, wrap) => Effect.gen(function* () {
+	const module = yield* validatorModule(entry, inputs, name);
+	const { source } = yield* bundle(wrap(module, name), entry);
+	return {
+		name,
+		source
+	};
+});
 const compileEvalEffect = (path) => Effect.gen(function* () {
 	const entry = resolve(path);
 	const loaded = yield* loadDefinition(entry);
@@ -165,13 +217,11 @@ const compileEvalEffect = (path) => Effect.gen(function* () {
 		cases: yield* Effect.forEach(definition.cases, (subject) => Effect.gen(function* () {
 			const hasValidator = typeof subject.validate === "function";
 			if (hasValidator === (typeof subject.verify === "string")) return yield* Effect.fail(/* @__PURE__ */ new Error(`${subject.name} must have exactly one of validate or verify`));
-			const validator = hasValidator ? {
-				name: subject.validate?.name || subject.name,
-				source: (yield* bundle(validatorEntry(yield* validatorModule(entry, loaded.inputs, subject.validate?.name || subject.name), subject.validate?.name || subject.name), entry)).source
-			} : null;
+			const validator = hasValidator ? yield* bundled(entry, loaded.inputs, subject.validate?.name || subject.name, validatorEntry) : null;
+			const setup = typeof subject.setup === "function" ? yield* bundled(entry, loaded.inputs, subject.setup.name || `${subject.name}-setup`, setupEntry) : null;
 			return {
 				name: subject.name,
-				setup: subject.setup ?? null,
+				setup,
 				...sourceFor(definition, subject, fallback),
 				validator,
 				variables: subject.variables ?? {},

package/dist/config.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_client = require("./client-BCZEcfVE.cjs");
+const require_client = require("./client-BdGNFO_D.cjs");
 let effect = require("effect");
 //#region src/client/config.ts
 const apiKeyConfig = effect.Config.redacted("ANPORD_API_KEY");

package/dist/config.mjs

@@ -1,4 +1,4 @@
-import { n as DEFAULT_BASE_URL } from "./client-BfhmGRQl.mjs";
+import { n as DEFAULT_BASE_URL } from "./client-POb2FNW1.mjs";
 import { Config } from "effect";
 //#region src/client/config.ts
 const apiKeyConfig = Config.redacted("ANPORD_API_KEY");

package/dist/eval.cjs

@@ -1,4 +1,4 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_compiler = require("./compiler-CZoX4jkx.cjs");
+const require_compiler = require("./compiler-7cyJPuPr.cjs");
 exports.compileEval = require_compiler.compileEval;
 exports.compileEvalEffect = require_compiler.compileEvalEffect;

package/dist/eval.d.cts

@@ -1,4 +1,4 @@
-import { t as PublicStartEvalRequest } from "./evals-api-sqVNeWet.cjs";
+import { t as PublicStartEvalRequest } from "./evals-api-BW7WsvHI.cjs";
 import { Effect } from "effect";
 //#region src/evals/compiler.d.ts
 declare const compileEvalEffect: (path: string) => Effect.Effect<{
@@ -10,7 +10,10 @@ declare const compileEvalEffect: (path: string) => Effect.Effect<{
     variables: Readonly<Record<string, string>>;
     verify: string | null;
     name: string;
-    setup: string | null;
+    setup: {
+      name: string;
+      source: string;
+    } | null;
   }[];
   prompt: string;
   tasks: {

package/dist/eval.d.mts

@@ -1,4 +1,4 @@
-import { t as PublicStartEvalRequest } from "./evals-api-DUNVUGUy.mjs";
+import { t as PublicStartEvalRequest } from "./evals-api-D-86RBA4.mjs";
 import { Effect } from "effect";
 //#region src/evals/compiler.d.ts
 declare const compileEvalEffect: (path: string) => Effect.Effect<{
@@ -10,7 +10,10 @@ declare const compileEvalEffect: (path: string) => Effect.Effect<{
     variables: Readonly<Record<string, string>>;
     verify: string | null;
     name: string;
-    setup: string | null;
+    setup: {
+      name: string;
+      source: string;
+    } | null;
   }[];
   prompt: string;
   tasks: {

package/dist/eval.mjs

@@ -1,2 +1,2 @@
-import { n as compileEvalEffect, t as compileEval } from "./compiler-CKiewwUc.mjs";
+import { n as compileEvalEffect, t as compileEval } from "./compiler-Dk5l0aof.mjs";
 export { compileEval, compileEvalEffect };

package/dist/{evals-api-sqVNeWet.d.cts → evals-api-BW7WsvHI.d.cts}

@@ -39,7 +39,10 @@ declare class Forbidden extends Forbidden_base {}
 declare const PublicStartEvalRequest: Schema.Struct<{
   cases: Schema.filter<Schema.Array$<Schema.refine<{
     readonly name: string;
-    readonly setup?: string | null | undefined;
+    readonly setup?: {
+      readonly name: string;
+      readonly source: string;
+    } | null | undefined;
     readonly source?: {
       readonly kind: "empty";
     } | {
@@ -62,7 +65,10 @@ declare const PublicStartEvalRequest: Schema.Struct<{
     readonly verify: string | null;
   }, Schema.Struct<{
     name: typeof Schema.String;
-    setup: Schema.optional<Schema.NullOr<typeof Schema.String>>;
+    setup: Schema.optional<Schema.NullOr<Schema.Struct<{
+      name: Schema.filter<Schema.filter<typeof Schema.String>>;
+      source: Schema.filter<Schema.filter<typeof Schema.String>>;
+    }>>>;
     source: Schema.optional<Schema.Union<[Schema.Struct<{
       kind: Schema.Literal<["empty"]>;
     }>, Schema.Struct<{

package/dist/{evals-api-DUNVUGUy.d.mts → evals-api-D-86RBA4.d.mts}

@@ -39,7 +39,10 @@ declare class Forbidden extends Forbidden_base {}
 declare const PublicStartEvalRequest: Schema.Struct<{
   cases: Schema.filter<Schema.Array$<Schema.refine<{
     readonly name: string;
-    readonly setup?: string | null | undefined;
+    readonly setup?: {
+      readonly name: string;
+      readonly source: string;
+    } | null | undefined;
     readonly source?: {
       readonly kind: "empty";
     } | {
@@ -62,7 +65,10 @@ declare const PublicStartEvalRequest: Schema.Struct<{
     readonly verify: string | null;
   }, Schema.Struct<{
     name: typeof Schema.String;
-    setup: Schema.optional<Schema.NullOr<typeof Schema.String>>;
+    setup: Schema.optional<Schema.NullOr<Schema.Struct<{
+      name: Schema.filter<Schema.filter<typeof Schema.String>>;
+      source: Schema.filter<Schema.filter<typeof Schema.String>>;
+    }>>>;
     source: Schema.optional<Schema.Union<[Schema.Struct<{
       kind: Schema.Literal<["empty"]>;
     }>, Schema.Struct<{

package/dist/index.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_client = require("./client-BCZEcfVE.cjs");
+const require_client = require("./client-BdGNFO_D.cjs");
 const require_errors = require("./errors-Boum34f5.cjs");
 const require_source = require("./source-DvlGawEw.cjs");
 let _effect_platform = require("@effect/platform");

package/dist/index.d.cts

@@ -1,5 +1,5 @@
-import { a as NotFound, i as Forbidden, n as BadRequest, o as Unauthorized, r as Conflict, t as PublicStartEvalRequest } from "./evals-api-sqVNeWet.cjs";
-import { C as StartedEval, S as RerunCellRequest, _ as EvalTrial, a as EvalCell, b as EvalVerdict, c as EvalDistribution, d as EvalProvider, f as EvalRun, g as EvalTask, h as EvalSource, i as CatalogueModel, l as EvalHarness, m as EvalRunSummary, n as files, o as EvalCellHistoryEntry, p as EvalRunStatus, r as repo, s as EvalComparison, t as empty, u as EvalJournalEntry, v as EvalTrialStatus, x as ModelCatalogue, y as EvalUsage } from "./source-DQ8HChlm.cjs";
+import { a as NotFound, i as Forbidden, n as BadRequest, o as Unauthorized, r as Conflict, t as PublicStartEvalRequest } from "./evals-api-BW7WsvHI.cjs";
+import { C as StartedEval, S as RerunCellRequest, _ as EvalTrial, a as EvalCell, b as EvalVerdict, c as EvalDistribution, d as EvalProvider, f as EvalRun, g as EvalTask, h as EvalSource, i as CatalogueModel, l as EvalHarness, m as EvalRunSummary, n as files, o as EvalCellHistoryEntry, p as EvalRunStatus, r as repo, s as EvalComparison, t as empty, u as EvalJournalEntry, v as EvalTrialStatus, x as ModelCatalogue, y as EvalUsage } from "./source-Bq8YhKOG.cjs";
 import { Brand, Effect, Layer, Redacted, Schema } from "effect";
 import { HttpClient, HttpClientResponse } from "@effect/platform";
 //#region ../schema/src/public/client.d.ts
@@ -78,7 +78,10 @@ declare const make: ({ apiKey, baseUrl }: ClientOptions) => Effect.Effect<{
       readonly payload: {
         readonly cases: readonly {
           readonly name: string;
-          readonly setup?: string | null | undefined;
+          readonly setup?: {
+            readonly name: string;
+            readonly source: string;
+          } | null | undefined;
           readonly source?: {
             readonly kind: "empty";
           } | {
@@ -138,7 +141,7 @@ declare const make: ({ apiKey, baseUrl }: ClientOptions) => Effect.Effect<{
           readonly prompt: string;
           readonly repoRef: string | null;
           readonly repoUrl: string | null;
-          readonly setupCommand: string | null;
+          readonly setupName: string | null;
           readonly validatorName: string | null;
           readonly verifyCommand: string | null;
           readonly workspace: string;
@@ -147,6 +150,9 @@ declare const make: ({ apiKey, baseUrl }: ClientOptions) => Effect.Effect<{
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -243,7 +249,7 @@ declare const make: ({ apiKey, baseUrl }: ClientOptions) => Effect.Effect<{
           readonly prompt: string;
           readonly repoRef: string | null;
           readonly repoUrl: string | null;
-          readonly setupCommand: string | null;
+          readonly setupName: string | null;
           readonly validatorName: string | null;
           readonly verifyCommand: string | null;
           readonly workspace: string;
@@ -252,6 +258,9 @@ declare const make: ({ apiKey, baseUrl }: ClientOptions) => Effect.Effect<{
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -341,6 +350,9 @@ declare const make: ({ apiKey, baseUrl }: ClientOptions) => Effect.Effect<{
         readonly passed: boolean;
         readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
         readonly commands: number;
+        readonly setupValue: {
+          readonly [x: string]: unknown;
+        } | null;
         readonly exitCode: number;
         readonly failedCommands: number;
         readonly filesChanged: readonly string[];
@@ -413,6 +425,9 @@ declare const make: ({ apiKey, baseUrl }: ClientOptions) => Effect.Effect<{
         readonly passed: boolean;
         readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
         readonly commands: number;
+        readonly setupValue: {
+          readonly [x: string]: unknown;
+        } | null;
         readonly exitCode: number;
         readonly failedCommands: number;
         readonly filesChanged: readonly string[];
@@ -763,7 +778,10 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
       readonly payload: {
         readonly cases: readonly {
           readonly name: string;
-          readonly setup?: string | null | undefined;
+          readonly setup?: {
+            readonly name: string;
+            readonly source: string;
+          } | null | undefined;
           readonly source?: {
             readonly kind: "empty";
           } | {
@@ -823,7 +841,7 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly prompt: string;
           readonly repoRef: string | null;
           readonly repoUrl: string | null;
-          readonly setupCommand: string | null;
+          readonly setupName: string | null;
           readonly validatorName: string | null;
           readonly verifyCommand: string | null;
           readonly workspace: string;
@@ -832,6 +850,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -928,7 +949,7 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly prompt: string;
           readonly repoRef: string | null;
           readonly repoUrl: string | null;
-          readonly setupCommand: string | null;
+          readonly setupName: string | null;
           readonly validatorName: string | null;
           readonly verifyCommand: string | null;
           readonly workspace: string;
@@ -937,6 +958,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -1026,6 +1050,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
         readonly passed: boolean;
         readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
         readonly commands: number;
+        readonly setupValue: {
+          readonly [x: string]: unknown;
+        } | null;
         readonly exitCode: number;
         readonly failedCommands: number;
         readonly filesChanged: readonly string[];
@@ -1098,6 +1125,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
         readonly passed: boolean;
         readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
         readonly commands: number;
+        readonly setupValue: {
+          readonly [x: string]: unknown;
+        } | null;
         readonly exitCode: number;
         readonly failedCommands: number;
         readonly filesChanged: readonly string[];
@@ -1446,7 +1476,10 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
       readonly payload: {
         readonly cases: readonly {
           readonly name: string;
-          readonly setup?: string | null | undefined;
+          readonly setup?: {
+            readonly name: string;
+            readonly source: string;
+          } | null | undefined;
           readonly source?: {
             readonly kind: "empty";
           } | {
@@ -1506,7 +1539,7 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly prompt: string;
           readonly repoRef: string | null;
           readonly repoUrl: string | null;
-          readonly setupCommand: string | null;
+          readonly setupName: string | null;
           readonly validatorName: string | null;
           readonly verifyCommand: string | null;
           readonly workspace: string;
@@ -1515,6 +1548,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -1611,7 +1647,7 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly prompt: string;
           readonly repoRef: string | null;
           readonly repoUrl: string | null;
-          readonly setupCommand: string | null;
+          readonly setupName: string | null;
           readonly validatorName: string | null;
           readonly verifyCommand: string | null;
           readonly workspace: string;
@@ -1620,6 +1656,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -1709,6 +1748,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
         readonly passed: boolean;
         readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
         readonly commands: number;
+        readonly setupValue: {
+          readonly [x: string]: unknown;
+        } | null;
         readonly exitCode: number;
         readonly failedCommands: number;
         readonly filesChanged: readonly string[];
@@ -1781,6 +1823,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
         readonly passed: boolean;
         readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
         readonly commands: number;
+        readonly setupValue: {
+          readonly [x: string]: unknown;
+        } | null;
         readonly exitCode: number;
         readonly failedCommands: number;
         readonly filesChanged: readonly string[];
@@ -2130,7 +2175,10 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
         readonly payload: {
           readonly cases: readonly {
             readonly name: string;
-            readonly setup?: string | null | undefined;
+            readonly setup?: {
+              readonly name: string;
+              readonly source: string;
+            } | null | undefined;
             readonly source?: {
               readonly kind: "empty";
             } | {
@@ -2190,7 +2238,7 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
             readonly prompt: string;
             readonly repoRef: string | null;
             readonly repoUrl: string | null;
-            readonly setupCommand: string | null;
+            readonly setupName: string | null;
             readonly validatorName: string | null;
             readonly verifyCommand: string | null;
             readonly workspace: string;
@@ -2199,6 +2247,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
             readonly passed: boolean;
             readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
             readonly commands: number;
+            readonly setupValue: {
+              readonly [x: string]: unknown;
+            } | null;
             readonly exitCode: number;
             readonly failedCommands: number;
             readonly filesChanged: readonly string[];
@@ -2295,7 +2346,7 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
             readonly prompt: string;
             readonly repoRef: string | null;
             readonly repoUrl: string | null;
-            readonly setupCommand: string | null;
+            readonly setupName: string | null;
             readonly validatorName: string | null;
             readonly verifyCommand: string | null;
             readonly workspace: string;
@@ -2304,6 +2355,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
             readonly passed: boolean;
             readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
             readonly commands: number;
+            readonly setupValue: {
+              readonly [x: string]: unknown;
+            } | null;
             readonly exitCode: number;
             readonly failedCommands: number;
             readonly filesChanged: readonly string[];
@@ -2393,6 +2447,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -2465,6 +2522,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -2956,10 +3016,24 @@ interface ValidatorCommandResult {
   readonly stderr: string;
   readonly stdout: string;
 }
+interface ExecOptions {
+  readonly cwd?: string;
+  readonly env?: Readonly<Record<string, string>>;
+  readonly timeoutMs?: number;
+}
+interface SetupContext {
+  readonly exec: (file: string, args?: readonly string[], options?: ExecOptions) => Promise<ValidatorCommandResult>;
+  readonly exists: (path: string) => Promise<boolean>;
+  readonly readText: (path: string) => Promise<string>;
+  readonly workspace: string;
+}
+type SetupValue = Readonly<Record<string, unknown>>;
+type WorkspaceSetup = (context: SetupContext) => Promise<SetupValue | undefined> | SetupValue | undefined;
 interface ValidatorContext {
   readonly exec: (command: string) => Promise<ValidatorCommandResult>;
   readonly exists: (path: string) => Promise<boolean>;
   readonly readText: (path: string) => Promise<string>;
+  readonly setup: Readonly<Record<string, unknown>>;
 }
 interface ValidatorResult {
   readonly message?: string;
@@ -2969,7 +3043,7 @@ type Validator = (context: ValidatorContext) => boolean | ValidatorResult | Prom
 type DeclaredSource = EvalSource | string;
 interface EvalCaseBase {
   readonly name: string;
-  readonly setup?: string | null;
+  readonly setup?: WorkspaceSetup | null;
   readonly source?: DeclaredSource;
   readonly variables?: Readonly<Record<string, string>>;
 }
@@ -2997,4 +3071,4 @@ type StartEvalRequest = Parameters<EvalsSurface["start"]>[0];
 type EvalCase = StartEvalRequest["cases"][number];
 type EvalTaskRequest = StartEvalRequest["tasks"][number];
 //#endregion
-export { Anpord, AnpordApi, type AnpordClient, AnpordError, type AnpordOptions, type AnpordPromptVariables, type CacheOptions, type CatalogueModel, type ClientOptions, DEFAULT_BASE_URL, EvalAborted, EvalCase, type EvalCaseDefinition, type EvalCell, type EvalCellHistoryEntry, type EvalComparison, type EvalDefinition, type EvalDistribution, type EvalHarness, type EvalJournalEntry, type EvalProvider, type EvalRun, type EvalRunStatus, type EvalRunSummary, type EvalSource, type EvalTask, EvalTaskRequest, EvalTimeout, type EvalTrial, type EvalTrialStatus, type EvalUsage, type EvalVerdict, type EvalsSurface, type GetPromptOptions, MissingApiKey, type ModelCatalogue, type PromptFallback, type PromptMetadata, type PromptResult, type PromptsSurface, type PublicPrompt, type PublicPromptSummary, type PublicPromptWithVersions, type PublicVersion, type RerunCellRequest, StartEvalRequest, type StartedEval, type Validator, type ValidatorCommandResult, type ValidatorContext, type ValidatorResult, type WaitOptions, defineEval, empty, files, layer, make, repo };
+export { Anpord, AnpordApi, type AnpordClient, AnpordError, type AnpordOptions, type AnpordPromptVariables, type CacheOptions, type CatalogueModel, type ClientOptions, DEFAULT_BASE_URL, EvalAborted, EvalCase, type EvalCaseDefinition, type EvalCell, type EvalCellHistoryEntry, type EvalComparison, type EvalDefinition, type EvalDistribution, type EvalHarness, type EvalJournalEntry, type EvalProvider, type EvalRun, type EvalRunStatus, type EvalRunSummary, type EvalSource, type EvalTask, EvalTaskRequest, EvalTimeout, type EvalTrial, type EvalTrialStatus, type EvalUsage, type EvalVerdict, type EvalsSurface, type ExecOptions, type GetPromptOptions, MissingApiKey, type ModelCatalogue, type PromptFallback, type PromptMetadata, type PromptResult, type PromptsSurface, type PublicPrompt, type PublicPromptSummary, type PublicPromptWithVersions, type PublicVersion, type RerunCellRequest, type SetupContext, type SetupValue, StartEvalRequest, type StartedEval, type Validator, type ValidatorCommandResult, type ValidatorContext, type ValidatorResult, type WaitOptions, type WorkspaceSetup, defineEval, empty, files, layer, make, repo };

package/dist/index.d.mts

@@ -1,5 +1,5 @@
-import { a as NotFound, i as Forbidden, n as BadRequest, o as Unauthorized, r as Conflict, t as PublicStartEvalRequest } from "./evals-api-DUNVUGUy.mjs";
-import { C as StartedEval, S as RerunCellRequest, _ as EvalTrial, a as EvalCell, b as EvalVerdict, c as EvalDistribution, d as EvalProvider, f as EvalRun, g as EvalTask, h as EvalSource, i as CatalogueModel, l as EvalHarness, m as EvalRunSummary, n as files, o as EvalCellHistoryEntry, p as EvalRunStatus, r as repo, s as EvalComparison, t as empty, u as EvalJournalEntry, v as EvalTrialStatus, x as ModelCatalogue, y as EvalUsage } from "./source-DQ8HChlm.mjs";
+import { a as NotFound, i as Forbidden, n as BadRequest, o as Unauthorized, r as Conflict, t as PublicStartEvalRequest } from "./evals-api-D-86RBA4.mjs";
+import { C as StartedEval, S as RerunCellRequest, _ as EvalTrial, a as EvalCell, b as EvalVerdict, c as EvalDistribution, d as EvalProvider, f as EvalRun, g as EvalTask, h as EvalSource, i as CatalogueModel, l as EvalHarness, m as EvalRunSummary, n as files, o as EvalCellHistoryEntry, p as EvalRunStatus, r as repo, s as EvalComparison, t as empty, u as EvalJournalEntry, v as EvalTrialStatus, x as ModelCatalogue, y as EvalUsage } from "./source-Bq8YhKOG.mjs";
 import { HttpClient, HttpClientResponse } from "@effect/platform";
 import { Brand, Effect, Layer, Redacted, Schema } from "effect";
 //#region ../schema/src/public/client.d.ts
@@ -78,7 +78,10 @@ declare const make: ({ apiKey, baseUrl }: ClientOptions) => Effect.Effect<{
       readonly payload: {
         readonly cases: readonly {
           readonly name: string;
-          readonly setup?: string | null | undefined;
+          readonly setup?: {
+            readonly name: string;
+            readonly source: string;
+          } | null | undefined;
           readonly source?: {
             readonly kind: "empty";
           } | {
@@ -138,7 +141,7 @@ declare const make: ({ apiKey, baseUrl }: ClientOptions) => Effect.Effect<{
           readonly prompt: string;
           readonly repoRef: string | null;
           readonly repoUrl: string | null;
-          readonly setupCommand: string | null;
+          readonly setupName: string | null;
           readonly validatorName: string | null;
           readonly verifyCommand: string | null;
           readonly workspace: string;
@@ -147,6 +150,9 @@ declare const make: ({ apiKey, baseUrl }: ClientOptions) => Effect.Effect<{
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -243,7 +249,7 @@ declare const make: ({ apiKey, baseUrl }: ClientOptions) => Effect.Effect<{
           readonly prompt: string;
           readonly repoRef: string | null;
           readonly repoUrl: string | null;
-          readonly setupCommand: string | null;
+          readonly setupName: string | null;
           readonly validatorName: string | null;
           readonly verifyCommand: string | null;
           readonly workspace: string;
@@ -252,6 +258,9 @@ declare const make: ({ apiKey, baseUrl }: ClientOptions) => Effect.Effect<{
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -341,6 +350,9 @@ declare const make: ({ apiKey, baseUrl }: ClientOptions) => Effect.Effect<{
         readonly passed: boolean;
         readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
         readonly commands: number;
+        readonly setupValue: {
+          readonly [x: string]: unknown;
+        } | null;
         readonly exitCode: number;
         readonly failedCommands: number;
         readonly filesChanged: readonly string[];
@@ -413,6 +425,9 @@ declare const make: ({ apiKey, baseUrl }: ClientOptions) => Effect.Effect<{
         readonly passed: boolean;
         readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
         readonly commands: number;
+        readonly setupValue: {
+          readonly [x: string]: unknown;
+        } | null;
         readonly exitCode: number;
         readonly failedCommands: number;
         readonly filesChanged: readonly string[];
@@ -763,7 +778,10 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
       readonly payload: {
         readonly cases: readonly {
           readonly name: string;
-          readonly setup?: string | null | undefined;
+          readonly setup?: {
+            readonly name: string;
+            readonly source: string;
+          } | null | undefined;
           readonly source?: {
             readonly kind: "empty";
           } | {
@@ -823,7 +841,7 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly prompt: string;
           readonly repoRef: string | null;
           readonly repoUrl: string | null;
-          readonly setupCommand: string | null;
+          readonly setupName: string | null;
           readonly validatorName: string | null;
           readonly verifyCommand: string | null;
           readonly workspace: string;
@@ -832,6 +850,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -928,7 +949,7 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly prompt: string;
           readonly repoRef: string | null;
           readonly repoUrl: string | null;
-          readonly setupCommand: string | null;
+          readonly setupName: string | null;
           readonly validatorName: string | null;
           readonly verifyCommand: string | null;
           readonly workspace: string;
@@ -937,6 +958,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -1026,6 +1050,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
         readonly passed: boolean;
         readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
         readonly commands: number;
+        readonly setupValue: {
+          readonly [x: string]: unknown;
+        } | null;
         readonly exitCode: number;
         readonly failedCommands: number;
         readonly filesChanged: readonly string[];
@@ -1098,6 +1125,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
         readonly passed: boolean;
         readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
         readonly commands: number;
+        readonly setupValue: {
+          readonly [x: string]: unknown;
+        } | null;
         readonly exitCode: number;
         readonly failedCommands: number;
         readonly filesChanged: readonly string[];
@@ -1446,7 +1476,10 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
       readonly payload: {
         readonly cases: readonly {
           readonly name: string;
-          readonly setup?: string | null | undefined;
+          readonly setup?: {
+            readonly name: string;
+            readonly source: string;
+          } | null | undefined;
           readonly source?: {
             readonly kind: "empty";
           } | {
@@ -1506,7 +1539,7 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly prompt: string;
           readonly repoRef: string | null;
           readonly repoUrl: string | null;
-          readonly setupCommand: string | null;
+          readonly setupName: string | null;
           readonly validatorName: string | null;
           readonly verifyCommand: string | null;
           readonly workspace: string;
@@ -1515,6 +1548,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -1611,7 +1647,7 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly prompt: string;
           readonly repoRef: string | null;
           readonly repoUrl: string | null;
-          readonly setupCommand: string | null;
+          readonly setupName: string | null;
           readonly validatorName: string | null;
           readonly verifyCommand: string | null;
           readonly workspace: string;
@@ -1620,6 +1656,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -1709,6 +1748,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
         readonly passed: boolean;
         readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
         readonly commands: number;
+        readonly setupValue: {
+          readonly [x: string]: unknown;
+        } | null;
         readonly exitCode: number;
         readonly failedCommands: number;
         readonly filesChanged: readonly string[];
@@ -1781,6 +1823,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
         readonly passed: boolean;
         readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
         readonly commands: number;
+        readonly setupValue: {
+          readonly [x: string]: unknown;
+        } | null;
         readonly exitCode: number;
         readonly failedCommands: number;
         readonly filesChanged: readonly string[];
@@ -2130,7 +2175,10 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
         readonly payload: {
           readonly cases: readonly {
             readonly name: string;
-            readonly setup?: string | null | undefined;
+            readonly setup?: {
+              readonly name: string;
+              readonly source: string;
+            } | null | undefined;
             readonly source?: {
               readonly kind: "empty";
             } | {
@@ -2190,7 +2238,7 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
             readonly prompt: string;
             readonly repoRef: string | null;
             readonly repoUrl: string | null;
-            readonly setupCommand: string | null;
+            readonly setupName: string | null;
             readonly validatorName: string | null;
             readonly verifyCommand: string | null;
             readonly workspace: string;
@@ -2199,6 +2247,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
             readonly passed: boolean;
             readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
             readonly commands: number;
+            readonly setupValue: {
+              readonly [x: string]: unknown;
+            } | null;
             readonly exitCode: number;
             readonly failedCommands: number;
             readonly filesChanged: readonly string[];
@@ -2295,7 +2346,7 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
             readonly prompt: string;
             readonly repoRef: string | null;
             readonly repoUrl: string | null;
-            readonly setupCommand: string | null;
+            readonly setupName: string | null;
             readonly validatorName: string | null;
             readonly verifyCommand: string | null;
             readonly workspace: string;
@@ -2304,6 +2355,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
             readonly passed: boolean;
             readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
             readonly commands: number;
+            readonly setupValue: {
+              readonly [x: string]: unknown;
+            } | null;
             readonly exitCode: number;
             readonly failedCommands: number;
             readonly filesChanged: readonly string[];
@@ -2393,6 +2447,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -2465,6 +2522,9 @@ declare const AnpordApi_base: import("effect/Context").TagClass<AnpordApi, "@anp
           readonly passed: boolean;
           readonly status: "passed" | "running" | "failed" | "queued" | "void" | "exceeded";
           readonly commands: number;
+          readonly setupValue: {
+            readonly [x: string]: unknown;
+          } | null;
           readonly exitCode: number;
           readonly failedCommands: number;
           readonly filesChanged: readonly string[];
@@ -2956,10 +3016,24 @@ interface ValidatorCommandResult {
   readonly stderr: string;
   readonly stdout: string;
 }
+interface ExecOptions {
+  readonly cwd?: string;
+  readonly env?: Readonly<Record<string, string>>;
+  readonly timeoutMs?: number;
+}
+interface SetupContext {
+  readonly exec: (file: string, args?: readonly string[], options?: ExecOptions) => Promise<ValidatorCommandResult>;
+  readonly exists: (path: string) => Promise<boolean>;
+  readonly readText: (path: string) => Promise<string>;
+  readonly workspace: string;
+}
+type SetupValue = Readonly<Record<string, unknown>>;
+type WorkspaceSetup = (context: SetupContext) => Promise<SetupValue | undefined> | SetupValue | undefined;
 interface ValidatorContext {
   readonly exec: (command: string) => Promise<ValidatorCommandResult>;
   readonly exists: (path: string) => Promise<boolean>;
   readonly readText: (path: string) => Promise<string>;
+  readonly setup: Readonly<Record<string, unknown>>;
 }
 interface ValidatorResult {
   readonly message?: string;
@@ -2969,7 +3043,7 @@ type Validator = (context: ValidatorContext) => boolean | ValidatorResult | Prom
 type DeclaredSource = EvalSource | string;
 interface EvalCaseBase {
   readonly name: string;
-  readonly setup?: string | null;
+  readonly setup?: WorkspaceSetup | null;
   readonly source?: DeclaredSource;
   readonly variables?: Readonly<Record<string, string>>;
 }
@@ -2997,4 +3071,4 @@ type StartEvalRequest = Parameters<EvalsSurface["start"]>[0];
 type EvalCase = StartEvalRequest["cases"][number];
 type EvalTaskRequest = StartEvalRequest["tasks"][number];
 //#endregion
-export { Anpord, AnpordApi, type AnpordClient, AnpordError, type AnpordOptions, type AnpordPromptVariables, type CacheOptions, type CatalogueModel, type ClientOptions, DEFAULT_BASE_URL, EvalAborted, EvalCase, type EvalCaseDefinition, type EvalCell, type EvalCellHistoryEntry, type EvalComparison, type EvalDefinition, type EvalDistribution, type EvalHarness, type EvalJournalEntry, type EvalProvider, type EvalRun, type EvalRunStatus, type EvalRunSummary, type EvalSource, type EvalTask, EvalTaskRequest, EvalTimeout, type EvalTrial, type EvalTrialStatus, type EvalUsage, type EvalVerdict, type EvalsSurface, type GetPromptOptions, MissingApiKey, type ModelCatalogue, type PromptFallback, type PromptMetadata, type PromptResult, type PromptsSurface, type PublicPrompt, type PublicPromptSummary, type PublicPromptWithVersions, type PublicVersion, type RerunCellRequest, StartEvalRequest, type StartedEval, type Validator, type ValidatorCommandResult, type ValidatorContext, type ValidatorResult, type WaitOptions, defineEval, empty, files, layer, make, repo };
+export { Anpord, AnpordApi, type AnpordClient, AnpordError, type AnpordOptions, type AnpordPromptVariables, type CacheOptions, type CatalogueModel, type ClientOptions, DEFAULT_BASE_URL, EvalAborted, EvalCase, type EvalCaseDefinition, type EvalCell, type EvalCellHistoryEntry, type EvalComparison, type EvalDefinition, type EvalDistribution, type EvalHarness, type EvalJournalEntry, type EvalProvider, type EvalRun, type EvalRunStatus, type EvalRunSummary, type EvalSource, type EvalTask, EvalTaskRequest, EvalTimeout, type EvalTrial, type EvalTrialStatus, type EvalUsage, type EvalVerdict, type EvalsSurface, type ExecOptions, type GetPromptOptions, MissingApiKey, type ModelCatalogue, type PromptFallback, type PromptMetadata, type PromptResult, type PromptsSurface, type PublicPrompt, type PublicPromptSummary, type PublicPromptWithVersions, type PublicVersion, type RerunCellRequest, type SetupContext, type SetupValue, StartEvalRequest, type StartedEval, type Validator, type ValidatorCommandResult, type ValidatorContext, type ValidatorResult, type WaitOptions, type WorkspaceSetup, defineEval, empty, files, layer, make, repo };

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { i as make, n as DEFAULT_BASE_URL, r as layer, t as AnpordApi } from "./client-BfhmGRQl.mjs";
+import { i as make, n as DEFAULT_BASE_URL, r as layer, t as AnpordApi } from "./client-POb2FNW1.mjs";
 import { i as tokenMatcher, n as MissingApiKey, r as asAnpordError, t as AnpordError } from "./errors-C9bQUcA3.mjs";
 import { n as files, r as repo, t as empty } from "./source-Fg1h2VNn.mjs";
 import { FetchHttpClient } from "@effect/platform";

package/dist/{source-DQ8HChlm.d.cts → source-Bq8YhKOG.d.cts}

@@ -68,6 +68,7 @@ declare const EvalJournalEntry: Schema.Union<[Schema.Struct<{
 type EvalJournalEntry = typeof EvalJournalEntry.Type;
 declare const EvalTrial: Schema.Struct<{
   commands: typeof Schema.Int;
+  setupValue: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
   exitCode: typeof Schema.Int;
   failedCommands: typeof Schema.Int;
   filesChanged: Schema.Array$<typeof Schema.String>;
@@ -175,7 +176,7 @@ declare const EvalCell: Schema.Struct<{
     prompt: typeof Schema.String;
     repoRef: Schema.NullOr<typeof Schema.String>;
     repoUrl: Schema.NullOr<typeof Schema.String>;
-    setupCommand: Schema.NullOr<typeof Schema.String>;
+    setupName: Schema.NullOr<typeof Schema.String>;
     validatorName: Schema.NullOr<typeof Schema.String>;
     verifyCommand: Schema.NullOr<typeof Schema.String>;
     workspace: typeof Schema.String;
@@ -184,6 +185,7 @@ declare const EvalCell: Schema.Struct<{
   taskIndex: typeof Schema.Int;
   trials: Schema.Array$<Schema.Struct<{
     commands: typeof Schema.Int;
+    setupValue: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
     exitCode: typeof Schema.Int;
     failedCommands: typeof Schema.Int;
     filesChanged: Schema.Array$<typeof Schema.String>;
@@ -270,7 +272,7 @@ declare const EvalRun: Schema.Struct<{
       prompt: typeof Schema.String;
       repoRef: Schema.NullOr<typeof Schema.String>;
       repoUrl: Schema.NullOr<typeof Schema.String>;
-      setupCommand: Schema.NullOr<typeof Schema.String>;
+      setupName: Schema.NullOr<typeof Schema.String>;
       validatorName: Schema.NullOr<typeof Schema.String>;
       verifyCommand: Schema.NullOr<typeof Schema.String>;
       workspace: typeof Schema.String;
@@ -279,6 +281,7 @@ declare const EvalRun: Schema.Struct<{
     taskIndex: typeof Schema.Int;
     trials: Schema.Array$<Schema.Struct<{
       commands: typeof Schema.Int;
+      setupValue: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
       exitCode: typeof Schema.Int;
       failedCommands: typeof Schema.Int;
       filesChanged: Schema.Array$<typeof Schema.String>;
@@ -387,6 +390,7 @@ declare const EvalCellHistoryEntry: Schema.Struct<{
   runId: typeof Schema.String;
   trials: Schema.Array$<Schema.Struct<{
     commands: typeof Schema.Int;
+    setupValue: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
     exitCode: typeof Schema.Int;
     failedCommands: typeof Schema.Int;
     filesChanged: Schema.Array$<typeof Schema.String>;

package/dist/{source-DQ8HChlm.d.mts → source-Bq8YhKOG.d.mts}

@@ -68,6 +68,7 @@ declare const EvalJournalEntry: Schema.Union<[Schema.Struct<{
 type EvalJournalEntry = typeof EvalJournalEntry.Type;
 declare const EvalTrial: Schema.Struct<{
   commands: typeof Schema.Int;
+  setupValue: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
   exitCode: typeof Schema.Int;
   failedCommands: typeof Schema.Int;
   filesChanged: Schema.Array$<typeof Schema.String>;
@@ -175,7 +176,7 @@ declare const EvalCell: Schema.Struct<{
     prompt: typeof Schema.String;
     repoRef: Schema.NullOr<typeof Schema.String>;
     repoUrl: Schema.NullOr<typeof Schema.String>;
-    setupCommand: Schema.NullOr<typeof Schema.String>;
+    setupName: Schema.NullOr<typeof Schema.String>;
     validatorName: Schema.NullOr<typeof Schema.String>;
     verifyCommand: Schema.NullOr<typeof Schema.String>;
     workspace: typeof Schema.String;
@@ -184,6 +185,7 @@ declare const EvalCell: Schema.Struct<{
   taskIndex: typeof Schema.Int;
   trials: Schema.Array$<Schema.Struct<{
     commands: typeof Schema.Int;
+    setupValue: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
     exitCode: typeof Schema.Int;
     failedCommands: typeof Schema.Int;
     filesChanged: Schema.Array$<typeof Schema.String>;
@@ -270,7 +272,7 @@ declare const EvalRun: Schema.Struct<{
       prompt: typeof Schema.String;
       repoRef: Schema.NullOr<typeof Schema.String>;
       repoUrl: Schema.NullOr<typeof Schema.String>;
-      setupCommand: Schema.NullOr<typeof Schema.String>;
+      setupName: Schema.NullOr<typeof Schema.String>;
       validatorName: Schema.NullOr<typeof Schema.String>;
       verifyCommand: Schema.NullOr<typeof Schema.String>;
       workspace: typeof Schema.String;
@@ -279,6 +281,7 @@ declare const EvalRun: Schema.Struct<{
     taskIndex: typeof Schema.Int;
     trials: Schema.Array$<Schema.Struct<{
       commands: typeof Schema.Int;
+      setupValue: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
       exitCode: typeof Schema.Int;
       failedCommands: typeof Schema.Int;
       filesChanged: Schema.Array$<typeof Schema.String>;
@@ -387,6 +390,7 @@ declare const EvalCellHistoryEntry: Schema.Struct<{
   runId: typeof Schema.String;
   trials: Schema.Array$<Schema.Struct<{
     commands: typeof Schema.Int;
+    setupValue: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
     exitCode: typeof Schema.Int;
     failedCommands: typeof Schema.Int;
     filesChanged: Schema.Array$<typeof Schema.String>;

package/dist/source.d.cts

@@ -1,2 +1,2 @@
-import { n as files, r as repo, t as empty } from "./source-DQ8HChlm.cjs";
+import { n as files, r as repo, t as empty } from "./source-Bq8YhKOG.cjs";
 export { empty, files, repo };

package/dist/source.d.mts

@@ -1,2 +1,2 @@
-import { n as files, r as repo, t as empty } from "./source-DQ8HChlm.mjs";
+import { n as files, r as repo, t as empty } from "./source-Bq8YhKOG.mjs";
 export { empty, files, repo };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "anpord",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "TypeScript SDK and CLI for coding agent evals and prompt management.",
   "license": "MIT",
   "type": "module",
@@ -71,12 +71,12 @@
   },
   "devDependencies": {
     "@anpord/schema": "workspace:*",
+    "@anpord/template": "workspace:*",
     "publint": "catalog:",
     "tsdown": "catalog:",
     "typescript": "catalog:"
   },
   "dependencies": {
-    "@anpord/template": "workspace:*",
     "@effect/cli": "^0.75.2",
     "@effect/platform": "^0.96.1",
     "@effect/platform-node": "^0.107.0",