anpord 0.1.11 → 0.1.13

package/dist/{types-oaMF6-E1.d.mts → types-Codjaq-o.d.mts}

@@ -1,5 +1,6 @@
-import "./errors-BK3f7Rdr.mjs";
-import { f as EvalSource, o as EvalHarness } from "./evals-BEgEmfiO.mjs";
+import "./errors-aGrd21jy.mjs";
+import { t as EvalJudge } from "./eval-judges-DPPUftbh.mjs";
+import { f as EvalSource, o as EvalHarness } from "./evals-B6aT3xgH.mjs";
 import { t as McpCall } from "./calls-DYqBi5FA.mjs";
 import { n as McpServerDefinition } from "./define-HK0eeSNn.mjs";
 import { t as CliCall } from "./calls-Drj_dZpu.mjs";
@@ -8,31 +9,89 @@ import { HttpApiEndpoint, HttpApiGroup, HttpApiMiddleware, HttpApiSecurity } fro
 import { Context, Schema } from "effect";
 //#region ../schema/src/public/evals-api.d.ts
 declare const PublicStartEvalRequest: Schema.Struct<{
+  trigger: Schema.optional<Schema.Struct<{
+    source: Schema.Literal<["dashboard", "api", "cli", "ci", "mcp"]>;
+    url: Schema.optional<Schema.filter<Schema.filter<typeof Schema.String>>>;
+  }>>;
   cases: Schema.filter<Schema.filter<Schema.Array$<Schema.refine<{
-    readonly cache?: {
-      readonly key: string;
-      readonly path: string;
-    } | undefined;
-    readonly name: string;
-    readonly prepare?: {
-      readonly name: string;
-      readonly source: string;
-    } | null | undefined;
     readonly source?: {
       readonly kind: "empty";
     } | {
+      readonly url: string;
       readonly kind: "repo";
       readonly ref: string | null;
-      readonly url: string;
     } | {
       readonly kind: "files";
       readonly files: {
         readonly [x: string]: string;
       };
     } | undefined;
-    readonly validator?: {
+    readonly cache?: {
+      readonly key: string;
+      readonly path: string;
+    } | undefined;
+    readonly name: string;
+    readonly prepare?: {
+      readonly source: string;
       readonly name: string;
+    } | null | undefined;
+    readonly validator?: ({
       readonly source: string;
+      readonly name: string;
+      readonly capture?: boolean | undefined;
+      readonly manifest?: readonly {
+        readonly name: string;
+        readonly index: number;
+      }[] | undefined;
+    } & {
+      readonly sourceFiles?: readonly {
+        readonly path: string;
+        readonly content: string;
+      }[] | undefined;
+    }) | {
+      readonly name: string;
+      readonly kind: "judged";
+      readonly capture?: boolean | undefined;
+      readonly sourceFiles?: readonly {
+        readonly path: string;
+        readonly content: string;
+      }[] | undefined;
+      readonly checks: readonly {
+        readonly source: string;
+        readonly name: string;
+        readonly capture?: boolean | undefined;
+        readonly manifest?: readonly {
+          readonly name: string;
+          readonly index: number;
+        }[] | undefined;
+      }[];
+      readonly judges: readonly ({
+        readonly harness?: undefined;
+        readonly name: string;
+        readonly kind: "judge";
+        readonly provider: "openai";
+        readonly model: string;
+        readonly rubric: string;
+        readonly expected?: string | undefined;
+        readonly choices: {
+          readonly [x: string]: number;
+        };
+        readonly threshold: number;
+        readonly timeoutMs: number;
+      } | {
+        readonly harness: "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor";
+        readonly name: string;
+        readonly kind: "judge";
+        readonly provider?: undefined;
+        readonly model: string;
+        readonly rubric: string;
+        readonly expected?: string | undefined;
+        readonly choices: {
+          readonly [x: string]: number;
+        };
+        readonly threshold: number;
+        readonly timeoutMs: number;
+      })[];
     } | null | undefined;
     readonly variables?: {
       readonly [x: string]: string;
@@ -58,10 +117,68 @@ declare const PublicStartEvalRequest: Schema.Struct<{
       files: Schema.Record$<typeof Schema.String, typeof Schema.String>;
       kind: Schema.Literal<["files"]>;
     }>]>>;
-    validator: Schema.optional<Schema.NullOr<Schema.Struct<{
+    validator: Schema.optional<Schema.NullOr<Schema.Union<[Schema.extend<Schema.Struct<{
+      capture: Schema.optional<typeof Schema.Boolean>;
+      manifest: Schema.optional<Schema.filter<Schema.filter<Schema.filter<Schema.Array$<Schema.Struct<{
+        index: Schema.refine<number, typeof Schema.NonNegative>;
+        name: Schema.filter<typeof Schema.String>;
+      }>>>>>>;
       name: Schema.filter<Schema.filter<typeof Schema.String>>;
       source: Schema.filter<Schema.filter<typeof Schema.String>>;
-    }>>>;
+    }>, Schema.Struct<{
+      sourceFiles: Schema.optional<Schema.filter<Schema.filter<Schema.filter<Schema.Array$<Schema.Struct<{
+        path: Schema.filter<Schema.filter<typeof Schema.String>>;
+        content: Schema.filter<typeof Schema.String>;
+      }>>>>>>;
+    }>>, Schema.filter<Schema.Struct<{
+      kind: Schema.Literal<["judged"]>;
+      capture: Schema.optional<typeof Schema.Boolean>;
+      name: Schema.filter<Schema.filter<typeof Schema.String>>;
+      checks: Schema.filter<Schema.Array$<Schema.Struct<{
+        capture: Schema.optional<typeof Schema.Boolean>;
+        manifest: Schema.optional<Schema.filter<Schema.filter<Schema.filter<Schema.Array$<Schema.Struct<{
+          index: Schema.refine<number, typeof Schema.NonNegative>;
+          name: Schema.filter<typeof Schema.String>;
+        }>>>>>>;
+        name: Schema.filter<Schema.filter<typeof Schema.String>>;
+        source: Schema.filter<Schema.filter<typeof Schema.String>>;
+      }>>>;
+      judges: Schema.filter<Schema.filter<Schema.Array$<Schema.Union<[Schema.Struct<{
+        provider: Schema.Literal<["openai"]>;
+        harness: Schema.optional<typeof Schema.Never>;
+        kind: Schema.Literal<["judge"]>;
+        name: Schema.filter<Schema.filter<typeof Schema.String>>;
+        model: Schema.filter<Schema.filter<typeof Schema.String>>;
+        rubric: Schema.filter<Schema.filter<typeof Schema.String>>;
+        expected: Schema.optional<Schema.filter<typeof Schema.String>>;
+        choices: Schema.filter<Schema.Record$<Schema.filter<typeof Schema.String>, Schema.filter<typeof Schema.Number>>>;
+        threshold: Schema.optionalWith<Schema.filter<typeof Schema.Number>, {
+          default: () => number;
+        }>;
+        timeoutMs: Schema.optionalWith<Schema.filter<typeof Schema.Int>, {
+          default: () => number;
+        }>;
+      }>, Schema.Struct<{
+        provider: Schema.optional<typeof Schema.Never>;
+        harness: Schema.SchemaClass<"codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor", "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor", never>;
+        kind: Schema.Literal<["judge"]>;
+        name: Schema.filter<Schema.filter<typeof Schema.String>>;
+        model: Schema.filter<Schema.filter<typeof Schema.String>>;
+        rubric: Schema.filter<Schema.filter<typeof Schema.String>>;
+        expected: Schema.optional<Schema.filter<typeof Schema.String>>;
+        choices: Schema.filter<Schema.Record$<Schema.filter<typeof Schema.String>, Schema.filter<typeof Schema.Number>>>;
+        threshold: Schema.optionalWith<Schema.filter<typeof Schema.Number>, {
+          default: () => number;
+        }>;
+        timeoutMs: Schema.optionalWith<Schema.filter<typeof Schema.Int>, {
+          default: () => number;
+        }>;
+      }>]>>>>;
+      sourceFiles: Schema.optional<Schema.filter<Schema.filter<Schema.filter<Schema.Array$<Schema.Struct<{
+        path: Schema.filter<Schema.filter<typeof Schema.String>>;
+        content: Schema.filter<typeof Schema.String>;
+      }>>>>>>;
+    }>>]>>>;
     variables: Schema.optional<Schema.Record$<typeof Schema.String, Schema.refine<string, typeof Schema.String>>>;
     verify: Schema.NullOr<Schema.refine<string, typeof Schema.String>>;
   }>>>>>;
@@ -82,7 +199,7 @@ declare const PublicStartEvalRequest: Schema.Struct<{
       readonly run?: string | undefined;
       readonly systemPrompt?: string | undefined;
     } | undefined;
-    readonly provider: "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel";
+    readonly sandbox?: "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel" | undefined;
   }, Schema.Struct<{
     harness: Schema.Literal<["codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command"]>;
     model: Schema.filter<typeof Schema.String>;
@@ -94,7 +211,7 @@ declare const PublicStartEvalRequest: Schema.Struct<{
       run: Schema.optional<typeof Schema.String>;
       systemPrompt: Schema.optional<typeof Schema.String>;
     }>>;
-    provider: Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>;
+    sandbox: Schema.optional<Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>>;
   }>>>>>;
   trials: Schema.filter<typeof Schema.Int>;
 }>;
@@ -102,13 +219,7 @@ type PublicStartEvalRequest = typeof PublicStartEvalRequest.Type;
 //#endregion
 //#region src/evals/types.d.ts
 type EvalTaskRequest = PublicStartEvalRequest["tasks"][number];
-/**
- * A profile directory beside the eval file.
- *
- * `dir` resolves against the eval file. Files under `home/` and `workspace/`
- * are shipped into the sandbox; an optional `profile.json` names a system
- * prompt, env, and for the command harness the install and run steps.
- */
+/** A profile directory beside the eval file; `dir` resolves against it. */
 interface ProfileRef {
   readonly dir: string;
   readonly name: string;
@@ -120,7 +231,7 @@ type HarnessRef = EvalHarness | {
 interface EvalTaskDefinition {
   readonly harness: HarnessRef;
   readonly model: EvalTaskRequest["model"];
-  readonly provider: EvalTaskRequest["provider"];
+  readonly sandbox?: EvalTaskRequest["sandbox"];
 }
 interface CommandResult {
   readonly exitCode: number;
@@ -132,27 +243,16 @@ interface ExecOptions {
   readonly env?: Readonly<Record<string, string>>;
   readonly timeoutMs?: number;
 }
-/**
- * A directory worth keeping between runs of the same case.
- *
- * Declared on the case rather than returned by the prepare, because a restore
- * has to happen before the prepare runs: something that only exists once it has
- * finished cannot say where to look. This is the shape CI caches use for the
- * same reason -- actions/cache names its path and key before the step, not
- * after.
- *
- * The key should name everything the contents depend on, a lockfile hash above
- * all, because entries are write-once: a key that already holds an entry keeps
- * it, which is what makes two runs preparing at once safe.
- */
+/** Entries are write-once, so the key must name everything the contents
+ * depend on -- a lockfile hash above all. */
 interface CaseCache {
   readonly key: string;
   /** Relative to the workspace. */
   readonly path: string;
 }
 interface PrepareContext {
-  /** True when the runner restored a cached directory before this ran, so a
-   * prepare can skip the work that produced it. */
+  /** True when the runner restored a cached directory, so a prepare can skip
+   * the work that produced it. */
   readonly cached: boolean;
   readonly exec: (file: string, args?: readonly string[], options?: ExecOptions) => Promise<CommandResult>;
   readonly exists: (path: string) => Promise<boolean>;
@@ -162,7 +262,7 @@ interface PrepareContext {
 type PrepareValue = Readonly<Record<string, unknown>>;
 type Prepare = (context: PrepareContext) => Promise<PrepareValue | undefined> | PrepareValue | undefined;
 interface ValidatorContext {
-  /** The agent's final reply, empty when it said nothing. */
+  /** Empty when the agent said nothing. */
   readonly answer: () => Promise<string>;
   readonly cli: {
     readonly calls: (cli?: string) => Promise<readonly CliCall[]>;
@@ -174,7 +274,7 @@ interface ValidatorContext {
   };
   readonly prepared: Readonly<Record<string, unknown>>;
   readonly readText: (path: string) => Promise<string>;
-  /** Every reply the agent made, oldest first, separated by a blank line. */
+  /** Every reply, oldest first, separated by a blank line. */
   readonly transcript: () => Promise<string>;
 }
 interface ValidatorResult {
@@ -184,8 +284,7 @@ interface ValidatorResult {
 type Validator = (context: ValidatorContext) => boolean | ValidatorResult | Promise<boolean | ValidatorResult>;
 type DeclaredSource = EvalSource | string;
 interface EvalCaseBase {
-  /** What a prepare builds that is worth keeping for the next run of this
-   * case. Restored before it runs, and saved after it succeeds. */
+  /** Restored before the prepare runs, saved after it succeeds. */
   readonly cache?: CaseCache;
   readonly name: string;
   readonly prepare?: Prepare | null;
@@ -193,13 +292,15 @@ interface EvalCaseBase {
   readonly variables?: Readonly<Record<string, string>>;
 }
 type EvalCaseDefinition = EvalCaseBase & ({
-  readonly validate: Validator;
+  readonly validate: Validator | EvalJudge | readonly (Validator | EvalJudge)[];
   readonly verify?: never;
 } | {
   readonly validate?: never;
   readonly verify: string;
 });
 interface EvalDefinition {
+  readonly captureSource?: boolean;
+  readonly captureValidation?: boolean;
   readonly cases: readonly EvalCaseDefinition[];
   readonly cli?: readonly CliDefinition[];
   readonly mcp?: readonly McpServerDefinition[];

package/dist/{types-H9L450Iu.d.cts → types-ISsteq95.d.cts}

@@ -1,5 +1,6 @@
-import "./errors-BK3f7Rdr.cjs";
-import { f as EvalSource, o as EvalHarness } from "./evals-BEgEmfiO.cjs";
+import "./errors-aGrd21jy.cjs";
+import { t as EvalJudge } from "./eval-judges-DPPUftbh.cjs";
+import { f as EvalSource, o as EvalHarness } from "./evals-B6aT3xgH.cjs";
 import { t as McpCall } from "./calls-DYqBi5FA.cjs";
 import { n as McpServerDefinition } from "./define-HK0eeSNn.cjs";
 import { t as CliCall } from "./calls-Drj_dZpu.cjs";
@@ -8,31 +9,89 @@ import { Schema } from "effect";
 import "@effect/platform";
 //#region ../schema/src/public/evals-api.d.ts
 declare const PublicStartEvalRequest: Schema.Struct<{
+  trigger: Schema.optional<Schema.Struct<{
+    source: Schema.Literal<["dashboard", "api", "cli", "ci", "mcp"]>;
+    url: Schema.optional<Schema.filter<Schema.filter<typeof Schema.String>>>;
+  }>>;
   cases: Schema.filter<Schema.filter<Schema.Array$<Schema.refine<{
-    readonly cache?: {
-      readonly key: string;
-      readonly path: string;
-    } | undefined;
-    readonly name: string;
-    readonly prepare?: {
-      readonly name: string;
-      readonly source: string;
-    } | null | undefined;
     readonly source?: {
       readonly kind: "empty";
     } | {
+      readonly url: string;
       readonly kind: "repo";
       readonly ref: string | null;
-      readonly url: string;
     } | {
       readonly kind: "files";
       readonly files: {
         readonly [x: string]: string;
       };
     } | undefined;
-    readonly validator?: {
+    readonly cache?: {
+      readonly key: string;
+      readonly path: string;
+    } | undefined;
+    readonly name: string;
+    readonly prepare?: {
+      readonly source: string;
       readonly name: string;
+    } | null | undefined;
+    readonly validator?: ({
       readonly source: string;
+      readonly name: string;
+      readonly capture?: boolean | undefined;
+      readonly manifest?: readonly {
+        readonly name: string;
+        readonly index: number;
+      }[] | undefined;
+    } & {
+      readonly sourceFiles?: readonly {
+        readonly path: string;
+        readonly content: string;
+      }[] | undefined;
+    }) | {
+      readonly name: string;
+      readonly kind: "judged";
+      readonly capture?: boolean | undefined;
+      readonly sourceFiles?: readonly {
+        readonly path: string;
+        readonly content: string;
+      }[] | undefined;
+      readonly checks: readonly {
+        readonly source: string;
+        readonly name: string;
+        readonly capture?: boolean | undefined;
+        readonly manifest?: readonly {
+          readonly name: string;
+          readonly index: number;
+        }[] | undefined;
+      }[];
+      readonly judges: readonly ({
+        readonly harness?: undefined;
+        readonly name: string;
+        readonly kind: "judge";
+        readonly provider: "openai";
+        readonly model: string;
+        readonly rubric: string;
+        readonly expected?: string | undefined;
+        readonly choices: {
+          readonly [x: string]: number;
+        };
+        readonly threshold: number;
+        readonly timeoutMs: number;
+      } | {
+        readonly harness: "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor";
+        readonly name: string;
+        readonly kind: "judge";
+        readonly provider?: undefined;
+        readonly model: string;
+        readonly rubric: string;
+        readonly expected?: string | undefined;
+        readonly choices: {
+          readonly [x: string]: number;
+        };
+        readonly threshold: number;
+        readonly timeoutMs: number;
+      })[];
     } | null | undefined;
     readonly variables?: {
       readonly [x: string]: string;
@@ -58,10 +117,68 @@ declare const PublicStartEvalRequest: Schema.Struct<{
       files: Schema.Record$<typeof Schema.String, typeof Schema.String>;
       kind: Schema.Literal<["files"]>;
     }>]>>;
-    validator: Schema.optional<Schema.NullOr<Schema.Struct<{
+    validator: Schema.optional<Schema.NullOr<Schema.Union<[Schema.extend<Schema.Struct<{
+      capture: Schema.optional<typeof Schema.Boolean>;
+      manifest: Schema.optional<Schema.filter<Schema.filter<Schema.filter<Schema.Array$<Schema.Struct<{
+        index: Schema.refine<number, typeof Schema.NonNegative>;
+        name: Schema.filter<typeof Schema.String>;
+      }>>>>>>;
       name: Schema.filter<Schema.filter<typeof Schema.String>>;
       source: Schema.filter<Schema.filter<typeof Schema.String>>;
-    }>>>;
+    }>, Schema.Struct<{
+      sourceFiles: Schema.optional<Schema.filter<Schema.filter<Schema.filter<Schema.Array$<Schema.Struct<{
+        path: Schema.filter<Schema.filter<typeof Schema.String>>;
+        content: Schema.filter<typeof Schema.String>;
+      }>>>>>>;
+    }>>, Schema.filter<Schema.Struct<{
+      kind: Schema.Literal<["judged"]>;
+      capture: Schema.optional<typeof Schema.Boolean>;
+      name: Schema.filter<Schema.filter<typeof Schema.String>>;
+      checks: Schema.filter<Schema.Array$<Schema.Struct<{
+        capture: Schema.optional<typeof Schema.Boolean>;
+        manifest: Schema.optional<Schema.filter<Schema.filter<Schema.filter<Schema.Array$<Schema.Struct<{
+          index: Schema.refine<number, typeof Schema.NonNegative>;
+          name: Schema.filter<typeof Schema.String>;
+        }>>>>>>;
+        name: Schema.filter<Schema.filter<typeof Schema.String>>;
+        source: Schema.filter<Schema.filter<typeof Schema.String>>;
+      }>>>;
+      judges: Schema.filter<Schema.filter<Schema.Array$<Schema.Union<[Schema.Struct<{
+        provider: Schema.Literal<["openai"]>;
+        harness: Schema.optional<typeof Schema.Never>;
+        kind: Schema.Literal<["judge"]>;
+        name: Schema.filter<Schema.filter<typeof Schema.String>>;
+        model: Schema.filter<Schema.filter<typeof Schema.String>>;
+        rubric: Schema.filter<Schema.filter<typeof Schema.String>>;
+        expected: Schema.optional<Schema.filter<typeof Schema.String>>;
+        choices: Schema.filter<Schema.Record$<Schema.filter<typeof Schema.String>, Schema.filter<typeof Schema.Number>>>;
+        threshold: Schema.optionalWith<Schema.filter<typeof Schema.Number>, {
+          default: () => number;
+        }>;
+        timeoutMs: Schema.optionalWith<Schema.filter<typeof Schema.Int>, {
+          default: () => number;
+        }>;
+      }>, Schema.Struct<{
+        provider: Schema.optional<typeof Schema.Never>;
+        harness: Schema.SchemaClass<"codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor", "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor", never>;
+        kind: Schema.Literal<["judge"]>;
+        name: Schema.filter<Schema.filter<typeof Schema.String>>;
+        model: Schema.filter<Schema.filter<typeof Schema.String>>;
+        rubric: Schema.filter<Schema.filter<typeof Schema.String>>;
+        expected: Schema.optional<Schema.filter<typeof Schema.String>>;
+        choices: Schema.filter<Schema.Record$<Schema.filter<typeof Schema.String>, Schema.filter<typeof Schema.Number>>>;
+        threshold: Schema.optionalWith<Schema.filter<typeof Schema.Number>, {
+          default: () => number;
+        }>;
+        timeoutMs: Schema.optionalWith<Schema.filter<typeof Schema.Int>, {
+          default: () => number;
+        }>;
+      }>]>>>>;
+      sourceFiles: Schema.optional<Schema.filter<Schema.filter<Schema.filter<Schema.Array$<Schema.Struct<{
+        path: Schema.filter<Schema.filter<typeof Schema.String>>;
+        content: Schema.filter<typeof Schema.String>;
+      }>>>>>>;
+    }>>]>>>;
     variables: Schema.optional<Schema.Record$<typeof Schema.String, Schema.refine<string, typeof Schema.String>>>;
     verify: Schema.NullOr<Schema.refine<string, typeof Schema.String>>;
   }>>>>>;
@@ -82,7 +199,7 @@ declare const PublicStartEvalRequest: Schema.Struct<{
       readonly run?: string | undefined;
       readonly systemPrompt?: string | undefined;
     } | undefined;
-    readonly provider: "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel";
+    readonly sandbox?: "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel" | undefined;
   }, Schema.Struct<{
     harness: Schema.Literal<["codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command"]>;
     model: Schema.filter<typeof Schema.String>;
@@ -94,7 +211,7 @@ declare const PublicStartEvalRequest: Schema.Struct<{
       run: Schema.optional<typeof Schema.String>;
       systemPrompt: Schema.optional<typeof Schema.String>;
     }>>;
-    provider: Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>;
+    sandbox: Schema.optional<Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>>;
   }>>>>>;
   trials: Schema.filter<typeof Schema.Int>;
 }>;
@@ -102,13 +219,7 @@ type PublicStartEvalRequest = typeof PublicStartEvalRequest.Type;
 //#endregion
 //#region src/evals/types.d.ts
 type EvalTaskRequest = PublicStartEvalRequest["tasks"][number];
-/**
- * A profile directory beside the eval file.
- *
- * `dir` resolves against the eval file. Files under `home/` and `workspace/`
- * are shipped into the sandbox; an optional `profile.json` names a system
- * prompt, env, and for the command harness the install and run steps.
- */
+/** A profile directory beside the eval file; `dir` resolves against it. */
 interface ProfileRef {
   readonly dir: string;
   readonly name: string;
@@ -120,7 +231,7 @@ type HarnessRef = EvalHarness | {
 interface EvalTaskDefinition {
   readonly harness: HarnessRef;
   readonly model: EvalTaskRequest["model"];
-  readonly provider: EvalTaskRequest["provider"];
+  readonly sandbox?: EvalTaskRequest["sandbox"];
 }
 interface CommandResult {
   readonly exitCode: number;
@@ -132,27 +243,16 @@ interface ExecOptions {
   readonly env?: Readonly<Record<string, string>>;
   readonly timeoutMs?: number;
 }
-/**
- * A directory worth keeping between runs of the same case.
- *
- * Declared on the case rather than returned by the prepare, because a restore
- * has to happen before the prepare runs: something that only exists once it has
- * finished cannot say where to look. This is the shape CI caches use for the
- * same reason -- actions/cache names its path and key before the step, not
- * after.
- *
- * The key should name everything the contents depend on, a lockfile hash above
- * all, because entries are write-once: a key that already holds an entry keeps
- * it, which is what makes two runs preparing at once safe.
- */
+/** Entries are write-once, so the key must name everything the contents
+ * depend on -- a lockfile hash above all. */
 interface CaseCache {
   readonly key: string;
   /** Relative to the workspace. */
   readonly path: string;
 }
 interface PrepareContext {
-  /** True when the runner restored a cached directory before this ran, so a
-   * prepare can skip the work that produced it. */
+  /** True when the runner restored a cached directory, so a prepare can skip
+   * the work that produced it. */
   readonly cached: boolean;
   readonly exec: (file: string, args?: readonly string[], options?: ExecOptions) => Promise<CommandResult>;
   readonly exists: (path: string) => Promise<boolean>;
@@ -162,7 +262,7 @@ interface PrepareContext {
 type PrepareValue = Readonly<Record<string, unknown>>;
 type Prepare = (context: PrepareContext) => Promise<PrepareValue | undefined> | PrepareValue | undefined;
 interface ValidatorContext {
-  /** The agent's final reply, empty when it said nothing. */
+  /** Empty when the agent said nothing. */
   readonly answer: () => Promise<string>;
   readonly cli: {
     readonly calls: (cli?: string) => Promise<readonly CliCall[]>;
@@ -174,7 +274,7 @@ interface ValidatorContext {
   };
   readonly prepared: Readonly<Record<string, unknown>>;
   readonly readText: (path: string) => Promise<string>;
-  /** Every reply the agent made, oldest first, separated by a blank line. */
+  /** Every reply, oldest first, separated by a blank line. */
   readonly transcript: () => Promise<string>;
 }
 interface ValidatorResult {
@@ -184,8 +284,7 @@ interface ValidatorResult {
 type Validator = (context: ValidatorContext) => boolean | ValidatorResult | Promise<boolean | ValidatorResult>;
 type DeclaredSource = EvalSource | string;
 interface EvalCaseBase {
-  /** What a prepare builds that is worth keeping for the next run of this
-   * case. Restored before it runs, and saved after it succeeds. */
+  /** Restored before the prepare runs, saved after it succeeds. */
   readonly cache?: CaseCache;
   readonly name: string;
   readonly prepare?: Prepare | null;
@@ -193,13 +292,15 @@ interface EvalCaseBase {
   readonly variables?: Readonly<Record<string, string>>;
 }
 type EvalCaseDefinition = EvalCaseBase & ({
-  readonly validate: Validator;
+  readonly validate: Validator | EvalJudge | readonly (Validator | EvalJudge)[];
   readonly verify?: never;
 } | {
   readonly validate?: never;
   readonly verify: string;
 });
 interface EvalDefinition {
+  readonly captureSource?: boolean;
+  readonly captureValidation?: boolean;
   readonly cases: readonly EvalCaseDefinition[];
   readonly cli?: readonly CliDefinition[];
   readonly mcp?: readonly McpServerDefinition[];