@leostera/evalkit 0.8.0 → 0.10.0

package/dist/index.d.mts

@@ -1,6 +1,6 @@
 import * as Schema from "effect/Schema";
 //#region ../core/src/identity.d.ts
-export declare const ResourceKindSchema: Schema.Literal<["run", "trial", "artifact"]>;
+export declare const ResourceKindSchema: Schema.Literal<["run", "artifact"]>;
 export type ResourceKind = typeof ResourceKindSchema.Type;
 /** Human-authored project-local identifier; never a generated resource URI. */
 export declare function authorId(value: string): string;
@@ -26,14 +26,14 @@ export type DatasetFileSource = string | Uint8Array | {
   file: string;
 };
 /**
- * Files for the trial workspace: a directory relative to the project root, or
+ * Files for the run workspace: a directory relative to the project root, or
  * a map from workspace path (below the dataset's mount) to its content.
  */
 export type DatasetFiles = string | Readonly<Record<string, DatasetFileSource>>;
-/** What one trial receives from a dataset. */
+/** What one run receives from a dataset. */
 export type DatasetContent = {
   files?: DatasetFiles;
-  /** Any JSON-convertible value; converted once when the trial starts. */
+  /** Any JSON-convertible value; converted once when the run starts. */
   value?: unknown;
 };
 export type DatasetLoadContext = {
@@ -41,12 +41,12 @@ export type DatasetLoadContext = {
   root: string;
   signal: AbortSignal;
 };
-export type DatasetTrialContext = {
+export type DatasetRunContext = {
   evalId: string;
   runId: string;
-  trialId: string;
-  trialIndex: number;
-  /** Per-trial random seed, so repeated trials can choose different data. */
+  /** Set when the run is one of several repeats launched together. */
+  batch?: Batch;
+  /** Per-run random seed, so repeated runs can choose different data. */
   seed: number;
   /** The matrix cell's parameters, or `{}`. */
   parameters: JsonObject;
@@ -70,10 +70,10 @@ export type StaticDatasetDefinition = DatasetCommon & {
   onRun?: never;
 };
 export type ComputedDatasetDefinition<TLoaded extends DatasetContent> = DatasetCommon & {
-  /** Once per process; the successful result is cached and shared by every trial. */
+  /** Once per process; the successful result is cached and shared by every run. */
   onLoad: (load: DatasetLoadContext) => TLoaded | Promise<TLoaded>;
-  /** Once per trial; defaults to using the loaded content as is. */
-  onRun?: (loaded: TLoaded, trial: DatasetTrialContext) => DatasetContent | Promise<DatasetContent>;
+  /** Once per run; defaults to using the loaded content as is. */
+  onRun?: (loaded: TLoaded, run: DatasetRunContext) => DatasetContent | Promise<DatasetContent>;
   files?: never;
   value?: never;
   valueFile?: never;
@@ -86,7 +86,7 @@ export type DatasetSource = {
 } | {
   kind: 'computed';
   onLoad: (load: DatasetLoadContext) => DatasetContent | Promise<DatasetContent>;
-  onRun?: (loaded: DatasetContent, trial: DatasetTrialContext) => DatasetContent | Promise<DatasetContent>;
+  onRun?: (loaded: DatasetContent, run: DatasetRunContext) => DatasetContent | Promise<DatasetContent>;
 };
 /** A checked dataset, as returned by `defineDataset`. Evals accept only these. */
 export type Dataset = {
@@ -101,10 +101,10 @@ export type Dataset = {
 /** Normalizes a relative POSIX path and rejects absolute or escaping ones. */
 export declare function relativeDatasetPath(value: string, label: string): string;
 /**
- * Declares named data that scenarios use: files placed in each trial's
+ * Declares named data that scenarios use: files placed in each run's
  * workspace, a value for the agent and scorers, or both. Static datasets give
  * `files`, `value` or `valueFile`; computed datasets give `onLoad` (cached per
- * process) and optionally `onRun` (per trial).
+ * process) and optionally `onRun` (per run).
  */
 export declare function defineDataset<TLoaded extends DatasetContent>(definition: StaticDatasetDefinition | ComputedDatasetDefinition<TLoaded>): Dataset;
 export declare function isDataset(value: unknown): value is Dataset;
@@ -144,6 +144,8 @@ export type AutEventInput = {
   timestamp?: string;
   latencyMs?: number;
   usage?: Usage;
+  /** The model that answered this turn, when the agent knows it. */
+  model?: string;
 } | {
   kind: 'completed';
   output?: unknown;
@@ -165,6 +167,8 @@ export type TurnResult = string | {
   reply?: unknown;
   toolCalls?: readonly TurnToolCall[];
   usage?: Usage;
+  /** The model that answered, recorded with the usage. */
+  model?: string;
 };
 /** Converts an adapter event into a recorded `AutEvent`. */
 export declare function normalizeAutEvent(input: AutEventInput, now: () => Date): AutEvent;
@@ -174,6 +178,89 @@ export declare function normalizeAutEvent(input: AutEventInput, now: () => Date)
  */
 export declare function turnResultEvents(result: TurnResult, turn: number, now: () => Date): AutEvent[];
 //#endregion
+//#region ../core/src/judge.d.ts
+/**
+ * How a judge must answer, and how EvalKit turns the answer into a score.
+ *
+ * - `verdict` (the default): a value from 0 to 1 and a reason, as JSON.
+ * - `choices`: one of a fixed set of answers, such as A/B/C, each worth a value.
+ * - `criteria`: a value from 0 to 1 per criterion; the score is their weighted mean.
+ */
+type JudgeAnswer = {
+  kind: 'verdict';
+} | {
+  kind: 'choices';
+  choices: JudgeChoice[];
+} | {
+  kind: 'criteria';
+  criteria: JudgeCriterion[];
+};
+/** One answer a `choices` judge can give. Keys are matched case-insensitively. */
+type JudgeChoice = {
+  key: string;
+  label: string;
+  value: number;
+};
+/** One thing a `criteria` judge scores. */
+type JudgeCriterion = {
+  name: string;
+  weight: number;
+  description?: string;
+};
+/**
+ * A judge that answers with one of these choices, in this order:
+ * `choices({ A: ['Accurate and helpful', 1], B: ['Partly right', 0.5], C: ['Wrong', 0] })`.
+ */
+export declare function choices(options: Record<string, readonly [label: string, value: number]>): JudgeAnswer;
+/**
+ * Criteria by name: a number is the weight; `{ weight, description }` also
+ * says what the criterion checks. Weights default to 1.
+ */
+type CriteriaOptions = Record<string, number | {
+  weight?: number;
+  description?: string;
+}>;
+/**
+ * A judge that scores each criterion from 0 to 1; the score is the weighted
+ * mean. A number is the weight; `{ weight, description }` also says what the
+ * criterion means. Weights default to 1.
+ */
+export declare function criteria(options: CriteriaOptions): JudgeAnswer;
+/**
+ * One criterion's result, as a scorer returns it and as results record it:
+ * a value with what it saw, or `reached: false` when the check never got to
+ * run, because something before it failed (say, the app would not start).
+ * A criterion not reached counts as 0; the score's `explanation` says why,
+ * once, instead of every criterion repeating it.
+ */
+type CriterionResult = ReachedCriterion | UnreachedCriterion;
+type ReachedCriterion = {
+  name: string;
+  /** From 0 to 1. */
+  value: number;
+  /** Default 1, or the declared weight. */
+  weight?: number;
+  /** Why this value. */
+  explanation?: string;
+  /** What the check observed, as plain data. */
+  evidence?: JsonValue;
+};
+type UnreachedCriterion = {
+  name: string;
+  reached: false;
+  weight?: number;
+  explanation?: string;
+};
+/** A criterion as a scorer returns it: its evidence may be any plain value. */
+type CriterionInput = (Omit<ReachedCriterion, 'evidence'> & {
+  evidence?: unknown;
+}) | UnreachedCriterion;
+/**
+ * The pass mark in words: `only 1`, `≥ 0.5`, or for choices the ones that
+ * pass, such as `A or B`.
+ */
+export declare function passMark(passAt: number | undefined, answer?: JudgeAnswer): string;
+//#endregion
 //#region ../core/src/catalog.d.ts
 /**
  * What the dashboard's Library shows about each definition. Only definition
@@ -184,6 +271,8 @@ export declare function turnResultEvents(result: TurnResult, turn: number, now:
 export type CatalogOptions = {
   /** Project root; definition sites are reported relative to it, never absolute. */
   root?: string;
+  /** The project's default judge agent (`judge` in evalkit.config.ts). */
+  judge?: AutAdapter;
 };
 export type EvalCatalogAgent = {
   name?: string;
@@ -235,14 +324,24 @@ export type EvalCatalogDataset = {
 export type EvalCatalogScorer = {
   name: string;
   kind: ScoringRule['kind'];
+  /** What it checks, in the author's words. */
+  description?: string;
   passAt?: number;
   supportsPartial?: boolean;
   /** Judges only. */
   rubric?: string;
+  /** Judges only: the agent that judges (its own, or the project's default). */
+  agent?: EvalCatalogAgent;
+  /** Judges only: the names of the inputs it sees. */
+  inputs?: string[];
+  /** Judges only: how it answers. */
+  answer?: JudgeAnswer;
+  /** Judges only: the prompt it receives, with `{input}` placeholders. */
+  prompt?: string;
   definedAt?: string;
 };
 export type EvalCatalogStep =
-/** `message` is absent when it is computed for each trial. */
+/** `message` is absent when it is computed for each run. */
 {
   kind: 'user';
   message?: string;
@@ -268,9 +367,9 @@ export type EvalCatalogEntry = {
   /** The separate agent that runs judge rules, if any. */
   judgeAgent?: EvalCatalogAgent;
   datasets: EvalCatalogDataset[];
-  trialCount: number;
+  /** Runs started each time the scenario runs (`repeat`). */
+  repeat: number;
   policy: {
-    trials: number;
     timeoutMs?: number;
     failfast?: boolean;
   };
@@ -339,11 +438,11 @@ type EvalkitConfig = {
   };
   execution?: {
     concurrency?: number;
-    trials?: number;
+    repeat?: number;
     maxCells?: number;
   };
   reportDir?: string;
-  /** The candidate files saved with each trial's results. */
+  /** The candidate files saved with each run's results. */
   artifacts?: {
     /**
      * Paths to leave out, as globs relative to the candidate workspace; a
@@ -352,14 +451,33 @@ type EvalkitConfig = {
      * A scenario's `workspace.ignore` patterns come after these.
      */
     exclude?: readonly string[];
+    /** Limits for files adapters `attach()`. */
+    attachments?: AttachmentLimits;
   };
-  /** Where each trial's real workspace lives during a run. Default `_evalkit-sandbox`. */
+  /**
+   * Where each run's real workspace lives. Default: outside the project, in
+   * `<tmp>/evalkit/<project>-<hash>`, so agents cannot read the project or
+   * other runs by looking around. Inside the project, `run` warns.
+   */
   sandboxDir?: string;
-  /** Which trial workspaces to keep in `sandboxDir` afterwards. Default `'failed'`: only trials that did not pass. */
+  /** Which run workspaces to keep in `sandboxDir` afterwards. Default `'failed'`: only runs that did not pass. */
   keepSandboxes?: KeepSandboxes;
   share?: ShareConfig;
+  /**
+   * The agent for every judge that names none: `judge(name, { rubric })`.
+   * Made with `defineAgent()`; it gets each run's parameters, so a matrix
+   * axis can choose its model.
+   */
+  judge?: AutAdapter;
 };
 type KeepSandboxes = 'failed' | 'all' | 'none';
+type AttachmentLimits = {
+  /** A larger attachment keeps its last this-many bytes. Default 5 MB. */
+  maxFileBytes?: number;
+  /** All of one run's attachments; past it, `attach()` rejects. Default 20 MB. */
+  maxRunBytes?: number;
+};
+export declare const DEFAULT_ATTACHMENT_LIMITS: Required<AttachmentLimits>;
 /** Left out of candidate snapshots unless `artifacts.exclude` says otherwise. */
 export declare const DEFAULT_ARTIFACT_EXCLUDE: readonly string[];
 /** HTTP header names and values. */
@@ -377,6 +495,32 @@ type ShareConfig = {
 /** Pure, typed project configuration. Loading/discovery belongs to the CLI. */
 export declare function defineConfig<const T extends EvalkitConfig>(configuration: T): T;
 //#endregion
+//#region ../core/src/usage.d.ts
+/**
+ * Token usage an agent reports for one turn. EvalKit never measures or infers
+ * it; adapters copy it from their provider.
+ */
+type Usage = {
+  /**
+   * Every input token the model read this turn: the prompt, history and tool
+   * results, **including** tokens read from or written to a prompt cache.
+   * (Some providers report cached tokens separately; add them in.)
+   */
+  inputTokens?: number;
+  /** Every token the model produced, including reasoning tokens. */
+  outputTokens?: number;
+  /** Of `inputTokens`, how many were read from a prompt cache. Informational. */
+  cachedInputTokens?: number;
+  /** The provider's own total; used only when input and output are not reported. */
+  totalTokens?: number;
+};
+/**
+ * The tokens a turn counts toward the `tokens` metric: `inputTokens +
+ * outputTokens` (cached input included), or `totalTokens` when neither is
+ * reported. Undefined when the turn reported no usage.
+ */
+export declare function turnTokens(usage: Usage | undefined): number | undefined;
+//#endregion
 //#region ../core/src/json.d.ts
 /** A value that cannot be stored in a report, with the path of the offending part. */
 export declare class NotJsonError extends Error {
@@ -413,7 +557,9 @@ export declare function definitionSite(value: unknown): DefinitionSite | undefin
 //#endregion
 //#region ../core/src/schema.d.ts
 /** Runtime schemas for serialized EvalKit report and API boundaries. */
-export declare const RunStatusSchema: Schema.Literal<["running", "completed", "failed", "cancelled"]>;
+export declare const RunStatusSchema: Schema.Literal<["running", "passed", "failed", "errored", "cancelled"]>;
+/** The report schema version this EvalKit writes and reads. */
+export declare const REPORT_SCHEMA_VERSION = 6;
 export declare const RecordedErrorSchema: Schema.Struct<{
   name: typeof Schema.String;
   message: typeof Schema.String;
@@ -440,8 +586,19 @@ export declare const RunSourceSchema: Schema.Struct<{
     url: Schema.optional<typeof Schema.String>;
   }>>;
 }>;
+export declare const BatchSchema: Schema.Struct<{
+  id: typeof Schema.String;
+  index: typeof Schema.Number;
+  size: typeof Schema.Number;
+}>;
+export declare const RunDatasetSchema: Schema.Struct<{
+  id: typeof Schema.String;
+  visibility: Schema.Literal<["candidate", "evaluator"]>;
+  source: Schema.Literal<["static", "computed"]>;
+  digest: typeof Schema.String;
+}>;
 export declare const RunMetadataSchema: Schema.Struct<{
-  schemaVersion: Schema.Literal<[2, 3, 4, 5]>;
+  schemaVersion: Schema.Literal<[6]>;
   runUri: Schema.filter<typeof Schema.String>;
   evalId: typeof Schema.String;
   suiteId: Schema.optional<typeof Schema.String>;
@@ -475,25 +632,11 @@ export declare const RunMetadataSchema: Schema.Struct<{
     pid: typeof Schema.Number;
     hostname: typeof Schema.String;
   }>>;
-}>;
-export declare const TrialMetadataSchema: Schema.Struct<{
-  schemaVersion: Schema.Literal<[2, 3, 4, 5]>;
-  runUri: Schema.filter<typeof Schema.String>;
-  trialUri: Schema.filter<typeof Schema.String>;
-  trialIndex: typeof Schema.Number;
-  evalId: typeof Schema.String;
-  parameters: Schema.optional<Schema.Record$<typeof Schema.String, Schema.Schema<JsonValue, JsonValue, never>>>;
-  matrix: Schema.optional<Schema.Struct<{
-    id: typeof Schema.String;
-    cellKey: typeof Schema.String;
-  }>>;
-  aut: Schema.optional<Schema.Struct<{
-    name: Schema.optional<typeof Schema.String>;
-    kind: typeof Schema.String;
+  batch: Schema.optional<Schema.Struct<{
     id: typeof Schema.String;
-    version: Schema.optional<typeof Schema.String>;
+    index: typeof Schema.Number;
+    size: typeof Schema.Number;
   }>>;
-  startedAt: typeof Schema.String;
   /** The effective snapshot ignore patterns: the project's, then the scenario's. */
   workspaceIgnore: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
   datasets: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
@@ -503,9 +646,9 @@ export declare const TrialMetadataSchema: Schema.Struct<{
     digest: typeof Schema.String;
   }>>>>;
 }>;
-/** On-disk manifests include a lifecycle status; older finalized manifests may still say running. */
+/** On-disk manifests include a lifecycle status. */
 export declare const RunManifestSchema: Schema.Struct<{
-  schemaVersion: Schema.Literal<[2, 3, 4, 5]>;
+  schemaVersion: Schema.Literal<[6]>;
   runUri: Schema.filter<typeof Schema.String>;
   evalId: typeof Schema.String;
   suiteId: Schema.optional<typeof Schema.String>;
@@ -539,26 +682,11 @@ export declare const RunManifestSchema: Schema.Struct<{
     pid: typeof Schema.Number;
     hostname: typeof Schema.String;
   }>>;
-  status: Schema.optional<Schema.Literal<["running", "completed", "failed", "cancelled"]>>;
-}>;
-export declare const TrialManifestSchema: Schema.Struct<{
-  schemaVersion: Schema.Literal<[2, 3, 4, 5]>;
-  runUri: Schema.filter<typeof Schema.String>;
-  trialUri: Schema.filter<typeof Schema.String>;
-  trialIndex: typeof Schema.Number;
-  evalId: typeof Schema.String;
-  parameters: Schema.optional<Schema.Record$<typeof Schema.String, Schema.Schema<JsonValue, JsonValue, never>>>;
-  matrix: Schema.optional<Schema.Struct<{
+  batch: Schema.optional<Schema.Struct<{
     id: typeof Schema.String;
-    cellKey: typeof Schema.String;
-  }>>;
-  aut: Schema.optional<Schema.Struct<{
-    name: Schema.optional<typeof Schema.String>;
-    kind: typeof Schema.String;
-    id: typeof Schema.String;
-    version: Schema.optional<typeof Schema.String>;
+    index: typeof Schema.Number;
+    size: typeof Schema.Number;
   }>>;
-  startedAt: typeof Schema.String;
   workspaceIgnore: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
   datasets: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
     id: typeof Schema.String;
@@ -566,11 +694,12 @@ export declare const TrialManifestSchema: Schema.Struct<{
     source: Schema.Literal<["static", "computed"]>;
     digest: typeof Schema.String;
   }>>>>;
-  status: Schema.optional<Schema.Literal<["running", "completed", "failed", "cancelled"]>>;
+  status: Schema.optional<Schema.Literal<["running", "passed", "failed", "errored", "cancelled"]>>;
 }>;
 export declare const UsageSchema: Schema.Struct<{
   inputTokens: Schema.optional<typeof Schema.Number>;
   outputTokens: Schema.optional<typeof Schema.Number>;
+  cachedInputTokens: Schema.optional<typeof Schema.Number>;
   totalTokens: Schema.optional<typeof Schema.Number>;
 }>;
 export declare const JudgeRunInfoSchema: Schema.Struct<{
@@ -580,14 +709,34 @@ export declare const JudgeRunInfoSchema: Schema.Struct<{
     name: Schema.optional<typeof Schema.String>;
     version: Schema.optional<typeof Schema.String>;
   }>>;
+  model: Schema.optional<typeof Schema.String>;
   usage: Schema.optional<Schema.Struct<{
     inputTokens: Schema.optional<typeof Schema.Number>;
     outputTokens: Schema.optional<typeof Schema.Number>;
+    cachedInputTokens: Schema.optional<typeof Schema.Number>;
     totalTokens: Schema.optional<typeof Schema.Number>;
   }>>;
+  prompt: Schema.optional<typeof Schema.String>;
+  reply: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+  attempts: Schema.optional<typeof Schema.Number>;
   events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
 }>;
 export declare const ScoreResultSchema: Schema.Struct<{
+  passAt: Schema.optional<typeof Schema.Number>;
+  choice: Schema.optional<typeof Schema.String>;
+  criteria: Schema.optional<Schema.mutable<Schema.Array$<Schema.Union<[Schema.Struct<{
+    name: typeof Schema.String;
+    value: typeof Schema.Number;
+    weight: Schema.optional<typeof Schema.Number>;
+    explanation: Schema.optional<typeof Schema.String>;
+    evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+  }>, Schema.Struct<{
+    name: typeof Schema.String;
+    reached: Schema.Literal<[false]>;
+    weight: Schema.optional<typeof Schema.Number>;
+    explanation: Schema.optional<typeof Schema.String>;
+  }>]>>>>;
+  attachments: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
   name: typeof Schema.String;
   kind: Schema.Literal<["predicate", "judge"]>;
   value: Schema.optional<typeof Schema.Number>;
@@ -601,11 +750,16 @@ export declare const ScoreResultSchema: Schema.Struct<{
       name: Schema.optional<typeof Schema.String>;
       version: Schema.optional<typeof Schema.String>;
     }>>;
+    model: Schema.optional<typeof Schema.String>;
     usage: Schema.optional<Schema.Struct<{
       inputTokens: Schema.optional<typeof Schema.Number>;
       outputTokens: Schema.optional<typeof Schema.Number>;
+      cachedInputTokens: Schema.optional<typeof Schema.Number>;
       totalTokens: Schema.optional<typeof Schema.Number>;
     }>>;
+    prompt: Schema.optional<typeof Schema.String>;
+    reply: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+    attempts: Schema.optional<typeof Schema.Number>;
     events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
   }>>;
   durationMs: typeof Schema.Number;
@@ -617,6 +771,21 @@ export declare const ScoreResultSchema: Schema.Struct<{
   skipped: Schema.optional<Schema.Literal<[true]>>;
 }>;
 export declare const CheckpointResultSchema: Schema.Struct<{
+  passAt: Schema.optional<typeof Schema.Number>;
+  choice: Schema.optional<typeof Schema.String>;
+  criteria: Schema.optional<Schema.mutable<Schema.Array$<Schema.Union<[Schema.Struct<{
+    name: typeof Schema.String;
+    value: typeof Schema.Number;
+    weight: Schema.optional<typeof Schema.Number>;
+    explanation: Schema.optional<typeof Schema.String>;
+    evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+  }>, Schema.Struct<{
+    name: typeof Schema.String;
+    reached: Schema.Literal<[false]>;
+    weight: Schema.optional<typeof Schema.Number>;
+    explanation: Schema.optional<typeof Schema.String>;
+  }>]>>>>;
+  attachments: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
   step: typeof Schema.Number;
   kind: Schema.Literal<["predicate", "judge", "expect-tool-call"]>;
   name: typeof Schema.String;
@@ -632,11 +801,16 @@ export declare const CheckpointResultSchema: Schema.Struct<{
       name: Schema.optional<typeof Schema.String>;
       version: Schema.optional<typeof Schema.String>;
     }>>;
+    model: Schema.optional<typeof Schema.String>;
     usage: Schema.optional<Schema.Struct<{
       inputTokens: Schema.optional<typeof Schema.Number>;
       outputTokens: Schema.optional<typeof Schema.Number>;
+      cachedInputTokens: Schema.optional<typeof Schema.Number>;
       totalTokens: Schema.optional<typeof Schema.Number>;
     }>>;
+    prompt: Schema.optional<typeof Schema.String>;
+    reply: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+    attempts: Schema.optional<typeof Schema.Number>;
     events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
   }>>;
   durationMs: Schema.optional<typeof Schema.Number>;
@@ -662,8 +836,23 @@ export declare const MetricResultSchema: Schema.Struct<{
     stack: Schema.optional<typeof Schema.String>;
   }>>;
 }>;
-export declare const TrialScoringSchema: Schema.Struct<{
+export declare const RunScoringSchema: Schema.Struct<{
   results: Schema.mutable<Schema.Array$<Schema.Struct<{
+    passAt: Schema.optional<typeof Schema.Number>;
+    choice: Schema.optional<typeof Schema.String>;
+    criteria: Schema.optional<Schema.mutable<Schema.Array$<Schema.Union<[Schema.Struct<{
+      name: typeof Schema.String;
+      value: typeof Schema.Number;
+      weight: Schema.optional<typeof Schema.Number>;
+      explanation: Schema.optional<typeof Schema.String>;
+      evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+    }>, Schema.Struct<{
+      name: typeof Schema.String;
+      reached: Schema.Literal<[false]>;
+      weight: Schema.optional<typeof Schema.Number>;
+      explanation: Schema.optional<typeof Schema.String>;
+    }>]>>>>;
+    attachments: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
     name: typeof Schema.String;
     kind: Schema.Literal<["predicate", "judge"]>;
     value: Schema.optional<typeof Schema.Number>;
@@ -677,11 +866,16 @@ export declare const TrialScoringSchema: Schema.Struct<{
         name: Schema.optional<typeof Schema.String>;
         version: Schema.optional<typeof Schema.String>;
       }>>;
+      model: Schema.optional<typeof Schema.String>;
       usage: Schema.optional<Schema.Struct<{
         inputTokens: Schema.optional<typeof Schema.Number>;
         outputTokens: Schema.optional<typeof Schema.Number>;
+        cachedInputTokens: Schema.optional<typeof Schema.Number>;
         totalTokens: Schema.optional<typeof Schema.Number>;
       }>>;
+      prompt: Schema.optional<typeof Schema.String>;
+      reply: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+      attempts: Schema.optional<typeof Schema.Number>;
       events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
     }>>;
     durationMs: typeof Schema.Number;
@@ -705,6 +899,21 @@ export declare const TrialScoringSchema: Schema.Struct<{
     }>>;
   }>>>>;
   checkpoints: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
+    passAt: Schema.optional<typeof Schema.Number>;
+    choice: Schema.optional<typeof Schema.String>;
+    criteria: Schema.optional<Schema.mutable<Schema.Array$<Schema.Union<[Schema.Struct<{
+      name: typeof Schema.String;
+      value: typeof Schema.Number;
+      weight: Schema.optional<typeof Schema.Number>;
+      explanation: Schema.optional<typeof Schema.String>;
+      evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+    }>, Schema.Struct<{
+      name: typeof Schema.String;
+      reached: Schema.Literal<[false]>;
+      weight: Schema.optional<typeof Schema.Number>;
+      explanation: Schema.optional<typeof Schema.String>;
+    }>]>>>>;
+    attachments: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
     step: typeof Schema.Number;
     kind: Schema.Literal<["predicate", "judge", "expect-tool-call"]>;
     name: typeof Schema.String;
@@ -720,11 +929,16 @@ export declare const TrialScoringSchema: Schema.Struct<{
         name: Schema.optional<typeof Schema.String>;
         version: Schema.optional<typeof Schema.String>;
       }>>;
+      model: Schema.optional<typeof Schema.String>;
       usage: Schema.optional<Schema.Struct<{
         inputTokens: Schema.optional<typeof Schema.Number>;
         outputTokens: Schema.optional<typeof Schema.Number>;
+        cachedInputTokens: Schema.optional<typeof Schema.Number>;
         totalTokens: Schema.optional<typeof Schema.Number>;
       }>>;
+      prompt: Schema.optional<typeof Schema.String>;
+      reply: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+      attempts: Schema.optional<typeof Schema.Number>;
       events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
     }>>;
     durationMs: Schema.optional<typeof Schema.Number>;
@@ -747,12 +961,32 @@ export declare const ArtifactEntrySchema: Schema.Struct<{
   kind: Schema.Literal<["file", "directory"]>;
   size: Schema.optional<typeof Schema.Number>;
 }>;
-export declare const TrialSummarySchema: Schema.Struct<{
-  status: Schema.Literal<["running", "completed", "failed", "cancelled"]>;
+export declare const AttachmentEntrySchema: Schema.Struct<{
+  name: typeof Schema.String;
+  size: typeof Schema.Number;
+  truncatedFrom: Schema.optional<typeof Schema.Number>;
+}>;
+export declare const RunSummarySchema: Schema.Struct<{
+  status: Schema.Literal<["running", "passed", "failed", "errored", "cancelled"]>;
   endedAt: typeof Schema.String;
   durationMs: Schema.optional<typeof Schema.Number>;
   scoring: Schema.optional<Schema.Struct<{
     results: Schema.mutable<Schema.Array$<Schema.Struct<{
+      passAt: Schema.optional<typeof Schema.Number>;
+      choice: Schema.optional<typeof Schema.String>;
+      criteria: Schema.optional<Schema.mutable<Schema.Array$<Schema.Union<[Schema.Struct<{
+        name: typeof Schema.String;
+        value: typeof Schema.Number;
+        weight: Schema.optional<typeof Schema.Number>;
+        explanation: Schema.optional<typeof Schema.String>;
+        evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+      }>, Schema.Struct<{
+        name: typeof Schema.String;
+        reached: Schema.Literal<[false]>;
+        weight: Schema.optional<typeof Schema.Number>;
+        explanation: Schema.optional<typeof Schema.String>;
+      }>]>>>>;
+      attachments: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
       name: typeof Schema.String;
       kind: Schema.Literal<["predicate", "judge"]>;
       value: Schema.optional<typeof Schema.Number>;
@@ -766,11 +1000,16 @@ export declare const TrialSummarySchema: Schema.Struct<{
           name: Schema.optional<typeof Schema.String>;
           version: Schema.optional<typeof Schema.String>;
         }>>;
+        model: Schema.optional<typeof Schema.String>;
         usage: Schema.optional<Schema.Struct<{
           inputTokens: Schema.optional<typeof Schema.Number>;
           outputTokens: Schema.optional<typeof Schema.Number>;
+          cachedInputTokens: Schema.optional<typeof Schema.Number>;
           totalTokens: Schema.optional<typeof Schema.Number>;
         }>>;
+        prompt: Schema.optional<typeof Schema.String>;
+        reply: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+        attempts: Schema.optional<typeof Schema.Number>;
         events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
       }>>;
       durationMs: typeof Schema.Number;
@@ -794,6 +1033,21 @@ export declare const TrialSummarySchema: Schema.Struct<{
       }>>;
     }>>>>;
     checkpoints: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
+      passAt: Schema.optional<typeof Schema.Number>;
+      choice: Schema.optional<typeof Schema.String>;
+      criteria: Schema.optional<Schema.mutable<Schema.Array$<Schema.Union<[Schema.Struct<{
+        name: typeof Schema.String;
+        value: typeof Schema.Number;
+        weight: Schema.optional<typeof Schema.Number>;
+        explanation: Schema.optional<typeof Schema.String>;
+        evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+      }>, Schema.Struct<{
+        name: typeof Schema.String;
+        reached: Schema.Literal<[false]>;
+        weight: Schema.optional<typeof Schema.Number>;
+        explanation: Schema.optional<typeof Schema.String>;
+      }>]>>>>;
+      attachments: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
       step: typeof Schema.Number;
       kind: Schema.Literal<["predicate", "judge", "expect-tool-call"]>;
       name: typeof Schema.String;
@@ -809,11 +1063,16 @@ export declare const TrialSummarySchema: Schema.Struct<{
           name: Schema.optional<typeof Schema.String>;
           version: Schema.optional<typeof Schema.String>;
         }>>;
+        model: Schema.optional<typeof Schema.String>;
         usage: Schema.optional<Schema.Struct<{
           inputTokens: Schema.optional<typeof Schema.Number>;
           outputTokens: Schema.optional<typeof Schema.Number>;
+          cachedInputTokens: Schema.optional<typeof Schema.Number>;
           totalTokens: Schema.optional<typeof Schema.Number>;
         }>>;
+        prompt: Schema.optional<typeof Schema.String>;
+        reply: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+        attempts: Schema.optional<typeof Schema.Number>;
         events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
       }>>;
       durationMs: Schema.optional<typeof Schema.Number>;
@@ -836,20 +1095,11 @@ export declare const TrialSummarySchema: Schema.Struct<{
     kind: Schema.Literal<["file", "directory"]>;
     size: Schema.optional<typeof Schema.Number>;
   }>>>>;
-  error: Schema.optional<Schema.Struct<{
+  attachments: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
     name: typeof Schema.String;
-    message: typeof Schema.String;
-    stack: Schema.optional<typeof Schema.String>;
-  }>>;
-}>;
-export declare const RunSummarySchema: Schema.Struct<{
-  status: Schema.Literal<["running", "completed", "failed", "cancelled"]>;
-  endedAt: typeof Schema.String;
-  durationMs: Schema.optional<typeof Schema.Number>;
-  trialCount: typeof Schema.Number;
-  passed: typeof Schema.Number;
-  failed: typeof Schema.Number;
-  cancelled: Schema.optional<typeof Schema.Number>;
+    size: typeof Schema.Number;
+    truncatedFrom: Schema.optional<typeof Schema.Number>;
+  }>>>>;
   error: Schema.optional<Schema.Struct<{
     name: typeof Schema.String;
     message: typeof Schema.String;
@@ -891,11 +1141,6 @@ export type AgentRuntime = {
   configuration?: JsonObject;
 };
 export type AgentRuntimes = Partial<Record<AgentRuntimeName, AgentRuntime>>;
-export type Usage = {
-  inputTokens?: number;
-  outputTokens?: number;
-  totalTokens?: number;
-};
 export type RecordedError = {
   name: string;
   message: string;
@@ -931,6 +1176,8 @@ export type AutEvent = {
   timestamp: string;
   latencyMs?: number;
   usage?: Usage;
+  /** The model that answered this turn, when the agent knows it. */
+  model?: string;
 } | {
   kind: 'completed';
   output?: JsonValue;
@@ -941,7 +1188,7 @@ export type AutEvent = {
   timestamp: string;
 };
 export type RunnerEvent = {
-  kind: 'trial-started';
+  kind: 'run-started';
   timestamp: string;
 } | {
   kind: 'transcript-step-started';
@@ -989,7 +1236,7 @@ export type RunnerEvent = {
   error: RecordedError;
   timestamp: string;
 } | {
-  kind: 'trial-completed';
+  kind: 'run-completed';
   timestamp: string;
 } | {
   kind: 'error';
@@ -1002,7 +1249,7 @@ export type TrajectoryEvent = (AutEvent & {
   source: 'runner';
 });
 export type WorkspaceView = {
-  /** Absolute path to this trial-scoped workspace. Do not persist it in reports. */
+  /** Absolute path to this run-scoped workspace. Do not persist it in reports. */
   root: string;
 };
 export type CandidateWorkspace = WorkspaceView;
@@ -1010,8 +1257,8 @@ export type EvaluatorWorkspace = WorkspaceView;
 export type AutContext = {
   runId: string;
   evalId: string;
-  trialId: string;
-  trialIndex: number;
+  /** Set when this run is one of several repeats launched together. */
+  batch?: Batch;
   metadata: JsonObject;
   /** Run-time parameters supplied by the eval definition or CLI invocation. */
   parameters?: JsonObject;
@@ -1032,6 +1279,14 @@ export type AutAdapter<TMessage = string> = {
     runtime?: AgentRuntime;
     /** Optional: stream events as they happen. Returning a turn result from `send` is enough for most agents. */
     onEvent: (event: AutEventInput) => void | Promise<void>;
+    /**
+     * Saves a file with the run's results, such as the agent's raw log,
+     * whether or not its sandbox is kept. `name` is a relative path of
+     * letters, digits, `.`, `_` and `-` (`logs/agent.log`); each name once.
+     * A file over the size limit keeps its last bytes; past the run's
+     * total limit it is rejected. See `artifacts.attachments` in the config.
+     */
+    attach: (name: string, content: string | Uint8Array) => Promise<void>;
   }): Promise<AutSession<TMessage>>;
 };
 export type AutSession<TMessage = string> = {
@@ -1058,13 +1313,14 @@ export type UserMessageInput = {
   parameters: JsonObject;
   /** Values of the scenario's candidate datasets, by dataset ID (never evaluator ones). */
   data: JsonObject;
-  /** The trial's random seed (also in `context.metadata.randomSeed`). */
+  /** The run's random seed (also in `context.metadata.randomSeed`). */
   seed: number;
-  trialIndex: number;
+  /** Set when this run is one of several repeats launched together. */
+  batch?: Batch;
 };
 export type UserStep = {
   kind: 'user';
-  /** A fixed message, or one computed for each trial. */
+  /** A fixed message, or one computed for each run. */
   message: string | ((input: UserMessageInput) => string);
 };
 export type AgentStep = {
@@ -1104,14 +1360,28 @@ export type TranscriptStep = UserStep | AgentStep | ScoringRule | ExpectToolCall
 /** Observes a tool call in the preceding turn; never invokes a tool. */
 export declare function expectToolCall(expected: ExpectToolCallStep['expected']): ExpectToolCallStep;
 /**
- * A user message. Pass a function to compute it for each trial, e.g. from a
+ * A user message. Pass a function to compute it for each run, e.g. from a
  * matrix axis: `user(({ parameters }) => `Set up OCaml with ${parameters.toolchain}.`)`.
  */
 export declare function user(message: UserStep['message']): UserStep;
 export declare function agent(expectation: JsonObject): AgentStep;
+/**
+ * What a judge saw and said. The built-in `judge()` records all of it; a
+ * predicate that calls a model itself can report `model`, `prompt` and
+ * `reply` so the dashboard shows them the same way.
+ */
 export type JudgeRunInfo = {
   agent?: AutIdentity;
+  /** The model the judge reported using (`turn-completed` `model`). */
+  model?: string;
+  /** Tokens the judge used: never counted in the agent's `tokens`. */
   usage?: Usage;
+  /** The exact prompt the judge received. */
+  prompt?: string;
+  /** Its reply, as text or JSON. */
+  reply?: JsonValue;
+  /** 2 when its first reply was not in the format asked for and it was asked again. */
+  attempts?: number;
   /** Judge-agent observations; never attributed to the AUT trajectory. */
   events?: JsonValue[];
 };
@@ -1120,18 +1390,27 @@ export type SkippedScore = {
   skipped: true;
   reason: string;
 };
-export type ScoreValue = number | SkippedScore | {
-  value: number;
+export type ScoreValue<Evidence = unknown> = number | SkippedScore | {
+  /** From 0 to 1. Without it, the weighted mean of `criteria`. */
+  value?: number;
   passed?: boolean;
   explanation?: string;
-  /** Any JSON-convertible value. */
-  evidence?: unknown;
-  /** Judge-agent evidence, never inferred from AUT usage. */
+  /**
+   * Any JSON-convertible value: what the scorer observed. A checkpoint's
+   * evidence reaches later scorers through `checkpoint(rule)`.
+   */
+  evidence?: Evidence;
+  /**
+   * Per-criterion results, each shown on its own; see `CriterionResult`.
+   * Return `{ name, reached: false }` for a check that never ran.
+   */
+  criteria?: CriterionInput[];
+  /** What a model-backed scorer saw and said; never counted as AUT usage. */
   judge?: JudgeRunInfo;
 };
 /**
- * Marks a scorer as not applicable to this trial. A skipped result neither
- * passes nor fails the trial and is left out of score averages.
+ * Marks a scorer as not applicable to this run. A skipped result neither
+ * passes nor fails the run and is left out of score averages.
  */
 export declare function skip(reason: string): SkippedScore;
 export type ScoreResult = {
@@ -1139,8 +1418,15 @@ export type ScoreResult = {
   kind: 'predicate' | 'judge';
   value?: number;
   passed?: boolean;
+  /** The pass mark the value was held to (1 unless the rule set `passAt`). */
+  passAt?: number;
   explanation?: string;
   evidence?: JsonValue;
+  /** A `choices` judge's answer, such as `B`. */
+  choice?: string;
+  criteria?: CriterionResult[];
+  /** Files its scorer saved with `attach()`, by name. */
+  attachments?: string[];
   judge?: JudgeRunInfo;
   durationMs: number;
   error?: RecordedError;
@@ -1169,26 +1455,88 @@ export type ScoringContext = {
    * example, in a container that is gone by final scoring).
    */
   checkpoints: readonly CheckpointResult[];
+  /**
+   * The latest result of a checkpoint in this scenario's transcript, looked
+   * up by the rule itself, so renaming it cannot break the lookup. A
+   * predicate's `evidence` keeps its type. Undefined until that step has
+   * run; throws for a rule that is not in the transcript.
+   */
+  checkpoint<Evidence>(rule: PredicateScorer<Evidence>): CheckpointOutcome<Evidence> | undefined;
+  checkpoint(rule: JudgeRule | ExpectToolCallStep): CheckpointOutcome<JsonValue> | undefined;
+  /**
+   * Saves a file with the run's results, like the agent's `attach()`: for
+   * example a container's log, or a file the agent reported, read before the
+   * workspace is gone. Same names, limits and rules.
+   */
+  attach(name: string, content: string | Uint8Array): Promise<void>;
 };
-export type PredicateScorer = {
+/**
+ * A checkpoint's result with typed evidence. Evidence is saved as JSON, so
+ * it arrives as its JSON form: a `Date` becomes a string, `undefined` fields
+ * are dropped.
+ */
+export type CheckpointOutcome<Evidence> = Omit<CheckpointResult, 'evidence'> & {
+  evidence?: Evidence;
+};
+export type PredicateScorer<Evidence = unknown> = {
   kind: 'predicate';
   name: string;
+  /** What it checks, in a sentence, shown wherever the scorer is. */
+  description?: string;
   supportsPartial?: boolean;
   /** A numeric result passes when it is at least this value. Default: only 1 passes. */
   passAt?: number;
-  run(context: ScoringContext): boolean | ScoreValue | Promise<boolean | ScoreValue>;
+  /**
+   * The checks it reports, declared up front so they show before a run.
+   * A declared criterion it leaves out is recorded as not reached; one it
+   * did not declare is an error.
+   */
+  criteria?: JudgeCriterion[];
+  run(context: ScoringContext): boolean | ScoreValue<Evidence> | Promise<boolean | ScoreValue<Evidence>>;
 };
+/** What a judge sees: one named input, read from the run. */
+export type JudgeInput = (scoring: ScoringContext) => JsonValue | undefined | Promise<JsonValue | undefined>;
+/**
+ * A rubric-graded scorer. EvalKit builds the prompt from the rubric, the
+ * inputs and the answer format, runs the judge agent, checks its answer
+ * (asking once more if it is not in the format asked for), and records
+ * what it saw and said.
+ */
 export type JudgeRule = {
   kind: 'judge';
   name: string;
   rubric: string;
+  /** What it checks, in a sentence, shown wherever the judge is. */
+  description?: string;
+  /**
+   * The agent that judges, made with `defineAgent()`. Default: `judge` in
+   * evalkit.config.ts. It gets the run's parameters, so a matrix axis can
+   * choose its model.
+   */
+  agent?: AutAdapter;
+  /**
+   * What the judge sees, by name. Default: `conversation` (this turn at a
+   * checkpoint, the whole run in final scoring) and `reply` (the last reply).
+   */
+  inputs?: Readonly<Record<string, JudgeInput>>;
+  /** How it answers. Default `verdict`: a value from 0 to 1 and a reason. */
+  answer: JudgeAnswer;
+  /** A value passes when it is at least this. Default: only 1 passes. */
+  passAt?: number;
   /** Run even when the scenario stopped early or execution failed. */
   supportsPartial?: boolean;
 };
 export type ScoringRule = PredicateScorer | JudgeRule;
-export declare function predicate(name: string, run: PredicateScorer['run'], options?: Pick<PredicateScorer, 'supportsPartial' | 'passAt'>): PredicateScorer;
+/** A scorer in code. Its `evidence` type carries through to `checkpoint(rule)`. */
+export declare function predicate<Evidence = unknown>(name: string, run: PredicateScorer<Evidence>['run'], options?: Pick<PredicateScorer, 'supportsPartial' | 'passAt' | 'description'> & {
+  /** The checks it reports by name: `{ 'cannot push': 'the share token gets 403' }` or `{ name: { weight, description } }`. */
+  criteria?: Record<string, string | number | {
+    weight?: number;
+    description?: string;
+  }>;
+}): PredicateScorer<Evidence>;
 /**
- * A number measured for every trial, such as tokens, retries or web searches.
+ * A number measured for every run, such as tokens, retries or web searches.
  * Metrics never decide pass or fail; they are averaged per run and shown on
  * the scoreboard. Return undefined when there is nothing to measure.
  */
@@ -1201,10 +1549,11 @@ export type Metric = {
   better?: 'lower' | 'higher';
   run(context: ScoringContext): number | undefined | Promise<number | undefined>;
 };
-export declare function metric(name: string, run: Metric['run'], options?: Pick<Metric, 'unit' | 'better'>): Metric;
-/** A metric's value for one trial. */
+/** A built-in metric's name is a type error here, as well as at definition time. */
+export declare function metric<const Name extends string>(name: Name extends BuiltInMetricName ? never : Name, run: Metric['run'], options?: Pick<Metric, 'unit' | 'better'>): Metric;
+/** A metric's value for one run. */
 export type MetricResult = {
-  name: string;
+  name: MetricName;
   /** Absent when the metric measured nothing or failed. */
   value?: number;
   unit?: string;
@@ -1213,27 +1562,30 @@ export type MetricResult = {
   builtIn?: true;
   error?: RecordedError;
 };
-export declare function judge(name: string, options: Pick<JudgeRule, 'rubric' | 'supportsPartial'>): JudgeRule;
+export declare function judge(name: string, options: Omit<JudgeRule, 'kind' | 'name' | 'answer'> & {
+  answer?: JudgeAnswer;
+}): JudgeRule;
 export type EvalPolicy = {
   timeoutMs?: number;
   /** Stop later transcript steps after a failed checkpoint assertion. Default false. */
   failfast?: boolean;
-  /** Number of independent trials requested by local and hosted runners. */
-  trials?: number;
 };
 export type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
   id: string;
   name?: string;
   agent: TAgent;
-  /** Separate judge agent; its model and tools are never inherited from the AUT. */
-  judge?: AutAdapter;
-  /** Data and files the scenario uses; each trial gets its own copy. */
+  /** Data and files the scenario uses; each run gets its own copy. */
   datasets?: readonly Dataset[];
   transcript: TranscriptStep[];
   scoring: ScoringRule[];
-  /** Numbers to measure for every trial; they never affect pass or fail. */
+  /** Numbers to measure for every run; they never affect pass or fail. */
   metrics?: readonly Metric[];
   policy?: EvalPolicy;
+  /**
+   * How many runs to start each time the scenario runs, as one batch: repeats
+   * under identical conditions, to see how often it passes. Default 1.
+   */
+  repeat?: number;
   /** Any JSON-convertible object; converted when the eval is defined. */
   metadata?: Record<string, unknown>;
   workspace?: EvalWorkspace;
@@ -1241,7 +1593,7 @@ export type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
 /** How EvalKit treats this scenario's candidate workspace. */
 export type EvalWorkspace = {
   /**
-   * Paths to leave out of the trial's snapshot, read like `.gitignore`: globs
+   * Paths to leave out of the run's snapshot, read like `.gitignore`: globs
    * relative to the workspace, where a pattern without `/` matches that name
    * at any depth and later patterns win. They come after the project's
    * `artifacts.exclude` (default `DEFAULT_ARTIFACT_EXCLUDE`), so `!pattern`
@@ -1251,6 +1603,10 @@ export type EvalWorkspace = {
   ignore?: readonly string[];
 };
 export declare function defineEval<const T extends EvalDefinition>(definition: T): T;
+/** A judge never grades with the agent it is judging. */
+export declare function assertJudgesIndependent(definition: Pick<EvalDefinition, 'agent' | 'transcript' | 'scoring'> & {
+  id: string;
+}, defaultJudge?: AutAdapter): void;
 export type EvalSuite<TEvals extends readonly EvalDefinition[] = readonly EvalDefinition[]> = {
   id: string;
   name?: string;
@@ -1288,7 +1644,18 @@ export type EvalSuiteMetadata = {
  * path-like grouping of evals; nesting is presentation derived from its ID.
  */
 export declare function registerEvals<const TRegistrations extends readonly EvalRegistration[]>(registrations: TRegistrations): EvalRegistry;
-export type RunStatus = 'running' | 'completed' | 'failed' | 'cancelled';
+/**
+ * How a run ended: `passed` or `failed` by its scores, `errored` when it could
+ * not finish (the agent, a dataset or EvalKit failed), `cancelled` by Ctrl+C.
+ */
+export type RunStatus = 'running' | 'passed' | 'failed' | 'errored' | 'cancelled';
+/** Runs launched together as repeats of one configuration (`repeat`). */
+export type Batch = {
+  id: string;
+  /** Zero-based position in the batch. */
+  index: number;
+  size: number;
+};
 /**
  * Where a run's eval code came from: the repository, eval project and commit.
  * Results are tagged with it; they are never stored in the repository itself.
@@ -1323,7 +1690,7 @@ export type RunProcess = {
   hostname: string;
 };
 export type RunMetadata = {
-  schemaVersion: 2 | 3 | 4 | 5;
+  schemaVersion: 6;
   runId: string;
   runUri: ResourceUri<'run'>;
   evalId: string;
@@ -1335,36 +1702,21 @@ export type RunMetadata = {
   };
   aut?: AutIdentity;
   startedAt: string;
-  /** Present from schema version 4 when the run's commit could be determined. */
+  /** Present when the run's commit could be determined. */
   source?: RunSource;
-  /** From schema version 5. */
   process?: RunProcess;
-};
-export type TrialMetadata = {
-  schemaVersion: 2 | 3 | 4 | 5;
-  runId: string;
-  runUri: ResourceUri<'run'>;
-  trialId: string;
-  trialUri: ResourceUri<'trial'>;
-  trialIndex: number;
-  evalId: string;
-  parameters?: JsonObject;
-  matrix?: {
-    id: string;
-    cellKey: string;
-  };
-  aut?: AutIdentity;
-  startedAt: string;
+  /** Set when this run is one of several repeats launched together. */
+  batch?: Batch;
   /** The effective snapshot ignore patterns: the project's `artifacts.exclude`, then the scenario's `workspace.ignore`. */
   workspaceIgnore?: string[];
-  /** From schema version 5: the datasets this trial used, with content digests. */
-  datasets?: TrialDataset[];
+  /** The datasets this run used, with content digests. */
+  datasets?: RunDataset[];
 };
-export type TrialDataset = {
+export type RunDataset = {
   id: string;
   visibility: DatasetVisibility;
   source: 'static' | 'computed';
-  /** SHA-256 of the trial's content: file paths and bytes, and the canonical value. */
+  /** SHA-256 of the run's content: file paths and bytes, and the canonical value. */
   digest: string;
 };
 export type CheckpointResult = {
@@ -1374,8 +1726,13 @@ export type CheckpointResult = {
   status: 'passed' | 'failed' | 'error' | 'skipped';
   value?: number;
   passed?: boolean;
+  passAt?: number;
   explanation?: string;
   evidence?: JsonValue;
+  choice?: string;
+  criteria?: CriterionResult[];
+  /** Files its scorer saved with `attach()`, by name. */
+  attachments?: string[];
   judge?: JudgeRunInfo;
   durationMs?: number;
   error?: RecordedError;
@@ -1384,9 +1741,21 @@ export type CheckpointResult = {
     id: string;
   };
 };
-/** Metrics EvalKit records for every trial; scenario metrics may not reuse these names. */
-export declare const BUILT_IN_METRICS: readonly string[];
-export type TrialScoring = {
+/**
+ * Metrics EvalKit records for every run; scenario metrics may not reuse
+ * these names.
+ *
+ * - `agent time` (ms): from the run's first event to its last one before
+ *   final scoring.
+ * - `tokens`: the sum of `turnTokens(usage)` over the turns that reported
+ *   usage, cached input included. Absent when no turn reported any.
+ * - `tool calls`: the agent's `tool-call` events.
+ */
+export declare const BUILT_IN_METRICS: readonly ['agent time', 'tokens', 'tool calls', 'judge tokens'];
+export type BuiltInMetricName = (typeof BUILT_IN_METRICS)[number];
+/** A built-in metric's name (suggested by editors), or a scenario's own. */
+export type MetricName = BuiltInMetricName | (string & {});
+export type RunScoring = {
   results: ScoreResult[];
   metrics?: MetricResult[];
   checkpoints?: CheckpointResult[];
@@ -1395,40 +1764,34 @@ export type TrialScoring = {
   passed: boolean;
 };
 export type ArtifactEntry = {
-  /** Path relative to this trial's artifact root. */
+  /** Path relative to this run's artifact root. */
   path: string;
   kind: 'file' | 'directory';
   size?: number;
 };
-export type TrialSummary = {
-  status: RunStatus;
-  endedAt: string;
-  durationMs?: number;
-  scoring?: TrialScoring;
-  artifacts?: ArtifactEntry[];
-  error?: RecordedError;
+/** A file an adapter attached with `attach()`, saved as `artifacts/attachments/<name>`. */
+export type AttachmentEntry = {
+  name: string;
+  /** Bytes saved. */
+  size: number;
+  /** The original size, when the file was cut to its last `maxFileBytes`. */
+  truncatedFrom?: number;
 };
 export type RunSummary = {
   status: RunStatus;
   endedAt: string;
   durationMs?: number;
-  trialCount: number;
-  passed: number;
-  failed: number;
-  /** Trials stopped by cancellation; counted neither as passed nor failed. */
-  cancelled?: number;
+  scoring?: RunScoring;
+  artifacts?: ArtifactEntry[];
+  attachments?: AttachmentEntry[];
   error?: RecordedError;
 };
-export type TrialWriter = {
-  appendEvent(event: TrajectoryEvent): Promise<void>;
-  writeScores(scoring: TrialScoring): Promise<void>;
-  /** Writes a file below this trial's artifact root. */
-  writeArtifact(path: string, data: Uint8Array): Promise<void>;
-  finalize(summary: TrialSummary): Promise<void>;
-};
 export type RunWriter = {
   readonly location: string;
-  startTrial(metadata: TrialMetadata): Promise<TrialWriter>;
+  appendEvent(event: TrajectoryEvent): Promise<void>;
+  writeScores(scoring: RunScoring): Promise<void>;
+  /** Writes a file below this run's artifact root. */
+  writeArtifact(path: string, data: Uint8Array): Promise<void>;
   finalize(summary: RunSummary): Promise<void>;
 };
 export type ReportStore = {
@@ -1436,35 +1799,18 @@ export type ReportStore = {
     catalog?: RunCatalog;
   }): Promise<RunWriter>;
 };
-export type TrialResult = {
+/** One finished run, as `runEval` returns it. */
+export type RunResult = {
   runId: string;
   runUri: ResourceUri<'run'>;
-  trialId: string;
-  trialUri: ResourceUri<'trial'>;
-  trialIndex: number;
   evalId: string;
   status: RunStatus;
+  batch?: Batch;
   reportLocation: string;
   durationMs?: number;
-  scoring?: TrialScoring;
+  scoring?: RunScoring;
   error?: RecordedError;
 };
-export type AggregateScoring = {
-  passed: number;
-  failed: number;
-  cancelled?: number;
-  passRate: number;
-  overall?: number;
-};
-export type RunResult = TrialResult & {
-  runId: string;
-  trialCount?: number;
-  passed?: number;
-  failed?: number;
-  aggregateScoring?: AggregateScoring;
-  /** Present for an aggregate multi-trial run. */
-  trials?: TrialResult[];
-};
 export declare function recordError(error: unknown): RecordedError;
 //#endregion
 //#region ../agents/src/index.d.ts
@@ -1476,5 +1822,5 @@ type PiAutOptions = {
 /** A local, one-prompt-per-turn AUT backed by Pi's non-interactive print mode. */
 export declare function piAgent(options?: PiAutOptions): AutAdapter;
 //#endregion
-export type { DefinitionSite, EvalMatrix, EvalMatrixCell, EvalkitConfig, KeepSandboxes, MatrixSelection, ShareConfig, ShareHeaders };
+export type { AttachmentLimits, CriteriaOptions, CriterionInput, CriterionResult, DefinitionSite, EvalMatrix, EvalMatrixCell, EvalkitConfig, JudgeAnswer, JudgeChoice, JudgeCriterion, KeepSandboxes, MatrixSelection, ReachedCriterion, ShareConfig, ShareHeaders, UnreachedCriterion, Usage };
 //# sourceMappingURL=index.d.mts.map