@leostera/evalkit 0.8.0 → 0.10.0

package/dist/runner.d.mts

@@ -41,13 +41,13 @@ export declare class CheckpointExecutionError extends CheckpointExecutionError_b
 declare const CancelledError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
   readonly _tag: "CancelledError";
 } & Readonly<A>;
-/** The run was cancelled (Ctrl+C); the trial stops and is finalized as cancelled. */
+/** The run was cancelled (Ctrl+C); the run stops and is finalized as cancelled. */
 export declare class CancelledError extends CancelledError_base<{
   message: string;
 }> {}
 //#endregion
 //#region ../core/src/identity.d.ts
-declare const ResourceKindSchema: Schema.Literal<["run", "trial", "artifact"]>;
+declare const ResourceKindSchema: Schema.Literal<["run", "artifact"]>;
 type ResourceKind = typeof ResourceKindSchema.Type;
 type ResourceUri<TKind extends ResourceKind = ResourceKind> = `evalkit:${TKind}:${string}`;
 //#endregion
@@ -59,14 +59,14 @@ type DatasetFileSource = string | Uint8Array | {
   file: string;
 };
 /**
- * Files for the trial workspace: a directory relative to the project root, or
+ * Files for the run workspace: a directory relative to the project root, or
  * a map from workspace path (below the dataset's mount) to its content.
  */
 type DatasetFiles = string | Readonly<Record<string, DatasetFileSource>>;
-/** What one trial receives from a dataset. */
+/** What one run receives from a dataset. */
 type DatasetContent = {
   files?: DatasetFiles;
-  /** Any JSON-convertible value; converted once when the trial starts. */
+  /** Any JSON-convertible value; converted once when the run starts. */
   value?: unknown;
 };
 type DatasetLoadContext = {
@@ -74,12 +74,12 @@ type DatasetLoadContext = {
   root: string;
   signal: AbortSignal;
 };
-type DatasetTrialContext = {
+type DatasetRunContext = {
   evalId: string;
   runId: string;
-  trialId: string;
-  trialIndex: number;
-  /** Per-trial random seed, so repeated trials can choose different data. */
+  /** Set when the run is one of several repeats launched together. */
+  batch?: Batch;
+  /** Per-run random seed, so repeated runs can choose different data. */
   seed: number;
   /** The matrix cell's parameters, or `{}`. */
   parameters: JsonObject;
@@ -93,7 +93,7 @@ type DatasetSource = {
 } | {
   kind: 'computed';
   onLoad: (load: DatasetLoadContext) => DatasetContent | Promise<DatasetContent>;
-  onRun?: (loaded: DatasetContent, trial: DatasetTrialContext) => DatasetContent | Promise<DatasetContent>;
+  onRun?: (loaded: DatasetContent, run: DatasetRunContext) => DatasetContent | Promise<DatasetContent>;
 };
 /** A checked dataset, as returned by `defineDataset`. Evals accept only these. */
 type Dataset = {
@@ -141,6 +141,8 @@ type AutEventInput = {
   timestamp?: string;
   latencyMs?: number;
   usage?: Usage;
+  /** The model that answered this turn, when the agent knows it. */
+  model?: string;
 } | {
   kind: 'completed';
   output?: unknown;
@@ -162,8 +164,69 @@ type TurnResult = string | {
   reply?: unknown;
   toolCalls?: readonly TurnToolCall[];
   usage?: Usage;
+  /** The model that answered, recorded with the usage. */
+  model?: string;
 };
 //#endregion
+//#region ../core/src/judge.d.ts
+/**
+ * How a judge must answer, and how EvalKit turns the answer into a score.
+ *
+ * - `verdict` (the default): a value from 0 to 1 and a reason, as JSON.
+ * - `choices`: one of a fixed set of answers, such as A/B/C, each worth a value.
+ * - `criteria`: a value from 0 to 1 per criterion; the score is their weighted mean.
+ */
+type JudgeAnswer = {
+  kind: 'verdict';
+} | {
+  kind: 'choices';
+  choices: JudgeChoice[];
+} | {
+  kind: 'criteria';
+  criteria: JudgeCriterion[];
+};
+/** One answer a `choices` judge can give. Keys are matched case-insensitively. */
+type JudgeChoice = {
+  key: string;
+  label: string;
+  value: number;
+};
+/** One thing a `criteria` judge scores. */
+type JudgeCriterion = {
+  name: string;
+  weight: number;
+  description?: string;
+};
+/**
+ * One criterion's result, as a scorer returns it and as results record it:
+ * a value with what it saw, or `reached: false` when the check never got to
+ * run, because something before it failed (say, the app would not start).
+ * A criterion not reached counts as 0; the score's `explanation` says why,
+ * once, instead of every criterion repeating it.
+ */
+type CriterionResult = ReachedCriterion | UnreachedCriterion;
+type ReachedCriterion = {
+  name: string;
+  /** From 0 to 1. */
+  value: number;
+  /** Default 1, or the declared weight. */
+  weight?: number;
+  /** Why this value. */
+  explanation?: string;
+  /** What the check observed, as plain data. */
+  evidence?: JsonValue;
+};
+type UnreachedCriterion = {
+  name: string;
+  reached: false;
+  weight?: number;
+  explanation?: string;
+};
+/** A criterion as a scorer returns it: its evidence may be any plain value. */
+type CriterionInput = (Omit<ReachedCriterion, 'evidence'> & {
+  evidence?: unknown;
+}) | UnreachedCriterion;
+//#endregion
 //#region ../core/src/catalog.d.ts
 type EvalCatalogAgent = {
   name?: string;
@@ -215,14 +278,24 @@ type EvalCatalogDataset = {
 type EvalCatalogScorer = {
   name: string;
   kind: ScoringRule['kind'];
+  /** What it checks, in the author's words. */
+  description?: string;
   passAt?: number;
   supportsPartial?: boolean;
   /** Judges only. */
   rubric?: string;
+  /** Judges only: the agent that judges (its own, or the project's default). */
+  agent?: EvalCatalogAgent;
+  /** Judges only: the names of the inputs it sees. */
+  inputs?: string[];
+  /** Judges only: how it answers. */
+  answer?: JudgeAnswer;
+  /** Judges only: the prompt it receives, with `{input}` placeholders. */
+  prompt?: string;
   definedAt?: string;
 };
 type EvalCatalogStep =
-/** `message` is absent when it is computed for each trial. */
+/** `message` is absent when it is computed for each run. */
 {
   kind: 'user';
   message?: string;
@@ -248,9 +321,9 @@ type EvalCatalogEntry = {
   /** The separate agent that runs judge rules, if any. */
   judgeAgent?: EvalCatalogAgent;
   datasets: EvalCatalogDataset[];
-  trialCount: number;
+  /** Runs started each time the scenario runs (`repeat`). */
+  repeat: number;
   policy: {
-    trials: number;
     timeoutMs?: number;
     failfast?: boolean;
   };
@@ -296,6 +369,32 @@ type EvalMatrix = EvalMatrixDefinition & {
 //#endregion
 //#region ../core/src/config.d.ts
 type KeepSandboxes = 'failed' | 'all' | 'none';
+type AttachmentLimits = {
+  /** A larger attachment keeps its last this-many bytes. Default 5 MB. */
+  maxFileBytes?: number;
+  /** All of one run's attachments; past it, `attach()` rejects. Default 20 MB. */
+  maxRunBytes?: number;
+};
+//#endregion
+//#region ../core/src/usage.d.ts
+/**
+ * Token usage an agent reports for one turn. EvalKit never measures or infers
+ * it; adapters copy it from their provider.
+ */
+type Usage = {
+  /**
+   * Every input token the model read this turn: the prompt, history and tool
+   * results, **including** tokens read from or written to a prompt cache.
+   * (Some providers report cached tokens separately; add them in.)
+   */
+  inputTokens?: number;
+  /** Every token the model produced, including reasoning tokens. */
+  outputTokens?: number;
+  /** Of `inputTokens`, how many were read from a prompt cache. Informational. */
+  cachedInputTokens?: number;
+  /** The provider's own total; used only when input and output are not reported. */
+  totalTokens?: number;
+};
 //#endregion
 //#region ../core/src/index.d.ts
 type JsonPrimitive = boolean | number | string | null;
@@ -322,11 +421,6 @@ type AgentRuntime = {
   configuration?: JsonObject;
 };
 type AgentRuntimes = Partial<Record<AgentRuntimeName, AgentRuntime>>;
-type Usage = {
-  inputTokens?: number;
-  outputTokens?: number;
-  totalTokens?: number;
-};
 type RecordedError = {
   name: string;
   message: string;
@@ -362,6 +456,8 @@ type AutEvent = {
   timestamp: string;
   latencyMs?: number;
   usage?: Usage;
+  /** The model that answered this turn, when the agent knows it. */
+  model?: string;
 } | {
   kind: 'completed';
   output?: JsonValue;
@@ -372,7 +468,7 @@ type AutEvent = {
   timestamp: string;
 };
 type RunnerEvent = {
-  kind: 'trial-started';
+  kind: 'run-started';
   timestamp: string;
 } | {
   kind: 'transcript-step-started';
@@ -420,7 +516,7 @@ type RunnerEvent = {
   error: RecordedError;
   timestamp: string;
 } | {
-  kind: 'trial-completed';
+  kind: 'run-completed';
   timestamp: string;
 } | {
   kind: 'error';
@@ -433,7 +529,7 @@ type TrajectoryEvent = (AutEvent & {
   source: 'runner';
 });
 type WorkspaceView = {
-  /** Absolute path to this trial-scoped workspace. Do not persist it in reports. */
+  /** Absolute path to this run-scoped workspace. Do not persist it in reports. */
   root: string;
 };
 type CandidateWorkspace = WorkspaceView;
@@ -441,8 +537,8 @@ type EvaluatorWorkspace = WorkspaceView;
 type AutContext = {
   runId: string;
   evalId: string;
-  trialId: string;
-  trialIndex: number;
+  /** Set when this run is one of several repeats launched together. */
+  batch?: Batch;
   metadata: JsonObject;
   /** Run-time parameters supplied by the eval definition or CLI invocation. */
   parameters?: JsonObject;
@@ -463,6 +559,14 @@ type AutAdapter<TMessage = string> = {
     runtime?: AgentRuntime;
     /** Optional: stream events as they happen. Returning a turn result from `send` is enough for most agents. */
     onEvent: (event: AutEventInput) => void | Promise<void>;
+    /**
+     * Saves a file with the run's results, such as the agent's raw log,
+     * whether or not its sandbox is kept. `name` is a relative path of
+     * letters, digits, `.`, `_` and `-` (`logs/agent.log`); each name once.
+     * A file over the size limit keeps its last bytes; past the run's
+     * total limit it is rejected. See `artifacts.attachments` in the config.
+     */
+    attach: (name: string, content: string | Uint8Array) => Promise<void>;
   }): Promise<AutSession<TMessage>>;
 };
 type AutSession<TMessage = string> = {
@@ -480,13 +584,14 @@ type UserMessageInput = {
   parameters: JsonObject;
   /** Values of the scenario's candidate datasets, by dataset ID (never evaluator ones). */
   data: JsonObject;
-  /** The trial's random seed (also in `context.metadata.randomSeed`). */
+  /** The run's random seed (also in `context.metadata.randomSeed`). */
   seed: number;
-  trialIndex: number;
+  /** Set when this run is one of several repeats launched together. */
+  batch?: Batch;
 };
 type UserStep = {
   kind: 'user';
-  /** A fixed message, or one computed for each trial. */
+  /** A fixed message, or one computed for each run. */
   message: string | ((input: UserMessageInput) => string);
 };
 type AgentStep = {
@@ -523,9 +628,23 @@ type ExpectToolCallStep = {
   };
 };
 type TranscriptStep = UserStep | AgentStep | ScoringRule | ExpectToolCallStep;
+/**
+ * What a judge saw and said. The built-in `judge()` records all of it; a
+ * predicate that calls a model itself can report `model`, `prompt` and
+ * `reply` so the dashboard shows them the same way.
+ */
 type JudgeRunInfo = {
   agent?: AutIdentity;
+  /** The model the judge reported using (`turn-completed` `model`). */
+  model?: string;
+  /** Tokens the judge used: never counted in the agent's `tokens`. */
   usage?: Usage;
+  /** The exact prompt the judge received. */
+  prompt?: string;
+  /** Its reply, as text or JSON. */
+  reply?: JsonValue;
+  /** 2 when its first reply was not in the format asked for and it was asked again. */
+  attempts?: number;
   /** Judge-agent observations; never attributed to the AUT trajectory. */
   events?: JsonValue[];
 };
@@ -534,13 +653,22 @@ type SkippedScore = {
   skipped: true;
   reason: string;
 };
-type ScoreValue = number | SkippedScore | {
-  value: number;
+type ScoreValue<Evidence = unknown> = number | SkippedScore | {
+  /** From 0 to 1. Without it, the weighted mean of `criteria`. */
+  value?: number;
   passed?: boolean;
   explanation?: string;
-  /** Any JSON-convertible value. */
-  evidence?: unknown;
-  /** Judge-agent evidence, never inferred from AUT usage. */
+  /**
+   * Any JSON-convertible value: what the scorer observed. A checkpoint's
+   * evidence reaches later scorers through `checkpoint(rule)`.
+   */
+  evidence?: Evidence;
+  /**
+   * Per-criterion results, each shown on its own; see `CriterionResult`.
+   * Return `{ name, reached: false }` for a check that never ran.
+   */
+  criteria?: CriterionInput[];
+  /** What a model-backed scorer saw and said; never counted as AUT usage. */
   judge?: JudgeRunInfo;
 };
 type ScoreResult = {
@@ -548,8 +676,15 @@ type ScoreResult = {
   kind: 'predicate' | 'judge';
   value?: number;
   passed?: boolean;
+  /** The pass mark the value was held to (1 unless the rule set `passAt`). */
+  passAt?: number;
   explanation?: string;
   evidence?: JsonValue;
+  /** A `choices` judge's answer, such as `B`. */
+  choice?: string;
+  criteria?: CriterionResult[];
+  /** Files its scorer saved with `attach()`, by name. */
+  attachments?: string[];
   judge?: JudgeRunInfo;
   durationMs: number;
   error?: RecordedError;
@@ -578,25 +713,80 @@ type ScoringContext = {
    * example, in a container that is gone by final scoring).
    */
   checkpoints: readonly CheckpointResult[];
+  /**
+   * The latest result of a checkpoint in this scenario's transcript, looked
+   * up by the rule itself, so renaming it cannot break the lookup. A
+   * predicate's `evidence` keeps its type. Undefined until that step has
+   * run; throws for a rule that is not in the transcript.
+   */
+  checkpoint<Evidence>(rule: PredicateScorer<Evidence>): CheckpointOutcome<Evidence> | undefined;
+  checkpoint(rule: JudgeRule | ExpectToolCallStep): CheckpointOutcome<JsonValue> | undefined;
+  /**
+   * Saves a file with the run's results, like the agent's `attach()`: for
+   * example a container's log, or a file the agent reported, read before the
+   * workspace is gone. Same names, limits and rules.
+   */
+  attach(name: string, content: string | Uint8Array): Promise<void>;
+};
+/**
+ * A checkpoint's result with typed evidence. Evidence is saved as JSON, so
+ * it arrives as its JSON form: a `Date` becomes a string, `undefined` fields
+ * are dropped.
+ */
+type CheckpointOutcome<Evidence> = Omit<CheckpointResult, 'evidence'> & {
+  evidence?: Evidence;
 };
-type PredicateScorer = {
+type PredicateScorer<Evidence = unknown> = {
   kind: 'predicate';
   name: string;
+  /** What it checks, in a sentence, shown wherever the scorer is. */
+  description?: string;
   supportsPartial?: boolean;
   /** A numeric result passes when it is at least this value. Default: only 1 passes. */
   passAt?: number;
-  run(context: ScoringContext): boolean | ScoreValue | Promise<boolean | ScoreValue>;
+  /**
+   * The checks it reports, declared up front so they show before a run.
+   * A declared criterion it leaves out is recorded as not reached; one it
+   * did not declare is an error.
+   */
+  criteria?: JudgeCriterion[];
+  run(context: ScoringContext): boolean | ScoreValue<Evidence> | Promise<boolean | ScoreValue<Evidence>>;
 };
+/** What a judge sees: one named input, read from the run. */
+type JudgeInput = (scoring: ScoringContext) => JsonValue | undefined | Promise<JsonValue | undefined>;
+/**
+ * A rubric-graded scorer. EvalKit builds the prompt from the rubric, the
+ * inputs and the answer format, runs the judge agent, checks its answer
+ * (asking once more if it is not in the format asked for), and records
+ * what it saw and said.
+ */
 type JudgeRule = {
   kind: 'judge';
   name: string;
   rubric: string;
+  /** What it checks, in a sentence, shown wherever the judge is. */
+  description?: string;
+  /**
+   * The agent that judges, made with `defineAgent()`. Default: `judge` in
+   * evalkit.config.ts. It gets the run's parameters, so a matrix axis can
+   * choose its model.
+   */
+  agent?: AutAdapter;
+  /**
+   * What the judge sees, by name. Default: `conversation` (this turn at a
+   * checkpoint, the whole run in final scoring) and `reply` (the last reply).
+   */
+  inputs?: Readonly<Record<string, JudgeInput>>;
+  /** How it answers. Default `verdict`: a value from 0 to 1 and a reason. */
+  answer: JudgeAnswer;
+  /** A value passes when it is at least this. Default: only 1 passes. */
+  passAt?: number;
   /** Run even when the scenario stopped early or execution failed. */
   supportsPartial?: boolean;
 };
 type ScoringRule = PredicateScorer | JudgeRule;
 /**
- * A number measured for every trial, such as tokens, retries or web searches.
+ * A number measured for every run, such as tokens, retries or web searches.
  * Metrics never decide pass or fail; they are averaged per run and shown on
  * the scoreboard. Return undefined when there is nothing to measure.
  */
@@ -609,9 +799,9 @@ type Metric = {
   better?: 'lower' | 'higher';
   run(context: ScoringContext): number | undefined | Promise<number | undefined>;
 };
-/** A metric's value for one trial. */
+/** A metric's value for one run. */
 type MetricResult = {
-  name: string;
+  name: MetricName;
   /** Absent when the metric measured nothing or failed. */
   value?: number;
   unit?: string;
@@ -624,22 +814,23 @@ type EvalPolicy = {
   timeoutMs?: number;
   /** Stop later transcript steps after a failed checkpoint assertion. Default false. */
   failfast?: boolean;
-  /** Number of independent trials requested by local and hosted runners. */
-  trials?: number;
 };
 type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
   id: string;
   name?: string;
   agent: TAgent;
-  /** Separate judge agent; its model and tools are never inherited from the AUT. */
-  judge?: AutAdapter;
-  /** Data and files the scenario uses; each trial gets its own copy. */
+  /** Data and files the scenario uses; each run gets its own copy. */
   datasets?: readonly Dataset[];
   transcript: TranscriptStep[];
   scoring: ScoringRule[];
-  /** Numbers to measure for every trial; they never affect pass or fail. */
+  /** Numbers to measure for every run; they never affect pass or fail. */
   metrics?: readonly Metric[];
   policy?: EvalPolicy;
+  /**
+   * How many runs to start each time the scenario runs, as one batch: repeats
+   * under identical conditions, to see how often it passes. Default 1.
+   */
+  repeat?: number;
   /** Any JSON-convertible object; converted when the eval is defined. */
   metadata?: Record<string, unknown>;
   workspace?: EvalWorkspace;
@@ -647,7 +838,7 @@ type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
 /** How EvalKit treats this scenario's candidate workspace. */
 type EvalWorkspace = {
   /**
-   * Paths to leave out of the trial's snapshot, read like `.gitignore`: globs
+   * Paths to leave out of the run's snapshot, read like `.gitignore`: globs
    * relative to the workspace, where a pattern without `/` matches that name
    * at any depth and later patterns win. They come after the project's
    * `artifacts.exclude` (default `DEFAULT_ARTIFACT_EXCLUDE`), so `!pattern`
@@ -666,7 +857,18 @@ type EvalSuiteMetadata = {
   name?: string;
   evalIds: string[];
 };
-type RunStatus = 'running' | 'completed' | 'failed' | 'cancelled';
+/**
+ * How a run ended: `passed` or `failed` by its scores, `errored` when it could
+ * not finish (the agent, a dataset or EvalKit failed), `cancelled` by Ctrl+C.
+ */
+type RunStatus = 'running' | 'passed' | 'failed' | 'errored' | 'cancelled';
+/** Runs launched together as repeats of one configuration (`repeat`). */
+type Batch = {
+  id: string;
+  /** Zero-based position in the batch. */
+  index: number;
+  size: number;
+};
 /**
  * Where a run's eval code came from: the repository, eval project and commit.
  * Results are tagged with it; they are never stored in the repository itself.
@@ -701,7 +903,7 @@ type RunProcess = {
   hostname: string;
 };
 type RunMetadata = {
-  schemaVersion: 2 | 3 | 4 | 5;
+  schemaVersion: 6;
   runId: string;
   runUri: ResourceUri<'run'>;
   evalId: string;
@@ -713,36 +915,21 @@ type RunMetadata = {
   };
   aut?: AutIdentity;
   startedAt: string;
-  /** Present from schema version 4 when the run's commit could be determined. */
+  /** Present when the run's commit could be determined. */
   source?: RunSource;
-  /** From schema version 5. */
   process?: RunProcess;
-};
-type TrialMetadata = {
-  schemaVersion: 2 | 3 | 4 | 5;
-  runId: string;
-  runUri: ResourceUri<'run'>;
-  trialId: string;
-  trialUri: ResourceUri<'trial'>;
-  trialIndex: number;
-  evalId: string;
-  parameters?: JsonObject;
-  matrix?: {
-    id: string;
-    cellKey: string;
-  };
-  aut?: AutIdentity;
-  startedAt: string;
+  /** Set when this run is one of several repeats launched together. */
+  batch?: Batch;
   /** The effective snapshot ignore patterns: the project's `artifacts.exclude`, then the scenario's `workspace.ignore`. */
   workspaceIgnore?: string[];
-  /** From schema version 5: the datasets this trial used, with content digests. */
-  datasets?: TrialDataset[];
+  /** The datasets this run used, with content digests. */
+  datasets?: RunDataset[];
 };
-type TrialDataset = {
+type RunDataset = {
   id: string;
   visibility: DatasetVisibility;
   source: 'static' | 'computed';
-  /** SHA-256 of the trial's content: file paths and bytes, and the canonical value. */
+  /** SHA-256 of the run's content: file paths and bytes, and the canonical value. */
   digest: string;
 };
 type CheckpointResult = {
@@ -752,8 +939,13 @@ type CheckpointResult = {
   status: 'passed' | 'failed' | 'error' | 'skipped';
   value?: number;
   passed?: boolean;
+  passAt?: number;
   explanation?: string;
   evidence?: JsonValue;
+  choice?: string;
+  criteria?: CriterionResult[];
+  /** Files its scorer saved with `attach()`, by name. */
+  attachments?: string[];
   judge?: JudgeRunInfo;
   durationMs?: number;
   error?: RecordedError;
@@ -762,7 +954,21 @@ type CheckpointResult = {
     id: string;
   };
 };
-type TrialScoring = {
+/**
+ * Metrics EvalKit records for every run; scenario metrics may not reuse
+ * these names.
+ *
+ * - `agent time` (ms): from the run's first event to its last one before
+ *   final scoring.
+ * - `tokens`: the sum of `turnTokens(usage)` over the turns that reported
+ *   usage, cached input included. Absent when no turn reported any.
+ * - `tool calls`: the agent's `tool-call` events.
+ */
+declare const BUILT_IN_METRICS: readonly ['agent time', 'tokens', 'tool calls', 'judge tokens'];
+type BuiltInMetricName = (typeof BUILT_IN_METRICS)[number];
+/** A built-in metric's name (suggested by editors), or a scenario's own. */
+type MetricName = BuiltInMetricName | (string & {});
+type RunScoring = {
   results: ScoreResult[];
   metrics?: MetricResult[];
   checkpoints?: CheckpointResult[];
@@ -771,40 +977,34 @@ type TrialScoring = {
   passed: boolean;
 };
 type ArtifactEntry = {
-  /** Path relative to this trial's artifact root. */
+  /** Path relative to this run's artifact root. */
   path: string;
   kind: 'file' | 'directory';
   size?: number;
 };
-type TrialSummary = {
-  status: RunStatus;
-  endedAt: string;
-  durationMs?: number;
-  scoring?: TrialScoring;
-  artifacts?: ArtifactEntry[];
-  error?: RecordedError;
+/** A file an adapter attached with `attach()`, saved as `artifacts/attachments/<name>`. */
+type AttachmentEntry = {
+  name: string;
+  /** Bytes saved. */
+  size: number;
+  /** The original size, when the file was cut to its last `maxFileBytes`. */
+  truncatedFrom?: number;
 };
 type RunSummary = {
   status: RunStatus;
   endedAt: string;
   durationMs?: number;
-  trialCount: number;
-  passed: number;
-  failed: number;
-  /** Trials stopped by cancellation; counted neither as passed nor failed. */
-  cancelled?: number;
+  scoring?: RunScoring;
+  artifacts?: ArtifactEntry[];
+  attachments?: AttachmentEntry[];
   error?: RecordedError;
 };
-type TrialWriter = {
-  appendEvent(event: TrajectoryEvent): Promise<void>;
-  writeScores(scoring: TrialScoring): Promise<void>;
-  /** Writes a file below this trial's artifact root. */
-  writeArtifact(path: string, data: Uint8Array): Promise<void>;
-  finalize(summary: TrialSummary): Promise<void>;
-};
 type RunWriter = {
   readonly location: string;
-  startTrial(metadata: TrialMetadata): Promise<TrialWriter>;
+  appendEvent(event: TrajectoryEvent): Promise<void>;
+  writeScores(scoring: RunScoring): Promise<void>;
+  /** Writes a file below this run's artifact root. */
+  writeArtifact(path: string, data: Uint8Array): Promise<void>;
   finalize(summary: RunSummary): Promise<void>;
 };
 type ReportStore = {
@@ -812,35 +1012,18 @@ type ReportStore = {
     catalog?: RunCatalog;
   }): Promise<RunWriter>;
 };
-type TrialResult = {
+/** One finished run, as `runEval` returns it. */
+type RunResult = {
   runId: string;
   runUri: ResourceUri<'run'>;
-  trialId: string;
-  trialUri: ResourceUri<'trial'>;
-  trialIndex: number;
   evalId: string;
   status: RunStatus;
+  batch?: Batch;
   reportLocation: string;
   durationMs?: number;
-  scoring?: TrialScoring;
+  scoring?: RunScoring;
   error?: RecordedError;
 };
-type AggregateScoring = {
-  passed: number;
-  failed: number;
-  cancelled?: number;
-  passRate: number;
-  overall?: number;
-};
-type RunResult = TrialResult & {
-  runId: string;
-  trialCount?: number;
-  passed?: number;
-  failed?: number;
-  aggregateScoring?: AggregateScoring;
-  /** Present for an aggregate multi-trial run. */
-  trials?: TrialResult[];
-};
 //#endregion
 //#region ../runner/src/local-report-store.d.ts
 /** Persists each run as a local tree rooted below `rootDirectory`. */
@@ -863,7 +1046,7 @@ export declare function staticDatasetFile(dataset: Dataset, root: string, file:
 //#endregion
 //#region ../runner/src/local-report-reader.d.ts
 export declare const readRunManifest: (root: string, runId: string) => Promise<{
-  readonly schemaVersion: 2 | 3 | 4 | 5;
+  readonly schemaVersion: 6;
   readonly runUri: string;
   readonly evalId: string;
   readonly suiteId?: string | undefined;
@@ -899,42 +1082,11 @@ export declare const readRunManifest: (root: string, runId: string) => Promise<{
     readonly pid: number;
     readonly hostname: string;
   } | undefined;
-  readonly status?: "cancelled" | "completed" | "failed" | "running" | undefined;
-}>;
-export declare const readRunSummary: (root: string, runId: string) => Promise<{
-  readonly status: "cancelled" | "completed" | "failed" | "running";
-  readonly endedAt: string;
-  readonly durationMs?: number | undefined;
-  readonly trialCount: number;
-  readonly passed: number;
-  readonly failed: number;
-  readonly cancelled?: number | undefined;
-  readonly error?: {
-    readonly name: string;
-    readonly message: string;
-    readonly stack?: string | undefined;
-  } | undefined;
-}>;
-export declare const readTrialManifest: (root: string, runId: string, trialId: string) => Promise<{
-  readonly schemaVersion: 2 | 3 | 4 | 5;
-  readonly runUri: string;
-  readonly trialUri: string;
-  readonly trialIndex: number;
-  readonly evalId: string;
-  readonly parameters?: {
-    readonly [x: string]: JsonValue;
-  } | undefined;
-  readonly matrix?: {
-    readonly id: string;
-    readonly cellKey: string;
-  } | undefined;
-  readonly aut?: {
-    readonly name?: string | undefined;
-    readonly kind: string;
+  readonly batch?: {
     readonly id: string;
-    readonly version?: string | undefined;
+    readonly index: number;
+    readonly size: number;
   } | undefined;
-  readonly startedAt: string;
   readonly workspaceIgnore?: string[] | undefined;
   readonly datasets?: {
     readonly id: string;
@@ -942,14 +1094,29 @@ export declare const readTrialManifest: (root: string, runId: string, trialId: s
     readonly source: "computed" | "static";
     readonly digest: string;
   }[] | undefined;
-  readonly status?: "cancelled" | "completed" | "failed" | "running" | undefined;
+  readonly status?: "cancelled" | "errored" | "failed" | "passed" | "running" | undefined;
 }>;
-export declare const readTrialSummary: (root: string, runId: string, trialId: string) => Promise<{
-  readonly status: "cancelled" | "completed" | "failed" | "running";
+export declare const readRunSummary: (root: string, runId: string) => Promise<{
+  readonly status: "cancelled" | "errored" | "failed" | "passed" | "running";
   readonly endedAt: string;
   readonly durationMs?: number | undefined;
   readonly scoring?: {
     readonly results: {
+      readonly passAt?: number | undefined;
+      readonly choice?: string | undefined;
+      readonly criteria?: ({
+        readonly name: string;
+        readonly value: number;
+        readonly weight?: number | undefined;
+        readonly explanation?: string | undefined;
+        readonly evidence?: JsonValue | undefined;
+      } | {
+        readonly name: string;
+        readonly reached: false;
+        readonly weight?: number | undefined;
+        readonly explanation?: string | undefined;
+      })[] | undefined;
+      readonly attachments?: string[] | undefined;
       readonly name: string;
       readonly kind: "judge" | "predicate";
       readonly value?: number | undefined;
@@ -963,11 +1130,16 @@ export declare const readTrialSummary: (root: string, runId: string, trialId: st
           readonly name?: string | undefined;
           readonly version?: string | undefined;
         } | undefined;
+        readonly model?: string | undefined;
         readonly usage?: {
           readonly inputTokens?: number | undefined;
           readonly outputTokens?: number | undefined;
+          readonly cachedInputTokens?: number | undefined;
           readonly totalTokens?: number | undefined;
         } | undefined;
+        readonly prompt?: string | undefined;
+        readonly reply?: JsonValue | undefined;
+        readonly attempts?: number | undefined;
         readonly events?: JsonValue[] | undefined;
       } | undefined;
       readonly durationMs: number;
@@ -991,6 +1163,21 @@ export declare const readTrialSummary: (root: string, runId: string, trialId: st
       } | undefined;
     }[] | undefined;
     readonly checkpoints?: {
+      readonly passAt?: number | undefined;
+      readonly choice?: string | undefined;
+      readonly criteria?: ({
+        readonly name: string;
+        readonly value: number;
+        readonly weight?: number | undefined;
+        readonly explanation?: string | undefined;
+        readonly evidence?: JsonValue | undefined;
+      } | {
+        readonly name: string;
+        readonly reached: false;
+        readonly weight?: number | undefined;
+        readonly explanation?: string | undefined;
+      })[] | undefined;
+      readonly attachments?: string[] | undefined;
       readonly step: number;
       readonly kind: "expect-tool-call" | "judge" | "predicate";
       readonly name: string;
@@ -1006,11 +1193,16 @@ export declare const readTrialSummary: (root: string, runId: string, trialId: st
           readonly name?: string | undefined;
           readonly version?: string | undefined;
         } | undefined;
+        readonly model?: string | undefined;
         readonly usage?: {
           readonly inputTokens?: number | undefined;
           readonly outputTokens?: number | undefined;
+          readonly cachedInputTokens?: number | undefined;
           readonly totalTokens?: number | undefined;
         } | undefined;
+        readonly prompt?: string | undefined;
+        readonly reply?: JsonValue | undefined;
+        readonly attempts?: number | undefined;
         readonly events?: JsonValue[] | undefined;
       } | undefined;
       readonly durationMs?: number | undefined;
@@ -1033,14 +1225,34 @@ export declare const readTrialSummary: (root: string, runId: string, trialId: st
     readonly kind: "directory" | "file";
     readonly size?: number | undefined;
   }[] | undefined;
+  readonly attachments?: {
+    readonly name: string;
+    readonly size: number;
+    readonly truncatedFrom?: number | undefined;
+  }[] | undefined;
   readonly error?: {
     readonly name: string;
     readonly message: string;
     readonly stack?: string | undefined;
   } | undefined;
 }>;
-export declare const readTrialScoring: (root: string, runId: string, trialId: string) => Promise<{
+export declare const readRunScoring: (root: string, runId: string) => Promise<{
   readonly results: {
+    readonly passAt?: number | undefined;
+    readonly choice?: string | undefined;
+    readonly criteria?: ({
+      readonly name: string;
+      readonly value: number;
+      readonly weight?: number | undefined;
+      readonly explanation?: string | undefined;
+      readonly evidence?: JsonValue | undefined;
+    } | {
+      readonly name: string;
+      readonly reached: false;
+      readonly weight?: number | undefined;
+      readonly explanation?: string | undefined;
+    })[] | undefined;
+    readonly attachments?: string[] | undefined;
     readonly name: string;
     readonly kind: "judge" | "predicate";
     readonly value?: number | undefined;
@@ -1054,11 +1266,16 @@ export declare const readTrialScoring: (root: string, runId: string, trialId: st
         readonly name?: string | undefined;
         readonly version?: string | undefined;
       } | undefined;
+      readonly model?: string | undefined;
       readonly usage?: {
         readonly inputTokens?: number | undefined;
         readonly outputTokens?: number | undefined;
+        readonly cachedInputTokens?: number | undefined;
         readonly totalTokens?: number | undefined;
       } | undefined;
+      readonly prompt?: string | undefined;
+      readonly reply?: JsonValue | undefined;
+      readonly attempts?: number | undefined;
       readonly events?: JsonValue[] | undefined;
     } | undefined;
     readonly durationMs: number;
@@ -1082,6 +1299,21 @@ export declare const readTrialScoring: (root: string, runId: string, trialId: st
     } | undefined;
   }[] | undefined;
   readonly checkpoints?: {
+    readonly passAt?: number | undefined;
+    readonly choice?: string | undefined;
+    readonly criteria?: ({
+      readonly name: string;
+      readonly value: number;
+      readonly weight?: number | undefined;
+      readonly explanation?: string | undefined;
+      readonly evidence?: JsonValue | undefined;
+    } | {
+      readonly name: string;
+      readonly reached: false;
+      readonly weight?: number | undefined;
+      readonly explanation?: string | undefined;
+    })[] | undefined;
+    readonly attachments?: string[] | undefined;
     readonly step: number;
     readonly kind: "expect-tool-call" | "judge" | "predicate";
     readonly name: string;
@@ -1097,11 +1329,16 @@ export declare const readTrialScoring: (root: string, runId: string, trialId: st
         readonly name?: string | undefined;
         readonly version?: string | undefined;
       } | undefined;
+      readonly model?: string | undefined;
       readonly usage?: {
         readonly inputTokens?: number | undefined;
         readonly outputTokens?: number | undefined;
+        readonly cachedInputTokens?: number | undefined;
         readonly totalTokens?: number | undefined;
       } | undefined;
+      readonly prompt?: string | undefined;
+      readonly reply?: JsonValue | undefined;
+      readonly attempts?: number | undefined;
       readonly events?: JsonValue[] | undefined;
     } | undefined;
     readonly durationMs?: number | undefined;
@@ -1119,7 +1356,7 @@ export declare const readTrialScoring: (root: string, runId: string, trialId: st
   readonly overall?: number | undefined;
   readonly passed: boolean;
 }>;
-export declare function readTrialEvents(root: string, runId: string, trialId: string): Promise<TrajectoryEvent[]>;
+export declare function readRunEvents(root: string, runId: string): Promise<TrajectoryEvent[]>;
 /**
  * True when a run's owning process is known to be gone: it ran on this host
  * and its PID no longer exists. Runs from other hosts are never judged.
@@ -1127,52 +1364,86 @@ export declare function readTrialEvents(root: string, runId: string, trialId: st
 export declare function ownerIsGone(owner: RunProcess | undefined): boolean;
 //#endregion
 //#region ../runner/src/matrix.d.ts
-/** Cells run; cancelled cells are neither passed nor failed. */
+/** Cells and runs; a run that errored counts as failed, a cancelled one as neither. */
 type MatrixSummary = {
   cells: number;
+  runs: number;
   passed: number;
   failed: number;
   cancelled: number;
 };
 /** Bounded workers pull cells from the iterator; no Cartesian job array is allocated. */
-export declare function runMatrix(matrix: EvalMatrix, options: Omit<RunEvalOptions, 'runId' | 'trialId' | 'parameters' | 'matrix'> & {
+export declare function runMatrix(matrix: EvalMatrix, options: Omit<RunEvalOptions, 'runId' | 'parameters' | 'matrix'> & {
   selection?: MatrixSelection;
   /** The suite each cell's run is recorded under, when `suiteId` is not given. */
   suiteFor?: (cell: EvalMatrixCell) => string | undefined;
-  onResult?: (cell: EvalMatrixCell, result: RunResult) => void | Promise<void>;
+  onResult?: (cell: EvalMatrixCell, runs: readonly RunResult[]) => void | Promise<void>;
 }): Effect.Effect<MatrixSummary, unknown>;
 //#endregion
+//#region ../runner/src/upgrade.d.ts
+/** What upgrading a results folder did, or would do with `dryRun`. */
+type UpgradeReport = {
+  /** Old runs converted (or that would be). */
+  upgraded: number;
+  /** Version 6 runs they became. */
+  runs: number;
+  /** Old runs with more than one trial: each trial became a run in a batch. */
+  batches: number;
+  /** Old runs left alone because they are still running. */
+  running: string[];
+};
+/**
+ * Rewrites results from report schema versions 2 to 5 into version 6, in
+ * place (RFD0008). An old run of one trial (most of them) becomes a run with
+ * the same ID, so links to it keep working. An old run of several trials
+ * becomes runs with the trials' IDs, in a batch with the old run's ID.
+ *
+ * Each new run is written under a temporary name and renamed into place;
+ * artifacts are moved, not copied, as the last step before that rename. An
+ * old run is removed only after its new runs exist, and one replaced under
+ * the same ID is set aside first, so a crash at any point leaves the old run
+ * or complete new ones, and re-running finishes the job. Version 6 runs are
+ * left alone.
+ */
+export declare function upgradeReports(root: string, options?: {
+  dryRun?: boolean;
+}): Promise<UpgradeReport>;
+//#endregion
 //#region ../runner/src/workspace.d.ts
-/** Where a trial's persistent workspace lives below `root`. */
-export declare function trialWorkspacePath(root: string, trialId: string): string;
+/** Where a run's persistent workspace lives below `root`. */
+export declare function runWorkspacePath(root: string, runId: string): string;
 //#endregion
 //#region ../runner/src/index.d.ts
 export type RunEvalOptions = {
   report: ReportStore;
+  /** The run's ID; only with one run (no `repeat`). */
   runId?: string;
-  trialId?: string;
   suiteId?: string;
-  /** Overrides the eval policy's requested number of independent trials. */
-  trials?: number;
-  /** Parameters made available to the AUT for this invocation. */
+  /** Overrides the scenario's `repeat`: how many runs to start, as one batch. */
+  repeat?: number;
+  /** Parameters made available to the AUT, and to judges, for this invocation. */
   parameters?: JsonObject;
-  /** Matrix provenance retained with every aggregate and trial manifest. */
+  /** The agent for judges that name none (`judge` in evalkit.config.ts). */
+  judge?: AutAdapter;
+  /** Matrix provenance recorded in every run's manifest. */
   matrix?: RunMetadata['matrix'];
   /** Selects a declared AUT runtime such as local, sandbox, or remote. */
   runtime?: AgentRuntimeName;
-  /** Maximum number of trials from this aggregate run executing concurrently. */
+  /** Maximum number of this batch's runs executing concurrently. */
   concurrency?: number;
-  /** Optional shared limiter for coordinating trials across aggregate runs. */
+  /** Optional shared limiter for coordinating runs across batches. */
   semaphore?: Effect.Semaphore;
-  /** Base directory for persistent local trial workspaces. Omit for an ephemeral OS temp workspace. */
+  /** Base directory for persistent local run workspaces. Omit for an ephemeral OS temp workspace. */
   workspaceRoot?: string;
-  /** With `workspaceRoot`: which trial workspaces to keep afterwards. Default `'failed'`. */
+  /** With `workspaceRoot`: which run workspaces to keep afterwards. Default `'failed'`. */
   keepWorkspaces?: KeepSandboxes;
   /**
    * Candidate paths left out of the artifact snapshot, before the scenario's
    * own `workspace.ignore`. Default `DEFAULT_ARTIFACT_EXCLUDE`.
    */
   artifactExclude?: readonly string[];
+  /** Size limits for files adapters and scorers `attach()`. Default `DEFAULT_ATTACHMENT_LIMITS`. */
+  attachmentLimits?: AttachmentLimits;
   /** Repository and commit the eval code came from, recorded in the manifest. */
   source?: RunSource;
   /** The project's evals and suites at run time, saved beside the manifest. */
@@ -1180,7 +1451,7 @@ export type RunEvalOptions = {
   /** Project root; dataset paths resolve against it. Default: the working directory. */
   root?: string;
   /**
-   * Aborting stops new trials from starting; running trials stop at their next
+   * Aborting stops new runs from starting; running ones stop at their next
    * step and are finalized as cancelled. Also passed to dataset loaders.
    */
   signal?: AbortSignal;
@@ -1188,20 +1459,23 @@ export type RunEvalOptions = {
    * Observes every trajectory event after it is recorded, for live progress.
    * It must not throw; an error it throws is ignored.
    */
-  onEvent?: (trial: TrialRef, event: TrajectoryEvent) => void;
+  onEvent?: (run: RunRef, event: TrajectoryEvent) => void;
   now?: () => Date;
 };
-/** Identifies the trial an observed event belongs to. */
-export type TrialRef = {
+/** Identifies the run an observed event belongs to. */
+export type RunRef = {
   runId: string;
-  trialId: string;
-  trialIndex: number;
+  batch?: Batch;
   evalId: string;
   suiteId?: string;
   parameters?: JsonObject;
 };
-/** Effect-native aggregate eval execution entry point. */
-export declare function runEval(definition: EvalDefinition, options: RunEvalOptions): Effect.Effect<RunResult, unknown>;
+/**
+ * Runs a scenario: once, or `repeat` times as one batch. Resolves to one
+ * result per run started; runs never started because of cancellation are
+ * left out.
+ */
+export declare function runEval(definition: EvalDefinition, options: RunEvalOptions): Effect.Effect<RunResult[], unknown>;
 //#endregion
-export type { DatasetFileEntry, MatrixSummary };
+export type { DatasetFileEntry, MatrixSummary, UpgradeReport };
 //# sourceMappingURL=runner.d.mts.map