@leostera/evalkit 0.7.0 → 0.9.0

package/dist/runner.d.mts

@@ -41,13 +41,13 @@ export declare class CheckpointExecutionError extends CheckpointExecutionError_b
 declare const CancelledError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
   readonly _tag: "CancelledError";
 } & Readonly<A>;
-/** The run was cancelled (Ctrl+C); the trial stops and is finalized as cancelled. */
+/** The run was cancelled (Ctrl+C); the run stops and is finalized as cancelled. */
 export declare class CancelledError extends CancelledError_base<{
   message: string;
 }> {}
 //#endregion
 //#region ../core/src/identity.d.ts
-declare const ResourceKindSchema: Schema.Literal<["run", "trial", "artifact"]>;
+declare const ResourceKindSchema: Schema.Literal<["run", "artifact"]>;
 type ResourceKind = typeof ResourceKindSchema.Type;
 type ResourceUri<TKind extends ResourceKind = ResourceKind> = `evalkit:${TKind}:${string}`;
 //#endregion
@@ -59,14 +59,14 @@ type DatasetFileSource = string | Uint8Array | {
   file: string;
 };
 /**
- * Files for the trial workspace: a directory relative to the project root, or
+ * Files for the run workspace: a directory relative to the project root, or
  * a map from workspace path (below the dataset's mount) to its content.
  */
 type DatasetFiles = string | Readonly<Record<string, DatasetFileSource>>;
-/** What one trial receives from a dataset. */
+/** What one run receives from a dataset. */
 type DatasetContent = {
   files?: DatasetFiles;
-  /** Any JSON-convertible value; converted once when the trial starts. */
+  /** Any JSON-convertible value; converted once when the run starts. */
   value?: unknown;
 };
 type DatasetLoadContext = {
@@ -74,12 +74,12 @@ type DatasetLoadContext = {
   root: string;
   signal: AbortSignal;
 };
-type DatasetTrialContext = {
+type DatasetRunContext = {
   evalId: string;
   runId: string;
-  trialId: string;
-  trialIndex: number;
-  /** Per-trial random seed, so repeated trials can choose different data. */
+  /** Set when the run is one of several repeats launched together. */
+  batch?: Batch;
+  /** Per-run random seed, so repeated runs can choose different data. */
   seed: number;
   /** The matrix cell's parameters, or `{}`. */
   parameters: JsonObject;
@@ -93,7 +93,7 @@ type DatasetSource = {
 } | {
   kind: 'computed';
   onLoad: (load: DatasetLoadContext) => DatasetContent | Promise<DatasetContent>;
-  onRun?: (loaded: DatasetContent, trial: DatasetTrialContext) => DatasetContent | Promise<DatasetContent>;
+  onRun?: (loaded: DatasetContent, run: DatasetRunContext) => DatasetContent | Promise<DatasetContent>;
 };
 /** A checked dataset, as returned by `defineDataset`. Evals accept only these. */
 type Dataset = {
@@ -222,7 +222,7 @@ type EvalCatalogScorer = {
   definedAt?: string;
 };
 type EvalCatalogStep =
-/** `message` is absent when it is computed for each trial. */
+/** `message` is absent when it is computed for each run. */
 {
   kind: 'user';
   message?: string;
@@ -248,9 +248,9 @@ type EvalCatalogEntry = {
   /** The separate agent that runs judge rules, if any. */
   judgeAgent?: EvalCatalogAgent;
   datasets: EvalCatalogDataset[];
-  trialCount: number;
+  /** Runs started each time the scenario runs (`repeat`). */
+  repeat: number;
   policy: {
-    trials: number;
     timeoutMs?: number;
     failfast?: boolean;
   };
@@ -296,6 +296,32 @@ type EvalMatrix = EvalMatrixDefinition & {
 //#endregion
 //#region ../core/src/config.d.ts
 type KeepSandboxes = 'failed' | 'all' | 'none';
+type AttachmentLimits = {
+  /** A larger attachment keeps its last this-many bytes. Default 5 MB. */
+  maxFileBytes?: number;
+  /** All of one run's attachments; past it, `attach()` rejects. Default 20 MB. */
+  maxRunBytes?: number;
+};
+//#endregion
+//#region ../core/src/usage.d.ts
+/**
+ * Token usage an agent reports for one turn. EvalKit never measures or infers
+ * it; adapters copy it from their provider.
+ */
+type Usage = {
+  /**
+   * Every input token the model read this turn: the prompt, history and tool
+   * results, **including** tokens read from or written to a prompt cache.
+   * (Some providers report cached tokens separately; add them in.)
+   */
+  inputTokens?: number;
+  /** Every token the model produced, including reasoning tokens. */
+  outputTokens?: number;
+  /** Of `inputTokens`, how many were read from a prompt cache. Informational. */
+  cachedInputTokens?: number;
+  /** The provider's own total; used only when input and output are not reported. */
+  totalTokens?: number;
+};
 //#endregion
 //#region ../core/src/index.d.ts
 type JsonPrimitive = boolean | number | string | null;
@@ -322,11 +348,6 @@ type AgentRuntime = {
   configuration?: JsonObject;
 };
 type AgentRuntimes = Partial<Record<AgentRuntimeName, AgentRuntime>>;
-type Usage = {
-  inputTokens?: number;
-  outputTokens?: number;
-  totalTokens?: number;
-};
 type RecordedError = {
   name: string;
   message: string;
@@ -372,7 +393,7 @@ type AutEvent = {
   timestamp: string;
 };
 type RunnerEvent = {
-  kind: 'trial-started';
+  kind: 'run-started';
   timestamp: string;
 } | {
   kind: 'transcript-step-started';
@@ -420,7 +441,7 @@ type RunnerEvent = {
   error: RecordedError;
   timestamp: string;
 } | {
-  kind: 'trial-completed';
+  kind: 'run-completed';
   timestamp: string;
 } | {
   kind: 'error';
@@ -433,7 +454,7 @@ type TrajectoryEvent = (AutEvent & {
   source: 'runner';
 });
 type WorkspaceView = {
-  /** Absolute path to this trial-scoped workspace. Do not persist it in reports. */
+  /** Absolute path to this run-scoped workspace. Do not persist it in reports. */
   root: string;
 };
 type CandidateWorkspace = WorkspaceView;
@@ -441,8 +462,8 @@ type EvaluatorWorkspace = WorkspaceView;
 type AutContext = {
   runId: string;
   evalId: string;
-  trialId: string;
-  trialIndex: number;
+  /** Set when this run is one of several repeats launched together. */
+  batch?: Batch;
   metadata: JsonObject;
   /** Run-time parameters supplied by the eval definition or CLI invocation. */
   parameters?: JsonObject;
@@ -463,6 +484,14 @@ type AutAdapter<TMessage = string> = {
     runtime?: AgentRuntime;
     /** Optional: stream events as they happen. Returning a turn result from `send` is enough for most agents. */
     onEvent: (event: AutEventInput) => void | Promise<void>;
+    /**
+     * Saves a file with the run's results, such as the agent's raw log,
+     * whether or not its sandbox is kept. `name` is a relative path of
+     * letters, digits, `.`, `_` and `-` (`logs/agent.log`); each name once.
+     * A file over the size limit keeps its last bytes; past the run's
+     * total limit it is rejected. See `artifacts.attachments` in the config.
+     */
+    attach: (name: string, content: string | Uint8Array) => Promise<void>;
   }): Promise<AutSession<TMessage>>;
 };
 type AutSession<TMessage = string> = {
@@ -480,13 +509,14 @@ type UserMessageInput = {
   parameters: JsonObject;
   /** Values of the scenario's candidate datasets, by dataset ID (never evaluator ones). */
   data: JsonObject;
-  /** The trial's random seed (also in `context.metadata.randomSeed`). */
+  /** The run's random seed (also in `context.metadata.randomSeed`). */
   seed: number;
-  trialIndex: number;
+  /** Set when this run is one of several repeats launched together. */
+  batch?: Batch;
 };
 type UserStep = {
   kind: 'user';
-  /** A fixed message, or one computed for each trial. */
+  /** A fixed message, or one computed for each run. */
   message: string | ((input: UserMessageInput) => string);
 };
 type AgentStep = {
@@ -534,12 +564,15 @@ type SkippedScore = {
   skipped: true;
   reason: string;
 };
-type ScoreValue = number | SkippedScore | {
+type ScoreValue<Evidence = unknown> = number | SkippedScore | {
   value: number;
   passed?: boolean;
   explanation?: string;
-  /** Any JSON-convertible value. */
-  evidence?: unknown;
+  /**
+   * Any JSON-convertible value: what the scorer observed. A checkpoint's
+   * evidence reaches later scorers through `checkpoint(rule)`.
+   */
+  evidence?: Evidence;
   /** Judge-agent evidence, never inferred from AUT usage. */
   judge?: JudgeRunInfo;
 };
@@ -578,14 +611,36 @@ type ScoringContext = {
    * example, in a container that is gone by final scoring).
    */
   checkpoints: readonly CheckpointResult[];
+  /**
+   * The latest result of a checkpoint in this scenario's transcript, looked
+   * up by the rule itself, so renaming it cannot break the lookup. A
+   * predicate's `evidence` keeps its type. Undefined until that step has
+   * run; throws for a rule that is not in the transcript.
+   */
+  checkpoint<Evidence>(rule: PredicateScorer<Evidence>): CheckpointOutcome<Evidence> | undefined;
+  checkpoint(rule: JudgeRule | ExpectToolCallStep): CheckpointOutcome<JsonValue> | undefined;
+  /**
+   * Saves a file with the run's results, like the agent's `attach()`: for
+   * example a container's log, or a file the agent reported, read before the
+   * workspace is gone. Same names, limits and rules.
+   */
+  attach(name: string, content: string | Uint8Array): Promise<void>;
+};
+/**
+ * A checkpoint's result with typed evidence. Evidence is saved as JSON, so
+ * it arrives as its JSON form: a `Date` becomes a string, `undefined` fields
+ * are dropped.
+ */
+type CheckpointOutcome<Evidence> = Omit<CheckpointResult, 'evidence'> & {
+  evidence?: Evidence;
 };
-type PredicateScorer = {
+type PredicateScorer<Evidence = unknown> = {
   kind: 'predicate';
   name: string;
   supportsPartial?: boolean;
   /** A numeric result passes when it is at least this value. Default: only 1 passes. */
   passAt?: number;
-  run(context: ScoringContext): boolean | ScoreValue | Promise<boolean | ScoreValue>;
+  run(context: ScoringContext): boolean | ScoreValue<Evidence> | Promise<boolean | ScoreValue<Evidence>>;
 };
 type JudgeRule = {
   kind: 'judge';
@@ -596,7 +651,7 @@ type JudgeRule = {
 };
 type ScoringRule = PredicateScorer | JudgeRule;
 /**
- * A number measured for every trial, such as tokens, retries or web searches.
+ * A number measured for every run, such as tokens, retries or web searches.
  * Metrics never decide pass or fail; they are averaged per run and shown on
  * the scoreboard. Return undefined when there is nothing to measure.
  */
@@ -609,9 +664,9 @@ type Metric = {
   better?: 'lower' | 'higher';
   run(context: ScoringContext): number | undefined | Promise<number | undefined>;
 };
-/** A metric's value for one trial. */
+/** A metric's value for one run. */
 type MetricResult = {
-  name: string;
+  name: MetricName;
   /** Absent when the metric measured nothing or failed. */
   value?: number;
   unit?: string;
@@ -624,8 +679,6 @@ type EvalPolicy = {
   timeoutMs?: number;
   /** Stop later transcript steps after a failed checkpoint assertion. Default false. */
   failfast?: boolean;
-  /** Number of independent trials requested by local and hosted runners. */
-  trials?: number;
 };
 type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
   id: string;
@@ -633,15 +686,33 @@ type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
   agent: TAgent;
   /** Separate judge agent; its model and tools are never inherited from the AUT. */
   judge?: AutAdapter;
-  /** Data and files the scenario uses; each trial gets its own copy. */
+  /** Data and files the scenario uses; each run gets its own copy. */
   datasets?: readonly Dataset[];
   transcript: TranscriptStep[];
   scoring: ScoringRule[];
-  /** Numbers to measure for every trial; they never affect pass or fail. */
+  /** Numbers to measure for every run; they never affect pass or fail. */
   metrics?: readonly Metric[];
   policy?: EvalPolicy;
+  /**
+   * How many runs to start each time the scenario runs, as one batch: repeats
+   * under identical conditions, to see how often it passes. Default 1.
+   */
+  repeat?: number;
   /** Any JSON-convertible object; converted when the eval is defined. */
   metadata?: Record<string, unknown>;
+  workspace?: EvalWorkspace;
+};
+/** How EvalKit treats this scenario's candidate workspace. */
+type EvalWorkspace = {
+  /**
+   * Paths to leave out of the run's snapshot, read like `.gitignore`: globs
+   * relative to the workspace, where a pattern without `/` matches that name
+   * at any depth and later patterns win. They come after the project's
+   * `artifacts.exclude` (default `DEFAULT_ARTIFACT_EXCLUDE`), so `!pattern`
+   * can keep a path the project leaves out, even inside an ignored directory:
+   * `['!_build/doc/**']` keeps generated docs while dropping the rest of `_build`.
+   */
+  ignore?: readonly string[];
 };
 type EvalRegistryMetadata = {
   id: string;
@@ -653,7 +724,18 @@ type EvalSuiteMetadata = {
   name?: string;
   evalIds: string[];
 };
-type RunStatus = 'running' | 'completed' | 'failed' | 'cancelled';
+/**
+ * How a run ended: `passed` or `failed` by its scores, `errored` when it could
+ * not finish (the agent, a dataset or EvalKit failed), `cancelled` by Ctrl+C.
+ */
+type RunStatus = 'running' | 'passed' | 'failed' | 'errored' | 'cancelled';
+/** Runs launched together as repeats of one configuration (`repeat`). */
+type Batch = {
+  id: string;
+  /** Zero-based position in the batch. */
+  index: number;
+  size: number;
+};
 /**
  * Where a run's eval code came from: the repository, eval project and commit.
  * Results are tagged with it; they are never stored in the repository itself.
@@ -688,7 +770,7 @@ type RunProcess = {
   hostname: string;
 };
 type RunMetadata = {
-  schemaVersion: 2 | 3 | 4 | 5;
+  schemaVersion: 6;
   runId: string;
   runUri: ResourceUri<'run'>;
   evalId: string;
@@ -700,34 +782,21 @@ type RunMetadata = {
   };
   aut?: AutIdentity;
   startedAt: string;
-  /** Present from schema version 4 when the run's commit could be determined. */
+  /** Present when the run's commit could be determined. */
   source?: RunSource;
-  /** From schema version 5. */
   process?: RunProcess;
-};
-type TrialMetadata = {
-  schemaVersion: 2 | 3 | 4 | 5;
-  runId: string;
-  runUri: ResourceUri<'run'>;
-  trialId: string;
-  trialUri: ResourceUri<'trial'>;
-  trialIndex: number;
-  evalId: string;
-  parameters?: JsonObject;
-  matrix?: {
-    id: string;
-    cellKey: string;
-  };
-  aut?: AutIdentity;
-  startedAt: string;
-  /** From schema version 5: the datasets this trial used, with content digests. */
-  datasets?: TrialDataset[];
-};
-type TrialDataset = {
+  /** Set when this run is one of several repeats launched together. */
+  batch?: Batch;
+  /** The effective snapshot ignore patterns: the project's `artifacts.exclude`, then the scenario's `workspace.ignore`. */
+  workspaceIgnore?: string[];
+  /** The datasets this run used, with content digests. */
+  datasets?: RunDataset[];
+};
+type RunDataset = {
   id: string;
   visibility: DatasetVisibility;
   source: 'static' | 'computed';
-  /** SHA-256 of the trial's content: file paths and bytes, and the canonical value. */
+  /** SHA-256 of the run's content: file paths and bytes, and the canonical value. */
   digest: string;
 };
 type CheckpointResult = {
@@ -747,7 +816,21 @@ type CheckpointResult = {
     id: string;
   };
 };
-type TrialScoring = {
+/**
+ * Metrics EvalKit records for every run; scenario metrics may not reuse
+ * these names.
+ *
+ * - `agent time` (ms): from the run's first event to its last one before
+ *   final scoring.
+ * - `tokens`: the sum of `turnTokens(usage)` over the turns that reported
+ *   usage, cached input included. Absent when no turn reported any.
+ * - `tool calls`: the agent's `tool-call` events.
+ */
+declare const BUILT_IN_METRICS: readonly ['agent time', 'tokens', 'tool calls'];
+type BuiltInMetricName = (typeof BUILT_IN_METRICS)[number];
+/** A built-in metric's name (suggested by editors), or a scenario's own. */
+type MetricName = BuiltInMetricName | (string & {});
+type RunScoring = {
   results: ScoreResult[];
   metrics?: MetricResult[];
   checkpoints?: CheckpointResult[];
@@ -756,40 +839,34 @@ type TrialScoring = {
   passed: boolean;
 };
 type ArtifactEntry = {
-  /** Path relative to this trial's artifact root. */
+  /** Path relative to this run's artifact root. */
   path: string;
   kind: 'file' | 'directory';
   size?: number;
 };
-type TrialSummary = {
-  status: RunStatus;
-  endedAt: string;
-  durationMs?: number;
-  scoring?: TrialScoring;
-  artifacts?: ArtifactEntry[];
-  error?: RecordedError;
+/** A file an adapter attached with `attach()`, saved as `artifacts/attachments/<name>`. */
+type AttachmentEntry = {
+  name: string;
+  /** Bytes saved. */
+  size: number;
+  /** The original size, when the file was cut to its last `maxFileBytes`. */
+  truncatedFrom?: number;
 };
 type RunSummary = {
   status: RunStatus;
   endedAt: string;
   durationMs?: number;
-  trialCount: number;
-  passed: number;
-  failed: number;
-  /** Trials stopped by cancellation; counted neither as passed nor failed. */
-  cancelled?: number;
+  scoring?: RunScoring;
+  artifacts?: ArtifactEntry[];
+  attachments?: AttachmentEntry[];
   error?: RecordedError;
 };
-type TrialWriter = {
-  appendEvent(event: TrajectoryEvent): Promise<void>;
-  writeScores(scoring: TrialScoring): Promise<void>;
-  /** Writes a file below this trial's artifact root. */
-  writeArtifact(path: string, data: Uint8Array): Promise<void>;
-  finalize(summary: TrialSummary): Promise<void>;
-};
 type RunWriter = {
   readonly location: string;
-  startTrial(metadata: TrialMetadata): Promise<TrialWriter>;
+  appendEvent(event: TrajectoryEvent): Promise<void>;
+  writeScores(scoring: RunScoring): Promise<void>;
+  /** Writes a file below this run's artifact root. */
+  writeArtifact(path: string, data: Uint8Array): Promise<void>;
   finalize(summary: RunSummary): Promise<void>;
 };
 type ReportStore = {
@@ -797,35 +874,18 @@ type ReportStore = {
     catalog?: RunCatalog;
   }): Promise<RunWriter>;
 };
-type TrialResult = {
+/** One finished run, as `runEval` returns it. */
+type RunResult = {
   runId: string;
   runUri: ResourceUri<'run'>;
-  trialId: string;
-  trialUri: ResourceUri<'trial'>;
-  trialIndex: number;
   evalId: string;
   status: RunStatus;
+  batch?: Batch;
   reportLocation: string;
   durationMs?: number;
-  scoring?: TrialScoring;
+  scoring?: RunScoring;
   error?: RecordedError;
 };
-type AggregateScoring = {
-  passed: number;
-  failed: number;
-  cancelled?: number;
-  passRate: number;
-  overall?: number;
-};
-type RunResult = TrialResult & {
-  runId: string;
-  trialCount?: number;
-  passed?: number;
-  failed?: number;
-  aggregateScoring?: AggregateScoring;
-  /** Present for an aggregate multi-trial run. */
-  trials?: TrialResult[];
-};
 //#endregion
 //#region ../runner/src/local-report-store.d.ts
 /** Persists each run as a local tree rooted below `rootDirectory`. */
@@ -848,7 +908,7 @@ export declare function staticDatasetFile(dataset: Dataset, root: string, file:
 //#endregion
 //#region ../runner/src/local-report-reader.d.ts
 export declare const readRunManifest: (root: string, runId: string) => Promise<{
-  readonly schemaVersion: 2 | 3 | 4 | 5;
+  readonly schemaVersion: 6;
   readonly runUri: string;
   readonly evalId: string;
   readonly suiteId?: string | undefined;
@@ -884,52 +944,22 @@ export declare const readRunManifest: (root: string, runId: string) => Promise<{
     readonly pid: number;
     readonly hostname: string;
   } | undefined;
-  readonly status?: "cancelled" | "completed" | "failed" | "running" | undefined;
-}>;
-export declare const readRunSummary: (root: string, runId: string) => Promise<{
-  readonly status: "cancelled" | "completed" | "failed" | "running";
-  readonly endedAt: string;
-  readonly durationMs?: number | undefined;
-  readonly trialCount: number;
-  readonly passed: number;
-  readonly failed: number;
-  readonly cancelled?: number | undefined;
-  readonly error?: {
-    readonly name: string;
-    readonly message: string;
-    readonly stack?: string | undefined;
-  } | undefined;
-}>;
-export declare const readTrialManifest: (root: string, runId: string, trialId: string) => Promise<{
-  readonly schemaVersion: 2 | 3 | 4 | 5;
-  readonly runUri: string;
-  readonly trialUri: string;
-  readonly trialIndex: number;
-  readonly evalId: string;
-  readonly parameters?: {
-    readonly [x: string]: JsonValue;
-  } | undefined;
-  readonly matrix?: {
-    readonly id: string;
-    readonly cellKey: string;
-  } | undefined;
-  readonly aut?: {
-    readonly name?: string | undefined;
-    readonly kind: string;
+  readonly batch?: {
     readonly id: string;
-    readonly version?: string | undefined;
+    readonly index: number;
+    readonly size: number;
   } | undefined;
-  readonly startedAt: string;
+  readonly workspaceIgnore?: string[] | undefined;
   readonly datasets?: {
     readonly id: string;
     readonly visibility: "candidate" | "evaluator";
     readonly source: "computed" | "static";
     readonly digest: string;
   }[] | undefined;
-  readonly status?: "cancelled" | "completed" | "failed" | "running" | undefined;
+  readonly status?: "cancelled" | "errored" | "failed" | "passed" | "running" | undefined;
 }>;
-export declare const readTrialSummary: (root: string, runId: string, trialId: string) => Promise<{
-  readonly status: "cancelled" | "completed" | "failed" | "running";
+export declare const readRunSummary: (root: string, runId: string) => Promise<{
+  readonly status: "cancelled" | "errored" | "failed" | "passed" | "running";
   readonly endedAt: string;
   readonly durationMs?: number | undefined;
   readonly scoring?: {
@@ -950,6 +980,7 @@ export declare const readTrialSummary: (root: string, runId: string, trialId: st
         readonly usage?: {
           readonly inputTokens?: number | undefined;
           readonly outputTokens?: number | undefined;
+          readonly cachedInputTokens?: number | undefined;
           readonly totalTokens?: number | undefined;
         } | undefined;
         readonly events?: JsonValue[] | undefined;
@@ -993,6 +1024,7 @@ export declare const readTrialSummary: (root: string, runId: string, trialId: st
         readonly usage?: {
           readonly inputTokens?: number | undefined;
           readonly outputTokens?: number | undefined;
+          readonly cachedInputTokens?: number | undefined;
           readonly totalTokens?: number | undefined;
         } | undefined;
         readonly events?: JsonValue[] | undefined;
@@ -1017,13 +1049,18 @@ export declare const readTrialSummary: (root: string, runId: string, trialId: st
     readonly kind: "directory" | "file";
     readonly size?: number | undefined;
   }[] | undefined;
+  readonly attachments?: {
+    readonly name: string;
+    readonly size: number;
+    readonly truncatedFrom?: number | undefined;
+  }[] | undefined;
   readonly error?: {
     readonly name: string;
     readonly message: string;
     readonly stack?: string | undefined;
   } | undefined;
 }>;
-export declare const readTrialScoring: (root: string, runId: string, trialId: string) => Promise<{
+export declare const readRunScoring: (root: string, runId: string) => Promise<{
   readonly results: {
     readonly name: string;
     readonly kind: "judge" | "predicate";
@@ -1041,6 +1078,7 @@ export declare const readTrialScoring: (root: string, runId: string, trialId: st
       readonly usage?: {
         readonly inputTokens?: number | undefined;
         readonly outputTokens?: number | undefined;
+        readonly cachedInputTokens?: number | undefined;
         readonly totalTokens?: number | undefined;
       } | undefined;
       readonly events?: JsonValue[] | undefined;
@@ -1084,6 +1122,7 @@ export declare const readTrialScoring: (root: string, runId: string, trialId: st
       readonly usage?: {
         readonly inputTokens?: number | undefined;
         readonly outputTokens?: number | undefined;
+        readonly cachedInputTokens?: number | undefined;
         readonly totalTokens?: number | undefined;
       } | undefined;
       readonly events?: JsonValue[] | undefined;
@@ -1103,7 +1142,7 @@ export declare const readTrialScoring: (root: string, runId: string, trialId: st
   readonly overall?: number | undefined;
   readonly passed: boolean;
 }>;
-export declare function readTrialEvents(root: string, runId: string, trialId: string): Promise<TrajectoryEvent[]>;
+export declare function readRunEvents(root: string, runId: string): Promise<TrajectoryEvent[]>;
 /**
  * True when a run's owning process is known to be gone: it ran on this host
  * and its PID no longer exists. Runs from other hosts are never judged.
@@ -1111,49 +1150,84 @@ export declare function readTrialEvents(root: string, runId: string, trialId: st
 export declare function ownerIsGone(owner: RunProcess | undefined): boolean;
 //#endregion
 //#region ../runner/src/matrix.d.ts
-/** Cells run; cancelled cells are neither passed nor failed. */
+/** Cells and runs; a run that errored counts as failed, a cancelled one as neither. */
 type MatrixSummary = {
   cells: number;
+  runs: number;
   passed: number;
   failed: number;
   cancelled: number;
 };
 /** Bounded workers pull cells from the iterator; no Cartesian job array is allocated. */
-export declare function runMatrix(matrix: EvalMatrix, options: Omit<RunEvalOptions, 'runId' | 'trialId' | 'parameters' | 'matrix'> & {
+export declare function runMatrix(matrix: EvalMatrix, options: Omit<RunEvalOptions, 'runId' | 'parameters' | 'matrix'> & {
   selection?: MatrixSelection;
   /** The suite each cell's run is recorded under, when `suiteId` is not given. */
   suiteFor?: (cell: EvalMatrixCell) => string | undefined;
-  onResult?: (cell: EvalMatrixCell, result: RunResult) => void | Promise<void>;
+  onResult?: (cell: EvalMatrixCell, runs: readonly RunResult[]) => void | Promise<void>;
 }): Effect.Effect<MatrixSummary, unknown>;
 //#endregion
+//#region ../runner/src/upgrade.d.ts
+/** What upgrading a results folder did, or would do with `dryRun`. */
+type UpgradeReport = {
+  /** Old runs converted (or that would be). */
+  upgraded: number;
+  /** Version 6 runs they became. */
+  runs: number;
+  /** Old runs with more than one trial: each trial became a run in a batch. */
+  batches: number;
+  /** Old runs left alone because they are still running. */
+  running: string[];
+};
+/**
+ * Rewrites results from report schema versions 2 to 5 into version 6, in
+ * place (RFD0008). An old run of one trial (most of them) becomes a run with
+ * the same ID, so links to it keep working. An old run of several trials
+ * becomes runs with the trials' IDs, in a batch with the old run's ID.
+ *
+ * Each new run is written under a temporary name and renamed into place;
+ * artifacts are moved, not copied, as the last step before that rename. An
+ * old run is removed only after its new runs exist, and one replaced under
+ * the same ID is set aside first, so a crash at any point leaves the old run
+ * or complete new ones, and re-running finishes the job. Version 6 runs are
+ * left alone.
+ */
+export declare function upgradeReports(root: string, options?: {
+  dryRun?: boolean;
+}): Promise<UpgradeReport>;
+//#endregion
 //#region ../runner/src/workspace.d.ts
-/** Where a trial's persistent workspace lives below `root`. */
-export declare function trialWorkspacePath(root: string, trialId: string): string;
+/** Where a run's persistent workspace lives below `root`. */
+export declare function runWorkspacePath(root: string, runId: string): string;
 //#endregion
 //#region ../runner/src/index.d.ts
 export type RunEvalOptions = {
   report: ReportStore;
+  /** The run's ID; only with one run (no `repeat`). */
   runId?: string;
-  trialId?: string;
   suiteId?: string;
-  /** Overrides the eval policy's requested number of independent trials. */
-  trials?: number;
+  /** Overrides the scenario's `repeat`: how many runs to start, as one batch. */
+  repeat?: number;
   /** Parameters made available to the AUT for this invocation. */
   parameters?: JsonObject;
-  /** Matrix provenance retained with every aggregate and trial manifest. */
+  /** Matrix provenance recorded in every run's manifest. */
   matrix?: RunMetadata['matrix'];
   /** Selects a declared AUT runtime such as local, sandbox, or remote. */
   runtime?: AgentRuntimeName;
-  /** Maximum number of trials from this aggregate run executing concurrently. */
+  /** Maximum number of this batch's runs executing concurrently. */
   concurrency?: number;
-  /** Optional shared limiter for coordinating trials across aggregate runs. */
+  /** Optional shared limiter for coordinating runs across batches. */
   semaphore?: Effect.Semaphore;
-  /** Base directory for persistent local trial workspaces. Omit for an ephemeral OS temp workspace. */
+  /** Base directory for persistent local run workspaces. Omit for an ephemeral OS temp workspace. */
   workspaceRoot?: string;
-  /** With `workspaceRoot`: which trial workspaces to keep afterwards. Default `'failed'`. */
+  /** With `workspaceRoot`: which run workspaces to keep afterwards. Default `'failed'`. */
   keepWorkspaces?: KeepSandboxes;
-  /** Candidate paths left out of the artifact snapshot. Default `DEFAULT_ARTIFACT_EXCLUDE`. */
+  /**
+   * Candidate paths left out of the artifact snapshot, before the scenario's
+   * own `workspace.ignore`. Default `DEFAULT_ARTIFACT_EXCLUDE`.
+   */
   artifactExclude?: readonly string[];
+  /** Size limits for files adapters and scorers `attach()`. Default `DEFAULT_ATTACHMENT_LIMITS`. */
+  attachmentLimits?: AttachmentLimits;
   /** Repository and commit the eval code came from, recorded in the manifest. */
   source?: RunSource;
   /** The project's evals and suites at run time, saved beside the manifest. */
@@ -1161,7 +1235,7 @@ export type RunEvalOptions = {
   /** Project root; dataset paths resolve against it. Default: the working directory. */
   root?: string;
   /**
-   * Aborting stops new trials from starting; running trials stop at their next
+   * Aborting stops new runs from starting; running ones stop at their next
    * step and are finalized as cancelled. Also passed to dataset loaders.
    */
   signal?: AbortSignal;
@@ -1169,20 +1243,23 @@ export type RunEvalOptions = {
    * Observes every trajectory event after it is recorded, for live progress.
    * It must not throw; an error it throws is ignored.
    */
-  onEvent?: (trial: TrialRef, event: TrajectoryEvent) => void;
+  onEvent?: (run: RunRef, event: TrajectoryEvent) => void;
   now?: () => Date;
 };
-/** Identifies the trial an observed event belongs to. */
-export type TrialRef = {
+/** Identifies the run an observed event belongs to. */
+export type RunRef = {
   runId: string;
-  trialId: string;
-  trialIndex: number;
+  batch?: Batch;
   evalId: string;
   suiteId?: string;
   parameters?: JsonObject;
 };
-/** Effect-native aggregate eval execution entry point. */
-export declare function runEval(definition: EvalDefinition, options: RunEvalOptions): Effect.Effect<RunResult, unknown>;
+/**
+ * Runs a scenario: once, or `repeat` times as one batch. Resolves to one
+ * result per run started; runs never started because of cancellation are
+ * left out.
+ */
+export declare function runEval(definition: EvalDefinition, options: RunEvalOptions): Effect.Effect<RunResult[], unknown>;
 //#endregion
-export type { DatasetFileEntry, MatrixSummary };
+export type { DatasetFileEntry, MatrixSummary, UpgradeReport };
 //# sourceMappingURL=runner.d.mts.map