@leostera/evalkit 0.7.0 → 0.9.0

package/dist/index.d.mts

@@ -1,6 +1,6 @@
 import * as Schema from "effect/Schema";
 //#region ../core/src/identity.d.ts
-export declare const ResourceKindSchema: Schema.Literal<["run", "trial", "artifact"]>;
+export declare const ResourceKindSchema: Schema.Literal<["run", "artifact"]>;
 export type ResourceKind = typeof ResourceKindSchema.Type;
 /** Human-authored project-local identifier; never a generated resource URI. */
 export declare function authorId(value: string): string;
@@ -26,14 +26,14 @@ export type DatasetFileSource = string | Uint8Array | {
   file: string;
 };
 /**
- * Files for the trial workspace: a directory relative to the project root, or
+ * Files for the run workspace: a directory relative to the project root, or
  * a map from workspace path (below the dataset's mount) to its content.
  */
 export type DatasetFiles = string | Readonly<Record<string, DatasetFileSource>>;
-/** What one trial receives from a dataset. */
+/** What one run receives from a dataset. */
 export type DatasetContent = {
   files?: DatasetFiles;
-  /** Any JSON-convertible value; converted once when the trial starts. */
+  /** Any JSON-convertible value; converted once when the run starts. */
   value?: unknown;
 };
 export type DatasetLoadContext = {
@@ -41,12 +41,12 @@ export type DatasetLoadContext = {
   root: string;
   signal: AbortSignal;
 };
-export type DatasetTrialContext = {
+export type DatasetRunContext = {
   evalId: string;
   runId: string;
-  trialId: string;
-  trialIndex: number;
-  /** Per-trial random seed, so repeated trials can choose different data. */
+  /** Set when the run is one of several repeats launched together. */
+  batch?: Batch;
+  /** Per-run random seed, so repeated runs can choose different data. */
   seed: number;
   /** The matrix cell's parameters, or `{}`. */
   parameters: JsonObject;
@@ -70,10 +70,10 @@ export type StaticDatasetDefinition = DatasetCommon & {
   onRun?: never;
 };
 export type ComputedDatasetDefinition<TLoaded extends DatasetContent> = DatasetCommon & {
-  /** Once per process; the successful result is cached and shared by every trial. */
+  /** Once per process; the successful result is cached and shared by every run. */
   onLoad: (load: DatasetLoadContext) => TLoaded | Promise<TLoaded>;
-  /** Once per trial; defaults to using the loaded content as is. */
-  onRun?: (loaded: TLoaded, trial: DatasetTrialContext) => DatasetContent | Promise<DatasetContent>;
+  /** Once per run; defaults to using the loaded content as is. */
+  onRun?: (loaded: TLoaded, run: DatasetRunContext) => DatasetContent | Promise<DatasetContent>;
   files?: never;
   value?: never;
   valueFile?: never;
@@ -86,7 +86,7 @@ export type DatasetSource = {
 } | {
   kind: 'computed';
   onLoad: (load: DatasetLoadContext) => DatasetContent | Promise<DatasetContent>;
-  onRun?: (loaded: DatasetContent, trial: DatasetTrialContext) => DatasetContent | Promise<DatasetContent>;
+  onRun?: (loaded: DatasetContent, run: DatasetRunContext) => DatasetContent | Promise<DatasetContent>;
 };
 /** A checked dataset, as returned by `defineDataset`. Evals accept only these. */
 export type Dataset = {
@@ -101,10 +101,10 @@ export type Dataset = {
 /** Normalizes a relative POSIX path and rejects absolute or escaping ones. */
 export declare function relativeDatasetPath(value: string, label: string): string;
 /**
- * Declares named data that scenarios use: files placed in each trial's
+ * Declares named data that scenarios use: files placed in each run's
  * workspace, a value for the agent and scorers, or both. Static datasets give
  * `files`, `value` or `valueFile`; computed datasets give `onLoad` (cached per
- * process) and optionally `onRun` (per trial).
+ * process) and optionally `onRun` (per run).
  */
 export declare function defineDataset<TLoaded extends DatasetContent>(definition: StaticDatasetDefinition | ComputedDatasetDefinition<TLoaded>): Dataset;
 export declare function isDataset(value: unknown): value is Dataset;
@@ -242,7 +242,7 @@ export type EvalCatalogScorer = {
   definedAt?: string;
 };
 export type EvalCatalogStep =
-/** `message` is absent when it is computed for each trial. */
+/** `message` is absent when it is computed for each run. */
 {
   kind: 'user';
   message?: string;
@@ -268,9 +268,9 @@ export type EvalCatalogEntry = {
   /** The separate agent that runs judge rules, if any. */
   judgeAgent?: EvalCatalogAgent;
   datasets: EvalCatalogDataset[];
-  trialCount: number;
+  /** Runs started each time the scenario runs (`repeat`). */
+  repeat: number;
   policy: {
-    trials: number;
     timeoutMs?: number;
     failfast?: boolean;
   };
@@ -339,26 +339,36 @@ type EvalkitConfig = {
   };
   execution?: {
     concurrency?: number;
-    trials?: number;
+    repeat?: number;
     maxCells?: number;
   };
   reportDir?: string;
-  /** The candidate files saved with each trial's results. */
+  /** The candidate files saved with each run's results. */
   artifacts?: {
     /**
      * Paths to leave out, as globs relative to the candidate workspace; a
      * pattern without `/` matches that name at any depth. Replaces the default,
      * `DEFAULT_ARTIFACT_EXCLUDE` (dependencies, build output, VCS metadata).
+     * A scenario's `workspace.ignore` patterns come after these.
      */
     exclude?: readonly string[];
+    /** Limits for files adapters `attach()`. */
+    attachments?: AttachmentLimits;
   };
-  /** Where each trial's real workspace lives during a run. Default `_evalkit-sandbox`. */
+  /** Where each run's real workspace lives during a run. Default `_evalkit-sandbox`. */
   sandboxDir?: string;
-  /** Which trial workspaces to keep in `sandboxDir` afterwards. Default `'failed'`: only trials that did not pass. */
+  /** Which run workspaces to keep in `sandboxDir` afterwards. Default `'failed'`: only runs that did not pass. */
   keepSandboxes?: KeepSandboxes;
   share?: ShareConfig;
 };
 type KeepSandboxes = 'failed' | 'all' | 'none';
+type AttachmentLimits = {
+  /** A larger attachment keeps its last this-many bytes. Default 5 MB. */
+  maxFileBytes?: number;
+  /** All of one run's attachments; past it, `attach()` rejects. Default 20 MB. */
+  maxRunBytes?: number;
+};
+export declare const DEFAULT_ATTACHMENT_LIMITS: Required<AttachmentLimits>;
 /** Left out of candidate snapshots unless `artifacts.exclude` says otherwise. */
 export declare const DEFAULT_ARTIFACT_EXCLUDE: readonly string[];
 /** HTTP header names and values. */
@@ -376,6 +386,32 @@ type ShareConfig = {
 /** Pure, typed project configuration. Loading/discovery belongs to the CLI. */
 export declare function defineConfig<const T extends EvalkitConfig>(configuration: T): T;
 //#endregion
+//#region ../core/src/usage.d.ts
+/**
+ * Token usage an agent reports for one turn. EvalKit never measures or infers
+ * it; adapters copy it from their provider.
+ */
+type Usage = {
+  /**
+   * Every input token the model read this turn: the prompt, history and tool
+   * results, **including** tokens read from or written to a prompt cache.
+   * (Some providers report cached tokens separately; add them in.)
+   */
+  inputTokens?: number;
+  /** Every token the model produced, including reasoning tokens. */
+  outputTokens?: number;
+  /** Of `inputTokens`, how many were read from a prompt cache. Informational. */
+  cachedInputTokens?: number;
+  /** The provider's own total; used only when input and output are not reported. */
+  totalTokens?: number;
+};
+/**
+ * The tokens a turn counts toward the `tokens` metric: `inputTokens +
+ * outputTokens` (cached input included), or `totalTokens` when neither is
+ * reported. Undefined when the turn reported no usage.
+ */
+export declare function turnTokens(usage: Usage | undefined): number | undefined;
+//#endregion
 //#region ../core/src/json.d.ts
 /** A value that cannot be stored in a report, with the path of the offending part. */
 export declare class NotJsonError extends Error {
@@ -412,7 +448,9 @@ export declare function definitionSite(value: unknown): DefinitionSite | undefin
 //#endregion
 //#region ../core/src/schema.d.ts
 /** Runtime schemas for serialized EvalKit report and API boundaries. */
-export declare const RunStatusSchema: Schema.Literal<["running", "completed", "failed", "cancelled"]>;
+export declare const RunStatusSchema: Schema.Literal<["running", "passed", "failed", "errored", "cancelled"]>;
+/** The report schema version this EvalKit writes and reads. */
+export declare const REPORT_SCHEMA_VERSION = 6;
 export declare const RecordedErrorSchema: Schema.Struct<{
   name: typeof Schema.String;
   message: typeof Schema.String;
@@ -439,8 +477,19 @@ export declare const RunSourceSchema: Schema.Struct<{
     url: Schema.optional<typeof Schema.String>;
   }>>;
 }>;
+export declare const BatchSchema: Schema.Struct<{
+  id: typeof Schema.String;
+  index: typeof Schema.Number;
+  size: typeof Schema.Number;
+}>;
+export declare const RunDatasetSchema: Schema.Struct<{
+  id: typeof Schema.String;
+  visibility: Schema.Literal<["candidate", "evaluator"]>;
+  source: Schema.Literal<["static", "computed"]>;
+  digest: typeof Schema.String;
+}>;
 export declare const RunMetadataSchema: Schema.Struct<{
-  schemaVersion: Schema.Literal<[2, 3, 4, 5]>;
+  schemaVersion: Schema.Literal<[6]>;
   runUri: Schema.filter<typeof Schema.String>;
   evalId: typeof Schema.String;
   suiteId: Schema.optional<typeof Schema.String>;
@@ -474,25 +523,13 @@ export declare const RunMetadataSchema: Schema.Struct<{
     pid: typeof Schema.Number;
     hostname: typeof Schema.String;
   }>>;
-}>;
-export declare const TrialMetadataSchema: Schema.Struct<{
-  schemaVersion: Schema.Literal<[2, 3, 4, 5]>;
-  runUri: Schema.filter<typeof Schema.String>;
-  trialUri: Schema.filter<typeof Schema.String>;
-  trialIndex: typeof Schema.Number;
-  evalId: typeof Schema.String;
-  parameters: Schema.optional<Schema.Record$<typeof Schema.String, Schema.Schema<JsonValue, JsonValue, never>>>;
-  matrix: Schema.optional<Schema.Struct<{
-    id: typeof Schema.String;
-    cellKey: typeof Schema.String;
-  }>>;
-  aut: Schema.optional<Schema.Struct<{
-    name: Schema.optional<typeof Schema.String>;
-    kind: typeof Schema.String;
+  batch: Schema.optional<Schema.Struct<{
     id: typeof Schema.String;
-    version: Schema.optional<typeof Schema.String>;
+    index: typeof Schema.Number;
+    size: typeof Schema.Number;
   }>>;
-  startedAt: typeof Schema.String;
+  /** The effective snapshot ignore patterns: the project's, then the scenario's. */
+  workspaceIgnore: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
   datasets: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
     id: typeof Schema.String;
     visibility: Schema.Literal<["candidate", "evaluator"]>;
@@ -500,9 +537,9 @@ export declare const TrialMetadataSchema: Schema.Struct<{
     digest: typeof Schema.String;
   }>>>>;
 }>;
-/** On-disk manifests include a lifecycle status; older finalized manifests may still say running. */
+/** On-disk manifests include a lifecycle status. */
 export declare const RunManifestSchema: Schema.Struct<{
-  schemaVersion: Schema.Literal<[2, 3, 4, 5]>;
+  schemaVersion: Schema.Literal<[6]>;
   runUri: Schema.filter<typeof Schema.String>;
   evalId: typeof Schema.String;
   suiteId: Schema.optional<typeof Schema.String>;
@@ -536,37 +573,24 @@ export declare const RunManifestSchema: Schema.Struct<{
     pid: typeof Schema.Number;
     hostname: typeof Schema.String;
   }>>;
-  status: Schema.optional<Schema.Literal<["running", "completed", "failed", "cancelled"]>>;
-}>;
-export declare const TrialManifestSchema: Schema.Struct<{
-  schemaVersion: Schema.Literal<[2, 3, 4, 5]>;
-  runUri: Schema.filter<typeof Schema.String>;
-  trialUri: Schema.filter<typeof Schema.String>;
-  trialIndex: typeof Schema.Number;
-  evalId: typeof Schema.String;
-  parameters: Schema.optional<Schema.Record$<typeof Schema.String, Schema.Schema<JsonValue, JsonValue, never>>>;
-  matrix: Schema.optional<Schema.Struct<{
-    id: typeof Schema.String;
-    cellKey: typeof Schema.String;
-  }>>;
-  aut: Schema.optional<Schema.Struct<{
-    name: Schema.optional<typeof Schema.String>;
-    kind: typeof Schema.String;
+  batch: Schema.optional<Schema.Struct<{
     id: typeof Schema.String;
-    version: Schema.optional<typeof Schema.String>;
+    index: typeof Schema.Number;
+    size: typeof Schema.Number;
   }>>;
-  startedAt: typeof Schema.String;
+  workspaceIgnore: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
   datasets: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
     id: typeof Schema.String;
     visibility: Schema.Literal<["candidate", "evaluator"]>;
     source: Schema.Literal<["static", "computed"]>;
     digest: typeof Schema.String;
   }>>>>;
-  status: Schema.optional<Schema.Literal<["running", "completed", "failed", "cancelled"]>>;
+  status: Schema.optional<Schema.Literal<["running", "passed", "failed", "errored", "cancelled"]>>;
 }>;
 export declare const UsageSchema: Schema.Struct<{
   inputTokens: Schema.optional<typeof Schema.Number>;
   outputTokens: Schema.optional<typeof Schema.Number>;
+  cachedInputTokens: Schema.optional<typeof Schema.Number>;
   totalTokens: Schema.optional<typeof Schema.Number>;
 }>;
 export declare const JudgeRunInfoSchema: Schema.Struct<{
@@ -579,6 +603,7 @@ export declare const JudgeRunInfoSchema: Schema.Struct<{
   usage: Schema.optional<Schema.Struct<{
     inputTokens: Schema.optional<typeof Schema.Number>;
     outputTokens: Schema.optional<typeof Schema.Number>;
+    cachedInputTokens: Schema.optional<typeof Schema.Number>;
     totalTokens: Schema.optional<typeof Schema.Number>;
   }>>;
   events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
@@ -600,6 +625,7 @@ export declare const ScoreResultSchema: Schema.Struct<{
     usage: Schema.optional<Schema.Struct<{
       inputTokens: Schema.optional<typeof Schema.Number>;
       outputTokens: Schema.optional<typeof Schema.Number>;
+      cachedInputTokens: Schema.optional<typeof Schema.Number>;
       totalTokens: Schema.optional<typeof Schema.Number>;
     }>>;
     events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
@@ -631,6 +657,7 @@ export declare const CheckpointResultSchema: Schema.Struct<{
     usage: Schema.optional<Schema.Struct<{
       inputTokens: Schema.optional<typeof Schema.Number>;
       outputTokens: Schema.optional<typeof Schema.Number>;
+      cachedInputTokens: Schema.optional<typeof Schema.Number>;
       totalTokens: Schema.optional<typeof Schema.Number>;
     }>>;
     events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
@@ -658,7 +685,7 @@ export declare const MetricResultSchema: Schema.Struct<{
     stack: Schema.optional<typeof Schema.String>;
   }>>;
 }>;
-export declare const TrialScoringSchema: Schema.Struct<{
+export declare const RunScoringSchema: Schema.Struct<{
   results: Schema.mutable<Schema.Array$<Schema.Struct<{
     name: typeof Schema.String;
     kind: Schema.Literal<["predicate", "judge"]>;
@@ -676,6 +703,7 @@ export declare const TrialScoringSchema: Schema.Struct<{
       usage: Schema.optional<Schema.Struct<{
         inputTokens: Schema.optional<typeof Schema.Number>;
         outputTokens: Schema.optional<typeof Schema.Number>;
+        cachedInputTokens: Schema.optional<typeof Schema.Number>;
         totalTokens: Schema.optional<typeof Schema.Number>;
       }>>;
       events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
@@ -719,6 +747,7 @@ export declare const TrialScoringSchema: Schema.Struct<{
       usage: Schema.optional<Schema.Struct<{
         inputTokens: Schema.optional<typeof Schema.Number>;
         outputTokens: Schema.optional<typeof Schema.Number>;
+        cachedInputTokens: Schema.optional<typeof Schema.Number>;
         totalTokens: Schema.optional<typeof Schema.Number>;
       }>>;
       events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
@@ -743,8 +772,13 @@ export declare const ArtifactEntrySchema: Schema.Struct<{
   kind: Schema.Literal<["file", "directory"]>;
   size: Schema.optional<typeof Schema.Number>;
 }>;
-export declare const TrialSummarySchema: Schema.Struct<{
-  status: Schema.Literal<["running", "completed", "failed", "cancelled"]>;
+export declare const AttachmentEntrySchema: Schema.Struct<{
+  name: typeof Schema.String;
+  size: typeof Schema.Number;
+  truncatedFrom: Schema.optional<typeof Schema.Number>;
+}>;
+export declare const RunSummarySchema: Schema.Struct<{
+  status: Schema.Literal<["running", "passed", "failed", "errored", "cancelled"]>;
   endedAt: typeof Schema.String;
   durationMs: Schema.optional<typeof Schema.Number>;
   scoring: Schema.optional<Schema.Struct<{
@@ -765,6 +799,7 @@ export declare const TrialSummarySchema: Schema.Struct<{
         usage: Schema.optional<Schema.Struct<{
           inputTokens: Schema.optional<typeof Schema.Number>;
           outputTokens: Schema.optional<typeof Schema.Number>;
+          cachedInputTokens: Schema.optional<typeof Schema.Number>;
           totalTokens: Schema.optional<typeof Schema.Number>;
         }>>;
         events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
@@ -808,6 +843,7 @@ export declare const TrialSummarySchema: Schema.Struct<{
         usage: Schema.optional<Schema.Struct<{
           inputTokens: Schema.optional<typeof Schema.Number>;
           outputTokens: Schema.optional<typeof Schema.Number>;
+          cachedInputTokens: Schema.optional<typeof Schema.Number>;
           totalTokens: Schema.optional<typeof Schema.Number>;
         }>>;
         events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
@@ -832,20 +868,11 @@ export declare const TrialSummarySchema: Schema.Struct<{
     kind: Schema.Literal<["file", "directory"]>;
     size: Schema.optional<typeof Schema.Number>;
   }>>>>;
-  error: Schema.optional<Schema.Struct<{
+  attachments: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
     name: typeof Schema.String;
-    message: typeof Schema.String;
-    stack: Schema.optional<typeof Schema.String>;
-  }>>;
-}>;
-export declare const RunSummarySchema: Schema.Struct<{
-  status: Schema.Literal<["running", "completed", "failed", "cancelled"]>;
-  endedAt: typeof Schema.String;
-  durationMs: Schema.optional<typeof Schema.Number>;
-  trialCount: typeof Schema.Number;
-  passed: typeof Schema.Number;
-  failed: typeof Schema.Number;
-  cancelled: Schema.optional<typeof Schema.Number>;
+    size: typeof Schema.Number;
+    truncatedFrom: Schema.optional<typeof Schema.Number>;
+  }>>>>;
   error: Schema.optional<Schema.Struct<{
     name: typeof Schema.String;
     message: typeof Schema.String;
@@ -887,11 +914,6 @@ export type AgentRuntime = {
   configuration?: JsonObject;
 };
 export type AgentRuntimes = Partial<Record<AgentRuntimeName, AgentRuntime>>;
-export type Usage = {
-  inputTokens?: number;
-  outputTokens?: number;
-  totalTokens?: number;
-};
 export type RecordedError = {
   name: string;
   message: string;
@@ -937,7 +959,7 @@ export type AutEvent = {
   timestamp: string;
 };
 export type RunnerEvent = {
-  kind: 'trial-started';
+  kind: 'run-started';
   timestamp: string;
 } | {
   kind: 'transcript-step-started';
@@ -985,7 +1007,7 @@ export type RunnerEvent = {
   error: RecordedError;
   timestamp: string;
 } | {
-  kind: 'trial-completed';
+  kind: 'run-completed';
   timestamp: string;
 } | {
   kind: 'error';
@@ -998,7 +1020,7 @@ export type TrajectoryEvent = (AutEvent & {
   source: 'runner';
 });
 export type WorkspaceView = {
-  /** Absolute path to this trial-scoped workspace. Do not persist it in reports. */
+  /** Absolute path to this run-scoped workspace. Do not persist it in reports. */
   root: string;
 };
 export type CandidateWorkspace = WorkspaceView;
@@ -1006,8 +1028,8 @@ export type EvaluatorWorkspace = WorkspaceView;
 export type AutContext = {
   runId: string;
   evalId: string;
-  trialId: string;
-  trialIndex: number;
+  /** Set when this run is one of several repeats launched together. */
+  batch?: Batch;
   metadata: JsonObject;
   /** Run-time parameters supplied by the eval definition or CLI invocation. */
   parameters?: JsonObject;
@@ -1028,6 +1050,14 @@ export type AutAdapter<TMessage = string> = {
     runtime?: AgentRuntime;
     /** Optional: stream events as they happen. Returning a turn result from `send` is enough for most agents. */
     onEvent: (event: AutEventInput) => void | Promise<void>;
+    /**
+     * Saves a file with the run's results, such as the agent's raw log,
+     * whether or not its sandbox is kept. `name` is a relative path of
+     * letters, digits, `.`, `_` and `-` (`logs/agent.log`); each name once.
+     * A file over the size limit keeps its last bytes; past the run's
+     * total limit it is rejected. See `artifacts.attachments` in the config.
+     */
+    attach: (name: string, content: string | Uint8Array) => Promise<void>;
   }): Promise<AutSession<TMessage>>;
 };
 export type AutSession<TMessage = string> = {
@@ -1054,13 +1084,14 @@ export type UserMessageInput = {
   parameters: JsonObject;
   /** Values of the scenario's candidate datasets, by dataset ID (never evaluator ones). */
   data: JsonObject;
-  /** The trial's random seed (also in `context.metadata.randomSeed`). */
+  /** The run's random seed (also in `context.metadata.randomSeed`). */
   seed: number;
-  trialIndex: number;
+  /** Set when this run is one of several repeats launched together. */
+  batch?: Batch;
 };
 export type UserStep = {
   kind: 'user';
-  /** A fixed message, or one computed for each trial. */
+  /** A fixed message, or one computed for each run. */
   message: string | ((input: UserMessageInput) => string);
 };
 export type AgentStep = {
@@ -1100,7 +1131,7 @@ export type TranscriptStep = UserStep | AgentStep | ScoringRule | ExpectToolCall
 /** Observes a tool call in the preceding turn; never invokes a tool. */
 export declare function expectToolCall(expected: ExpectToolCallStep['expected']): ExpectToolCallStep;
 /**
- * A user message. Pass a function to compute it for each trial, e.g. from a
+ * A user message. Pass a function to compute it for each run, e.g. from a
  * matrix axis: `user(({ parameters }) => `Set up OCaml with ${parameters.toolchain}.`)`.
  */
 export declare function user(message: UserStep['message']): UserStep;
@@ -1116,18 +1147,21 @@ export type SkippedScore = {
   skipped: true;
   reason: string;
 };
-export type ScoreValue = number | SkippedScore | {
+export type ScoreValue<Evidence = unknown> = number | SkippedScore | {
   value: number;
   passed?: boolean;
   explanation?: string;
-  /** Any JSON-convertible value. */
-  evidence?: unknown;
+  /**
+   * Any JSON-convertible value: what the scorer observed. A checkpoint's
+   * evidence reaches later scorers through `checkpoint(rule)`.
+   */
+  evidence?: Evidence;
   /** Judge-agent evidence, never inferred from AUT usage. */
   judge?: JudgeRunInfo;
 };
 /**
- * Marks a scorer as not applicable to this trial. A skipped result neither
- * passes nor fails the trial and is left out of score averages.
+ * Marks a scorer as not applicable to this run. A skipped result neither
+ * passes nor fails the run and is left out of score averages.
  */
 export declare function skip(reason: string): SkippedScore;
 export type ScoreResult = {
@@ -1165,14 +1199,36 @@ export type ScoringContext = {
    * example, in a container that is gone by final scoring).
    */
   checkpoints: readonly CheckpointResult[];
+  /**
+   * The latest result of a checkpoint in this scenario's transcript, looked
+   * up by the rule itself, so renaming it cannot break the lookup. A
+   * predicate's `evidence` keeps its type. Undefined until that step has
+   * run; throws for a rule that is not in the transcript.
+   */
+  checkpoint<Evidence>(rule: PredicateScorer<Evidence>): CheckpointOutcome<Evidence> | undefined;
+  checkpoint(rule: JudgeRule | ExpectToolCallStep): CheckpointOutcome<JsonValue> | undefined;
+  /**
+   * Saves a file with the run's results, like the agent's `attach()`: for
+   * example a container's log, or a file the agent reported, read before the
+   * workspace is gone. Same names, limits and rules.
+   */
+  attach(name: string, content: string | Uint8Array): Promise<void>;
+};
+/**
+ * A checkpoint's result with typed evidence. Evidence is saved as JSON, so
+ * it arrives as its JSON form: a `Date` becomes a string, `undefined` fields
+ * are dropped.
+ */
+export type CheckpointOutcome<Evidence> = Omit<CheckpointResult, 'evidence'> & {
+  evidence?: Evidence;
 };
-export type PredicateScorer = {
+export type PredicateScorer<Evidence = unknown> = {
   kind: 'predicate';
   name: string;
   supportsPartial?: boolean;
   /** A numeric result passes when it is at least this value. Default: only 1 passes. */
   passAt?: number;
-  run(context: ScoringContext): boolean | ScoreValue | Promise<boolean | ScoreValue>;
+  run(context: ScoringContext): boolean | ScoreValue<Evidence> | Promise<boolean | ScoreValue<Evidence>>;
 };
 export type JudgeRule = {
   kind: 'judge';
@@ -1182,9 +1238,10 @@ export type JudgeRule = {
   supportsPartial?: boolean;
 };
 export type ScoringRule = PredicateScorer | JudgeRule;
-export declare function predicate(name: string, run: PredicateScorer['run'], options?: Pick<PredicateScorer, 'supportsPartial' | 'passAt'>): PredicateScorer;
+/** A scorer in code. Its `evidence` type carries through to `checkpoint(rule)`. */
+export declare function predicate<Evidence = unknown>(name: string, run: PredicateScorer<Evidence>['run'], options?: Pick<PredicateScorer, 'supportsPartial' | 'passAt'>): PredicateScorer<Evidence>;
 /**
- * A number measured for every trial, such as tokens, retries or web searches.
+ * A number measured for every run, such as tokens, retries or web searches.
  * Metrics never decide pass or fail; they are averaged per run and shown on
  * the scoreboard. Return undefined when there is nothing to measure.
  */
@@ -1197,10 +1254,11 @@ export type Metric = {
   better?: 'lower' | 'higher';
   run(context: ScoringContext): number | undefined | Promise<number | undefined>;
 };
-export declare function metric(name: string, run: Metric['run'], options?: Pick<Metric, 'unit' | 'better'>): Metric;
-/** A metric's value for one trial. */
+/** A built-in metric's name is a type error here, as well as at definition time. */
+export declare function metric<const Name extends string>(name: Name extends BuiltInMetricName ? never : Name, run: Metric['run'], options?: Pick<Metric, 'unit' | 'better'>): Metric;
+/** A metric's value for one run. */
 export type MetricResult = {
-  name: string;
+  name: MetricName;
   /** Absent when the metric measured nothing or failed. */
   value?: number;
   unit?: string;
@@ -1214,8 +1272,6 @@ export type EvalPolicy = {
   timeoutMs?: number;
   /** Stop later transcript steps after a failed checkpoint assertion. Default false. */
   failfast?: boolean;
-  /** Number of independent trials requested by local and hosted runners. */
-  trials?: number;
 };
 export type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
   id: string;
@@ -1223,15 +1279,33 @@ export type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
   agent: TAgent;
   /** Separate judge agent; its model and tools are never inherited from the AUT. */
   judge?: AutAdapter;
-  /** Data and files the scenario uses; each trial gets its own copy. */
+  /** Data and files the scenario uses; each run gets its own copy. */
   datasets?: readonly Dataset[];
   transcript: TranscriptStep[];
   scoring: ScoringRule[];
-  /** Numbers to measure for every trial; they never affect pass or fail. */
+  /** Numbers to measure for every run; they never affect pass or fail. */
   metrics?: readonly Metric[];
   policy?: EvalPolicy;
+  /**
+   * How many runs to start each time the scenario runs, as one batch: repeats
+   * under identical conditions, to see how often it passes. Default 1.
+   */
+  repeat?: number;
   /** Any JSON-convertible object; converted when the eval is defined. */
   metadata?: Record<string, unknown>;
+  workspace?: EvalWorkspace;
+};
+/** How EvalKit treats this scenario's candidate workspace. */
+export type EvalWorkspace = {
+  /**
+   * Paths to leave out of the run's snapshot, read like `.gitignore`: globs
+   * relative to the workspace, where a pattern without `/` matches that name
+   * at any depth and later patterns win. They come after the project's
+   * `artifacts.exclude` (default `DEFAULT_ARTIFACT_EXCLUDE`), so `!pattern`
+   * can keep a path the project leaves out, even inside an ignored directory:
+   * `['!_build/doc/**']` keeps generated docs while dropping the rest of `_build`.
+   */
+  ignore?: readonly string[];
 };
 export declare function defineEval<const T extends EvalDefinition>(definition: T): T;
 export type EvalSuite<TEvals extends readonly EvalDefinition[] = readonly EvalDefinition[]> = {
@@ -1271,7 +1345,18 @@ export type EvalSuiteMetadata = {
  * path-like grouping of evals; nesting is presentation derived from its ID.
  */
 export declare function registerEvals<const TRegistrations extends readonly EvalRegistration[]>(registrations: TRegistrations): EvalRegistry;
-export type RunStatus = 'running' | 'completed' | 'failed' | 'cancelled';
+/**
+ * How a run ended: `passed` or `failed` by its scores, `errored` when it could
+ * not finish (the agent, a dataset or EvalKit failed), `cancelled` by Ctrl+C.
+ */
+export type RunStatus = 'running' | 'passed' | 'failed' | 'errored' | 'cancelled';
+/** Runs launched together as repeats of one configuration (`repeat`). */
+export type Batch = {
+  id: string;
+  /** Zero-based position in the batch. */
+  index: number;
+  size: number;
+};
 /**
  * Where a run's eval code came from: the repository, eval project and commit.
  * Results are tagged with it; they are never stored in the repository itself.
@@ -1306,7 +1391,7 @@ export type RunProcess = {
   hostname: string;
 };
 export type RunMetadata = {
-  schemaVersion: 2 | 3 | 4 | 5;
+  schemaVersion: 6;
   runId: string;
   runUri: ResourceUri<'run'>;
   evalId: string;
@@ -1318,34 +1403,21 @@ export type RunMetadata = {
   };
   aut?: AutIdentity;
   startedAt: string;
-  /** Present from schema version 4 when the run's commit could be determined. */
+  /** Present when the run's commit could be determined. */
   source?: RunSource;
-  /** From schema version 5. */
   process?: RunProcess;
-};
-export type TrialMetadata = {
-  schemaVersion: 2 | 3 | 4 | 5;
-  runId: string;
-  runUri: ResourceUri<'run'>;
-  trialId: string;
-  trialUri: ResourceUri<'trial'>;
-  trialIndex: number;
-  evalId: string;
-  parameters?: JsonObject;
-  matrix?: {
-    id: string;
-    cellKey: string;
-  };
-  aut?: AutIdentity;
-  startedAt: string;
-  /** From schema version 5: the datasets this trial used, with content digests. */
-  datasets?: TrialDataset[];
-};
-export type TrialDataset = {
+  /** Set when this run is one of several repeats launched together. */
+  batch?: Batch;
+  /** The effective snapshot ignore patterns: the project's `artifacts.exclude`, then the scenario's `workspace.ignore`. */
+  workspaceIgnore?: string[];
+  /** The datasets this run used, with content digests. */
+  datasets?: RunDataset[];
+};
+export type RunDataset = {
   id: string;
   visibility: DatasetVisibility;
   source: 'static' | 'computed';
-  /** SHA-256 of the trial's content: file paths and bytes, and the canonical value. */
+  /** SHA-256 of the run's content: file paths and bytes, and the canonical value. */
   digest: string;
 };
 export type CheckpointResult = {
@@ -1365,9 +1437,21 @@ export type CheckpointResult = {
     id: string;
   };
 };
-/** Metrics EvalKit records for every trial; scenario metrics may not reuse these names. */
-export declare const BUILT_IN_METRICS: readonly string[];
-export type TrialScoring = {
+/**
+ * Metrics EvalKit records for every run; scenario metrics may not reuse
+ * these names.
+ *
+ * - `agent time` (ms): from the run's first event to its last one before
+ *   final scoring.
+ * - `tokens`: the sum of `turnTokens(usage)` over the turns that reported
+ *   usage, cached input included. Absent when no turn reported any.
+ * - `tool calls`: the agent's `tool-call` events.
+ */
+export declare const BUILT_IN_METRICS: readonly ['agent time', 'tokens', 'tool calls'];
+export type BuiltInMetricName = (typeof BUILT_IN_METRICS)[number];
+/** A built-in metric's name (suggested by editors), or a scenario's own. */
+export type MetricName = BuiltInMetricName | (string & {});
+export type RunScoring = {
   results: ScoreResult[];
   metrics?: MetricResult[];
   checkpoints?: CheckpointResult[];
@@ -1376,40 +1460,34 @@ export type TrialScoring = {
   passed: boolean;
 };
 export type ArtifactEntry = {
-  /** Path relative to this trial's artifact root. */
+  /** Path relative to this run's artifact root. */
   path: string;
   kind: 'file' | 'directory';
   size?: number;
 };
-export type TrialSummary = {
-  status: RunStatus;
-  endedAt: string;
-  durationMs?: number;
-  scoring?: TrialScoring;
-  artifacts?: ArtifactEntry[];
-  error?: RecordedError;
+/** A file an adapter attached with `attach()`, saved as `artifacts/attachments/<name>`. */
+export type AttachmentEntry = {
+  name: string;
+  /** Bytes saved. */
+  size: number;
+  /** The original size, when the file was cut to its last `maxFileBytes`. */
+  truncatedFrom?: number;
 };
 export type RunSummary = {
   status: RunStatus;
   endedAt: string;
   durationMs?: number;
-  trialCount: number;
-  passed: number;
-  failed: number;
-  /** Trials stopped by cancellation; counted neither as passed nor failed. */
-  cancelled?: number;
+  scoring?: RunScoring;
+  artifacts?: ArtifactEntry[];
+  attachments?: AttachmentEntry[];
   error?: RecordedError;
 };
-export type TrialWriter = {
-  appendEvent(event: TrajectoryEvent): Promise<void>;
-  writeScores(scoring: TrialScoring): Promise<void>;
-  /** Writes a file below this trial's artifact root. */
-  writeArtifact(path: string, data: Uint8Array): Promise<void>;
-  finalize(summary: TrialSummary): Promise<void>;
-};
 export type RunWriter = {
   readonly location: string;
-  startTrial(metadata: TrialMetadata): Promise<TrialWriter>;
+  appendEvent(event: TrajectoryEvent): Promise<void>;
+  writeScores(scoring: RunScoring): Promise<void>;
+  /** Writes a file below this run's artifact root. */
+  writeArtifact(path: string, data: Uint8Array): Promise<void>;
   finalize(summary: RunSummary): Promise<void>;
 };
 export type ReportStore = {
@@ -1417,35 +1495,18 @@ export type ReportStore = {
     catalog?: RunCatalog;
   }): Promise<RunWriter>;
 };
-export type TrialResult = {
+/** One finished run, as `runEval` returns it. */
+export type RunResult = {
   runId: string;
   runUri: ResourceUri<'run'>;
-  trialId: string;
-  trialUri: ResourceUri<'trial'>;
-  trialIndex: number;
   evalId: string;
   status: RunStatus;
+  batch?: Batch;
   reportLocation: string;
   durationMs?: number;
-  scoring?: TrialScoring;
+  scoring?: RunScoring;
   error?: RecordedError;
 };
-export type AggregateScoring = {
-  passed: number;
-  failed: number;
-  cancelled?: number;
-  passRate: number;
-  overall?: number;
-};
-export type RunResult = TrialResult & {
-  runId: string;
-  trialCount?: number;
-  passed?: number;
-  failed?: number;
-  aggregateScoring?: AggregateScoring;
-  /** Present for an aggregate multi-trial run. */
-  trials?: TrialResult[];
-};
 export declare function recordError(error: unknown): RecordedError;
 //#endregion
 //#region ../agents/src/index.d.ts
@@ -1457,5 +1518,5 @@ type PiAutOptions = {
 /** A local, one-prompt-per-turn AUT backed by Pi's non-interactive print mode. */
 export declare function piAgent(options?: PiAutOptions): AutAdapter;
 //#endregion
-export type { DefinitionSite, EvalMatrix, EvalMatrixCell, EvalkitConfig, KeepSandboxes, MatrixSelection, ShareConfig, ShareHeaders };
+export type { AttachmentLimits, DefinitionSite, EvalMatrix, EvalMatrixCell, EvalkitConfig, KeepSandboxes, MatrixSelection, ShareConfig, ShareHeaders, Usage };
 //# sourceMappingURL=index.d.mts.map