@leostera/evalkit 0.10.0 → 0.11.0

package/dist/runner.d.mts

@@ -1,51 +1,6 @@
-import { Effect } from "effect";
 import * as Schema from "effect/Schema";
-//#region ../runner/src/errors.d.ts
-declare const AutExecutionError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
-  readonly _tag: "AutExecutionError";
-} & Readonly<A>;
-export declare class AutExecutionError extends AutExecutionError_base<{
-  cause: unknown;
-  message?: string;
-}> {}
-declare const DatasetError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
-  readonly _tag: "DatasetError";
-} & Readonly<A>;
-export declare class DatasetError extends DatasetError_base<{
-  cause: unknown;
-  message?: string;
-}> {}
-declare const ReportError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
-  readonly _tag: "ReportError";
-} & Readonly<A>;
-export declare class ReportError extends ReportError_base<{
-  cause: unknown;
-  message?: string;
-}> {}
-declare const ScoringError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
-  readonly _tag: "ScoringError";
-} & Readonly<A>;
-export declare class ScoringError extends ScoringError_base<{
-  cause: unknown;
-  message?: string;
-}> {}
-declare const CheckpointExecutionError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
-  readonly _tag: "CheckpointExecutionError";
-} & Readonly<A>;
-export declare class CheckpointExecutionError extends CheckpointExecutionError_base<{
-  step: number;
-  name: string;
-  message: string;
-  cause: unknown;
-}> {}
-declare const CancelledError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
-  readonly _tag: "CancelledError";
-} & Readonly<A>;
-/** The run was cancelled (Ctrl+C); the run stops and is finalized as cancelled. */
-export declare class CancelledError extends CancelledError_base<{
-  message: string;
-}> {}
-//#endregion
+import { AssistantBlock, EntryRecord, MessageRecord, ModelRef, Origin, StopReason, Transcript, Transcript as Transcript$1, TreeRecord, TreeRecord as TreeRecord$1, Usage, UserBlock } from "@leostera/atxf";
+import { Effect } from "effect";
 //#region ../core/src/identity.d.ts
 declare const ResourceKindSchema: Schema.Literal<["run", "artifact"]>;
 type ResourceKind = typeof ResourceKindSchema.Type;
@@ -106,66 +61,276 @@ type Dataset = {
   readonly source: DatasetSource;
 };
 //#endregion
-//#region ../core/src/aut-events.d.ts
+//#region ../core/src/usage.d.ts
 /**
- * Events as adapters emit them. Payloads accept any JSON-convertible value and
- * timestamps are optional; EvalKit converts them once into `AutEvent`s.
+ * Token usage as an adapter reports it: ATXF's buckets, each optional.
+ * The buckets are disjoint: `input` excludes cached tokens, which are
+ * `cacheRead` (read from a prompt cache) and `cacheWrite` (written to one);
+ * `output` includes `reasoning`. Convert your provider's conventions: one
+ * that counts cached tokens inside its prompt tokens reports
+ * `input: promptTokens - cachedTokens, cacheRead: cachedTokens`.
  */
-type AutEventInput = {
-  kind: 'started';
-  timestamp?: string;
-} | {
-  kind: 'message';
-  role: 'system' | 'user' | 'assistant' | 'tool';
-  content: unknown;
-  timestamp?: string;
-} | {
-  kind: 'tool-call';
-  id: string;
-  name: string;
-  arguments?: unknown;
-  timestamp?: string;
-} | {
-  kind: 'tool-result';
-  id: string;
-  name?: string;
-  result?: unknown;
-  timestamp?: string;
-} | {
-  kind: 'turn-started';
-  turn: number;
-  timestamp?: string;
-} | {
-  kind: 'turn-completed';
-  turn: number;
-  timestamp?: string;
-  latencyMs?: number;
-  usage?: Usage;
-  /** The model that answered this turn, when the agent knows it. */
-  model?: string;
-} | {
-  kind: 'completed';
-  output?: unknown;
-  timestamp?: string;
-} | {
-  kind: 'error';
-  error: RecordedError;
-  timestamp?: string;
+type UsageInput = {
+  input?: number;
+  output?: number;
+  cacheRead?: number;
+  cacheWrite?: number;
+  reasoning?: number;
+  /** The provider's total; used only when no bucket is reported. */
+  total?: number;
+};
+//#endregion
+//#region ../core/src/agent-transcript.d.ts
+type DistributiveOmit<T, K extends PropertyKey> = T extends unknown ? Omit<T, K> : never;
+/** An ATXF tree record to append: `id`, `time` and `parentId` are filled in when absent. */
+type RecordInput = DistributiveOmit<TreeRecord, 'id' | 'time' | 'parentId' | 'hash'> & {
+  id?: string;
+  time?: string;
+  parentId?: string | null;
+};
+/** An assistant message: what the model said and the tools it called. */
+type AssistantInput = {
+  /** Text, or ATXF blocks: `text`, `reasoning`, `tool_call` and `image`. */
+  content: string | AssistantBlock[];
+  /** `provider/id` (`openai/gpt-5`), or an ATXF model reference. */
+  model?: string | ModelRef;
+  /** This response's usage, in ATXF's buckets. */
+  usage?: UsageInput;
+  stop?: StopReason;
+  error?: {
+    message: string;
+    code?: string;
+  };
+};
+/**
+ * Where an adapter records what its agent did, as an ATXF transcript
+ * (https://atxf.leostera.dev). Each method appends one record and returns
+ * its ID. EvalKit writes each scripted user message itself, unless the
+ * adapter writes it (or imports a transcript holding it) first.
+ */
+type TranscriptWriter = {
+  /** A message the agent received. */
+  user(content: string | UserBlock[], options?: {
+    origin?: Origin;
+  }): string;
+  assistant(message: AssistantInput): string;
+  /**
+   * A tool call's result. A string is the result's text; any other
+   * JSON-convertible value is kept as is in `details.result` and shown as
+   * JSON.
+   */
+  toolResult(callId: string, result: unknown, options?: {
+    name?: string;
+    isError?: boolean;
+  }): string;
+  /** Any other ATXF tree record, such as `reasoning` settings, a compaction or a sub-agent link. */
+  append(record: RecordInput): string;
+  /**
+   * Appends an ATXF transcript's records that were not imported before (by
+   * ID), keeping their IDs and native records: for agents whose sessions
+   * ATXF converts (`pi.toAtxf(await readFile(sessionFile, 'utf8'))`, from `@leostera/atxf/pi`). Import the whole session
+   * after each turn; a user message matching the one EvalKit is about to
+   * write stands for it.
+   */
+  import(transcript: Transcript): void;
+  /**
+   * Marks a turn's start or end, as the agent saw it: `turn('start')`
+   * returns a turn ID to pass to `turn('end', { turnId, outcome })`.
+   */
+  turn(phase: 'start' | 'end', options?: {
+    turnId?: string;
+    outcome?: 'succeeded' | 'failed' | 'interrupted';
+    error?: {
+      message: string;
+      code?: string;
+    };
+  }): string;
+  /**
+   * Keeps a sub-agent's ATXF transcript with the run, in
+   * `transcripts/<id>.atxf.jsonl`, linked from this one by an `agent_link`
+   * record with its `transcriptId` (append one, or import a session that
+   * has it). Its steps and tokens count toward the run's limits. Pass the
+   * whole transcript again as it grows; it replaces the one kept.
+   */
+  subagent(transcript: Transcript): void;
 };
 type TurnToolCall = {
   name: string;
   arguments?: unknown;
-  /** When given, a `tool-result` event is recorded after the call. */
+  /** When given, the call's result is recorded after it. */
   result?: unknown;
   id?: string;
 };
-/** What `send()` may return instead of (or as well as) emitting events. A string is the reply. */
+/** What `send()` may return instead of writing to the transcript. A string is the reply. */
 type TurnResult = string | {
+  /** Text, or a JSON value (recorded as its JSON text). */
   reply?: unknown;
   toolCalls?: readonly TurnToolCall[];
-  usage?: Usage;
-  /** The model that answered, recorded with the usage. */
-  model?: string;
+  usage?: UsageInput;
+  /** The model that answered. */
+  model?: string | ModelRef;
+};
+//#endregion
+//#region ../core/src/isolation.d.ts
+/**
+ * Where a run's processes run, and the shell adapters and scorers use to
+ * run them (RFD0009). Types and constructors only: the runner implements
+ * them.
+ */
+/** Commands run on the host, in the run's workspace. The default. */
+type NoIsolation = {
+  kind: 'none';
+};
+/**
+ * What a run's container can reach: nothing, only the project's fixtures,
+ * or whatever the container runtime allows by default.
+ */
+type ContainerNetwork = 'none' | 'fixtures' | 'full';
+/** A host path mounted read-only into every run's container. */
+type ContainerMount = {
+  host: string;
+  path: string;
+};
+/** One container per run, from `image`, with the candidate workspace at `/workspace`. */
+type ContainerIsolation = {
+  kind: 'container';
+  /** An image reference (`node:22`), or a Dockerfile path relative to the project (`./evals/Dockerfile`). */
+  image: string;
+  network: ContainerNetwork;
+  /** The `docker`-compatible command to run: `docker` (the default) or `podman`. */
+  runtime: string;
+  /** Names of host environment variables passed into the container; nothing else is. */
+  env: readonly string[];
+  mounts: readonly ContainerMount[];
+};
+/** macOS only: each run's processes run under a generated `sandbox-exec` profile. */
+type SandboxExecIsolation = {
+  kind: 'sandbox-exec';
+  /** Host paths the run may read and write besides its workspace, such as a cache (`~/.npm`). */
+  allow: readonly string[];
+  /** Host paths the run may only read, such as a toolchain or an agent CLI's install. */
+  read: readonly string[];
+  /** More host paths the run may not touch, such as a fixture's state; `allow` and `read` still win inside them. */
+  deny: readonly string[];
+  /**
+   * Names of host environment variables the run gets, besides `PATH`,
+   * `HOME`, `TMPDIR`, `USER`, `LANG` and `TERM`; undefined passes the whole
+   * environment. A command's own `env` is added on top.
+   */
+  env?: readonly string[];
+};
+type Isolation = NoIsolation | ContainerIsolation | SandboxExecIsolation;
+/**
+ * A service runs reach, such as a fake store or API, started on the host
+ * before the first run that needs it and stopped after the invocation's
+ * last.
+ */
+type Fixture = {
+  /** The command that starts it, run with `sh -c` in the project root. */
+  start: string;
+  /** A URL polled until it answers, or a command that must exit 0. */
+  ready?: string;
+  /** A command run before the fixture is stopped. */
+  stop?: string;
+  /** The port it listens on; taken from `ready` when that is a URL with one. */
+  port?: number;
+};
+/** A fixture as one run reaches it. */
+type FixtureAddress = {
+  /** Where the run reaches it: `http://localhost:8787`, or `http://fixtures.<name>:8787` from a container. */
+  url: string;
+};
+/** A process's exit: a signal shows as 128 plus its number, a timeout as 124. */
+type ShellExit = {
+  exitCode: number;
+  /** The signal that stopped it, if one did. */
+  signal?: string;
+  /** True when `timeoutMs` passed and the process was stopped. */
+  timedOut?: true;
+};
+type ShellResult = ShellExit & {
+  stdout: string;
+  stderr: string;
+};
+type ShellOptions = {
+  /** Relative to the workspace. */
+  cwd?: string;
+  /** Added to the environment the process gets. */
+  env?: Readonly<Record<string, string>>;
+  /** Stops the process when aborted. Default: the run's signal. */
+  signal?: AbortSignal;
+  /** Stops the process after this many milliseconds. */
+  timeoutMs?: number;
+  /** Written to the process's standard input, which is then closed. */
+  input?: string;
+};
+/** A process started with `spawn`. Its output is always read, so it never blocks on a full pipe. */
+type ShellProcess = {
+  /** Its process ID where it runs (inside the container, for container isolation). */
+  readonly pid: number | undefined;
+  /** Its standard output so far: the last megabyte. */
+  stdout(): string;
+  stderr(): string;
+  /** Calls `listener` with each chunk of output as it arrives; returns a function that stops. */
+  onOutput(listener: (chunk: string, stream: 'stdout' | 'stderr') => void): () => void;
+  /** Resolves when it exits. */
+  wait(): Promise<ShellExit>;
+  /** Signals it and the processes it started. Default `SIGTERM`. */
+  kill(signal?: string): Promise<void>;
+};
+/**
+ * The run's shell: commands, long-lived processes and files, wherever the
+ * run's isolation puts them. One per run, shared by the adapter and the
+ * run's scorers; processes it started are stopped when the run ends. A
+ * string command runs with `sh -c`; an array runs as it is.
+ */
+type Shell = {
+  readonly isolation: Isolation['kind'];
+  /** The workspace as commands see it: its host path, or `/workspace` in a container. */
+  readonly root: string;
+  /**
+   * The run's private directory, outside the workspace, as commands see it
+   * (`/evalkit-home` in a container): for an agent CLI's config and session
+   * files. It is never snapshotted, and is removed with the run.
+   */
+  readonly home: string;
+  exec(command: string | readonly string[], options?: ShellOptions): Promise<ShellResult>;
+  spawn(command: string | readonly string[], options?: Omit<ShellOptions, 'timeoutMs'>): ShellProcess;
+  /** Relative to the workspace, or absolute inside `root` or `home`. */
+  readFile(path: string): Promise<string>;
+  writeFile(path: string, content: string | Uint8Array): Promise<void>;
+  /** A directory's entries, relative to the workspace; directories end with `/`. */
+  list(path?: string): Promise<string[]>;
+};
+//#endregion
+//#region ../core/src/limits.d.ts
+/** What a run may use before EvalKit stops it. */
+type Limits = {
+  /** Assistant messages, one per model call, sub-agents included. */
+  steps?: number;
+  /** The `tokens` metric: uncached input, cache writes and output. Cache reads do not count. */
+  tokens?: number;
+  /** Milliseconds from the session's start to the end of its last turn; scoring is not counted. */
+  time?: number;
+};
+type LimitKind = keyof Limits;
+/** Limits as written in config and scenarios: `time` may be a duration such as `'20m'`. */
+type LimitsInput = {
+  steps?: number;
+  tokens?: number;
+  /** Milliseconds, or `'90s'`, `'20m'`, `'1h'`. */
+  time?: number | string;
+};
+/** Which limit a run hit, its value, and how much was used. */
+type LimitReached = {
+  limit: LimitKind;
+  value: number;
+  used: number;
+};
+//#endregion
+//#region ../core/src/transcript.d.ts
+type AssistantMessage = MessageRecord & {
+  role: 'assistant';
 };
 //#endregion
 //#region ../core/src/judge.d.ts
@@ -324,9 +489,10 @@ type EvalCatalogEntry = {
   /** Runs started each time the scenario runs (`repeat`). */
   repeat: number;
   policy: {
-    timeoutMs?: number;
     failfast?: boolean;
   };
+  /** The scenario's own limits; the project's, a cell's and `--limit` may add to them. */
+  limits?: Limits;
   transcript: EvalCatalogStep[];
   scorers: EvalCatalogScorer[];
   metadata?: JsonValue;
@@ -360,6 +526,15 @@ type EvalMatrixDefinition = {
   cases?: readonly JsonObject[];
   /** Partial patterns over declared dimensions; matching cells are never run. */
   exclude?: readonly JsonObject[];
+  /**
+   * Limits to compare, as axes: `{ steps: [40, 80] }`. Each becomes the axis
+   * `limits.steps`, so cells show it and `-a limits.steps=80` selects it.
+   */
+  limits?: Readonly<{
+    steps?: readonly number[];
+    tokens?: readonly number[];
+    time?: readonly (number | string)[];
+  }>;
 };
 type EvalMatrix = EvalMatrixDefinition & {
   kind: 'matrix';
@@ -376,26 +551,6 @@ type AttachmentLimits = {
   maxRunBytes?: number;
 };
 //#endregion
-//#region ../core/src/usage.d.ts
-/**
- * Token usage an agent reports for one turn. EvalKit never measures or infers
- * it; adapters copy it from their provider.
- */
-type Usage = {
-  /**
-   * Every input token the model read this turn: the prompt, history and tool
-   * results, **including** tokens read from or written to a prompt cache.
-   * (Some providers report cached tokens separately; add them in.)
-   */
-  inputTokens?: number;
-  /** Every token the model produced, including reasoning tokens. */
-  outputTokens?: number;
-  /** Of `inputTokens`, how many were read from a prompt cache. Informational. */
-  cachedInputTokens?: number;
-  /** The provider's own total; used only when input and output are not reported. */
-  totalTokens?: number;
-};
-//#endregion
 //#region ../core/src/index.d.ts
 type JsonPrimitive = boolean | number | string | null;
 type JsonValue = JsonPrimitive | JsonValue[] | {
@@ -426,47 +581,6 @@ type RecordedError = {
   message: string;
   stack?: string;
 };
-type AutEvent = {
-  kind: 'started';
-  timestamp: string;
-} | {
-  kind: 'message';
-  role: 'system' | 'user' | 'assistant' | 'tool';
-  content: JsonValue;
-  timestamp: string;
-} | {
-  kind: 'tool-call';
-  id: string;
-  name: string;
-  arguments: JsonValue;
-  timestamp: string;
-} | {
-  kind: 'tool-result';
-  id: string;
-  name?: string;
-  result: JsonValue;
-  timestamp: string;
-} | {
-  kind: 'turn-started';
-  turn: number;
-  timestamp: string;
-} | {
-  kind: 'turn-completed';
-  turn: number;
-  timestamp: string;
-  latencyMs?: number;
-  usage?: Usage;
-  /** The model that answered this turn, when the agent knows it. */
-  model?: string;
-} | {
-  kind: 'completed';
-  output?: JsonValue;
-  timestamp: string;
-} | {
-  kind: 'error';
-  error: RecordedError;
-  timestamp: string;
-};
 type RunnerEvent = {
   kind: 'run-started';
   timestamp: string;
@@ -515,7 +629,10 @@ type RunnerEvent = {
   scorer: string;
   error: RecordedError;
   timestamp: string;
-} | {
+} | ({
+  kind: 'limit-reached';
+  timestamp: string;
+} & LimitReached) | {
   kind: 'run-completed';
   timestamp: string;
 } | {
@@ -523,11 +640,9 @@ type RunnerEvent = {
   error: RecordedError;
   timestamp: string;
 };
-type TrajectoryEvent = (AutEvent & {
-  source: 'aut';
-}) | (RunnerEvent & {
-  source: 'runner';
-});
+type RunEvent = RunnerEvent & {
+  at: string | null;
+};
 type WorkspaceView = {
   /** Absolute path to this run-scoped workspace. Do not persist it in reports. */
   root: string;
@@ -545,6 +660,20 @@ type AutContext = {
   runtime?: AgentRuntimeName;
   /** Values of the scenario's candidate datasets, by dataset ID. */
   data: JsonObject;
+  /**
+   * Aborts when the run hits a limit or is cancelled. Pass it to model calls
+   * and tool processes so they stop with the run.
+   */
+  signal: AbortSignal;
+  /** The limits in force for this run; see `limits` in the config. */
+  limits: Limits;
+  /**
+   * The run's shell: run the agent and its tools through it, so they run
+   * where the project's `isolation` puts them. Scorers get the same one.
+   */
+  shell: Shell;
+  /** The project's fixtures, as this run reaches them, by name. */
+  fixtures: Readonly<Record<string, FixtureAddress>>;
   /** Contains only resources visible to the AUT/model. */
   workspace: CandidateWorkspace;
   /** Private evaluator capability for trusted AUT adapters; never expose this to the model. */
@@ -557,8 +686,11 @@ type AutAdapter<TMessage = string> = {
   start(options: {
     context: AutContext;
     runtime?: AgentRuntime;
-    /** Optional: stream events as they happen. Returning a turn result from `send` is enough for most agents. */
-    onEvent: (event: AutEventInput) => void | Promise<void>;
+    /**
+     * Where to record what the agent does, as ATXF. Optional: returning a
+     * turn result from `send` is enough for most agents.
+     */
+    transcript: TranscriptWriter;
     /**
      * Saves a file with the run's results, such as the agent's raw log,
      * whether or not its sandbox is kept. `name` is a relative path of
@@ -598,19 +730,15 @@ type AgentStep = {
   kind: 'agent';
   expectation: JsonObject;
 };
+/** One user step and what the agent did in answer to it. */
 type TurnView = {
   userStepIndex: number;
-  events: readonly (AutEvent & {
-    source: 'aut';
-  })[];
-  assistantMessages: readonly (Extract<AutEvent, {
-    kind: 'message';
-  }> & {
-    role: 'assistant';
-    source: 'aut';
-  })[];
+  /** The transcript records of this turn, on the active path, in order. */
+  records: readonly TreeRecord$1[];
+  assistantMessages: readonly AssistantMessage[];
   toolCalls: readonly {
-    eventIndex: number;
+    /** The assistant message holding the call. */
+    messageId: string;
     id: string;
     name: string;
     arguments: JsonValue;
@@ -635,7 +763,7 @@ type TranscriptStep = UserStep | AgentStep | ScoringRule | ExpectToolCallStep;
  */
 type JudgeRunInfo = {
   agent?: AutIdentity;
-  /** The model the judge reported using (`turn-completed` `model`). */
+  /** The model the judge reported using, as `provider/id`. */
   model?: string;
   /** Tokens the judge used: never counted in the agent's `tokens`. */
   usage?: Usage;
@@ -645,8 +773,8 @@ type JudgeRunInfo = {
   reply?: JsonValue;
   /** 2 when its first reply was not in the format asked for and it was asked again. */
   attempts?: number;
-  /** Judge-agent observations; never attributed to the AUT trajectory. */
-  events?: JsonValue[];
+  /** The judge's own session, as ATXF; never part of the agent's transcript. */
+  transcript?: Transcript$1;
 };
 /** A scorer's "does not apply" result; see `skip()`. */
 type SkippedScore = {
@@ -697,14 +825,21 @@ type ArtifactView = {
 };
 type ScoringContext = {
   context: AutContext;
-  trajectory: {
-    events: readonly TrajectoryEvent[];
-  };
+  /**
+   * What the agent did, as an ATXF transcript: all of it for final scorers,
+   * up to this step for checkpoints. `messages(ctx)`, `toolCalls(ctx)` and
+   * `lastReply(ctx)` read it.
+   */
+  transcript: Transcript$1;
+  /** Sub-agent transcripts the agent kept, by transcript ID. */
+  subagents: Readonly<Record<string, Transcript$1>>;
+  /** EvalKit's events so far: steps, checkpoints and scorers. */
+  runEvents: readonly RunEvent[];
   artifacts: ArtifactView;
   /** Most recent completed turn; absent if no user message has been sent. */
   turn?: TurnView;
-  /** Content of the last assistant message of `turn`. */
-  output?: JsonValue;
+  /** The text of the last assistant message of `turn`. */
+  output?: string;
   /** Values of all the scenario's datasets (candidate and evaluator), by dataset ID. */
   data: JsonObject;
   /**
@@ -811,7 +946,6 @@ type MetricResult = {
   error?: RecordedError;
 };
 type EvalPolicy = {
-  timeoutMs?: number;
   /** Stop later transcript steps after a failed checkpoint assertion. Default false. */
   failfast?: boolean;
 };
@@ -831,6 +965,11 @@ type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
    * under identical conditions, to see how often it passes. Default 1.
    */
   repeat?: number;
+  /**
+   * What each run may use before EvalKit stops it: `{ steps: 40, time: '20m' }`.
+   * Over the project's `limits`; a matrix cell's and `--limit` come after.
+   */
+  limits?: LimitsInput;
   /** Any JSON-convertible object; converted when the eval is defined. */
   metadata?: Record<string, unknown>;
   workspace?: EvalWorkspace;
@@ -859,9 +998,11 @@ type EvalSuiteMetadata = {
 };
 /**
  * How a run ended: `passed` or `failed` by its scores, `errored` when it could
- * not finish (the agent, a dataset or EvalKit failed), `cancelled` by Ctrl+C.
+ * not finish (the agent, a dataset or EvalKit failed), `cancelled` by Ctrl+C,
+ * `limited` when it hit one of its limits (graded by scorers that support
+ * partial work, and never a pass).
  */
-type RunStatus = 'running' | 'passed' | 'failed' | 'errored' | 'cancelled';
+type RunStatus = 'running' | 'passed' | 'failed' | 'errored' | 'cancelled' | 'limited';
 /** Runs launched together as repeats of one configuration (`repeat`). */
 type Batch = {
   id: string;
@@ -903,7 +1044,7 @@ type RunProcess = {
   hostname: string;
 };
 type RunMetadata = {
-  schemaVersion: 6;
+  schemaVersion: 7;
   runId: string;
   runUri: ResourceUri<'run'>;
   evalId: string;
@@ -924,6 +1065,18 @@ type RunMetadata = {
   workspaceIgnore?: string[];
   /** The datasets this run used, with content digests. */
   datasets?: RunDataset[];
+  /** The limits in force. */
+  limits?: Limits;
+  /** Where its processes ran; absent for runs on the host. */
+  isolation?: RunIsolation;
+};
+/** A run's isolation as its manifest records it: no host paths, no environment. */
+type RunIsolation = {
+  kind: 'container';
+  image: string;
+  network: ContainerNetwork;
+} | {
+  kind: 'sandbox-exec';
 };
 type RunDataset = {
   id: string;
@@ -949,8 +1102,9 @@ type CheckpointResult = {
   judge?: JudgeRunInfo;
   durationMs?: number;
   error?: RecordedError;
+  /** The tool call it matched: the assistant message holding it, and the call's ID. */
   matchedToolCall?: {
-    eventIndex: number;
+    messageId: string;
     id: string;
   };
 };
@@ -958,13 +1112,17 @@ type CheckpointResult = {
  * Metrics EvalKit records for every run; scenario metrics may not reuse
  * these names.
  *
- * - `agent time` (ms): from the run's first event to its last one before
- *   final scoring.
- * - `tokens`: the sum of `turnTokens(usage)` over the turns that reported
- *   usage, cached input included. Absent when no turn reported any.
- * - `tool calls`: the agent's `tool-call` events.
+ * - `agent time` (ms): from the session's start to the end of its last turn.
+ * - `tokens`: uncached input, cache writes and output, summed over the
+ *   transcript (`uncachedTokens`). Absent when nothing reported usage.
+ * - `cached tokens`: input tokens read from a prompt cache.
+ * - `output tokens`: output, reasoning included.
+ * - `cache hit rate`: the share of input tokens read from a cache, 0 to 1.
+ * - `steps`: assistant messages, one per model call.
+ * - `tool calls`: tool calls in the transcript.
+ * - `judge tokens`: `tokens` over the judges' transcripts, never the agent's.
  */
-declare const BUILT_IN_METRICS: readonly ['agent time', 'tokens', 'tool calls', 'judge tokens'];
+declare const BUILT_IN_METRICS: readonly ['agent time', 'tokens', 'cached tokens', 'output tokens', 'cache hit rate', 'steps', 'tool calls', 'judge tokens'];
 type BuiltInMetricName = (typeof BUILT_IN_METRICS)[number];
 /** A built-in metric's name (suggested by editors), or a scenario's own. */
 type MetricName = BuiltInMetricName | (string & {});
@@ -998,10 +1156,17 @@ type RunSummary = {
   artifacts?: ArtifactEntry[];
   attachments?: AttachmentEntry[];
   error?: RecordedError;
+  /** The limit a `limited` run hit. */
+  limit?: LimitReached;
 };
 type RunWriter = {
   readonly location: string;
-  appendEvent(event: TrajectoryEvent): Promise<void>;
+  /** Appends one of EvalKit's events to `run-events.jsonl`. */
+  appendEvent(event: RunEvent): Promise<void>;
+  /** Appends an ATXF record (the header first) to `transcript.atxf.jsonl`. */
+  appendTranscript(line: Transcript$1['header'] | Transcript$1['records'][number]): Promise<void>;
+  /** Writes a sub-agent's whole transcript (JSONL), replacing the one kept. */
+  writeSubagent(id: string, text: string): Promise<void>;
   writeScores(scoring: RunScoring): Promise<void>;
   /** Writes a file below this run's artifact root. */
   writeArtifact(path: string, data: Uint8Array): Promise<void>;
@@ -1023,8 +1188,86 @@ type RunResult = {
   durationMs?: number;
   scoring?: RunScoring;
   error?: RecordedError;
+  /** The limit a `limited` run hit. */
+  limit?: LimitReached;
 };
 //#endregion
+//#region ../runner/src/errors.d.ts
+declare const AutExecutionError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "AutExecutionError";
+} & Readonly<A>;
+export declare class AutExecutionError extends AutExecutionError_base<{
+  cause: unknown;
+  message?: string;
+}> {}
+declare const DatasetError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "DatasetError";
+} & Readonly<A>;
+export declare class DatasetError extends DatasetError_base<{
+  cause: unknown;
+  message?: string;
+}> {}
+declare const ReportError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "ReportError";
+} & Readonly<A>;
+export declare class ReportError extends ReportError_base<{
+  cause: unknown;
+  message?: string;
+}> {}
+declare const ScoringError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "ScoringError";
+} & Readonly<A>;
+export declare class ScoringError extends ScoringError_base<{
+  cause: unknown;
+  message?: string;
+}> {}
+declare const CheckpointExecutionError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "CheckpointExecutionError";
+} & Readonly<A>;
+export declare class CheckpointExecutionError extends CheckpointExecutionError_base<{
+  step: number;
+  name: string;
+  message: string;
+  cause: unknown;
+}> {}
+declare const CancelledError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "CancelledError";
+} & Readonly<A>;
+/** The run was cancelled (Ctrl+C); the run stops and is finalized as cancelled. */
+export declare class CancelledError extends CancelledError_base<{
+  message: string;
+}> {}
+//#endregion
+//#region ../runner/src/shell/fixtures.d.ts
+type FixtureSet = {
+  /** Starts every fixture, once; later calls wait for the same start. Throws when one never becomes ready. */
+  ensure(): Promise<void>;
+  /** True once `ensure` was called. */
+  readonly started: boolean;
+  /** The fixtures as a run with this isolation reaches them. */
+  addresses(isolation: Isolation): Record<string, FixtureAddress>;
+  /** The internal network containers share with the fixtures' proxies, created on first use. */
+  network(runtime: string): Promise<string>;
+  /** Each fixture's output so far. */
+  logs(): Array<{
+    name: string;
+    content: string;
+  }>;
+  /** Runs each `stop` command, then stops the fixtures, their proxies and network. */
+  stop(): Promise<void>;
+};
+/**
+ * The project's fixtures for one invocation: started on the host, in the
+ * project root, the first time a run needs them, and stopped after its last.
+ */
+export declare function fixtureSet(fixtures: Readonly<Record<string, Fixture>>, options: {
+  root: string;
+}): FixtureSet;
+//#endregion
+//#region ../runner/src/shell/container-shell.d.ts
+/** Removes a run's kept container, if there is one; true when it removed one. */
+export declare function removeRunContainer(runId: string, runtime?: string): Promise<boolean>;
+//#endregion
 //#region ../runner/src/local-report-store.d.ts
 /** Persists each run as a local tree rooted below `rootDirectory`. */
 export declare function localReportStore(rootDirectory: string): ReportStore;
@@ -1046,7 +1289,7 @@ export declare function staticDatasetFile(dataset: Dataset, root: string, file:
 //#endregion
 //#region ../runner/src/local-report-reader.d.ts
 export declare const readRunManifest: (root: string, runId: string) => Promise<{
-  readonly schemaVersion: 6;
+  readonly schemaVersion: 7;
   readonly runUri: string;
   readonly evalId: string;
   readonly suiteId?: string | undefined;
@@ -1094,10 +1337,22 @@ export declare const readRunManifest: (root: string, runId: string) => Promise<{
     readonly source: "computed" | "static";
     readonly digest: string;
   }[] | undefined;
-  readonly status?: "cancelled" | "errored" | "failed" | "passed" | "running" | undefined;
+  readonly limits?: {
+    readonly steps?: number | undefined;
+    readonly tokens?: number | undefined;
+    readonly time?: number | undefined;
+  } | undefined;
+  readonly isolation?: {
+    readonly kind: "container";
+    readonly image: string;
+    readonly network: "fixtures" | "full" | "none";
+  } | {
+    readonly kind: "sandbox-exec";
+  } | undefined;
+  readonly status?: "cancelled" | "errored" | "failed" | "limited" | "passed" | "running" | undefined;
 }>;
 export declare const readRunSummary: (root: string, runId: string) => Promise<{
-  readonly status: "cancelled" | "errored" | "failed" | "passed" | "running";
+  readonly status: "cancelled" | "errored" | "failed" | "limited" | "passed" | "running";
   readonly endedAt: string;
   readonly durationMs?: number | undefined;
   readonly scoring?: {
@@ -1132,15 +1387,25 @@ export declare const readRunSummary: (root: string, runId: string) => Promise<{
         } | undefined;
         readonly model?: string | undefined;
         readonly usage?: {
-          readonly inputTokens?: number | undefined;
-          readonly outputTokens?: number | undefined;
-          readonly cachedInputTokens?: number | undefined;
-          readonly totalTokens?: number | undefined;
+          readonly input: number;
+          readonly output: number;
+          readonly cacheRead?: number | undefined;
+          readonly cacheWrite?: number | undefined;
+          readonly reasoning?: number | undefined;
+          readonly total?: number | undefined;
+          readonly cost?: {
+            readonly currency?: string | undefined;
+            readonly input?: number | undefined;
+            readonly output?: number | undefined;
+            readonly cacheRead?: number | undefined;
+            readonly cacheWrite?: number | undefined;
+            readonly total: number;
+          } | undefined;
         } | undefined;
         readonly prompt?: string | undefined;
         readonly reply?: JsonValue | undefined;
         readonly attempts?: number | undefined;
-        readonly events?: JsonValue[] | undefined;
+        readonly transcript?: Transcript$1 | undefined;
       } | undefined;
       readonly durationMs: number;
       readonly error?: {
@@ -1195,15 +1460,25 @@ export declare const readRunSummary: (root: string, runId: string) => Promise<{
         } | undefined;
         readonly model?: string | undefined;
         readonly usage?: {
-          readonly inputTokens?: number | undefined;
-          readonly outputTokens?: number | undefined;
-          readonly cachedInputTokens?: number | undefined;
-          readonly totalTokens?: number | undefined;
+          readonly input: number;
+          readonly output: number;
+          readonly cacheRead?: number | undefined;
+          readonly cacheWrite?: number | undefined;
+          readonly reasoning?: number | undefined;
+          readonly total?: number | undefined;
+          readonly cost?: {
+            readonly currency?: string | undefined;
+            readonly input?: number | undefined;
+            readonly output?: number | undefined;
+            readonly cacheRead?: number | undefined;
+            readonly cacheWrite?: number | undefined;
+            readonly total: number;
+          } | undefined;
         } | undefined;
         readonly prompt?: string | undefined;
         readonly reply?: JsonValue | undefined;
         readonly attempts?: number | undefined;
-        readonly events?: JsonValue[] | undefined;
+        readonly transcript?: Transcript$1 | undefined;
       } | undefined;
       readonly durationMs?: number | undefined;
       readonly error?: {
@@ -1212,7 +1487,7 @@ export declare const readRunSummary: (root: string, runId: string) => Promise<{
         readonly stack?: string | undefined;
       } | undefined;
       readonly matchedToolCall?: {
-        readonly eventIndex: number;
+        readonly messageId: string;
         readonly id: string;
       } | undefined;
     }[] | undefined;
@@ -1235,6 +1510,11 @@ export declare const readRunSummary: (root: string, runId: string) => Promise<{
     readonly message: string;
     readonly stack?: string | undefined;
   } | undefined;
+  readonly limit?: {
+    readonly limit: "steps" | "time" | "tokens";
+    readonly value: number;
+    readonly used: number;
+  } | undefined;
 }>;
 export declare const readRunScoring: (root: string, runId: string) => Promise<{
   readonly results: {
@@ -1268,15 +1548,25 @@ export declare const readRunScoring: (root: string, runId: string) => Promise<{
       } | undefined;
       readonly model?: string | undefined;
       readonly usage?: {
-        readonly inputTokens?: number | undefined;
-        readonly outputTokens?: number | undefined;
-        readonly cachedInputTokens?: number | undefined;
-        readonly totalTokens?: number | undefined;
+        readonly input: number;
+        readonly output: number;
+        readonly cacheRead?: number | undefined;
+        readonly cacheWrite?: number | undefined;
+        readonly reasoning?: number | undefined;
+        readonly total?: number | undefined;
+        readonly cost?: {
+          readonly currency?: string | undefined;
+          readonly input?: number | undefined;
+          readonly output?: number | undefined;
+          readonly cacheRead?: number | undefined;
+          readonly cacheWrite?: number | undefined;
+          readonly total: number;
+        } | undefined;
       } | undefined;
       readonly prompt?: string | undefined;
       readonly reply?: JsonValue | undefined;
       readonly attempts?: number | undefined;
-      readonly events?: JsonValue[] | undefined;
+      readonly transcript?: Transcript$1 | undefined;
     } | undefined;
     readonly durationMs: number;
     readonly error?: {
@@ -1331,15 +1621,25 @@ export declare const readRunScoring: (root: string, runId: string) => Promise<{
       } | undefined;
       readonly model?: string | undefined;
       readonly usage?: {
-        readonly inputTokens?: number | undefined;
-        readonly outputTokens?: number | undefined;
-        readonly cachedInputTokens?: number | undefined;
-        readonly totalTokens?: number | undefined;
+        readonly input: number;
+        readonly output: number;
+        readonly cacheRead?: number | undefined;
+        readonly cacheWrite?: number | undefined;
+        readonly reasoning?: number | undefined;
+        readonly total?: number | undefined;
+        readonly cost?: {
+          readonly currency?: string | undefined;
+          readonly input?: number | undefined;
+          readonly output?: number | undefined;
+          readonly cacheRead?: number | undefined;
+          readonly cacheWrite?: number | undefined;
+          readonly total: number;
+        } | undefined;
       } | undefined;
       readonly prompt?: string | undefined;
       readonly reply?: JsonValue | undefined;
       readonly attempts?: number | undefined;
-      readonly events?: JsonValue[] | undefined;
+      readonly transcript?: Transcript$1 | undefined;
     } | undefined;
     readonly durationMs?: number | undefined;
     readonly error?: {
@@ -1348,7 +1648,7 @@ export declare const readRunScoring: (root: string, runId: string) => Promise<{
       readonly stack?: string | undefined;
     } | undefined;
     readonly matchedToolCall?: {
-      readonly eventIndex: number;
+      readonly messageId: string;
       readonly id: string;
     } | undefined;
   }[] | undefined;
@@ -1356,7 +1656,15 @@ export declare const readRunScoring: (root: string, runId: string) => Promise<{
   readonly overall?: number | undefined;
   readonly passed: boolean;
 }>;
-export declare function readRunEvents(root: string, runId: string): Promise<TrajectoryEvent[]>;
+/** EvalKit's events for a run, in order. */
+export declare function readRunEvents(root: string, runId: string): Promise<RunEvent[]>;
+/** A run's sub-agent transcripts, by transcript ID; none when it kept none. */
+export declare function readRunSubagents(root: string, runId: string): Promise<Record<string, Transcript$1>>;
+/**
+ * The agent's ATXF transcript for a run. A running run's last line may be
+ * half written; it is left out until it is complete.
+ */
+export declare function readRunTranscript(root: string, runId: string): Promise<Transcript$1>;
 /**
  * True when a run's owning process is known to be gone: it ran on this host
  * and its PID no longer exists. Runs from other hosts are never judged.
@@ -1364,12 +1672,16 @@ export declare function readRunEvents(root: string, runId: string): Promise<Traj
 export declare function ownerIsGone(owner: RunProcess | undefined): boolean;
 //#endregion
 //#region ../runner/src/matrix.d.ts
-/** Cells and runs; a run that errored counts as failed, a cancelled one as neither. */
+/**
+ * Cells and runs; a run that errored counts as failed, one stopped at a
+ * limit as limited, a cancelled one as none of them.
+ */
 type MatrixSummary = {
   cells: number;
   runs: number;
   passed: number;
   failed: number;
+  limited: number;
   cancelled: number;
 };
 /** Bounded workers pull cells from the iterator; no Cartesian job array is allocated. */
@@ -1391,6 +1703,13 @@ type UpgradeReport = {
   batches: number;
   /** Old runs left alone because they are still running. */
   running: string[];
+  /** Version 6 runs whose trajectory became an ATXF transcript (or would). */
+  transcripts: number;
+  /** Runs that could not be converted, left as they were, with why. */
+  failed: Array<{
+    runId: string;
+    reason: string;
+  }>;
 };
 /**
  * Rewrites results from report schema versions 2 to 5 into version 6, in
@@ -1403,7 +1722,8 @@ type UpgradeReport = {
  * old run is removed only after its new runs exist, and one replaced under
  * the same ID is set aside first, so a crash at any point leaves the old run
  * or complete new ones, and re-running finishes the job. Version 6 runs are
- * left alone.
+ * left alone. Then every version 6 run is converted to version 7, its
+ * trajectory into an ATXF transcript (RFD0009).
  */
 export declare function upgradeReports(root: string, options?: {
   dryRun?: boolean;
@@ -1423,6 +1743,10 @@ export type RunEvalOptions = {
   repeat?: number;
   /** Parameters made available to the AUT, and to judges, for this invocation. */
   parameters?: JsonObject;
+  /** Where each run's processes run, through `context.shell`. Default `none()`: on the host. */
+  isolation?: Isolation;
+  /** The project's fixtures, shared by an invocation's runs; started by the first run. */
+  fixtures?: FixtureSet;
   /** The agent for judges that name none (`judge` in evalkit.config.ts). */
   judge?: AutAdapter;
   /** Matrix provenance recorded in every run's manifest. */
@@ -1456,12 +1780,29 @@ export type RunEvalOptions = {
    */
   signal?: AbortSignal;
   /**
-   * Observes every trajectory event after it is recorded, for live progress.
-   * It must not throw; an error it throws is ignored.
+   * What each run may use before EvalKit stops it. `base` is the project's
+   * limits; the scenario's own come after it, then a matrix cell's
+   * `limits.<kind>` parameters, then `override` (`--limit`), kind by kind.
    */
-  onEvent?: (run: RunRef, event: TrajectoryEvent) => void;
+  limits?: {
+    base?: Limits;
+    override?: Limits;
+  };
+  /**
+   * Observes every transcript record and run event once recorded, for live
+   * progress. It must not throw; an error it throws is ignored.
+   */
+  onUpdate?: (run: RunRef, update: RunUpdate) => void;
   now?: () => Date;
 };
+/** A record appended to a run's transcript, or one of EvalKit's events. */
+export type RunUpdate = {
+  kind: 'record';
+  record: EntryRecord;
+} | {
+  kind: 'event';
+  event: RunEvent;
+};
 /** Identifies the run an observed event belongs to. */
 export type RunRef = {
   runId: string;
@@ -1477,5 +1818,5 @@ export type RunRef = {
  */
 export declare function runEval(definition: EvalDefinition, options: RunEvalOptions): Effect.Effect<RunResult[], unknown>;
 //#endregion
-export type { DatasetFileEntry, MatrixSummary, UpgradeReport };
+export type { DatasetFileEntry, FixtureSet, MatrixSummary, UpgradeReport };
 //# sourceMappingURL=runner.d.mts.map