@leostera/evalkit 0.4.0 → 0.6.0

package/dist/runner.d.mts

@@ -1,5 +1,5 @@
-import { $ as TrajectoryEvent, B as ReportStore, F as JsonValue, H as RunMetadata, Jt as EvalMatrixCell, N as JsonObject, U as RunResult, V as RunCatalog, W as RunSource, Yt as MatrixSelection, n as AgentRuntimeName, qt as EvalMatrix, y as EvalDefinition } from "./index-U6TUTBf7.mjs";
 import { Effect } from "effect";
+import * as Schema from "effect/Schema";
 //#region ../runner/src/errors.d.ts
 declare const AutExecutionError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
   readonly _tag: "AutExecutionError";
@@ -8,10 +8,10 @@ export declare class AutExecutionError extends AutExecutionError_base<{
   cause: unknown;
   message?: string;
 }> {}
-declare const FixtureError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
-  readonly _tag: "FixtureError";
+declare const DatasetError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "DatasetError";
 } & Readonly<A>;
-export declare class FixtureError extends FixtureError_base<{
+export declare class DatasetError extends DatasetError_base<{
   cause: unknown;
   message?: string;
 }> {}
@@ -38,14 +38,765 @@ export declare class CheckpointExecutionError extends CheckpointExecutionError_b
   message: string;
   cause: unknown;
 }> {}
+declare const CancelledError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "CancelledError";
+} & Readonly<A>;
+/** The run was cancelled (Ctrl+C); the trial stops and is finalized as cancelled. */
+export declare class CancelledError extends CancelledError_base<{
+  message: string;
+}> {}
+//#endregion
+//#region ../core/src/identity.d.ts
+declare const ResourceKindSchema: Schema.Literal<["run", "trial", "artifact"]>;
+type ResourceKind = typeof ResourceKindSchema.Type;
+type ResourceUri<TKind extends ResourceKind = ResourceKind> = `evalkit:${TKind}:${string}`;
+//#endregion
+//#region ../core/src/dataset.d.ts
+/** Who can see a dataset: the agent (and scorers), or only scorers. */
+type DatasetVisibility = 'candidate' | 'evaluator';
+/** Inline text or bytes, or a file relative to the project root. */
+type DatasetFileSource = string | Uint8Array | {
+  file: string;
+};
+/**
+ * Files for the trial workspace: a directory relative to the project root, or
+ * a map from workspace path (below the dataset's mount) to its content.
+ */
+type DatasetFiles = string | Readonly<Record<string, DatasetFileSource>>;
+/** What one trial receives from a dataset. */
+type DatasetContent = {
+  files?: DatasetFiles;
+  /** Any JSON-convertible value; converted once when the trial starts. */
+  value?: unknown;
+};
+type DatasetLoadContext = {
+  /** The project root; relative paths in returned content resolve against it. */
+  root: string;
+  signal: AbortSignal;
+};
+type DatasetTrialContext = {
+  evalId: string;
+  runId: string;
+  trialId: string;
+  trialIndex: number;
+  /** Per-trial random seed, so repeated trials can choose different data. */
+  seed: number;
+  /** The matrix cell's parameters, or `{}`. */
+  parameters: JsonObject;
+  signal: AbortSignal;
+};
+type DatasetSource = {
+  kind: 'static';
+  files?: DatasetFiles;
+  value?: unknown;
+  valueFile?: string;
+} | {
+  kind: 'computed';
+  onLoad: (load: DatasetLoadContext) => DatasetContent | Promise<DatasetContent>;
+  onRun?: (loaded: DatasetContent, trial: DatasetTrialContext) => DatasetContent | Promise<DatasetContent>;
+};
+/** A checked dataset, as returned by `defineDataset`. Evals accept only these. */
+type Dataset = {
+  readonly kind: 'dataset';
+  readonly id: string;
+  readonly name?: string;
+  readonly visibility: DatasetVisibility;
+  /** Normalized relative POSIX path; `.` is the workspace root. */
+  readonly mount: string;
+  readonly source: DatasetSource;
+};
+//#endregion
+//#region ../core/src/aut-events.d.ts
+/**
+ * Events as adapters emit them. Payloads accept any JSON-convertible value and
+ * timestamps are optional; EvalKit converts them once into `AutEvent`s.
+ */
+type AutEventInput = {
+  kind: 'started';
+  timestamp?: string;
+} | {
+  kind: 'message';
+  role: 'system' | 'user' | 'assistant' | 'tool';
+  content: unknown;
+  timestamp?: string;
+} | {
+  kind: 'tool-call';
+  id: string;
+  name: string;
+  arguments?: unknown;
+  timestamp?: string;
+} | {
+  kind: 'tool-result';
+  id: string;
+  name?: string;
+  result?: unknown;
+  timestamp?: string;
+} | {
+  kind: 'turn-started';
+  turn: number;
+  timestamp?: string;
+} | {
+  kind: 'turn-completed';
+  turn: number;
+  timestamp?: string;
+  latencyMs?: number;
+  usage?: Usage;
+} | {
+  kind: 'completed';
+  output?: unknown;
+  timestamp?: string;
+} | {
+  kind: 'error';
+  error: RecordedError;
+  timestamp?: string;
+};
+type TurnToolCall = {
+  name: string;
+  arguments?: unknown;
+  /** When given, a `tool-result` event is recorded after the call. */
+  result?: unknown;
+  id?: string;
+};
+/** What `send()` may return instead of (or as well as) emitting events. A string is the reply. */
+type TurnResult = string | {
+  reply?: unknown;
+  toolCalls?: readonly TurnToolCall[];
+  usage?: Usage;
+};
+//#endregion
+//#region ../core/src/catalog.d.ts
+type EvalCatalogAgent = {
+  name?: string;
+  kind: string;
+  id?: string;
+  version?: string;
+  runtimes: Array<{
+    name: AgentRuntimeName;
+    kind: string;
+  }>;
+  /** `file:line` of its `defineAgent` call, relative to the project. */
+  definedAt?: string;
+};
+/** A value's shape, without its content. */
+type ValueShape = {
+  type: 'array';
+  items: number;
+} | {
+  type: 'object';
+  keys: string[];
+} | {
+  type: 'string' | 'number' | 'boolean' | 'null';
+};
+type EvalCatalogDataset = {
+  id: string;
+  name?: string;
+  visibility: DatasetVisibility;
+  source: 'static' | 'computed';
+  mount: string;
+  /** Static datasets: a directory relative to the project, or the workspace paths of inline files. */
+  files?: {
+    directory: string;
+  } | {
+    paths: string[];
+  };
+  /** Static datasets: the inline value's shape, or the value file's path. */
+  value?: {
+    shape: ValueShape;
+  } | {
+    file: string;
+  };
+  /** Computed datasets: which hooks are defined. */
+  hooks?: {
+    onLoad: true;
+    onRun: boolean;
+  };
+  definedAt?: string;
+};
+type EvalCatalogScorer = {
+  name: string;
+  kind: ScoringRule['kind'];
+  passAt?: number;
+  supportsPartial?: boolean;
+  /** Judges only. */
+  rubric?: string;
+  definedAt?: string;
+};
+type EvalCatalogStep = {
+  kind: 'user';
+  message: string;
+} | {
+  kind: 'agent';
+  expectation: JsonValue;
+} | {
+  kind: 'expect-tool-call';
+  name: string;
+  arguments?: JsonValue;
+} |
+/** A predicate or judge checked at this point of the conversation. */
+{
+  kind: 'checkpoint';
+  scorer: EvalCatalogScorer;
+};
+type EvalCatalogEntry = {
+  id: string;
+  path: string;
+  name?: string;
+  suiteId?: string;
+  agent: EvalCatalogAgent;
+  /** The separate agent that runs judge rules, if any. */
+  judgeAgent?: EvalCatalogAgent;
+  datasets: EvalCatalogDataset[];
+  trialCount: number;
+  policy: {
+    trials: number;
+    timeoutMs?: number;
+    failfast?: boolean;
+  };
+  transcript: EvalCatalogStep[];
+  scorers: EvalCatalogScorer[];
+  metadata?: JsonValue;
+  /** `file:line` of its `defineEval` call, relative to the project. */
+  definedAt?: string;
+};
+//#endregion
+//#region ../core/src/matrix.d.ts
+type MatrixSelection = {
+  /** Base eval IDs. Unknown values fail before dispatch. */
+  evals?: readonly string[];
+  /** Select existing dimension values, rather than overwriting every cell with the same value. */
+  parameters?: Readonly<Record<string, readonly JsonValue[]>>;
+  /** Overrides for non-dimension execution settings (e.g. maxTokens). */
+  overrides?: JsonObject;
+};
+type EvalMatrixCell = {
+  matrixId: string;
+  /** Stable canonical key including matrix, base eval and effective parameters. */
+  key: string;
+  eval: EvalDefinition;
+  parameters: JsonObject;
+};
+type EvalMatrixDefinition = {
+  id: string;
+  name?: string;
+  evals: readonly EvalDefinition[];
+  parameters: Readonly<Record<string, readonly JsonValue[]>>;
+  defaults?: JsonObject;
+  /** Complete correlated assignments, crossed with independent parameters. */
+  cases?: readonly JsonObject[];
+  /** Partial patterns over declared dimensions; matching cells are never run. */
+  exclude?: readonly JsonObject[];
+};
+type EvalMatrix = EvalMatrixDefinition & {
+  kind: 'matrix';
+  count(selection?: MatrixSelection): number;
+  cells(selection?: MatrixSelection): IterableIterator<EvalMatrixCell>;
+};
+//#endregion
+//#region ../core/src/index.d.ts
+type JsonPrimitive = boolean | number | string | null;
+type JsonValue = JsonPrimitive | JsonValue[] | {
+  [key: string]: JsonValue;
+};
+type JsonObject = {
+  [key: string]: JsonValue;
+};
+type AutIdentity = {
+  /** Human-readable logical agent name shown in catalogs and dashboards. */
+  name?: string;
+  kind: string;
+  id: string;
+  version?: string;
+};
+type AgentRuntimeName = 'local' | 'sandbox' | 'remote';
+/**
+ * A serializable runtime requirement for an AUT. Concrete runtime packages own
+ * the interpretation of `kind` and `configuration`.
+ */
+type AgentRuntime = {
+  kind: string;
+  configuration?: JsonObject;
+};
+type AgentRuntimes = Partial<Record<AgentRuntimeName, AgentRuntime>>;
+type Usage = {
+  inputTokens?: number;
+  outputTokens?: number;
+  totalTokens?: number;
+};
+type RecordedError = {
+  name: string;
+  message: string;
+  stack?: string;
+};
+type AutEvent = {
+  kind: 'started';
+  timestamp: string;
+} | {
+  kind: 'message';
+  role: 'system' | 'user' | 'assistant' | 'tool';
+  content: JsonValue;
+  timestamp: string;
+} | {
+  kind: 'tool-call';
+  id: string;
+  name: string;
+  arguments: JsonValue;
+  timestamp: string;
+} | {
+  kind: 'tool-result';
+  id: string;
+  name?: string;
+  result: JsonValue;
+  timestamp: string;
+} | {
+  kind: 'turn-started';
+  turn: number;
+  timestamp: string;
+} | {
+  kind: 'turn-completed';
+  turn: number;
+  timestamp: string;
+  latencyMs?: number;
+  usage?: Usage;
+} | {
+  kind: 'completed';
+  output?: JsonValue;
+  timestamp: string;
+} | {
+  kind: 'error';
+  error: RecordedError;
+  timestamp: string;
+};
+type RunnerEvent = {
+  kind: 'trial-started';
+  timestamp: string;
+} | {
+  kind: 'transcript-step-started';
+  step: number;
+  timestamp: string;
+} | {
+  kind: 'transcript-step-completed';
+  step: number;
+  timestamp: string;
+} | {
+  kind: 'transcript-step-skipped';
+  step: number;
+  timestamp: string;
+} | {
+  kind: 'checkpoint-started';
+  step: number;
+  name: string;
+  timestamp: string;
+} | {
+  kind: 'checkpoint-completed';
+  step: number;
+  status: 'passed' | 'failed' | 'skipped';
+  timestamp: string;
+} | {
+  kind: 'checkpoint-error';
+  step: number;
+  error: RecordedError;
+  timestamp: string;
+} | {
+  kind: 'scorer-started';
+  scorer: string;
+  timestamp: string;
+} | {
+  kind: 'scorer-completed';
+  scorer: string;
+  /** Absent when the scorer returned `skip()`. */
+  value?: number;
+  skipped?: true;
+  timestamp: string;
+} | {
+  kind: 'scorer-failed';
+  scorer: string;
+  error: RecordedError;
+  timestamp: string;
+} | {
+  kind: 'trial-completed';
+  timestamp: string;
+} | {
+  kind: 'error';
+  error: RecordedError;
+  timestamp: string;
+};
+type TrajectoryEvent = (AutEvent & {
+  source: 'aut';
+}) | (RunnerEvent & {
+  source: 'runner';
+});
+type WorkspaceView = {
+  /** Absolute path to this trial-scoped workspace. Do not persist it in reports. */
+  root: string;
+};
+type CandidateWorkspace = WorkspaceView;
+type EvaluatorWorkspace = WorkspaceView;
+type AutContext = {
+  runId: string;
+  evalId: string;
+  trialId: string;
+  trialIndex: number;
+  metadata: JsonObject;
+  /** Run-time parameters supplied by the eval definition or CLI invocation. */
+  parameters?: JsonObject;
+  runtime?: AgentRuntimeName;
+  /** Values of the scenario's candidate datasets, by dataset ID. */
+  data: JsonObject;
+  /** Contains only resources visible to the AUT/model. */
+  workspace: CandidateWorkspace;
+  /** Private evaluator capability for trusted AUT adapters; never expose this to the model. */
+  evaluatorWorkspace: EvaluatorWorkspace;
+};
+type AutAdapter<TMessage = string> = {
+  readonly identity?: AutIdentity;
+  /** Available execution environments for this AUT. Default: `{ local: { kind: 'in-process' } }`. */
+  readonly runtimes?: AgentRuntimes;
+  start(options: {
+    context: AutContext;
+    runtime?: AgentRuntime;
+    /** Optional: stream events as they happen. Returning a turn result from `send` is enough for most agents. */
+    onEvent: (event: AutEventInput) => void | Promise<void>;
+  }): Promise<AutSession<TMessage>>;
+};
+type AutSession<TMessage = string> = {
+  /**
+   * Resolves once the AUT has completed the turn initiated by this message.
+   * It may return the turn's reply and tool calls instead of emitting events.
+   */
+  send(message: TMessage): Promise<void | TurnResult>;
+  /** Optional; defaults to doing nothing. */
+  close?(): Promise<void>;
+};
+type UserStep = {
+  kind: 'user';
+  message: string;
+};
+type AgentStep = {
+  kind: 'agent';
+  expectation: JsonObject;
+};
+type TurnView = {
+  userStepIndex: number;
+  events: readonly (AutEvent & {
+    source: 'aut';
+  })[];
+  assistantMessages: readonly (Extract<AutEvent, {
+    kind: 'message';
+  }> & {
+    role: 'assistant';
+    source: 'aut';
+  })[];
+  toolCalls: readonly {
+    eventIndex: number;
+    id: string;
+    name: string;
+    arguments: JsonValue;
+    resultObservation: 'observed' | 'absent' | 'ambiguous';
+    result?: JsonValue;
+  }[];
+  lastAssistantText?: string;
+};
+type ExpectToolCallStep = {
+  kind: 'expect-tool-call';
+  name: string;
+  expected: {
+    name: string;
+    arguments?: JsonValue;
+  };
+};
+type TranscriptStep = UserStep | AgentStep | ScoringRule | ExpectToolCallStep;
+type JudgeRunInfo = {
+  agent?: AutIdentity;
+  usage?: Usage;
+  /** Judge-agent observations; never attributed to the AUT trajectory. */
+  events?: JsonValue[];
+};
+/** A scorer's "does not apply" result; see `skip()`. */
+type SkippedScore = {
+  skipped: true;
+  reason: string;
+};
+type ScoreValue = number | SkippedScore | {
+  value: number;
+  passed?: boolean;
+  explanation?: string;
+  /** Any JSON-convertible value. */
+  evidence?: unknown;
+  /** Judge-agent evidence, never inferred from AUT usage. */
+  judge?: JudgeRunInfo;
+};
+type ScoreResult = {
+  name: string;
+  kind: 'predicate' | 'judge';
+  value?: number;
+  passed?: boolean;
+  explanation?: string;
+  evidence?: JsonValue;
+  judge?: JudgeRunInfo;
+  durationMs: number;
+  error?: RecordedError;
+  /** The scorer returned `skip()`; `explanation` holds the reason and there is no value. */
+  skipped?: true;
+};
+type ArtifactView = {
+  candidate: CandidateWorkspace;
+  evaluator: EvaluatorWorkspace;
+};
+type ScoringContext = {
+  context: AutContext;
+  trajectory: {
+    events: readonly TrajectoryEvent[];
+  };
+  artifacts: ArtifactView;
+  /** Most recent completed turn; absent if no user message has been sent. */
+  turn?: TurnView;
+  /** Content of the last assistant message of `turn`. */
+  output?: JsonValue;
+  /** Values of all the scenario's datasets (candidate and evaluator), by dataset ID. */
+  data: JsonObject;
+};
+type PredicateScorer = {
+  kind: 'predicate';
+  name: string;
+  supportsPartial?: boolean;
+  /** A numeric result passes when it is at least this value. Default: only 1 passes. */
+  passAt?: number;
+  run(context: ScoringContext): boolean | ScoreValue | Promise<boolean | ScoreValue>;
+};
+type JudgeRule = {
+  kind: 'judge';
+  name: string;
+  rubric: string;
+  /** Run even when the scenario stopped early or execution failed. */
+  supportsPartial?: boolean;
+};
+type ScoringRule = PredicateScorer | JudgeRule;
+type EvalPolicy = {
+  timeoutMs?: number;
+  /** Stop later transcript steps after a failed checkpoint assertion. Default false. */
+  failfast?: boolean;
+  /** Number of independent trials requested by local and hosted runners. */
+  trials?: number;
+};
+type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
+  id: string;
+  name?: string;
+  agent: TAgent;
+  /** Separate judge agent; its model and tools are never inherited from the AUT. */
+  judge?: AutAdapter;
+  /** Data and files the scenario uses; each trial gets its own copy. */
+  datasets?: readonly Dataset[];
+  transcript: TranscriptStep[];
+  scoring: ScoringRule[];
+  policy?: EvalPolicy;
+  /** Any JSON-convertible object; converted when the eval is defined. */
+  metadata?: Record<string, unknown>;
+};
+type EvalRegistryMetadata = {
+  id: string;
+  name?: string;
+  suiteId?: string;
+};
+type EvalSuiteMetadata = {
+  id: string;
+  name?: string;
+  evalIds: string[];
+};
+type RunStatus = 'running' | 'completed' | 'failed' | 'cancelled';
+/**
+ * Where a run's eval code came from: the repository, eval project and commit.
+ * Results are tagged with it; they are never stored in the repository itself.
+ */
+type RunSource = {
+  /** Normalized repository key, e.g. `gitlab.example.com/group/repo`. */
+  repo: string;
+  /** Eval project directory relative to the repository root; `""` at the root. */
+  project: string;
+  /** Full commit SHA. */
+  commit: string;
+  ref?: string;
+  committedAt?: string;
+  /** True when the working tree had uncommitted changes at run time. */
+  dirty: boolean;
+  ci?: {
+    provider: string;
+    pipeline?: string;
+    job?: string;
+    url?: string;
+  };
+};
+/** The project's evals and suites when a run started; written beside its manifest. */
+type RunCatalog = {
+  catalog: EvalCatalogEntry[];
+  evals: EvalRegistryMetadata[];
+  suites: EvalSuiteMetadata[];
+};
+/** The process that owns a running run, so readers can tell an interrupted run from a live one. */
+type RunProcess = {
+  pid: number;
+  hostname: string;
+};
+type RunMetadata = {
+  schemaVersion: 2 | 3 | 4 | 5;
+  runId: string;
+  runUri: ResourceUri<'run'>;
+  evalId: string;
+  suiteId?: string;
+  parameters?: JsonObject;
+  matrix?: {
+    id: string;
+    cellKey: string;
+  };
+  aut?: AutIdentity;
+  startedAt: string;
+  /** Present from schema version 4 when the run's commit could be determined. */
+  source?: RunSource;
+  /** From schema version 5. */
+  process?: RunProcess;
+};
+type TrialMetadata = {
+  schemaVersion: 2 | 3 | 4 | 5;
+  runId: string;
+  runUri: ResourceUri<'run'>;
+  trialId: string;
+  trialUri: ResourceUri<'trial'>;
+  trialIndex: number;
+  evalId: string;
+  parameters?: JsonObject;
+  matrix?: {
+    id: string;
+    cellKey: string;
+  };
+  aut?: AutIdentity;
+  startedAt: string;
+  /** From schema version 5: the datasets this trial used, with content digests. */
+  datasets?: TrialDataset[];
+};
+type TrialDataset = {
+  id: string;
+  visibility: DatasetVisibility;
+  source: 'static' | 'computed';
+  /** SHA-256 of the trial's content: file paths and bytes, and the canonical value. */
+  digest: string;
+};
+type CheckpointResult = {
+  step: number;
+  kind: 'predicate' | 'judge' | 'expect-tool-call';
+  name: string;
+  status: 'passed' | 'failed' | 'error' | 'skipped';
+  value?: number;
+  passed?: boolean;
+  explanation?: string;
+  evidence?: JsonValue;
+  judge?: JudgeRunInfo;
+  durationMs?: number;
+  error?: RecordedError;
+  matchedToolCall?: {
+    eventIndex: number;
+    id: string;
+  };
+};
+type TrialScoring = {
+  results: ScoreResult[];
+  checkpoints?: CheckpointResult[];
+  skippedScorers?: string[];
+  overall?: number;
+  passed: boolean;
+};
+type ArtifactEntry = {
+  /** Path relative to this trial's artifact root. */
+  path: string;
+  kind: 'file' | 'directory';
+  size?: number;
+};
+type TrialSummary = {
+  status: RunStatus;
+  endedAt: string;
+  durationMs?: number;
+  scoring?: TrialScoring;
+  artifacts?: ArtifactEntry[];
+  error?: RecordedError;
+};
+type RunSummary = {
+  status: RunStatus;
+  endedAt: string;
+  durationMs?: number;
+  trialCount: number;
+  passed: number;
+  failed: number;
+  /** Trials stopped by cancellation; counted neither as passed nor failed. */
+  cancelled?: number;
+  error?: RecordedError;
+};
+type TrialWriter = {
+  appendEvent(event: TrajectoryEvent): Promise<void>;
+  writeScores(scoring: TrialScoring): Promise<void>;
+  /** Writes a file below this trial's artifact root. */
+  writeArtifact(path: string, data: Uint8Array): Promise<void>;
+  finalize(summary: TrialSummary): Promise<void>;
+};
+type RunWriter = {
+  readonly location: string;
+  startTrial(metadata: TrialMetadata): Promise<TrialWriter>;
+  finalize(summary: RunSummary): Promise<void>;
+};
+type ReportStore = {
+  startRun(metadata: RunMetadata, context?: {
+    catalog?: RunCatalog;
+  }): Promise<RunWriter>;
+};
+type TrialResult = {
+  runId: string;
+  runUri: ResourceUri<'run'>;
+  trialId: string;
+  trialUri: ResourceUri<'trial'>;
+  trialIndex: number;
+  evalId: string;
+  status: RunStatus;
+  reportLocation: string;
+  durationMs?: number;
+  scoring?: TrialScoring;
+  error?: RecordedError;
+};
+type AggregateScoring = {
+  passed: number;
+  failed: number;
+  cancelled?: number;
+  passRate: number;
+  overall?: number;
+};
+type RunResult = TrialResult & {
+  runId: string;
+  trialCount?: number;
+  passed?: number;
+  failed?: number;
+  aggregateScoring?: AggregateScoring;
+  /** Present for an aggregate multi-trial run. */
+  trials?: TrialResult[];
+};
 //#endregion
 //#region ../runner/src/local-report-store.d.ts
 /** Persists each run as a local tree rooted below `rootDirectory`. */
 export declare function localReportStore(rootDirectory: string): ReportStore;
 //#endregion
+//#region ../runner/src/datasets.d.ts
+/** A file a static dataset places below its mount, for browsing. */
+type DatasetFileEntry = {
+  path: string;
+  size: number;
+};
+/**
+ * The files a static dataset places in every workspace, relative to its
+ * mount. Computed datasets return `undefined`: their files only exist once
+ * `onLoad` runs, and browsing must never run project code.
+ */
+export declare function staticDatasetFiles(dataset: Dataset, root: string): Promise<DatasetFileEntry[] | undefined>;
+/** One file of a static dataset, by its path below the mount. */
+export declare function staticDatasetFile(dataset: Dataset, root: string, file: string): Promise<Uint8Array<ArrayBuffer>>;
+//#endregion
 //#region ../runner/src/local-report-reader.d.ts
 export declare const readRunManifest: (root: string, runId: string) => Promise<{
-  readonly schemaVersion: 2 | 3 | 4;
+  readonly schemaVersion: 2 | 3 | 4 | 5;
   readonly runUri: string;
   readonly evalId: string;
   readonly suiteId?: string | undefined;
@@ -77,6 +828,10 @@ export declare const readRunManifest: (root: string, runId: string) => Promise<{
       readonly url?: string | undefined;
     } | undefined;
   } | undefined;
+  readonly process?: {
+    readonly pid: number;
+    readonly hostname: string;
+  } | undefined;
   readonly status?: "cancelled" | "completed" | "failed" | "running" | undefined;
 }>;
 export declare const readRunSummary: (root: string, runId: string) => Promise<{
@@ -86,6 +841,7 @@ export declare const readRunSummary: (root: string, runId: string) => Promise<{
   readonly trialCount: number;
   readonly passed: number;
   readonly failed: number;
+  readonly cancelled?: number | undefined;
   readonly error?: {
     readonly name: string;
     readonly message: string;
@@ -93,7 +849,7 @@ export declare const readRunSummary: (root: string, runId: string) => Promise<{
   } | undefined;
 }>;
 export declare const readTrialManifest: (root: string, runId: string, trialId: string) => Promise<{
-  readonly schemaVersion: 2 | 3 | 4;
+  readonly schemaVersion: 2 | 3 | 4 | 5;
   readonly runUri: string;
   readonly trialUri: string;
   readonly trialIndex: number;
@@ -112,6 +868,12 @@ export declare const readTrialManifest: (root: string, runId: string, trialId: s
     readonly version?: string | undefined;
   } | undefined;
   readonly startedAt: string;
+  readonly datasets?: {
+    readonly id: string;
+    readonly visibility: "candidate" | "evaluator";
+    readonly source: "computed" | "static";
+    readonly digest: string;
+  }[] | undefined;
   readonly status?: "cancelled" | "completed" | "failed" | "running" | undefined;
 }>;
 export declare const readTrialSummary: (root: string, runId: string, trialId: string) => Promise<{
@@ -146,6 +908,7 @@ export declare const readTrialSummary: (root: string, runId: string, trialId: st
         readonly message: string;
         readonly stack?: string | undefined;
       } | undefined;
+      readonly skipped?: true | undefined;
     }[];
     readonly checkpoints?: {
       readonly step: number;
@@ -224,6 +987,7 @@ export declare const readTrialScoring: (root: string, runId: string, trialId: st
       readonly message: string;
       readonly stack?: string | undefined;
     } | undefined;
+    readonly skipped?: true | undefined;
   }[];
   readonly checkpoints?: {
     readonly step: number;
@@ -264,17 +1028,27 @@ export declare const readTrialScoring: (root: string, runId: string, trialId: st
   readonly passed: boolean;
 }>;
 export declare function readTrialEvents(root: string, runId: string, trialId: string): Promise<TrajectoryEvent[]>;
+/**
+ * True when a run's owning process is known to be gone: it ran on this host
+ * and its PID no longer exists. Runs from other hosts are never judged.
+ */
+export declare function ownerIsGone(owner: RunProcess | undefined): boolean;
 //#endregion
 //#region ../runner/src/matrix.d.ts
+/** Cells run; cancelled cells are neither passed nor failed. */
+type MatrixSummary = {
+  cells: number;
+  passed: number;
+  failed: number;
+  cancelled: number;
+};
 /** Bounded workers pull cells from the iterator; no Cartesian job array is allocated. */
 export declare function runMatrix(matrix: EvalMatrix, options: Omit<RunEvalOptions, 'runId' | 'trialId' | 'parameters' | 'matrix'> & {
   selection?: MatrixSelection;
+  /** The suite each cell's run is recorded under, when `suiteId` is not given. */
+  suiteFor?: (cell: EvalMatrixCell) => string | undefined;
   onResult?: (cell: EvalMatrixCell, result: RunResult) => void | Promise<void>;
-}): Effect.Effect<{
-  cells: number;
-  passed: number;
-  failed: number;
-}, unknown>;
+}): Effect.Effect<MatrixSummary, unknown>;
 //#endregion
 //#region ../runner/src/index.d.ts
 export type RunEvalOptions = {
@@ -300,9 +1074,31 @@ export type RunEvalOptions = {
   source?: RunSource;
   /** The project's evals and suites at run time, saved beside the manifest. */
   catalog?: RunCatalog;
+  /** Project root; dataset paths resolve against it. Default: the working directory. */
+  root?: string;
+  /**
+   * Aborting stops new trials from starting; running trials stop at their next
+   * step and are finalized as cancelled. Also passed to dataset loaders.
+   */
+  signal?: AbortSignal;
+  /**
+   * Observes every trajectory event after it is recorded, for live progress.
+   * It must not throw; an error it throws is ignored.
+   */
+  onEvent?: (trial: TrialRef, event: TrajectoryEvent) => void;
   now?: () => Date;
 };
+/** Identifies the trial an observed event belongs to. */
+export type TrialRef = {
+  runId: string;
+  trialId: string;
+  trialIndex: number;
+  evalId: string;
+  suiteId?: string;
+  parameters?: JsonObject;
+};
 /** Effect-native aggregate eval execution entry point. */
 export declare function runEval(definition: EvalDefinition, options: RunEvalOptions): Effect.Effect<RunResult, unknown>;
 //#endregion
+export type { DatasetFileEntry, MatrixSummary };
 //# sourceMappingURL=runner.d.mts.map