@leostera/evalkit 0.5.0 → 0.6.1

package/dist/index.d.mts

@@ -1,4 +1,1348 @@
-import { $ as TrajectoryEvent, $t as canonicalParameters, A as FixtureContext, At as JsonValueSchema, B as ReportStore, Bt as TrialManifestSchema, C as EvalRegistryMetadata, Ct as registerEvals, D as ExpectToolCallStep, Dt as ArtifactEntrySchema, E as EvaluatorWorkspace, Et as ApiErrorSchema, F as JsonValue, Ft as RunSourceSchema, G as RunStatus, Gt as EvalkitConfig, H as RunMetadata, Ht as TrialScoringSchema, I as JudgeRule, It as RunStatusSchema, J as RunnerEvent, Jt as defineConfig, K as RunSummary, Kt as ShareConfig, L as JudgeRunInfo, Lt as RunSummarySchema, M as InlineFixture, Mt as RecordedErrorSchema, N as JsonObject, Nt as RunManifestSchema, O as FileFixture, Ot as AutIdentitySchema, P as JsonPrimitive, Pt as RunMetadataSchema, Q as ScoringRule, Qt as defineEvalMatrix, R as PredicateScorer, Rt as ScoreResultSchema, S as EvalRegistry, St as recordError, T as EvalSuiteMetadata, Tt as parseTrajectoryJsonl, U as RunResult, Ut as TrialSummarySchema, V as RunCatalog, Vt as TrialMetadataSchema, W as RunSource, Wt as UsageSchema, X as ScoreValue, Xt as EvalMatrixCell, Y as ScoreResult, Yt as EvalMatrix, Z as ScoringContext, Zt as MatrixSelection, _ as EvalCatalogEntry, _t as expectToolCall, a as AggregateScoring, an as UuidSchema, at as TrialWriter, b as EvalPolicy, bt as judge, c as AutAdapter, cn as resourceUri, ct as UserStep, d as AutIdentity, dt as authoringId, en as ResourceKind, et as TranscriptStep, f as AutSession, ft as defineAgent, g as DynamicFixture, gt as dynamic, h as DirectoryFixture, ht as directory, i as AgentStep, in as Uuid, it as TrialSummary, j as FixtureVisibility, jt as JudgeRunInfoSchema, k as Fixture, kt as CheckpointResultSchema, l as AutContext, ln as resourceUriSchema, lt as WorkspaceView, m as CheckpointResult, mt as defineSuite, n as AgentRuntimeName, nn as ResourceUri, nt as TrialResult, o as ArtifactEntry, on as authorId, ot as TurnView, p as CandidateWorkspace, pt as defineEval, q as RunWriter, qt as ShareHeaders, r as AgentRuntimes, rn as ResourceUriSchema, rt as TrialScoring, s as ArtifactView, sn as parseResourceUri, st as Usage, t as AgentRuntime, tn as ResourceKindSchema, tt as TrialMetadata, u as AutEvent, un as resourceUuid, ut as agent, v as EvalCatalogFixture, vt as file, w as EvalSuite, wt as user, x as EvalRegistration, xt as predicate, y as EvalDefinition, yt as inlineFile, z as RecordedError, zt as TrajectoryEventSchema } from "./index-lOw0fmak.mjs";
+import * as Schema from "effect/Schema";
+//#region ../core/src/identity.d.ts
+export declare const ResourceKindSchema: Schema.Literal<["run", "trial", "artifact"]>;
+export type ResourceKind = typeof ResourceKindSchema.Type;
+/** Human-authored project-local identifier; never a generated resource URI. */
+export declare function authorId(value: string): string;
+/** RFC 4122 textual UUID. EvalKit accepts v4/v7 and other valid UUID versions. */
+export declare const UuidSchema: Schema.filter<typeof Schema.String>;
+export type Uuid = typeof UuidSchema.Type;
+export type ResourceUri<TKind extends ResourceKind = ResourceKind> = `evalkit:${TKind}:${string}`;
+export declare const ResourceUriSchema: Schema.filter<typeof Schema.String>;
+export declare function resourceUriSchema<TKind extends ResourceKind>(kind: TKind): Schema.filter<typeof Schema.String>;
+export declare function resourceUri<TKind extends ResourceKind>(kind: TKind, uuid: Uuid): ResourceUri<TKind>;
+export declare function parseResourceUri<TKind extends ResourceKind>(value: string, expectedKind?: TKind): {
+  kind: TKind;
+  uuid: Uuid;
+  uri: ResourceUri<TKind>;
+};
+export declare function resourceUuid(value: string, expectedKind?: ResourceKind): Uuid;
+//#endregion
+//#region ../core/src/dataset.d.ts
+/** Who can see a dataset: the agent (and scorers), or only scorers. */
+export type DatasetVisibility = 'candidate' | 'evaluator';
+/** Inline text or bytes, or a file relative to the project root. */
+export type DatasetFileSource = string | Uint8Array | {
+  file: string;
+};
+/**
+ * Files for the trial workspace: a directory relative to the project root, or
+ * a map from workspace path (below the dataset's mount) to its content.
+ */
+export type DatasetFiles = string | Readonly<Record<string, DatasetFileSource>>;
+/** What one trial receives from a dataset. */
+export type DatasetContent = {
+  files?: DatasetFiles;
+  /** Any JSON-convertible value; converted once when the trial starts. */
+  value?: unknown;
+};
+export type DatasetLoadContext = {
+  /** The project root; relative paths in returned content resolve against it. */
+  root: string;
+  signal: AbortSignal;
+};
+export type DatasetTrialContext = {
+  evalId: string;
+  runId: string;
+  trialId: string;
+  trialIndex: number;
+  /** Per-trial random seed, so repeated trials can choose different data. */
+  seed: number;
+  /** The matrix cell's parameters, or `{}`. */
+  parameters: JsonObject;
+  signal: AbortSignal;
+};
+type DatasetCommon = {
+  id: string;
+  name?: string;
+  /** Default `candidate`. */
+  visibility?: DatasetVisibility;
+  /** Workspace directory for the dataset's files. Default: the dataset ID. `.` is the workspace root. */
+  mount?: string;
+};
+export type StaticDatasetDefinition = DatasetCommon & {
+  files?: DatasetFiles;
+  /** An inline value. Exclusive with `valueFile`. */
+  value?: unknown;
+  /** A `.json` file, or a `.jsonl` file read as an array of objects, relative to the project root. */
+  valueFile?: string;
+  onLoad?: never;
+  onRun?: never;
+};
+export type ComputedDatasetDefinition<TLoaded extends DatasetContent> = DatasetCommon & {
+  /** Once per process; the successful result is cached and shared by every trial. */
+  onLoad: (load: DatasetLoadContext) => TLoaded | Promise<TLoaded>;
+  /** Once per trial; defaults to using the loaded content as is. */
+  onRun?: (loaded: TLoaded, trial: DatasetTrialContext) => DatasetContent | Promise<DatasetContent>;
+  files?: never;
+  value?: never;
+  valueFile?: never;
+};
+export type DatasetSource = {
+  kind: 'static';
+  files?: DatasetFiles;
+  value?: unknown;
+  valueFile?: string;
+} | {
+  kind: 'computed';
+  onLoad: (load: DatasetLoadContext) => DatasetContent | Promise<DatasetContent>;
+  onRun?: (loaded: DatasetContent, trial: DatasetTrialContext) => DatasetContent | Promise<DatasetContent>;
+};
+/** A checked dataset, as returned by `defineDataset`. Evals accept only these. */
+export type Dataset = {
+  readonly kind: 'dataset';
+  readonly id: string;
+  readonly name?: string;
+  readonly visibility: DatasetVisibility;
+  /** Normalized relative POSIX path; `.` is the workspace root. */
+  readonly mount: string;
+  readonly source: DatasetSource;
+};
+/** Normalizes a relative POSIX path and rejects absolute or escaping ones. */
+export declare function relativeDatasetPath(value: string, label: string): string;
+/**
+ * Declares named data that scenarios use: files placed in each trial's
+ * workspace, a value for the agent and scorers, or both. Static datasets give
+ * `files`, `value` or `valueFile`; computed datasets give `onLoad` (cached per
+ * process) and optionally `onRun` (per trial).
+ */
+export declare function defineDataset<TLoaded extends DatasetContent>(definition: StaticDatasetDefinition | ComputedDatasetDefinition<TLoaded>): Dataset;
+export declare function isDataset(value: unknown): value is Dataset;
+//#endregion
+//#region ../core/src/aut-events.d.ts
+/**
+ * Events as adapters emit them. Payloads accept any JSON-convertible value and
+ * timestamps are optional; EvalKit converts them once into `AutEvent`s.
+ */
+export type AutEventInput = {
+  kind: 'started';
+  timestamp?: string;
+} | {
+  kind: 'message';
+  role: 'system' | 'user' | 'assistant' | 'tool';
+  content: unknown;
+  timestamp?: string;
+} | {
+  kind: 'tool-call';
+  id: string;
+  name: string;
+  arguments?: unknown;
+  timestamp?: string;
+} | {
+  kind: 'tool-result';
+  id: string;
+  name?: string;
+  result?: unknown;
+  timestamp?: string;
+} | {
+  kind: 'turn-started';
+  turn: number;
+  timestamp?: string;
+} | {
+  kind: 'turn-completed';
+  turn: number;
+  timestamp?: string;
+  latencyMs?: number;
+  usage?: Usage;
+} | {
+  kind: 'completed';
+  output?: unknown;
+  timestamp?: string;
+} | {
+  kind: 'error';
+  error: RecordedError;
+  timestamp?: string;
+};
+export type TurnToolCall = {
+  name: string;
+  arguments?: unknown;
+  /** When given, a `tool-result` event is recorded after the call. */
+  result?: unknown;
+  id?: string;
+};
+/** What `send()` may return instead of (or as well as) emitting events. A string is the reply. */
+export type TurnResult = string | {
+  reply?: unknown;
+  toolCalls?: readonly TurnToolCall[];
+  usage?: Usage;
+};
+/** Converts an adapter event into a recorded `AutEvent`. */
+export declare function normalizeAutEvent(input: AutEventInput, now: () => Date): AutEvent;
+/**
+ * The events a returned turn result stands for: each tool call (and its
+ * result), then the reply, then the usage as a `turn-completed` event.
+ */
+export declare function turnResultEvents(result: TurnResult, turn: number, now: () => Date): AutEvent[];
+//#endregion
+//#region ../core/src/catalog.d.ts
+/**
+ * What the dashboard's Library shows about each definition. Only definition
+ * data: names, kinds, settings, shapes and where it was defined. Never source
+ * code, and never dataset values (evaluator data must not reach the agent's
+ * side or a shared report), only their shape.
+ */
+export type CatalogOptions = {
+  /** Project root; definition sites are reported relative to it, never absolute. */
+  root?: string;
+};
+export type EvalCatalogAgent = {
+  name?: string;
+  kind: string;
+  id?: string;
+  version?: string;
+  runtimes: Array<{
+    name: AgentRuntimeName;
+    kind: string;
+  }>;
+  /** `file:line` of its `defineAgent` call, relative to the project. */
+  definedAt?: string;
+};
+/** A value's shape, without its content. */
+export type ValueShape = {
+  type: 'array';
+  items: number;
+} | {
+  type: 'object';
+  keys: string[];
+} | {
+  type: 'string' | 'number' | 'boolean' | 'null';
+};
+export type EvalCatalogDataset = {
+  id: string;
+  name?: string;
+  visibility: DatasetVisibility;
+  source: 'static' | 'computed';
+  mount: string;
+  /** Static datasets: a directory relative to the project, or the workspace paths of inline files. */
+  files?: {
+    directory: string;
+  } | {
+    paths: string[];
+  };
+  /** Static datasets: the inline value's shape, or the value file's path. */
+  value?: {
+    shape: ValueShape;
+  } | {
+    file: string;
+  };
+  /** Computed datasets: which hooks are defined. */
+  hooks?: {
+    onLoad: true;
+    onRun: boolean;
+  };
+  definedAt?: string;
+};
+export type EvalCatalogScorer = {
+  name: string;
+  kind: ScoringRule['kind'];
+  passAt?: number;
+  supportsPartial?: boolean;
+  /** Judges only. */
+  rubric?: string;
+  definedAt?: string;
+};
+export type EvalCatalogStep = {
+  kind: 'user';
+  message: string;
+} | {
+  kind: 'agent';
+  expectation: JsonValue;
+} | {
+  kind: 'expect-tool-call';
+  name: string;
+  arguments?: JsonValue;
+} |
+/** A predicate or judge checked at this point of the conversation. */
+{
+  kind: 'checkpoint';
+  scorer: EvalCatalogScorer;
+};
+export type EvalCatalogEntry = {
+  id: string;
+  path: string;
+  name?: string;
+  suiteId?: string;
+  agent: EvalCatalogAgent;
+  /** The separate agent that runs judge rules, if any. */
+  judgeAgent?: EvalCatalogAgent;
+  datasets: EvalCatalogDataset[];
+  trialCount: number;
+  policy: {
+    trials: number;
+    timeoutMs?: number;
+    failfast?: boolean;
+  };
+  transcript: EvalCatalogStep[];
+  scorers: EvalCatalogScorer[];
+  metadata?: JsonValue;
+  /** `file:line` of its `defineEval` call, relative to the project. */
+  definedAt?: string;
+};
+export declare function catalogEntry(evaluation: EvalDefinition, suite: EvalSuite | undefined, options?: CatalogOptions): EvalCatalogEntry;
+//#endregion
+//#region ../core/src/canonical-json.d.ts
+/** Canonical JSON: keys sorted recursively, array order preserved. Rejects non-JSON inputs. */
+export declare function canonicalParameters(value: JsonValue): string;
+//#endregion
+//#region ../core/src/matrix.d.ts
+type MatrixSelection = {
+  /** Base eval IDs. Unknown values fail before dispatch. */
+  evals?: readonly string[];
+  /** Select existing dimension values, rather than overwriting every cell with the same value. */
+  parameters?: Readonly<Record<string, readonly JsonValue[]>>;
+  /** Overrides for non-dimension execution settings (e.g. maxTokens). */
+  overrides?: JsonObject;
+};
+type EvalMatrixCell = {
+  matrixId: string;
+  /** Stable canonical key including matrix, base eval and effective parameters. */
+  key: string;
+  eval: EvalDefinition;
+  parameters: JsonObject;
+};
+type EvalMatrixDefinition = {
+  id: string;
+  name?: string;
+  evals: readonly EvalDefinition[];
+  parameters: Readonly<Record<string, readonly JsonValue[]>>;
+  defaults?: JsonObject;
+  /** Complete correlated assignments, crossed with independent parameters. */
+  cases?: readonly JsonObject[];
+  /** Partial patterns over declared dimensions; matching cells are never run. */
+  exclude?: readonly JsonObject[];
+};
+type EvalMatrix = EvalMatrixDefinition & {
+  kind: 'matrix';
+  count(selection?: MatrixSelection): number;
+  cells(selection?: MatrixSelection): IterableIterator<EvalMatrixCell>;
+};
+export declare function defineEvalMatrix<const T extends EvalMatrixDefinition>(definition: T): EvalMatrix & T;
+//#endregion
+//#region ../core/src/config.d.ts
+type EvalkitConfig = {
+  /** Where scenarios are discovered. Default `scenarios`. Paths are relative to evalkit.config.js/ts. */
+  testDir?: string;
+  include?: readonly string[];
+  exclude?: readonly string[];
+  discovery?: {
+    /** Each directory directly below `testDir` is a suite. Default true. */
+    suites?: boolean;
+  };
+  /** Explicit definitions are an alternative to discovery (e.g. hosted registries). */
+  evals?: readonly EvalDefinition[];
+  registry?: EvalRegistry;
+  /** Applied to every discovered eval. Values are forwarded, never provider-interpreted. */
+  matrix?: Omit<EvalMatrixDefinition, 'evals' | 'id'> & {
+    id?: string;
+  };
+  execution?: {
+    concurrency?: number;
+    trials?: number;
+    maxCells?: number;
+  };
+  reportDir?: string;
+  sandboxDir?: string;
+  share?: ShareConfig;
+};
+/** HTTP header names and values. */
+type ShareHeaders = Record<string, string>;
+type ShareConfig = {
+  /** Where `evalkit share` uploads results. `EVALKIT_SHARE_URL` overrides it. */
+  url: string;
+  /**
+   * Extra headers for every request to the share server, e.g. credentials for
+   * an auth proxy in front of it. A function is called once per `evalkit share`
+   * (it may run a command to fetch a token). Values are never printed.
+   */
+  headers?: ShareHeaders | (() => ShareHeaders | Promise<ShareHeaders>);
+};
+/** Pure, typed project configuration. Loading/discovery belongs to the CLI. */
+export declare function defineConfig<const T extends EvalkitConfig>(configuration: T): T;
+//#endregion
+//#region ../core/src/json.d.ts
+/** A value that cannot be stored in a report, with the path of the offending part. */
+export declare class NotJsonError extends Error {
+  readonly path: string;
+  readonly reason: string;
+  readonly name = "NotJsonError";
+  constructor(path: string, reason: string);
+}
+/**
+ * Converts an author-supplied value to JSON once, where it enters EvalKit.
+ * Honours `toJSON()` (so `Date` becomes an ISO string) and drops `undefined`
+ * object fields. Rejects functions, symbols, bigints, non-finite numbers and
+ * cycles, naming the path of the bad value.
+ */
+export declare function toJson(value: unknown, path?: string): JsonValue;
+/** `toJson` for values that must be objects, such as metadata. */
+export declare function toJsonObject(value: unknown, path: string): {
+  [key: string]: JsonValue;
+};
+//#endregion
+//#region ../core/src/definition-site.d.ts
+/**
+ * Where an agent, dataset, judge or scenario was defined: the file and line of
+ * the call to its `define…` helper, read from the call stack the way test
+ * runners locate tests. Best effort: absent when the stack has no file frame
+ * (some bundlers, Workers). Kept off the object's own enumerable keys so it
+ * never changes equality, JSON or spreading.
+ */
+type DefinitionSite = {
+  file: string;
+  line: number;
+};
+export declare function definitionSite(value: unknown): DefinitionSite | undefined;
+//#endregion
+//#region ../core/src/schema.d.ts
+/** Runtime schemas for serialized EvalKit report and API boundaries. */
+export declare const RunStatusSchema: Schema.Literal<["running", "completed", "failed", "cancelled"]>;
+export declare const RecordedErrorSchema: Schema.Struct<{
+  name: typeof Schema.String;
+  message: typeof Schema.String;
+  stack: Schema.optional<typeof Schema.String>;
+}>;
+export declare const AutIdentitySchema: Schema.Struct<{
+  name: Schema.optional<typeof Schema.String>;
+  kind: typeof Schema.String;
+  id: typeof Schema.String;
+  version: Schema.optional<typeof Schema.String>;
+}>;
+export declare const JsonValueSchema: Schema.Schema<JsonValue>;
+export declare const RunSourceSchema: Schema.Struct<{
+  repo: typeof Schema.String;
+  project: typeof Schema.String;
+  commit: typeof Schema.String;
+  ref: Schema.optional<typeof Schema.String>;
+  committedAt: Schema.optional<typeof Schema.String>;
+  dirty: typeof Schema.Boolean;
+  ci: Schema.optional<Schema.Struct<{
+    provider: typeof Schema.String;
+    pipeline: Schema.optional<typeof Schema.String>;
+    job: Schema.optional<typeof Schema.String>;
+    url: Schema.optional<typeof Schema.String>;
+  }>>;
+}>;
+export declare const RunMetadataSchema: Schema.Struct<{
+  schemaVersion: Schema.Literal<[2, 3, 4, 5]>;
+  runUri: Schema.filter<typeof Schema.String>;
+  evalId: typeof Schema.String;
+  suiteId: Schema.optional<typeof Schema.String>;
+  parameters: Schema.optional<Schema.Record$<typeof Schema.String, Schema.Schema<JsonValue, JsonValue, never>>>;
+  matrix: Schema.optional<Schema.Struct<{
+    id: typeof Schema.String;
+    cellKey: typeof Schema.String;
+  }>>;
+  aut: Schema.optional<Schema.Struct<{
+    name: Schema.optional<typeof Schema.String>;
+    kind: typeof Schema.String;
+    id: typeof Schema.String;
+    version: Schema.optional<typeof Schema.String>;
+  }>>;
+  startedAt: typeof Schema.String;
+  source: Schema.optional<Schema.Struct<{
+    repo: typeof Schema.String;
+    project: typeof Schema.String;
+    commit: typeof Schema.String;
+    ref: Schema.optional<typeof Schema.String>;
+    committedAt: Schema.optional<typeof Schema.String>;
+    dirty: typeof Schema.Boolean;
+    ci: Schema.optional<Schema.Struct<{
+      provider: typeof Schema.String;
+      pipeline: Schema.optional<typeof Schema.String>;
+      job: Schema.optional<typeof Schema.String>;
+      url: Schema.optional<typeof Schema.String>;
+    }>>;
+  }>>;
+  process: Schema.optional<Schema.Struct<{
+    pid: typeof Schema.Number;
+    hostname: typeof Schema.String;
+  }>>;
+}>;
+export declare const TrialMetadataSchema: Schema.Struct<{
+  schemaVersion: Schema.Literal<[2, 3, 4, 5]>;
+  runUri: Schema.filter<typeof Schema.String>;
+  trialUri: Schema.filter<typeof Schema.String>;
+  trialIndex: typeof Schema.Number;
+  evalId: typeof Schema.String;
+  parameters: Schema.optional<Schema.Record$<typeof Schema.String, Schema.Schema<JsonValue, JsonValue, never>>>;
+  matrix: Schema.optional<Schema.Struct<{
+    id: typeof Schema.String;
+    cellKey: typeof Schema.String;
+  }>>;
+  aut: Schema.optional<Schema.Struct<{
+    name: Schema.optional<typeof Schema.String>;
+    kind: typeof Schema.String;
+    id: typeof Schema.String;
+    version: Schema.optional<typeof Schema.String>;
+  }>>;
+  startedAt: typeof Schema.String;
+  datasets: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
+    id: typeof Schema.String;
+    visibility: Schema.Literal<["candidate", "evaluator"]>;
+    source: Schema.Literal<["static", "computed"]>;
+    digest: typeof Schema.String;
+  }>>>>;
+}>;
+/** On-disk manifests include a lifecycle status; older finalized manifests may still say running. */
+export declare const RunManifestSchema: Schema.Struct<{
+  schemaVersion: Schema.Literal<[2, 3, 4, 5]>;
+  runUri: Schema.filter<typeof Schema.String>;
+  evalId: typeof Schema.String;
+  suiteId: Schema.optional<typeof Schema.String>;
+  parameters: Schema.optional<Schema.Record$<typeof Schema.String, Schema.Schema<JsonValue, JsonValue, never>>>;
+  matrix: Schema.optional<Schema.Struct<{
+    id: typeof Schema.String;
+    cellKey: typeof Schema.String;
+  }>>;
+  aut: Schema.optional<Schema.Struct<{
+    name: Schema.optional<typeof Schema.String>;
+    kind: typeof Schema.String;
+    id: typeof Schema.String;
+    version: Schema.optional<typeof Schema.String>;
+  }>>;
+  startedAt: typeof Schema.String;
+  source: Schema.optional<Schema.Struct<{
+    repo: typeof Schema.String;
+    project: typeof Schema.String;
+    commit: typeof Schema.String;
+    ref: Schema.optional<typeof Schema.String>;
+    committedAt: Schema.optional<typeof Schema.String>;
+    dirty: typeof Schema.Boolean;
+    ci: Schema.optional<Schema.Struct<{
+      provider: typeof Schema.String;
+      pipeline: Schema.optional<typeof Schema.String>;
+      job: Schema.optional<typeof Schema.String>;
+      url: Schema.optional<typeof Schema.String>;
+    }>>;
+  }>>;
+  process: Schema.optional<Schema.Struct<{
+    pid: typeof Schema.Number;
+    hostname: typeof Schema.String;
+  }>>;
+  status: Schema.optional<Schema.Literal<["running", "completed", "failed", "cancelled"]>>;
+}>;
+export declare const TrialManifestSchema: Schema.Struct<{
+  schemaVersion: Schema.Literal<[2, 3, 4, 5]>;
+  runUri: Schema.filter<typeof Schema.String>;
+  trialUri: Schema.filter<typeof Schema.String>;
+  trialIndex: typeof Schema.Number;
+  evalId: typeof Schema.String;
+  parameters: Schema.optional<Schema.Record$<typeof Schema.String, Schema.Schema<JsonValue, JsonValue, never>>>;
+  matrix: Schema.optional<Schema.Struct<{
+    id: typeof Schema.String;
+    cellKey: typeof Schema.String;
+  }>>;
+  aut: Schema.optional<Schema.Struct<{
+    name: Schema.optional<typeof Schema.String>;
+    kind: typeof Schema.String;
+    id: typeof Schema.String;
+    version: Schema.optional<typeof Schema.String>;
+  }>>;
+  startedAt: typeof Schema.String;
+  datasets: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
+    id: typeof Schema.String;
+    visibility: Schema.Literal<["candidate", "evaluator"]>;
+    source: Schema.Literal<["static", "computed"]>;
+    digest: typeof Schema.String;
+  }>>>>;
+  status: Schema.optional<Schema.Literal<["running", "completed", "failed", "cancelled"]>>;
+}>;
+export declare const UsageSchema: Schema.Struct<{
+  inputTokens: Schema.optional<typeof Schema.Number>;
+  outputTokens: Schema.optional<typeof Schema.Number>;
+  totalTokens: Schema.optional<typeof Schema.Number>;
+}>;
+export declare const JudgeRunInfoSchema: Schema.Struct<{
+  agent: Schema.optional<Schema.Struct<{
+    id: typeof Schema.String;
+    kind: typeof Schema.String;
+    name: Schema.optional<typeof Schema.String>;
+    version: Schema.optional<typeof Schema.String>;
+  }>>;
+  usage: Schema.optional<Schema.Struct<{
+    inputTokens: Schema.optional<typeof Schema.Number>;
+    outputTokens: Schema.optional<typeof Schema.Number>;
+    totalTokens: Schema.optional<typeof Schema.Number>;
+  }>>;
+  events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
+}>;
+export declare const ScoreResultSchema: Schema.Struct<{
+  name: typeof Schema.String;
+  kind: Schema.Literal<["predicate", "judge"]>;
+  value: Schema.optional<typeof Schema.Number>;
+  passed: Schema.optional<typeof Schema.Boolean>;
+  explanation: Schema.optional<typeof Schema.String>;
+  evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+  judge: Schema.optional<Schema.Struct<{
+    agent: Schema.optional<Schema.Struct<{
+      id: typeof Schema.String;
+      kind: typeof Schema.String;
+      name: Schema.optional<typeof Schema.String>;
+      version: Schema.optional<typeof Schema.String>;
+    }>>;
+    usage: Schema.optional<Schema.Struct<{
+      inputTokens: Schema.optional<typeof Schema.Number>;
+      outputTokens: Schema.optional<typeof Schema.Number>;
+      totalTokens: Schema.optional<typeof Schema.Number>;
+    }>>;
+    events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
+  }>>;
+  durationMs: typeof Schema.Number;
+  error: Schema.optional<Schema.Struct<{
+    name: typeof Schema.String;
+    message: typeof Schema.String;
+    stack: Schema.optional<typeof Schema.String>;
+  }>>;
+  skipped: Schema.optional<Schema.Literal<[true]>>;
+}>;
+export declare const CheckpointResultSchema: Schema.Struct<{
+  step: typeof Schema.Number;
+  kind: Schema.Literal<["predicate", "judge", "expect-tool-call"]>;
+  name: typeof Schema.String;
+  status: Schema.Literal<["passed", "failed", "error", "skipped"]>;
+  value: Schema.optional<typeof Schema.Number>;
+  passed: Schema.optional<typeof Schema.Boolean>;
+  explanation: Schema.optional<typeof Schema.String>;
+  evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+  judge: Schema.optional<Schema.Struct<{
+    agent: Schema.optional<Schema.Struct<{
+      id: typeof Schema.String;
+      kind: typeof Schema.String;
+      name: Schema.optional<typeof Schema.String>;
+      version: Schema.optional<typeof Schema.String>;
+    }>>;
+    usage: Schema.optional<Schema.Struct<{
+      inputTokens: Schema.optional<typeof Schema.Number>;
+      outputTokens: Schema.optional<typeof Schema.Number>;
+      totalTokens: Schema.optional<typeof Schema.Number>;
+    }>>;
+    events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
+  }>>;
+  durationMs: Schema.optional<typeof Schema.Number>;
+  error: Schema.optional<Schema.Struct<{
+    name: typeof Schema.String;
+    message: typeof Schema.String;
+    stack: Schema.optional<typeof Schema.String>;
+  }>>;
+  matchedToolCall: Schema.optional<Schema.Struct<{
+    eventIndex: typeof Schema.Number;
+    id: typeof Schema.String;
+  }>>;
+}>;
+export declare const TrialScoringSchema: Schema.Struct<{
+  results: Schema.mutable<Schema.Array$<Schema.Struct<{
+    name: typeof Schema.String;
+    kind: Schema.Literal<["predicate", "judge"]>;
+    value: Schema.optional<typeof Schema.Number>;
+    passed: Schema.optional<typeof Schema.Boolean>;
+    explanation: Schema.optional<typeof Schema.String>;
+    evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+    judge: Schema.optional<Schema.Struct<{
+      agent: Schema.optional<Schema.Struct<{
+        id: typeof Schema.String;
+        kind: typeof Schema.String;
+        name: Schema.optional<typeof Schema.String>;
+        version: Schema.optional<typeof Schema.String>;
+      }>>;
+      usage: Schema.optional<Schema.Struct<{
+        inputTokens: Schema.optional<typeof Schema.Number>;
+        outputTokens: Schema.optional<typeof Schema.Number>;
+        totalTokens: Schema.optional<typeof Schema.Number>;
+      }>>;
+      events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
+    }>>;
+    durationMs: typeof Schema.Number;
+    error: Schema.optional<Schema.Struct<{
+      name: typeof Schema.String;
+      message: typeof Schema.String;
+      stack: Schema.optional<typeof Schema.String>;
+    }>>;
+    skipped: Schema.optional<Schema.Literal<[true]>>;
+  }>>>;
+  checkpoints: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
+    step: typeof Schema.Number;
+    kind: Schema.Literal<["predicate", "judge", "expect-tool-call"]>;
+    name: typeof Schema.String;
+    status: Schema.Literal<["passed", "failed", "error", "skipped"]>;
+    value: Schema.optional<typeof Schema.Number>;
+    passed: Schema.optional<typeof Schema.Boolean>;
+    explanation: Schema.optional<typeof Schema.String>;
+    evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+    judge: Schema.optional<Schema.Struct<{
+      agent: Schema.optional<Schema.Struct<{
+        id: typeof Schema.String;
+        kind: typeof Schema.String;
+        name: Schema.optional<typeof Schema.String>;
+        version: Schema.optional<typeof Schema.String>;
+      }>>;
+      usage: Schema.optional<Schema.Struct<{
+        inputTokens: Schema.optional<typeof Schema.Number>;
+        outputTokens: Schema.optional<typeof Schema.Number>;
+        totalTokens: Schema.optional<typeof Schema.Number>;
+      }>>;
+      events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
+    }>>;
+    durationMs: Schema.optional<typeof Schema.Number>;
+    error: Schema.optional<Schema.Struct<{
+      name: typeof Schema.String;
+      message: typeof Schema.String;
+      stack: Schema.optional<typeof Schema.String>;
+    }>>;
+    matchedToolCall: Schema.optional<Schema.Struct<{
+      eventIndex: typeof Schema.Number;
+      id: typeof Schema.String;
+    }>>;
+  }>>>>;
+  skippedScorers: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
+  overall: Schema.optional<typeof Schema.Number>;
+  passed: typeof Schema.Boolean;
+}>;
+export declare const ArtifactEntrySchema: Schema.Struct<{
+  path: typeof Schema.String;
+  kind: Schema.Literal<["file", "directory"]>;
+  size: Schema.optional<typeof Schema.Number>;
+}>;
+export declare const TrialSummarySchema: Schema.Struct<{
+  status: Schema.Literal<["running", "completed", "failed", "cancelled"]>;
+  endedAt: typeof Schema.String;
+  durationMs: Schema.optional<typeof Schema.Number>;
+  scoring: Schema.optional<Schema.Struct<{
+    results: Schema.mutable<Schema.Array$<Schema.Struct<{
+      name: typeof Schema.String;
+      kind: Schema.Literal<["predicate", "judge"]>;
+      value: Schema.optional<typeof Schema.Number>;
+      passed: Schema.optional<typeof Schema.Boolean>;
+      explanation: Schema.optional<typeof Schema.String>;
+      evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+      judge: Schema.optional<Schema.Struct<{
+        agent: Schema.optional<Schema.Struct<{
+          id: typeof Schema.String;
+          kind: typeof Schema.String;
+          name: Schema.optional<typeof Schema.String>;
+          version: Schema.optional<typeof Schema.String>;
+        }>>;
+        usage: Schema.optional<Schema.Struct<{
+          inputTokens: Schema.optional<typeof Schema.Number>;
+          outputTokens: Schema.optional<typeof Schema.Number>;
+          totalTokens: Schema.optional<typeof Schema.Number>;
+        }>>;
+        events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
+      }>>;
+      durationMs: typeof Schema.Number;
+      error: Schema.optional<Schema.Struct<{
+        name: typeof Schema.String;
+        message: typeof Schema.String;
+        stack: Schema.optional<typeof Schema.String>;
+      }>>;
+      skipped: Schema.optional<Schema.Literal<[true]>>;
+    }>>>;
+    checkpoints: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
+      step: typeof Schema.Number;
+      kind: Schema.Literal<["predicate", "judge", "expect-tool-call"]>;
+      name: typeof Schema.String;
+      status: Schema.Literal<["passed", "failed", "error", "skipped"]>;
+      value: Schema.optional<typeof Schema.Number>;
+      passed: Schema.optional<typeof Schema.Boolean>;
+      explanation: Schema.optional<typeof Schema.String>;
+      evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+      judge: Schema.optional<Schema.Struct<{
+        agent: Schema.optional<Schema.Struct<{
+          id: typeof Schema.String;
+          kind: typeof Schema.String;
+          name: Schema.optional<typeof Schema.String>;
+          version: Schema.optional<typeof Schema.String>;
+        }>>;
+        usage: Schema.optional<Schema.Struct<{
+          inputTokens: Schema.optional<typeof Schema.Number>;
+          outputTokens: Schema.optional<typeof Schema.Number>;
+          totalTokens: Schema.optional<typeof Schema.Number>;
+        }>>;
+        events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
+      }>>;
+      durationMs: Schema.optional<typeof Schema.Number>;
+      error: Schema.optional<Schema.Struct<{
+        name: typeof Schema.String;
+        message: typeof Schema.String;
+        stack: Schema.optional<typeof Schema.String>;
+      }>>;
+      matchedToolCall: Schema.optional<Schema.Struct<{
+        eventIndex: typeof Schema.Number;
+        id: typeof Schema.String;
+      }>>;
+    }>>>>;
+    skippedScorers: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
+    overall: Schema.optional<typeof Schema.Number>;
+    passed: typeof Schema.Boolean;
+  }>>;
+  artifacts: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
+    path: typeof Schema.String;
+    kind: Schema.Literal<["file", "directory"]>;
+    size: Schema.optional<typeof Schema.Number>;
+  }>>>>;
+  error: Schema.optional<Schema.Struct<{
+    name: typeof Schema.String;
+    message: typeof Schema.String;
+    stack: Schema.optional<typeof Schema.String>;
+  }>>;
+}>;
+export declare const RunSummarySchema: Schema.Struct<{
+  status: Schema.Literal<["running", "completed", "failed", "cancelled"]>;
+  endedAt: typeof Schema.String;
+  durationMs: Schema.optional<typeof Schema.Number>;
+  trialCount: typeof Schema.Number;
+  passed: typeof Schema.Number;
+  failed: typeof Schema.Number;
+  cancelled: Schema.optional<typeof Schema.Number>;
+  error: Schema.optional<Schema.Struct<{
+    name: typeof Schema.String;
+    message: typeof Schema.String;
+    stack: Schema.optional<typeof Schema.String>;
+  }>>;
+}>;
+export declare const TrajectoryEventSchema: Schema.Schema<TrajectoryEvent>;
+export declare const ApiErrorSchema: Schema.Struct<{
+  error: typeof Schema.String;
+  message: typeof Schema.String;
+}>;
+//#endregion
+//#region ../core/src/trajectory.d.ts
+/** Decode one private report's JSONL trajectory; reject malformed/non-event lines with their line number. */
+export declare function parseTrajectoryJsonl(contents: string): TrajectoryEvent[];
+//#endregion
+//#region ../core/src/index.d.ts
+export type JsonPrimitive = boolean | number | string | null;
+export type JsonValue = JsonPrimitive | JsonValue[] | {
+  [key: string]: JsonValue;
+};
+export type JsonObject = {
+  [key: string]: JsonValue;
+};
+export type AutIdentity = {
+  /** Human-readable logical agent name shown in catalogs and dashboards. */
+  name?: string;
+  kind: string;
+  id: string;
+  version?: string;
+};
+export type AgentRuntimeName = 'local' | 'sandbox' | 'remote';
+/**
+ * A serializable runtime requirement for an AUT. Concrete runtime packages own
+ * the interpretation of `kind` and `configuration`.
+ */
+export type AgentRuntime = {
+  kind: string;
+  configuration?: JsonObject;
+};
+export type AgentRuntimes = Partial<Record<AgentRuntimeName, AgentRuntime>>;
+export type Usage = {
+  inputTokens?: number;
+  outputTokens?: number;
+  totalTokens?: number;
+};
+export type RecordedError = {
+  name: string;
+  message: string;
+  stack?: string;
+};
+export type AutEvent = {
+  kind: 'started';
+  timestamp: string;
+} | {
+  kind: 'message';
+  role: 'system' | 'user' | 'assistant' | 'tool';
+  content: JsonValue;
+  timestamp: string;
+} | {
+  kind: 'tool-call';
+  id: string;
+  name: string;
+  arguments: JsonValue;
+  timestamp: string;
+} | {
+  kind: 'tool-result';
+  id: string;
+  name?: string;
+  result: JsonValue;
+  timestamp: string;
+} | {
+  kind: 'turn-started';
+  turn: number;
+  timestamp: string;
+} | {
+  kind: 'turn-completed';
+  turn: number;
+  timestamp: string;
+  latencyMs?: number;
+  usage?: Usage;
+} | {
+  kind: 'completed';
+  output?: JsonValue;
+  timestamp: string;
+} | {
+  kind: 'error';
+  error: RecordedError;
+  timestamp: string;
+};
+export type RunnerEvent = {
+  kind: 'trial-started';
+  timestamp: string;
+} | {
+  kind: 'transcript-step-started';
+  step: number;
+  /** The user message sent at this step; absent for checkpoints. */
+  message?: string;
+  timestamp: string;
+} | {
+  kind: 'transcript-step-completed';
+  step: number;
+  timestamp: string;
+} | {
+  kind: 'transcript-step-skipped';
+  step: number;
+  timestamp: string;
+} | {
+  kind: 'checkpoint-started';
+  step: number;
+  name: string;
+  timestamp: string;
+} | {
+  kind: 'checkpoint-completed';
+  step: number;
+  status: 'passed' | 'failed' | 'skipped';
+  timestamp: string;
+} | {
+  kind: 'checkpoint-error';
+  step: number;
+  error: RecordedError;
+  timestamp: string;
+} | {
+  kind: 'scorer-started';
+  scorer: string;
+  timestamp: string;
+} | {
+  kind: 'scorer-completed';
+  scorer: string;
+  /** Absent when the scorer returned `skip()`. */
+  value?: number;
+  skipped?: true;
+  timestamp: string;
+} | {
+  kind: 'scorer-failed';
+  scorer: string;
+  error: RecordedError;
+  timestamp: string;
+} | {
+  kind: 'trial-completed';
+  timestamp: string;
+} | {
+  kind: 'error';
+  error: RecordedError;
+  timestamp: string;
+};
+export type TrajectoryEvent = (AutEvent & {
+  source: 'aut';
+}) | (RunnerEvent & {
+  source: 'runner';
+});
+export type WorkspaceView = {
+  /** Absolute path to this trial-scoped workspace. Do not persist it in reports. */
+  root: string;
+};
+export type CandidateWorkspace = WorkspaceView;
+export type EvaluatorWorkspace = WorkspaceView;
+export type AutContext = {
+  runId: string;
+  evalId: string;
+  trialId: string;
+  trialIndex: number;
+  metadata: JsonObject;
+  /** Run-time parameters supplied by the eval definition or CLI invocation. */
+  parameters?: JsonObject;
+  runtime?: AgentRuntimeName;
+  /** Values of the scenario's candidate datasets, by dataset ID. */
+  data: JsonObject;
+  /** Contains only resources visible to the AUT/model. */
+  workspace: CandidateWorkspace;
+  /** Private evaluator capability for trusted AUT adapters; never expose this to the model. */
+  evaluatorWorkspace: EvaluatorWorkspace;
+};
+export type AutAdapter<TMessage = string> = {
+  readonly identity?: AutIdentity;
+  /** Available execution environments for this AUT. Default: `{ local: { kind: 'in-process' } }`. */
+  readonly runtimes?: AgentRuntimes;
+  start(options: {
+    context: AutContext;
+    runtime?: AgentRuntime;
+    /** Optional: stream events as they happen. Returning a turn result from `send` is enough for most agents. */
+    onEvent: (event: AutEventInput) => void | Promise<void>;
+  }): Promise<AutSession<TMessage>>;
+};
+export type AutSession<TMessage = string> = {
+  /**
+   * Resolves once the AUT has completed the turn initiated by this message.
+   * It may return the turn's reply and tool calls instead of emitting events.
+   */
+  send(message: TMessage): Promise<void | TurnResult>;
+  /** Optional; defaults to doing nothing. */
+  close?(): Promise<void>;
+};
+/** Used when an agent declares no runtimes. */
+export declare const DEFAULT_AGENT_RUNTIMES: AgentRuntimes;
+/** The runtimes an agent can run in, applying the default. */
+export declare function agentRuntimes(agent: AutAdapter<any>): AgentRuntimes;
+/**
+ * Defines an Agent Under Test without coupling it to a runtime or transport.
+ * It preserves the adapter's concrete type for application-specific helpers.
+ */
+export declare function defineAgent<const T extends AutAdapter>(agent: T): T;
+export type UserStep = {
+  kind: 'user';
+  message: string;
+};
+export type AgentStep = {
+  kind: 'agent';
+  expectation: JsonObject;
+};
+export type TurnView = {
+  userStepIndex: number;
+  events: readonly (AutEvent & {
+    source: 'aut';
+  })[];
+  assistantMessages: readonly (Extract<AutEvent, {
+    kind: 'message';
+  }> & {
+    role: 'assistant';
+    source: 'aut';
+  })[];
+  toolCalls: readonly {
+    eventIndex: number;
+    id: string;
+    name: string;
+    arguments: JsonValue;
+    resultObservation: 'observed' | 'absent' | 'ambiguous';
+    result?: JsonValue;
+  }[];
+  lastAssistantText?: string;
+};
+export type ExpectToolCallStep = {
+  kind: 'expect-tool-call';
+  name: string;
+  expected: {
+    name: string;
+    arguments?: JsonValue;
+  };
+};
+export type TranscriptStep = UserStep | AgentStep | ScoringRule | ExpectToolCallStep;
+/** Observes a tool call in the preceding turn; never invokes a tool. */
+export declare function expectToolCall(expected: ExpectToolCallStep['expected']): ExpectToolCallStep;
+export declare function user(message: string): UserStep;
+export declare function agent(expectation: JsonObject): AgentStep;
+export type JudgeRunInfo = {
+  agent?: AutIdentity;
+  usage?: Usage;
+  /** Judge-agent observations; never attributed to the AUT trajectory. */
+  events?: JsonValue[];
+};
+/** A scorer's "does not apply" result; see `skip()`. */
+export type SkippedScore = {
+  skipped: true;
+  reason: string;
+};
+export type ScoreValue = number | SkippedScore | {
+  value: number;
+  passed?: boolean;
+  explanation?: string;
+  /** Any JSON-convertible value. */
+  evidence?: unknown;
+  /** Judge-agent evidence, never inferred from AUT usage. */
+  judge?: JudgeRunInfo;
+};
+/**
+ * Marks a scorer as not applicable to this trial. A skipped result neither
+ * passes nor fails the trial and is left out of score averages.
+ */
+export declare function skip(reason: string): SkippedScore;
+export type ScoreResult = {
+  name: string;
+  kind: 'predicate' | 'judge';
+  value?: number;
+  passed?: boolean;
+  explanation?: string;
+  evidence?: JsonValue;
+  judge?: JudgeRunInfo;
+  durationMs: number;
+  error?: RecordedError;
+  /** The scorer returned `skip()`; `explanation` holds the reason and there is no value. */
+  skipped?: true;
+};
+export type ArtifactView = {
+  candidate: CandidateWorkspace;
+  evaluator: EvaluatorWorkspace;
+};
+export type ScoringContext = {
+  context: AutContext;
+  trajectory: {
+    events: readonly TrajectoryEvent[];
+  };
+  artifacts: ArtifactView;
+  /** Most recent completed turn; absent if no user message has been sent. */
+  turn?: TurnView;
+  /** Content of the last assistant message of `turn`. */
+  output?: JsonValue;
+  /** Values of all the scenario's datasets (candidate and evaluator), by dataset ID. */
+  data: JsonObject;
+};
+export type PredicateScorer = {
+  kind: 'predicate';
+  name: string;
+  supportsPartial?: boolean;
+  /** A numeric result passes when it is at least this value. Default: only 1 passes. */
+  passAt?: number;
+  run(context: ScoringContext): boolean | ScoreValue | Promise<boolean | ScoreValue>;
+};
+export type JudgeRule = {
+  kind: 'judge';
+  name: string;
+  rubric: string;
+  /** Run even when the scenario stopped early or execution failed. */
+  supportsPartial?: boolean;
+};
+export type ScoringRule = PredicateScorer | JudgeRule;
+export declare function predicate(name: string, run: PredicateScorer['run'], options?: Pick<PredicateScorer, 'supportsPartial' | 'passAt'>): PredicateScorer;
+export declare function judge(name: string, options: Pick<JudgeRule, 'rubric' | 'supportsPartial'>): JudgeRule;
+export type EvalPolicy = {
+  timeoutMs?: number;
+  /** Stop later transcript steps after a failed checkpoint assertion. Default false. */
+  failfast?: boolean;
+  /** Number of independent trials requested by local and hosted runners. */
+  trials?: number;
+};
+export type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
+  id: string;
+  name?: string;
+  agent: TAgent;
+  /** Separate judge agent; its model and tools are never inherited from the AUT. */
+  judge?: AutAdapter;
+  /** Data and files the scenario uses; each trial gets its own copy. */
+  datasets?: readonly Dataset[];
+  transcript: TranscriptStep[];
+  scoring: ScoringRule[];
+  policy?: EvalPolicy;
+  /** Any JSON-convertible object; converted when the eval is defined. */
+  metadata?: Record<string, unknown>;
+};
+export declare function defineEval<const T extends EvalDefinition>(definition: T): T;
+export type EvalSuite<TEvals extends readonly EvalDefinition[] = readonly EvalDefinition[]> = {
+  id: string;
+  name?: string;
+  evals: TEvals;
+};
+export declare function defineSuite<const TSuite extends EvalSuite>(suite: TSuite): TSuite;
+export type EvalRegistration = EvalDefinition | EvalSuite | EvalMatrix;
+export declare function authoringId(value: {
+  id: string;
+}): string;
+export type EvalRegistry = {
+  /** All registered evals, flattened from standalone entries and suites. */
+  evals: readonly EvalDefinition[];
+  suites: readonly EvalSuite[];
+  matrices: readonly EvalMatrix[];
+  get(uri: string): EvalDefinition | undefined;
+  getSuite(uri: string): EvalSuite | undefined;
+  metadata(): EvalRegistryMetadata[];
+  suiteMetadata(): EvalSuiteMetadata[];
+  /** Definition data for the dashboard; pass the project root to include where each thing is defined. */
+  catalog(options?: CatalogOptions): EvalCatalogEntry[];
+};
+export type EvalRegistryMetadata = {
+  id: string;
+  name?: string;
+  suiteId?: string;
+};
+export type EvalSuiteMetadata = {
+  id: string;
+  name?: string;
+  evalIds: string[];
+};
+/**
+ * Creates an explicit, statically imported eval registry. A suite is a
+ * path-like grouping of evals; nesting is presentation derived from its ID.
+ */
+export declare function registerEvals<const TRegistrations extends readonly EvalRegistration[]>(registrations: TRegistrations): EvalRegistry;
+export type RunStatus = 'running' | 'completed' | 'failed' | 'cancelled';
+/**
+ * Where a run's eval code came from: the repository, eval project and commit.
+ * Results are tagged with it; they are never stored in the repository itself.
+ */
+export type RunSource = {
+  /** Normalized repository key, e.g. `gitlab.example.com/group/repo`. */
+  repo: string;
+  /** Eval project directory relative to the repository root; `""` at the root. */
+  project: string;
+  /** Full commit SHA. */
+  commit: string;
+  ref?: string;
+  committedAt?: string;
+  /** True when the working tree had uncommitted changes at run time. */
+  dirty: boolean;
+  ci?: {
+    provider: string;
+    pipeline?: string;
+    job?: string;
+    url?: string;
+  };
+};
+/** The project's evals and suites when a run started; written beside its manifest. */
+export type RunCatalog = {
+  catalog: EvalCatalogEntry[];
+  evals: EvalRegistryMetadata[];
+  suites: EvalSuiteMetadata[];
+};
+/** The process that owns a running run, so readers can tell an interrupted run from a live one. */
+export type RunProcess = {
+  pid: number;
+  hostname: string;
+};
+export type RunMetadata = {
+  schemaVersion: 2 | 3 | 4 | 5;
+  runId: string;
+  runUri: ResourceUri<'run'>;
+  evalId: string;
+  suiteId?: string;
+  parameters?: JsonObject;
+  matrix?: {
+    id: string;
+    cellKey: string;
+  };
+  aut?: AutIdentity;
+  startedAt: string;
+  /** Present from schema version 4 when the run's commit could be determined. */
+  source?: RunSource;
+  /** From schema version 5. */
+  process?: RunProcess;
+};
+export type TrialMetadata = {
+  schemaVersion: 2 | 3 | 4 | 5;
+  runId: string;
+  runUri: ResourceUri<'run'>;
+  trialId: string;
+  trialUri: ResourceUri<'trial'>;
+  trialIndex: number;
+  evalId: string;
+  parameters?: JsonObject;
+  matrix?: {
+    id: string;
+    cellKey: string;
+  };
+  aut?: AutIdentity;
+  startedAt: string;
+  /** From schema version 5: the datasets this trial used, with content digests. */
+  datasets?: TrialDataset[];
+};
+export type TrialDataset = {
+  id: string;
+  visibility: DatasetVisibility;
+  source: 'static' | 'computed';
+  /** SHA-256 of the trial's content: file paths and bytes, and the canonical value. */
+  digest: string;
+};
+export type CheckpointResult = {
+  step: number;
+  kind: 'predicate' | 'judge' | 'expect-tool-call';
+  name: string;
+  status: 'passed' | 'failed' | 'error' | 'skipped';
+  value?: number;
+  passed?: boolean;
+  explanation?: string;
+  evidence?: JsonValue;
+  judge?: JudgeRunInfo;
+  durationMs?: number;
+  error?: RecordedError;
+  matchedToolCall?: {
+    eventIndex: number;
+    id: string;
+  };
+};
+export type TrialScoring = {
+  results: ScoreResult[];
+  checkpoints?: CheckpointResult[];
+  skippedScorers?: string[];
+  overall?: number;
+  passed: boolean;
+};
+export type ArtifactEntry = {
+  /** Path relative to this trial's artifact root. */
+  path: string;
+  kind: 'file' | 'directory';
+  size?: number;
+};
+export type TrialSummary = {
+  status: RunStatus;
+  endedAt: string;
+  durationMs?: number;
+  scoring?: TrialScoring;
+  artifacts?: ArtifactEntry[];
+  error?: RecordedError;
+};
+export type RunSummary = {
+  status: RunStatus;
+  endedAt: string;
+  durationMs?: number;
+  trialCount: number;
+  passed: number;
+  failed: number;
+  /** Trials stopped by cancellation; counted neither as passed nor failed. */
+  cancelled?: number;
+  error?: RecordedError;
+};
+export type TrialWriter = {
+  appendEvent(event: TrajectoryEvent): Promise<void>;
+  writeScores(scoring: TrialScoring): Promise<void>;
+  /** Writes a file below this trial's artifact root. */
+  writeArtifact(path: string, data: Uint8Array): Promise<void>;
+  finalize(summary: TrialSummary): Promise<void>;
+};
+export type RunWriter = {
+  readonly location: string;
+  startTrial(metadata: TrialMetadata): Promise<TrialWriter>;
+  finalize(summary: RunSummary): Promise<void>;
+};
+export type ReportStore = {
+  startRun(metadata: RunMetadata, context?: {
+    catalog?: RunCatalog;
+  }): Promise<RunWriter>;
+};
+export type TrialResult = {
+  runId: string;
+  runUri: ResourceUri<'run'>;
+  trialId: string;
+  trialUri: ResourceUri<'trial'>;
+  trialIndex: number;
+  evalId: string;
+  status: RunStatus;
+  reportLocation: string;
+  durationMs?: number;
+  scoring?: TrialScoring;
+  error?: RecordedError;
+};
+export type AggregateScoring = {
+  passed: number;
+  failed: number;
+  cancelled?: number;
+  passRate: number;
+  overall?: number;
+};
+export type RunResult = TrialResult & {
+  runId: string;
+  trialCount?: number;
+  passed?: number;
+  failed?: number;
+  aggregateScoring?: AggregateScoring;
+  /** Present for an aggregate multi-trial run. */
+  trials?: TrialResult[];
+};
+export declare function recordError(error: unknown): RecordedError;
+//#endregion
 //#region ../agents/src/index.d.ts
 type PiAutOptions = {
   command?: string;
@@ -8,5 +1352,5 @@ type PiAutOptions = {
 /** A local, one-prompt-per-turn AUT backed by Pi's non-interactive print mode. */
 export declare function piAgent(options?: PiAutOptions): AutAdapter;
 //#endregion
-export { AgentRuntime, AgentRuntimeName, AgentRuntimes, AgentStep, AggregateScoring, ApiErrorSchema, ArtifactEntry, ArtifactEntrySchema, ArtifactView, AutAdapter, AutContext, AutEvent, AutIdentity, AutIdentitySchema, AutSession, CandidateWorkspace, CheckpointResult, CheckpointResultSchema, DirectoryFixture, DynamicFixture, EvalCatalogEntry, EvalCatalogFixture, EvalDefinition, type EvalMatrix, type EvalMatrixCell, EvalPolicy, EvalRegistration, EvalRegistry, EvalRegistryMetadata, EvalSuite, EvalSuiteMetadata, type EvalkitConfig, EvaluatorWorkspace, ExpectToolCallStep, FileFixture, Fixture, FixtureContext, FixtureVisibility, InlineFixture, JsonObject, JsonPrimitive, JsonValue, JsonValueSchema, JudgeRule, JudgeRunInfo, JudgeRunInfoSchema, type MatrixSelection, PredicateScorer, RecordedError, RecordedErrorSchema, ReportStore, ResourceKind, ResourceKindSchema, ResourceUri, ResourceUriSchema, RunCatalog, RunManifestSchema, RunMetadata, RunMetadataSchema, RunResult, RunSource, RunSourceSchema, RunStatus, RunStatusSchema, RunSummary, RunSummarySchema, RunWriter, RunnerEvent, ScoreResult, ScoreResultSchema, ScoreValue, ScoringContext, ScoringRule, type ShareConfig, type ShareHeaders, TrajectoryEvent, TrajectoryEventSchema, TranscriptStep, TrialManifestSchema, TrialMetadata, TrialMetadataSchema, TrialResult, TrialScoring, TrialScoringSchema, TrialSummary, TrialSummarySchema, TrialWriter, TurnView, Usage, UsageSchema, UserStep, Uuid, UuidSchema, WorkspaceView, agent, authorId, authoringId, canonicalParameters, defineAgent, defineConfig, defineEval, defineEvalMatrix, defineSuite, directory, dynamic, expectToolCall, file, inlineFile, judge, parseResourceUri, parseTrajectoryJsonl, predicate, recordError, registerEvals, resourceUri, resourceUriSchema, resourceUuid, user };
+export type { DefinitionSite, EvalMatrix, EvalMatrixCell, EvalkitConfig, MatrixSelection, ShareConfig, ShareHeaders };
 //# sourceMappingURL=index.d.mts.map