@leostera/evalkit 0.5.0 → 0.6.1

package/dist/index-lOw0fmak.d.mts

@@ -1,1028 +0,0 @@
-import * as Schema from "effect/Schema";
-//#region ../core/src/identity.d.ts
-declare const ResourceKindSchema: Schema.Literal<["run", "trial", "artifact"]>;
-type ResourceKind = typeof ResourceKindSchema.Type;
-/** Human-authored project-local identifier; never a generated resource URI. */
-declare function authorId(value: string): string;
-/** RFC 4122 textual UUID. EvalKit accepts v4/v7 and other valid UUID versions. */
-declare const UuidSchema: Schema.filter<typeof Schema.String>;
-type Uuid = typeof UuidSchema.Type;
-type ResourceUri<TKind extends ResourceKind = ResourceKind> = `evalkit:${TKind}:${string}`;
-declare const ResourceUriSchema: Schema.filter<typeof Schema.String>;
-declare function resourceUriSchema<TKind extends ResourceKind>(kind: TKind): Schema.filter<typeof Schema.String>;
-declare function resourceUri<TKind extends ResourceKind>(kind: TKind, uuid: Uuid): ResourceUri<TKind>;
-declare function parseResourceUri<TKind extends ResourceKind>(value: string, expectedKind?: TKind): {
-  kind: TKind;
-  uuid: Uuid;
-  uri: ResourceUri<TKind>;
-};
-declare function resourceUuid(value: string, expectedKind?: ResourceKind): Uuid;
-//#endregion
-//#region ../core/src/canonical-json.d.ts
-/** Canonical JSON: keys sorted recursively, array order preserved. Rejects non-JSON inputs. */
-declare function canonicalParameters(value: JsonValue): string;
-//#endregion
-//#region ../core/src/matrix.d.ts
-type MatrixSelection = {
-  /** Base eval IDs. Unknown values fail before dispatch. */
-  evals?: readonly string[];
-  /** Select existing dimension values, rather than overwriting every cell with the same value. */
-  parameters?: Readonly<Record<string, readonly JsonValue[]>>;
-  /** Overrides for non-dimension execution settings (e.g. maxTokens). */
-  overrides?: JsonObject;
-};
-type EvalMatrixCell = {
-  matrixId: string;
-  /** Stable canonical key including matrix, base eval and effective parameters. */
-  key: string;
-  eval: EvalDefinition;
-  parameters: JsonObject;
-};
-type EvalMatrixDefinition = {
-  id: string;
-  name?: string;
-  evals: readonly EvalDefinition[];
-  parameters: Readonly<Record<string, readonly JsonValue[]>>;
-  defaults?: JsonObject;
-  /** Complete correlated assignments, crossed with independent parameters. */
-  cases?: readonly JsonObject[];
-  /** Partial patterns over declared dimensions; matching cells are never run. */
-  exclude?: readonly JsonObject[];
-};
-type EvalMatrix = EvalMatrixDefinition & {
-  kind: 'matrix';
-  count(selection?: MatrixSelection): number;
-  cells(selection?: MatrixSelection): IterableIterator<EvalMatrixCell>;
-};
-declare function defineEvalMatrix<const T extends EvalMatrixDefinition>(definition: T): EvalMatrix & T;
-//#endregion
-//#region ../core/src/config.d.ts
-type EvalkitConfig = {
-  /** Paths are relative to evalkit.config.js/ts, not the invoking shell directory. */
-  testDir?: string;
-  include?: readonly string[];
-  exclude?: readonly string[];
-  /** Explicit definitions are an alternative to discovery (e.g. hosted registries). */
-  evals?: readonly EvalDefinition[];
-  registry?: EvalRegistry;
-  /** Applied to every discovered eval. Values are forwarded, never provider-interpreted. */
-  matrix?: Omit<EvalMatrixDefinition, 'evals' | 'id'> & {
-    id?: string;
-  };
-  execution?: {
-    concurrency?: number;
-    trials?: number;
-    maxCells?: number;
-  };
-  reportDir?: string;
-  sandboxDir?: string;
-  share?: ShareConfig;
-};
-/** HTTP header names and values. */
-type ShareHeaders = Record<string, string>;
-type ShareConfig = {
-  /** Where `evalkit share` uploads results. `EVALKIT_SHARE_URL` overrides it. */
-  url: string;
-  /**
-   * Extra headers for every request to the share server, e.g. credentials for
-   * an auth proxy in front of it. A function is called once per `evalkit share`
-   * (it may run a command to fetch a token). Values are never printed.
-   */
-  headers?: ShareHeaders | (() => ShareHeaders | Promise<ShareHeaders>);
-};
-/** Pure, typed project configuration. Loading/discovery belongs to the CLI. */
-declare function defineConfig<const T extends EvalkitConfig>(configuration: T): T;
-//#endregion
-//#region ../core/src/schema.d.ts
-/** Runtime schemas for serialized EvalKit report and API boundaries. */
-declare const RunStatusSchema: Schema.Literal<["running", "completed", "failed", "cancelled"]>;
-declare const RecordedErrorSchema: Schema.Struct<{
-  name: typeof Schema.String;
-  message: typeof Schema.String;
-  stack: Schema.optional<typeof Schema.String>;
-}>;
-declare const AutIdentitySchema: Schema.Struct<{
-  name: Schema.optional<typeof Schema.String>;
-  kind: typeof Schema.String;
-  id: typeof Schema.String;
-  version: Schema.optional<typeof Schema.String>;
-}>;
-declare const JsonValueSchema: Schema.Schema<JsonValue>;
-declare const RunSourceSchema: Schema.Struct<{
-  repo: typeof Schema.String;
-  project: typeof Schema.String;
-  commit: typeof Schema.String;
-  ref: Schema.optional<typeof Schema.String>;
-  committedAt: Schema.optional<typeof Schema.String>;
-  dirty: typeof Schema.Boolean;
-  ci: Schema.optional<Schema.Struct<{
-    provider: typeof Schema.String;
-    pipeline: Schema.optional<typeof Schema.String>;
-    job: Schema.optional<typeof Schema.String>;
-    url: Schema.optional<typeof Schema.String>;
-  }>>;
-}>;
-declare const RunMetadataSchema: Schema.Struct<{
-  schemaVersion: Schema.Literal<[2, 3, 4]>;
-  runUri: Schema.filter<typeof Schema.String>;
-  evalId: typeof Schema.String;
-  suiteId: Schema.optional<typeof Schema.String>;
-  parameters: Schema.optional<Schema.Record$<typeof Schema.String, Schema.Schema<JsonValue, JsonValue, never>>>;
-  matrix: Schema.optional<Schema.Struct<{
-    id: typeof Schema.String;
-    cellKey: typeof Schema.String;
-  }>>;
-  aut: Schema.optional<Schema.Struct<{
-    name: Schema.optional<typeof Schema.String>;
-    kind: typeof Schema.String;
-    id: typeof Schema.String;
-    version: Schema.optional<typeof Schema.String>;
-  }>>;
-  startedAt: typeof Schema.String;
-  source: Schema.optional<Schema.Struct<{
-    repo: typeof Schema.String;
-    project: typeof Schema.String;
-    commit: typeof Schema.String;
-    ref: Schema.optional<typeof Schema.String>;
-    committedAt: Schema.optional<typeof Schema.String>;
-    dirty: typeof Schema.Boolean;
-    ci: Schema.optional<Schema.Struct<{
-      provider: typeof Schema.String;
-      pipeline: Schema.optional<typeof Schema.String>;
-      job: Schema.optional<typeof Schema.String>;
-      url: Schema.optional<typeof Schema.String>;
-    }>>;
-  }>>;
-}>;
-declare const TrialMetadataSchema: Schema.Struct<{
-  schemaVersion: Schema.Literal<[2, 3, 4]>;
-  runUri: Schema.filter<typeof Schema.String>;
-  trialUri: Schema.filter<typeof Schema.String>;
-  trialIndex: typeof Schema.Number;
-  evalId: typeof Schema.String;
-  parameters: Schema.optional<Schema.Record$<typeof Schema.String, Schema.Schema<JsonValue, JsonValue, never>>>;
-  matrix: Schema.optional<Schema.Struct<{
-    id: typeof Schema.String;
-    cellKey: typeof Schema.String;
-  }>>;
-  aut: Schema.optional<Schema.Struct<{
-    name: Schema.optional<typeof Schema.String>;
-    kind: typeof Schema.String;
-    id: typeof Schema.String;
-    version: Schema.optional<typeof Schema.String>;
-  }>>;
-  startedAt: typeof Schema.String;
-}>;
-/** On-disk manifests include a lifecycle status; older finalized manifests may still say running. */
-declare const RunManifestSchema: Schema.Struct<{
-  schemaVersion: Schema.Literal<[2, 3, 4]>;
-  runUri: Schema.filter<typeof Schema.String>;
-  evalId: typeof Schema.String;
-  suiteId: Schema.optional<typeof Schema.String>;
-  parameters: Schema.optional<Schema.Record$<typeof Schema.String, Schema.Schema<JsonValue, JsonValue, never>>>;
-  matrix: Schema.optional<Schema.Struct<{
-    id: typeof Schema.String;
-    cellKey: typeof Schema.String;
-  }>>;
-  aut: Schema.optional<Schema.Struct<{
-    name: Schema.optional<typeof Schema.String>;
-    kind: typeof Schema.String;
-    id: typeof Schema.String;
-    version: Schema.optional<typeof Schema.String>;
-  }>>;
-  startedAt: typeof Schema.String;
-  source: Schema.optional<Schema.Struct<{
-    repo: typeof Schema.String;
-    project: typeof Schema.String;
-    commit: typeof Schema.String;
-    ref: Schema.optional<typeof Schema.String>;
-    committedAt: Schema.optional<typeof Schema.String>;
-    dirty: typeof Schema.Boolean;
-    ci: Schema.optional<Schema.Struct<{
-      provider: typeof Schema.String;
-      pipeline: Schema.optional<typeof Schema.String>;
-      job: Schema.optional<typeof Schema.String>;
-      url: Schema.optional<typeof Schema.String>;
-    }>>;
-  }>>;
-  status: Schema.optional<Schema.Literal<["running", "completed", "failed", "cancelled"]>>;
-}>;
-declare const TrialManifestSchema: Schema.Struct<{
-  schemaVersion: Schema.Literal<[2, 3, 4]>;
-  runUri: Schema.filter<typeof Schema.String>;
-  trialUri: Schema.filter<typeof Schema.String>;
-  trialIndex: typeof Schema.Number;
-  evalId: typeof Schema.String;
-  parameters: Schema.optional<Schema.Record$<typeof Schema.String, Schema.Schema<JsonValue, JsonValue, never>>>;
-  matrix: Schema.optional<Schema.Struct<{
-    id: typeof Schema.String;
-    cellKey: typeof Schema.String;
-  }>>;
-  aut: Schema.optional<Schema.Struct<{
-    name: Schema.optional<typeof Schema.String>;
-    kind: typeof Schema.String;
-    id: typeof Schema.String;
-    version: Schema.optional<typeof Schema.String>;
-  }>>;
-  startedAt: typeof Schema.String;
-  status: Schema.optional<Schema.Literal<["running", "completed", "failed", "cancelled"]>>;
-}>;
-declare const UsageSchema: Schema.Struct<{
-  inputTokens: Schema.optional<typeof Schema.Number>;
-  outputTokens: Schema.optional<typeof Schema.Number>;
-  totalTokens: Schema.optional<typeof Schema.Number>;
-}>;
-declare const JudgeRunInfoSchema: Schema.Struct<{
-  agent: Schema.optional<Schema.Struct<{
-    id: typeof Schema.String;
-    kind: typeof Schema.String;
-    name: Schema.optional<typeof Schema.String>;
-    version: Schema.optional<typeof Schema.String>;
-  }>>;
-  usage: Schema.optional<Schema.Struct<{
-    inputTokens: Schema.optional<typeof Schema.Number>;
-    outputTokens: Schema.optional<typeof Schema.Number>;
-    totalTokens: Schema.optional<typeof Schema.Number>;
-  }>>;
-  events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
-}>;
-declare const ScoreResultSchema: Schema.Struct<{
-  name: typeof Schema.String;
-  kind: Schema.Literal<["predicate", "judge"]>;
-  value: Schema.optional<typeof Schema.Number>;
-  passed: Schema.optional<typeof Schema.Boolean>;
-  explanation: Schema.optional<typeof Schema.String>;
-  evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
-  judge: Schema.optional<Schema.Struct<{
-    agent: Schema.optional<Schema.Struct<{
-      id: typeof Schema.String;
-      kind: typeof Schema.String;
-      name: Schema.optional<typeof Schema.String>;
-      version: Schema.optional<typeof Schema.String>;
-    }>>;
-    usage: Schema.optional<Schema.Struct<{
-      inputTokens: Schema.optional<typeof Schema.Number>;
-      outputTokens: Schema.optional<typeof Schema.Number>;
-      totalTokens: Schema.optional<typeof Schema.Number>;
-    }>>;
-    events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
-  }>>;
-  durationMs: typeof Schema.Number;
-  error: Schema.optional<Schema.Struct<{
-    name: typeof Schema.String;
-    message: typeof Schema.String;
-    stack: Schema.optional<typeof Schema.String>;
-  }>>;
-}>;
-declare const CheckpointResultSchema: Schema.Struct<{
-  step: typeof Schema.Number;
-  kind: Schema.Literal<["predicate", "judge", "expect-tool-call"]>;
-  name: typeof Schema.String;
-  status: Schema.Literal<["passed", "failed", "error", "skipped"]>;
-  value: Schema.optional<typeof Schema.Number>;
-  passed: Schema.optional<typeof Schema.Boolean>;
-  explanation: Schema.optional<typeof Schema.String>;
-  evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
-  judge: Schema.optional<Schema.Struct<{
-    agent: Schema.optional<Schema.Struct<{
-      id: typeof Schema.String;
-      kind: typeof Schema.String;
-      name: Schema.optional<typeof Schema.String>;
-      version: Schema.optional<typeof Schema.String>;
-    }>>;
-    usage: Schema.optional<Schema.Struct<{
-      inputTokens: Schema.optional<typeof Schema.Number>;
-      outputTokens: Schema.optional<typeof Schema.Number>;
-      totalTokens: Schema.optional<typeof Schema.Number>;
-    }>>;
-    events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
-  }>>;
-  durationMs: Schema.optional<typeof Schema.Number>;
-  error: Schema.optional<Schema.Struct<{
-    name: typeof Schema.String;
-    message: typeof Schema.String;
-    stack: Schema.optional<typeof Schema.String>;
-  }>>;
-  matchedToolCall: Schema.optional<Schema.Struct<{
-    eventIndex: typeof Schema.Number;
-    id: typeof Schema.String;
-  }>>;
-}>;
-declare const TrialScoringSchema: Schema.Struct<{
-  results: Schema.mutable<Schema.Array$<Schema.Struct<{
-    name: typeof Schema.String;
-    kind: Schema.Literal<["predicate", "judge"]>;
-    value: Schema.optional<typeof Schema.Number>;
-    passed: Schema.optional<typeof Schema.Boolean>;
-    explanation: Schema.optional<typeof Schema.String>;
-    evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
-    judge: Schema.optional<Schema.Struct<{
-      agent: Schema.optional<Schema.Struct<{
-        id: typeof Schema.String;
-        kind: typeof Schema.String;
-        name: Schema.optional<typeof Schema.String>;
-        version: Schema.optional<typeof Schema.String>;
-      }>>;
-      usage: Schema.optional<Schema.Struct<{
-        inputTokens: Schema.optional<typeof Schema.Number>;
-        outputTokens: Schema.optional<typeof Schema.Number>;
-        totalTokens: Schema.optional<typeof Schema.Number>;
-      }>>;
-      events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
-    }>>;
-    durationMs: typeof Schema.Number;
-    error: Schema.optional<Schema.Struct<{
-      name: typeof Schema.String;
-      message: typeof Schema.String;
-      stack: Schema.optional<typeof Schema.String>;
-    }>>;
-  }>>>;
-  checkpoints: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
-    step: typeof Schema.Number;
-    kind: Schema.Literal<["predicate", "judge", "expect-tool-call"]>;
-    name: typeof Schema.String;
-    status: Schema.Literal<["passed", "failed", "error", "skipped"]>;
-    value: Schema.optional<typeof Schema.Number>;
-    passed: Schema.optional<typeof Schema.Boolean>;
-    explanation: Schema.optional<typeof Schema.String>;
-    evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
-    judge: Schema.optional<Schema.Struct<{
-      agent: Schema.optional<Schema.Struct<{
-        id: typeof Schema.String;
-        kind: typeof Schema.String;
-        name: Schema.optional<typeof Schema.String>;
-        version: Schema.optional<typeof Schema.String>;
-      }>>;
-      usage: Schema.optional<Schema.Struct<{
-        inputTokens: Schema.optional<typeof Schema.Number>;
-        outputTokens: Schema.optional<typeof Schema.Number>;
-        totalTokens: Schema.optional<typeof Schema.Number>;
-      }>>;
-      events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
-    }>>;
-    durationMs: Schema.optional<typeof Schema.Number>;
-    error: Schema.optional<Schema.Struct<{
-      name: typeof Schema.String;
-      message: typeof Schema.String;
-      stack: Schema.optional<typeof Schema.String>;
-    }>>;
-    matchedToolCall: Schema.optional<Schema.Struct<{
-      eventIndex: typeof Schema.Number;
-      id: typeof Schema.String;
-    }>>;
-  }>>>>;
-  skippedScorers: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
-  overall: Schema.optional<typeof Schema.Number>;
-  passed: typeof Schema.Boolean;
-}>;
-declare const ArtifactEntrySchema: Schema.Struct<{
-  path: typeof Schema.String;
-  kind: Schema.Literal<["file", "directory"]>;
-  size: Schema.optional<typeof Schema.Number>;
-}>;
-declare const TrialSummarySchema: Schema.Struct<{
-  status: Schema.Literal<["running", "completed", "failed", "cancelled"]>;
-  endedAt: typeof Schema.String;
-  durationMs: Schema.optional<typeof Schema.Number>;
-  scoring: Schema.optional<Schema.Struct<{
-    results: Schema.mutable<Schema.Array$<Schema.Struct<{
-      name: typeof Schema.String;
-      kind: Schema.Literal<["predicate", "judge"]>;
-      value: Schema.optional<typeof Schema.Number>;
-      passed: Schema.optional<typeof Schema.Boolean>;
-      explanation: Schema.optional<typeof Schema.String>;
-      evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
-      judge: Schema.optional<Schema.Struct<{
-        agent: Schema.optional<Schema.Struct<{
-          id: typeof Schema.String;
-          kind: typeof Schema.String;
-          name: Schema.optional<typeof Schema.String>;
-          version: Schema.optional<typeof Schema.String>;
-        }>>;
-        usage: Schema.optional<Schema.Struct<{
-          inputTokens: Schema.optional<typeof Schema.Number>;
-          outputTokens: Schema.optional<typeof Schema.Number>;
-          totalTokens: Schema.optional<typeof Schema.Number>;
-        }>>;
-        events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
-      }>>;
-      durationMs: typeof Schema.Number;
-      error: Schema.optional<Schema.Struct<{
-        name: typeof Schema.String;
-        message: typeof Schema.String;
-        stack: Schema.optional<typeof Schema.String>;
-      }>>;
-    }>>>;
-    checkpoints: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
-      step: typeof Schema.Number;
-      kind: Schema.Literal<["predicate", "judge", "expect-tool-call"]>;
-      name: typeof Schema.String;
-      status: Schema.Literal<["passed", "failed", "error", "skipped"]>;
-      value: Schema.optional<typeof Schema.Number>;
-      passed: Schema.optional<typeof Schema.Boolean>;
-      explanation: Schema.optional<typeof Schema.String>;
-      evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
-      judge: Schema.optional<Schema.Struct<{
-        agent: Schema.optional<Schema.Struct<{
-          id: typeof Schema.String;
-          kind: typeof Schema.String;
-          name: Schema.optional<typeof Schema.String>;
-          version: Schema.optional<typeof Schema.String>;
-        }>>;
-        usage: Schema.optional<Schema.Struct<{
-          inputTokens: Schema.optional<typeof Schema.Number>;
-          outputTokens: Schema.optional<typeof Schema.Number>;
-          totalTokens: Schema.optional<typeof Schema.Number>;
-        }>>;
-        events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
-      }>>;
-      durationMs: Schema.optional<typeof Schema.Number>;
-      error: Schema.optional<Schema.Struct<{
-        name: typeof Schema.String;
-        message: typeof Schema.String;
-        stack: Schema.optional<typeof Schema.String>;
-      }>>;
-      matchedToolCall: Schema.optional<Schema.Struct<{
-        eventIndex: typeof Schema.Number;
-        id: typeof Schema.String;
-      }>>;
-    }>>>>;
-    skippedScorers: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
-    overall: Schema.optional<typeof Schema.Number>;
-    passed: typeof Schema.Boolean;
-  }>>;
-  artifacts: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
-    path: typeof Schema.String;
-    kind: Schema.Literal<["file", "directory"]>;
-    size: Schema.optional<typeof Schema.Number>;
-  }>>>>;
-  error: Schema.optional<Schema.Struct<{
-    name: typeof Schema.String;
-    message: typeof Schema.String;
-    stack: Schema.optional<typeof Schema.String>;
-  }>>;
-}>;
-declare const RunSummarySchema: Schema.Struct<{
-  status: Schema.Literal<["running", "completed", "failed", "cancelled"]>;
-  endedAt: typeof Schema.String;
-  durationMs: Schema.optional<typeof Schema.Number>;
-  trialCount: typeof Schema.Number;
-  passed: typeof Schema.Number;
-  failed: typeof Schema.Number;
-  error: Schema.optional<Schema.Struct<{
-    name: typeof Schema.String;
-    message: typeof Schema.String;
-    stack: Schema.optional<typeof Schema.String>;
-  }>>;
-}>;
-declare const TrajectoryEventSchema: Schema.Schema<TrajectoryEvent>;
-declare const ApiErrorSchema: Schema.Struct<{
-  error: typeof Schema.String;
-  message: typeof Schema.String;
-}>;
-//#endregion
-//#region ../core/src/trajectory.d.ts
-/** Decode one private report's JSONL trajectory; reject malformed/non-event lines with their line number. */
-declare function parseTrajectoryJsonl(contents: string): TrajectoryEvent[];
-//#endregion
-//#region ../core/src/index.d.ts
-type JsonPrimitive = boolean | number | string | null;
-type JsonValue = JsonPrimitive | JsonValue[] | {
-  [key: string]: JsonValue;
-};
-type JsonObject = {
-  [key: string]: JsonValue;
-};
-type AutIdentity = {
-  /** Human-readable logical agent name shown in catalogs and dashboards. */
-  name?: string;
-  kind: string;
-  id: string;
-  version?: string;
-};
-type AgentRuntimeName = 'local' | 'sandbox' | 'remote';
-/**
- * A serializable runtime requirement for an AUT. Concrete runtime packages own
- * the interpretation of `kind` and `configuration`.
- */
-type AgentRuntime = {
-  kind: string;
-  configuration?: JsonObject;
-};
-type AgentRuntimes = Partial<Record<AgentRuntimeName, AgentRuntime>>;
-type Usage = {
-  inputTokens?: number;
-  outputTokens?: number;
-  totalTokens?: number;
-};
-type RecordedError = {
-  name: string;
-  message: string;
-  stack?: string;
-};
-type AutEvent = {
-  kind: 'started';
-  timestamp: string;
-} | {
-  kind: 'message';
-  role: 'system' | 'user' | 'assistant' | 'tool';
-  content: JsonValue;
-  timestamp: string;
-} | {
-  kind: 'tool-call';
-  id: string;
-  name: string;
-  arguments: JsonValue;
-  timestamp: string;
-} | {
-  kind: 'tool-result';
-  id: string;
-  name?: string;
-  result: JsonValue;
-  timestamp: string;
-} | {
-  kind: 'turn-started';
-  turn: number;
-  timestamp: string;
-} | {
-  kind: 'turn-completed';
-  turn: number;
-  timestamp: string;
-  latencyMs?: number;
-  usage?: Usage;
-} | {
-  kind: 'completed';
-  output?: JsonValue;
-  timestamp: string;
-} | {
-  kind: 'error';
-  error: RecordedError;
-  timestamp: string;
-};
-type RunnerEvent = {
-  kind: 'trial-started';
-  timestamp: string;
-} | {
-  kind: 'transcript-step-started';
-  step: number;
-  timestamp: string;
-} | {
-  kind: 'transcript-step-completed';
-  step: number;
-  timestamp: string;
-} | {
-  kind: 'transcript-step-skipped';
-  step: number;
-  timestamp: string;
-} | {
-  kind: 'checkpoint-started';
-  step: number;
-  name: string;
-  timestamp: string;
-} | {
-  kind: 'checkpoint-completed';
-  step: number;
-  status: 'passed' | 'failed';
-  timestamp: string;
-} | {
-  kind: 'checkpoint-error';
-  step: number;
-  error: RecordedError;
-  timestamp: string;
-} | {
-  kind: 'scorer-started';
-  scorer: string;
-  timestamp: string;
-} | {
-  kind: 'scorer-completed';
-  scorer: string;
-  value: number;
-  timestamp: string;
-} | {
-  kind: 'scorer-failed';
-  scorer: string;
-  error: RecordedError;
-  timestamp: string;
-} | {
-  kind: 'trial-completed';
-  timestamp: string;
-} | {
-  kind: 'error';
-  error: RecordedError;
-  timestamp: string;
-};
-type TrajectoryEvent = (AutEvent & {
-  source: 'aut';
-}) | (RunnerEvent & {
-  source: 'runner';
-});
-type WorkspaceView = {
-  /** Absolute path to this trial-scoped workspace. Do not persist it in reports. */
-  root: string;
-};
-type CandidateWorkspace = WorkspaceView;
-type EvaluatorWorkspace = WorkspaceView;
-type AutContext = {
-  runId: string;
-  evalId: string;
-  trialId: string;
-  trialIndex: number;
-  metadata: JsonObject;
-  /** Run-time parameters supplied by the eval definition or CLI invocation. */
-  parameters?: JsonObject;
-  runtime?: AgentRuntimeName;
-  /** Contains only resources visible to the AUT/model. */
-  workspace: CandidateWorkspace;
-  /** Private evaluator capability for trusted AUT adapters; never expose this to the model. */
-  evaluatorWorkspace: EvaluatorWorkspace;
-};
-type FixtureContext = Omit<AutContext, 'workspace' | 'evaluatorWorkspace'>;
-type AutAdapter<TMessage = string> = {
-  readonly identity?: AutIdentity;
-  /** Available execution environments for this AUT. */
-  readonly runtimes?: AgentRuntimes;
-  start(options: {
-    context: AutContext;
-    runtime?: AgentRuntime;
-    onEvent: (event: AutEvent) => void | Promise<void>;
-  }): Promise<AutSession<TMessage>>;
-};
-type AutSession<TMessage = string> = {
-  /** Resolves once the AUT has completed the turn initiated by this message. */
-  send(message: TMessage): Promise<void>;
-  close(): Promise<void>;
-};
-/**
- * Defines an Agent Under Test without coupling it to a runtime or transport.
- * It preserves the adapter's concrete type for application-specific helpers.
- */
-declare function defineAgent<const T extends AutAdapter>(agent: T): T;
-type FixtureVisibility = 'candidate' | 'evaluator';
-type DirectoryFixture = {
-  kind: 'directory';
-  src: string;
-  dst: string;
-  visibility: FixtureVisibility;
-};
-type FileFixture = {
-  kind: 'file';
-  src: string;
-  dst: string;
-  visibility: FixtureVisibility;
-};
-type InlineFixture = {
-  kind: 'inline';
-  file: string;
-  data: string;
-  visibility: FixtureVisibility;
-};
-type DynamicFixture = {
-  kind: 'dynamic';
-  create(context: FixtureContext): Fixture | Fixture[] | Promise<Fixture | Fixture[]>;
-};
-type Fixture = DirectoryFixture | FileFixture | InlineFixture | DynamicFixture;
-/**
- * Declares a directory relative to the invoking project's working directory.
- * The short form is candidate-visible and copies beneath the source basename.
- */
-declare function directory(src: string, options?: Partial<Omit<DirectoryFixture, 'kind' | 'src'>>): DirectoryFixture;
-declare function file(src: string, options: Omit<FileFixture, 'kind' | 'src'>): FileFixture;
-declare function inlineFile(file: string, data: string, visibility: FixtureVisibility): InlineFixture;
-declare function dynamic(create: DynamicFixture['create']): DynamicFixture;
-type UserStep = {
-  kind: 'user';
-  message: string;
-};
-type AgentStep = {
-  kind: 'agent';
-  expectation: JsonObject;
-};
-type TurnView = {
-  userStepIndex: number;
-  events: readonly (AutEvent & {
-    source: 'aut';
-  })[];
-  assistantMessages: readonly (Extract<AutEvent, {
-    kind: 'message';
-  }> & {
-    role: 'assistant';
-    source: 'aut';
-  })[];
-  toolCalls: readonly {
-    eventIndex: number;
-    id: string;
-    name: string;
-    arguments: JsonValue;
-    resultObservation: 'observed' | 'absent' | 'ambiguous';
-    result?: JsonValue;
-  }[];
-  lastAssistantText?: string;
-};
-type ExpectToolCallStep = {
-  kind: 'expect-tool-call';
-  name: string;
-  expected: {
-    name: string;
-    arguments?: JsonValue;
-  };
-};
-type TranscriptStep = UserStep | AgentStep | ScoringRule | ExpectToolCallStep;
-/** Observes a tool call in the preceding turn; never invokes a tool. */
-declare function expectToolCall(expected: ExpectToolCallStep['expected']): ExpectToolCallStep;
-declare function user(message: string): UserStep;
-declare function agent(expectation: JsonObject): AgentStep;
-type JudgeRunInfo = {
-  agent?: AutIdentity;
-  usage?: Usage;
-  /** Judge-agent observations; never attributed to the AUT trajectory. */
-  events?: JsonValue[];
-};
-type ScoreValue = number | {
-  value: number;
-  passed?: boolean;
-  explanation?: string;
-  evidence?: JsonValue;
-  /** Judge-agent evidence, never inferred from AUT usage. */
-  judge?: JudgeRunInfo;
-};
-type ScoreResult = {
-  name: string;
-  kind: 'predicate' | 'judge';
-  value?: number;
-  passed?: boolean;
-  explanation?: string;
-  evidence?: JsonValue;
-  judge?: JudgeRunInfo;
-  durationMs: number;
-  error?: RecordedError;
-};
-type ArtifactView = {
-  candidate: CandidateWorkspace;
-  evaluator: EvaluatorWorkspace;
-};
-type ScoringContext = {
-  context: AutContext;
-  trajectory: {
-    events: readonly TrajectoryEvent[];
-  };
-  artifacts: ArtifactView;
-  /** Most recent completed turn; absent if no user message has been sent. */
-  turn?: TurnView;
-};
-type PredicateScorer = {
-  kind: 'predicate';
-  name: string;
-  supportsPartial?: boolean;
-  run(context: ScoringContext): boolean | ScoreValue | Promise<boolean | ScoreValue>;
-};
-type JudgeRule = {
-  kind: 'judge';
-  name: string;
-  rubric: string;
-  /** Run even when the scenario stopped early or execution failed. */
-  supportsPartial?: boolean;
-};
-type ScoringRule = PredicateScorer | JudgeRule;
-declare function predicate(name: string, run: PredicateScorer['run'], options?: Pick<PredicateScorer, 'supportsPartial'>): PredicateScorer;
-declare function judge(name: string, options: Pick<JudgeRule, 'rubric' | 'supportsPartial'>): JudgeRule;
-type EvalPolicy = {
-  timeoutMs?: number;
-  /** Stop later transcript steps after a failed checkpoint assertion. Default false. */
-  failfast?: boolean;
-  /** Number of independent trials requested by local and hosted runners. */
-  trials?: number;
-};
-type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
-  id: string;
-  name?: string;
-  agent: TAgent;
-  /** Separate judge agent; its model and tools are never inherited from the AUT. */
-  judge?: AutAdapter;
-  fixtures?: Fixture[];
-  transcript: TranscriptStep[];
-  scoring: ScoringRule[];
-  policy?: EvalPolicy;
-  metadata?: JsonObject;
-};
-declare function defineEval<const T extends EvalDefinition>(definition: T): T;
-type EvalSuite<TEvals extends readonly EvalDefinition[] = readonly EvalDefinition[]> = {
-  id: string;
-  name?: string;
-  evals: TEvals;
-};
-declare function defineSuite<const TSuite extends EvalSuite>(suite: TSuite): TSuite;
-type EvalRegistration = EvalDefinition | EvalSuite | EvalMatrix;
-declare function authoringId(value: {
-  id: string;
-}): string;
-type EvalRegistry = {
-  /** All registered evals, flattened from standalone entries and suites. */
-  evals: readonly EvalDefinition[];
-  suites: readonly EvalSuite[];
-  matrices: readonly EvalMatrix[];
-  get(uri: string): EvalDefinition | undefined;
-  getSuite(uri: string): EvalSuite | undefined;
-  metadata(): EvalRegistryMetadata[];
-  suiteMetadata(): EvalSuiteMetadata[];
-  catalog(): EvalCatalogEntry[];
-};
-type EvalRegistryMetadata = {
-  id: string;
-  name?: string;
-  suiteId?: string;
-};
-type EvalSuiteMetadata = {
-  id: string;
-  name?: string;
-  evalIds: string[];
-};
-type EvalCatalogFixture = {
-  kind: Fixture['kind'];
-  source: string;
-  destination?: string;
-  visibility?: FixtureVisibility;
-};
-type EvalCatalogEntry = {
-  id: string;
-  path: string;
-  name?: string;
-  suiteId?: string;
-  agent: {
-    name?: string;
-    kind: string;
-    id?: string;
-    version?: string;
-    runtimes: Array<{
-      name: AgentRuntimeName;
-      kind: string;
-    }>;
-  };
-  fixtures: EvalCatalogFixture[];
-  trialCount: number;
-  scorers: Array<{
-    name: string;
-    kind: ScoringRule['kind'];
-  }>;
-};
-/**
- * Creates an explicit, statically imported eval registry. A suite is a
- * path-like grouping of evals; nesting is presentation derived from its ID.
- */
-declare function registerEvals<const TRegistrations extends readonly EvalRegistration[]>(registrations: TRegistrations): EvalRegistry;
-type RunStatus = 'running' | 'completed' | 'failed' | 'cancelled';
-/**
- * Where a run's eval code came from: the repository, eval project and commit.
- * Results are tagged with it; they are never stored in the repository itself.
- */
-type RunSource = {
-  /** Normalized repository key, e.g. `gitlab.example.com/group/repo`. */
-  repo: string;
-  /** Eval project directory relative to the repository root; `""` at the root. */
-  project: string;
-  /** Full commit SHA. */
-  commit: string;
-  ref?: string;
-  committedAt?: string;
-  /** True when the working tree had uncommitted changes at run time. */
-  dirty: boolean;
-  ci?: {
-    provider: string;
-    pipeline?: string;
-    job?: string;
-    url?: string;
-  };
-};
-/** The project's evals and suites when a run started; written beside its manifest. */
-type RunCatalog = {
-  catalog: EvalCatalogEntry[];
-  evals: EvalRegistryMetadata[];
-  suites: EvalSuiteMetadata[];
-};
-type RunMetadata = {
-  schemaVersion: 2 | 3 | 4;
-  runId: string;
-  runUri: ResourceUri<'run'>;
-  evalId: string;
-  suiteId?: string;
-  parameters?: JsonObject;
-  matrix?: {
-    id: string;
-    cellKey: string;
-  };
-  aut?: AutIdentity;
-  startedAt: string;
-  /** Present from schema version 4 when the run's commit could be determined. */
-  source?: RunSource;
-};
-type TrialMetadata = {
-  schemaVersion: 2 | 3 | 4;
-  runId: string;
-  runUri: ResourceUri<'run'>;
-  trialId: string;
-  trialUri: ResourceUri<'trial'>;
-  trialIndex: number;
-  evalId: string;
-  parameters?: JsonObject;
-  matrix?: {
-    id: string;
-    cellKey: string;
-  };
-  aut?: AutIdentity;
-  startedAt: string;
-};
-type CheckpointResult = {
-  step: number;
-  kind: 'predicate' | 'judge' | 'expect-tool-call';
-  name: string;
-  status: 'passed' | 'failed' | 'error' | 'skipped';
-  value?: number;
-  passed?: boolean;
-  explanation?: string;
-  evidence?: JsonValue;
-  judge?: JudgeRunInfo;
-  durationMs?: number;
-  error?: RecordedError;
-  matchedToolCall?: {
-    eventIndex: number;
-    id: string;
-  };
-};
-type TrialScoring = {
-  results: ScoreResult[];
-  checkpoints?: CheckpointResult[];
-  skippedScorers?: string[];
-  overall?: number;
-  passed: boolean;
-};
-type ArtifactEntry = {
-  /** Path relative to this trial's artifact root. */
-  path: string;
-  kind: 'file' | 'directory';
-  size?: number;
-};
-type TrialSummary = {
-  status: RunStatus;
-  endedAt: string;
-  durationMs?: number;
-  scoring?: TrialScoring;
-  artifacts?: ArtifactEntry[];
-  error?: RecordedError;
-};
-type RunSummary = {
-  status: RunStatus;
-  endedAt: string;
-  durationMs?: number;
-  trialCount: number;
-  passed: number;
-  failed: number;
-  error?: RecordedError;
-};
-type TrialWriter = {
-  appendEvent(event: TrajectoryEvent): Promise<void>;
-  writeScores(scoring: TrialScoring): Promise<void>;
-  /** Writes a file below this trial's artifact root. */
-  writeArtifact(path: string, data: Uint8Array): Promise<void>;
-  finalize(summary: TrialSummary): Promise<void>;
-};
-type RunWriter = {
-  readonly location: string;
-  startTrial(metadata: TrialMetadata): Promise<TrialWriter>;
-  finalize(summary: RunSummary): Promise<void>;
-};
-type ReportStore = {
-  startRun(metadata: RunMetadata, context?: {
-    catalog?: RunCatalog;
-  }): Promise<RunWriter>;
-};
-type TrialResult = {
-  runId: string;
-  runUri: ResourceUri<'run'>;
-  trialId: string;
-  trialUri: ResourceUri<'trial'>;
-  trialIndex: number;
-  evalId: string;
-  status: RunStatus;
-  reportLocation: string;
-  durationMs?: number;
-  scoring?: TrialScoring;
-  error?: RecordedError;
-};
-type AggregateScoring = {
-  passed: number;
-  failed: number;
-  passRate: number;
-  overall?: number;
-};
-type RunResult = TrialResult & {
-  runId: string;
-  trialCount?: number;
-  passed?: number;
-  failed?: number;
-  aggregateScoring?: AggregateScoring;
-  /** Present for an aggregate multi-trial run. */
-  trials?: TrialResult[];
-};
-declare function recordError(error: unknown): RecordedError;
-//#endregion
-export { TrajectoryEvent as $, canonicalParameters as $t, FixtureContext as A, JsonValueSchema as At, ReportStore as B, TrialManifestSchema as Bt, EvalRegistryMetadata as C, registerEvals as Ct, ExpectToolCallStep as D, ArtifactEntrySchema as Dt, EvaluatorWorkspace as E, ApiErrorSchema as Et, JsonValue as F, RunSourceSchema as Ft, RunStatus as G, EvalkitConfig as Gt, RunMetadata as H, TrialScoringSchema as Ht, JudgeRule as I, RunStatusSchema as It, RunnerEvent as J, defineConfig as Jt, RunSummary as K, ShareConfig as Kt, JudgeRunInfo as L, RunSummarySchema as Lt, InlineFixture as M, RecordedErrorSchema as Mt, JsonObject as N, RunManifestSchema as Nt, FileFixture as O, AutIdentitySchema as Ot, JsonPrimitive as P, RunMetadataSchema as Pt, ScoringRule as Q, defineEvalMatrix as Qt, PredicateScorer as R, ScoreResultSchema as Rt, EvalRegistry as S, recordError as St, EvalSuiteMetadata as T, parseTrajectoryJsonl as Tt, RunResult as U, TrialSummarySchema as Ut, RunCatalog as V, TrialMetadataSchema as Vt, RunSource as W, UsageSchema as Wt, ScoreValue as X, EvalMatrixCell as Xt, ScoreResult as Y, EvalMatrix as Yt, ScoringContext as Z, MatrixSelection as Zt, EvalCatalogEntry as _, expectToolCall as _t, AggregateScoring as a, UuidSchema as an, TrialWriter as at, EvalPolicy as b, judge as bt, AutAdapter as c, resourceUri as cn, UserStep as ct, AutIdentity as d, authoringId as dt, ResourceKind as en, TranscriptStep as et, AutSession as f, defineAgent as ft, DynamicFixture as g, dynamic as gt, DirectoryFixture as h, directory as ht, AgentStep as i, Uuid as in, TrialSummary as it, FixtureVisibility as j, JudgeRunInfoSchema as jt, Fixture as k, CheckpointResultSchema as kt, AutContext as l, resourceUriSchema as ln, WorkspaceView as lt, CheckpointResult as m, defineSuite as mt, AgentRuntimeName as n, ResourceUri as nn, TrialResult as nt, ArtifactEntry as o, authorId as on, TurnView as ot, CandidateWorkspace as p, defineEval as pt, RunWriter as q, ShareHeaders as qt, AgentRuntimes as r, ResourceUriSchema as rn, TrialScoring as rt, ArtifactView as s, parseResourceUri as sn, Usage as st, AgentRuntime as t, ResourceKindSchema as tn, TrialMetadata as tt, AutEvent as u, resourceUuid as un, agent as ut, EvalCatalogFixture as v, file as vt, EvalSuite as w, user as wt, EvalRegistration as x, predicate as xt, EvalDefinition as y, inlineFile as yt, RecordedError as z, TrajectoryEventSchema as zt };
-//# sourceMappingURL=index-lOw0fmak.d.mts.map