@leostera/evalkit 0.9.0 → 0.10.0

package/dist/index.d.mts

@@ -144,6 +144,8 @@ export type AutEventInput = {
   timestamp?: string;
   latencyMs?: number;
   usage?: Usage;
+  /** The model that answered this turn, when the agent knows it. */
+  model?: string;
 } | {
   kind: 'completed';
   output?: unknown;
@@ -165,6 +167,8 @@ export type TurnResult = string | {
   reply?: unknown;
   toolCalls?: readonly TurnToolCall[];
   usage?: Usage;
+  /** The model that answered, recorded with the usage. */
+  model?: string;
 };
 /** Converts an adapter event into a recorded `AutEvent`. */
 export declare function normalizeAutEvent(input: AutEventInput, now: () => Date): AutEvent;
@@ -174,6 +178,89 @@ export declare function normalizeAutEvent(input: AutEventInput, now: () => Date)
  */
 export declare function turnResultEvents(result: TurnResult, turn: number, now: () => Date): AutEvent[];
 //#endregion
+//#region ../core/src/judge.d.ts
+/**
+ * How a judge must answer, and how EvalKit turns the answer into a score.
+ *
+ * - `verdict` (the default): a value from 0 to 1 and a reason, as JSON.
+ * - `choices`: one of a fixed set of answers, such as A/B/C, each worth a value.
+ * - `criteria`: a value from 0 to 1 per criterion; the score is their weighted mean.
+ */
+type JudgeAnswer = {
+  kind: 'verdict';
+} | {
+  kind: 'choices';
+  choices: JudgeChoice[];
+} | {
+  kind: 'criteria';
+  criteria: JudgeCriterion[];
+};
+/** One answer a `choices` judge can give. Keys are matched case-insensitively. */
+type JudgeChoice = {
+  key: string;
+  label: string;
+  value: number;
+};
+/** One thing a `criteria` judge scores. */
+type JudgeCriterion = {
+  name: string;
+  weight: number;
+  description?: string;
+};
+/**
+ * A judge that answers with one of these choices, in this order:
+ * `choices({ A: ['Accurate and helpful', 1], B: ['Partly right', 0.5], C: ['Wrong', 0] })`.
+ */
+export declare function choices(options: Record<string, readonly [label: string, value: number]>): JudgeAnswer;
+/**
+ * Criteria by name: a number is the weight; `{ weight, description }` also
+ * says what the criterion checks. Weights default to 1.
+ */
+type CriteriaOptions = Record<string, number | {
+  weight?: number;
+  description?: string;
+}>;
+/**
+ * A judge that scores each criterion from 0 to 1; the score is the weighted
+ * mean. A number is the weight; `{ weight, description }` also says what the
+ * criterion means. Weights default to 1.
+ */
+export declare function criteria(options: CriteriaOptions): JudgeAnswer;
+/**
+ * One criterion's result, as a scorer returns it and as results record it:
+ * a value with what it saw, or `reached: false` when the check never got to
+ * run, because something before it failed (say, the app would not start).
+ * A criterion not reached counts as 0; the score's `explanation` says why,
+ * once, instead of every criterion repeating it.
+ */
+type CriterionResult = ReachedCriterion | UnreachedCriterion;
+type ReachedCriterion = {
+  name: string;
+  /** From 0 to 1. */
+  value: number;
+  /** Default 1, or the declared weight. */
+  weight?: number;
+  /** Why this value. */
+  explanation?: string;
+  /** What the check observed, as plain data. */
+  evidence?: JsonValue;
+};
+type UnreachedCriterion = {
+  name: string;
+  reached: false;
+  weight?: number;
+  explanation?: string;
+};
+/** A criterion as a scorer returns it: its evidence may be any plain value. */
+type CriterionInput = (Omit<ReachedCriterion, 'evidence'> & {
+  evidence?: unknown;
+}) | UnreachedCriterion;
+/**
+ * The pass mark in words: `only 1`, `≥ 0.5`, or for choices the ones that
+ * pass, such as `A or B`.
+ */
+export declare function passMark(passAt: number | undefined, answer?: JudgeAnswer): string;
+//#endregion
 //#region ../core/src/catalog.d.ts
 /**
  * What the dashboard's Library shows about each definition. Only definition
@@ -184,6 +271,8 @@ export declare function turnResultEvents(result: TurnResult, turn: number, now:
 export type CatalogOptions = {
   /** Project root; definition sites are reported relative to it, never absolute. */
   root?: string;
+  /** The project's default judge agent (`judge` in evalkit.config.ts). */
+  judge?: AutAdapter;
 };
 export type EvalCatalogAgent = {
   name?: string;
@@ -235,10 +324,20 @@ export type EvalCatalogDataset = {
 export type EvalCatalogScorer = {
   name: string;
   kind: ScoringRule['kind'];
+  /** What it checks, in the author's words. */
+  description?: string;
   passAt?: number;
   supportsPartial?: boolean;
   /** Judges only. */
   rubric?: string;
+  /** Judges only: the agent that judges (its own, or the project's default). */
+  agent?: EvalCatalogAgent;
+  /** Judges only: the names of the inputs it sees. */
+  inputs?: string[];
+  /** Judges only: how it answers. */
+  answer?: JudgeAnswer;
+  /** Judges only: the prompt it receives, with `{input}` placeholders. */
+  prompt?: string;
   definedAt?: string;
 };
 export type EvalCatalogStep =
@@ -355,11 +454,21 @@ type EvalkitConfig = {
     /** Limits for files adapters `attach()`. */
     attachments?: AttachmentLimits;
   };
-  /** Where each run's real workspace lives during a run. Default `_evalkit-sandbox`. */
+  /**
+   * Where each run's real workspace lives. Default: outside the project, in
+   * `<tmp>/evalkit/<project>-<hash>`, so agents cannot read the project or
+   * other runs by looking around. Inside the project, `run` warns.
+   */
   sandboxDir?: string;
   /** Which run workspaces to keep in `sandboxDir` afterwards. Default `'failed'`: only runs that did not pass. */
   keepSandboxes?: KeepSandboxes;
   share?: ShareConfig;
+  /**
+   * The agent for every judge that names none: `judge(name, { rubric })`.
+   * Made with `defineAgent()`; it gets each run's parameters, so a matrix
+   * axis can choose its model.
+   */
+  judge?: AutAdapter;
 };
 type KeepSandboxes = 'failed' | 'all' | 'none';
 type AttachmentLimits = {
@@ -600,15 +709,34 @@ export declare const JudgeRunInfoSchema: Schema.Struct<{
     name: Schema.optional<typeof Schema.String>;
     version: Schema.optional<typeof Schema.String>;
   }>>;
+  model: Schema.optional<typeof Schema.String>;
   usage: Schema.optional<Schema.Struct<{
     inputTokens: Schema.optional<typeof Schema.Number>;
     outputTokens: Schema.optional<typeof Schema.Number>;
     cachedInputTokens: Schema.optional<typeof Schema.Number>;
     totalTokens: Schema.optional<typeof Schema.Number>;
   }>>;
+  prompt: Schema.optional<typeof Schema.String>;
+  reply: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+  attempts: Schema.optional<typeof Schema.Number>;
   events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
 }>;
 export declare const ScoreResultSchema: Schema.Struct<{
+  passAt: Schema.optional<typeof Schema.Number>;
+  choice: Schema.optional<typeof Schema.String>;
+  criteria: Schema.optional<Schema.mutable<Schema.Array$<Schema.Union<[Schema.Struct<{
+    name: typeof Schema.String;
+    value: typeof Schema.Number;
+    weight: Schema.optional<typeof Schema.Number>;
+    explanation: Schema.optional<typeof Schema.String>;
+    evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+  }>, Schema.Struct<{
+    name: typeof Schema.String;
+    reached: Schema.Literal<[false]>;
+    weight: Schema.optional<typeof Schema.Number>;
+    explanation: Schema.optional<typeof Schema.String>;
+  }>]>>>>;
+  attachments: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
   name: typeof Schema.String;
   kind: Schema.Literal<["predicate", "judge"]>;
   value: Schema.optional<typeof Schema.Number>;
@@ -622,12 +750,16 @@ export declare const ScoreResultSchema: Schema.Struct<{
       name: Schema.optional<typeof Schema.String>;
       version: Schema.optional<typeof Schema.String>;
     }>>;
+    model: Schema.optional<typeof Schema.String>;
     usage: Schema.optional<Schema.Struct<{
       inputTokens: Schema.optional<typeof Schema.Number>;
       outputTokens: Schema.optional<typeof Schema.Number>;
       cachedInputTokens: Schema.optional<typeof Schema.Number>;
       totalTokens: Schema.optional<typeof Schema.Number>;
     }>>;
+    prompt: Schema.optional<typeof Schema.String>;
+    reply: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+    attempts: Schema.optional<typeof Schema.Number>;
     events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
   }>>;
   durationMs: typeof Schema.Number;
@@ -639,6 +771,21 @@ export declare const ScoreResultSchema: Schema.Struct<{
   skipped: Schema.optional<Schema.Literal<[true]>>;
 }>;
 export declare const CheckpointResultSchema: Schema.Struct<{
+  passAt: Schema.optional<typeof Schema.Number>;
+  choice: Schema.optional<typeof Schema.String>;
+  criteria: Schema.optional<Schema.mutable<Schema.Array$<Schema.Union<[Schema.Struct<{
+    name: typeof Schema.String;
+    value: typeof Schema.Number;
+    weight: Schema.optional<typeof Schema.Number>;
+    explanation: Schema.optional<typeof Schema.String>;
+    evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+  }>, Schema.Struct<{
+    name: typeof Schema.String;
+    reached: Schema.Literal<[false]>;
+    weight: Schema.optional<typeof Schema.Number>;
+    explanation: Schema.optional<typeof Schema.String>;
+  }>]>>>>;
+  attachments: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
   step: typeof Schema.Number;
   kind: Schema.Literal<["predicate", "judge", "expect-tool-call"]>;
   name: typeof Schema.String;
@@ -654,12 +801,16 @@ export declare const CheckpointResultSchema: Schema.Struct<{
       name: Schema.optional<typeof Schema.String>;
       version: Schema.optional<typeof Schema.String>;
     }>>;
+    model: Schema.optional<typeof Schema.String>;
     usage: Schema.optional<Schema.Struct<{
       inputTokens: Schema.optional<typeof Schema.Number>;
       outputTokens: Schema.optional<typeof Schema.Number>;
       cachedInputTokens: Schema.optional<typeof Schema.Number>;
       totalTokens: Schema.optional<typeof Schema.Number>;
     }>>;
+    prompt: Schema.optional<typeof Schema.String>;
+    reply: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+    attempts: Schema.optional<typeof Schema.Number>;
     events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
   }>>;
   durationMs: Schema.optional<typeof Schema.Number>;
@@ -687,6 +838,21 @@ export declare const MetricResultSchema: Schema.Struct<{
 }>;
 export declare const RunScoringSchema: Schema.Struct<{
   results: Schema.mutable<Schema.Array$<Schema.Struct<{
+    passAt: Schema.optional<typeof Schema.Number>;
+    choice: Schema.optional<typeof Schema.String>;
+    criteria: Schema.optional<Schema.mutable<Schema.Array$<Schema.Union<[Schema.Struct<{
+      name: typeof Schema.String;
+      value: typeof Schema.Number;
+      weight: Schema.optional<typeof Schema.Number>;
+      explanation: Schema.optional<typeof Schema.String>;
+      evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+    }>, Schema.Struct<{
+      name: typeof Schema.String;
+      reached: Schema.Literal<[false]>;
+      weight: Schema.optional<typeof Schema.Number>;
+      explanation: Schema.optional<typeof Schema.String>;
+    }>]>>>>;
+    attachments: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
     name: typeof Schema.String;
     kind: Schema.Literal<["predicate", "judge"]>;
     value: Schema.optional<typeof Schema.Number>;
@@ -700,12 +866,16 @@ export declare const RunScoringSchema: Schema.Struct<{
         name: Schema.optional<typeof Schema.String>;
         version: Schema.optional<typeof Schema.String>;
       }>>;
+      model: Schema.optional<typeof Schema.String>;
       usage: Schema.optional<Schema.Struct<{
         inputTokens: Schema.optional<typeof Schema.Number>;
         outputTokens: Schema.optional<typeof Schema.Number>;
         cachedInputTokens: Schema.optional<typeof Schema.Number>;
         totalTokens: Schema.optional<typeof Schema.Number>;
       }>>;
+      prompt: Schema.optional<typeof Schema.String>;
+      reply: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+      attempts: Schema.optional<typeof Schema.Number>;
       events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
     }>>;
     durationMs: typeof Schema.Number;
@@ -729,6 +899,21 @@ export declare const RunScoringSchema: Schema.Struct<{
     }>>;
   }>>>>;
   checkpoints: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
+    passAt: Schema.optional<typeof Schema.Number>;
+    choice: Schema.optional<typeof Schema.String>;
+    criteria: Schema.optional<Schema.mutable<Schema.Array$<Schema.Union<[Schema.Struct<{
+      name: typeof Schema.String;
+      value: typeof Schema.Number;
+      weight: Schema.optional<typeof Schema.Number>;
+      explanation: Schema.optional<typeof Schema.String>;
+      evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+    }>, Schema.Struct<{
+      name: typeof Schema.String;
+      reached: Schema.Literal<[false]>;
+      weight: Schema.optional<typeof Schema.Number>;
+      explanation: Schema.optional<typeof Schema.String>;
+    }>]>>>>;
+    attachments: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
     step: typeof Schema.Number;
     kind: Schema.Literal<["predicate", "judge", "expect-tool-call"]>;
     name: typeof Schema.String;
@@ -744,12 +929,16 @@ export declare const RunScoringSchema: Schema.Struct<{
         name: Schema.optional<typeof Schema.String>;
         version: Schema.optional<typeof Schema.String>;
       }>>;
+      model: Schema.optional<typeof Schema.String>;
       usage: Schema.optional<Schema.Struct<{
         inputTokens: Schema.optional<typeof Schema.Number>;
         outputTokens: Schema.optional<typeof Schema.Number>;
         cachedInputTokens: Schema.optional<typeof Schema.Number>;
         totalTokens: Schema.optional<typeof Schema.Number>;
       }>>;
+      prompt: Schema.optional<typeof Schema.String>;
+      reply: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+      attempts: Schema.optional<typeof Schema.Number>;
       events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
     }>>;
     durationMs: Schema.optional<typeof Schema.Number>;
@@ -783,6 +972,21 @@ export declare const RunSummarySchema: Schema.Struct<{
   durationMs: Schema.optional<typeof Schema.Number>;
   scoring: Schema.optional<Schema.Struct<{
     results: Schema.mutable<Schema.Array$<Schema.Struct<{
+      passAt: Schema.optional<typeof Schema.Number>;
+      choice: Schema.optional<typeof Schema.String>;
+      criteria: Schema.optional<Schema.mutable<Schema.Array$<Schema.Union<[Schema.Struct<{
+        name: typeof Schema.String;
+        value: typeof Schema.Number;
+        weight: Schema.optional<typeof Schema.Number>;
+        explanation: Schema.optional<typeof Schema.String>;
+        evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+      }>, Schema.Struct<{
+        name: typeof Schema.String;
+        reached: Schema.Literal<[false]>;
+        weight: Schema.optional<typeof Schema.Number>;
+        explanation: Schema.optional<typeof Schema.String>;
+      }>]>>>>;
+      attachments: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
       name: typeof Schema.String;
       kind: Schema.Literal<["predicate", "judge"]>;
       value: Schema.optional<typeof Schema.Number>;
@@ -796,12 +1000,16 @@ export declare const RunSummarySchema: Schema.Struct<{
           name: Schema.optional<typeof Schema.String>;
           version: Schema.optional<typeof Schema.String>;
         }>>;
+        model: Schema.optional<typeof Schema.String>;
         usage: Schema.optional<Schema.Struct<{
           inputTokens: Schema.optional<typeof Schema.Number>;
           outputTokens: Schema.optional<typeof Schema.Number>;
           cachedInputTokens: Schema.optional<typeof Schema.Number>;
           totalTokens: Schema.optional<typeof Schema.Number>;
         }>>;
+        prompt: Schema.optional<typeof Schema.String>;
+        reply: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+        attempts: Schema.optional<typeof Schema.Number>;
         events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
       }>>;
       durationMs: typeof Schema.Number;
@@ -825,6 +1033,21 @@ export declare const RunSummarySchema: Schema.Struct<{
       }>>;
     }>>>>;
     checkpoints: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
+      passAt: Schema.optional<typeof Schema.Number>;
+      choice: Schema.optional<typeof Schema.String>;
+      criteria: Schema.optional<Schema.mutable<Schema.Array$<Schema.Union<[Schema.Struct<{
+        name: typeof Schema.String;
+        value: typeof Schema.Number;
+        weight: Schema.optional<typeof Schema.Number>;
+        explanation: Schema.optional<typeof Schema.String>;
+        evidence: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+      }>, Schema.Struct<{
+        name: typeof Schema.String;
+        reached: Schema.Literal<[false]>;
+        weight: Schema.optional<typeof Schema.Number>;
+        explanation: Schema.optional<typeof Schema.String>;
+      }>]>>>>;
+      attachments: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
       step: typeof Schema.Number;
       kind: Schema.Literal<["predicate", "judge", "expect-tool-call"]>;
       name: typeof Schema.String;
@@ -840,12 +1063,16 @@ export declare const RunSummarySchema: Schema.Struct<{
           name: Schema.optional<typeof Schema.String>;
           version: Schema.optional<typeof Schema.String>;
         }>>;
+        model: Schema.optional<typeof Schema.String>;
         usage: Schema.optional<Schema.Struct<{
           inputTokens: Schema.optional<typeof Schema.Number>;
           outputTokens: Schema.optional<typeof Schema.Number>;
           cachedInputTokens: Schema.optional<typeof Schema.Number>;
           totalTokens: Schema.optional<typeof Schema.Number>;
         }>>;
+        prompt: Schema.optional<typeof Schema.String>;
+        reply: Schema.optional<Schema.Schema<JsonValue, JsonValue, never>>;
+        attempts: Schema.optional<typeof Schema.Number>;
         events: Schema.optional<Schema.mutable<Schema.Array$<Schema.Schema<JsonValue, JsonValue, never>>>>;
       }>>;
       durationMs: Schema.optional<typeof Schema.Number>;
@@ -949,6 +1176,8 @@ export type AutEvent = {
   timestamp: string;
   latencyMs?: number;
   usage?: Usage;
+  /** The model that answered this turn, when the agent knows it. */
+  model?: string;
 } | {
   kind: 'completed';
   output?: JsonValue;
@@ -1136,9 +1365,23 @@ export declare function expectToolCall(expected: ExpectToolCallStep['expected'])
  */
 export declare function user(message: UserStep['message']): UserStep;
 export declare function agent(expectation: JsonObject): AgentStep;
+/**
+ * What a judge saw and said. The built-in `judge()` records all of it; a
+ * predicate that calls a model itself can report `model`, `prompt` and
+ * `reply` so the dashboard shows them the same way.
+ */
 export type JudgeRunInfo = {
   agent?: AutIdentity;
+  /** The model the judge reported using (`turn-completed` `model`). */
+  model?: string;
+  /** Tokens the judge used: never counted in the agent's `tokens`. */
   usage?: Usage;
+  /** The exact prompt the judge received. */
+  prompt?: string;
+  /** Its reply, as text or JSON. */
+  reply?: JsonValue;
+  /** 2 when its first reply was not in the format asked for and it was asked again. */
+  attempts?: number;
   /** Judge-agent observations; never attributed to the AUT trajectory. */
   events?: JsonValue[];
 };
@@ -1148,7 +1391,8 @@ export type SkippedScore = {
   reason: string;
 };
 export type ScoreValue<Evidence = unknown> = number | SkippedScore | {
-  value: number;
+  /** From 0 to 1. Without it, the weighted mean of `criteria`. */
+  value?: number;
   passed?: boolean;
   explanation?: string;
   /**
@@ -1156,7 +1400,12 @@ export type ScoreValue<Evidence = unknown> = number | SkippedScore | {
    * evidence reaches later scorers through `checkpoint(rule)`.
    */
   evidence?: Evidence;
-  /** Judge-agent evidence, never inferred from AUT usage. */
+  /**
+   * Per-criterion results, each shown on its own; see `CriterionResult`.
+   * Return `{ name, reached: false }` for a check that never ran.
+   */
+  criteria?: CriterionInput[];
+  /** What a model-backed scorer saw and said; never counted as AUT usage. */
   judge?: JudgeRunInfo;
 };
 /**
@@ -1169,8 +1418,15 @@ export type ScoreResult = {
   kind: 'predicate' | 'judge';
   value?: number;
   passed?: boolean;
+  /** The pass mark the value was held to (1 unless the rule set `passAt`). */
+  passAt?: number;
   explanation?: string;
   evidence?: JsonValue;
+  /** A `choices` judge's answer, such as `B`. */
+  choice?: string;
+  criteria?: CriterionResult[];
+  /** Files its scorer saved with `attach()`, by name. */
+  attachments?: string[];
   judge?: JudgeRunInfo;
   durationMs: number;
   error?: RecordedError;
@@ -1225,21 +1481,60 @@ export type CheckpointOutcome<Evidence> = Omit<CheckpointResult, 'evidence'> & {
 export type PredicateScorer<Evidence = unknown> = {
   kind: 'predicate';
   name: string;
+  /** What it checks, in a sentence, shown wherever the scorer is. */
+  description?: string;
   supportsPartial?: boolean;
   /** A numeric result passes when it is at least this value. Default: only 1 passes. */
   passAt?: number;
+  /**
+   * The checks it reports, declared up front so they show before a run.
+   * A declared criterion it leaves out is recorded as not reached; one it
+   * did not declare is an error.
+   */
+  criteria?: JudgeCriterion[];
   run(context: ScoringContext): boolean | ScoreValue<Evidence> | Promise<boolean | ScoreValue<Evidence>>;
 };
+/** What a judge sees: one named input, read from the run. */
+export type JudgeInput = (scoring: ScoringContext) => JsonValue | undefined | Promise<JsonValue | undefined>;
+/**
+ * A rubric-graded scorer. EvalKit builds the prompt from the rubric, the
+ * inputs and the answer format, runs the judge agent, checks its answer
+ * (asking once more if it is not in the format asked for), and records
+ * what it saw and said.
+ */
 export type JudgeRule = {
   kind: 'judge';
   name: string;
   rubric: string;
+  /** What it checks, in a sentence, shown wherever the judge is. */
+  description?: string;
+  /**
+   * The agent that judges, made with `defineAgent()`. Default: `judge` in
+   * evalkit.config.ts. It gets the run's parameters, so a matrix axis can
+   * choose its model.
+   */
+  agent?: AutAdapter;
+  /**
+   * What the judge sees, by name. Default: `conversation` (this turn at a
+   * checkpoint, the whole run in final scoring) and `reply` (the last reply).
+   */
+  inputs?: Readonly<Record<string, JudgeInput>>;
+  /** How it answers. Default `verdict`: a value from 0 to 1 and a reason. */
+  answer: JudgeAnswer;
+  /** A value passes when it is at least this. Default: only 1 passes. */
+  passAt?: number;
   /** Run even when the scenario stopped early or execution failed. */
   supportsPartial?: boolean;
 };
 export type ScoringRule = PredicateScorer | JudgeRule;
 /** A scorer in code. Its `evidence` type carries through to `checkpoint(rule)`. */
-export declare function predicate<Evidence = unknown>(name: string, run: PredicateScorer<Evidence>['run'], options?: Pick<PredicateScorer, 'supportsPartial' | 'passAt'>): PredicateScorer<Evidence>;
+export declare function predicate<Evidence = unknown>(name: string, run: PredicateScorer<Evidence>['run'], options?: Pick<PredicateScorer, 'supportsPartial' | 'passAt' | 'description'> & {
+  /** The checks it reports by name: `{ 'cannot push': 'the share token gets 403' }` or `{ name: { weight, description } }`. */
+  criteria?: Record<string, string | number | {
+    weight?: number;
+    description?: string;
+  }>;
+}): PredicateScorer<Evidence>;
 /**
  * A number measured for every run, such as tokens, retries or web searches.
  * Metrics never decide pass or fail; they are averaged per run and shown on
@@ -1267,7 +1562,9 @@ export type MetricResult = {
   builtIn?: true;
   error?: RecordedError;
 };
-export declare function judge(name: string, options: Pick<JudgeRule, 'rubric' | 'supportsPartial'>): JudgeRule;
+export declare function judge(name: string, options: Omit<JudgeRule, 'kind' | 'name' | 'answer'> & {
+  answer?: JudgeAnswer;
+}): JudgeRule;
 export type EvalPolicy = {
   timeoutMs?: number;
   /** Stop later transcript steps after a failed checkpoint assertion. Default false. */
@@ -1277,8 +1574,6 @@ export type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
   id: string;
   name?: string;
   agent: TAgent;
-  /** Separate judge agent; its model and tools are never inherited from the AUT. */
-  judge?: AutAdapter;
   /** Data and files the scenario uses; each run gets its own copy. */
   datasets?: readonly Dataset[];
   transcript: TranscriptStep[];
@@ -1308,6 +1603,10 @@ export type EvalWorkspace = {
   ignore?: readonly string[];
 };
 export declare function defineEval<const T extends EvalDefinition>(definition: T): T;
+/** A judge never grades with the agent it is judging. */
+export declare function assertJudgesIndependent(definition: Pick<EvalDefinition, 'agent' | 'transcript' | 'scoring'> & {
+  id: string;
+}, defaultJudge?: AutAdapter): void;
 export type EvalSuite<TEvals extends readonly EvalDefinition[] = readonly EvalDefinition[]> = {
   id: string;
   name?: string;
@@ -1427,8 +1726,13 @@ export type CheckpointResult = {
   status: 'passed' | 'failed' | 'error' | 'skipped';
   value?: number;
   passed?: boolean;
+  passAt?: number;
   explanation?: string;
   evidence?: JsonValue;
+  choice?: string;
+  criteria?: CriterionResult[];
+  /** Files its scorer saved with `attach()`, by name. */
+  attachments?: string[];
   judge?: JudgeRunInfo;
   durationMs?: number;
   error?: RecordedError;
@@ -1447,7 +1751,7 @@ export type CheckpointResult = {
  *   usage, cached input included. Absent when no turn reported any.
  * - `tool calls`: the agent's `tool-call` events.
  */
-export declare const BUILT_IN_METRICS: readonly ['agent time', 'tokens', 'tool calls'];
+export declare const BUILT_IN_METRICS: readonly ['agent time', 'tokens', 'tool calls', 'judge tokens'];
 export type BuiltInMetricName = (typeof BUILT_IN_METRICS)[number];
 /** A built-in metric's name (suggested by editors), or a scenario's own. */
 export type MetricName = BuiltInMetricName | (string & {});
@@ -1518,5 +1822,5 @@ type PiAutOptions = {
 /** A local, one-prompt-per-turn AUT backed by Pi's non-interactive print mode. */
 export declare function piAgent(options?: PiAutOptions): AutAdapter;
 //#endregion
-export type { AttachmentLimits, DefinitionSite, EvalMatrix, EvalMatrixCell, EvalkitConfig, KeepSandboxes, MatrixSelection, ShareConfig, ShareHeaders, Usage };
+export type { AttachmentLimits, CriteriaOptions, CriterionInput, CriterionResult, DefinitionSite, EvalMatrix, EvalMatrixCell, EvalkitConfig, JudgeAnswer, JudgeChoice, JudgeCriterion, KeepSandboxes, MatrixSelection, ReachedCriterion, ShareConfig, ShareHeaders, UnreachedCriterion, Usage };
 //# sourceMappingURL=index.d.mts.map

package/dist/index.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.mts","names":[],"sources":["../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/catalog.ts","../../core/src/canonical-json.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/usage.ts","../../core/src/json.ts","../../core/src/definition-site.ts","../../core/src/schema.ts","../../core/src/trajectory.ts","../../core/src/index.ts","../../agents/src/index.ts"],"mappings":";;qBAEa,oBAAkB,OAAA;YACnB,sBAAsB,mBAAmB;;wBAGrC,SAAS;;qBAOZ,YAAU,OAAA,cAAA,OAAA;YAKX,cAAc,WAAW;YAEzB,YAAY,cAAc,eAAe,2BACxC;qBAEA,mBAAiB,OAAA,cAAA,OAAA;wBAMd,kBAAkB,cAAc,cAAc,MAAM,QAAK,OAAA,cAAA,OAAA;wBAWzD,YAAY,cAAc,cACxC,MAAM,OACN,MAAM,OACL,YAAY;wBAIC,iBAAiB,cAAc,cAC7C,eACA,eAAe;EACZ,MAAM;EAAO,MAAM;EAAM,KAAK,YAAY;;wBAY/B,aAAa,eAAe,eAAe,eAAe;;;;YCvD9D;;YAGA,6BAA6B;EAAe;;;;;;YAM5C,wBAAwB,SAAS,eAAe;;YAGhD;EACV,QAAQ;;EAER;;YAGU;;EAEV;EACA,QAAQ;;YAGE;EACV;EACA;;EAEA,QAAQ;;EAER;;EAEA,YAAY;EACZ,QAAQ;;KAGL;EACH;EACA;;EAEA,aAAa;;EAEb;;YAGU,0BAA0B;EACpC,QAAQ;;EAER;;EAEA;EACA;EACA;;YAGU,0BAA0B,gBAAgB,kBACpD;;EAEE,SAAS,MAAM,uBAAuB,UAAU,QAAQ;;EAExD,SACE,QAAQ,SACR,KAAK,sBACF,iBAAiB,QAAQ;EAC9B;EACA;EACA;;YAGQ;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,KAAK,sBACF,iBAAiB,QAAQ;;;YAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;wBAIH,oBAAoB,eAAe;;;;;;;wBAuBnC,cAAc,gBAAgB,gBAC5C,YAAY,0BAA0B,0BAA0B,WAC/D;wBAmFa,UAAU,iBAAiB,SAAS;;;;;;;YC/MxC;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;YAE/B;EACV;EACA;;EAEA;EACA;;;YAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;;wBAIE,kBACd,OAAO,eACP,WAAW,OACV;;;;;wBA0Da,iBACd,QAAQ,YACR,cACA,WAAW,OACV;;;;;;;;;YCtGS;;EAEV;;YAGU;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;YAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;YAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;YAGU;EACV;EACA,MAAM;EACN;EACA;;EAEA;EACA;;YAGU;;;EAEN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;YAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;;EAEV;EACA;IAAU;IAAoB;;EAC9B,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;wBAGc,aACd,YAAY,gBACZ,OAAO,uBACP,UAAS,iBACR;;;;wBC7Fa,oBAAoB,OAAO;;;KCK/B;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;wBAWvC,uBAAuB,UAAU,sBAC/C,YAAY,IACX,aAAa;;;KCjDJ;;EAEV;EACA;EACA;EACA;;IAEE;;;EAGF,iBAAiB;EACjB,WAAW;;EAEX,SAAS,KAAK;IAA0C;;EACxD;IAAc;IAAsB;IAAiB;;EACrD;;EAEA;;;;;;;IAOE;;IAEA,cAAc;;;EAGhB;;EAEA,gBAAgB;EAChB,QAAQ;;KAGE;KAEA;;EAEV;;EAEA;;qBAGW,2BAA2B,SAAS;;qBAMpC;;KAWD,eAAe;KAEf;;EAEV;;;;;;EAMA,UAAU,sBAAsB,eAAe,QAAQ;;;wBAIzC,mBAAmB,UAAU,eAC3C,eAAe,IACd;;;;;;;KC5ES;;;;;;EAMV;;EAEA;;EAEA;;EAEA;;;;;;;wBAQc,WAAW,OAAO;;;;qBCrBrB,qBAAqB;WAGrB;WACA;WAHO;EAClB,YACW,cACA;;;;;;;;wBAYG,OAAO,gBAAgB,gBAAiB;;wBA4CxC,aACd,gBACA;GACI,cAAc;;;;;;;;;;;KC3DR;EAAmB;EAAc;;wBAqB7B,eAAe,iBAAiB;;;;qBCvBnC,iBAAe,OAAA;;qBASf;qBAEA,qBAAmB,OAAA;;;;;qBAMnB,mBAAiB,OAAA;;;;;;qBAQjB,iBAAiB,OAAO,OAAO;qBAoB/B,iBAAe,OAAA;;;;;;;;;;;;;;qBAiBf,aAAW,OAAA;;;;;qBAMX,kBAAgB,OAAA;;;;;;qBAOhB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAoBjB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAKjB,aAAW,OAAA;;;;;;qBAOX,oBAAkB,OAAA;;;;;;;;;;;;;;;qBAalB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAYjB,wBAAsB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAgBtB,oBAAkB,OAAA;;;;;;;;;;;;qBAQlB,kBAAgB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAUhB,qBAAmB,OAAA;;;;;qBAKnB,uBAAqB,OAAA;;;;;qBAKrB,kBAAgB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAoJhB,uBAAuB,OAAO,OAAO;qBAGrC,gBAAc,OAAA;;;;;;;wBCvUX,qBAAqB,mBAAmB;;;YCmB5C;YACA,YACV,gBAAgB;GAAiB,cAAc;;YACrC;GAAgB,cAAc;;YAE9B;;EAEV;EACA;EACA;EACA;;YAGU;;;;;YAMA;EACV;EACA,gBAAgB;;YAGN,gBAAgB,QAAQ,OAAO,kBAAkB;YAKjD;EACV;EACA;EACA;;YAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;YAE/B;EACN;EAAqB;;EAErB;EACA;;EAEA;EACA;;EAEA;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAuB;;EACvB;EAAe,OAAO;EAAe;;YAE/B,mBACT;EAAa;MAAoB;EAAgB;;YAExC;;EAEV;;YAGU,qBAAqB;YACrB,qBAAqB;YAErB;EACV;EACA;;EAEA,QAAQ;EACR,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;EAEN,WAAW;;EAEX,oBAAoB;;YAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;IAEV,UAAU,OAAO,yBAAyB;;;;;;;;IAQ1C,SAAS,cAAc,kBAAkB,eAAe;MACtD,QAAQ,WAAW;;YAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;;qBAIC,wBAAwB;;wBAKrB,cAAc,OAAO,kBAAkB;;;;;wBAUvC,kBAAkB,UAAU,YAAY,OAAO,IAAI;;YAMvD;;EAEV,YAAY;;EAEZ,MAAM;;EAEN;;EAEA,QAAQ;;YAGE;EACV;;EAEA,oBAAoB,OAAO;;YAGjB;EACV;EACA,aAAa;;YAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;YAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;YAE5B,iBACV,WAAW,YAAY,cAAc;;wBAGvB,eACd,UAAU,iCACT;;;;;wBAUa,KAAK,SAAS,sBAAsB;wBAIpC,MAAM,aAAa,aAAa;YAIpC;EACV,QAAQ;EACR,QAAQ;;EAER,SAAS;;;YAIC;EAAiB;EAAe;;YAEhC,WAAW,+BAEnB;EAEE;EACA;EACA;;;;;EAKA,WAAW;;EAEX,QAAQ;;;;;;wBAOE,KAAK,iBAAiB;YAK1B;EACV;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;;EAER;;YAGU;EACV,WAAW;EACX,WAAW;;YAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;EAEP,SAAS;;EAET,MAAM;;;;;;EAMN,sBAAsB;;;;;;;EAOtB,WAAW,UACT,MAAM,gBAAgB,YACrB,kBAAkB;EACrB,WACE,MAAM,YAAY,qBACjB,kBAAkB;;;;;;EAMrB,OAAO,cAAc,kBAAkB,aAAa;;;;;;;YAQ1C,kBAAkB,YAAY,KAAK;EAC7C,WAAW;;YAGD,gBAAgB;EAC1B;EACA;EACA;;EAEA;EACA,IACE,SAAS,2BACE,WAAW,YAAY,kBAAkB,WAAW;;YAGvD;EACV;EACA;EACA;;EAEA;;YAGU,cAAc,kBAAkB;;wBAG5B,UAAU,oBACxB,cACA,KAAK,gBAAgB,kBACrB,UAAS,KAAK,iDACb,gBAAgB;;;;;;YAqBP;EACV;EACA;;EAEA;;EAEA;EACA,IACE,SAAS,sCACa;;;wBAIV,aAAa,qBAC3B,MAAM,aAAa,4BAA4B,MAC/C,KAAK,eACL,UAAS,KAAK,6BACb;;YAMS;EACV,MAAM;;EAEN;EACA;EACA;;EAEA;EACA,QAAQ;;wBAGM,MACd,cACA,SAAS,KAAK,2CACb;YAMS;EACV;;EAEA;;YAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,QAAQ;;EAER,oBAAoB;EACpB,YAAY;EACZ,SAAS;;EAET,mBAAmB;EACnB,SAAS;;;;;EAKT;;EAEA,WAAW;EACX,YAAY;;;YAIF;;;;;;;;;EASV;;wBAGc,iBAAiB,UAAU,gBAAgB,YAAY,IAAI;YAuD/D,UACV,wBAAwB,4BAA4B;EAEpD;EACA;EACA,OAAO;;wBAGO,kBAAkB,eAAe,WAC/C,OAAO,SACN;YAKS,mBAAmB,iBAAiB,YAAY;wBAE5C,YAAY;EAAS;;YAkBzB;;EAEV,gBAAgB;EAChB,iBAAiB;EACjB,mBAAmB;EACnB,IAAI,cAAc;EAClB,SAAS,cAAc;EACvB,YAAY;EACZ,iBAAiB;;EAEjB,QAAQ,UAAU,iBAAiB;;YAGzB;EACV;EACA;EACA;;YAGU;EACV;EACA;EACA;;;;;;wBAOc,oBACR,gCAAgC,oBACtC,eAAe,iBAAiB;;;;;YA2EtB;;YAIA;EACV;;EAEA;EACA;;;;;;YAOU;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;YAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;YAIE;EAAe;EAAa;;YAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;EACT,UAAU;;EAEV,QAAQ;;EAER;;EAEA,WAAW;;YAGD;EACV;EACA,YAAY;EACZ;;EAEA;;YAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;;;;;;;;;;;qBAa7B;YACD,4BAA4B;;YAE5B,aAAa;YAEb;EACV,SAAS;EACT,UAAU;EACV,cAAc;EACd;EACA;EACA;;YAGU;;EAEV;EACA;EACA;;;YAIU;EACV;;EAEA;;EAEA;;YAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,cAAc;EACd,QAAQ;;YAGE;WACD;EACT,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,aAAa;;EAElC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,aAAa;;YAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;;YAID;EACV;EACA,QAAQ;EACR;EACA,QAAQ;EACR,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;wBAYM,YAAY,iBAAiB;;;KCp2BjC;EACV;;EAEA;;;wBAIc,QAAQ,UAAS,eAAoB"}
+{"version":3,"file":"index.d.mts","names":[],"sources":["../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/judge.ts","../../core/src/catalog.ts","../../core/src/canonical-json.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/usage.ts","../../core/src/json.ts","../../core/src/definition-site.ts","../../core/src/schema.ts","../../core/src/trajectory.ts","../../core/src/index.ts","../../agents/src/index.ts"],"mappings":";;qBAEa,oBAAkB,OAAA;YACnB,sBAAsB,mBAAmB;;wBAGrC,SAAS;;qBAOZ,YAAU,OAAA,cAAA,OAAA;YAKX,cAAc,WAAW;YAEzB,YAAY,cAAc,eAAe,2BACxC;qBAEA,mBAAiB,OAAA,cAAA,OAAA;wBAMd,kBAAkB,cAAc,cAAc,MAAM,QAAK,OAAA,cAAA,OAAA;wBAWzD,YAAY,cAAc,cACxC,MAAM,OACN,MAAM,OACL,YAAY;wBAIC,iBAAiB,cAAc,cAC7C,eACA,eAAe;EACZ,MAAM;EAAO,MAAM;EAAM,KAAK,YAAY;;wBAY/B,aAAa,eAAe,eAAe,eAAe;;;;YCvD9D;;YAGA,6BAA6B;EAAe;;;;;;YAM5C,wBAAwB,SAAS,eAAe;;YAGhD;EACV,QAAQ;;EAER;;YAGU;;EAEV;EACA,QAAQ;;YAGE;EACV;EACA;;EAEA,QAAQ;;EAER;;EAEA,YAAY;EACZ,QAAQ;;KAGL;EACH;EACA;;EAEA,aAAa;;EAEb;;YAGU,0BAA0B;EACpC,QAAQ;;EAER;;EAEA;EACA;EACA;;YAGU,0BAA0B,gBAAgB,kBACpD;;EAEE,SAAS,MAAM,uBAAuB,UAAU,QAAQ;;EAExD,SACE,QAAQ,SACR,KAAK,sBACF,iBAAiB,QAAQ;EAC9B;EACA;EACA;;YAGQ;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,KAAK,sBACF,iBAAiB,QAAQ;;;YAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;wBAIH,oBAAoB,eAAe;;;;;;;wBAuBnC,cAAc,gBAAgB,gBAC5C,YAAY,0BAA0B,0BAA0B,WAC/D;wBAmFa,UAAU,iBAAiB,SAAS;;;;;;;YC/MxC;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;;EAEA;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;YAE/B;EACV;EACA;;EAEA;EACA;;;YAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;EAER;;;wBAIU,kBACd,OAAO,eACP,WAAW,OACV;;;;;wBA0Da,iBACd,QAAQ,YACR,cACA,WAAW,OACV;;;;;;;;;;KCtHS;EACN;;EACA;EAAiB,SAAS;;EAC1B;EAAkB,UAAU;;;KAGtB;EAAgB;EAAa;EAAe;;;KAG5C;EACV;EACA;EACA;;;;;;wBAkBc,QACd,SAAS,yBAAyB,eAAe,kBAChD;;;;;KA6BS,kBAAkB;EAEjB;EAAiB;;;;;;;wBAQd,SAAS,SAAS,kBAAkB;;;;;;;;KA4BxC,kBAAkB,mBAAmB;KAErC;EACV;;EAEA;;EAEA;;EAEA;;EAEA,WAAW;;KAGD;EACV;EACA;EACA;EACA;;;KAIU,kBACP,KAAK;EAAkC;KACxC;;;;;wBAkNY,SACd,4BACA,SAAS;;;;;;;;;YCjUC;;EAEV;;EAEA,QAAQ;;YAGE;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;YAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;YAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;YAGU;EACV;EACA,MAAM;;EAEN;EACA;EACA;;EAEA;;EAEA,QAAQ;;EAER;;EAEA,SAAS;;EAET;EACA;;YAGU;;;EAEN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;YAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;;EAEV;EACA;IAAU;IAAoB;;EAC9B,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;wBAGc,aACd,YAAY,gBACZ,OAAO,uBACP,UAAS,iBACR;;;;wBC3Ga,oBAAoB,OAAO;;;KCK/B;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;wBAWvC,uBAAuB,UAAU,sBAC/C,YAAY,IACX,aAAa;;;KCjDJ;;EAEV;EACA;EACA;EACA;;IAEE;;;EAGF,iBAAiB;EACjB,WAAW;;EAEX,SAAS,KAAK;IAA0C;;EACxD;IAAc;IAAsB;IAAiB;;EACrD;;EAEA;;;;;;;IAOE;;IAEA,cAAc;;;;;;;EAOhB;;EAEA,gBAAgB;EAChB,QAAQ;;;;;;EAMR,QAAQ;;KAGE;KAEA;;EAEV;;EAEA;;qBAGW,2BAA2B,SAAS;;qBAMpC;;KAWD,eAAe;KAEf;;EAEV;;;;;;EAMA,UAAU,sBAAsB,eAAe,QAAQ;;;wBAIzC,mBAAmB,UAAU,eAC3C,eAAe,IACd;;;;;;;KCtFS;;;;;;EAMV;;EAEA;;EAEA;;EAEA;;;;;;;wBAQc,WAAW,OAAO;;;;qBCrBrB,qBAAqB;WAGrB;WACA;WAHO;EAClB,YACW,cACA;;;;;;;;wBAYG,OAAO,gBAAgB,gBAAiB;;wBA4CxC,aACd,gBACA;GACI,cAAc;;;;;;;;;;;KC3DR;EAAmB;EAAc;;wBAqB7B,eAAe,iBAAiB;;;;qBCvBnC,iBAAe,OAAA;;qBASf;qBAEA,qBAAmB,OAAA;;;;;qBAMnB,mBAAiB,OAAA;;;;;;qBAQjB,iBAAiB,OAAO,OAAO;qBAoB/B,iBAAe,OAAA;;;;;;;;;;;;;;qBAiBf,aAAW,OAAA;;;;;qBAMX,kBAAgB,OAAA;;;;;;qBAOhB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAoBjB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAKjB,aAAW,OAAA;;;;;;qBAOX,oBAAkB,OAAA;;;;;;;;;;;;;;;;;;;qBA2ClB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAajB,wBAAsB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAiBtB,oBAAkB,OAAA;;;;;;;;;;;;qBAQlB,kBAAgB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAUhB,qBAAmB,OAAA;;;;;qBAKnB,uBAAqB,OAAA;;;;;qBAKrB,kBAAgB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAqJhB,uBAAuB,OAAO,OAAO;qBAGrC,gBAAc,OAAA;;;;;;;wBCxWX,qBAAqB,mBAAmB;;;YCwC5C;YACA,YACV,gBAAgB;GAAiB,cAAc;;YACrC;GAAgB,cAAc;;YAE9B;;EAEV;EACA;EACA;EACA;;YAGU;;;;;YAMA;EACV;EACA,gBAAgB;;YAGN,gBAAgB,QAAQ,OAAO,kBAAkB;YAKjD;EACV;EACA;EACA;;YAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;;EAEA;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;YAE/B;EACN;EAAqB;;EAErB;EACA;;EAEA;EACA;;EAEA;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAuB;;EACvB;EAAe,OAAO;EAAe;;YAE/B,mBACT;EAAa;MAAoB;EAAgB;;YAExC;;EAEV;;YAGU,qBAAqB;YACrB,qBAAqB;YAErB;EACV;EACA;;EAEA,QAAQ;EACR,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;EAEN,WAAW;;EAEX,oBAAoB;;YAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;IAEV,UAAU,OAAO,yBAAyB;;;;;;;;IAQ1C,SAAS,cAAc,kBAAkB,eAAe;MACtD,QAAQ,WAAW;;YAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;;qBAIC,wBAAwB;;wBAKrB,cAAc,OAAO,kBAAkB;;;;;wBAUvC,kBAAkB,UAAU,YAAY,OAAO,IAAI;;YAMvD;;EAEV,YAAY;;EAEZ,MAAM;;EAEN;;EAEA,QAAQ;;YAGE;EACV;;EAEA,oBAAoB,OAAO;;YAGjB;EACV;EACA,aAAa;;YAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;YAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;YAE5B,iBACV,WAAW,YAAY,cAAc;;wBAGvB,eACd,UAAU,iCACT;;;;;wBAUa,KAAK,SAAS,sBAAsB;wBAIpC,MAAM,aAAa,aAAa;;;;;;YASpC;EACV,QAAQ;;EAER;;EAEA,QAAQ;;EAER;;EAEA,QAAQ;;EAER;;EAEA,SAAS;;;YAIC;EAAiB;EAAe;;YAEhC,WAAW,+BAEnB;;EAGE;EACA;EACA;;;;;EAKA,WAAW;;;;;EAKX,WAAW;;EAEX,QAAQ;;;;;;wBAOE,KAAK,iBAAiB;YAK1B;EACV;EACA;EACA;EACA;;EAEA;EACA;EACA,WAAW;;EAEX;EACA,WAAW;;EAEX;EACA,QAAQ;EACR;EACA,QAAQ;;EAER;;YAGU;EACV,WAAW;EACX,WAAW;;YAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;EAEP,SAAS;;EAET,MAAM;;;;;;EAMN,sBAAsB;;;;;;;EAOtB,WAAW,UACT,MAAM,gBAAgB,YACrB,kBAAkB;EACrB,WACE,MAAM,YAAY,qBACjB,kBAAkB;;;;;;EAMrB,OAAO,cAAc,kBAAkB,aAAa;;;;;;;YAQ1C,kBAAkB,YAAY,KAAK;EAC7C,WAAW;;YAGD,gBAAgB;EAC1B;EACA;;EAEA;EACA;;EAEA;;;;;;EAMA,WAAW;EACX,IACE,SAAS,2BACE,WAAW,YAAY,kBAAkB,WAAW;;;YAIvD,cACV,SAAS,mBACN,wBAAwB,QAAQ;;;;;;;YAQzB;EACV;EACA;EACA;;EAEA;;;;;;EAMA,QAAQ;;;;;EAKR,SAAS,SAAS,eAAe;;EAEjC,QAAQ;;EAER;;EAEA;;YAGU,cAAc,kBAAkB;;wBAG5B,UAAU,oBACxB,cACA,KAAK,gBAAgB,kBACrB,UAAS,KACP;;EAIA,WAAW;IAEW;IAAiB;;IAGxC,gBAAgB;;;;;;YA6BP;EACV;EACA;;EAEA;;EAEA;EACA,IACE,SAAS,sCACa;;;wBAIV,aAAa,qBAC3B,MAAM,aAAa,4BAA4B,MAC/C,KAAK,eACL,UAAS,KAAK,6BACb;;YAMS;EACV,MAAM;;EAEN;EACA;EACA;;EAEA;EACA,QAAQ;;wBAKM,MACd,cACA,SAAS,KAAK;EACZ,SAAS;IAEV;YA+BS;EACV;;EAEA;;YAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,oBAAoB;EACpB,YAAY;EACZ,SAAS;;EAET,mBAAmB;EACnB,SAAS;;;;;EAKT;;EAEA,WAAW;EACX,YAAY;;;YAIF;;;;;;;;;EASV;;wBAGc,iBAAiB,UAAU,gBAAgB,YAAY,IAAI;;wBA6C3D,wBACd,YAAY,KAAK;EACf;GAEF,eAAe;YAgBL,UACV,wBAAwB,4BAA4B;EAEpD;EACA;EACA,OAAO;;wBAGO,kBAAkB,eAAe,WAC/C,OAAO,SACN;YAKS,mBAAmB,iBAAiB,YAAY;wBAE5C,YAAY;EAAS;;YAkBzB;;EAEV,gBAAgB;EAChB,iBAAiB;EACjB,mBAAmB;EACnB,IAAI,cAAc;EAClB,SAAS,cAAc;EACvB,YAAY;EACZ,iBAAiB;;EAEjB,QAAQ,UAAU,iBAAiB;;YAGzB;EACV;EACA;EACA;;YAGU;EACV;EACA;EACA;;;;;;wBAOc,oBACR,gCAAgC,oBACtC,eAAe,iBAAiB;;;;;YA2EtB;;YAIA;EACV;;EAEA;EACA;;;;;;YAOU;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;YAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;YAIE;EAAe;EAAa;;YAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;EACT,UAAU;;EAEV,QAAQ;;EAER;;EAEA,WAAW;;YAGD;EACV;EACA,YAAY;EACZ;;EAEA;;YAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX;EACA,WAAW;;EAEX;EACA,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;;;;;;;;;;;qBAa7B;YAMD,4BAA4B;;YAE5B,aAAa;YAEb;EACV,SAAS;EACT,UAAU;EACV,cAAc;EACd;EACA;EACA;;YAGU;;EAEV;EACA;EACA;;;YAIU;EACV;;EAEA;;EAEA;;YAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,cAAc;EACd,QAAQ;;YAGE;WACD;EACT,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,aAAa;;EAElC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,aAAa;;YAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;;YAID;EACV;EACA,QAAQ;EACR;EACA,QAAQ;EACR,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;wBAYM,YAAY,iBAAiB;;;KC1/BjC;EACV;;EAEA;;;wBAIc,QAAQ,UAAS,eAAoB"}

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { $ as relativeDatasetPath, A as RunManifestSchema, B as turnResultEvents, C as CheckpointResultSchema, D as REPORT_SCHEMA_VERSION, E as MetricResultSchema, F as RunSummarySchema, G as DEFAULT_ATTACHMENT_LIMITS, H as canonicalParameters, I as ScoreResultSchema, J as NotJsonError, K as defineConfig, L as TrajectoryEventSchema, M as RunScoringSchema, N as RunSourceSchema, O as RecordedErrorSchema, P as RunStatusSchema, Q as isDataset, R as UsageSchema, S as BatchSchema, T as JudgeRunInfoSchema, U as turnTokens, V as defineEvalMatrix, W as DEFAULT_ARTIFACT_EXCLUDE, X as toJsonObject, Y as toJson, Z as defineDataset, _ as parseTrajectoryJsonl, a as authoringId, at as parseResourceUri, b as AttachmentEntrySchema, c as defineSuite, ct as resourceUuid, d as metric, et as definitionSite, f as predicate, g as user, h as skip, i as agentRuntimes, it as authorId, j as RunMetadataSchema, k as RunDatasetSchema, l as expectToolCall, m as registerEvals, n as DEFAULT_AGENT_RUNTIMES, nt as ResourceUriSchema, o as defineAgent, ot as resourceUri, p as recordError, q as catalogEntry, r as agent, rt as UuidSchema, s as defineEval, st as resourceUriSchema, t as BUILT_IN_METRICS, tt as ResourceKindSchema, u as judge, v as ApiErrorSchema, w as JsonValueSchema, x as AutIdentitySchema, y as ArtifactEntrySchema, z as normalizeAutEvent } from "./src-BrfIXfO2.mjs";
+import { A as RunDatasetSchema, B as normalizeAutEvent, C as BatchSchema, D as MetricResultSchema, E as JudgeRunInfoSchema, F as RunStatusSchema, G as DEFAULT_ARTIFACT_EXCLUDE, H as defineEvalMatrix, I as RunSummarySchema, J as catalogEntry, K as DEFAULT_ATTACHMENT_LIMITS, L as ScoreResultSchema, M as RunMetadataSchema, N as RunScoringSchema, O as REPORT_SCHEMA_VERSION, P as RunSourceSchema, Q as criteria, R as TrajectoryEventSchema, S as AutIdentitySchema, T as JsonValueSchema, U as canonicalParameters, V as turnResultEvents, W as turnTokens, Z as choices, _ as user, _t as resourceUriSchema, a as assertJudgesIndependent, at as toJson, b as ArtifactEntrySchema, c as defineEval, ct as isDataset, d as judge, dt as ResourceKindSchema, f as metric, ft as ResourceUriSchema, g as skip, gt as resourceUri, h as registerEvals, ht as parseResourceUri, i as agentRuntimes, it as NotJsonError, j as RunManifestSchema, k as RecordedErrorSchema, l as defineSuite, lt as relativeDatasetPath, m as recordError, mt as authorId, n as DEFAULT_AGENT_RUNTIMES, o as authoringId, ot as toJsonObject, p as predicate, pt as UuidSchema, q as defineConfig, r as agent, s as defineAgent, st as defineDataset, t as BUILT_IN_METRICS, tt as passMark, u as expectToolCall, ut as definitionSite, v as parseTrajectoryJsonl, vt as resourceUuid, w as CheckpointResultSchema, x as AttachmentEntrySchema, y as ApiErrorSchema, z as UsageSchema } from "./src-B14fXAGp.mjs";
 //#region ../agents/src/index.ts
 /** A local, one-prompt-per-turn AUT backed by Pi's non-interactive print mode. */
 function piAgent(options = {}) {
@@ -68,4 +68,4 @@ function piAgent(options = {}) {
 	});
 }
 //#endregion
-export { ApiErrorSchema, ArtifactEntrySchema, AttachmentEntrySchema, AutIdentitySchema, BUILT_IN_METRICS, BatchSchema, CheckpointResultSchema, DEFAULT_AGENT_RUNTIMES, DEFAULT_ARTIFACT_EXCLUDE, DEFAULT_ATTACHMENT_LIMITS, JsonValueSchema, JudgeRunInfoSchema, MetricResultSchema, NotJsonError, REPORT_SCHEMA_VERSION, RecordedErrorSchema, ResourceKindSchema, ResourceUriSchema, RunDatasetSchema, RunManifestSchema, RunMetadataSchema, RunScoringSchema, RunSourceSchema, RunStatusSchema, RunSummarySchema, ScoreResultSchema, TrajectoryEventSchema, UsageSchema, UuidSchema, agent, agentRuntimes, authorId, authoringId, canonicalParameters, catalogEntry, defineAgent, defineConfig, defineDataset, defineEval, defineEvalMatrix, defineSuite, definitionSite, expectToolCall, isDataset, judge, metric, normalizeAutEvent, parseResourceUri, parseTrajectoryJsonl, piAgent, predicate, recordError, registerEvals, relativeDatasetPath, resourceUri, resourceUriSchema, resourceUuid, skip, toJson, toJsonObject, turnResultEvents, turnTokens, user };
+export { ApiErrorSchema, ArtifactEntrySchema, AttachmentEntrySchema, AutIdentitySchema, BUILT_IN_METRICS, BatchSchema, CheckpointResultSchema, DEFAULT_AGENT_RUNTIMES, DEFAULT_ARTIFACT_EXCLUDE, DEFAULT_ATTACHMENT_LIMITS, JsonValueSchema, JudgeRunInfoSchema, MetricResultSchema, NotJsonError, REPORT_SCHEMA_VERSION, RecordedErrorSchema, ResourceKindSchema, ResourceUriSchema, RunDatasetSchema, RunManifestSchema, RunMetadataSchema, RunScoringSchema, RunSourceSchema, RunStatusSchema, RunSummarySchema, ScoreResultSchema, TrajectoryEventSchema, UsageSchema, UuidSchema, agent, agentRuntimes, assertJudgesIndependent, authorId, authoringId, canonicalParameters, catalogEntry, choices, criteria, defineAgent, defineConfig, defineDataset, defineEval, defineEvalMatrix, defineSuite, definitionSite, expectToolCall, isDataset, judge, metric, normalizeAutEvent, parseResourceUri, parseTrajectoryJsonl, passMark, piAgent, predicate, recordError, registerEvals, relativeDatasetPath, resourceUri, resourceUriSchema, resourceUuid, skip, toJson, toJsonObject, turnResultEvents, turnTokens, user };