@leostera/evalkit 0.9.0 → 0.10.0

package/dist/runner.d.mts

@@ -141,6 +141,8 @@ type AutEventInput = {
   timestamp?: string;
   latencyMs?: number;
   usage?: Usage;
+  /** The model that answered this turn, when the agent knows it. */
+  model?: string;
 } | {
   kind: 'completed';
   output?: unknown;
@@ -162,8 +164,69 @@ type TurnResult = string | {
   reply?: unknown;
   toolCalls?: readonly TurnToolCall[];
   usage?: Usage;
+  /** The model that answered, recorded with the usage. */
+  model?: string;
 };
 //#endregion
+//#region ../core/src/judge.d.ts
+/**
+ * How a judge must answer, and how EvalKit turns the answer into a score.
+ *
+ * - `verdict` (the default): a value from 0 to 1 and a reason, as JSON.
+ * - `choices`: one of a fixed set of answers, such as A/B/C, each worth a value.
+ * - `criteria`: a value from 0 to 1 per criterion; the score is their weighted mean.
+ */
+type JudgeAnswer = {
+  kind: 'verdict';
+} | {
+  kind: 'choices';
+  choices: JudgeChoice[];
+} | {
+  kind: 'criteria';
+  criteria: JudgeCriterion[];
+};
+/** One answer a `choices` judge can give. Keys are matched case-insensitively. */
+type JudgeChoice = {
+  key: string;
+  label: string;
+  value: number;
+};
+/** One thing a `criteria` judge scores. */
+type JudgeCriterion = {
+  name: string;
+  weight: number;
+  description?: string;
+};
+/**
+ * One criterion's result, as a scorer returns it and as results record it:
+ * a value with what it saw, or `reached: false` when the check never got to
+ * run, because something before it failed (say, the app would not start).
+ * A criterion not reached counts as 0; the score's `explanation` says why,
+ * once, instead of every criterion repeating it.
+ */
+type CriterionResult = ReachedCriterion | UnreachedCriterion;
+type ReachedCriterion = {
+  name: string;
+  /** From 0 to 1. */
+  value: number;
+  /** Default 1, or the declared weight. */
+  weight?: number;
+  /** Why this value. */
+  explanation?: string;
+  /** What the check observed, as plain data. */
+  evidence?: JsonValue;
+};
+type UnreachedCriterion = {
+  name: string;
+  reached: false;
+  weight?: number;
+  explanation?: string;
+};
+/** A criterion as a scorer returns it: its evidence may be any plain value. */
+type CriterionInput = (Omit<ReachedCriterion, 'evidence'> & {
+  evidence?: unknown;
+}) | UnreachedCriterion;
+//#endregion
 //#region ../core/src/catalog.d.ts
 type EvalCatalogAgent = {
   name?: string;
@@ -215,10 +278,20 @@ type EvalCatalogDataset = {
 type EvalCatalogScorer = {
   name: string;
   kind: ScoringRule['kind'];
+  /** What it checks, in the author's words. */
+  description?: string;
   passAt?: number;
   supportsPartial?: boolean;
   /** Judges only. */
   rubric?: string;
+  /** Judges only: the agent that judges (its own, or the project's default). */
+  agent?: EvalCatalogAgent;
+  /** Judges only: the names of the inputs it sees. */
+  inputs?: string[];
+  /** Judges only: how it answers. */
+  answer?: JudgeAnswer;
+  /** Judges only: the prompt it receives, with `{input}` placeholders. */
+  prompt?: string;
   definedAt?: string;
 };
 type EvalCatalogStep =
@@ -383,6 +456,8 @@ type AutEvent = {
   timestamp: string;
   latencyMs?: number;
   usage?: Usage;
+  /** The model that answered this turn, when the agent knows it. */
+  model?: string;
 } | {
   kind: 'completed';
   output?: JsonValue;
@@ -553,9 +628,23 @@ type ExpectToolCallStep = {
   };
 };
 type TranscriptStep = UserStep | AgentStep | ScoringRule | ExpectToolCallStep;
+/**
+ * What a judge saw and said. The built-in `judge()` records all of it; a
+ * predicate that calls a model itself can report `model`, `prompt` and
+ * `reply` so the dashboard shows them the same way.
+ */
 type JudgeRunInfo = {
   agent?: AutIdentity;
+  /** The model the judge reported using (`turn-completed` `model`). */
+  model?: string;
+  /** Tokens the judge used: never counted in the agent's `tokens`. */
   usage?: Usage;
+  /** The exact prompt the judge received. */
+  prompt?: string;
+  /** Its reply, as text or JSON. */
+  reply?: JsonValue;
+  /** 2 when its first reply was not in the format asked for and it was asked again. */
+  attempts?: number;
   /** Judge-agent observations; never attributed to the AUT trajectory. */
   events?: JsonValue[];
 };
@@ -565,7 +654,8 @@ type SkippedScore = {
   reason: string;
 };
 type ScoreValue<Evidence = unknown> = number | SkippedScore | {
-  value: number;
+  /** From 0 to 1. Without it, the weighted mean of `criteria`. */
+  value?: number;
   passed?: boolean;
   explanation?: string;
   /**
@@ -573,7 +663,12 @@ type ScoreValue<Evidence = unknown> = number | SkippedScore | {
    * evidence reaches later scorers through `checkpoint(rule)`.
    */
   evidence?: Evidence;
-  /** Judge-agent evidence, never inferred from AUT usage. */
+  /**
+   * Per-criterion results, each shown on its own; see `CriterionResult`.
+   * Return `{ name, reached: false }` for a check that never ran.
+   */
+  criteria?: CriterionInput[];
+  /** What a model-backed scorer saw and said; never counted as AUT usage. */
   judge?: JudgeRunInfo;
 };
 type ScoreResult = {
@@ -581,8 +676,15 @@ type ScoreResult = {
   kind: 'predicate' | 'judge';
   value?: number;
   passed?: boolean;
+  /** The pass mark the value was held to (1 unless the rule set `passAt`). */
+  passAt?: number;
   explanation?: string;
   evidence?: JsonValue;
+  /** A `choices` judge's answer, such as `B`. */
+  choice?: string;
+  criteria?: CriterionResult[];
+  /** Files its scorer saved with `attach()`, by name. */
+  attachments?: string[];
   judge?: JudgeRunInfo;
   durationMs: number;
   error?: RecordedError;
@@ -637,15 +739,48 @@ type CheckpointOutcome<Evidence> = Omit<CheckpointResult, 'evidence'> & {
 type PredicateScorer<Evidence = unknown> = {
   kind: 'predicate';
   name: string;
+  /** What it checks, in a sentence, shown wherever the scorer is. */
+  description?: string;
   supportsPartial?: boolean;
   /** A numeric result passes when it is at least this value. Default: only 1 passes. */
   passAt?: number;
+  /**
+   * The checks it reports, declared up front so they show before a run.
+   * A declared criterion it leaves out is recorded as not reached; one it
+   * did not declare is an error.
+   */
+  criteria?: JudgeCriterion[];
   run(context: ScoringContext): boolean | ScoreValue<Evidence> | Promise<boolean | ScoreValue<Evidence>>;
 };
+/** What a judge sees: one named input, read from the run. */
+type JudgeInput = (scoring: ScoringContext) => JsonValue | undefined | Promise<JsonValue | undefined>;
+/**
+ * A rubric-graded scorer. EvalKit builds the prompt from the rubric, the
+ * inputs and the answer format, runs the judge agent, checks its answer
+ * (asking once more if it is not in the format asked for), and records
+ * what it saw and said.
+ */
 type JudgeRule = {
   kind: 'judge';
   name: string;
   rubric: string;
+  /** What it checks, in a sentence, shown wherever the judge is. */
+  description?: string;
+  /**
+   * The agent that judges, made with `defineAgent()`. Default: `judge` in
+   * evalkit.config.ts. It gets the run's parameters, so a matrix axis can
+   * choose its model.
+   */
+  agent?: AutAdapter;
+  /**
+   * What the judge sees, by name. Default: `conversation` (this turn at a
+   * checkpoint, the whole run in final scoring) and `reply` (the last reply).
+   */
+  inputs?: Readonly<Record<string, JudgeInput>>;
+  /** How it answers. Default `verdict`: a value from 0 to 1 and a reason. */
+  answer: JudgeAnswer;
+  /** A value passes when it is at least this. Default: only 1 passes. */
+  passAt?: number;
   /** Run even when the scenario stopped early or execution failed. */
   supportsPartial?: boolean;
 };
@@ -684,8 +819,6 @@ type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
   id: string;
   name?: string;
   agent: TAgent;
-  /** Separate judge agent; its model and tools are never inherited from the AUT. */
-  judge?: AutAdapter;
   /** Data and files the scenario uses; each run gets its own copy. */
   datasets?: readonly Dataset[];
   transcript: TranscriptStep[];
@@ -806,8 +939,13 @@ type CheckpointResult = {
   status: 'passed' | 'failed' | 'error' | 'skipped';
   value?: number;
   passed?: boolean;
+  passAt?: number;
   explanation?: string;
   evidence?: JsonValue;
+  choice?: string;
+  criteria?: CriterionResult[];
+  /** Files its scorer saved with `attach()`, by name. */
+  attachments?: string[];
   judge?: JudgeRunInfo;
   durationMs?: number;
   error?: RecordedError;
@@ -826,7 +964,7 @@ type CheckpointResult = {
  *   usage, cached input included. Absent when no turn reported any.
  * - `tool calls`: the agent's `tool-call` events.
  */
-declare const BUILT_IN_METRICS: readonly ['agent time', 'tokens', 'tool calls'];
+declare const BUILT_IN_METRICS: readonly ['agent time', 'tokens', 'tool calls', 'judge tokens'];
 type BuiltInMetricName = (typeof BUILT_IN_METRICS)[number];
 /** A built-in metric's name (suggested by editors), or a scenario's own. */
 type MetricName = BuiltInMetricName | (string & {});
@@ -964,6 +1102,21 @@ export declare const readRunSummary: (root: string, runId: string) => Promise<{
   readonly durationMs?: number | undefined;
   readonly scoring?: {
     readonly results: {
+      readonly passAt?: number | undefined;
+      readonly choice?: string | undefined;
+      readonly criteria?: ({
+        readonly name: string;
+        readonly value: number;
+        readonly weight?: number | undefined;
+        readonly explanation?: string | undefined;
+        readonly evidence?: JsonValue | undefined;
+      } | {
+        readonly name: string;
+        readonly reached: false;
+        readonly weight?: number | undefined;
+        readonly explanation?: string | undefined;
+      })[] | undefined;
+      readonly attachments?: string[] | undefined;
       readonly name: string;
       readonly kind: "judge" | "predicate";
       readonly value?: number | undefined;
@@ -977,12 +1130,16 @@ export declare const readRunSummary: (root: string, runId: string) => Promise<{
           readonly name?: string | undefined;
           readonly version?: string | undefined;
         } | undefined;
+        readonly model?: string | undefined;
         readonly usage?: {
           readonly inputTokens?: number | undefined;
           readonly outputTokens?: number | undefined;
           readonly cachedInputTokens?: number | undefined;
           readonly totalTokens?: number | undefined;
         } | undefined;
+        readonly prompt?: string | undefined;
+        readonly reply?: JsonValue | undefined;
+        readonly attempts?: number | undefined;
         readonly events?: JsonValue[] | undefined;
       } | undefined;
       readonly durationMs: number;
@@ -1006,6 +1163,21 @@ export declare const readRunSummary: (root: string, runId: string) => Promise<{
       } | undefined;
     }[] | undefined;
     readonly checkpoints?: {
+      readonly passAt?: number | undefined;
+      readonly choice?: string | undefined;
+      readonly criteria?: ({
+        readonly name: string;
+        readonly value: number;
+        readonly weight?: number | undefined;
+        readonly explanation?: string | undefined;
+        readonly evidence?: JsonValue | undefined;
+      } | {
+        readonly name: string;
+        readonly reached: false;
+        readonly weight?: number | undefined;
+        readonly explanation?: string | undefined;
+      })[] | undefined;
+      readonly attachments?: string[] | undefined;
       readonly step: number;
       readonly kind: "expect-tool-call" | "judge" | "predicate";
       readonly name: string;
@@ -1021,12 +1193,16 @@ export declare const readRunSummary: (root: string, runId: string) => Promise<{
           readonly name?: string | undefined;
           readonly version?: string | undefined;
         } | undefined;
+        readonly model?: string | undefined;
         readonly usage?: {
           readonly inputTokens?: number | undefined;
           readonly outputTokens?: number | undefined;
           readonly cachedInputTokens?: number | undefined;
           readonly totalTokens?: number | undefined;
         } | undefined;
+        readonly prompt?: string | undefined;
+        readonly reply?: JsonValue | undefined;
+        readonly attempts?: number | undefined;
         readonly events?: JsonValue[] | undefined;
       } | undefined;
       readonly durationMs?: number | undefined;
@@ -1062,6 +1238,21 @@ export declare const readRunSummary: (root: string, runId: string) => Promise<{
 }>;
 export declare const readRunScoring: (root: string, runId: string) => Promise<{
   readonly results: {
+    readonly passAt?: number | undefined;
+    readonly choice?: string | undefined;
+    readonly criteria?: ({
+      readonly name: string;
+      readonly value: number;
+      readonly weight?: number | undefined;
+      readonly explanation?: string | undefined;
+      readonly evidence?: JsonValue | undefined;
+    } | {
+      readonly name: string;
+      readonly reached: false;
+      readonly weight?: number | undefined;
+      readonly explanation?: string | undefined;
+    })[] | undefined;
+    readonly attachments?: string[] | undefined;
     readonly name: string;
     readonly kind: "judge" | "predicate";
     readonly value?: number | undefined;
@@ -1075,12 +1266,16 @@ export declare const readRunScoring: (root: string, runId: string) => Promise<{
         readonly name?: string | undefined;
         readonly version?: string | undefined;
       } | undefined;
+      readonly model?: string | undefined;
       readonly usage?: {
         readonly inputTokens?: number | undefined;
         readonly outputTokens?: number | undefined;
         readonly cachedInputTokens?: number | undefined;
         readonly totalTokens?: number | undefined;
       } | undefined;
+      readonly prompt?: string | undefined;
+      readonly reply?: JsonValue | undefined;
+      readonly attempts?: number | undefined;
       readonly events?: JsonValue[] | undefined;
     } | undefined;
     readonly durationMs: number;
@@ -1104,6 +1299,21 @@ export declare const readRunScoring: (root: string, runId: string) => Promise<{
     } | undefined;
   }[] | undefined;
   readonly checkpoints?: {
+    readonly passAt?: number | undefined;
+    readonly choice?: string | undefined;
+    readonly criteria?: ({
+      readonly name: string;
+      readonly value: number;
+      readonly weight?: number | undefined;
+      readonly explanation?: string | undefined;
+      readonly evidence?: JsonValue | undefined;
+    } | {
+      readonly name: string;
+      readonly reached: false;
+      readonly weight?: number | undefined;
+      readonly explanation?: string | undefined;
+    })[] | undefined;
+    readonly attachments?: string[] | undefined;
     readonly step: number;
     readonly kind: "expect-tool-call" | "judge" | "predicate";
     readonly name: string;
@@ -1119,12 +1329,16 @@ export declare const readRunScoring: (root: string, runId: string) => Promise<{
         readonly name?: string | undefined;
         readonly version?: string | undefined;
       } | undefined;
+      readonly model?: string | undefined;
       readonly usage?: {
         readonly inputTokens?: number | undefined;
         readonly outputTokens?: number | undefined;
         readonly cachedInputTokens?: number | undefined;
         readonly totalTokens?: number | undefined;
       } | undefined;
+      readonly prompt?: string | undefined;
+      readonly reply?: JsonValue | undefined;
+      readonly attempts?: number | undefined;
       readonly events?: JsonValue[] | undefined;
     } | undefined;
     readonly durationMs?: number | undefined;
@@ -1207,8 +1421,10 @@ export type RunEvalOptions = {
   suiteId?: string;
   /** Overrides the scenario's `repeat`: how many runs to start, as one batch. */
   repeat?: number;
-  /** Parameters made available to the AUT for this invocation. */
+  /** Parameters made available to the AUT, and to judges, for this invocation. */
   parameters?: JsonObject;
+  /** The agent for judges that name none (`judge` in evalkit.config.ts). */
+  judge?: AutAdapter;
   /** Matrix provenance recorded in every run's manifest. */
   matrix?: RunMetadata['matrix'];
   /** Selects a declared AUT runtime such as local, sandbox, or remote. */

package/dist/runner.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.mts","names":["_$_evalkit_core0.JsonValue","_$_evalkit_core0.TrajectoryEvent"],"sources":["../../runner/src/errors.ts","../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/catalog.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/usage.ts","../../core/src/index.ts","../../runner/src/local-report-store.ts","../../runner/src/datasets.ts","../../runner/src/local-report-reader.ts","../../runner/src/matrix.ts","../../runner/src/upgrade.ts","../../runner/src/workspace.ts","../../runner/src/index.ts"],"mappings":";;;;;;qBAEa,0BAA0B;EACrC;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBAGW,oBAAoB;EAC/B;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBASW,iCAAiC;EAG5C;EACA;EACA;EACA;;;;;;qBAIW,uBAAuB;EAClC;;;;cCrCW,oBAAkB,OAAA;KACnB,sBAAsB,mBAAmB;KAiBzC,YAAY,cAAc,eAAe,2BACxC;;;;KCdD;;KAGA,6BAA6B;EAAe;;;;;;KAM5C,wBAAwB,SAAS,eAAe;;KAGhD;EACV,QAAQ;;EAER;;KAGU;;EAEV;EACA,QAAQ;;KAGE;EACV;EACA;;EAEA,QAAQ;;EAER;;EAEA,YAAY;EACZ,QAAQ;;KAoCE;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,KAAK,sBACF,iBAAiB,QAAQ;;;KAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;;;;;;KC/FP;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;KAE/B;EACV;EACA;;EAEA;EACA;;;KAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;;;KC5BF;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;KAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;KAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;KAGU;EACV;EACA,MAAM;EACN;EACA;;EAEA;EACA;;KAGU;;;EAEN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;KAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;;EAEV;EACA;IAAU;IAAoB;;EAC9B,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;;;KCjFU;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;;;KCD3C;KAEA;;EAEV;;EAEA;;;;;;;;KCxCU;;;;;;EAMV;;EAEA;;EAEA;;EAEA;;;;KCQU;KACA,YACV,gBAAgB;GAAiB,cAAc;;KACrC;GAAgB,cAAc;;KAE9B;;EAEV;EACA;EACA;EACA;;KAGU;;;;;KAMA;EACV;EACA,gBAAgB;;KAGN,gBAAgB,QAAQ,OAAO,kBAAkB;KAKjD;EACV;EACA;EACA;;KAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;KAE/B;EACN;EAAqB;;EAErB;EACA;;EAEA;EACA;;EAEA;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAuB;;EACvB;EAAe,OAAO;EAAe;;KAE/B,mBACT;EAAa;MAAoB;EAAgB;;KAExC;;EAEV;;KAGU,qBAAqB;KACrB,qBAAqB;KAErB;EACV;EACA;;EAEA,QAAQ;EACR,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;EAEN,WAAW;;EAEX,oBAAoB;;KAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;IAEV,UAAU,OAAO,yBAAyB;;;;;;;;IAQ1C,SAAS,cAAc,kBAAkB,eAAe;MACtD,QAAQ,WAAW;;KAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;;KAyBA;;EAEV,YAAY;;EAEZ,MAAM;;EAEN;;EAEA,QAAQ;;KAGE;EACV;;EAEA,oBAAoB,OAAO;;KAGjB;EACV;EACA,aAAa;;KAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;KAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;KAE5B,iBACV,WAAW,YAAY,cAAc;KAuB3B;EACV,QAAQ;EACR,QAAQ;;EAER,SAAS;;;KAIC;EAAiB;EAAe;;KAEhC,WAAW,+BAEnB;EAEE;EACA;EACA;;;;;EAKA,WAAW;;EAEX,QAAQ;;KAYF;EACV;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;;EAER;;KAGU;EACV,WAAW;EACX,WAAW;;KAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;EAEP,SAAS;;EAET,MAAM;;;;;;EAMN,sBAAsB;;;;;;;EAOtB,WAAW,UACT,MAAM,gBAAgB,YACrB,kBAAkB;EACrB,WACE,MAAM,YAAY,qBACjB,kBAAkB;;;;;;EAMrB,OAAO,cAAc,kBAAkB,aAAa;;;;;;;KAQ1C,kBAAkB,YAAY,KAAK;EAC7C,WAAW;;KAGD,gBAAgB;EAC1B;EACA;EACA;;EAEA;EACA,IACE,SAAS,2BACE,WAAW,YAAY,kBAAkB,WAAW;;KAGvD;EACV;EACA;EACA;;EAEA;;KAGU,cAAc,kBAAkB;;;;;;KA4BhC;EACV;EACA;;EAEA;;EAEA;EACA,IACE,SAAS,sCACa;;;KAcd;EACV,MAAM;;EAEN;EACA;EACA;;EAEA;EACA,QAAQ;;KAYE;EACV;;EAEA;;KAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,QAAQ;;EAER,oBAAoB;EACpB,YAAY;EACZ,SAAS;;EAET,mBAAmB;EACnB,SAAS;;;;;EAKT;;EAEA,WAAW;EACX,YAAY;;;KAIF;;;;;;;;;EASV;;KA0GU;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;;;;;KAoFU;;KAIA;EACV;;EAEA;EACA;;;;;;KAOU;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;KAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;KAIE;EAAe;EAAa;;KAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;EACT,UAAU;;EAEV,QAAQ;;EAER;;EAEA,WAAW;;KAGD;EACV;EACA,YAAY;EACZ;;EAEA;;KAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;;;;;;;;;;;cAa7B;KACD,4BAA4B;;KAE5B,aAAa;KAEb;EACV,SAAS;EACT,UAAU;EACV,cAAc;EACd;EACA;EACA;;KAGU;;EAEV;EACA;EACA;;;KAIU;EACV;;EAEA;;EAEA;;KAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,cAAc;EACd,QAAQ;;KAGE;WACD;EACT,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,aAAa;;EAElC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,aAAa;;KAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;;KAID;EACV;EACA,QAAQ;EACR;EACA,QAAQ;EACR,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;;;;wBCrwBM,iBAAiB,wBAAwB;;;;KCmK7C;EAAqB;EAAc;;;;;;;wBAOzB,mBACpB,SAAS,SACT,eACC,QAAQ;;wBA6BW,kBACpB,SAAS,SACT,cACA,eACC,QAAQ,WAAW;;;qBC7RT,kBAAe,cAAgB,kBAAe;;;;;;0BAA9CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAEA,iBAAc,cAAgB,kBAAe;;;;;;;;;;;0BAA7CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAEA,iBAAc,cAAgB,kBAAe;;;;;;;wBAA7CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBAES,cAAc,cAAc,gBAAa,QAAzCC;;;;;wBAUN,YAAY,OAAO;;;;KC3BvB;EACV;EACA;EACA;EACA;EACA;;;wBAIc,UACd,QAAQ,YACR,SAAS,KAAK;EACZ,YAAY;;EAEZ,YAAY,MAAM;EAClB,YACE,MAAM,gBACN,eAAe,uBACL;IAEb,OAAO,OAAO;;;;KCTL;;EAEV;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;;wBAsBoB,eACpB,cACA;EAAW;IACV,QAAQ;;;;wBCtCK,iBAAiB,cAAc;;;YC8DnC;EACV,QAAQ;;EAER;EACA;;EAEA;;EAEA,aAAa;;EAEb,SAAS;;EAET,UAAU;;EAEV;;EAEA,YAAY,OAAO;;EAEnB;;EAEA,iBAAiB;;;;;EAKjB;;EAEA,mBAAmB;;EAEnB,SAAS;;EAET,UAAU;;EAEV;;;;;EAKA,SAAS;;;;;EAKT,WAAW,KAAK,QAAQ,OAAO;EAC/B,YAAY;;;YAIF;EACV;EACA,QAAQ;EACR;EACA;EACA,aAAa;;;;;;;wBAUC,QACd,YAAY,gBACZ,SAAS,iBACR,OAAO,OAAO"}
+{"version":3,"file":"runner.d.mts","names":["_$_evalkit_core0.JsonValue","_$_evalkit_core0.TrajectoryEvent"],"sources":["../../runner/src/errors.ts","../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/judge.ts","../../core/src/catalog.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/usage.ts","../../core/src/index.ts","../../runner/src/local-report-store.ts","../../runner/src/datasets.ts","../../runner/src/local-report-reader.ts","../../runner/src/matrix.ts","../../runner/src/upgrade.ts","../../runner/src/workspace.ts","../../runner/src/index.ts"],"mappings":";;;;;;qBAEa,0BAA0B;EACrC;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBAGW,oBAAoB;EAC/B;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBASW,iCAAiC;EAG5C;EACA;EACA;EACA;;;;;;qBAIW,uBAAuB;EAClC;;;;cCrCW,oBAAkB,OAAA;KACnB,sBAAsB,mBAAmB;KAiBzC,YAAY,cAAc,eAAe,2BACxC;;;;KCdD;;KAGA,6BAA6B;EAAe;;;;;;KAM5C,wBAAwB,SAAS,eAAe;;KAGhD;EACV,QAAQ;;EAER;;KAGU;;EAEV;EACA,QAAQ;;KAGE;EACV;EACA;;EAEA,QAAQ;;EAER;;EAEA,YAAY;EACZ,QAAQ;;KAoCE;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,KAAK,sBACF,iBAAiB,QAAQ;;;KAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;;;;;;KC/FP;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;;EAEA;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;KAE/B;EACV;EACA;;EAEA;EACA;;;KAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;EAER;;;;;;;;;;;KCjDM;EACN;;EACA;EAAiB,SAAS;;EAC1B;EAAkB,UAAU;;;KAGtB;EAAgB;EAAa;EAAe;;;KAG5C;EACV;EACA;EACA;;;;;;;;;KAuFU,kBAAkB,mBAAmB;KAErC;EACV;;EAEA;;EAEA;;EAEA;;EAEA,WAAW;;KAGD;EACV;EACA;EACA;EACA;;;KAIU,kBACP,KAAK;EAAkC;KACxC;;;KCtGQ;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;KAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;KAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;KAGU;EACV;EACA,MAAM;;EAEN;EACA;EACA;;EAEA;;EAEA,QAAQ;;EAER;;EAEA,SAAS;;EAET;EACA;;KAGU;;;EAEN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;KAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;;EAEV;EACA;IAAU;IAAoB;;EAC9B,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;;;KC/FU;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;;;KCS3C;KAEA;;EAEV;;EAEA;;;;;;;;KClDU;;;;;;EAMV;;EAEA;;EAEA;;EAEA;;;;KC6BU;KACA,YACV,gBAAgB;GAAiB,cAAc;;KACrC;GAAgB,cAAc;;KAE9B;;EAEV;EACA;EACA;EACA;;KAGU;;;;;KAMA;EACV;EACA,gBAAgB;;KAGN,gBAAgB,QAAQ,OAAO,kBAAkB;KAKjD;EACV;EACA;EACA;;KAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;;EAEA;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;KAE/B;EACN;EAAqB;;EAErB;EACA;;EAEA;EACA;;EAEA;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAuB;;EACvB;EAAe,OAAO;EAAe;;KAE/B,mBACT;EAAa;MAAoB;EAAgB;;KAExC;;EAEV;;KAGU,qBAAqB;KACrB,qBAAqB;KAErB;EACV;EACA;;EAEA,QAAQ;EACR,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;EAEN,WAAW;;EAEX,oBAAoB;;KAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;IAEV,UAAU,OAAO,yBAAyB;;;;;;;;IAQ1C,SAAS,cAAc,kBAAkB,eAAe;MACtD,QAAQ,WAAW;;KAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;;KAyBA;;EAEV,YAAY;;EAEZ,MAAM;;EAEN;;EAEA,QAAQ;;KAGE;EACV;;EAEA,oBAAoB,OAAO;;KAGjB;EACV;EACA,aAAa;;KAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;KAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;KAE5B,iBACV,WAAW,YAAY,cAAc;;;;;;KA4B3B;EACV,QAAQ;;EAER;;EAEA,QAAQ;;EAER;;EAEA,QAAQ;;EAER;;EAEA,SAAS;;;KAIC;EAAiB;EAAe;;KAEhC,WAAW,+BAEnB;;EAGE;EACA;EACA;;;;;EAKA,WAAW;;;;;EAKX,WAAW;;EAEX,QAAQ;;KAYF;EACV;EACA;EACA;EACA;;EAEA;EACA;EACA,WAAW;;EAEX;EACA,WAAW;;EAEX;EACA,QAAQ;EACR;EACA,QAAQ;;EAER;;KAGU;EACV,WAAW;EACX,WAAW;;KAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;EAEP,SAAS;;EAET,MAAM;;;;;;EAMN,sBAAsB;;;;;;;EAOtB,WAAW,UACT,MAAM,gBAAgB,YACrB,kBAAkB;EACrB,WACE,MAAM,YAAY,qBACjB,kBAAkB;;;;;;EAMrB,OAAO,cAAc,kBAAkB,aAAa;;;;;;;KAQ1C,kBAAkB,YAAY,KAAK;EAC7C,WAAW;;KAGD,gBAAgB;EAC1B;EACA;;EAEA;EACA;;EAEA;;;;;;EAMA,WAAW;EACX,IACE,SAAS,2BACE,WAAW,YAAY,kBAAkB,WAAW;;;KAIvD,cACV,SAAS,mBACN,wBAAwB,QAAQ;;;;;;;KAQzB;EACV;EACA;EACA;;EAEA;;;;;;EAMA,QAAQ;;;;;EAKR,SAAS,SAAS,eAAe;;EAEjC,QAAQ;;EAER;;EAEA;;KAGU,cAAc,kBAAkB;;;;;;KA6ChC;EACV;EACA;;EAEA;;EAEA;EACA,IACE,SAAS,sCACa;;;KAcd;EACV,MAAM;;EAEN;EACA;EACA;;EAEA;EACA,QAAQ;;KAyCE;EACV;;EAEA;;KAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,oBAAoB;EACpB,YAAY;EACZ,SAAS;;EAET,mBAAmB;EACnB,SAAS;;;;;EAKT;;EAEA,WAAW;EACX,YAAY;;;KAIF;;;;;;;;;EASV;;KAoHU;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;;;;;KAoFU;;KAIA;EACV;;EAEA;EACA;;;;;;KAOU;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;KAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;KAIE;EAAe;EAAa;;KAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;EACT,UAAU;;EAEV,QAAQ;;EAER;;EAEA,WAAW;;KAGD;EACV;EACA,YAAY;EACZ;;EAEA;;KAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX;EACA,WAAW;;EAEX;EACA,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;;;;;;;;;;;cAa7B;KAMD,4BAA4B;;KAE5B,aAAa;KAEb;EACV,SAAS;EACT,UAAU;EACV,cAAc;EACd;EACA;EACA;;KAGU;;EAEV;EACA;EACA;;;KAIU;EACV;;EAEA;;EAEA;;KAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,cAAc;EACd,QAAQ;;KAGE;WACD;EACT,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,aAAa;;EAElC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,aAAa;;KAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;;KAID;EACV;EACA,QAAQ;EACR;EACA,QAAQ;EACR,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;;;;wBC35BM,iBAAiB,wBAAwB;;;;KCmK7C;EAAqB;EAAc;;;;;;;wBAOzB,mBACpB,SAAS,SACT,eACC,QAAQ;;wBA6BW,kBACpB,SAAS,SACT,cACA,eACC,QAAQ,WAAW;;;qBC7RT,kBAAe,cAAgB,kBAAe;;;;;;0BAA9CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAEA,iBAAc,cAAgB,kBAAe;;;;;;;;;;;;;4BAA7CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAEA,iBAAc,cAAgB,kBAAe;;;;;;;;;0BAA7CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBAES,cAAc,cAAc,gBAAa,QAAzCC;;;;;wBAUN,YAAY,OAAO;;;;KC3BvB;EACV;EACA;EACA;EACA;EACA;;;wBAIc,UACd,QAAQ,YACR,SAAS,KAAK;EACZ,YAAY;;EAEZ,YAAY,MAAM;EAClB,YACE,MAAM,gBACN,eAAe,uBACL;IAEb,OAAO,OAAO;;;;KCTL;;EAEV;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;;wBAsBoB,eACpB,cACA;EAAW;IACV,QAAQ;;;;wBCtCK,iBAAiB,cAAc;;;YCgEnC;EACV,QAAQ;;EAER;EACA;;EAEA;;EAEA,aAAa;;EAEb,QAAQ;;EAER,SAAS;;EAET,UAAU;;EAEV;;EAEA,YAAY,OAAO;;EAEnB;;EAEA,iBAAiB;;;;;EAKjB;;EAEA,mBAAmB;;EAEnB,SAAS;;EAET,UAAU;;EAEV;;;;;EAKA,SAAS;;;;;EAKT,WAAW,KAAK,QAAQ,OAAO;EAC/B,YAAY;;;YAIF;EACV;EACA,QAAQ;EACR;EACA;EACA,aAAa;;;;;;;wBAUC,QACd,YAAY,gBACZ,SAAS,iBACR,OAAO,OAAO"}

package/dist/runner.mjs

@@ -1,2 +1,2 @@
-import { _ as DatasetError, a as readRunEvents, c as readRunSummary, d as staticDatasetFile, f as staticDatasetFiles, g as CheckpointExecutionError, h as CancelledError, i as ownerIsGone, l as localReportStore, m as AutExecutionError, n as upgradeReports, o as readRunManifest, r as runMatrix, s as readRunScoring, t as runEval, u as runWorkspacePath, v as ReportError, y as ScoringError } from "./src-CcdLZqFk.mjs";
+import { _ as DatasetError, a as readRunEvents, c as readRunSummary, d as staticDatasetFile, f as staticDatasetFiles, g as CheckpointExecutionError, h as CancelledError, i as ownerIsGone, l as localReportStore, m as AutExecutionError, n as upgradeReports, o as readRunManifest, r as runMatrix, s as readRunScoring, t as runEval, u as runWorkspacePath, v as ReportError, y as ScoringError } from "./src-BfSQKOJP.mjs";
 export { AutExecutionError, CancelledError, CheckpointExecutionError, DatasetError, ReportError, ScoringError, localReportStore, ownerIsGone, readRunEvents, readRunManifest, readRunScoring, readRunSummary, runEval, runMatrix, runWorkspacePath, staticDatasetFile, staticDatasetFiles, upgradeReports };

package/dist/share-worker.mjs

@@ -1,4 +1,4 @@
-import { r as createShareApp, t as ShareIndexDb } from "./index-db-BbVblaHz.mjs";
+import { r as createShareApp, t as ShareIndexDb } from "./index-db-B1FeXzmS.mjs";
 import { t as r2Files } from "./files-DkDQs2JL.mjs";
 //#region ../share/src/index-do.ts
 const METHODS = [

package/dist/share.mjs

@@ -1,4 +1,4 @@
-import { a as parseUpload, c as validRepoKey, d as decodeBundle, f as encodeBundle, i as normalizeBasePath, l as dashboardIndex, n as asyncIndex, o as allowedRunPath, r as createShareApp, s as repoKey, t as ShareIndexDb, u as BUNDLE_FORMAT } from "./index-db-BbVblaHz.mjs";
+import { a as parseUpload, c as validRepoKey, d as decodeBundle, f as encodeBundle, i as normalizeBasePath, l as dashboardIndex, n as asyncIndex, o as allowedRunPath, r as createShareApp, s as repoKey, t as ShareIndexDb, u as BUNDLE_FORMAT } from "./index-db-B1FeXzmS.mjs";
 import { t as r2Files } from "./files-DkDQs2JL.mjs";
-import { i as directoryFiles, n as directoryAssets, r as sqliteIndex, t as createDirectoryShareApp } from "./src-Bn6fnZnF.mjs";
+import { i as directoryFiles, n as directoryAssets, r as sqliteIndex, t as createDirectoryShareApp } from "./src-Bccv9igQ.mjs";
 export { BUNDLE_FORMAT, ShareIndexDb, allowedRunPath, asyncIndex, createDirectoryShareApp, createShareApp, dashboardIndex, decodeBundle, directoryAssets, directoryFiles, encodeBundle, normalizeBasePath, parseUpload, r2Files, repoKey, sqliteIndex, validRepoKey };

package/dist/skills/evalkit/SKILL.md

@@ -22,11 +22,11 @@ For a cross-cutting change, use all relevant subskills; for a focused task, load
 
 ## Invariants
 
-- Work from the eval project's directory (for the included project, `examples/starter`). By default the CLI discovers default-exported `scenarios/**/*.eval.ts` without a config or registry, and each directory directly below `scenarios/` is a suite. The starter demonstrates this zero-config discovery path; `examples/configured-matrix` demonstrates optional config and a runnable sweep. An optional `evalkit.config.js`/`.ts` customizes discovery and execution, or can supply an explicit registry/eval list. Dataset paths and `_evalkit-results` / `_evalkit-sandbox` resolve relative to the config directory or project root.
+- Work from the eval project's directory (for the included project, `examples/starter`). By default the CLI discovers default-exported `scenarios/**/*.eval.ts` without a config or registry, and each directory directly below `scenarios/` is a suite. The starter demonstrates this zero-config discovery path; `examples/configured-matrix` demonstrates optional config and a runnable sweep. An optional `evalkit.config.js`/`.ts` customizes discovery and execution, or can supply an explicit registry/eval list. Dataset paths and `_evalkit-results` resolve relative to the config directory or project root; run workspaces live outside the project in `$TMPDIR/evalkit/<project>-<hash>/` unless `sandboxDir` is set.
 - A scenario is one concrete conversation in its own small file. Share agents (`agents/`), judges (`judges/`) and datasets (`datasets/`) by importing them; never generate scenarios in loops or from datasets.
 - Authored suites, scenarios, datasets, agents, and matrices use stable lowercase kebab-case IDs. Only runs and runs receive generated runtime URIs.
 - Keep evaluator datasets/answers private from the AUT and model unless a trusted adapter deliberately reads them and includes selected content in its prompt. Such inclusion is a deliberate treatment (for example, a memory-on condition), not a live retrieval test. Local sandboxes retain evaluator files; review reports and sandboxes before sharing.
-- Executable transcript steps are `user(...)`, `predicate(...)`, `judge(...)`, and `expectToolCall(...)`. Final scoring supports both `predicate(...)` and `judge(...)`; `judge(...)` requires a separate eval-level judge adapter. `agent(...)` transcript steps are not executable. `agentsSdk()` is unimplemented, `judgeScorer(...)` is not part of the current API, and `policy.timeoutMs` is not enforced. Verify current code/docs before proposing capabilities.
+- Executable transcript steps are `user(...)`, `predicate(...)`, `judge(...)`, and `expectToolCall(...)`. Final scoring supports both `predicate(...)` and `judge(...)`; `judge(...)` uses its own `agent` or the project's `judge` in `evalkit.config.ts`, never the AUT. `agent(...)` transcript steps are not executable. `agentsSdk()` is unimplemented, `judgeScorer(...)` is not part of the current API, and `policy.timeoutMs` is not enforced. Verify current code/docs before proposing capabilities.
 - The built-in `piAgent()` invokes Pi with `--no-tools`, once per user message, and does not preserve conversational state. It cannot test live MCP retrieval/tool use. For that, implement a custom AUT adapter that explicitly enables the desired tools and emits observed `tool-call`/`tool-result` events; do not claim a tool was used unless the adapter records it.
 - For an intervention/A-B eval, hold task, model, and other settings constant; vary only the intervention parameter, use repeated runs and a negative control, and report exactly what was varied. A memory snapshot injected into the prompt tests memory-content usefulness, not retrieval quality. Start with a dry run and narrow smoke test; Pi-backed evaluations can incur provider costs.
 - Run `bun run check` and relevant `bun test` suites when changing code. Check `git status --short` first and preserve unrelated changes.

package/dist/skills/evalkit-evals/SKILL.md

@@ -8,7 +8,7 @@ description: Define or modify EvalKit scenarios (evals) and suites. Use when add
 Read `../evalkit/SKILL.md`, `../../../sites/evalkit.leostera.dev/src/content/docs/manual/project-structure.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/scoring-and-evals.md`, `../../../examples/starter/scenarios/greeting.eval.ts`, and `../../../examples/configured-matrix/evalkit.config.ts`.
 
 1. Default-export one scenario per file from `scenarios/**/*.eval.ts` using `defineEval({ id: 'my-scenario', name, agent, transcript, scoring, datasets?, repeat?, policy? })`. The CLI discovers these files automatically; no registry or config is required. Choose a stable lowercase kebab-case ID. `metadata` is optional; `name` is a display label. Import shared agents, judges and datasets; keep each scenario short and concrete (a suite of a hundred cases is a hundred files), and never generate scenarios in a loop.
-2. Executable transcript steps are `user('message')`, inline `predicate(...)`, `judge(...)`, and `expectToolCall(...)`. User messages call `session.send` in order on one fresh session per run; predicates/judges/tool-call assertions run at checkpoints. `judge(...)` requires a separate eval-level `judge` adapter. `agent(...)` transcript steps are not executable. `user(({ parameters, data, seed, batch }) => ...)` computes a message per run, so one scenario can vary its prompt by matrix axis or candidate dataset instead of being copied per variant; `repeat: n` starts `n` independent runs as one batch (default 1, positive integer; `--repeat` and `execution.repeat` override it); `policy.timeoutMs` is not enforced.
+2. Executable transcript steps are `user('message')`, inline `predicate(...)`, `judge(...)`, and `expectToolCall(...)`. User messages call `session.send` in order on one fresh session per run; predicates/judges/tool-call assertions run at checkpoints. A `judge(...)` names its judge `agent`, or uses `judge` in `evalkit.config.ts`; it is never the AUT. `agent(...)` transcript steps are not executable. `user(({ parameters, data, seed, batch }) => ...)` computes a message per run, so one scenario can vary its prompt by matrix axis or candidate dataset instead of being copied per variant; `repeat: n` starts `n` independent runs as one batch (default 1, positive integer; `--repeat` and `execution.repeat` override it); `policy.timeoutMs` is not enforced.
 3. Suites are directories: every file below `scenarios/billing/` is in suite `billing` (kebab-case directory names; deeper directories stay in the top one). Files directly in `scenarios/` have no suite; `discovery.suites: false` turns this off. Only use an explicit registry (`registry: registerEvals([defineSuite({ id, evals })])` in the config) when you need full control; it replaces discovery, and duplicate eval IDs throw.
 4. For an intervention comparison, hold task/model/prompt constant and vary one explicit matrix axis (for example `memory: ['off', 'on']`). Repeat runs and include a negative control where possible. Record whether the treatment injects a saved memory or performs live retrieval: those answer different questions. A memory dataset can be evaluator-visible to the trusted adapter, but do not expose evaluator roots or reference answers to the AUT unless revealing the chosen treatment is intentional.
 5. From the project directory run `bun run evalkit run <eval-id>`. Use `bun run evalkit run --suite <suite-id>` for a suite directory. IDs are exact human-defined strings. Inspect per-run `scoring.json` and the run `summary.json`; completed execution is not the same as passing scores.

package/dist/skills/evalkit-judges/SKILL.md

@@ -5,13 +5,13 @@ description: Write or debug EvalKit scoring rules and judges. Use for predicate
 
 # Score EvalKit runs
 
-Read `../evalkit/SKILL.md`, `../../../sites/evalkit.leostera.dev/src/content/docs/manual/scoring-and-evals.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/results.md`, `../../../packages/runner/src/index.ts` (`normalizeScore`, `scoreSummary`, `runPredicate`), and `../../../examples/starter/judges/greeting.ts`.
+Read `../evalkit/SKILL.md`, `../../../sites/evalkit.leostera.dev/src/content/docs/manual/scoring-and-evals.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/results.md`, `../../../packages/runner/src/score.ts` (`normalizeScore`), `../../../packages/runner/src/judge-agent.ts` (prompt, retry, recorded fields), `../../../packages/core/src/judge.ts` (`choices`, `criteria`, `judgePrompt`), `../../../examples/starter/judges/greeting.ts`, and `../../../examples/judges/` (choices, criteria and evidence predicates with a matrix-chosen judge model).
 
-EvalKit currently executes deterministic `predicate(name, callback)` scorers and agent-backed `judge(name, { rubric })` rules. A judge rule requires a **separate eval-level `judge` adapter**; it is not inferred from or shared with the AUT. The `judgeScorer(...)` API is not implemented—do not suggest it.
+EvalKit currently executes deterministic `predicate(name, callback)` scorers and agent-backed `judge(name, { rubric })` rules. A judge rule runs on its own `agent` or the project's `judge` (`defineConfig({ judge })`); it is never the AUT. Give every rule a `description` saying what it checks: the dashboard, `--dry-run` and failure lines show it. The `judgeScorer(...)` API is not implemented—do not suggest it.
 
 1. Add a named `predicate(...)` for deterministic checks or a `judge(...)` descriptor for rubric-based LLM assessment, then reference it from the eval's `scoring` array or supported transcript checkpoint. The predicate callback receives `{ context, output, data, trajectory, artifacts, checkpoints, turn? }`; `checkpoints` are earlier checkpoint results (all of them in final scoring), so a checkpoint that inspects live state before the session closes can hand its `evidence` to final scorers. Use `output` for the final assistant reply and `data[id]` for dataset values (expected answers belong in an evaluator dataset); only filter `trajectory.events` (by `source: 'aut'` and `kind`) for tool calls or intermediate messages. `context` includes per-run IDs, metadata, random seed and optional parameters; workspace roots are absolute local paths.
 2. A predicate returns a boolean, a finite score from 0 to 1, `{ value, passed?, explanation?, evidence? }` (evidence may be any plain value), or `skip(reason)` when the check does not apply. Without `passed`, only `value === 1` passes; give `predicate(name, fn, { passAt: 0.8 })` for a threshold. A skipped result neither passes nor fails and is left out of averages; never fake "not applicable" as a pass. Use clear evidence for debugging.
-3. A `judge(...)` rule sends a rubric plus the current turn or trajectory to the separate judge adapter, which must emit a JSON verdict. Validate that the judge is independent from the AUT, runs in an appropriate workspace, and does not receive secrets. Judge events/prompts may contain sensitive data and incur model cost; cost is not inferred unless the adapter emits usage.
+3. A `judge(...)` rule: EvalKit builds the prompt from `rubric`, `inputs` (`{ name: (ctx) => value }`, default `conversation` and `reply`) and `answer` (`choices({ A: [label, value] })`, `criteria({ name: weight })`, or the default 0–1 verdict), reads the reply, and asks once more if it cannot. Set `passAt` (default only 1). The judge agent receives `context.parameters`, so a matrix axis such as `judgeModel` can choose its model; it should return `{ reply, usage, model }`. Results save the prompt, reply, conversation, usage, model and `passAt`; judging tokens are the built-in `judge tokens` metric, apart from `tokens`. A predicate may return `criteria` instead of `value`; the score is their weighted mean. Validate that the judge is independent from the AUT, runs in an appropriate workspace, and does not receive secrets. Judge events/prompts may contain sensitive data and incur model cost; cost is not inferred unless the adapter emits usage.
 4. Normal scorers are skipped after primary execution errors unless declared with `{ supportsPartial: true }` and the required context/workspace exists. Scorer exceptions and invalid scores become recorded failures; never mask them as passes.
 5. Scorers run after the AUT closes; use `node:fs/promises` to inspect candidate/evaluator files before workspace cleanup. Keep evaluator references and secrets out of AUT events, evidence, and exported reports. Reading one evaluator dataset in a trusted adapter and deliberately injecting it as an experiment treatment is valid, but measures injected context—not whether the AUT retrieved it itself.
 6. Numbers to compare rather than gate on (tokens, time, retries, mistakes, web searches) are **metrics**, not scorers: `metric(name, (context) => number | undefined, { unit?, better? })` in the eval's `metrics` array. Never fake one as a predicate that always passes. Agent time, tokens (when the adapter reports usage) and tool calls are recorded automatically; have the adapter emit `turn-completed` usage so tokens appear. Means show on the run page and scoreboard.

package/dist/skills/evalkit-results/SKILL.md

@@ -11,4 +11,4 @@ Read `../evalkit/SKILL.md` and `../../../sites/evalkit.leostera.dev/src/content/
 2. Each `_evalkit-results/<run-uuid>/` is one run (one execution): `manifest.json`, `catalog.json`, `trajectory.jsonl`, `scoring.json`, `summary.json`, and `artifacts/{candidate,attachments}/`. `trajectory.jsonl` contains one JSON event per line. `repeat: n` starts `n` runs sharing `manifest.batch: { id, index, size }`. Results saved before EvalKit 0.9 (a run holding `runs/`) are skipped until `bun run evalkit upgrade` converts them; it only reads the config, so run it first when updating.
 3. A run's `summary.json` `status` is `passed`, `failed` (finished, but a score or checkpoint did not pass), `errored` (could not finish), or `cancelled` (never a failure). `scoring.overall` is an unweighted mean of valid values, not a pass rate. For a batch, count how many of its runs passed. For CI, use `jq -e '.status == "passed"' "_evalkit-results/<run-uuid>/summary.json"` after replacing the placeholder; the CLI exits nonzero when any selected run does not pass.
 4. `--json` emits one JSON object per finished run with its cell key, eval ID, effective parameters, and run result (including `batch`). Read the saved run trajectory for adapter-emitted turn latency or token usage; EvalKit does not infer provider cost, latency, or token counts. Compare matrix conditions at both cell and run level and avoid claiming significance from a small pilot.
-5. The dashboard can start runs and inspect events, scoring, and candidate files. `_evalkit-sandbox/<run-uuid>/{candidate,evaluator}/` retains **both** local workspaces; evaluator secrets can live there. Treat reports and sandboxes as sensitive, apply Git ignore/retention rules, and don't share raw data without review. They may be deleted after inspection.
+5. The dashboard can start runs and inspect events, scoring, and candidate files. Run workspaces live outside the project, in `$TMPDIR/evalkit/<project>-<hash>/`<run-uuid>/{candidate,evaluator}/ (`sandboxDir` overrides it; inside the project, agents can read its source and other runs, and `run` warns), and kept ones retain **both** workspaces; evaluator secrets can live there. Treat reports and sandboxes as sensitive, apply Git ignore/retention rules, and don't share raw data without review. They may be deleted after inspection.