@leostera/evalkit 0.6.1 → 0.8.0

package/dist/dashboard/index.html

@@ -4,8 +4,8 @@
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>EvalKit</title>
-    <script type="module" crossorigin src="/assets/index-CTJpd8q8.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-YAabTN3X.css">
+    <script type="module" crossorigin src="/assets/index-xlVsW6ZR.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-C12IeG2y.css">
   </head>
   <body>
     <div id="root"></div>

package/dist/index.d.mts

@@ -241,9 +241,11 @@ export type EvalCatalogScorer = {
   rubric?: string;
   definedAt?: string;
 };
-export type EvalCatalogStep = {
+export type EvalCatalogStep =
+/** `message` is absent when it is computed for each trial. */
+{
   kind: 'user';
-  message: string;
+  message?: string;
 } | {
   kind: 'agent';
   expectation: JsonValue;
@@ -341,13 +343,29 @@ type EvalkitConfig = {
     maxCells?: number;
   };
   reportDir?: string;
+  /** The candidate files saved with each trial's results. */
+  artifacts?: {
+    /**
+     * Paths to leave out, as globs relative to the candidate workspace; a
+     * pattern without `/` matches that name at any depth. Replaces the default,
+     * `DEFAULT_ARTIFACT_EXCLUDE` (dependencies, build output, VCS metadata).
+     * A scenario's `workspace.ignore` patterns come after these.
+     */
+    exclude?: readonly string[];
+  };
+  /** Where each trial's real workspace lives during a run. Default `_evalkit-sandbox`. */
   sandboxDir?: string;
+  /** Which trial workspaces to keep in `sandboxDir` afterwards. Default `'failed'`: only trials that did not pass. */
+  keepSandboxes?: KeepSandboxes;
   share?: ShareConfig;
 };
+type KeepSandboxes = 'failed' | 'all' | 'none';
+/** Left out of candidate snapshots unless `artifacts.exclude` says otherwise. */
+export declare const DEFAULT_ARTIFACT_EXCLUDE: readonly string[];
 /** HTTP header names and values. */
 type ShareHeaders = Record<string, string>;
 type ShareConfig = {
-  /** Where `evalkit share` uploads results. `EVALKIT_SHARE_URL` overrides it. */
+  /** Where `evalkit share` uploads results. `--server <url>` overrides it. */
   url: string;
   /**
    * Extra headers for every request to the share server, e.g. credentials for
@@ -476,6 +494,8 @@ export declare const TrialMetadataSchema: Schema.Struct<{
     version: Schema.optional<typeof Schema.String>;
   }>>;
   startedAt: typeof Schema.String;
+  /** The effective snapshot ignore patterns: the project's, then the scenario's. */
+  workspaceIgnore: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
   datasets: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
     id: typeof Schema.String;
     visibility: Schema.Literal<["candidate", "evaluator"]>;
@@ -539,6 +559,7 @@ export declare const TrialManifestSchema: Schema.Struct<{
     version: Schema.optional<typeof Schema.String>;
   }>>;
   startedAt: typeof Schema.String;
+  workspaceIgnore: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
   datasets: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
     id: typeof Schema.String;
     visibility: Schema.Literal<["candidate", "evaluator"]>;
@@ -629,6 +650,18 @@ export declare const CheckpointResultSchema: Schema.Struct<{
     id: typeof Schema.String;
   }>>;
 }>;
+export declare const MetricResultSchema: Schema.Struct<{
+  name: typeof Schema.String;
+  value: Schema.optional<typeof Schema.Number>;
+  unit: Schema.optional<typeof Schema.String>;
+  better: Schema.optional<Schema.Literal<["lower", "higher"]>>;
+  builtIn: Schema.optional<Schema.Literal<[true]>>;
+  error: Schema.optional<Schema.Struct<{
+    name: typeof Schema.String;
+    message: typeof Schema.String;
+    stack: Schema.optional<typeof Schema.String>;
+  }>>;
+}>;
 export declare const TrialScoringSchema: Schema.Struct<{
   results: Schema.mutable<Schema.Array$<Schema.Struct<{
     name: typeof Schema.String;
@@ -659,6 +692,18 @@ export declare const TrialScoringSchema: Schema.Struct<{
     }>>;
     skipped: Schema.optional<Schema.Literal<[true]>>;
   }>>>;
+  metrics: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
+    name: typeof Schema.String;
+    value: Schema.optional<typeof Schema.Number>;
+    unit: Schema.optional<typeof Schema.String>;
+    better: Schema.optional<Schema.Literal<["lower", "higher"]>>;
+    builtIn: Schema.optional<Schema.Literal<[true]>>;
+    error: Schema.optional<Schema.Struct<{
+      name: typeof Schema.String;
+      message: typeof Schema.String;
+      stack: Schema.optional<typeof Schema.String>;
+    }>>;
+  }>>>>;
   checkpoints: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
     step: typeof Schema.Number;
     kind: Schema.Literal<["predicate", "judge", "expect-tool-call"]>;
@@ -736,6 +781,18 @@ export declare const TrialSummarySchema: Schema.Struct<{
       }>>;
       skipped: Schema.optional<Schema.Literal<[true]>>;
     }>>>;
+    metrics: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
+      name: typeof Schema.String;
+      value: Schema.optional<typeof Schema.Number>;
+      unit: Schema.optional<typeof Schema.String>;
+      better: Schema.optional<Schema.Literal<["lower", "higher"]>>;
+      builtIn: Schema.optional<Schema.Literal<[true]>>;
+      error: Schema.optional<Schema.Struct<{
+        name: typeof Schema.String;
+        message: typeof Schema.String;
+        stack: Schema.optional<typeof Schema.String>;
+      }>>;
+    }>>>>;
     checkpoints: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
       step: typeof Schema.Number;
       kind: Schema.Literal<["predicate", "judge", "expect-tool-call"]>;
@@ -995,9 +1052,20 @@ export declare function agentRuntimes(agent: AutAdapter<any>): AgentRuntimes;
  * It preserves the adapter's concrete type for application-specific helpers.
  */
 export declare function defineAgent<const T extends AutAdapter>(agent: T): T;
+/** What a computed user message may depend on: only what the agent is given too. */
+export type UserMessageInput = {
+  /** The matrix cell's parameters and CLI overrides. */
+  parameters: JsonObject;
+  /** Values of the scenario's candidate datasets, by dataset ID (never evaluator ones). */
+  data: JsonObject;
+  /** The trial's random seed (also in `context.metadata.randomSeed`). */
+  seed: number;
+  trialIndex: number;
+};
 export type UserStep = {
   kind: 'user';
-  message: string;
+  /** A fixed message, or one computed for each trial. */
+  message: string | ((input: UserMessageInput) => string);
 };
 export type AgentStep = {
   kind: 'agent';
@@ -1035,7 +1103,11 @@ export type ExpectToolCallStep = {
 export type TranscriptStep = UserStep | AgentStep | ScoringRule | ExpectToolCallStep;
 /** Observes a tool call in the preceding turn; never invokes a tool. */
 export declare function expectToolCall(expected: ExpectToolCallStep['expected']): ExpectToolCallStep;
-export declare function user(message: string): UserStep;
+/**
+ * A user message. Pass a function to compute it for each trial, e.g. from a
+ * matrix axis: `user(({ parameters }) => `Set up OCaml with ${parameters.toolchain}.`)`.
+ */
+export declare function user(message: UserStep['message']): UserStep;
 export declare function agent(expectation: JsonObject): AgentStep;
 export type JudgeRunInfo = {
   agent?: AutIdentity;
@@ -1091,6 +1163,12 @@ export type ScoringContext = {
   output?: JsonValue;
   /** Values of all the scenario's datasets (candidate and evaluator), by dataset ID. */
   data: JsonObject;
+  /**
+   * Results of the transcript's checkpoints so far, in order: all of them for
+   * final scorers. A checkpoint's `evidence` can carry what it observed (for
+   * example, in a container that is gone by final scoring).
+   */
+  checkpoints: readonly CheckpointResult[];
 };
 export type PredicateScorer = {
   kind: 'predicate';
@@ -1109,6 +1187,32 @@ export type JudgeRule = {
 };
 export type ScoringRule = PredicateScorer | JudgeRule;
 export declare function predicate(name: string, run: PredicateScorer['run'], options?: Pick<PredicateScorer, 'supportsPartial' | 'passAt'>): PredicateScorer;
+/**
+ * A number measured for every trial, such as tokens, retries or web searches.
+ * Metrics never decide pass or fail; they are averaged per run and shown on
+ * the scoreboard. Return undefined when there is nothing to measure.
+ */
+export type Metric = {
+  kind: 'metric';
+  name: string;
+  /** Shown next to values: `ms` formats as a duration, others as numbers. */
+  unit?: string;
+  /** Which direction is an improvement, for comparisons. */
+  better?: 'lower' | 'higher';
+  run(context: ScoringContext): number | undefined | Promise<number | undefined>;
+};
+export declare function metric(name: string, run: Metric['run'], options?: Pick<Metric, 'unit' | 'better'>): Metric;
+/** A metric's value for one trial. */
+export type MetricResult = {
+  name: string;
+  /** Absent when the metric measured nothing or failed. */
+  value?: number;
+  unit?: string;
+  better?: 'lower' | 'higher';
+  /** Measured by EvalKit itself (agent time, tokens, tool calls). */
+  builtIn?: true;
+  error?: RecordedError;
+};
 export declare function judge(name: string, options: Pick<JudgeRule, 'rubric' | 'supportsPartial'>): JudgeRule;
 export type EvalPolicy = {
   timeoutMs?: number;
@@ -1127,9 +1231,24 @@ export type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
   datasets?: readonly Dataset[];
   transcript: TranscriptStep[];
   scoring: ScoringRule[];
+  /** Numbers to measure for every trial; they never affect pass or fail. */
+  metrics?: readonly Metric[];
   policy?: EvalPolicy;
   /** Any JSON-convertible object; converted when the eval is defined. */
   metadata?: Record<string, unknown>;
+  workspace?: EvalWorkspace;
+};
+/** How EvalKit treats this scenario's candidate workspace. */
+export type EvalWorkspace = {
+  /**
+   * Paths to leave out of the trial's snapshot, read like `.gitignore`: globs
+   * relative to the workspace, where a pattern without `/` matches that name
+   * at any depth and later patterns win. They come after the project's
+   * `artifacts.exclude` (default `DEFAULT_ARTIFACT_EXCLUDE`), so `!pattern`
+   * can keep a path the project leaves out, even inside an ignored directory:
+   * `['!_build/doc/**']` keeps generated docs while dropping the rest of `_build`.
+   */
+  ignore?: readonly string[];
 };
 export declare function defineEval<const T extends EvalDefinition>(definition: T): T;
 export type EvalSuite<TEvals extends readonly EvalDefinition[] = readonly EvalDefinition[]> = {
@@ -1236,6 +1355,8 @@ export type TrialMetadata = {
   };
   aut?: AutIdentity;
   startedAt: string;
+  /** The effective snapshot ignore patterns: the project's `artifacts.exclude`, then the scenario's `workspace.ignore`. */
+  workspaceIgnore?: string[];
   /** From schema version 5: the datasets this trial used, with content digests. */
   datasets?: TrialDataset[];
 };
@@ -1263,8 +1384,11 @@ export type CheckpointResult = {
     id: string;
   };
 };
+/** Metrics EvalKit records for every trial; scenario metrics may not reuse these names. */
+export declare const BUILT_IN_METRICS: readonly string[];
 export type TrialScoring = {
   results: ScoreResult[];
+  metrics?: MetricResult[];
   checkpoints?: CheckpointResult[];
   skippedScorers?: string[];
   overall?: number;
@@ -1352,5 +1476,5 @@ type PiAutOptions = {
 /** A local, one-prompt-per-turn AUT backed by Pi's non-interactive print mode. */
 export declare function piAgent(options?: PiAutOptions): AutAdapter;
 //#endregion
-export type { DefinitionSite, EvalMatrix, EvalMatrixCell, EvalkitConfig, MatrixSelection, ShareConfig, ShareHeaders };
+export type { DefinitionSite, EvalMatrix, EvalMatrixCell, EvalkitConfig, KeepSandboxes, MatrixSelection, ShareConfig, ShareHeaders };
 //# sourceMappingURL=index.d.mts.map

package/dist/index.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.mts","names":[],"sources":["../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/catalog.ts","../../core/src/canonical-json.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/json.ts","../../core/src/definition-site.ts","../../core/src/schema.ts","../../core/src/trajectory.ts","../../core/src/index.ts","../../agents/src/index.ts"],"mappings":";;qBAEa,oBAAkB,OAAA;YACnB,sBAAsB,mBAAmB;;wBAGrC,SAAS;;qBAOZ,YAAU,OAAA,cAAA,OAAA;YAKX,cAAc,WAAW;YAEzB,YAAY,cAAc,eAAe,2BACxC;qBAEA,mBAAiB,OAAA,cAAA,OAAA;wBAMd,kBAAkB,cAAc,cAAc,MAAM,QAAK,OAAA,cAAA,OAAA;wBAWzD,YAAY,cAAc,cACxC,MAAM,OACN,MAAM,OACL,YAAY;wBAIC,iBAAiB,cAAc,cAC7C,eACA,eAAe;EACZ,MAAM;EAAO,MAAM;EAAM,KAAK,YAAY;;wBAY/B,aAAa,eAAe,eAAe,eAAe;;;;YCvD9D;;YAGA,6BAA6B;EAAe;;;;;;YAM5C,wBAAwB,SAAS,eAAe;;YAGhD;EACV,QAAQ;;EAER;;YAGU;;EAEV;EACA,QAAQ;;YAGE;EACV;EACA;EACA;EACA;;EAEA;;EAEA,YAAY;EACZ,QAAQ;;KAGL;EACH;EACA;;EAEA,aAAa;;EAEb;;YAGU,0BAA0B;EACpC,QAAQ;;EAER;;EAEA;EACA;EACA;;YAGU,0BAA0B,gBAAgB,kBACpD;;EAEE,SAAS,MAAM,uBAAuB,UAAU,QAAQ;;EAExD,SACE,QAAQ,SACR,OAAO,wBACJ,iBAAiB,QAAQ;EAC9B;EACA;EACA;;YAGQ;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,OAAO,wBACJ,iBAAiB,QAAQ;;;YAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;wBAIH,oBAAoB,eAAe;;;;;;;wBAuBnC,cAAc,gBAAgB,gBAC5C,YAAY,0BAA0B,0BAA0B,WAC/D;wBAmFa,UAAU,iBAAiB,SAAS;;;;;;;YC/MxC;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;YAE/B;EACV;EACA;;EAEA;EACA;;;YAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;;wBAIE,kBACd,OAAO,eACP,WAAW,OACV;;;;;wBA0Da,iBACd,QAAQ,YACR,cACA,WAAW,OACV;;;;;;;;;YCtGS;;EAEV;;YAGU;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;YAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;YAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;YAGU;EACV;EACA,MAAM;EACN;EACA;;EAEA;EACA;;YAGU;EACN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;YAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;EACV;EACA;IAAU;IAAgB;IAAoB;;EAC9C,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;wBAGc,aACd,YAAY,gBACZ,OAAO,uBACP,UAAS,iBACR;;;;wBC3Fa,oBAAoB,OAAO;;;KCK/B;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;wBAWvC,uBAAuB,UAAU,sBAC/C,YAAY,IACX,aAAa;;;KCjDJ;;EAEV;EACA;EACA;EACA;;IAEE;;;EAGF,iBAAiB;EACjB,WAAW;;EAEX,SAAS,KAAK;IAA0C;;EACxD;IAAc;IAAsB;IAAiB;;EACrD;EACA;EACA,QAAQ;;;KAIE,eAAe;KAEf;;EAEV;;;;;;EAMA,UAAU,sBAAsB,eAAe,QAAQ;;;wBAIzC,mBAAmB,UAAU,eAC3C,eAAe,IACd;;;;qBCrCU,qBAAqB;WAGrB;WACA;WAHO;EAClB,YACW,cACA;;;;;;;;wBAYG,OAAO,gBAAgB,gBAAiB;;wBA4CxC,aACd,gBACA;GACI,cAAc;;;;;;;;;;;KC3DR;EAAmB;EAAc;;wBAqB7B,eAAe,iBAAiB;;;;qBCvBnC,iBAAe,OAAA;qBAOf,qBAAmB,OAAA;;;;;qBAMnB,mBAAiB,OAAA;;;;;;qBAQjB,iBAAiB,OAAO,OAAO;qBAoB/B,iBAAe,OAAA;;;;;;;;;;;;;;qBAiBf,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAejB,qBAAmB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;qBAyBnB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAIjB,qBAAmB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;qBAKnB,aAAW,OAAA;;;;;qBAMX,oBAAkB,OAAA;;;;;;;;;;;;;;qBAalB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAYjB,wBAAsB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAgBtB,oBAAkB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBASlB,qBAAmB,OAAA;;;;;qBAKnB,oBAAkB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBASlB,kBAAgB,OAAA;;;;;;;;;;;;;;qBAmJhB,uBAAuB,OAAO,OAAO;qBAGrC,gBAAc,OAAA;;;;;;;wBCvUX,qBAAqB,mBAAmB;;;YCc5C;YACA,YACV,gBAAgB;GAAiB,cAAc;;YACrC;GAAgB,cAAc;;YAE9B;;EAEV;EACA;EACA;EACA;;YAGU;;;;;YAMA;EACV;EACA,gBAAgB;;YAGN,gBAAgB,QAAQ,OAAO,kBAAkB;YAEjD;EACV;EACA;EACA;;YAGU;EACV;EACA;EACA;;YAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;YAE/B;EACN;EAAuB;;EAEvB;EACA;;EAEA;EACA;;EAEA;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAyB;;EACzB;EAAe,OAAO;EAAe;;YAE/B,mBACT;EAAa;MAAoB;EAAgB;;YAExC;;EAEV;;YAGU,qBAAqB;YACrB,qBAAqB;YAErB;EACV;EACA;EACA;EACA;EACA,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;EAEN,WAAW;;EAEX,oBAAoB;;YAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;IAEV,UAAU,OAAO,yBAAyB;MACxC,QAAQ,WAAW;;YAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;;qBAIC,wBAAwB;;wBAKrB,cAAc,OAAO,kBAAkB;;;;;wBAUvC,kBAAkB,UAAU,YAAY,OAAO,IAAI;YAKvD;EACV;EACA;;YAGU;EACV;EACA,aAAa;;YAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;YAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;YAE5B,iBACV,WAAW,YAAY,cAAc;;wBAGvB,eACd,UAAU,iCACT;wBAMa,KAAK,kBAAkB;wBAIvB,MAAM,aAAa,aAAa;YAIpC;EACV,QAAQ;EACR,QAAQ;;EAER,SAAS;;;YAIC;EAAiB;EAAe;;YAEhC,sBAER;EAEE;EACA;EACA;;EAEA;;EAEA,QAAQ;;;;;;wBAOE,KAAK,iBAAiB;YAK1B;EACV;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;;EAER;;YAGU;EACV,WAAW;EACX,WAAW;;YAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;EAEP,SAAS;;EAET,MAAM;;YAGI;EACV;EACA;EACA;;EAEA;EACA,IACE,SAAS,2BACE,aAAa,kBAAkB;;YAGlC;EACV;EACA;EACA;;EAEA;;YAGU,cAAc,kBAAkB;wBAE5B,UACd,cACA,KAAK,wBACL,UAAS,KAAK,iDACb;wBAgBa,MACd,cACA,SAAS,KAAK,2CACb;YAMS;EACV;;EAEA;;EAEA;;YAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,QAAQ;;EAER,oBAAoB;EACpB,YAAY;EACZ,SAAS;EACT,SAAS;;EAET,WAAW;;wBAGG,iBAAiB,UAAU,gBAAgB,YAAY,IAAI;YAyC/D,UACV,wBAAwB,4BAA4B;EAEpD;EACA;EACA,OAAO;;wBAGO,kBAAkB,eAAe,WAC/C,OAAO,SACN;YAKS,mBAAmB,iBAAiB,YAAY;wBAE5C,YAAY;EAAS;;YAkBzB;;EAEV,gBAAgB;EAChB,iBAAiB;EACjB,mBAAmB;EACnB,IAAI,cAAc;EAClB,SAAS,cAAc;EACvB,YAAY;EACZ,iBAAiB;;EAEjB,QAAQ,UAAU,iBAAiB;;YAGzB;EACV;EACA;EACA;;YAGU;EACV;EACA;EACA;;;;;;wBAOc,oBACR,gCAAgC,oBACtC,eAAe,iBAAiB;YAuEtB;;;;;YAMA;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;YAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;YAIE;EAAe;EAAa;;YAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;;EAET,UAAU;;YAGA;EACV;EACA;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,WAAW;;YAGD;EACV;EACA,YAAY;EACZ;;EAEA;;YAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;YAG9B;EACV,SAAS;EACT,cAAc;EACd;EACA;EACA;;YAGU;;EAEV;EACA;EACA;;YAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,QAAQ;;YAGE;EACV,QAAQ;EACR;EACA;EACA;EACA;EACA;;EAEA;EACA,QAAQ;;YAGE;EACV,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,eAAe;;EAEpC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,eAAe;;YAGvB;WACD;EACT,WAAW,UAAU,gBAAgB,QAAQ;EAC7C,SAAS,SAAS,aAAa;;YAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;YAGD;EACV;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;YAGE;EACV;EACA;EACA;EACA;EACA;;YAGU,YAAY;EACtB;EACA;EACA;EACA;EACA,mBAAmB;;EAEnB,SAAS;;wBAYK,YAAY,iBAAiB;;;KCxuBjC;EACV;;EAEA;;;wBAIc,QAAQ,UAAS,eAAoB"}
+{"version":3,"file":"index.d.mts","names":[],"sources":["../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/catalog.ts","../../core/src/canonical-json.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/json.ts","../../core/src/definition-site.ts","../../core/src/schema.ts","../../core/src/trajectory.ts","../../core/src/index.ts","../../agents/src/index.ts"],"mappings":";;qBAEa,oBAAkB,OAAA;YACnB,sBAAsB,mBAAmB;;wBAGrC,SAAS;;qBAOZ,YAAU,OAAA,cAAA,OAAA;YAKX,cAAc,WAAW;YAEzB,YAAY,cAAc,eAAe,2BACxC;qBAEA,mBAAiB,OAAA,cAAA,OAAA;wBAMd,kBAAkB,cAAc,cAAc,MAAM,QAAK,OAAA,cAAA,OAAA;wBAWzD,YAAY,cAAc,cACxC,MAAM,OACN,MAAM,OACL,YAAY;wBAIC,iBAAiB,cAAc,cAC7C,eACA,eAAe;EACZ,MAAM;EAAO,MAAM;EAAM,KAAK,YAAY;;wBAY/B,aAAa,eAAe,eAAe,eAAe;;;;YCvD9D;;YAGA,6BAA6B;EAAe;;;;;;YAM5C,wBAAwB,SAAS,eAAe;;YAGhD;EACV,QAAQ;;EAER;;YAGU;;EAEV;EACA,QAAQ;;YAGE;EACV;EACA;EACA;EACA;;EAEA;;EAEA,YAAY;EACZ,QAAQ;;KAGL;EACH;EACA;;EAEA,aAAa;;EAEb;;YAGU,0BAA0B;EACpC,QAAQ;;EAER;;EAEA;EACA;EACA;;YAGU,0BAA0B,gBAAgB,kBACpD;;EAEE,SAAS,MAAM,uBAAuB,UAAU,QAAQ;;EAExD,SACE,QAAQ,SACR,OAAO,wBACJ,iBAAiB,QAAQ;EAC9B;EACA;EACA;;YAGQ;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,OAAO,wBACJ,iBAAiB,QAAQ;;;YAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;wBAIH,oBAAoB,eAAe;;;;;;;wBAuBnC,cAAc,gBAAgB,gBAC5C,YAAY,0BAA0B,0BAA0B,WAC/D;wBAmFa,UAAU,iBAAiB,SAAS;;;;;;;YC/MxC;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;YAE/B;EACV;EACA;;EAEA;EACA;;;YAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;;wBAIE,kBACd,OAAO,eACP,WAAW,OACV;;;;;wBA0Da,iBACd,QAAQ,YACR,cACA,WAAW,OACV;;;;;;;;;YCtGS;;EAEV;;YAGU;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;YAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;YAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;YAGU;EACV;EACA,MAAM;EACN;EACA;;EAEA;EACA;;YAGU;;;EAEN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;YAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;EACV;EACA;IAAU;IAAgB;IAAoB;;EAC9C,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;wBAGc,aACd,YAAY,gBACZ,OAAO,uBACP,UAAS,iBACR;;;;wBC5Fa,oBAAoB,OAAO;;;KCK/B;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;wBAWvC,uBAAuB,UAAU,sBAC/C,YAAY,IACX,aAAa;;;KCjDJ;;EAEV;EACA;EACA;EACA;;IAEE;;;EAGF,iBAAiB;EACjB,WAAW;;EAEX,SAAS,KAAK;IAA0C;;EACxD;IAAc;IAAsB;IAAiB;;EACrD;;EAEA;;;;;;;IAOE;;;EAGF;;EAEA,gBAAgB;EAChB,QAAQ;;KAGE;;qBAGC;;KAWD,eAAe;KAEf;;EAEV;;;;;;EAMA,UAAU,sBAAsB,eAAe,QAAQ;;;wBAIzC,mBAAmB,UAAU,eAC3C,eAAe,IACd;;;;qBC/DU,qBAAqB;WAGrB;WACA;WAHO;EAClB,YACW,cACA;;;;;;;;wBAYG,OAAO,gBAAgB,gBAAiB;;wBA4CxC,aACd,gBACA;GACI,cAAc;;;;;;;;;;;KC3DR;EAAmB;EAAc;;wBAqB7B,eAAe,iBAAiB;;;;qBCvBnC,iBAAe,OAAA;qBAOf,qBAAmB,OAAA;;;;;qBAMnB,mBAAiB,OAAA;;;;;;qBAQjB,iBAAiB,OAAO,OAAO;qBAoB/B,iBAAe,OAAA;;;;;;;;;;;;;;qBAiBf,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAejB,qBAAmB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBA2BnB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAIjB,qBAAmB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAKnB,aAAW,OAAA;;;;;qBAMX,oBAAkB,OAAA;;;;;;;;;;;;;;qBAalB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAYjB,wBAAsB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAgBtB,oBAAkB,OAAA;;;;;;;;;;;;qBAQlB,oBAAkB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAUlB,qBAAmB,OAAA;;;;;qBAKnB,oBAAkB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBASlB,kBAAgB,OAAA;;;;;;;;;;;;;;qBAmJhB,uBAAuB,OAAO,OAAO;qBAGrC,gBAAc,OAAA;;;;;;;wBClVX,qBAAqB,mBAAmB;;;YCgB5C;YACA,YACV,gBAAgB;GAAiB,cAAc;;YACrC;GAAgB,cAAc;;YAE9B;;EAEV;EACA;EACA;EACA;;YAGU;;;;;YAMA;EACV;EACA,gBAAgB;;YAGN,gBAAgB,QAAQ,OAAO,kBAAkB;YAEjD;EACV;EACA;EACA;;YAGU;EACV;EACA;EACA;;YAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;YAE/B;EACN;EAAuB;;EAEvB;EACA;;EAEA;EACA;;EAEA;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAyB;;EACzB;EAAe,OAAO;EAAe;;YAE/B,mBACT;EAAa;MAAoB;EAAgB;;YAExC;;EAEV;;YAGU,qBAAqB;YACrB,qBAAqB;YAErB;EACV;EACA;EACA;EACA;EACA,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;EAEN,WAAW;;EAEX,oBAAoB;;YAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;IAEV,UAAU,OAAO,yBAAyB;MACxC,QAAQ,WAAW;;YAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;;qBAIC,wBAAwB;;wBAKrB,cAAc,OAAO,kBAAkB;;;;;wBAUvC,kBAAkB,UAAU,YAAY,OAAO,IAAI;;YAMvD;;EAEV,YAAY;;EAEZ,MAAM;;EAEN;EACA;;YAGU;EACV;;EAEA,oBAAoB,OAAO;;YAGjB;EACV;EACA,aAAa;;YAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;YAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;YAE5B,iBACV,WAAW,YAAY,cAAc;;wBAGvB,eACd,UAAU,iCACT;;;;;wBAUa,KAAK,SAAS,sBAAsB;wBAIpC,MAAM,aAAa,aAAa;YAIpC;EACV,QAAQ;EACR,QAAQ;;EAER,SAAS;;;YAIC;EAAiB;EAAe;;YAEhC,sBAER;EAEE;EACA;EACA;;EAEA;;EAEA,QAAQ;;;;;;wBAOE,KAAK,iBAAiB;YAK1B;EACV;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;;EAER;;YAGU;EACV,WAAW;EACX,WAAW;;YAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;EAEP,SAAS;;EAET,MAAM;;;;;;EAMN,sBAAsB;;YAGZ;EACV;EACA;EACA;;EAEA;EACA,IACE,SAAS,2BACE,aAAa,kBAAkB;;YAGlC;EACV;EACA;EACA;;EAEA;;YAGU,cAAc,kBAAkB;wBAE5B,UACd,cACA,KAAK,wBACL,UAAS,KAAK,iDACb;;;;;;YAqBS;EACV;EACA;;EAEA;;EAEA;EACA,IACE,SAAS,sCACa;;wBAGV,OACd,cACA,KAAK,eACL,UAAS,KAAK,6BACb;;YAMS;EACV;;EAEA;EACA;EACA;;EAEA;EACA,QAAQ;;wBAGM,MACd,cACA,SAAS,KAAK,2CACb;YAMS;EACV;;EAEA;;EAEA;;YAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,QAAQ;;EAER,oBAAoB;EACpB,YAAY;EACZ,SAAS;;EAET,mBAAmB;EACnB,SAAS;;EAET,WAAW;EACX,YAAY;;;YAIF;;;;;;;;;EASV;;wBAGc,iBAAiB,UAAU,gBAAgB,YAAY,IAAI;YA0D/D,UACV,wBAAwB,4BAA4B;EAEpD;EACA;EACA,OAAO;;wBAGO,kBAAkB,eAAe,WAC/C,OAAO,SACN;YAKS,mBAAmB,iBAAiB,YAAY;wBAE5C,YAAY;EAAS;;YAkBzB;;EAEV,gBAAgB;EAChB,iBAAiB;EACjB,mBAAmB;EACnB,IAAI,cAAc;EAClB,SAAS,cAAc;EACvB,YAAY;EACZ,iBAAiB;;EAEjB,QAAQ,UAAU,iBAAiB;;YAGzB;EACV;EACA;EACA;;YAGU;EACV;EACA;EACA;;;;;;wBAOc,oBACR,gCAAgC,oBACtC,eAAe,iBAAiB;YAuEtB;;;;;YAMA;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;YAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;YAIE;EAAe;EAAa;;YAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;;EAET,UAAU;;YAGA;EACV;EACA;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA;;EAEA,WAAW;;YAGD;EACV;EACA,YAAY;EACZ;;EAEA;;YAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;;qBAI7B;YAMD;EACV,SAAS;EACT,UAAU;EACV,cAAc;EACd;EACA;EACA;;YAGU;;EAEV;EACA;EACA;;YAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,QAAQ;;YAGE;EACV,QAAQ;EACR;EACA;EACA;EACA;EACA;;EAEA;EACA,QAAQ;;YAGE;EACV,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,eAAe;;EAEpC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,eAAe;;YAGvB;WACD;EACT,WAAW,UAAU,gBAAgB,QAAQ;EAC7C,SAAS,SAAS,aAAa;;YAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;YAGD;EACV;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;YAGE;EACV;EACA;EACA;EACA;EACA;;YAGU,YAAY;EACtB;EACA;EACA;EACA;EACA,mBAAmB;;EAEnB,SAAS;;wBAYK,YAAY,iBAAiB;;;KCj1BjC;EACV;;EAEA;;;wBAIc,QAAQ,UAAS,eAAoB"}

package/dist/index.mjs

@@ -1,10 +1,4 @@
-import { $ as resourceUriSchema, A as TrialManifestSchema, B as NotJsonError, C as RunManifestSchema, D as RunSummarySchema, E as RunStatusSchema, F as normalizeAutEvent, G as relativeDatasetPath, H as toJsonObject, I as turnResultEvents, J as ResourceUriSchema, K as definitionSite, L as defineEvalMatrix, M as TrialScoringSchema, N as TrialSummarySchema, O as ScoreResultSchema, P as UsageSchema, Q as resourceUri, R as canonicalParameters, S as RecordedErrorSchema, T as RunSourceSchema, U as defineDataset, V as toJson, W as isDataset, X as authorId, Y as UuidSchema, Z as parseResourceUri, _ as ArtifactEntrySchema, a as defineAgent, b as JsonValueSchema, c as expectToolCall, d as recordError, et as resourceUuid, f as registerEvals, g as ApiErrorSchema, h as parseTrajectoryJsonl, i as authoringId, j as TrialMetadataSchema, k as TrajectoryEventSchema, l as judge, m as user, n as agent, o as defineEval, p as skip, q as ResourceKindSchema, r as agentRuntimes, s as defineSuite, t as DEFAULT_AGENT_RUNTIMES, u as predicate, v as AutIdentitySchema, w as RunMetadataSchema, x as JudgeRunInfoSchema, y as CheckpointResultSchema, z as catalogEntry } from "./src-CZPufWN_.mjs";
-//#region ../core/src/config.ts
-/** Pure, typed project configuration. Loading/discovery belongs to the CLI. */
-function defineConfig(configuration) {
-	return configuration;
-}
-//#endregion
+import { $ as ResourceUriSchema, A as RunSummarySchema, B as defineEvalMatrix, C as JudgeRunInfoSchema, D as RunMetadataSchema, E as RunManifestSchema, F as TrialScoringSchema, G as NotJsonError, H as DEFAULT_ARTIFACT_EXCLUDE, I as TrialSummarySchema, J as defineDataset, K as toJson, L as UsageSchema, M as TrajectoryEventSchema, N as TrialManifestSchema, O as RunSourceSchema, P as TrialMetadataSchema, Q as ResourceKindSchema, R as normalizeAutEvent, S as JsonValueSchema, T as RecordedErrorSchema, U as defineConfig, V as canonicalParameters, W as catalogEntry, X as relativeDatasetPath, Y as isDataset, Z as definitionSite, _ as parseTrajectoryJsonl, a as authoringId, at as resourceUuid, b as AutIdentitySchema, c as defineSuite, d as metric, et as UuidSchema, f as predicate, g as user, h as skip, i as agentRuntimes, it as resourceUriSchema, j as ScoreResultSchema, k as RunStatusSchema, l as expectToolCall, m as registerEvals, n as DEFAULT_AGENT_RUNTIMES, nt as parseResourceUri, o as defineAgent, p as recordError, q as toJsonObject, r as agent, rt as resourceUri, s as defineEval, t as BUILT_IN_METRICS, tt as authorId, u as judge, v as ApiErrorSchema, w as MetricResultSchema, x as CheckpointResultSchema, y as ArtifactEntrySchema, z as turnResultEvents } from "./src-LeGSyDD2.mjs";
 //#region ../agents/src/index.ts
 /** A local, one-prompt-per-turn AUT backed by Pi's non-interactive print mode. */
 function piAgent(options = {}) {
@@ -74,4 +68,4 @@ function piAgent(options = {}) {
 	});
 }
 //#endregion
-export { ApiErrorSchema, ArtifactEntrySchema, AutIdentitySchema, CheckpointResultSchema, DEFAULT_AGENT_RUNTIMES, JsonValueSchema, JudgeRunInfoSchema, NotJsonError, RecordedErrorSchema, ResourceKindSchema, ResourceUriSchema, RunManifestSchema, RunMetadataSchema, RunSourceSchema, RunStatusSchema, RunSummarySchema, ScoreResultSchema, TrajectoryEventSchema, TrialManifestSchema, TrialMetadataSchema, TrialScoringSchema, TrialSummarySchema, UsageSchema, UuidSchema, agent, agentRuntimes, authorId, authoringId, canonicalParameters, catalogEntry, defineAgent, defineConfig, defineDataset, defineEval, defineEvalMatrix, defineSuite, definitionSite, expectToolCall, isDataset, judge, normalizeAutEvent, parseResourceUri, parseTrajectoryJsonl, piAgent, predicate, recordError, registerEvals, relativeDatasetPath, resourceUri, resourceUriSchema, resourceUuid, skip, toJson, toJsonObject, turnResultEvents, user };
+export { ApiErrorSchema, ArtifactEntrySchema, AutIdentitySchema, BUILT_IN_METRICS, CheckpointResultSchema, DEFAULT_AGENT_RUNTIMES, DEFAULT_ARTIFACT_EXCLUDE, JsonValueSchema, JudgeRunInfoSchema, MetricResultSchema, NotJsonError, RecordedErrorSchema, ResourceKindSchema, ResourceUriSchema, RunManifestSchema, RunMetadataSchema, RunSourceSchema, RunStatusSchema, RunSummarySchema, ScoreResultSchema, TrajectoryEventSchema, TrialManifestSchema, TrialMetadataSchema, TrialScoringSchema, TrialSummarySchema, UsageSchema, UuidSchema, agent, agentRuntimes, authorId, authoringId, canonicalParameters, catalogEntry, defineAgent, defineConfig, defineDataset, defineEval, defineEvalMatrix, defineSuite, definitionSite, expectToolCall, isDataset, judge, metric, normalizeAutEvent, parseResourceUri, parseTrajectoryJsonl, piAgent, predicate, recordError, registerEvals, relativeDatasetPath, resourceUri, resourceUriSchema, resourceUuid, skip, toJson, toJsonObject, turnResultEvents, user };

package/dist/runner.d.mts

@@ -221,9 +221,11 @@ type EvalCatalogScorer = {
   rubric?: string;
   definedAt?: string;
 };
-type EvalCatalogStep = {
+type EvalCatalogStep =
+/** `message` is absent when it is computed for each trial. */
+{
   kind: 'user';
-  message: string;
+  message?: string;
 } | {
   kind: 'agent';
   expectation: JsonValue;
@@ -292,6 +294,9 @@ type EvalMatrix = EvalMatrixDefinition & {
   cells(selection?: MatrixSelection): IterableIterator<EvalMatrixCell>;
 };
 //#endregion
+//#region ../core/src/config.d.ts
+type KeepSandboxes = 'failed' | 'all' | 'none';
+//#endregion
 //#region ../core/src/index.d.ts
 type JsonPrimitive = boolean | number | string | null;
 type JsonValue = JsonPrimitive | JsonValue[] | {
@@ -469,9 +474,20 @@ type AutSession<TMessage = string> = {
   /** Optional; defaults to doing nothing. */
   close?(): Promise<void>;
 };
+/** What a computed user message may depend on: only what the agent is given too. */
+type UserMessageInput = {
+  /** The matrix cell's parameters and CLI overrides. */
+  parameters: JsonObject;
+  /** Values of the scenario's candidate datasets, by dataset ID (never evaluator ones). */
+  data: JsonObject;
+  /** The trial's random seed (also in `context.metadata.randomSeed`). */
+  seed: number;
+  trialIndex: number;
+};
 type UserStep = {
   kind: 'user';
-  message: string;
+  /** A fixed message, or one computed for each trial. */
+  message: string | ((input: UserMessageInput) => string);
 };
 type AgentStep = {
   kind: 'agent';
@@ -556,6 +572,12 @@ type ScoringContext = {
   output?: JsonValue;
   /** Values of all the scenario's datasets (candidate and evaluator), by dataset ID. */
   data: JsonObject;
+  /**
+   * Results of the transcript's checkpoints so far, in order: all of them for
+   * final scorers. A checkpoint's `evidence` can carry what it observed (for
+   * example, in a container that is gone by final scoring).
+   */
+  checkpoints: readonly CheckpointResult[];
 };
 type PredicateScorer = {
   kind: 'predicate';
@@ -573,6 +595,31 @@ type JudgeRule = {
   supportsPartial?: boolean;
 };
 type ScoringRule = PredicateScorer | JudgeRule;
+/**
+ * A number measured for every trial, such as tokens, retries or web searches.
+ * Metrics never decide pass or fail; they are averaged per run and shown on
+ * the scoreboard. Return undefined when there is nothing to measure.
+ */
+type Metric = {
+  kind: 'metric';
+  name: string;
+  /** Shown next to values: `ms` formats as a duration, others as numbers. */
+  unit?: string;
+  /** Which direction is an improvement, for comparisons. */
+  better?: 'lower' | 'higher';
+  run(context: ScoringContext): number | undefined | Promise<number | undefined>;
+};
+/** A metric's value for one trial. */
+type MetricResult = {
+  name: string;
+  /** Absent when the metric measured nothing or failed. */
+  value?: number;
+  unit?: string;
+  better?: 'lower' | 'higher';
+  /** Measured by EvalKit itself (agent time, tokens, tool calls). */
+  builtIn?: true;
+  error?: RecordedError;
+};
 type EvalPolicy = {
   timeoutMs?: number;
   /** Stop later transcript steps after a failed checkpoint assertion. Default false. */
@@ -590,9 +637,24 @@ type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
   datasets?: readonly Dataset[];
   transcript: TranscriptStep[];
   scoring: ScoringRule[];
+  /** Numbers to measure for every trial; they never affect pass or fail. */
+  metrics?: readonly Metric[];
   policy?: EvalPolicy;
   /** Any JSON-convertible object; converted when the eval is defined. */
   metadata?: Record<string, unknown>;
+  workspace?: EvalWorkspace;
+};
+/** How EvalKit treats this scenario's candidate workspace. */
+type EvalWorkspace = {
+  /**
+   * Paths to leave out of the trial's snapshot, read like `.gitignore`: globs
+   * relative to the workspace, where a pattern without `/` matches that name
+   * at any depth and later patterns win. They come after the project's
+   * `artifacts.exclude` (default `DEFAULT_ARTIFACT_EXCLUDE`), so `!pattern`
+   * can keep a path the project leaves out, even inside an ignored directory:
+   * `['!_build/doc/**']` keeps generated docs while dropping the rest of `_build`.
+   */
+  ignore?: readonly string[];
 };
 type EvalRegistryMetadata = {
   id: string;
@@ -671,6 +733,8 @@ type TrialMetadata = {
   };
   aut?: AutIdentity;
   startedAt: string;
+  /** The effective snapshot ignore patterns: the project's `artifacts.exclude`, then the scenario's `workspace.ignore`. */
+  workspaceIgnore?: string[];
   /** From schema version 5: the datasets this trial used, with content digests. */
   datasets?: TrialDataset[];
 };
@@ -700,6 +764,7 @@ type CheckpointResult = {
 };
 type TrialScoring = {
   results: ScoreResult[];
+  metrics?: MetricResult[];
   checkpoints?: CheckpointResult[];
   skippedScorers?: string[];
   overall?: number;
@@ -870,6 +935,7 @@ export declare const readTrialManifest: (root: string, runId: string, trialId: s
     readonly version?: string | undefined;
   } | undefined;
   readonly startedAt: string;
+  readonly workspaceIgnore?: string[] | undefined;
   readonly datasets?: {
     readonly id: string;
     readonly visibility: "candidate" | "evaluator";
@@ -912,6 +978,18 @@ export declare const readTrialSummary: (root: string, runId: string, trialId: st
       } | undefined;
       readonly skipped?: true | undefined;
     }[];
+    readonly metrics?: {
+      readonly name: string;
+      readonly value?: number | undefined;
+      readonly unit?: string | undefined;
+      readonly better?: "higher" | "lower" | undefined;
+      readonly builtIn?: true | undefined;
+      readonly error?: {
+        readonly name: string;
+        readonly message: string;
+        readonly stack?: string | undefined;
+      } | undefined;
+    }[] | undefined;
     readonly checkpoints?: {
       readonly step: number;
       readonly kind: "expect-tool-call" | "judge" | "predicate";
@@ -991,6 +1069,18 @@ export declare const readTrialScoring: (root: string, runId: string, trialId: st
     } | undefined;
     readonly skipped?: true | undefined;
   }[];
+  readonly metrics?: {
+    readonly name: string;
+    readonly value?: number | undefined;
+    readonly unit?: string | undefined;
+    readonly better?: "higher" | "lower" | undefined;
+    readonly builtIn?: true | undefined;
+    readonly error?: {
+      readonly name: string;
+      readonly message: string;
+      readonly stack?: string | undefined;
+    } | undefined;
+  }[] | undefined;
   readonly checkpoints?: {
     readonly step: number;
     readonly kind: "expect-tool-call" | "judge" | "predicate";
@@ -1052,6 +1142,10 @@ export declare function runMatrix(matrix: EvalMatrix, options: Omit<RunEvalOptio
   onResult?: (cell: EvalMatrixCell, result: RunResult) => void | Promise<void>;
 }): Effect.Effect<MatrixSummary, unknown>;
 //#endregion
+//#region ../runner/src/workspace.d.ts
+/** Where a trial's persistent workspace lives below `root`. */
+export declare function trialWorkspacePath(root: string, trialId: string): string;
+//#endregion
 //#region ../runner/src/index.d.ts
 export type RunEvalOptions = {
   report: ReportStore;
@@ -1072,6 +1166,13 @@ export type RunEvalOptions = {
   semaphore?: Effect.Semaphore;
   /** Base directory for persistent local trial workspaces. Omit for an ephemeral OS temp workspace. */
   workspaceRoot?: string;
+  /** With `workspaceRoot`: which trial workspaces to keep afterwards. Default `'failed'`. */
+  keepWorkspaces?: KeepSandboxes;
+  /**
+   * Candidate paths left out of the artifact snapshot, before the scenario's
+   * own `workspace.ignore`. Default `DEFAULT_ARTIFACT_EXCLUDE`.
+   */
+  artifactExclude?: readonly string[];
   /** Repository and commit the eval code came from, recorded in the manifest. */
   source?: RunSource;
   /** The project's evals and suites at run time, saved beside the manifest. */

package/dist/runner.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.mts","names":["_$_evalkit_core0.JsonValue","_$_evalkit_core0.TrajectoryEvent"],"sources":["../../runner/src/errors.ts","../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/catalog.ts","../../core/src/matrix.ts","../../core/src/index.ts","../../runner/src/local-report-store.ts","../../runner/src/datasets.ts","../../runner/src/local-report-reader.ts","../../runner/src/matrix.ts","../../runner/src/index.ts"],"mappings":";;;;;;qBAEa,0BAA0B;EACrC;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBAGW,oBAAoB;EAC/B;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBASW,iCAAiC;EAG5C;EACA;EACA;EACA;;;;;;qBAIW,uBAAuB;EAClC;;;;cCrCW,oBAAkB,OAAA;KACnB,sBAAsB,mBAAmB;KAiBzC,YAAY,cAAc,eAAe,2BACxC;;;;KCdD;;KAGA,6BAA6B;EAAe;;;;;;KAM5C,wBAAwB,SAAS,eAAe;;KAGhD;EACV,QAAQ;;EAER;;KAGU;;EAEV;EACA,QAAQ;;KAGE;EACV;EACA;EACA;EACA;;EAEA;;EAEA,YAAY;EACZ,QAAQ;;KAoCE;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,OAAO,wBACJ,iBAAiB,QAAQ;;;KAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;;;;;;KC/FP;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;KAE/B;EACV;EACA;;EAEA;EACA;;;KAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;;;KC5BF;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;KAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;KAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;KAGU;EACV;EACA,MAAM;EACN;EACA;;EAEA;EACA;;KAGU;EACN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;KAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;EACV;EACA;IAAU;IAAgB;IAAoB;;EAC9C,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;;;KC/EU;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;;;KCpB3C;KACA,YACV,gBAAgB;GAAiB,cAAc;;KACrC;GAAgB,cAAc;;KAE9B;;EAEV;EACA;EACA;EACA;;KAGU;;;;;KAMA;EACV;EACA,gBAAgB;;KAGN,gBAAgB,QAAQ,OAAO,kBAAkB;KAEjD;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;KAE/B;EACN;EAAuB;;EAEvB;EACA;;EAEA;EACA;;EAEA;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAyB;;EACzB;EAAe,OAAO;EAAe;;KAE/B,mBACT;EAAa;MAAoB;EAAgB;;KAExC;;EAEV;;KAGU,qBAAqB;KACrB,qBAAqB;KAErB;EACV;EACA;EACA;EACA;EACA,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;EAEN,WAAW;;EAEX,oBAAoB;;KAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;IAEV,UAAU,OAAO,yBAAyB;MACxC,QAAQ,WAAW;;KAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;KAwBA;EACV;EACA;;KAGU;EACV;EACA,aAAa;;KAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;KAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;KAE5B,iBACV,WAAW,YAAY,cAAc;KAmB3B;EACV,QAAQ;EACR,QAAQ;;EAER,SAAS;;;KAIC;EAAiB;EAAe;;KAEhC,sBAER;EAEE;EACA;EACA;;EAEA;;EAEA,QAAQ;;KAYF;EACV;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;;EAER;;KAGU;EACV,WAAW;EACX,WAAW;;KAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;EAEP,SAAS;;EAET,MAAM;;KAGI;EACV;EACA;EACA;;EAEA;EACA,IACE,SAAS,2BACE,aAAa,kBAAkB;;KAGlC;EACV;EACA;EACA;;EAEA;;KAGU,cAAc,kBAAkB;KA+BhC;EACV;;EAEA;;EAEA;;KAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,QAAQ;;EAER,oBAAoB;EACpB,YAAY;EACZ,SAAS;EACT,SAAS;;EAET,WAAW;;KA4FD;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAgFU;;;;;KAMA;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;KAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;KAIE;EAAe;EAAa;;KAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;;EAET,UAAU;;KAGA;EACV;EACA;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,WAAW;;KAGD;EACV;EACA,YAAY;EACZ;;EAEA;;KAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;KAG9B;EACV,SAAS;EACT,cAAc;EACd;EACA;EACA;;KAGU;;EAEV;EACA;EACA;;KAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,QAAQ;;KAGE;EACV,QAAQ;EACR;EACA;EACA;EACA;EACA;;EAEA;EACA,QAAQ;;KAGE;EACV,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,eAAe;;EAEpC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,eAAe;;KAGvB;WACD;EACT,WAAW,UAAU,gBAAgB,QAAQ;EAC7C,SAAS,SAAS,aAAa;;KAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;KAGD;EACV;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;KAGE;EACV;EACA;EACA;EACA;EACA;;KAGU,YAAY;EACtB;EACA;EACA;EACA;EACA,mBAAmB;;EAEnB,SAAS;;;;;wBCnmBK,iBAAiB,wBAAwB;;;;KCiI7C;EAAqB;EAAc;;;;;;;wBAOzB,mBACpB,SAAS,SACT,eACC,QAAQ;;wBA6BW,kBACpB,SAAS,SACT,cACA,eACC,QAAQ,WAAW;;;qBC3RT,kBAAe,cAAgB,kBAAe;;;;;;0BAA9CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAEA,iBAAc,cAAgB,kBAAe;;;;;;;;;;;;;;qBAE7C,oBAAiB,cAChB,eACC,oBACE;;;;;;;0BAHJA;;;;;;;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;;;;;0BAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;wBAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBASS,gBACpB,cACA,eACA,kBAAe,QAHKC;;;;;wBAiBN,YAAY,OAAO;;;;KC7DvB;EACV;EACA;EACA;EACA;;;wBAIc,UACd,QAAQ,YACR,SAAS,KACP;EAGA,YAAY;;EAEZ,YAAY,MAAM;EAClB,YACE,MAAM,gBACN,QAAQ,qBACE;IAEb,OAAO,OAAO;;;YCkCL;EACV,QAAQ;EACR;EACA;EACA;;EAEA;;EAEA,aAAa;;EAEb,SAAS;;EAET,UAAU;;EAEV;;EAEA,YAAY,OAAO;;EAEnB;;EAEA,SAAS;;EAET,UAAU;;EAEV;;;;;EAKA,SAAS;;;;;EAKT,WAAW,OAAO,UAAU,OAAO;EACnC,YAAY;;;YAIF;EACV;EACA;EACA;EACA;EACA;EACA,aAAa;;;wBASC,QACd,YAAY,gBACZ,SAAS,iBACR,OAAO,OAAO"}
+{"version":3,"file":"runner.d.mts","names":["_$_evalkit_core0.JsonValue","_$_evalkit_core0.TrajectoryEvent"],"sources":["../../runner/src/errors.ts","../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/catalog.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/index.ts","../../runner/src/local-report-store.ts","../../runner/src/datasets.ts","../../runner/src/local-report-reader.ts","../../runner/src/matrix.ts","../../runner/src/workspace.ts","../../runner/src/index.ts"],"mappings":";;;;;;qBAEa,0BAA0B;EACrC;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBAGW,oBAAoB;EAC/B;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBASW,iCAAiC;EAG5C;EACA;EACA;EACA;;;;;;qBAIW,uBAAuB;EAClC;;;;cCrCW,oBAAkB,OAAA;KACnB,sBAAsB,mBAAmB;KAiBzC,YAAY,cAAc,eAAe,2BACxC;;;;KCdD;;KAGA,6BAA6B;EAAe;;;;;;KAM5C,wBAAwB,SAAS,eAAe;;KAGhD;EACV,QAAQ;;EAER;;KAGU;;EAEV;EACA,QAAQ;;KAGE;EACV;EACA;EACA;EACA;;EAEA;;EAEA,YAAY;EACZ,QAAQ;;KAoCE;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,OAAO,wBACJ,iBAAiB,QAAQ;;;KAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;;;;;;KC/FP;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;KAE/B;EACV;EACA;;EAEA;EACA;;;KAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;;;KC5BF;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;KAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;KAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;KAGU;EACV;EACA,MAAM;EACN;EACA;;EAEA;EACA;;KAGU;;;EAEN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;KAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;EACV;EACA;IAAU;IAAgB;IAAoB;;EAC9C,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;;;KChFU;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;;;KCH3C;;;KCfA;KACA,YACV,gBAAgB;GAAiB,cAAc;;KACrC;GAAgB,cAAc;;KAE9B;;EAEV;EACA;EACA;EACA;;KAGU;;;;;KAMA;EACV;EACA,gBAAgB;;KAGN,gBAAgB,QAAQ,OAAO,kBAAkB;KAEjD;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;KAE/B;EACN;EAAuB;;EAEvB;EACA;;EAEA;EACA;;EAEA;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAyB;;EACzB;EAAe,OAAO;EAAe;;KAE/B,mBACT;EAAa;MAAoB;EAAgB;;KAExC;;EAEV;;KAGU,qBAAqB;KACrB,qBAAqB;KAErB;EACV;EACA;EACA;EACA;EACA,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;EAEN,WAAW;;EAEX,oBAAoB;;KAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;IAEV,UAAU,OAAO,yBAAyB;MACxC,QAAQ,WAAW;;KAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;;KAyBA;;EAEV,YAAY;;EAEZ,MAAM;;EAEN;EACA;;KAGU;EACV;;EAEA,oBAAoB,OAAO;;KAGjB;EACV;EACA,aAAa;;KAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;KAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;KAE5B,iBACV,WAAW,YAAY,cAAc;KAuB3B;EACV,QAAQ;EACR,QAAQ;;EAER,SAAS;;;KAIC;EAAiB;EAAe;;KAEhC,sBAER;EAEE;EACA;EACA;;EAEA;;EAEA,QAAQ;;KAYF;EACV;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;;EAER;;KAGU;EACV,WAAW;EACX,WAAW;;KAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;EAEP,SAAS;;EAET,MAAM;;;;;;EAMN,sBAAsB;;KAGZ;EACV;EACA;EACA;;EAEA;EACA,IACE,SAAS,2BACE,aAAa,kBAAkB;;KAGlC;EACV;EACA;EACA;;EAEA;;KAGU,cAAc,kBAAkB;;;;;;KA2BhC;EACV;EACA;;EAEA;;EAEA;EACA,IACE,SAAS,sCACa;;;KAad;EACV;;EAEA;EACA;EACA;;EAEA;EACA,QAAQ;;KAYE;EACV;;EAEA;;EAEA;;KAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,QAAQ;;EAER,oBAAoB;EACpB,YAAY;EACZ,SAAS;;EAET,mBAAmB;EACnB,SAAS;;EAET,WAAW;EACX,YAAY;;;KAIF;;;;;;;;;EASV;;KA6GU;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAgFU;;;;;KAMA;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;KAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;KAIE;EAAe;EAAa;;KAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;;EAET,UAAU;;KAGA;EACV;EACA;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA;;EAEA,WAAW;;KAGD;EACV;EACA,YAAY;EACZ;;EAEA;;KAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;KAU9B;EACV,SAAS;EACT,UAAU;EACV,cAAc;EACd;EACA;EACA;;KAGU;;EAEV;EACA;EACA;;KAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,QAAQ;;KAGE;EACV,QAAQ;EACR;EACA;EACA;EACA;EACA;;EAEA;EACA,QAAQ;;KAGE;EACV,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,eAAe;;EAEpC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,eAAe;;KAGvB;WACD;EACT,WAAW,UAAU,gBAAgB,QAAQ;EAC7C,SAAS,SAAS,aAAa;;KAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;KAGD;EACV;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;KAGE;EACV;EACA;EACA;EACA;EACA;;KAGU,YAAY;EACtB;EACA;EACA;EACA;EACA,mBAAmB;;EAEnB,SAAS;;;;;wBC5sBK,iBAAiB,wBAAwB;;;;KCiI7C;EAAqB;EAAc;;;;;;;wBAOzB,mBACpB,SAAS,SACT,eACC,QAAQ;;wBA6BW,kBACpB,SAAS,SACT,cACA,eACC,QAAQ,WAAW;;;qBC3RT,kBAAe,cAAgB,kBAAe;;;;;;0BAA9CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAEA,iBAAc,cAAgB,kBAAe;;;;;;;;;;;;;;qBAE7C,oBAAiB,cAChB,eACC,oBACE;;;;;;;0BAHJA;;;;;;;;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;;;;;0BAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;wBAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBASS,gBACpB,cACA,eACA,kBAAe,QAHKC;;;;;wBAiBN,YAAY,OAAO;;;;KC7DvB;EACV;EACA;EACA;EACA;;;wBAIc,UACd,QAAQ,YACR,SAAS,KACP;EAGA,YAAY;;EAEZ,YAAY,MAAM;EAClB,YACE,MAAM,gBACN,QAAQ,qBACE;IAEb,OAAO,OAAO;;;;wBChBD,mBAAmB,cAAc;;;YC0DrC;EACV,QAAQ;EACR;EACA;EACA;;EAEA;;EAEA,aAAa;;EAEb,SAAS;;EAET,UAAU;;EAEV;;EAEA,YAAY,OAAO;;EAEnB;;EAEA,iBAAiB;;;;;EAKjB;;EAEA,SAAS;;EAET,UAAU;;EAEV;;;;;EAKA,SAAS;;;;;EAKT,WAAW,OAAO,UAAU,OAAO;EACnC,YAAY;;;YAIF;EACV;EACA;EACA;EACA;EACA;EACA,aAAa;;;wBASC,QACd,YAAY,gBACZ,SAAS,iBACR,OAAO,OAAO"}

package/dist/runner.mjs

@@ -1,2 +1,2 @@
-import { _ as ReportError, a as readRunSummary, c as readTrialScoring, d as staticDatasetFile, f as staticDatasetFiles, g as DatasetError, h as CheckpointExecutionError, i as readRunManifest, l as readTrialSummary, m as CancelledError, n as runMatrix, o as readTrialEvents, p as AutExecutionError, r as ownerIsGone, s as readTrialManifest, t as runEval, u as localReportStore, v as ScoringError } from "./src-CKqOzeBi.mjs";
-export { AutExecutionError, CancelledError, CheckpointExecutionError, DatasetError, ReportError, ScoringError, localReportStore, ownerIsGone, readRunManifest, readRunSummary, readTrialEvents, readTrialManifest, readTrialScoring, readTrialSummary, runEval, runMatrix, staticDatasetFile, staticDatasetFiles };
+import { _ as CheckpointExecutionError, a as readRunSummary, b as ScoringError, c as readTrialScoring, d as trialWorkspacePath, f as staticDatasetFile, g as CancelledError, h as AutExecutionError, i as readRunManifest, l as readTrialSummary, n as runMatrix, o as readTrialEvents, p as staticDatasetFiles, r as ownerIsGone, s as readTrialManifest, t as runEval, u as localReportStore, v as DatasetError, y as ReportError } from "./src-mm8L_IxM.mjs";
+export { AutExecutionError, CancelledError, CheckpointExecutionError, DatasetError, ReportError, ScoringError, localReportStore, ownerIsGone, readRunManifest, readRunSummary, readTrialEvents, readTrialManifest, readTrialScoring, readTrialSummary, runEval, runMatrix, staticDatasetFile, staticDatasetFiles, trialWorkspacePath };

package/dist/skills/evalkit-evals/SKILL.md

@@ -8,7 +8,7 @@ description: Define or modify EvalKit scenarios (evals) and suites. Use when add
 Read `../evalkit/SKILL.md`, `../../../sites/evalkit.leostera.dev/src/content/docs/manual/project-structure.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/scoring-and-evals.md`, `../../../examples/starter/scenarios/greeting.eval.ts`, and `../../../examples/configured-matrix/evalkit.config.ts`.
 
 1. Default-export one scenario per file from `scenarios/**/*.eval.ts` using `defineEval({ id: 'my-scenario', name, agent, transcript, scoring, datasets?, policy? })`. The CLI discovers these files automatically; no registry or config is required. Choose a stable lowercase kebab-case ID. `metadata` is optional; `name` is a display label. Import shared agents, judges and datasets; keep each scenario short and concrete (a suite of a hundred cases is a hundred files), and never generate scenarios in a loop.
-2. Executable transcript steps are `user('message')`, inline `predicate(...)`, `judge(...)`, and `expectToolCall(...)`. User messages call `session.send` in order on one fresh session per trial; predicates/judges/tool-call assertions run at checkpoints. `judge(...)` requires a separate eval-level `judge` adapter. `agent(...)` transcript steps are not executable. `{{randomSeed}}` is replaced per trial; `policy.trials` controls independent trials (default 1, positive integer); `policy.timeoutMs` is not enforced.
+2. Executable transcript steps are `user('message')`, inline `predicate(...)`, `judge(...)`, and `expectToolCall(...)`. User messages call `session.send` in order on one fresh session per trial; predicates/judges/tool-call assertions run at checkpoints. `judge(...)` requires a separate eval-level `judge` adapter. `agent(...)` transcript steps are not executable. `user(({ parameters, data, seed, trialIndex }) => ...)` computes a message per trial, so one scenario can vary its prompt by matrix axis or candidate dataset instead of being copied per variant; `policy.trials` controls independent trials (default 1, positive integer); `policy.timeoutMs` is not enforced.
 3. Suites are directories: every file below `scenarios/billing/` is in suite `billing` (kebab-case directory names; deeper directories stay in the top one). Files directly in `scenarios/` have no suite; `discovery.suites: false` turns this off. Only use an explicit registry (`registry: registerEvals([defineSuite({ id, evals })])` in the config) when you need full control; it replaces discovery, and duplicate eval IDs throw.
 4. For an intervention comparison, hold task/model/prompt constant and vary one explicit matrix axis (for example `memory: ['off', 'on']`). Repeat trials and include a negative control where possible. Record whether the treatment injects a saved memory or performs live retrieval: those answer different questions. A memory dataset can be evaluator-visible to the trusted adapter, but do not expose evaluator roots or reference answers to the AUT unless revealing the chosen treatment is intentional.
 5. From the project directory run `bun run evalkit run <eval-id>`. Use `bun run evalkit run --suite <suite-id>` for a suite directory. IDs are exact human-defined strings. Inspect per-trial `scoring.json` and the run `summary.json`; completed execution is not the same as passing scores.

package/dist/skills/evalkit-judges/SKILL.md

@@ -9,9 +9,10 @@ Read `../evalkit/SKILL.md`, `../../../sites/evalkit.leostera.dev/src/content/doc
 
 EvalKit currently executes deterministic `predicate(name, callback)` scorers and agent-backed `judge(name, { rubric })` rules. A judge rule requires a **separate eval-level `judge` adapter**; it is not inferred from or shared with the AUT. The `judgeScorer(...)` API is not implemented—do not suggest it.
 
-1. Add a named `predicate(...)` for deterministic checks or a `judge(...)` descriptor for rubric-based LLM assessment, then reference it from the eval's `scoring` array or supported transcript checkpoint. The predicate callback receives `{ context, output, data, trajectory, artifacts, turn? }`. Use `output` for the final assistant reply and `data[id]` for dataset values (expected answers belong in an evaluator dataset); only filter `trajectory.events` (by `source: 'aut'` and `kind`) for tool calls or intermediate messages. `context` includes per-trial IDs, metadata, random seed and optional parameters; workspace roots are absolute local paths.
+1. Add a named `predicate(...)` for deterministic checks or a `judge(...)` descriptor for rubric-based LLM assessment, then reference it from the eval's `scoring` array or supported transcript checkpoint. The predicate callback receives `{ context, output, data, trajectory, artifacts, checkpoints, turn? }`; `checkpoints` are earlier checkpoint results (all of them in final scoring), so a checkpoint that inspects live state before the session closes can hand its `evidence` to final scorers. Use `output` for the final assistant reply and `data[id]` for dataset values (expected answers belong in an evaluator dataset); only filter `trajectory.events` (by `source: 'aut'` and `kind`) for tool calls or intermediate messages. `context` includes per-trial IDs, metadata, random seed and optional parameters; workspace roots are absolute local paths.
 2. A predicate returns a boolean, a finite score from 0 to 1, `{ value, passed?, explanation?, evidence? }` (evidence may be any plain value), or `skip(reason)` when the check does not apply. Without `passed`, only `value === 1` passes; give `predicate(name, fn, { passAt: 0.8 })` for a threshold. A skipped result neither passes nor fails and is left out of averages; never fake "not applicable" as a pass. Use clear evidence for debugging.
 3. A `judge(...)` rule sends a rubric plus the current turn or trajectory to the separate judge adapter, which must emit a JSON verdict. Validate that the judge is independent from the AUT, runs in an appropriate workspace, and does not receive secrets. Judge events/prompts may contain sensitive data and incur model cost; cost is not inferred unless the adapter emits usage.
 4. Normal scorers are skipped after primary execution errors unless declared with `{ supportsPartial: true }` and the required context/workspace exists. Scorer exceptions and invalid scores become recorded failures; never mask them as passes.
 5. Scorers run after the AUT closes; use `node:fs/promises` to inspect candidate/evaluator files before workspace cleanup. Keep evaluator references and secrets out of AUT events, evidence, and exported reports. Reading one evaluator dataset in a trusted adapter and deliberately injecting it as an experiment treatment is valid, but measures injected context—not whether the AUT retrieved it itself.
-6. Test passing, failing, and missing-output trajectories; inspect `scoring.json` (`results`, `overall`, `passed`) and run `summary.json`. `overall` is an unweighted mean of valid scores, not the pass rate. Execution `status: 'completed'` can still have failed scoring; run-level `failed` is the meaningful aggregate gate.
+6. Numbers to compare rather than gate on (tokens, time, retries, mistakes, web searches) are **metrics**, not scorers: `metric(name, (context) => number | undefined, { unit?, better? })` in the eval's `metrics` array. Never fake one as a predicate that always passes. Agent time, tokens (when the adapter reports usage) and tool calls are recorded automatically; have the adapter emit `turn-completed` usage so tokens appear. Means show on the run page and scoreboard.
+7. Test passing, failing, and missing-output trajectories; inspect `scoring.json` (`results`, `overall`, `passed`) and run `summary.json`. `overall` is an unweighted mean of valid scores, not the pass rate. Execution `status: 'completed'` can still have failed scoring; run-level `failed` is the meaningful aggregate gate.

package/dist/skills/evalkit-results/SKILL.md

@@ -7,7 +7,7 @@ description: Run Evalkit locally and interpret or process evaluation results. Us
 
 Read `../evalkit/SKILL.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/index.md` (First run) and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/cli-and-matrices.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/results.md`. Verify report semantics in `../../../packages/runner/src/index.ts`, `../../../packages/runner/src/local-report-store.ts`, and `../../../packages/cli/src/index.ts` before automation.
 
-1. Run from the project root containing default-exported `scenarios/**/*.eval.ts` (as in `examples/starter`, or use an optional config as in `examples/configured-matrix`), not this skill directory. Start with `bun run evalkit run <eval-id>`; run all evals only when intended (Pi-backed evals can cost tokens). `bun run evalkit run --suite <suite-id>`, `bun run evalkit run --matrix <matrix-id>`, and `bun run evalkit dev` are also available; `bun run evalkit build [--out dir]` writes a read-only static dashboard snapshot (reports plus candidate artifacts, never evaluator workspaces) that must still be reviewed before sharing; it asks to run evals first (always runs them when there are no results; `--run`/`--no-run` skip the prompt). `bun run evalkit tidy` archives runs of evals no longer in the project (marker `archived.json`, nothing deleted); archived runs are hidden in the dashboard and excluded from builds. Runs record `source` (repo, project, commit, ref, dirty) in `manifest.json`; `bun run evalkit share` uploads runs for HEAD to the configured share server (`share.url` or `EVALKIT_SHARE_URL`; `share.headers` (object or function) adds headers for an auth proxy in front of the server; `--all`, `--allow-dirty`, `--no-workspace`) and prints the URL. `bun run evalkit share serve` runs a local share server. Set `EVALKIT_NO_OPEN=1` or `PORT=...` for dashboard behavior, and `EVALKIT_LOG_LEVEL=debug` to see the runner's structured logs. `--concurrency` bounds cell parallelism. In CI, add `--junit <file>` (one test case per scenario and matrix cell) and gate with `bun run evalkit compare --base <branch> --fail-on regression` (or `--base-dir <results>`; `--markdown <file>` for an MR comment, `--fail-on score-drop=0.05`). `run` exits 1 on any failure or a selected file that failed to load (`--skip-broken` runs the rest), 130 when cancelled with Ctrl+C; cancelled runs are never failures. Datasets' digests are recorded per trial; `compare` lists changed datasets. For baseline/treatment studies, inspect a `--dry-run` first, keep model/task/prompt constant, and identify whether the treatment is injected context or live tool retrieval.
+1. Run from the project root containing default-exported `scenarios/**/*.eval.ts` (as in `examples/starter`, or use an optional config as in `examples/configured-matrix`), not this skill directory. Start with `bun run evalkit run <eval-id>`; run all evals only when intended (Pi-backed evals can cost tokens). `bun run evalkit run --suite <suite-id>`, `bun run evalkit run --matrix <matrix-id>`, and `bun run evalkit dev` are also available; `bun run evalkit build [--out dir]` writes a read-only static dashboard snapshot (reports plus candidate artifacts, never evaluator workspaces) that must still be reviewed before sharing; it asks to run evals first (always runs them when there are no results; `--run`/`--no-run` skip the prompt). `bun run evalkit tidy` archives runs of evals no longer in the project (marker `archived.json`, nothing deleted); archived runs are hidden in the dashboard and excluded from builds. Runs record `source` (repo, project, commit, ref, dirty) in `manifest.json`; `bun run evalkit share` uploads runs for HEAD to the configured share server (`share.url` or `EVALKIT_SHARE_URL`; `share.headers` (object or function) adds headers for an auth proxy in front of the server; `--all`, `--allow-dirty`, `--no-workspace`) and prints the URL. `bun run evalkit share serve` runs a local share server. Pass `dev --no-open` or `dev --port <n>` (`-p`) for dashboard behavior, and `--log-level debug` (on `run`, `dev` or `build`) to see the runner's structured logs. `evalkit <command> --help` lists every flag. `--concurrency` bounds cell parallelism. In CI, add `--junit <file>` (one test case per scenario and matrix cell) and gate with `bun run evalkit compare --base <branch> --fail-on regression` (or `--base-dir <results>`; `--markdown <file>` for an MR comment, `--fail-on score-drop=0.05`). `run` exits 1 on any failure or a selected file that failed to load (`--skip-broken` runs the rest), 130 when cancelled with Ctrl+C; cancelled runs are never failures. Datasets' digests are recorded per trial; `compare` lists changed datasets. For baseline/treatment studies, inspect a `--dry-run` first, keep model/task/prompt constant, and identify whether the treatment is injected context or live tool retrieval.
 2. Each `_evalkit-results/<run-uuid>/` has `manifest.json`, `summary.json`, and `trials/<trial-uuid>/{manifest.json,trajectory.jsonl,scoring.json,summary.json,artifacts/candidate/...}`. `trajectory.jsonl` contains one JSON event per line; filter by `source` and `kind`. `scoring.json` records scorer values/explanations/errors; candidate snapshot files are report artifacts.
 3. Read run `summary.json` fields `passed`, `failed`, `trialCount` for success rates. Trial `status: 'completed'` means execution finished, **not** that its scorers passed. `scoring.overall` is an unweighted mean of valid values, not a pass rate. For CI, use `jq -e '.failed == 0 and .trialCount > 0' "_evalkit-results/<run-uuid>/summary.json"` after replacing the placeholder; CLI exits nonzero when a matrix cell fails; the report supplies trial-level detail.
 4. `--json` emits one JSON object per completed matrix cell with its key, eval ID, effective parameters, and run result. Read the saved trial trajectory for adapter-emitted turn latency or token usage; EvalKit does not infer provider cost, latency, or token counts. Compare matrix conditions at both cell and trial level and avoid claiming significance from a small pilot.