@cursor/july 0.1.16 → 0.1.17

package/dist/evals.d.ts

@@ -44,33 +44,62 @@
  *   ],
  * });
  * ```
+ *
+ * ## Severity: gates and scores
+ *
+ * Every assertion returns a chainable handle, so severity rides on the
+ * assertion instead of a separate thresholds map. Gates are hard (a miss fails
+ * the eval and `eval` exits non-zero); soft assertions are tracked scores that
+ * mark the eval `scored` and only fail the build under `--strict`:
+ *
+ * ```ts
+ * t.succeeded();                                        // gate (default)
+ * t.calledTool("get_weather").soft();                   // tracked, no gate
+ * t.check(t.reply, similarity("Sunny, 72F")).atLeast(0.8);
+ * t.judge.closedQA("cites a source").atLeast(0.6);      // LLM-graded
+ * ```
+ *
+ * Matchers narrow tool assertions (`{ input, output, status, count }` accept a
+ * literal, a RegExp, or a predicate), and each turn returned by `t.send(...)`
+ * carries the same vocabulary scoped to that turn.
  */
+import type { EvalAssertionResult, EvalAssertions, EvalVerdict } from "./evals/assertions.js";
+import type { Expectation } from "./evals/expect.js";
+import type { EvalJudge, EvalJudgeConfig } from "./evals/judge.js";
+import type { EvalReporter } from "./evals/results.js";
 import type { AgentCloudOptions, SessionEvent } from "./types.js";
-export type Expectation = {
-    kind: "includes";
-    value: string | RegExp;
-} | {
-    kind: "equals";
-    value: unknown;
-} | {
-    kind: "satisfies";
-    predicate: (value: unknown) => boolean;
-    label: string;
-};
-export declare function includes(value: string | RegExp): Expectation;
-export declare function equals(value: unknown): Expectation;
-export declare function satisfies(predicate: (value: unknown) => boolean, label: string): Expectation;
-export interface EvalAssertionResult {
-    name: string;
-    passed: boolean;
-    detail?: string;
-}
-export interface EvalTurnResult {
+export type { EvalAssertionHandle, EvalAssertionResult, EvalAssertionSeverity, EvalAssertions, EvalVerdict, } from "./evals/assertions.js";
+export { computeVerdict, verdictFailsRun } from "./evals/assertions.js";
+export type { EvalSchemaLike, Expectation } from "./evals/expect.js";
+export { equals, includes, matches, normalizedSimilarity, satisfies, similarity, } from "./evals/expect.js";
+export type { EvalJudge, EvalJudgeCallOptions, EvalJudgeConfig, EvalJudgeGraderName, } from "./evals/judge.js";
+export { EVAL_JUDGE_INJECTION_GUARD, fenceUntrusted, } from "./evals/judge.js";
+export type { EvalCountMatcher, EvalEventMatcher, EvalEventOrderMatcher, EvalMatcher, EvalSubagentMatcher, EvalToolCallMatcher, EvalToolCallStatus, } from "./evals/matchers.js";
+export type { EvalReporter, EvalReporterEvalInfo, EvalReporterTarget, EvalRunResult, EvalRunSummary, } from "./evals/results.js";
+export type { EvalInputRequestFact, EvalRunFacts, EvalSubagentFact, EvalToolCallFact, } from "./evals/run-facts.js";
+export { deriveRunFacts } from "./evals/run-facts.js";
+/**
+ * One settled turn. Carries the same assertion vocabulary as `t`, narrowed to
+ * this turn's events, so an intermediate turn can be graded before the next
+ * `send` overwrites `t.reply`:
+ *
+ * ```ts
+ * const draft = await t.send("Draft the summary.");
+ * draft.calledTool("read_file");
+ * t.check(draft.message, includes("Summary"));
+ * ```
+ */
+export interface EvalTurnResult extends EvalAssertions {
     message: string;
     sessionId: string;
     events: SessionEvent[];
+    /** Tool names requested during this turn, in order. */
     toolCalls: string[];
     ok: boolean;
+    /** 1-based turn index within the case. */
+    index: number;
+    /** Throws when this turn failed. Only needed when later steps depend on it. */
+    expectOk(): EvalTurnResult;
 }
 export interface EvalSendOptions {
     /** Absolute harness `cwd` for a new session (`SendMessageOptions.workspaceDir`). */
@@ -84,7 +113,15 @@ export interface EvalSendOptions {
      */
     cloud?: AgentCloudOptions;
 }
-export interface EvalTestContext {
+/**
+ * The eval driver and assertion surface.
+ *
+ * `t` both drives the agent and grades it. Assertions on `t` read the whole run
+ * once `test` finishes; assertions on the turn returned by `t.send(...)` read
+ * only that turn. Recorded assertions never throw, so one run reports every
+ * failure rather than dying on the first.
+ */
+export interface EvalTestContext extends EvalAssertions {
     /** Send a turn and wait until the session parks (or fails). */
     send(message: string, options?: EvalSendOptions): Promise<EvalTurnResult>;
     /** Last assistant text, or null before the first send. */
@@ -92,6 +129,8 @@ export interface EvalTestContext {
     readonly sessionId: string | undefined;
     /** All events captured so far across turns. */
     readonly events: SessionEvent[];
+    /** Turns settled so far, oldest first. */
+    readonly turns: readonly EvalTurnResult[];
     /**
      * 1-based index of this run when {@link EvalConfig.iterations} /
      * {@link EvalDatapoint.iterations} expands a datapoint. Always `1` when
@@ -100,16 +139,25 @@ export interface EvalTestContext {
     readonly iteration: number;
     /** Total iterations for this datapoint (default `1`). */
     readonly iterations: number;
-    /** Gate: run did not fail. */
-    succeeded(): void;
-    /** Gate: at least one completed call to `name`. */
-    calledTool(name: string): void;
-    /** Gate: no call to `name`. */
-    notCalledTool(name: string): void;
-    /** Gate: joined assistant text matches. */
-    messageIncludes(token: string | RegExp): void;
-    /** Gate a value with an {@link Expectation} builder. */
-    check(value: unknown, assertion: Expectation): void;
+    /** Aborts when the case hits its timeout. */
+    readonly signal: AbortSignal;
+    /**
+     * LLM-as-judge assertions. Soft by default (tracked, never fatal); add a bar
+     * with `.atLeast(0.7)` or promote with `.gate(0.8)`. Requires a judge model
+     * from `defineEvalConfig({ judge })`, `defineEval({ judge })`, or the call.
+     */
+    readonly judge: EvalJudge;
+    /**
+     * Record a gate that must pass before the script can safely continue.
+     * Returns `value` when it passes and stops the test body when it does not,
+     * without adding a duplicate execution error.
+     */
+    require<T>(value: T, assertion: Expectation): Promise<T>;
+    /**
+     * End the case as skipped. Skipped cases are reported separately and never
+     * change the exit code. Call it before sending messages.
+     */
+    skip(reason: string): never;
     /** Debug line (surfaced under `agent-serve eval --verbose`). */
     log(message: string): void;
     /**
@@ -139,6 +187,12 @@ export interface EvalDatapoint {
      * concurrent case.
      */
     iterations?: number;
+    /** Free-form data carried onto the result and every reporter. */
+    metadata?: Record<string, unknown>;
+    /** Judge model for this datapoint (overrides `evals.config.ts`). */
+    judge?: EvalJudgeConfig;
+    /** Destinations for just this datapoint, in addition to config reporters. */
+    reporters?: EvalReporter[];
     test: (t: EvalTestContext) => Promise<void>;
 }
 /**
@@ -154,6 +208,12 @@ export interface EvalConfig {
      * Case-level {@link EvalDatapoint.iterations} overrides this.
      */
     iterations?: number;
+    /** Free-form data carried onto every result in this file and to reporters. */
+    metadata?: Record<string, unknown>;
+    /** Judge model for this file (overrides `evals.config.ts`). */
+    judge?: EvalJudgeConfig;
+    /** Destinations for this file, in addition to config reporters. */
+    reporters?: EvalReporter[];
     /** Single-datapoint shorthand. Case id = file path under `evals/`. */
     test?: (t: EvalTestContext) => Promise<void>;
     /** Multiple datapoints in this file. Each id = `<fileId>/<case.id>`. */
@@ -168,6 +228,9 @@ export interface EvalCaseDefinition {
     iterations?: number;
     /** 1-based iteration index for this expanded runnable case. */
     iteration?: number;
+    metadata?: Record<string, unknown>;
+    judge?: EvalJudgeConfig;
+    reporters?: EvalReporter[];
     test: (t: EvalTestContext) => Promise<void>;
     readonly __agentServe: "eval";
 }
@@ -239,6 +302,10 @@ export interface EvalCaseSnapshot {
     tags?: string[];
     status: EvalCaseRunStatus;
     ok?: boolean;
+    /** Full grade, including `scored` (soft threshold miss) and `skipped`. */
+    verdict?: EvalVerdict;
+    /** Reason passed to `t.skip(...)`. */
+    skipReason?: string;
     assertions?: EvalAssertionResult[];
     sessionId?: string;
     durationMs?: number;
@@ -246,6 +313,8 @@ export interface EvalCaseSnapshot {
     logs?: string[];
     /** Structured scores from {@link EvalTestContext.metric}. */
     metrics?: Record<string, string | number>;
+    /** Author-supplied metadata from `defineEval({ metadata })`. */
+    metadata?: Record<string, unknown>;
     inputs?: string[];
     finalText?: string;
     tools?: string[];
@@ -266,6 +335,13 @@ export interface EvalRunSnapshot {
     summary: {
         passed: number;
         failed: number;
+        /**
+         * Cases that only missed a soft threshold (fatal only under `--strict`).
+         * Optional: a batch persisted by an older serve process has no such count.
+         */
+        scored?: number;
+        /** Cases ended by `t.skip(...)`; they never change the exit code. */
+        skipped?: number;
         total: number;
         done: number;
     };
@@ -326,6 +402,17 @@ export interface EvalConfigFile {
      * request limits.
      */
     maxConcurrency: number;
+    /**
+     * Default judge model for `t.judge.*` across this tree. Optional: a tree of
+     * fully deterministic evals never needs one.
+     */
+    judge?: EvalJudgeConfig;
+    /**
+     * Destinations that observe **every** case in the run — the right place for a
+     * shared one, so you do not repeat it in each eval file. `--skip-report`
+     * suppresses them for a local iteration loop.
+     */
+    reporters?: EvalReporter[];
     /**
      * Optional. Max batches kept in the **playground / `/v1/dev/evals*`**
      * history window (in-memory, and pruned from the `defineStorage` evals

package/dist/evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../src/evals.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AAIH,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAElE,MAAM,MAAM,WAAW,GACnB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,GAC5C;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,GAClC;IACE,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;IACvC,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEN,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,WAAW,CAE5D;AAED,wBAAgB,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,WAAW,CAElD;AAED,wBAAgB,SAAS,CACvB,SAAS,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,EACtC,KAAK,EAAE,MAAM,GACZ,WAAW,CAEb;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,EAAE,EAAE,OAAO,CAAC;CACb;AAED,MAAM,WAAW,eAAe;IAC9B,oFAAoF;IACpF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qEAAqE;IACrE,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC;;;;OAIG;IACH,KAAK,CAAC,EAAE,iBAAiB,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,+DAA+D;IAC/D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAC1E,0DAA0D;IAC1D,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,+CAA+C;IAC/C,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC;IAChC;;;;OAIG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,yDAAyD;IACzD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,8BAA8B;IAC9B,SAAS,IAAI,IAAI,CAAC;IAClB,mDAAmD;IACnD,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,+BAA+B;IAC/B,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,2CAA2C;IAC3C,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAC9C,wDAAwD;IACxD,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,GAAG,IAAI,CAAC;IACpD,gEAAgE;IAChE,GAAG,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B;;;;OAIG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;CACpD;AAED,0EAA0E;AAC1E,eAAO,MAAM,mBAAmB,MAAM,CAAC;AAEvC,0EAA0E;AAC1E,MAAM,WAAW,aAAa;IAC5B;;;;OAIG;IACH,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7C;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,wEAAwE;IACxE,KAAK,CAAC,EAAE,aAAa,EAAE,CAAC;CACzB;AAED,sEAAsE;AACtE,MAAM,WAAW,kBAAkB;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG;IAAE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5E;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAK9D;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAoC7D;AAaD;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,cAAc,GACzB,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,kBAAkB,CAAA;CAAE,CAAC,CA8BvD;AAiCD;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAE3E;AAED;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,YAAY,CAAC;AAE1C,sDAAsD;AACtD,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,GAClC,OAAO,CAET;AAED;;;;GAIG;AACH,wBAAgB,2BAA2B,CACzC,CAAC,SAAS;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAExC,GAAG,EAAE,SAAS,CAAC,EAAE,EACjB,QAAQ,EAAE,SAAS,CAAC,EAAE,EACtB,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,GAC9B,CAAC,EAAE,CAgBL;AAED;;;;;GAKG;AACH,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAE9C,gGAAgG;AAChG,eAAO,MAAM,gCAAgC,KAAK,CAAC;AAEnD,iEAAiE;AACjE,eAAO,MAAM,wBAAwB,MAAM,CAAC;AAE5C,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;AAC/D,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE/E,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,iBAAiB,CAAC;IAC1B,EAAE,CAAC,EAAE,OAAO,CAAC;IACb,UAAU,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IAC1C,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,KAAK,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,OAAO,CAAC;QACf,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,OAAO,CAAC;KAClB,CAAC,CAAC;CACJ;AAED,gFAAgF;AAChF,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,eAAe,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACzE,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,MAAM,EAAE;QACN,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,CAAC;CACH;AAED,8EAA8E;AAC9E,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IACjC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8DAA8D;IAC9D,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,wFAAwF;AACxF,MAAM,WAAW,qBAAqB;IACpC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,IAAI,OAAO,CAAC;QACd,KAAK,EAAE,gBAAgB,EAAE,CAAC;QAC1B,MAAM,EAAE,kBAAkB,CAAC;KAC5B,CAAC,CAAC;IACH,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAAC;IACzD,kBAAkB,IAAI,OAAO,CAAC;QAC5B,IAAI,EAAE,eAAe,EAAE,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;IACH,KAAK,CAAC,KAAK,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAC/D,mDAAmD;IACnD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;;;;;;;OAQG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,cAAc,GAAG,cAAc,CAQvE;AAED;;GAEG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAMlE;AAED,yDAAyD;AACzD,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAKrE;AAED;;;GAGG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASnE"}
+{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../src/evals.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AAEH,OAAO,KAAK,EACV,mBAAmB,EACnB,cAAc,EACd,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAKlE,YAAY,EACV,mBAAmB,EACnB,mBAAmB,EACnB,qBAAqB,EACrB,cAAc,EACd,WAAW,GACZ,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxE,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,EACL,MAAM,EACN,QAAQ,EACR,OAAO,EACP,oBAAoB,EACpB,SAAS,EACT,UAAU,GACX,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACV,SAAS,EACT,oBAAoB,EACpB,eAAe,EACf,mBAAmB,GACpB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EACL,0BAA0B,EAC1B,cAAc,GACf,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,gBAAgB,EAChB,gBAAgB,EAChB,qBAAqB,EACrB,WAAW,EACX,mBAAmB,EACnB,mBAAmB,EACnB,kBAAkB,GACnB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,YAAY,EACZ,oBAAoB,EACpB,kBAAkB,EAClB,aAAa,EACb,cAAc,GACf,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EACV,oBAAoB,EACpB,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD;;;;;;;;;;GAUG;AACH,MAAM,WAAW,cAAe,SAAQ,cAAc;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,uDAAuD;IACvD,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,EAAE,EAAE,OAAO,CAAC;IACZ,0CAA0C;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,+EAA+E;IAC/E,QAAQ,IAAI,cAAc,CAAC;CAC5B;AAED,MAAM,WAAW,eAAe;IAC9B,oFAAoF;IACpF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qEAAqE;IACrE,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC;;;;OAIG;IACH,KAAK,CAAC,EAAE,iBAAiB,CAAC;CAC3B;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,eAAgB,SAAQ,cAAc;IACrD,+DAA+D;IAC/D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAC1E,0DAA0D;IAC1D,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,+CAA+C;IAC/C,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC;IAChC,0CAA0C;IAC1C,QAAQ,CAAC,KAAK,EAAE,SAAS,cAAc,EAAE,CAAC;IAC1C;;;;OAIG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,yDAAyD;IACzD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,6CAA6C;IAC7C,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B;;;;OAIG;IACH,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzD;;;OAGG;IACH,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC;IAC5B,gEAAgE;IAChE,GAAG,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B;;;;OAIG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;CACpD;AAED,0EAA0E;AAC1E,eAAO,MAAM,mBAAmB,MAAM,CAAC;AAEvC,0EAA0E;AAC1E,MAAM,WAAW,aAAa;IAC5B;;;;OAIG;IACH,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,oEAAoE;IACpE,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,6EAA6E;IAC7E,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;IAC3B,IAAI,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7C;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,+DAA+D;IAC/D,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,mEAAmE;IACnE,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;IAC3B,sEAAsE;IACtE,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,wEAAwE;IACxE,KAAK,CAAC,EAAE,aAAa,EAAE,CAAC;CACzB;AAED,sEAAsE;AACtE,MAAM,WAAW,kBAAkB;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;IAC3B,IAAI,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG;IAAE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5E;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAK9D;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAoC7D;AAaD;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,cAAc,GACzB,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,kBAAkB,CAAA;CAAE,CAAC,CAwCvD;AAqDD;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAE3E;AAED;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,YAAY,CAAC;AAE1C,sDAAsD;AACtD,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,GAClC,OAAO,CAET;AAED;;;;GAIG;AACH,wBAAgB,2BAA2B,CACzC,CAAC,SAAS;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAExC,GAAG,EAAE,SAAS,CAAC,EAAE,EACjB,QAAQ,EAAE,SAAS,CAAC,EAAE,EACtB,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,GAC9B,CAAC,EAAE,CAgBL;AAED;;;;;GAKG;AACH,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAE9C,gGAAgG;AAChG,eAAO,MAAM,gCAAgC,KAAK,CAAC;AAEnD,iEAAiE;AACjE,eAAO,MAAM,wBAAwB,MAAM,CAAC;AAE5C,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;AAC/D,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE/E,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,iBAAiB,CAAC;IAC1B,EAAE,CAAC,EAAE,OAAO,CAAC;IACb,0EAA0E;IAC1E,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,sCAAsC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IAC1C,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,KAAK,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,OAAO,CAAC;QACf,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,OAAO,CAAC;KAClB,CAAC,CAAC;CACJ;AAED,gFAAgF;AAChF,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,eAAe,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf;;;WAGG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,qEAAqE;QACrE,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,MAAM,EAAE;QACN,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,CAAC;CACH;AAED,8EAA8E;AAC9E,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IACjC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8DAA8D;IAC9D,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,wFAAwF;AACxF,MAAM,WAAW,qBAAqB;IACpC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,IAAI,OAAO,CAAC;QACd,KAAK,EAAE,gBAAgB,EAAE,CAAC;QAC1B,MAAM,EAAE,kBAAkB,CAAC;KAC5B,CAAC,CAAC;IACH,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAAC;IACzD,kBAAkB,IAAI,OAAO,CAAC;QAC5B,IAAI,EAAE,eAAe,EAAE,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;IACH,KAAK,CAAC,KAAK,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAC/D,mDAAmD;IACnD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB;;;;OAIG;IACH,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;IAC3B;;;;;;;;OAQG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,cAAc,GAAG,cAAc,CAUvE;AAED;;GAEG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAMlE;AAED,yDAAyD;AACzD,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAKrE;AAED;;;GAGG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASnE"}

package/dist/evals.js

@@ -44,18 +44,33 @@
  *   ],
  * });
  * ```
+ *
+ * ## Severity: gates and scores
+ *
+ * Every assertion returns a chainable handle, so severity rides on the
+ * assertion instead of a separate thresholds map. Gates are hard (a miss fails
+ * the eval and `eval` exits non-zero); soft assertions are tracked scores that
+ * mark the eval `scored` and only fail the build under `--strict`:
+ *
+ * ```ts
+ * t.succeeded();                                        // gate (default)
+ * t.calledTool("get_weather").soft();                   // tracked, no gate
+ * t.check(t.reply, similarity("Sunny, 72F")).atLeast(0.8);
+ * t.judge.closedQA("cites a source").atLeast(0.6);      // LLM-graded
+ * ```
+ *
+ * Matchers narrow tool assertions (`{ input, output, status, count }` accept a
+ * literal, a RegExp, or a predicate), and each turn returned by `t.send(...)`
+ * carries the same vocabulary scoped to that turn.
  */
 import { normalizePositiveInt } from "./internal/bounded-int.js";
 import { brandDefinition } from "./internal/brand.js";
-export function includes(value) {
-    return { kind: "includes", value };
-}
-export function equals(value) {
-    return { kind: "equals", value };
-}
-export function satisfies(predicate, label) {
-    return { kind: "satisfies", predicate, label };
-}
+export { computeVerdict, verdictFailsRun } from "./evals/assertions.js";
+export { equals, includes, matches, normalizedSimilarity, satisfies, similarity, } from "./evals/expect.js";
+// A custom judge built on `t.judge.model` embeds the same attacker-controlled
+// agent output the built-in graders do, so it needs the same two protections.
+export { EVAL_JUDGE_INJECTION_GUARD, fenceUntrusted, } from "./evals/judge.js";
+export { deriveRunFacts } from "./evals/run-facts.js";
 /** Hard ceiling for {@link EvalConfig.iterations} / case `iterations`. */
 export const EVAL_MAX_ITERATIONS = 100;
 /**
@@ -122,16 +137,22 @@ export function expandEvalDefinition(fileId, definition) {
     var _a;
     if (definition.cases !== undefined) {
         return definition.cases.flatMap((datapoint) => {
-            var _a, _b, _c, _d, _e;
+            var _a, _b, _c, _d, _e, _f;
             const description = (_a = datapoint.description) !== null && _a !== void 0 ? _a : definition.description;
             const timeoutMs = (_b = datapoint.timeoutMs) !== null && _b !== void 0 ? _b : definition.timeoutMs;
             const iterations = normalizeEvalIterations((_d = (_c = datapoint.iterations) !== null && _c !== void 0 ? _c : definition.iterations) !== null && _d !== void 0 ? _d : 1);
+            const metadata = datapoint.metadata === undefined && definition.metadata === undefined
+                ? undefined
+                : Object.assign(Object.assign({}, definition.metadata), datapoint.metadata);
             return expandIterations({
                 baseId: `${fileId}/${datapoint.id}`,
                 description,
                 tags: (_e = datapoint.tags) !== null && _e !== void 0 ? _e : definition.tags,
                 timeoutMs,
                 iterations,
+                metadata,
+                judge: (_f = datapoint.judge) !== null && _f !== void 0 ? _f : definition.judge,
+                reporters: mergeReporters(definition.reporters, datapoint.reporters),
                 test: datapoint.test,
             });
         });
@@ -146,9 +167,22 @@ export function expandEvalDefinition(fileId, definition) {
         tags: definition.tags,
         timeoutMs: definition.timeoutMs,
         iterations,
+        metadata: definition.metadata,
+        judge: definition.judge,
+        reporters: definition.reporters,
         test: definition.test,
     });
 }
+/** Union file-level and case-level reporters, keeping each instance once. */
+function mergeReporters(fileLevel, caseLevel) {
+    if (fileLevel === undefined) {
+        return caseLevel;
+    }
+    if (caseLevel === undefined) {
+        return fileLevel;
+    }
+    return [...new Set([...fileLevel, ...caseLevel])];
+}
 function expandIterations(args) {
     const out = [];
     for (let iteration = 1; iteration <= args.iterations; iteration++) {
@@ -158,7 +192,7 @@ function expandIterations(args) {
             : `${args.description} (iteration ${iteration}/${args.iterations})`;
         out.push({
             id,
-            definition: brandDefinition("eval", Object.assign(Object.assign(Object.assign(Object.assign({}, (description !== undefined ? { description } : {})), (args.tags !== undefined ? { tags: args.tags } : {})), (args.timeoutMs !== undefined ? { timeoutMs: args.timeoutMs } : {})), { iterations: args.iterations, iteration, test: args.test })),
+            definition: brandDefinition("eval", Object.assign(Object.assign(Object.assign(Object.assign({}, (description !== undefined ? { description } : {})), (args.tags !== undefined ? { tags: args.tags } : {})), (args.timeoutMs !== undefined ? { timeoutMs: args.timeoutMs } : {})), { metadata: args.metadata, judge: args.judge, reporters: args.reporters, iterations: args.iterations, iteration, test: args.test })),
         });
     }
     return out;
@@ -222,7 +256,7 @@ export const EVAL_MAX_PLAYGROUND_RUNS = 500;
  */
 export function defineEvalConfig(config) {
     var _a;
-    return Object.assign(Object.assign({}, (config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs })), { maxConcurrency: normalizeEvalMaxConcurrency(config.maxConcurrency), maxPlaygroundRuns: normalizeEvalMaxPlaygroundRuns((_a = config.maxPlaygroundRuns) !== null && _a !== void 0 ? _a : EVAL_DEFAULT_MAX_PLAYGROUND_RUNS) });
+    return Object.assign(Object.assign({}, (config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs })), { maxConcurrency: normalizeEvalMaxConcurrency(config.maxConcurrency), maxPlaygroundRuns: normalizeEvalMaxPlaygroundRuns((_a = config.maxPlaygroundRuns) !== null && _a !== void 0 ? _a : EVAL_DEFAULT_MAX_PLAYGROUND_RUNS), judge: config.judge, reporters: config.reporters });
 }
 /**
  * Validate and enforce the hard concurrency ceiling.

package/dist/index.d.ts

@@ -35,7 +35,7 @@ export type { StorageABTable, StorageConfig, StorageContext, StorageDefinition,
 export { defineStorage, STORAGE_KEY_ROOT, storageKeys, } from "./storage.js";
 export { defineTool } from "./tools.js";
 export type * from "./types.js";
-export { DEFAULT_MODEL, DIRECT_TOOL_CALL_CHANNEL_ID, isCursorAccountTransport, isPeerConnectionTransport, isSymbolicConnectionTransport, MAX_RUNNING_TURNS_LIMIT, } from "./types.js";
+export { cursorAccountServersHasMixedWildcard, DEFAULT_MODEL, DIRECT_TOOL_CALL_CHANNEL_ID, isCursorAccountTransport, isPeerConnectionTransport, isSymbolicConnectionTransport, MAX_RUNNING_TURNS_LIMIT, normalizeCursorAccountServers, } from "./types.js";
 /**
  * Runtime config for an agent — authored at `agent/agent.ts` (the root
  * agent) or `agent/subagents/<id>/agent.ts` (a subagent).

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAKH,OAAO,EACL,KAAK,uBAAuB,EAE7B,MAAM,yBAAyB,CAAC;AAEjC,OAAO,KAAK,EACV,WAAW,EACX,qBAAqB,EACrB,YAAY,EACZ,gBAAgB,EAChB,kBAAkB,EAClB,sBAAsB,EACtB,YAAY,EACb,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAEnC,OAAO,EACL,QAAQ,EACR,UAAU,EACV,MAAM,EACN,aAAa,EACb,GAAG,EACH,WAAW,EACX,QAAQ,EACR,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,GAAG,GACJ,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,YAAY,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,EACX,kBAAkB,GACnB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,4BAA4B,EAC5B,6BAA6B,EAC7B,sBAAsB,EACtB,uBAAuB,EACvB,6BAA6B,GAC9B,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EACL,2BAA2B,EAC3B,uBAAuB,GACxB,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,uBAAuB,EACvB,oBAAoB,GACrB,MAAM,+BAA+B,CAAC;AACvC,YAAY,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,4BAA4B,EAC5B,mBAAmB,EACnB,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EACV,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,kBAAkB,EAClB,eAAe,EACf,YAAY,EACZ,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,oBAAoB,GACrB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,YAAY,EACV,cAAc,EACd,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,GACrB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,WAAW,GACZ,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,mBAAmB,YAAY,CAAC;AAChC,OAAO,EACL,aAAa,EACb,2BAA2B,EAC3B,wBAAwB,EACxB,yBAAyB,EACzB,6BAA6B,EAC7B,uBAAuB,GACxB,MAAM,YAAY,CAAC;AAEpB;;;GAGG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,qBAAqB,CAEtE;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,kBAAkB,GACzB,sBAAsB,CAExB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,MAAM,GAAG,YAAY,EACnC,OAAO,GAAE,YAAY,GAAG;IACtB,SAAS,CAAC,EAAE,uBAAuB,CAAC;IACpC;;;;OAIG;IACH,IAAI,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC;CACtB,GACL,OAAO,CAAC,gBAAgB,CAAC,CAqB3B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAKH,OAAO,EACL,KAAK,uBAAuB,EAE7B,MAAM,yBAAyB,CAAC;AAEjC,OAAO,KAAK,EACV,WAAW,EACX,qBAAqB,EACrB,YAAY,EACZ,gBAAgB,EAChB,kBAAkB,EAClB,sBAAsB,EACtB,YAAY,EACb,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAEnC,OAAO,EACL,QAAQ,EACR,UAAU,EACV,MAAM,EACN,aAAa,EACb,GAAG,EACH,WAAW,EACX,QAAQ,EACR,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,GAAG,GACJ,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,YAAY,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,EACX,kBAAkB,GACnB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,4BAA4B,EAC5B,6BAA6B,EAC7B,sBAAsB,EACtB,uBAAuB,EACvB,6BAA6B,GAC9B,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EACL,2BAA2B,EAC3B,uBAAuB,GACxB,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,uBAAuB,EACvB,oBAAoB,GACrB,MAAM,+BAA+B,CAAC;AACvC,YAAY,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,4BAA4B,EAC5B,mBAAmB,EACnB,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EACV,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,kBAAkB,EAClB,eAAe,EACf,YAAY,EACZ,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,oBAAoB,GACrB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,YAAY,EACV,cAAc,EACd,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,GACrB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,WAAW,GACZ,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,mBAAmB,YAAY,CAAC;AAChC,OAAO,EACL,oCAAoC,EACpC,aAAa,EACb,2BAA2B,EAC3B,wBAAwB,EACxB,yBAAyB,EACzB,6BAA6B,EAC7B,uBAAuB,EACvB,6BAA6B,GAC9B,MAAM,YAAY,CAAC;AAEpB;;;GAGG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,qBAAqB,CAEtE;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,kBAAkB,GACzB,sBAAsB,CAExB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,MAAM,GAAG,YAAY,EACnC,OAAO,GAAE,YAAY,GAAG;IACtB,SAAS,CAAC,EAAE,uBAAuB,CAAC;IACpC;;;;OAIG;IACH,IAAI,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC;CACtB,GACL,OAAO,CAAC,gBAAgB,CAAC,CAqB3B"}

package/dist/index.js

@@ -52,7 +52,7 @@ export { defineSchedule } from "./schedules.js";
 export { defineSkill } from "./skills.js";
 export { defineStorage, STORAGE_KEY_ROOT, storageKeys, } from "./storage.js";
 export { defineTool } from "./tools.js";
-export { DEFAULT_MODEL, DIRECT_TOOL_CALL_CHANNEL_ID, isCursorAccountTransport, isPeerConnectionTransport, isSymbolicConnectionTransport, MAX_RUNNING_TURNS_LIMIT, } from "./types.js";
+export { cursorAccountServersHasMixedWildcard, DEFAULT_MODEL, DIRECT_TOOL_CALL_CHANNEL_ID, isCursorAccountTransport, isPeerConnectionTransport, isSymbolicConnectionTransport, MAX_RUNNING_TURNS_LIMIT, normalizeCursorAccountServers, } from "./types.js";
 /**
  * Runtime config for an agent — authored at `agent/agent.ts` (the root
  * agent) or `agent/subagents/<id>/agent.ts` (a subagent).

package/dist/internal/cli-ax.d.ts

@@ -6,6 +6,7 @@
  * filesystem-first evals under evals/ (*.eval.ts). Live A/B metrics
  * (`defineAB` under `agent/ab`) plug into serve — there is no `ab` CLI.
  */
+import type { EvalAssertionResult } from "../evals.js";
 export interface AxCliOptions {
     cwd?: string;
     dir: string;
@@ -36,6 +37,19 @@ export interface AxCliOptions {
     tags?: string[];
     /** Eval ids / path prefixes. */
     evalIds?: string[];
+    /** `eval`: a soft threshold miss (`scored`) also fails the exit code. */
+    strict?: boolean;
+    /** `eval`: write JUnit XML to this path. */
+    junit?: string;
+    /**
+     * `eval`: run-artifact directory. Defaults to a timestamped directory under
+     * `<dir>/.agent-serve/evals/`; `false` disables artifacts entirely.
+     */
+    artifacts?: string | false;
+    /** `eval`: skip reporters declared in `evals.config.ts` and eval files. */
+    skipReport?: boolean;
+    /** `eval`: override `evals.config.ts` `maxConcurrency`. */
+    maxConcurrency?: number;
     /** Scaffold target directory for `init`. */
     initTarget?: string;
     /** Tool name for `call`. */
@@ -145,6 +159,19 @@ export declare function cmdEvalCancel(runId: string | undefined, options: AxCliO
  * the batch is still running (for poll loops after `--no-wait`).
  */
 export declare function cmdEvalStatus(runId: string | undefined, options: AxCliOptions): Promise<number>;
+/**
+ * Where a local run writes its artifacts.
+ *
+ * Defaults to a timestamped directory under the project's gitignored
+ * `.agent-serve/evals/`, so the full story of a failure is always on disk
+ * without the eval author opting in. `--no-artifacts` turns it off.
+ */
+export declare function resolveEvalArtifactsDir(projectRoot: string, options: Pick<AxCliOptions, "artifacts">): string | undefined;
+/**
+ * One assertion line. Soft assertions read `~` rather than `✗` when they miss,
+ * so a tracked score is visibly different from a failed gate.
+ */
+export declare function formatEvalAssertionLine(assertion: EvalAssertionResult): string;
 /**
  * `logs` — follow deploy progress (`--prod`) then runtime agent logs.
  * Local / `--url` tails `/v1/logs` only.

package/dist/internal/cli-ax.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cli-ax.d.ts","sourceRoot":"","sources":["../../src/internal/cli-ax.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AA6DH,MAAM,WAAW,YAAY;IAC3B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,KAAK,CAAC;IAC/B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,gCAAgC;IAChC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,4CAA4C;IAC5C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6BAA6B;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;OAIG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,oEAAoE;IACpE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yCAAyC;IACzC,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,6CAA6C;IAC7C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,4DAA4D;IAC5D,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,sEAAsE;IACtE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7B;AAsED,wBAAsB,aAAa,CACjC,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,mBAAmB,CAAC,CAmB9B;AAED,wBAAsB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAoFpE;AAED;;;GAGG;AACH;;;;;;;;GAQG;AACH,wBAAsB,SAAS,CAC7B,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,MAAM,CAAC,CAcjB;AAUD,wBAAsB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CA8GpE;AAED,wBAAsB,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CA4GnE;AAED;;;GAGG;AACH,wBAAsB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CA0DpE;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAe1E;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,MAAM,GAAG,KAAK,CAAC,GAC1C,OAAO,GAAG,KAAK,GAAG,YAAY,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAepD;AAiDD,wBAAsB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CA4LpE;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,CAAC,GAC5D,MAAM,CAcR;AAsID;;GAEG;AACH,wBAAsB,aAAa,CACjC,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,MAAM,CAAC,CAwDjB;AAED;;;;;GAKG;AACH,wBAAsB,aAAa,CACjC,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,MAAM,CAAC,CA8FjB;AA8KD;;;GAGG;AACH,wBAAsB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAgFpE;AAmCD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAkCxE;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,MAAM,CAAC,CAkDjB;AAED;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAoF1E"}
+{"version":3,"file":"cli-ax.d.ts","sourceRoot":"","sources":["../../src/internal/cli-ax.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,OAAO,KAAK,EACV,mBAAmB,EAIpB,MAAM,aAAa,CAAC;AAwDrB,MAAM,WAAW,YAAY;IAC3B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,KAAK,CAAC;IAC/B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,gCAAgC;IAChC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,yEAAyE;IACzE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,4CAA4C;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IAC3B,2EAA2E;IAC3E,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,2DAA2D;IAC3D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,4CAA4C;IAC5C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6BAA6B;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;OAIG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,oEAAoE;IACpE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yCAAyC;IACzC,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,6CAA6C;IAC7C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,4DAA4D;IAC5D,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,sEAAsE;IACtE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7B;AAsED,wBAAsB,aAAa,CACjC,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,mBAAmB,CAAC,CAmB9B;AAED,wBAAsB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAoFpE;AAED;;;GAGG;AACH;;;;;;;;GAQG;AACH,wBAAsB,SAAS,CAC7B,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,MAAM,CAAC,CAcjB;AAUD,wBAAsB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CA8GpE;AAED,wBAAsB,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CA4GnE;AAED;;;GAGG;AACH,wBAAsB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CA0DpE;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAe1E;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,MAAM,GAAG,KAAK,CAAC,GAC1C,OAAO,GAAG,KAAK,GAAG,YAAY,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAepD;AAiDD,wBAAsB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAkMpE;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,CAAC,GAC5D,MAAM,CAcR;AAsID;;GAEG;AACH,wBAAsB,aAAa,CACjC,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,MAAM,CAAC,CAwDjB;AAED;;;;;GAKG;AACH,wBAAsB,aAAa,CACjC,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,MAAM,CAAC,CA8FjB;AAmGD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,WAAW,CAAC,GACvC,MAAM,GAAG,SAAS,CASpB;AA8HD;;;GAGG;AACH,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,mBAAmB,GAC7B,MAAM,CAgBR;AAED;;;GAGG;AACH,wBAAsB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAgFpE;AAmCD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAkCxE;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,MAAM,CAAC,CAkDjB;AAED;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAoF1E"}

package/dist/internal/cli-ax.js

@@ -18,6 +18,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
+import { Artifacts, JUnit } from "../evals/reporters.js";
 import { serve } from "../index.js";
 import { lookupSessionContinuation, runChat } from "./chat-client.js";
 import { cmdLogin } from "./cli-cursor.js";
@@ -616,6 +617,7 @@ export function cmdEval(options) {
             // batch lands in playground history and defineStorage evals (when set).
             if (mode === "local") {
                 const progress = stream ? createStreamProgress() : undefined;
+                const artifactsDir = resolveEvalArtifactsDir(projectRoot, options);
                 try {
                     const results = yield runDiscoveredEvals({
                         projectRoot,
@@ -623,6 +625,11 @@ export function cmdEval(options) {
                         filterIds,
                         tags,
                         timeoutMs: options.timeoutMs,
+                        maxConcurrency: options.maxConcurrency,
+                        strict: options.strict === true,
+                        skipReport: options.skipReport === true,
+                        reporters: buildCliEvalReporters(options, artifactsDir),
+                        target: { baseUrl: target.agentUrl, mode: "local" },
                         bearerToken: options.bearerToken,
                         headers: target.headers,
                         verbose: Boolean(options.verbose) || stream,
@@ -632,7 +639,7 @@ export function cmdEval(options) {
                         process.stderr.write("No matching evals found.\n");
                         return 2;
                     }
-                    return printEvalResults(results, options);
+                    return printEvalResults(results, options, undefined, artifactsDir);
                 }
                 finally {
                     progress === null || progress === void 0 ? void 0 : progress.close();
@@ -1031,33 +1038,76 @@ function createRemoteEvalProgress(enabled) {
         },
     };
 }
-function printEvalResults(results, options, batch) {
+/**
+ * Where a local run writes its artifacts.
+ *
+ * Defaults to a timestamped directory under the project's gitignored
+ * `.agent-serve/evals/`, so the full story of a failure is always on disk
+ * without the eval author opting in. `--no-artifacts` turns it off.
+ */
+export function resolveEvalArtifactsDir(projectRoot, options) {
+    if (options.artifacts === false) {
+        return undefined;
+    }
+    if (typeof options.artifacts === "string" && options.artifacts !== "") {
+        return resolve(options.artifacts);
+    }
+    const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+    return join(projectRoot, ".agent-serve", "evals", stamp);
+}
+/** JUnit / artifacts reporters requested via CLI flags. */
+function buildCliEvalReporters(options, artifactsDir) {
+    const reporters = [];
+    if (artifactsDir !== undefined) {
+        reporters.push(Artifacts({ dir: artifactsDir }));
+    }
+    if (options.junit !== undefined && options.junit !== "") {
+        reporters.push(JUnit({ filePath: options.junit }));
+    }
+    return reporters;
+}
+function evalVerdictOf(result) {
+    var _a;
+    return (_a = result.verdict) !== null && _a !== void 0 ? _a : (result.ok ? "passed" : "failed");
+}
+const EVAL_VERDICT_LABEL = {
+    passed: "PASS",
+    failed: "FAIL",
+    scored: "SCORED",
+    skipped: "SKIP",
+};
+function printEvalResults(results, options, batch, artifactsDir) {
     var _a, _b;
-    const passed = results.filter((r) => r.ok).length;
-    const failed = results.length - passed;
+    const strict = options.strict === true;
+    const counts = { passed: 0, failed: 0, scored: 0, skipped: 0 };
+    for (const result of results) {
+        counts[evalVerdictOf(result)]++;
+    }
     const batchFailed = (batch === null || batch === void 0 ? void 0 : batch.status) === "failed";
-    const ok = failed === 0 && !batchFailed;
+    const ok = counts.failed === 0 && !(strict && counts.scored > 0) && !batchFailed;
     if ((batch === null || batch === void 0 ? void 0 : batch.error) !== undefined && batch.error !== "" && !options.json) {
         // Always surface harness-level failures (not only --verbose).
         process.stderr.write(`eval run error: ${batch.error}\n`);
     }
     if (options.json) {
-        console.log(JSON.stringify(Object.assign(Object.assign(Object.assign({ ok,
-            passed,
-            failed,
-            results }, ((batch === null || batch === void 0 ? void 0 : batch.runId) === undefined ? {} : { runId: batch.runId })), ((batch === null || batch === void 0 ? void 0 : batch.status) === undefined ? {} : { status: batch.status })), ((batch === null || batch === void 0 ? void 0 : batch.error) === undefined || batch.error === ""
+        console.log(JSON.stringify(Object.assign(Object.assign(Object.assign(Object.assign({ ok, passed: counts.passed, failed: counts.failed, scored: counts.scored, skipped: counts.skipped, strict,
+            results }, (artifactsDir === undefined ? {} : { artifactsDir })), ((batch === null || batch === void 0 ? void 0 : batch.runId) === undefined ? {} : { runId: batch.runId })), ((batch === null || batch === void 0 ? void 0 : batch.status) === undefined ? {} : { status: batch.status })), ((batch === null || batch === void 0 ? void 0 : batch.error) === undefined || batch.error === ""
             ? {}
             : { error: batch.error })), null, 2));
     }
     else {
         for (const r of results) {
-            process.stdout.write(`${r.ok ? "PASS" : "FAIL"}  ${r.id} (${(r.durationMs / 1000).toFixed(1)}s)`);
+            const verdict = evalVerdictOf(r);
+            process.stdout.write(`${EVAL_VERDICT_LABEL[verdict]}  ${r.id} (${(r.durationMs / 1000).toFixed(1)}s)`);
             if (r.description !== undefined) {
                 process.stdout.write(` — ${r.description}`);
             }
             process.stdout.write("\n");
+            if (r.skipReason !== undefined) {
+                process.stdout.write(`  skipped: ${r.skipReason}\n`);
+            }
             for (const a of r.assertions) {
-                process.stdout.write(`  ${a.passed ? "✓" : "✗"} ${a.name}${a.detail !== undefined ? ` (${a.detail})` : ""}\n`);
+                process.stdout.write(`  ${formatEvalAssertionLine(a)}\n`);
             }
             for (const [name, value] of Object.entries((_a = r.metrics) !== null && _a !== void 0 ? _a : {})) {
                 process.stdout.write(`  metric ${name}=${String(value)}\n`);
@@ -1072,10 +1122,38 @@ function printEvalResults(results, options, batch) {
                 process.stdout.write(`  reply: ${r.finalText.slice(0, 200).replace(/\s+/g, " ")}\n`);
             }
         }
-        process.stdout.write(`\n${passed} passed, ${failed} failed, ${results.length} total\n`);
+        const extra = [];
+        if (counts.scored > 0) {
+            extra.push(`${counts.scored} scored${strict ? " (fatal: --strict)" : ""}`);
+        }
+        if (counts.skipped > 0) {
+            extra.push(`${counts.skipped} skipped`);
+        }
+        process.stdout.write(`\n${counts.passed} passed, ${counts.failed} failed${extra.length === 0 ? "" : `, ${extra.join(", ")}`}, ${results.length} total\n`);
+        if (artifactsDir !== undefined) {
+            process.stdout.write(`artifacts: ${artifactsDir}\n`);
+        }
     }
     return ok ? 0 : 1;
 }
+/**
+ * One assertion line. Soft assertions read `~` rather than `✗` when they miss,
+ * so a tracked score is visibly different from a failed gate.
+ */
+export function formatEvalAssertionLine(assertion) {
+    var _a;
+    const soft = ((_a = assertion.severity) !== null && _a !== void 0 ? _a : "gate") === "soft";
+    const mark = assertion.passed ? "✓" : soft ? "~" : "✗";
+    const parts = [];
+    if (assertion.score !== undefined) {
+        parts.push(`score=${assertion.score.toFixed(3)}${assertion.threshold === undefined ? "" : ` >= ${assertion.threshold}`}`);
+    }
+    if (assertion.detail !== undefined) {
+        parts.push(assertion.detail);
+    }
+    const suffix = parts.length === 0 ? "" : ` (${parts.join(", ")})`;
+    return `${mark} ${assertion.name}${soft ? " [soft]" : ""}${suffix}`;
+}
 /**
  * `logs` — follow deploy progress (`--prod`) then runtime agent logs.
  * Local / `--url` tails `/v1/logs` only.