@cursor/july 0.1.16 → 0.1.18

package/dist/evals.d.ts

@@ -44,33 +44,62 @@
  *   ],
  * });
  * ```
+ *
+ * ## Severity: gates and scores
+ *
+ * Every assertion returns a chainable handle, so severity rides on the
+ * assertion instead of a separate thresholds map. Gates are hard (a miss fails
+ * the eval and `eval` exits non-zero); soft assertions are tracked scores that
+ * mark the eval `scored` and only fail the build under `--strict`:
+ *
+ * ```ts
+ * t.succeeded();                                        // gate (default)
+ * t.calledTool("get_weather").soft();                   // tracked, no gate
+ * t.check(t.reply, similarity("Sunny, 72F")).atLeast(0.8);
+ * t.judge.closedQA("cites a source").atLeast(0.6);      // LLM-graded
+ * ```
+ *
+ * Matchers narrow tool assertions (`{ input, output, status, count }` accept a
+ * literal, a RegExp, or a predicate), and each turn returned by `t.send(...)`
+ * carries the same vocabulary scoped to that turn.
  */
+import type { EvalAssertionResult, EvalAssertions, EvalVerdict } from "./evals/assertions.js";
+import type { Expectation } from "./evals/expect.js";
+import type { EvalJudge, EvalJudgeConfig } from "./evals/judge.js";
+import type { EvalReporter } from "./evals/results.js";
 import type { AgentCloudOptions, SessionEvent } from "./types.js";
-export type Expectation = {
-    kind: "includes";
-    value: string | RegExp;
-} | {
-    kind: "equals";
-    value: unknown;
-} | {
-    kind: "satisfies";
-    predicate: (value: unknown) => boolean;
-    label: string;
-};
-export declare function includes(value: string | RegExp): Expectation;
-export declare function equals(value: unknown): Expectation;
-export declare function satisfies(predicate: (value: unknown) => boolean, label: string): Expectation;
-export interface EvalAssertionResult {
-    name: string;
-    passed: boolean;
-    detail?: string;
-}
-export interface EvalTurnResult {
+export type { EvalAssertionHandle, EvalAssertionResult, EvalAssertionSeverity, EvalAssertions, EvalVerdict, } from "./evals/assertions.js";
+export { computeVerdict, verdictFailsRun } from "./evals/assertions.js";
+export type { EvalSchemaLike, Expectation } from "./evals/expect.js";
+export { equals, includes, matches, normalizedSimilarity, satisfies, similarity, } from "./evals/expect.js";
+export type { EvalJudge, EvalJudgeCallOptions, EvalJudgeConfig, EvalJudgeGraderName, } from "./evals/judge.js";
+export { EVAL_JUDGE_INJECTION_GUARD, fenceUntrusted, } from "./evals/judge.js";
+export type { EvalCountMatcher, EvalEventMatcher, EvalEventOrderMatcher, EvalMatcher, EvalSubagentMatcher, EvalToolCallMatcher, EvalToolCallStatus, } from "./evals/matchers.js";
+export type { EvalReporter, EvalReporterEvalInfo, EvalReporterTarget, EvalRunResult, EvalRunSummary, } from "./evals/results.js";
+export type { EvalInputRequestFact, EvalRunFacts, EvalSubagentFact, EvalToolCallFact, } from "./evals/run-facts.js";
+export { deriveRunFacts } from "./evals/run-facts.js";
+/**
+ * One settled turn. Carries the same assertion vocabulary as `t`, narrowed to
+ * this turn's events, so an intermediate turn can be graded before the next
+ * `send` overwrites `t.reply`:
+ *
+ * ```ts
+ * const draft = await t.send("Draft the summary.");
+ * draft.calledTool("read_file");
+ * t.check(draft.message, includes("Summary"));
+ * ```
+ */
+export interface EvalTurnResult extends EvalAssertions {
     message: string;
     sessionId: string;
     events: SessionEvent[];
+    /** Tool names requested during this turn, in order. */
     toolCalls: string[];
     ok: boolean;
+    /** 1-based turn index within the case. */
+    index: number;
+    /** Throws when this turn failed. Only needed when later steps depend on it. */
+    expectOk(): EvalTurnResult;
 }
 export interface EvalSendOptions {
     /** Absolute harness `cwd` for a new session (`SendMessageOptions.workspaceDir`). */
@@ -84,7 +113,15 @@ export interface EvalSendOptions {
      */
     cloud?: AgentCloudOptions;
 }
-export interface EvalTestContext {
+/**
+ * The eval driver and assertion surface.
+ *
+ * `t` both drives the agent and grades it. Assertions on `t` read the whole run
+ * once `test` finishes; assertions on the turn returned by `t.send(...)` read
+ * only that turn. Recorded assertions never throw, so one run reports every
+ * failure rather than dying on the first.
+ */
+export interface EvalTestContext extends EvalAssertions {
     /** Send a turn and wait until the session parks (or fails). */
     send(message: string, options?: EvalSendOptions): Promise<EvalTurnResult>;
     /** Last assistant text, or null before the first send. */
@@ -92,6 +129,8 @@ export interface EvalTestContext {
     readonly sessionId: string | undefined;
     /** All events captured so far across turns. */
     readonly events: SessionEvent[];
+    /** Turns settled so far, oldest first. */
+    readonly turns: readonly EvalTurnResult[];
     /**
      * 1-based index of this run when {@link EvalConfig.iterations} /
      * {@link EvalDatapoint.iterations} expands a datapoint. Always `1` when
@@ -100,16 +139,25 @@ export interface EvalTestContext {
     readonly iteration: number;
     /** Total iterations for this datapoint (default `1`). */
     readonly iterations: number;
-    /** Gate: run did not fail. */
-    succeeded(): void;
-    /** Gate: at least one completed call to `name`. */
-    calledTool(name: string): void;
-    /** Gate: no call to `name`. */
-    notCalledTool(name: string): void;
-    /** Gate: joined assistant text matches. */
-    messageIncludes(token: string | RegExp): void;
-    /** Gate a value with an {@link Expectation} builder. */
-    check(value: unknown, assertion: Expectation): void;
+    /** Aborts when the case hits its timeout. */
+    readonly signal: AbortSignal;
+    /**
+     * LLM-as-judge assertions. Soft by default (tracked, never fatal); add a bar
+     * with `.atLeast(0.7)` or promote with `.gate(0.8)`. Requires a judge model
+     * from `defineEvalConfig({ judge })`, `defineEval({ judge })`, or the call.
+     */
+    readonly judge: EvalJudge;
+    /**
+     * Record a gate that must pass before the script can safely continue.
+     * Returns `value` when it passes and stops the test body when it does not,
+     * without adding a duplicate execution error.
+     */
+    require<T>(value: T, assertion: Expectation): Promise<T>;
+    /**
+     * End the case as skipped. Skipped cases are reported separately and never
+     * change the exit code. Call it before sending messages.
+     */
+    skip(reason: string): never;
     /** Debug line (surfaced under `agent-serve eval --verbose`). */
     log(message: string): void;
     /**
@@ -139,6 +187,12 @@ export interface EvalDatapoint {
      * concurrent case.
      */
     iterations?: number;
+    /** Free-form data carried onto the result and every reporter. */
+    metadata?: Record<string, unknown>;
+    /** Judge model for this datapoint (overrides `evals.config.ts`). */
+    judge?: EvalJudgeConfig;
+    /** Destinations for just this datapoint, in addition to config reporters. */
+    reporters?: EvalReporter[];
     test: (t: EvalTestContext) => Promise<void>;
 }
 /**
@@ -154,6 +208,12 @@ export interface EvalConfig {
      * Case-level {@link EvalDatapoint.iterations} overrides this.
      */
     iterations?: number;
+    /** Free-form data carried onto every result in this file and to reporters. */
+    metadata?: Record<string, unknown>;
+    /** Judge model for this file (overrides `evals.config.ts`). */
+    judge?: EvalJudgeConfig;
+    /** Destinations for this file, in addition to config reporters. */
+    reporters?: EvalReporter[];
     /** Single-datapoint shorthand. Case id = file path under `evals/`. */
     test?: (t: EvalTestContext) => Promise<void>;
     /** Multiple datapoints in this file. Each id = `<fileId>/<case.id>`. */
@@ -168,6 +228,9 @@ export interface EvalCaseDefinition {
     iterations?: number;
     /** 1-based iteration index for this expanded runnable case. */
     iteration?: number;
+    metadata?: Record<string, unknown>;
+    judge?: EvalJudgeConfig;
+    reporters?: EvalReporter[];
     test: (t: EvalTestContext) => Promise<void>;
     readonly __agentServe: "eval";
 }
@@ -239,6 +302,10 @@ export interface EvalCaseSnapshot {
     tags?: string[];
     status: EvalCaseRunStatus;
     ok?: boolean;
+    /** Full grade, including `scored` (soft threshold miss) and `skipped`. */
+    verdict?: EvalVerdict;
+    /** Reason passed to `t.skip(...)`. */
+    skipReason?: string;
     assertions?: EvalAssertionResult[];
     sessionId?: string;
     durationMs?: number;
@@ -246,6 +313,8 @@ export interface EvalCaseSnapshot {
     logs?: string[];
     /** Structured scores from {@link EvalTestContext.metric}. */
     metrics?: Record<string, string | number>;
+    /** Author-supplied metadata from `defineEval({ metadata })`. */
+    metadata?: Record<string, unknown>;
     inputs?: string[];
     finalText?: string;
     tools?: string[];
@@ -266,6 +335,13 @@ export interface EvalRunSnapshot {
     summary: {
         passed: number;
         failed: number;
+        /**
+         * Cases that only missed a soft threshold (fatal only under `--strict`).
+         * Optional: a batch persisted by an older serve process has no such count.
+         */
+        scored?: number;
+        /** Cases ended by `t.skip(...)`; they never change the exit code. */
+        skipped?: number;
         total: number;
         done: number;
     };
@@ -326,6 +402,17 @@ export interface EvalConfigFile {
      * request limits.
      */
     maxConcurrency: number;
+    /**
+     * Default judge model for `t.judge.*` across this tree. Optional: a tree of
+     * fully deterministic evals never needs one.
+     */
+    judge?: EvalJudgeConfig;
+    /**
+     * Destinations that observe **every** case in the run — the right place for a
+     * shared one, so you do not repeat it in each eval file. `--skip-report`
+     * suppresses them for a local iteration loop.
+     */
+    reporters?: EvalReporter[];
     /**
      * Optional. Max batches kept in the **playground / `/v1/dev/evals*`**
      * history window (in-memory, and pruned from the `defineStorage` evals

package/dist/evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../src/evals.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AAIH,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAElE,MAAM,MAAM,WAAW,GACnB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,GAC5C;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,GAClC;IACE,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;IACvC,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEN,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,WAAW,CAE5D;AAED,wBAAgB,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,WAAW,CAElD;AAED,wBAAgB,SAAS,CACvB,SAAS,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,EACtC,KAAK,EAAE,MAAM,GACZ,WAAW,CAEb;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,EAAE,EAAE,OAAO,CAAC;CACb;AAED,MAAM,WAAW,eAAe;IAC9B,oFAAoF;IACpF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qEAAqE;IACrE,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC;;;;OAIG;IACH,KAAK,CAAC,EAAE,iBAAiB,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,+DAA+D;IAC/D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAC1E,0DAA0D;IAC1D,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,+CAA+C;IAC/C,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC;IAChC;;;;OAIG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,yDAAyD;IACzD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,8BAA8B;IAC9B,SAAS,IAAI,IAAI,CAAC;IAClB,mDAAmD;IACnD,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,+BAA+B;IAC/B,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,2CAA2C;IAC3C,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAC9C,wDAAwD;IACxD,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,GAAG,IAAI,CAAC;IACpD,gEAAgE;IAChE,GAAG,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B;;;;OAIG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;CACpD;AAED,0EAA0E;AAC1E,eAAO,MAAM,mBAAmB,MAAM,CAAC;AAEvC,0EAA0E;AAC1E,MAAM,WAAW,aAAa;IAC5B;;;;OAIG;IACH,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7C;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,wEAAwE;IACxE,KAAK,CAAC,EAAE,aAAa,EAAE,CAAC;CACzB;AAED,sEAAsE;AACtE,MAAM,WAAW,kBAAkB;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG;IAAE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5E;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAK9D;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAoC7D;AAaD;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,cAAc,GACzB,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,kBAAkB,CAAA;CAAE,CAAC,CA8BvD;AAiCD;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAE3E;AAED;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,YAAY,CAAC;AAE1C,sDAAsD;AACtD,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,GAClC,OAAO,CAET;AAED;;;;GAIG;AACH,wBAAgB,2BAA2B,CACzC,CAAC,SAAS;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAExC,GAAG,EAAE,SAAS,CAAC,EAAE,EACjB,QAAQ,EAAE,SAAS,CAAC,EAAE,EACtB,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,GAC9B,CAAC,EAAE,CAgBL;AAED;;;;;GAKG;AACH,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAE9C,gGAAgG;AAChG,eAAO,MAAM,gCAAgC,KAAK,CAAC;AAEnD,iEAAiE;AACjE,eAAO,MAAM,wBAAwB,MAAM,CAAC;AAE5C,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;AAC/D,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE/E,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,iBAAiB,CAAC;IAC1B,EAAE,CAAC,EAAE,OAAO,CAAC;IACb,UAAU,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IAC1C,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,KAAK,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,OAAO,CAAC;QACf,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,OAAO,CAAC;KAClB,CAAC,CAAC;CACJ;AAED,gFAAgF;AAChF,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,eAAe,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACzE,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,MAAM,EAAE;QACN,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,CAAC;CACH;AAED,8EAA8E;AAC9E,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IACjC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8DAA8D;IAC9D,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,wFAAwF;AACxF,MAAM,WAAW,qBAAqB;IACpC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,IAAI,OAAO,CAAC;QACd,KAAK,EAAE,gBAAgB,EAAE,CAAC;QAC1B,MAAM,EAAE,kBAAkB,CAAC;KAC5B,CAAC,CAAC;IACH,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAAC;IACzD,kBAAkB,IAAI,OAAO,CAAC;QAC5B,IAAI,EAAE,eAAe,EAAE,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;IACH,KAAK,CAAC,KAAK,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAC/D,mDAAmD;IACnD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;;;;;;;OAQG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,cAAc,GAAG,cAAc,CAQvE;AAED;;GAEG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAMlE;AAED,yDAAyD;AACzD,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAKrE;AAED;;;GAGG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASnE"}
+{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../src/evals.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AAEH,OAAO,KAAK,EACV,mBAAmB,EACnB,cAAc,EACd,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAKlE,YAAY,EACV,mBAAmB,EACnB,mBAAmB,EACnB,qBAAqB,EACrB,cAAc,EACd,WAAW,GACZ,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxE,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,EACL,MAAM,EACN,QAAQ,EACR,OAAO,EACP,oBAAoB,EACpB,SAAS,EACT,UAAU,GACX,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACV,SAAS,EACT,oBAAoB,EACpB,eAAe,EACf,mBAAmB,GACpB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EACL,0BAA0B,EAC1B,cAAc,GACf,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,gBAAgB,EAChB,gBAAgB,EAChB,qBAAqB,EACrB,WAAW,EACX,mBAAmB,EACnB,mBAAmB,EACnB,kBAAkB,GACnB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,YAAY,EACZ,oBAAoB,EACpB,kBAAkB,EAClB,aAAa,EACb,cAAc,GACf,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EACV,oBAAoB,EACpB,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD;;;;;;;;;;GAUG;AACH,MAAM,WAAW,cAAe,SAAQ,cAAc;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,uDAAuD;IACvD,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,EAAE,EAAE,OAAO,CAAC;IACZ,0CAA0C;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,+EAA+E;IAC/E,QAAQ,IAAI,cAAc,CAAC;CAC5B;AAED,MAAM,WAAW,eAAe;IAC9B,oFAAoF;IACpF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qEAAqE;IACrE,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC;;;;OAIG;IACH,KAAK,CAAC,EAAE,iBAAiB,CAAC;CAC3B;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,eAAgB,SAAQ,cAAc;IACrD,+DAA+D;IAC/D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAC1E,0DAA0D;IAC1D,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,+CAA+C;IAC/C,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC;IAChC,0CAA0C;IAC1C,QAAQ,CAAC,KAAK,EAAE,SAAS,cAAc,EAAE,CAAC;IAC1C;;;;OAIG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,yDAAyD;IACzD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,6CAA6C;IAC7C,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B;;;;OAIG;IACH,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzD;;;OAGG;IACH,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC;IAC5B,gEAAgE;IAChE,GAAG,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B;;;;OAIG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;CACpD;AAED,0EAA0E;AAC1E,eAAO,MAAM,mBAAmB,MAAM,CAAC;AAEvC,0EAA0E;AAC1E,MAAM,WAAW,aAAa;IAC5B;;;;OAIG;IACH,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,oEAAoE;IACpE,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,6EAA6E;IAC7E,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;IAC3B,IAAI,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7C;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,+DAA+D;IAC/D,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,mEAAmE;IACnE,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;IAC3B,sEAAsE;IACtE,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,wEAAwE;IACxE,KAAK,CAAC,EAAE,aAAa,EAAE,CAAC;CACzB;AAED,sEAAsE;AACtE,MAAM,WAAW,kBAAkB;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;IAC3B,IAAI,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG;IAAE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5E;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAK9D;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAoC7D;AAaD;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,cAAc,GACzB,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,kBAAkB,CAAA;CAAE,CAAC,CAwCvD;AAqDD;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAE3E;AAED;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,YAAY,CAAC;AAE1C,sDAAsD;AACtD,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,GAClC,OAAO,CAET;AAED;;;;GAIG;AACH,wBAAgB,2BAA2B,CACzC,CAAC,SAAS;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAExC,GAAG,EAAE,SAAS,CAAC,EAAE,EACjB,QAAQ,EAAE,SAAS,CAAC,EAAE,EACtB,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,GAC9B,CAAC,EAAE,CAgBL;AAED;;;;;GAKG;AACH,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAE9C,gGAAgG;AAChG,eAAO,MAAM,gCAAgC,KAAK,CAAC;AAEnD,iEAAiE;AACjE,eAAO,MAAM,wBAAwB,MAAM,CAAC;AAE5C,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;AAC/D,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE/E,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,iBAAiB,CAAC;IAC1B,EAAE,CAAC,EAAE,OAAO,CAAC;IACb,0EAA0E;IAC1E,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,sCAAsC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IAC1C,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,KAAK,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,OAAO,CAAC;QACf,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,OAAO,CAAC;KAClB,CAAC,CAAC;CACJ;AAED,gFAAgF;AAChF,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,eAAe,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf;;;WAGG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,qEAAqE;QACrE,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,MAAM,EAAE;QACN,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,CAAC;CACH;AAED,8EAA8E;AAC9E,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IACjC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8DAA8D;IAC9D,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,wFAAwF;AACxF,MAAM,WAAW,qBAAqB;IACpC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,IAAI,OAAO,CAAC;QACd,KAAK,EAAE,gBAAgB,EAAE,CAAC;QAC1B,MAAM,EAAE,kBAAkB,CAAC;KAC5B,CAAC,CAAC;IACH,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAAC;IACzD,kBAAkB,IAAI,OAAO,CAAC;QAC5B,IAAI,EAAE,eAAe,EAAE,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;IACH,KAAK,CAAC,KAAK,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAC/D,mDAAmD;IACnD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB;;;;OAIG;IACH,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;IAC3B;;;;;;;;OAQG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,cAAc,GAAG,cAAc,CAUvE;AAED;;GAEG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAMlE;AAED,yDAAyD;AACzD,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAKrE;AAED;;;GAGG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASnE"}

package/dist/evals.js

@@ -44,18 +44,33 @@
  *   ],
  * });
  * ```
+ *
+ * ## Severity: gates and scores
+ *
+ * Every assertion returns a chainable handle, so severity rides on the
+ * assertion instead of a separate thresholds map. Gates are hard (a miss fails
+ * the eval and `eval` exits non-zero); soft assertions are tracked scores that
+ * mark the eval `scored` and only fail the build under `--strict`:
+ *
+ * ```ts
+ * t.succeeded();                                        // gate (default)
+ * t.calledTool("get_weather").soft();                   // tracked, no gate
+ * t.check(t.reply, similarity("Sunny, 72F")).atLeast(0.8);
+ * t.judge.closedQA("cites a source").atLeast(0.6);      // LLM-graded
+ * ```
+ *
+ * Matchers narrow tool assertions (`{ input, output, status, count }` accept a
+ * literal, a RegExp, or a predicate), and each turn returned by `t.send(...)`
+ * carries the same vocabulary scoped to that turn.
  */
 import { normalizePositiveInt } from "./internal/bounded-int.js";
 import { brandDefinition } from "./internal/brand.js";
-export function includes(value) {
-    return { kind: "includes", value };
-}
-export function equals(value) {
-    return { kind: "equals", value };
-}
-export function satisfies(predicate, label) {
-    return { kind: "satisfies", predicate, label };
-}
+export { computeVerdict, verdictFailsRun } from "./evals/assertions.js";
+export { equals, includes, matches, normalizedSimilarity, satisfies, similarity, } from "./evals/expect.js";
+// A custom judge built on `t.judge.model` embeds the same attacker-controlled
+// agent output the built-in graders do, so it needs the same two protections.
+export { EVAL_JUDGE_INJECTION_GUARD, fenceUntrusted, } from "./evals/judge.js";
+export { deriveRunFacts } from "./evals/run-facts.js";
 /** Hard ceiling for {@link EvalConfig.iterations} / case `iterations`. */
 export const EVAL_MAX_ITERATIONS = 100;
 /**
@@ -122,16 +137,22 @@ export function expandEvalDefinition(fileId, definition) {
     var _a;
     if (definition.cases !== undefined) {
         return definition.cases.flatMap((datapoint) => {
-            var _a, _b, _c, _d, _e;
+            var _a, _b, _c, _d, _e, _f;
             const description = (_a = datapoint.description) !== null && _a !== void 0 ? _a : definition.description;
             const timeoutMs = (_b = datapoint.timeoutMs) !== null && _b !== void 0 ? _b : definition.timeoutMs;
             const iterations = normalizeEvalIterations((_d = (_c = datapoint.iterations) !== null && _c !== void 0 ? _c : definition.iterations) !== null && _d !== void 0 ? _d : 1);
+            const metadata = datapoint.metadata === undefined && definition.metadata === undefined
+                ? undefined
+                : Object.assign(Object.assign({}, definition.metadata), datapoint.metadata);
             return expandIterations({
                 baseId: `${fileId}/${datapoint.id}`,
                 description,
                 tags: (_e = datapoint.tags) !== null && _e !== void 0 ? _e : definition.tags,
                 timeoutMs,
                 iterations,
+                metadata,
+                judge: (_f = datapoint.judge) !== null && _f !== void 0 ? _f : definition.judge,
+                reporters: mergeReporters(definition.reporters, datapoint.reporters),
                 test: datapoint.test,
             });
         });
@@ -146,9 +167,22 @@ export function expandEvalDefinition(fileId, definition) {
         tags: definition.tags,
         timeoutMs: definition.timeoutMs,
         iterations,
+        metadata: definition.metadata,
+        judge: definition.judge,
+        reporters: definition.reporters,
         test: definition.test,
     });
 }
+/** Union file-level and case-level reporters, keeping each instance once. */
+function mergeReporters(fileLevel, caseLevel) {
+    if (fileLevel === undefined) {
+        return caseLevel;
+    }
+    if (caseLevel === undefined) {
+        return fileLevel;
+    }
+    return [...new Set([...fileLevel, ...caseLevel])];
+}
 function expandIterations(args) {
     const out = [];
     for (let iteration = 1; iteration <= args.iterations; iteration++) {
@@ -158,7 +192,7 @@ function expandIterations(args) {
             : `${args.description} (iteration ${iteration}/${args.iterations})`;
         out.push({
             id,
-            definition: brandDefinition("eval", Object.assign(Object.assign(Object.assign(Object.assign({}, (description !== undefined ? { description } : {})), (args.tags !== undefined ? { tags: args.tags } : {})), (args.timeoutMs !== undefined ? { timeoutMs: args.timeoutMs } : {})), { iterations: args.iterations, iteration, test: args.test })),
+            definition: brandDefinition("eval", Object.assign(Object.assign(Object.assign(Object.assign({}, (description !== undefined ? { description } : {})), (args.tags !== undefined ? { tags: args.tags } : {})), (args.timeoutMs !== undefined ? { timeoutMs: args.timeoutMs } : {})), { metadata: args.metadata, judge: args.judge, reporters: args.reporters, iterations: args.iterations, iteration, test: args.test })),
         });
     }
     return out;
@@ -222,7 +256,7 @@ export const EVAL_MAX_PLAYGROUND_RUNS = 500;
  */
 export function defineEvalConfig(config) {
     var _a;
-    return Object.assign(Object.assign({}, (config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs })), { maxConcurrency: normalizeEvalMaxConcurrency(config.maxConcurrency), maxPlaygroundRuns: normalizeEvalMaxPlaygroundRuns((_a = config.maxPlaygroundRuns) !== null && _a !== void 0 ? _a : EVAL_DEFAULT_MAX_PLAYGROUND_RUNS) });
+    return Object.assign(Object.assign({}, (config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs })), { maxConcurrency: normalizeEvalMaxConcurrency(config.maxConcurrency), maxPlaygroundRuns: normalizeEvalMaxPlaygroundRuns((_a = config.maxPlaygroundRuns) !== null && _a !== void 0 ? _a : EVAL_DEFAULT_MAX_PLAYGROUND_RUNS), judge: config.judge, reporters: config.reporters });
 }
 /**
  * Validate and enforce the hard concurrency ceiling.

package/dist/index.d.ts

@@ -35,7 +35,7 @@ export type { StorageABTable, StorageConfig, StorageContext, StorageDefinition,
 export { defineStorage, STORAGE_KEY_ROOT, storageKeys, } from "./storage.js";
 export { defineTool } from "./tools.js";
 export type * from "./types.js";
-export { DEFAULT_MODEL, DIRECT_TOOL_CALL_CHANNEL_ID, isCursorAccountTransport, isPeerConnectionTransport, isSymbolicConnectionTransport, MAX_RUNNING_TURNS_LIMIT, } from "./types.js";
+export { cursorAccountServersHasMixedWildcard, DEFAULT_MODEL, DIRECT_TOOL_CALL_CHANNEL_ID, isCursorAccountTransport, isPeerConnectionTransport, isSymbolicConnectionTransport, MAX_RUNNING_TURNS_LIMIT, normalizeCursorAccountServers, } from "./types.js";
 /**
  * Runtime config for an agent — authored at `agent/agent.ts` (the root
  * agent) or `agent/subagents/<id>/agent.ts` (a subagent).

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAKH,OAAO,EACL,KAAK,uBAAuB,EAE7B,MAAM,yBAAyB,CAAC;AAEjC,OAAO,KAAK,EACV,WAAW,EACX,qBAAqB,EACrB,YAAY,EACZ,gBAAgB,EAChB,kBAAkB,EAClB,sBAAsB,EACtB,YAAY,EACb,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAEnC,OAAO,EACL,QAAQ,EACR,UAAU,EACV,MAAM,EACN,aAAa,EACb,GAAG,EACH,WAAW,EACX,QAAQ,EACR,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,GAAG,GACJ,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,YAAY,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,EACX,kBAAkB,GACnB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,4BAA4B,EAC5B,6BAA6B,EAC7B,sBAAsB,EACtB,uBAAuB,EACvB,6BAA6B,GAC9B,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EACL,2BAA2B,EAC3B,uBAAuB,GACxB,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,uBAAuB,EACvB,oBAAoB,GACrB,MAAM,+BAA+B,CAAC;AACvC,YAAY,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,4BAA4B,EAC5B,mBAAmB,EACnB,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EACV,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,kBAAkB,EAClB,eAAe,EACf,YAAY,EACZ,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,oBAAoB,GACrB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,YAAY,EACV,cAAc,EACd,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,GACrB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,WAAW,GACZ,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,mBAAmB,YAAY,CAAC;AAChC,OAAO,EACL,aAAa,EACb,2BAA2B,EAC3B,wBAAwB,EACxB,yBAAyB,EACzB,6BAA6B,EAC7B,uBAAuB,GACxB,MAAM,YAAY,CAAC;AAEpB;;;GAGG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,qBAAqB,CAEtE;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,kBAAkB,GACzB,sBAAsB,CAExB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,MAAM,GAAG,YAAY,EACnC,OAAO,GAAE,YAAY,GAAG;IACtB,SAAS,CAAC,EAAE,uBAAuB,CAAC;IACpC;;;;OAIG;IACH,IAAI,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC;CACtB,GACL,OAAO,CAAC,gBAAgB,CAAC,CAqB3B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAKH,OAAO,EACL,KAAK,uBAAuB,EAE7B,MAAM,yBAAyB,CAAC;AAEjC,OAAO,KAAK,EACV,WAAW,EACX,qBAAqB,EACrB,YAAY,EACZ,gBAAgB,EAChB,kBAAkB,EAClB,sBAAsB,EACtB,YAAY,EACb,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAEnC,OAAO,EACL,QAAQ,EACR,UAAU,EACV,MAAM,EACN,aAAa,EACb,GAAG,EACH,WAAW,EACX,QAAQ,EACR,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,GAAG,GACJ,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,YAAY,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,EACX,kBAAkB,GACnB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,4BAA4B,EAC5B,6BAA6B,EAC7B,sBAAsB,EACtB,uBAAuB,EACvB,6BAA6B,GAC9B,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EACL,2BAA2B,EAC3B,uBAAuB,GACxB,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,uBAAuB,EACvB,oBAAoB,GACrB,MAAM,+BAA+B,CAAC;AACvC,YAAY,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,4BAA4B,EAC5B,mBAAmB,EACnB,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EACV,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,kBAAkB,EAClB,eAAe,EACf,YAAY,EACZ,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,oBAAoB,GACrB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,YAAY,EACV,cAAc,EACd,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,GACrB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,WAAW,GACZ,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,mBAAmB,YAAY,CAAC;AAChC,OAAO,EACL,oCAAoC,EACpC,aAAa,EACb,2BAA2B,EAC3B,wBAAwB,EACxB,yBAAyB,EACzB,6BAA6B,EAC7B,uBAAuB,EACvB,6BAA6B,GAC9B,MAAM,YAAY,CAAC;AAEpB;;;GAGG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,qBAAqB,CAEtE;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,kBAAkB,GACzB,sBAAsB,CAExB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,MAAM,GAAG,YAAY,EACnC,OAAO,GAAE,YAAY,GAAG;IACtB,SAAS,CAAC,EAAE,uBAAuB,CAAC;IACpC;;;;OAIG;IACH,IAAI,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC;CACtB,GACL,OAAO,CAAC,gBAAgB,CAAC,CAqB3B"}

package/dist/index.js

@@ -52,7 +52,7 @@ export { defineSchedule } from "./schedules.js";
 export { defineSkill } from "./skills.js";
 export { defineStorage, STORAGE_KEY_ROOT, storageKeys, } from "./storage.js";
 export { defineTool } from "./tools.js";
-export { DEFAULT_MODEL, DIRECT_TOOL_CALL_CHANNEL_ID, isCursorAccountTransport, isPeerConnectionTransport, isSymbolicConnectionTransport, MAX_RUNNING_TURNS_LIMIT, } from "./types.js";
+export { cursorAccountServersHasMixedWildcard, DEFAULT_MODEL, DIRECT_TOOL_CALL_CHANNEL_ID, isCursorAccountTransport, isPeerConnectionTransport, isSymbolicConnectionTransport, MAX_RUNNING_TURNS_LIMIT, normalizeCursorAccountServers, } from "./types.js";
 /**
  * Runtime config for an agent — authored at `agent/agent.ts` (the root
  * agent) or `agent/subagents/<id>/agent.ts` (a subagent).

package/dist/internal/builtin-tools/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Built-in tools (`AgentConfig.builtinTools`): framework-provided
+ * model-facing tools, materialized as ordinary server tools at discovery
+ * time so turns, direct calls, `info`, and the playground treat them
+ * exactly like authored tools.
+ */
+import type { BuiltinToolsConfig, DiscoveredTool } from "../../types.js";
+/** Resolve the enabled built-in tools, in stable catalog order. */
+export declare function builtinTools(config: BuiltinToolsConfig | undefined): DiscoveredTool[];
+//# sourceMappingURL=index.d.ts.map

package/dist/internal/builtin-tools/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/internal/builtin-tools/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,kBAAkB,EAClB,cAAc,EAEf,MAAM,gBAAgB,CAAC;AAIxB,mEAAmE;AACnE,wBAAgB,YAAY,CAC1B,MAAM,EAAE,kBAAkB,GAAG,SAAS,GACrC,cAAc,EAAE,CAMlB"}

package/dist/internal/builtin-tools/index.js

@@ -0,0 +1,27 @@
+/**
+ * Built-in tools (`AgentConfig.builtinTools`): framework-provided
+ * model-facing tools, materialized as ordinary server tools at discovery
+ * time so turns, direct calls, `info`, and the playground treat them
+ * exactly like authored tools.
+ */
+import { toolInputJsonSchema } from "../json-schema.js";
+import { reminderTools } from "./reminders.js";
+/** Resolve the enabled built-in tools, in stable catalog order. */
+export function builtinTools(config) {
+    const tools = [];
+    if ((config === null || config === void 0 ? void 0 : config.reminders) === true) {
+        tools.push(...toDiscovered(reminderTools));
+    }
+    return tools;
+}
+function toDiscovered(byName) {
+    return Object.entries(byName).map(([name, definition]) => ({
+        name,
+        description: definition.description,
+        inputSchema: toolInputJsonSchema(definition.inputSchema),
+        execution: "server",
+        needsApproval: definition.needsApproval === true ||
+            typeof definition.needsApproval === "function",
+        definition,
+    }));
+}

package/dist/internal/builtin-tools/reminders.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Built-in reminder tools (`builtinTools: { reminders: true }`): durable
+ * wakes bound to the current conversation over `host.reminders`. The
+ * framework's reminder fire prompt tells the model to call
+ * `reminders_cancel` by name, so these tool names must not change.
+ */
+import type { ToolDefinition } from "../../types.js";
+export declare const reminderTools: Record<string, ToolDefinition>;
+//# sourceMappingURL=reminders.d.ts.map

package/dist/internal/builtin-tools/reminders.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reminders.d.ts","sourceRoot":"","sources":["../../../src/internal/builtin-tools/reminders.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAIV,cAAc,EACf,MAAM,gBAAgB,CAAC;AA2IxB,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAIxD,CAAC"}

package/dist/internal/builtin-tools/reminders.js

@@ -0,0 +1,138 @@
+/**
+ * Built-in reminder tools (`builtinTools: { reminders: true }`): durable
+ * wakes bound to the current conversation over `host.reminders`. The
+ * framework's reminder fire prompt tells the model to call
+ * `reminders_cancel` by name, so these tool names must not change.
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { z } from "zod";
+import { defineTool } from "../../tools.js";
+import { parseDuration } from "../duration.js";
+/** Floor for model-armed recurring wakes: each fire spends a model turn. */
+const MIN_EVERY_MS = 60000;
+function api(ctx) {
+    const reminders = ctx.host.reminders;
+    if (reminders === undefined) {
+        throw new Error("Reminders are disabled on this host.");
+    }
+    return reminders;
+}
+function conversation(ctx) {
+    const token = ctx.session.continuationKey;
+    if (token == null) {
+        throw new Error("This session is not bound to a followable conversation; reminders cannot wake it.");
+    }
+    return { channelId: ctx.session.channelId, continuationToken: token };
+}
+/** Reminders visible to this conversation (tokens are channel-local). */
+function listMine(ctx, status) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const conv = conversation(ctx);
+        const all = yield api(ctx).list({
+            status,
+            continuationToken: conv.continuationToken,
+        });
+        return all.filter((r) => r.channelId === conv.channelId);
+    });
+}
+function toResult(r) {
+    return {
+        reminderId: r.id,
+        purpose: r.purpose,
+        schedule: `${r.scheduleKind} ${r.scheduleValue}`,
+        nextFireAt: r.nextFireAt,
+        until: r.untilPrompt,
+    };
+}
+const create = defineTool({
+    description: "Set a reminder for yourself in this conversation. When it fires, your prompt returns here as a follow-up turn. Use it to re-check state that needs time to settle. One-shot (delay) or recurring (every).",
+    inputSchema: z.object({
+        prompt: z
+            .string()
+            .min(1)
+            .describe("What to do when you wake. Keep it generic ('re-check CI and report only if the status changed') and re-read live state on wake instead of replaying stale details."),
+        delay: z
+            .string()
+            .optional()
+            .describe('One-shot: wake once after this duration, e.g. "20m" or "2h". Provide exactly one of delay or every.'),
+        every: z
+            .string()
+            .optional()
+            .describe('Recurring: wake at this interval, e.g. "30m", until cancelled or the stop condition is met. Provide exactly one of delay or every.'),
+        purpose: z
+            .string()
+            .min(1)
+            .max(64)
+            .optional()
+            .describe('Short slug naming the reminder, e.g. "ci_recheck". Defaults to "follow_up".'),
+        until: z
+            .string()
+            .optional()
+            .describe("Recurring only: plain-language stop condition you will honor by calling reminders_cancel, e.g. 'Cancel once CI is green or the PR is merged.'"),
+    }),
+    execute(input, ctx) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            if ((input.delay !== undefined) === (input.every !== undefined)) {
+                throw new Error("Provide exactly one of delay or every.");
+            }
+            if (input.every !== undefined &&
+                parseDuration(input.every) < MIN_EVERY_MS) {
+                throw new Error('Recurring reminders must use every >= "1m".');
+            }
+            const conv = conversation(ctx);
+            const reminder = yield api(ctx).create({
+                purpose: (_a = input.purpose) !== null && _a !== void 0 ? _a : "follow_up",
+                channelId: conv.channelId,
+                continuationToken: conv.continuationToken,
+                prompt: input.prompt,
+                delay: input.delay,
+                every: input.every,
+                until: input.until,
+            });
+            return toResult(reminder);
+        });
+    },
+});
+const list = defineTool({
+    description: "List the armed reminders bound to this conversation. Check before arming duplicates.",
+    inputSchema: z.object({}),
+    execute(_input, ctx) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return { reminders: (yield listMine(ctx, "armed")).map(toResult) };
+        });
+    },
+});
+const cancel = defineTool({
+    description: "Cancel one of this conversation's reminders by id, the moment its stop condition is met or it is no longer needed.",
+    inputSchema: z.object({
+        id: z
+            .string()
+            .min(1)
+            .describe("Reminder id from reminders_create or reminders_list."),
+    }),
+    execute(input, ctx) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const mine = (yield listMine(ctx)).find((r) => r.id === input.id);
+            if (mine === undefined) {
+                throw new Error(`No reminder "${input.id}" in this conversation.`);
+            }
+            const cancelled = yield api(ctx).cancel(input.id);
+            return { reminderId: input.id, status: (_a = cancelled === null || cancelled === void 0 ? void 0 : cancelled.status) !== null && _a !== void 0 ? _a : "cancelled" };
+        });
+    },
+});
+export const reminderTools = {
+    reminders_create: create,
+    reminders_list: list,
+    reminders_cancel: cancel,
+};