@cursor/july 0.1.16 → 0.1.18

package/dist/evals/assertions.d.ts

@@ -0,0 +1,190 @@
+/**
+ * Assertion recording and the scoped assertion vocabulary.
+ *
+ * Recorded assertions never throw: the recorder collects every result so one
+ * run reports all failures instead of dying on the first. Severity rides on the
+ * returned handle, so there is no separate thresholds map:
+ *
+ * ```ts
+ * t.succeeded();                                  // gate (hard)
+ * t.calledTool("get_weather").soft();             // tracked, does not gate
+ * t.check(t.reply, similarity("Sunny")).atLeast(0.8);
+ * ```
+ *
+ * The same vocabulary is available on the run (`t`) and on each turn returned
+ * by `t.send(...)`, which narrows the events under inspection to that turn.
+ */
+import type { SessionEvent } from "../types.js";
+import type { Expectation } from "./expect.js";
+import type { EvalCountMatcher, EvalEventMatcher, EvalEventOrderMatcher, EvalMatcher, EvalSubagentMatcher, EvalToolCallMatcher } from "./matchers.js";
+import type { EvalInputRequestFact, EvalToolCallFact } from "./run-facts.js";
+/** Hard gates fail the eval; soft assertions are tracked scores. */
+export type EvalAssertionSeverity = "gate" | "soft";
+/**
+ * Overall grade for one eval case.
+ *
+ * - `passed` — every gate passed and no soft threshold was missed.
+ * - `failed` — a gate failed, or the test body threw.
+ * - `scored` — only soft thresholds were missed: visible, and fatal only under
+ *   `--strict`.
+ * - `skipped` — `t.skip(reason)`, or a judge with no credentials available.
+ */
+export type EvalVerdict = "passed" | "failed" | "scored" | "skipped";
+export interface EvalAssertionResult {
+    name: string;
+    passed: boolean;
+    detail?: string;
+    /** Defaults to `gate`; omitted on the wire for gates to stay compact. */
+    severity?: EvalAssertionSeverity;
+    /** 0–1 score for scored assertions (`similarity`, judges). */
+    score?: number;
+    /** Bar a scored assertion must clear. */
+    threshold?: number;
+    /** Set when the assertion inspects one turn rather than the whole run. */
+    scope?: string;
+}
+/**
+ * Chainable severity control returned by every assertion.
+ *
+ * - `.gate(threshold?)` is hard: a miss fails the eval and `eval` exits 1.
+ * - `.soft(threshold?)` is tracked: a below-threshold miss marks the eval
+ *   `scored`, fatal only under `--strict`. With no threshold it never fails.
+ * - `.atLeast(threshold)` is soft with a bar.
+ */
+export interface EvalAssertionHandle {
+    gate(threshold?: number): EvalAssertionHandle;
+    soft(threshold?: number): EvalAssertionHandle;
+    atLeast(threshold: number): EvalAssertionHandle;
+}
+/** Thrown by `t.require(...)` to stop the test body without an error result. */
+export declare class EvalRequireFailure extends Error {
+    readonly assertionName: string;
+    constructor(assertionName: string);
+}
+/** Thrown by `t.skip(reason)` to end the case as skipped. */
+export declare class EvalSkipSignal extends Error {
+    readonly reason: string;
+    constructor(reason: string);
+}
+export interface RecordAssertionInput {
+    name: string;
+    passed: boolean;
+    detail?: string;
+    severity?: EvalAssertionSeverity;
+    score?: number;
+    threshold?: number;
+    scope?: string;
+}
+/**
+ * Collects assertion results and any in-flight judge scoring for one case.
+ */
+export declare class EvalAssertionRecorder {
+    private readonly slots;
+    private readonly pending;
+    record(input: RecordAssertionInput): EvalAssertionHandle;
+    /**
+     * Reserve a slot now and fill it in when `settle` resolves. Judges start when
+     * recorded so several graders overlap; {@link waitForPending} joins them.
+     */
+    recordAsync(input: RecordAssertionInput, settle: Promise<Partial<RecordAssertionInput>>): EvalAssertionHandle;
+    /** Await every async assertion started during the test body. */
+    waitForPending(): Promise<void>;
+    results(): EvalAssertionResult[];
+    get count(): number;
+}
+/**
+ * Grade a case from its recorded assertions.
+ *
+ * With no assertions at all, the caller's fallback applies (at least one
+ * successful turn), which keeps pre-severity eval files working unchanged.
+ */
+export declare function computeVerdict(results: readonly EvalAssertionResult[]): Exclude<EvalVerdict, "skipped">;
+/** Whether a verdict should fail the process exit code. */
+export declare function verdictFailsRun(verdict: EvalVerdict, strict: boolean): boolean;
+/**
+ * The deterministic assertion vocabulary, available on the run (`t`) and on
+ * each turn returned by `t.send(...)`.
+ */
+export interface EvalAssertions {
+    /**
+     * Gate: the run did not fail and is not parked on an unanswered approval.
+     * A session left open for the next message is a healthy end state.
+     */
+    succeeded(): EvalAssertionHandle;
+    /** Gate: the run cleanly parked on an unanswered human approval request. */
+    parked(): EvalAssertionHandle;
+    /** Gate: joined assistant text contains `token`. */
+    messageIncludes(token: string | RegExp): EvalAssertionHandle;
+    /**
+     * Gate: a matching call to `name` happened.
+     *
+     * With no matcher this is request-based (any lifecycle state), so it keeps
+     * meaning "the agent used this tool". Pass `{ status: "completed" }` to
+     * require the call actually returned, or `{ input, output, count }` to
+     * constrain it further.
+     */
+    calledTool(name: string, matcher?: EvalToolCallMatcher): EvalAssertionHandle;
+    /** Gate: no request for `name` in any lifecycle state. */
+    notCalledTool(name: string): EvalAssertionHandle;
+    /**
+     * Gate: the agent opened the skill's `SKILL.md`.
+     *
+     * Skills are materialized as files rather than exposed as a tool, so this
+     * matches any tool call whose input references `skills/<name>/SKILL.md`
+     * (a file read, a grep, or a shell `cat` all count).
+     */
+    loadedSkill(name: string): EvalAssertionHandle;
+    /** Gate: tool requests appear in this relative order (extra calls allowed). */
+    toolOrder(names: readonly string[]): EvalAssertionHandle;
+    /** Gate: no tool calls at all. */
+    usedNoTools(): EvalAssertionHandle;
+    /** Gate: at most `max` tool calls. */
+    maxToolCalls(max: number): EvalAssertionHandle;
+    /** Gate: no tool call reported an error. */
+    noFailedActions(): EvalAssertionHandle;
+    /** Gate: a matching subagent delegation happened. */
+    calledSubagent(name: string, matcher?: EvalSubagentMatcher): EvalAssertionHandle;
+    /** Gate: at least one matching event of `type` occurred. */
+    event(type: string, matcher?: EvalEventMatcher): EvalAssertionHandle;
+    /** Gate: no matching event of `type` occurred. */
+    notEvent(type: string, matcher?: EvalEventMatcher): EvalAssertionHandle;
+    /** Gate: matching event groups occur in this relative order. */
+    eventOrder(matchers: readonly EvalEventOrderMatcher[]): EvalAssertionHandle;
+    /** Gate: escape hatch for any predicate over the typed event stream. */
+    eventsSatisfy(label: string, predicate: (events: readonly SessionEvent[]) => boolean): EvalAssertionHandle;
+    /** Grade an explicit value with an `Expectation` builder. */
+    check(value: unknown, assertion: Expectation): EvalAssertionHandle;
+    /**
+     * Record a 0–1 score the eval computed itself, for a benchmark the built-in
+     * builders cannot express (recall against a gold set, a custom judge).
+     *
+     * Soft and tracked-only by default, so a new benchmark can report its number
+     * for a while before anyone gates on it. Add the bar when the number is
+     * trustworthy: `t.score("recall", recall).atLeast(0.6)`.
+     */
+    score(name: string, value: number): EvalAssertionHandle;
+    /**
+     * Record a gate and return the matching tool call so dependent code can read
+     * its input/output. Stops the test body when no call matches.
+     */
+    requireToolCall(name: string, matcher?: EvalToolCallMatcher): EvalToolCallFact;
+    /**
+     * Record a gate that exactly one approval request is pending, and return it.
+     * Stops the test body otherwise.
+     */
+    requireInputRequest(filter?: {
+        toolName?: EvalMatcher<string>;
+        args?: EvalMatcher;
+    }): EvalInputRequestFact;
+}
+export interface CreateAssertionsOptions {
+    recorder: EvalAssertionRecorder;
+    /** Events under inspection. Read lazily so run scope sees later turns. */
+    events: () => readonly SessionEvent[];
+    /** Label prefix for turn-scoped assertions, e.g. `turn[2]`. */
+    scope?: string;
+}
+/** Build the assertion vocabulary bound to one scope. */
+export declare function createAssertions(options: CreateAssertionsOptions): EvalAssertions;
+export type { EvalCountMatcher, EvalInputRequestFact, EvalToolCallFact };
+//# sourceMappingURL=assertions.d.ts.map

package/dist/evals/assertions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../../src/evals/assertions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/C,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAChB,qBAAqB,EACrB,WAAW,EACX,mBAAmB,EACnB,mBAAmB,EACpB,MAAM,eAAe,CAAC;AAEvB,OAAO,KAAK,EACV,oBAAoB,EAEpB,gBAAgB,EACjB,MAAM,gBAAgB,CAAC;AAGxB,oEAAoE;AACpE,MAAM,MAAM,qBAAqB,GAAG,MAAM,GAAG,MAAM,CAAC;AAEpD;;;;;;;;GAQG;AACH,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAErE,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yEAAyE;IACzE,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,8DAA8D;IAC9D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yCAAyC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0EAA0E;IAC1E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,mBAAmB;IAClC,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC9C,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC9C,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,mBAAmB,CAAC;CACjD;AAQD,gFAAgF;AAChF,qBAAa,kBAAmB,SAAQ,KAAK;IAC/B,QAAQ,CAAC,aAAa,EAAE,MAAM;IAA1C,YAAqB,aAAa,EAAE,MAAM,EAGzC;CACF;AAED,6DAA6D;AAC7D,qBAAa,cAAe,SAAQ,KAAK;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM;IAAnC,YAAqB,MAAM,EAAE,MAAM,EAGlC;CACF;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAuB;IAC7C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4B;IAEpD,MAAM,CAAC,KAAK,EAAE,oBAAoB,GAAG,mBAAmB,CAcvD;IAED;;;OAGG;IACH,WAAW,CACT,KAAK,EAAE,oBAAoB,EAC3B,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC,GAC7C,mBAAmB,CA0BrB;IAED,gEAAgE;IAC1D,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAKpC;IAED,OAAO,IAAI,mBAAmB,EAAE,CAW/B;IAED,IAAI,KAAK,IAAI,MAAM,CAElB;CACF;AA2CD;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,OAAO,EAAE,SAAS,mBAAmB,EAAE,GACtC,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAcjC;AAED,2DAA2D;AAC3D,wBAAgB,eAAe,CAC7B,OAAO,EAAE,WAAW,EACpB,MAAM,EAAE,OAAO,GACd,OAAO,CAKT;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,SAAS,IAAI,mBAAmB,CAAC;IACjC,4EAA4E;IAC5E,MAAM,IAAI,mBAAmB,CAAC;IAC9B,oDAAoD;IACpD,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,mBAAmB,CAAC;IAC7D;;;;;;;OAOG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,mBAAmB,CAAC;IAC7E,0DAA0D;IAC1D,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAAC;IACjD;;;;;;OAMG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC/C,+EAA+E;IAC/E,SAAS,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,mBAAmB,CAAC;IACzD,kCAAkC;IAClC,WAAW,IAAI,mBAAmB,CAAC;IACnC,sCAAsC;IACtC,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC/C,4CAA4C;IAC5C,eAAe,IAAI,mBAAmB,CAAC;IACvC,qDAAqD;IACrD,cAAc,CACZ,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,mBAAmB,GAC5B,mBAAmB,CAAC;IACvB,4DAA4D;IAC5D,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,mBAAmB,CAAC;IACrE,kDAAkD;IAClD,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,mBAAmB,CAAC;IACxE,gEAAgE;IAChE,UAAU,CAAC,QAAQ,EAAE,SAAS,qBAAqB,EAAE,GAAG,mBAAmB,CAAC;IAC5E,wEAAwE;IACxE,aAAa,CACX,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE,KAAK,OAAO,GACtD,mBAAmB,CAAC;IACvB,6DAA6D;IAC7D,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,GAAG,mBAAmB,CAAC;IACnE;;;;;;;OAOG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,mBAAmB,CAAC;IACxD;;;OAGG;IACH,eAAe,CACb,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,mBAAmB,GAC5B,gBAAgB,CAAC;IACpB;;;OAGG;IACH,mBAAmB,CAAC,MAAM,CAAC,EAAE;QAC3B,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC,EAAE,WAAW,CAAC;KACpB,GAAG,oBAAoB,CAAC;CAC1B;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,qBAAqB,CAAC;IAChC,0EAA0E;IAC1E,MAAM,EAAE,MAAM,SAAS,YAAY,EAAE,CAAC;IACtC,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,yDAAyD;AACzD,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,uBAAuB,GAC/B,cAAc,CA8UhB;AAED,YAAY,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,CAAC"}

package/dist/evals/assertions.js

@@ -0,0 +1,461 @@
+/**
+ * Assertion recording and the scoped assertion vocabulary.
+ *
+ * Recorded assertions never throw: the recorder collects every result so one
+ * run reports all failures instead of dying on the first. Severity rides on the
+ * returned handle, so there is no separate thresholds map:
+ *
+ * ```ts
+ * t.succeeded();                                  // gate (hard)
+ * t.calledTool("get_weather").soft();             // tracked, does not gate
+ * t.check(t.reply, similarity("Sunny")).atLeast(0.8);
+ * ```
+ *
+ * The same vocabulary is available on the run (`t`) and on each turn returned
+ * by `t.send(...)`, which narrows the events under inspection to that turn.
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { evaluateExpectation, expectationSeverity } from "./expect.js";
+import { describeMatcherObject, matchCount, matchValue } from "./matchers.js";
+import { deriveRunFacts } from "./run-facts.js";
+/** Thrown by `t.require(...)` to stop the test body without an error result. */
+export class EvalRequireFailure extends Error {
+    constructor(assertionName) {
+        super(`required assertion failed: ${assertionName}`);
+        this.assertionName = assertionName;
+        this.name = "EvalRequireFailure";
+    }
+}
+/** Thrown by `t.skip(reason)` to end the case as skipped. */
+export class EvalSkipSignal extends Error {
+    constructor(reason) {
+        super(reason);
+        this.reason = reason;
+        this.name = "EvalSkipSignal";
+    }
+}
+/**
+ * Collects assertion results and any in-flight judge scoring for one case.
+ */
+export class EvalAssertionRecorder {
+    constructor() {
+        this.slots = [];
+        this.pending = [];
+    }
+    record(input) {
+        var _a;
+        const slot = {
+            name: input.name,
+            passed: input.passed,
+            rawPassed: input.passed,
+            detail: input.detail,
+            severity: (_a = input.severity) !== null && _a !== void 0 ? _a : "gate",
+            score: input.score,
+            threshold: input.threshold,
+            scope: input.scope,
+        };
+        applyThreshold(slot);
+        this.slots.push(slot);
+        return handleFor(slot);
+    }
+    /**
+     * Reserve a slot now and fill it in when `settle` resolves. Judges start when
+     * recorded so several graders overlap; {@link waitForPending} joins them.
+     */
+    recordAsync(input, settle) {
+        const handle = this.record(input);
+        const slot = this.slots[this.slots.length - 1];
+        this.pending.push(settle.then((patch) => {
+            Object.assign(slot, patch);
+            if (patch.passed !== undefined) {
+                slot.rawPassed = patch.passed;
+            }
+            applyThreshold(slot);
+        }, (error) => {
+            // Grading blew up, which is a broken assertion rather than a low
+            // score. Soft severity covers "the model scored badly", so promote to
+            // a gate here or the failure would be invisible in the verdict.
+            slot.rawPassed = false;
+            slot.passed = false;
+            slot.score = undefined;
+            slot.threshold = undefined;
+            slot.severity = "gate";
+            slot.detail = error instanceof Error ? error.message : String(error);
+        }));
+        return handle;
+    }
+    /** Await every async assertion started during the test body. */
+    waitForPending() {
+        return __awaiter(this, void 0, void 0, function* () {
+            while (this.pending.length > 0) {
+                const inFlight = this.pending.splice(0, this.pending.length);
+                yield Promise.all(inFlight);
+            }
+        });
+    }
+    results() {
+        return this.slots.map((slot) => ({
+            name: slot.name,
+            passed: slot.passed,
+            detail: slot.detail,
+            // Keep the wire shape compact: `gate` is the default everywhere.
+            severity: slot.severity === "gate" ? undefined : slot.severity,
+            score: slot.score,
+            threshold: slot.threshold,
+            scope: slot.scope,
+        }));
+    }
+    get count() {
+        return this.slots.length;
+    }
+}
+/**
+ * Re-derive `passed` from the score whenever a threshold is in play.
+ *
+ * A tracked score with no bar always passes — there is nothing to miss. A
+ * boolean assertion keeps its recorded outcome even when demoted to soft, so
+ * the report stays honest; severity decides the verdict, not the `passed` flag.
+ */
+function applyThreshold(slot) {
+    var _a;
+    if (slot.threshold === undefined) {
+        slot.passed = slot.score !== undefined ? true : slot.rawPassed;
+        return;
+    }
+    const score = (_a = slot.score) !== null && _a !== void 0 ? _a : (slot.rawPassed ? 1 : 0);
+    slot.passed = score >= slot.threshold;
+}
+function handleFor(slot) {
+    const handle = {
+        gate(threshold) {
+            slot.severity = "gate";
+            if (threshold !== undefined) {
+                slot.threshold = threshold;
+            }
+            applyThreshold(slot);
+            return handle;
+        },
+        soft(threshold) {
+            slot.severity = "soft";
+            if (threshold !== undefined) {
+                slot.threshold = threshold;
+            }
+            applyThreshold(slot);
+            return handle;
+        },
+        atLeast(threshold) {
+            return handle.soft(threshold);
+        },
+    };
+    return handle;
+}
+/**
+ * Grade a case from its recorded assertions.
+ *
+ * With no assertions at all, the caller's fallback applies (at least one
+ * successful turn), which keeps pre-severity eval files working unchanged.
+ */
+export function computeVerdict(results) {
+    const failedGate = results.some((r) => { var _a; return ((_a = r.severity) !== null && _a !== void 0 ? _a : "gate") === "gate" && !r.passed; });
+    if (failedGate) {
+        return "failed";
+    }
+    const missedSoftBar = results.some((r) => { var _a; return ((_a = r.severity) !== null && _a !== void 0 ? _a : "gate") === "soft" &&
+        r.threshold !== undefined &&
+        !r.passed; });
+    return missedSoftBar ? "scored" : "passed";
+}
+/** Whether a verdict should fail the process exit code. */
+export function verdictFailsRun(verdict, strict) {
+    if (verdict === "failed") {
+        return true;
+    }
+    return strict && verdict === "scored";
+}
+/** Build the assertion vocabulary bound to one scope. */
+export function createAssertions(options) {
+    const { recorder, events, scope } = options;
+    const facts = () => deriveRunFacts(events());
+    const name = (text) => scope === undefined ? text : `${scope}.${text}`;
+    const record = (input) => recorder.record(Object.assign(Object.assign({}, input), { name: name(input.name), scope }));
+    const matchingToolCalls = (toolName, matcher) => facts().toolCalls.filter((call) => {
+        if (call.toolName !== toolName) {
+            return false;
+        }
+        if (matcher === undefined) {
+            return true;
+        }
+        if (matcher.status !== undefined && call.status !== matcher.status) {
+            return false;
+        }
+        if (!matchValue(call.input, matcher.input)) {
+            return false;
+        }
+        if (!matchValue(call.output, matcher.output)) {
+            return false;
+        }
+        return true;
+    });
+    const matchingEvents = (type, matcher) => events().filter((event) => event.type === type && matchValue(event.data, matcher === null || matcher === void 0 ? void 0 : matcher.data));
+    return {
+        succeeded() {
+            var _a;
+            const run = facts();
+            const detail = !run.ok
+                ? ((_a = run.failureMessage) !== null && _a !== void 0 ? _a : "turn failed")
+                : run.parked
+                    ? `parked on ${run.pendingInputRequests.length} unanswered approval request(s)`
+                    : undefined;
+            return record({
+                name: "succeeded",
+                passed: run.ok && !run.parked,
+                detail,
+            });
+        },
+        parked() {
+            const run = facts();
+            const pending = run.pendingInputRequests;
+            return record({
+                name: "parked",
+                passed: run.parked && run.turnsFailed === 0,
+                detail: run.parked
+                    ? pending.map((r) => r.toolName).join(", ")
+                    : "no unanswered approval request",
+            });
+        },
+        messageIncludes(token) {
+            const text = facts().assistantText;
+            const passed = typeof token === "string" ? text.includes(token) : token.test(text);
+            return record({
+                name: `messageIncludes(${String(token)})`,
+                passed,
+                detail: passed
+                    ? undefined
+                    : `text=${JSON.stringify(text.slice(0, 120))}`,
+            });
+        },
+        calledTool(toolName, matcher) {
+            const matched = matchingToolCalls(toolName, matcher);
+            const passed = matchCount(matched.length, matcher === null || matcher === void 0 ? void 0 : matcher.count);
+            const observed = facts()
+                .toolCalls.map((c) => c.toolName)
+                .join(", ");
+            return record({
+                name: `calledTool(${toolName}${describeMatcherObject(matcher)})`,
+                passed,
+                detail: passed
+                    ? undefined
+                    : `matched ${matched.length}; called [${observed}]`,
+            });
+        },
+        notCalledTool(toolName) {
+            const matched = matchingToolCalls(toolName, undefined);
+            return record({
+                name: `notCalledTool(${toolName})`,
+                passed: matched.length === 0,
+                detail: matched.length === 0 ? undefined : "tool was called",
+            });
+        },
+        loadedSkill(skill) {
+            const needle = `skills/${skill}/SKILL.md`;
+            const matched = facts().toolCalls.filter((call) => { var _a; return JSON.stringify((_a = call.input) !== null && _a !== void 0 ? _a : "").includes(needle); });
+            return record({
+                name: `loadedSkill(${skill})`,
+                passed: matched.length > 0,
+                detail: matched.length > 0 ? undefined : `no tool call referenced ${needle}`,
+            });
+        },
+        toolOrder(names) {
+            const called = facts().toolCalls.map((c) => c.toolName);
+            let cursor = 0;
+            for (const expected of names) {
+                const found = called.indexOf(expected, cursor);
+                if (found === -1) {
+                    return record({
+                        name: `toolOrder([${names.join(", ")}])`,
+                        passed: false,
+                        detail: `missing ${expected} after position ${cursor}; called [${called.join(", ")}]`,
+                    });
+                }
+                cursor = found + 1;
+            }
+            return record({
+                name: `toolOrder([${names.join(", ")}])`,
+                passed: true,
+            });
+        },
+        usedNoTools() {
+            const called = facts().toolCalls;
+            return record({
+                name: "usedNoTools",
+                passed: called.length === 0,
+                detail: called.length === 0
+                    ? undefined
+                    : `called [${called.map((c) => c.toolName).join(", ")}]`,
+            });
+        },
+        maxToolCalls(max) {
+            const total = facts().toolCalls.length;
+            return record({
+                name: `maxToolCalls(${max})`,
+                passed: total <= max,
+                detail: total <= max ? undefined : `made ${total} tool calls`,
+            });
+        },
+        noFailedActions() {
+            const failed = facts().failedToolCalls;
+            return record({
+                name: "noFailedActions",
+                passed: failed.length === 0,
+                detail: failed.length === 0
+                    ? undefined
+                    : failed.map((c) => c.toolName).join(", "),
+            });
+        },
+        calledSubagent(subagentName, matcher) {
+            const matched = facts().subagents.filter((sub) => {
+                if (sub.name !== subagentName) {
+                    return false;
+                }
+                if (matcher === undefined) {
+                    return true;
+                }
+                if (matcher.status !== undefined && sub.status !== matcher.status) {
+                    return false;
+                }
+                if (!matchValue(sub.callId, matcher.callId)) {
+                    return false;
+                }
+                return matchValue(sub.output, matcher.output);
+            });
+            const passed = matchCount(matched.length, matcher === null || matcher === void 0 ? void 0 : matcher.count);
+            return record({
+                name: `calledSubagent(${subagentName}${describeMatcherObject(matcher)})`,
+                passed,
+                detail: passed
+                    ? undefined
+                    : `matched ${matched.length}; delegated [${facts()
+                        .subagents.map((s) => { var _a; return (_a = s.name) !== null && _a !== void 0 ? _a : s.callId; })
+                        .join(", ")}]`,
+            });
+        },
+        event(type, matcher) {
+            const matched = matchingEvents(type, matcher);
+            const passed = matchCount(matched.length, matcher === null || matcher === void 0 ? void 0 : matcher.count);
+            return record({
+                name: `event(${type}${describeMatcherObject(matcher)})`,
+                passed,
+                detail: passed ? undefined : `matched ${matched.length}`,
+            });
+        },
+        notEvent(type, matcher) {
+            const matched = matchingEvents(type, matcher);
+            return record({
+                name: `notEvent(${type}${describeMatcherObject(matcher)})`,
+                passed: matched.length === 0,
+                detail: matched.length === 0 ? undefined : `matched ${matched.length}`,
+            });
+        },
+        eventOrder(matchers) {
+            const stream = events();
+            const label = `eventOrder([${matchers.map((m) => m.type).join(", ")}])`;
+            let cursor = 0;
+            for (const matcher of matchers) {
+                const required = typeof matcher.count === "number" ? matcher.count : 1;
+                let seen = 0;
+                while (seen < required && cursor < stream.length) {
+                    const event = stream[cursor];
+                    cursor++;
+                    if (event.type === matcher.type &&
+                        matchValue(event.data, matcher.data)) {
+                        seen++;
+                    }
+                }
+                if (seen < required) {
+                    return record({
+                        name: label,
+                        passed: false,
+                        detail: `only ${seen}/${required} ${matcher.type} event(s) in order`,
+                    });
+                }
+            }
+            return record({ name: label, passed: true });
+        },
+        eventsSatisfy(label, predicate) {
+            let passed = false;
+            let detail;
+            try {
+                passed = predicate(events()) === true;
+            }
+            catch (error) {
+                detail = error instanceof Error ? error.message : String(error);
+            }
+            return record({
+                name: `eventsSatisfy(${label})`,
+                passed,
+                detail: passed ? undefined : (detail !== null && detail !== void 0 ? detail : label),
+            });
+        },
+        check(value, assertion) {
+            const outcome = evaluateExpectation(value, assertion);
+            return record({
+                name: `check(${assertion.kind})`,
+                passed: outcome.passed,
+                detail: outcome.detail,
+                score: outcome.score,
+                severity: expectationSeverity(assertion),
+            });
+        },
+        score(scoreName, value) {
+            const finite = Number.isFinite(value);
+            return record({
+                name: `score(${scoreName})`,
+                // A NaN score is a bug in the eval's own math, so fail loudly rather
+                // than reporting a number nobody can act on.
+                passed: finite,
+                severity: "soft",
+                score: finite ? value : undefined,
+                detail: finite ? undefined : `score is not a finite number: ${value}`,
+            });
+        },
+        requireToolCall(toolName, matcher) {
+            const matched = matchingToolCalls(toolName, matcher);
+            const assertionName = `requireToolCall(${toolName})`;
+            const first = matched[0];
+            record({
+                name: assertionName,
+                passed: first !== undefined,
+                detail: first === undefined ? "no matching tool call" : undefined,
+            });
+            if (first === undefined) {
+                throw new EvalRequireFailure(name(assertionName));
+            }
+            return first;
+        },
+        requireInputRequest(filter) {
+            const pending = facts().pendingInputRequests.filter((request) => matchValue(request.toolName, filter === null || filter === void 0 ? void 0 : filter.toolName) &&
+                matchValue(request.args, filter === null || filter === void 0 ? void 0 : filter.args));
+            const assertionName = "requireInputRequest";
+            const passed = pending.length === 1;
+            record({
+                name: assertionName,
+                passed,
+                detail: passed
+                    ? undefined
+                    : `expected exactly 1 pending approval request, found ${pending.length}`,
+            });
+            if (!passed) {
+                throw new EvalRequireFailure(name(assertionName));
+            }
+            return pending[0];
+        },
+    };
+}