@cursor/bdk 0.2.1

package/dist/evals/assertions.js

@@ -0,0 +1,459 @@
+/**
+ * Assertion recording and the scoped assertion vocabulary.
+ *
+ * Recorded assertions never throw: the recorder collects every result so one
+ * run reports all failures instead of dying on the first. Severity rides on the
+ * returned handle, so there is no separate thresholds map:
+ *
+ * ```ts
+ * t.succeeded();                                  // gate (hard)
+ * t.calledTool("get_weather").soft();             // tracked, does not gate
+ * t.check(t.reply, similarity("Sunny")).atLeast(0.8);
+ * ```
+ *
+ * The same vocabulary is available on the run (`t`) and on each turn returned
+ * by `t.send(...)`, which narrows the events under inspection to that turn.
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { evaluateExpectation, expectationSeverity } from "./expect.js";
+import { describeMatcherObject, matchCount, matchValue } from "./matchers.js";
+import { deriveRunFacts } from "./run-facts.js";
+/** Thrown by `t.require(...)` to stop the test body without an error result. */
+export class EvalRequireFailure extends Error {
+    constructor(assertionName) {
+        super(`required assertion failed: ${assertionName}`);
+        this.assertionName = assertionName;
+        this.name = "EvalRequireFailure";
+    }
+}
+/** Thrown by `t.skip(reason)` to end the case as skipped. */
+export class EvalSkipSignal extends Error {
+    constructor(reason) {
+        super(reason);
+        this.reason = reason;
+        this.name = "EvalSkipSignal";
+    }
+}
+/**
+ * Collects assertion results and any in-flight judge scoring for one case.
+ */
+export class EvalAssertionRecorder {
+    constructor() {
+        this.slots = [];
+        this.pending = [];
+    }
+    record(input) {
+        var _a;
+        const slot = {
+            name: input.name,
+            passed: input.passed,
+            rawPassed: input.passed,
+            detail: input.detail,
+            severity: (_a = input.severity) !== null && _a !== void 0 ? _a : "gate",
+            score: input.score,
+            threshold: input.threshold,
+            scope: input.scope,
+        };
+        applyThreshold(slot);
+        this.slots.push(slot);
+        return handleFor(slot);
+    }
+    /**
+     * Reserve a slot now and fill it in when `settle` resolves. Judges start when
+     * recorded so several graders overlap; {@link waitForPending} joins them.
+     */
+    recordAsync(input, settle) {
+        const handle = this.record(input);
+        const slot = this.slots[this.slots.length - 1];
+        this.pending.push(settle.then(patch => {
+            Object.assign(slot, patch);
+            if (patch.passed !== undefined) {
+                slot.rawPassed = patch.passed;
+            }
+            applyThreshold(slot);
+        }, (error) => {
+            // Grading blew up, which is a broken assertion rather than a low
+            // score. Soft severity covers "the model scored badly", so promote to
+            // a gate here or the failure would be invisible in the verdict.
+            slot.rawPassed = false;
+            slot.passed = false;
+            slot.score = undefined;
+            slot.threshold = undefined;
+            slot.severity = "gate";
+            slot.detail = error instanceof Error ? error.message : String(error);
+        }));
+        return handle;
+    }
+    /** Await every async assertion started during the test body. */
+    waitForPending() {
+        return __awaiter(this, void 0, void 0, function* () {
+            while (this.pending.length > 0) {
+                const inFlight = this.pending.splice(0, this.pending.length);
+                yield Promise.all(inFlight);
+            }
+        });
+    }
+    results() {
+        return this.slots.map(slot => ({
+            name: slot.name,
+            passed: slot.passed,
+            detail: slot.detail,
+            // Keep the wire shape compact: `gate` is the default everywhere.
+            severity: slot.severity === "gate" ? undefined : slot.severity,
+            score: slot.score,
+            threshold: slot.threshold,
+            scope: slot.scope,
+        }));
+    }
+    get count() {
+        return this.slots.length;
+    }
+}
+/**
+ * Re-derive `passed` from the score whenever a threshold is in play.
+ *
+ * A tracked score with no bar always passes — there is nothing to miss. A
+ * boolean assertion keeps its recorded outcome even when demoted to soft, so
+ * the report stays honest; severity decides the verdict, not the `passed` flag.
+ */
+function applyThreshold(slot) {
+    var _a;
+    if (slot.threshold === undefined) {
+        slot.passed = slot.score !== undefined ? true : slot.rawPassed;
+        return;
+    }
+    const score = (_a = slot.score) !== null && _a !== void 0 ? _a : (slot.rawPassed ? 1 : 0);
+    slot.passed = score >= slot.threshold;
+}
+function handleFor(slot) {
+    const handle = {
+        gate(threshold) {
+            slot.severity = "gate";
+            if (threshold !== undefined) {
+                slot.threshold = threshold;
+            }
+            applyThreshold(slot);
+            return handle;
+        },
+        soft(threshold) {
+            slot.severity = "soft";
+            if (threshold !== undefined) {
+                slot.threshold = threshold;
+            }
+            applyThreshold(slot);
+            return handle;
+        },
+        atLeast(threshold) {
+            return handle.soft(threshold);
+        },
+    };
+    return handle;
+}
+/**
+ * Grade a case from its recorded assertions.
+ *
+ * With no assertions at all, the caller's fallback applies (at least one
+ * successful turn), which keeps pre-severity eval files working unchanged.
+ */
+export function computeVerdict(results) {
+    const failedGate = results.some(r => { var _a; return ((_a = r.severity) !== null && _a !== void 0 ? _a : "gate") === "gate" && !r.passed; });
+    if (failedGate) {
+        return "failed";
+    }
+    const missedSoftBar = results.some(r => { var _a; return ((_a = r.severity) !== null && _a !== void 0 ? _a : "gate") === "soft" && r.threshold !== undefined && !r.passed; });
+    return missedSoftBar ? "scored" : "passed";
+}
+/** Whether a verdict should fail the process exit code. */
+export function verdictFailsRun(verdict, strict) {
+    if (verdict === "failed") {
+        return true;
+    }
+    return strict && verdict === "scored";
+}
+/** Build the assertion vocabulary bound to one scope. */
+export function createAssertions(options) {
+    const { recorder, events, scope } = options;
+    const facts = () => deriveRunFacts(events());
+    const name = (text) => (scope === undefined ? text : `${scope}.${text}`);
+    const record = (input) => recorder.record(Object.assign(Object.assign({}, input), { name: name(input.name), scope }));
+    const matchingToolCalls = (toolName, matcher) => facts().toolCalls.filter(call => {
+        if (call.toolName !== toolName) {
+            return false;
+        }
+        if (matcher === undefined) {
+            return true;
+        }
+        if (matcher.status !== undefined && call.status !== matcher.status) {
+            return false;
+        }
+        if (!matchValue(call.input, matcher.input)) {
+            return false;
+        }
+        if (!matchValue(call.output, matcher.output)) {
+            return false;
+        }
+        return true;
+    });
+    const matchingEvents = (type, matcher) => events().filter(event => event.type === type && matchValue(event.data, matcher === null || matcher === void 0 ? void 0 : matcher.data));
+    return {
+        succeeded() {
+            var _a;
+            const run = facts();
+            const detail = !run.ok
+                ? ((_a = run.failureMessage) !== null && _a !== void 0 ? _a : "turn failed")
+                : run.parked
+                    ? `parked on ${run.pendingInputRequests.length} unanswered approval request(s)`
+                    : undefined;
+            return record({
+                name: "succeeded",
+                passed: run.ok && !run.parked,
+                detail,
+            });
+        },
+        parked() {
+            const run = facts();
+            const pending = run.pendingInputRequests;
+            return record({
+                name: "parked",
+                passed: run.parked && run.turnsFailed === 0,
+                detail: run.parked
+                    ? pending.map(r => r.toolName).join(", ")
+                    : "no unanswered approval request",
+            });
+        },
+        messageIncludes(token) {
+            const text = facts().assistantText;
+            const passed = typeof token === "string" ? text.includes(token) : token.test(text);
+            return record({
+                name: `messageIncludes(${String(token)})`,
+                passed,
+                detail: passed ? undefined : `text=${JSON.stringify(text.slice(0, 120))}`,
+            });
+        },
+        calledTool(toolName, matcher) {
+            const matched = matchingToolCalls(toolName, matcher);
+            const passed = matchCount(matched.length, matcher === null || matcher === void 0 ? void 0 : matcher.count);
+            const observed = facts()
+                .toolCalls.map(c => c.toolName)
+                .join(", ");
+            return record({
+                name: `calledTool(${toolName}${describeMatcherObject(matcher)})`,
+                passed,
+                detail: passed ? undefined : `matched ${matched.length}; called [${observed}]`,
+            });
+        },
+        notCalledTool(toolName) {
+            const matched = matchingToolCalls(toolName, undefined);
+            return record({
+                name: `notCalledTool(${toolName})`,
+                passed: matched.length === 0,
+                detail: matched.length === 0 ? undefined : "tool was called",
+            });
+        },
+        loadedSkill(skill) {
+            const needle = `skills/${skill}/SKILL.md`;
+            const matched = facts().toolCalls.filter(call => { var _a; return JSON.stringify((_a = call.input) !== null && _a !== void 0 ? _a : "").includes(needle); });
+            return record({
+                name: `loadedSkill(${skill})`,
+                passed: matched.length > 0,
+                detail: matched.length > 0 ? undefined : `no tool call referenced ${needle}`,
+            });
+        },
+        toolOrder(names) {
+            const called = facts().toolCalls.map(c => c.toolName);
+            let cursor = 0;
+            for (const expected of names) {
+                const found = called.indexOf(expected, cursor);
+                if (found === -1) {
+                    return record({
+                        name: `toolOrder([${names.join(", ")}])`,
+                        passed: false,
+                        detail: `missing ${expected} after position ${cursor}; called [${called.join(", ")}]`,
+                    });
+                }
+                cursor = found + 1;
+            }
+            return record({
+                name: `toolOrder([${names.join(", ")}])`,
+                passed: true,
+            });
+        },
+        usedNoTools() {
+            const called = facts().toolCalls;
+            return record({
+                name: "usedNoTools",
+                passed: called.length === 0,
+                detail: called.length === 0 ? undefined : `called [${called.map(c => c.toolName).join(", ")}]`,
+            });
+        },
+        maxToolCalls(max) {
+            const total = facts().toolCalls.length;
+            return record({
+                name: `maxToolCalls(${max})`,
+                passed: total <= max,
+                detail: total <= max ? undefined : `made ${total} tool calls`,
+            });
+        },
+        noFailedActions() {
+            const failed = facts().failedToolCalls;
+            return record({
+                name: "noFailedActions",
+                passed: failed.length === 0,
+                detail: failed.length === 0 ? undefined : failed.map(c => c.toolName).join(", "),
+            });
+        },
+        calledSubagent(subagentName, matcher) {
+            const matched = facts().subagents.filter(sub => {
+                if (sub.name !== subagentName) {
+                    return false;
+                }
+                if (matcher === undefined) {
+                    return true;
+                }
+                if (matcher.status !== undefined && sub.status !== matcher.status) {
+                    return false;
+                }
+                if (!matchValue(sub.callId, matcher.callId)) {
+                    return false;
+                }
+                return matchValue(sub.output, matcher.output);
+            });
+            const passed = matchCount(matched.length, matcher === null || matcher === void 0 ? void 0 : matcher.count);
+            return record({
+                name: `calledSubagent(${subagentName}${describeMatcherObject(matcher)})`,
+                passed,
+                detail: passed
+                    ? undefined
+                    : `matched ${matched.length}; delegated [${facts()
+                        .subagents.map(s => { var _a; return (_a = s.name) !== null && _a !== void 0 ? _a : s.callId; })
+                        .join(", ")}]`,
+            });
+        },
+        taggedArtifact(kind, predicate) {
+            const tagged = events().flatMap(event => event.type === "artifact.tagged" ? [event.data] : []);
+            const matched = tagged.filter(record => (kind === undefined || record.kind === kind) &&
+                (predicate === undefined || predicate(record) === true));
+            return record({
+                name: `taggedArtifact(${kind !== null && kind !== void 0 ? kind : ""})`,
+                passed: matched.length > 0,
+                detail: matched.length > 0 ? undefined : `tagged kinds: [${tagged.map(r => r.kind).join(", ")}]`,
+            });
+        },
+        event(type, matcher) {
+            const matched = matchingEvents(type, matcher);
+            const passed = matchCount(matched.length, matcher === null || matcher === void 0 ? void 0 : matcher.count);
+            return record({
+                name: `event(${type}${describeMatcherObject(matcher)})`,
+                passed,
+                detail: passed ? undefined : `matched ${matched.length}`,
+            });
+        },
+        notEvent(type, matcher) {
+            const matched = matchingEvents(type, matcher);
+            return record({
+                name: `notEvent(${type}${describeMatcherObject(matcher)})`,
+                passed: matched.length === 0,
+                detail: matched.length === 0 ? undefined : `matched ${matched.length}`,
+            });
+        },
+        eventOrder(matchers) {
+            const stream = events();
+            const label = `eventOrder([${matchers.map(m => m.type).join(", ")}])`;
+            let cursor = 0;
+            for (const matcher of matchers) {
+                const required = typeof matcher.count === "number" ? matcher.count : 1;
+                let seen = 0;
+                while (seen < required && cursor < stream.length) {
+                    const event = stream[cursor];
+                    cursor++;
+                    if (event.type === matcher.type && matchValue(event.data, matcher.data)) {
+                        seen++;
+                    }
+                }
+                if (seen < required) {
+                    return record({
+                        name: label,
+                        passed: false,
+                        detail: `only ${seen}/${required} ${matcher.type} event(s) in order`,
+                    });
+                }
+            }
+            return record({ name: label, passed: true });
+        },
+        eventsSatisfy(label, predicate) {
+            let passed = false;
+            let detail;
+            try {
+                passed = predicate(events()) === true;
+            }
+            catch (error) {
+                detail = error instanceof Error ? error.message : String(error);
+            }
+            return record({
+                name: `eventsSatisfy(${label})`,
+                passed,
+                detail: passed ? undefined : (detail !== null && detail !== void 0 ? detail : label),
+            });
+        },
+        check(value, assertion) {
+            const outcome = evaluateExpectation(value, assertion);
+            return record({
+                name: `check(${assertion.kind})`,
+                passed: outcome.passed,
+                detail: outcome.detail,
+                score: outcome.score,
+                severity: expectationSeverity(assertion),
+            });
+        },
+        score(scoreName, value) {
+            const finite = Number.isFinite(value);
+            return record({
+                name: `score(${scoreName})`,
+                // A NaN score is a bug in the eval's own math, so fail loudly rather
+                // than reporting a number nobody can act on.
+                passed: finite,
+                severity: "soft",
+                score: finite ? value : undefined,
+                detail: finite ? undefined : `score is not a finite number: ${value}`,
+            });
+        },
+        requireToolCall(toolName, matcher) {
+            const matched = matchingToolCalls(toolName, matcher);
+            const assertionName = `requireToolCall(${toolName})`;
+            const first = matched[0];
+            record({
+                name: assertionName,
+                passed: first !== undefined,
+                detail: first === undefined ? "no matching tool call" : undefined,
+            });
+            if (first === undefined) {
+                throw new EvalRequireFailure(name(assertionName));
+            }
+            return first;
+        },
+        requireInputRequest(filter) {
+            const pending = facts().pendingInputRequests.filter(request => matchValue(request.toolName, filter === null || filter === void 0 ? void 0 : filter.toolName) && matchValue(request.args, filter === null || filter === void 0 ? void 0 : filter.args));
+            const assertionName = "requireInputRequest";
+            const passed = pending.length === 1;
+            record({
+                name: assertionName,
+                passed,
+                detail: passed
+                    ? undefined
+                    : `expected exactly 1 pending approval request, found ${pending.length}`,
+            });
+            if (!passed) {
+                throw new EvalRequireFailure(name(assertionName));
+            }
+            return pending[0];
+        },
+    };
+}

package/dist/evals/expect.d.ts

@@ -0,0 +1,95 @@
+/**
+ * Value assertion builders for `t.check(value, …)`.
+ *
+ * Pick the cheapest builder that captures what "correct" means:
+ *
+ * | Builder                | Scores                                        | Default |
+ * | ---------------------- | --------------------------------------------- | ------- |
+ * | `includes(value)`      | substring or RegExp over the coerced string    | gate    |
+ * | `equals(value)`        | deep structural equality                      | gate    |
+ * | `matches(schema)`      | Standard Schema (e.g. Zod) validation          | gate    |
+ * | `similarity(expected)` | normalized Levenshtein, 1 = identical         | soft    |
+ * | `satisfies(fn, label)` | custom boolean predicate                      | gate    |
+ *
+ * When exact match is too strict but an LLM judge is overkill, `similarity` is
+ * the middle ground: it records a 0–1 score, so `.atLeast(0.8)` sets the bar.
+ */
+/** One Standard Schema validation issue. */
+export interface StandardSchemaIssue {
+    message: string;
+    /** Property path to the offending value, when the schema reports one. */
+    path?: ReadonlyArray<PropertyKey | {
+        key: PropertyKey;
+    }>;
+}
+/** A schema exposing the Standard Schema v1 `~standard` interface. */
+export interface StandardSchemaLike {
+    "~standard": {
+        validate: (value: unknown) => {
+            issues?: ReadonlyArray<StandardSchemaIssue>;
+        } | Promise<unknown>;
+    };
+}
+/** A Zod-style schema; supported so pre-Standard-Schema versions still work. */
+export interface SafeParseSchemaLike {
+    safeParse: (value: unknown) => {
+        success: boolean;
+        error?: {
+            message?: string;
+        };
+    };
+}
+export type EvalSchemaLike = StandardSchemaLike | SafeParseSchemaLike;
+export type Expectation = {
+    kind: "includes";
+    value: string | RegExp;
+} | {
+    kind: "equals";
+    value: unknown;
+} | {
+    kind: "matches";
+    schema: EvalSchemaLike;
+} | {
+    kind: "similarity";
+    expected: string;
+} | {
+    kind: "satisfies";
+    predicate: (value: unknown) => boolean;
+    label: string;
+};
+/**
+ * Gate: the coerced string contains `value` (or matches the RegExp).
+ * A structured value is coerced to JSON first, so this works on parsed output.
+ */
+export declare function includes(value: string | RegExp): Expectation;
+/** Gate: deep structural equality. */
+export declare function equals(value: unknown): Expectation;
+/**
+ * Gate: the value validates against a Standard Schema (Zod, Valibot, …).
+ * Use it to assert the shape of parsed JSON instead of spot-checking keys.
+ */
+export declare function matches(schema: EvalSchemaLike): Expectation;
+/**
+ * Soft score: normalized Levenshtein similarity against `expected`
+ * (1 = identical, 0 = nothing in common). Tracked-only until you add a bar
+ * with `.atLeast(0.8)` or promote it with `.gate(0.8)`.
+ */
+export declare function similarity(expected: string): Expectation;
+/** Gate: a custom predicate. `label` is the failure detail. */
+export declare function satisfies(predicate: (value: unknown) => boolean, label: string): Expectation;
+/** Default severity for an expectation, before `.gate()` / `.soft()`. */
+export declare function expectationSeverity(assertion: Expectation): "gate" | "soft";
+export interface ExpectationOutcome {
+    passed: boolean;
+    detail?: string;
+    /** Present for scored builders (`similarity`). */
+    score?: number;
+}
+/** Grade `value` against `assertion`. */
+export declare function evaluateExpectation(value: unknown, assertion: Expectation): ExpectationOutcome;
+/**
+ * Levenshtein distance normalized to a 0–1 similarity over the longer string.
+ * Uses the two-row form, so memory is O(min(len)) rather than O(len²).
+ */
+export declare function normalizedSimilarity(actual: string, expected: string): number;
+//# sourceMappingURL=expect.d.ts.map

package/dist/evals/expect.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"expect.d.ts","sourceRoot":"","sources":["../../src/evals/expect.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAIH,4CAA4C;AAC5C,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,yEAAyE;IACzE,IAAI,CAAC,EAAE,aAAa,CAAC,WAAW,GAAG;QAAE,GAAG,EAAE,WAAW,CAAA;KAAE,CAAC,CAAC;CAC1D;AAED,sEAAsE;AACtE,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE;QACX,QAAQ,EAAE,CACR,KAAK,EAAE,OAAO,KACX;YAAE,MAAM,CAAC,EAAE,aAAa,CAAC,mBAAmB,CAAC,CAAA;SAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;KACzE,CAAC;CACH;AAED,gFAAgF;AAChF,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK;QAC7B,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE;YAAE,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KAC9B,CAAC;CACH;AAED,MAAM,MAAM,cAAc,GAAG,kBAAkB,GAAG,mBAAmB,CAAC;AAEtE,MAAM,MAAM,WAAW,GACnB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,GAC5C;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,GAClC;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,cAAc,CAAA;CAAE,GAC3C;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GACxC;IACE,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;IACvC,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEN;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,WAAW,CAE5D;AAED,sCAAsC;AACtC,wBAAgB,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,WAAW,CAElD;AAED;;;GAGG;AACH,wBAAgB,OAAO,CAAC,MAAM,EAAE,cAAc,GAAG,WAAW,CAE3D;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,WAAW,CAExD;AAED,+DAA+D;AAC/D,wBAAgB,SAAS,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW,CAE5F;AAKD,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,WAAW,GAAG,MAAM,GAAG,MAAM,CAE3E;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AA8DD,yCAAyC;AACzC,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,GAAG,kBAAkB,CAwC9F;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAS7E"}