@cursor/july 0.1.15 → 0.1.17

package/dist/evals/reporters.js

@@ -0,0 +1,221 @@
+/**
+ * Built-in eval reporters.
+ *
+ * The CLI already prints a console summary; reporters add destinations on top.
+ * Attach them in `evals/evals.config.ts` to observe every case in the run, or
+ * on a single `defineEval({ reporters })` to scope a destination to that case.
+ *
+ * ```ts
+ * // evals/evals.config.ts
+ * import { defineEvalConfig } from "@cursor/july/evals";
+ * import { JUnit } from "@cursor/july/evals/reporters";
+ *
+ * export default defineEvalConfig({
+ *   maxConcurrency: 20,
+ *   reporters: [JUnit({ filePath: ".agent-serve/junit.xml" })],
+ * });
+ * ```
+ *
+ * `agentkit eval --junit <path>` and `--artifacts <dir>` attach these same two
+ * reporters without touching the eval files, which is usually the better fit:
+ * CI owns the output paths, not the eval author.
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+/**
+ * Write JUnit XML so CI shows per-case annotations.
+ *
+ * Each case becomes one `<testcase>` named by its path-derived id. Failed gates
+ * and execution errors become `<failure>`, `t.skip(reason)` becomes `<skipped>`,
+ * and a soft threshold miss becomes a `<failure>` only under `--strict` (it is
+ * reported as a `<system-out>` score line otherwise).
+ */
+export function JUnit(options) {
+    return {
+        onRunComplete(summary) {
+            return __awaiter(this, void 0, void 0, function* () {
+                var _a;
+                const xml = renderJUnitXml(summary, (_a = options.suiteName) !== null && _a !== void 0 ? _a : "agentkit-evals");
+                const target = resolve(options.filePath);
+                yield mkdir(dirname(target), { recursive: true });
+                yield writeFile(target, xml, "utf8");
+            });
+        },
+    };
+}
+/** Render a {@link EvalRunSummary} as JUnit XML. Exported for tests. */
+export function renderJUnitXml(summary, suiteName) {
+    var _a, _b, _c, _d;
+    const lines = [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        `<testsuites name="${escapeXml(suiteName)}" tests="${summary.total}" failures="${summary.failed + (summary.strict ? summary.scored : 0)}" skipped="${summary.skipped}" time="${(summary.durationMs / 1000).toFixed(3)}">`,
+        `  <testsuite name="${escapeXml(suiteName)}" tests="${summary.total}" failures="${summary.failed + (summary.strict ? summary.scored : 0)}" skipped="${summary.skipped}" time="${(summary.durationMs / 1000).toFixed(3)}">`,
+    ];
+    for (const result of summary.results) {
+        const verdict = (_a = result.verdict) !== null && _a !== void 0 ? _a : (result.ok ? "passed" : "failed");
+        const time = (result.durationMs / 1000).toFixed(3);
+        lines.push(`    <testcase name="${escapeXml(result.id)}" classname="${escapeXml(suiteName)}" time="${time}">`);
+        if (verdict === "skipped") {
+            lines.push(`      <skipped message="${escapeXml((_b = result.skipReason) !== null && _b !== void 0 ? _b : "skipped")}"/>`);
+        }
+        else if (verdict === "failed" ||
+            (verdict === "scored" && summary.strict)) {
+            const failures = result.assertions.filter((a) => !a.passed);
+            const message = (_d = (_c = result.error) !== null && _c !== void 0 ? _c : failures
+                .map((a) => `${a.name}${a.detail === undefined ? "" : ` (${a.detail})`}`)
+                .join("; ")) !== null && _d !== void 0 ? _d : verdict;
+            lines.push(`      <failure message="${escapeXml(message === "" ? verdict : message)}" type="${escapeXml(verdict)}"/>`);
+        }
+        const systemOut = renderSystemOut(result);
+        if (systemOut !== "") {
+            lines.push(`      <system-out>${escapeXml(systemOut)}</system-out>`);
+        }
+        lines.push("    </testcase>");
+    }
+    lines.push("  </testsuite>", "</testsuites>", "");
+    return lines.join("\n");
+}
+function renderSystemOut(result) {
+    const parts = [];
+    for (const assertion of result.assertions) {
+        if (assertion.score === undefined) {
+            continue;
+        }
+        const bar = assertion.threshold === undefined ? "" : ` (>= ${assertion.threshold})`;
+        parts.push(`${assertion.name}=${assertion.score.toFixed(3)}${bar}`);
+    }
+    for (const [name, value] of Object.entries(result.metrics)) {
+        parts.push(`${name}=${String(value)}`);
+    }
+    parts.push(...result.logs);
+    return parts.join("\n");
+}
+function escapeXml(text) {
+    return stripIllegalXmlChars(text)
+        .replace(/&/g, "&amp;")
+        .replace(/</g, "&lt;")
+        .replace(/>/g, "&gt;")
+        .replace(/"/g, "&quot;")
+        .replace(/'/g, "&apos;");
+}
+/**
+ * Drop characters XML 1.0 forbids outright.
+ *
+ * Tool output routinely carries stray control bytes, and they are illegal in
+ * XML even when escaped — a CI provider would reject the whole report. Tab,
+ * newline, and carriage return are the only legal ones below 0x20.
+ */
+function stripIllegalXmlChars(text) {
+    let out = "";
+    for (const char of text) {
+        const code = char.codePointAt(0);
+        if (code >= 0x20 || code === 0x09 || code === 0x0a || code === 0x0d) {
+            out += char;
+        }
+    }
+    return out;
+}
+/**
+ * Persist the full story of a run to disk.
+ *
+ * Console output stays tight on purpose; when a case fails, the artifact has
+ * everything — every assertion with its score, the captured event stream, the
+ * tool calls, and `t.log` lines.
+ *
+ * ```text
+ * <dir>/summary.json                 run totals + config
+ * <dir>/results.jsonl                one line per case, in completion order
+ * <dir>/evals/<case-id>.json         per-case detail
+ * ```
+ */
+export function Artifacts(options) {
+    const root = resolve(options.dir);
+    const jsonlPath = join(root, "results.jsonl");
+    const lines = [];
+    let target;
+    let discovered = [];
+    return {
+        onRunStart(evals, runTarget) {
+            return __awaiter(this, void 0, void 0, function* () {
+                discovered = evals;
+                target = runTarget;
+                yield mkdir(join(root, "evals"), { recursive: true });
+            });
+        },
+        onEvalComplete(result) {
+            return __awaiter(this, void 0, void 0, function* () {
+                var _a;
+                lines.push(JSON.stringify({
+                    id: result.id,
+                    verdict: (_a = result.verdict) !== null && _a !== void 0 ? _a : (result.ok ? "passed" : "failed"),
+                    ok: result.ok,
+                    durationMs: result.durationMs,
+                }));
+                const file = join(root, "evals", `${sanitizeCaseId(result.id)}.json`);
+                yield mkdir(dirname(file), { recursive: true });
+                yield writeFile(file, `${JSON.stringify(result, null, 2)}\n`, "utf8");
+                yield writeFile(jsonlPath, `${lines.join("\n")}\n`, "utf8");
+            });
+        },
+        onRunComplete(summary) {
+            return __awaiter(this, void 0, void 0, function* () {
+                yield mkdir(root, { recursive: true });
+                const { results } = summary, totals = __rest(summary, ["results"]);
+                yield writeFile(join(root, "summary.json"), `${JSON.stringify(Object.assign(Object.assign({}, totals), { caseCount: results.length, target, discovered: discovered.map((e) => e.id) }), null, 2)}\n`, "utf8");
+            });
+        },
+    };
+}
+/**
+ * Turn a case id into a safe relative path.
+ *
+ * Ids are already path-shaped (`weather/nyc`), and `defineEval` rejects `..` in
+ * case ids, but the artifact writer sanitizes anyway so a hand-built result can
+ * never escape the run directory.
+ */
+export function sanitizeCaseId(id) {
+    return id
+        .split("/")
+        .map((segment) => segment.replace(/[^A-Za-z0-9._-]/g, "_"))
+        .filter((segment) => segment !== "" && segment !== "." && segment !== "..")
+        .join("/");
+}
+/** Fan one lifecycle call out to several reporters, isolating failures. */
+export function combineReporters(reporters, onError) {
+    const forEach = (run) => __awaiter(this, void 0, void 0, function* () {
+        for (const reporter of reporters) {
+            try {
+                yield run(reporter);
+            }
+            catch (error) {
+                // A broken destination must not fail the eval run itself.
+                onError === null || onError === void 0 ? void 0 : onError(error);
+            }
+        }
+    });
+    return {
+        onRunStart: (evals, target) => forEach((r) => { var _a; return (_a = r.onRunStart) === null || _a === void 0 ? void 0 : _a.call(r, evals, target); }),
+        onEvalComplete: (result) => forEach((r) => { var _a; return (_a = r.onEvalComplete) === null || _a === void 0 ? void 0 : _a.call(r, result); }),
+        onRunComplete: (summary) => forEach((r) => { var _a; return (_a = r.onRunComplete) === null || _a === void 0 ? void 0 : _a.call(r, summary); }),
+    };
+}

package/dist/evals/results.d.ts

@@ -0,0 +1,103 @@
+/**
+ * Result and reporter types shared by the runner, the CLI, and reporters.
+ *
+ * Pure types only: reporters that touch the filesystem live in
+ * `@cursor/july/evals/reporters`, so the authoring surface stays importable
+ * from anywhere.
+ */
+import type { EvalAssertionResult, EvalVerdict } from "./assertions.js";
+/** One finished eval case. */
+export interface EvalRunResult {
+    id: string;
+    path: string;
+    description?: string;
+    /**
+     * No hard gate failed. Unchanged from before severity existed: a soft
+     * threshold miss (`verdict: "scored"`) leaves this `true` and is fatal only
+     * under `--strict`.
+     */
+    ok: boolean;
+    /** Full grade, including `scored` and `skipped`. */
+    verdict?: EvalVerdict;
+    /** Reason passed to `t.skip(...)`. */
+    skipReason?: string;
+    assertions: EvalAssertionResult[];
+    sessionId?: string;
+    /** Messages passed to `t.send` (eval inputs). */
+    inputs: string[];
+    finalText?: string;
+    tools?: string[];
+    /** Tool calls with args/results from the trajectory. */
+    toolCalls?: Array<{
+        toolName: string;
+        args?: unknown;
+        output?: unknown;
+        isError: boolean;
+    }>;
+    error?: string;
+    logs: string[];
+    /** Structured scores from `t.metric(...)`. */
+    metrics: Record<string, string | number>;
+    /** Author-supplied metadata from `defineEval({ metadata })`. */
+    metadata?: Record<string, unknown>;
+    tags?: string[];
+    durationMs: number;
+}
+/** Aggregate outcome of one `eval` invocation. */
+export interface EvalRunSummary {
+    total: number;
+    passed: number;
+    failed: number;
+    /** Cases that only missed a soft threshold. */
+    scored: number;
+    skipped: number;
+    /** Whether soft threshold misses were treated as failures. */
+    strict: boolean;
+    startedAt: string;
+    finishedAt: string;
+    durationMs: number;
+    results: readonly EvalRunResult[];
+}
+/** One discovered case, as handed to {@link EvalReporter.onRunStart}. */
+export interface EvalReporterEvalInfo {
+    id: string;
+    fileId: string;
+    description?: string;
+    tags?: string[];
+}
+/** Where the run executed. */
+export interface EvalReporterTarget {
+    baseUrl: string;
+    /** `local` boots an ephemeral server; the others target a running one. */
+    mode: "local" | "url" | "prod";
+}
+/**
+ * A destination for eval results.
+ *
+ * The runner grades everything itself; a reporter only ships results out. All
+ * three hooks are optional and may return a promise for async work such as an
+ * upload or a file write.
+ *
+ * ```ts
+ * // evals/evals.config.ts
+ * export default defineEvalConfig({
+ *   maxConcurrency: 20,
+ *   reporters: [JUnit({ filePath: ".agent-serve/junit.xml" })],
+ * });
+ * ```
+ */
+export interface EvalReporter {
+    /** Fires once before any case runs. */
+    onRunStart?(evals: readonly EvalReporterEvalInfo[], target: EvalReporterTarget): void | Promise<void>;
+    /** Fires after each observed case, with its assertions and verdict. */
+    onEvalComplete?(result: EvalRunResult): void | Promise<void>;
+    /** Fires once with the aggregated summary. */
+    onRunComplete?(summary: EvalRunSummary): void | Promise<void>;
+}
+/** Tally verdicts for {@link EvalRunSummary}. */
+export declare function summarizeEvalResults(results: readonly EvalRunResult[], options: {
+    strict: boolean;
+    startedAt: string;
+    finishedAt: string;
+}): EvalRunSummary;
+//# sourceMappingURL=results.d.ts.map

package/dist/evals/results.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"results.d.ts","sourceRoot":"","sources":["../../src/evals/results.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,mBAAmB,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAExE,8BAA8B;AAC9B,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,EAAE,EAAE,OAAO,CAAC;IACZ,oDAAoD;IACpD,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,sCAAsC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,mBAAmB,EAAE,CAAC;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,wDAAwD;IACxD,SAAS,CAAC,EAAE,KAAK,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,OAAO,CAAC;QACf,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,OAAO,CAAC;KAClB,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,8CAA8C;IAC9C,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACzC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,kDAAkD;AAClD,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,SAAS,aAAa,EAAE,CAAC;CACnC;AAED,yEAAyE;AACzE,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,8BAA8B;AAC9B,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,0EAA0E;IAC1E,IAAI,EAAE,OAAO,GAAG,KAAK,GAAG,MAAM,CAAC;CAChC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,YAAY;IAC3B,uCAAuC;IACvC,UAAU,CAAC,CACT,KAAK,EAAE,SAAS,oBAAoB,EAAE,EACtC,MAAM,EAAE,kBAAkB,GACzB,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,uEAAuE;IACvE,cAAc,CAAC,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7D,8CAA8C;IAC9C,aAAa,CAAC,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/D;AAED,iDAAiD;AACjD,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,SAAS,aAAa,EAAE,EACjC,OAAO,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAClE,cAAc,CAqBhB"}

package/dist/evals/results.js

@@ -0,0 +1,28 @@
+/**
+ * Result and reporter types shared by the runner, the CLI, and reporters.
+ *
+ * Pure types only: reporters that touch the filesystem live in
+ * `@cursor/july/evals/reporters`, so the authoring surface stays importable
+ * from anywhere.
+ */
+/** Tally verdicts for {@link EvalRunSummary}. */
+export function summarizeEvalResults(results, options) {
+    const verdictOf = (result) => { var _a; return (_a = result.verdict) !== null && _a !== void 0 ? _a : (result.ok ? "passed" : "failed"); };
+    const counts = { passed: 0, failed: 0, scored: 0, skipped: 0 };
+    for (const result of results) {
+        counts[verdictOf(result)]++;
+    }
+    return {
+        total: results.length,
+        passed: counts.passed,
+        failed: counts.failed,
+        scored: counts.scored,
+        skipped: counts.skipped,
+        strict: options.strict,
+        startedAt: options.startedAt,
+        finishedAt: options.finishedAt,
+        durationMs: new Date(options.finishedAt).getTime() -
+            new Date(options.startedAt).getTime(),
+        results,
+    };
+}

package/dist/evals/run-facts.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Typed facts derived from a captured session event stream.
+ *
+ * Assertions never walk raw events: they read this normalized view, which
+ * tracks each tool call's lifecycle (requested → result / approval decision),
+ * subagent delegations, and unanswered human-in-the-loop (HITL) approvals.
+ */
+import type { SessionEvent } from "../types.js";
+import type { EvalToolCallStatus } from "./matchers.js";
+/** One tool call, in request order, with its resolved lifecycle state. */
+export interface EvalToolCallFact {
+    callId: string;
+    toolName: string;
+    /** Arguments from `actions.requested` (absent for result-only calls). */
+    input?: unknown;
+    /** Payload from `action.result`. */
+    output?: unknown;
+    status: EvalToolCallStatus;
+    /** Approval outcome when the tool required a human decision. */
+    approvalDecision?: "approve" | "deny";
+    /** Set when the call ran inside a subagent. */
+    parentCallId?: string;
+    turnId?: string;
+    /** 0-based position in request order across the whole run. */
+    index: number;
+}
+/** One subagent delegation (`task`). */
+export interface EvalSubagentFact {
+    callId: string;
+    name?: string;
+    description?: string;
+    /** The delegating `task` tool call's result payload, once it settled. */
+    output?: unknown;
+    status: "completed" | "pending";
+    turnId?: string;
+    index: number;
+}
+/** One human approval request raised by a `needsApproval` tool. */
+export interface EvalInputRequestFact {
+    callId: string;
+    toolName: string;
+    args?: unknown;
+    resolved: boolean;
+    decision?: "approve" | "deny";
+}
+export interface EvalRunFacts {
+    /** Every tool call in request order, including nested subagent calls. */
+    toolCalls: EvalToolCallFact[];
+    subagents: EvalSubagentFact[];
+    inputRequests: EvalInputRequestFact[];
+    /** Approval requests still awaiting a human decision. */
+    pendingInputRequests: EvalInputRequestFact[];
+    /** True when the run stopped on an unanswered approval request. */
+    parked: boolean;
+    turns: number;
+    turnsFailed: number;
+    /** First failure message from `turn.failed` / `session.failed`. */
+    failureMessage?: string;
+    /** No failed turn and at least one turn. */
+    ok: boolean;
+    /** Every finished assistant message, in order (excludes subagent text). */
+    assistantMessages: string[];
+    /** All assistant text joined with newlines. */
+    assistantText: string;
+    /** Tool calls that reported `isError`. */
+    failedToolCalls: EvalToolCallFact[];
+}
+/**
+ * Collapse a session event list into {@link EvalRunFacts}.
+ *
+ * Tool calls are keyed by `callId`, so a result event updates the call that
+ * `actions.requested` created rather than appending a duplicate. Result-only
+ * calls (no matching request) are still recorded, which keeps assertions honest
+ * against transports that drop the request event.
+ */
+export declare function deriveRunFacts(events: readonly SessionEvent[]): EvalRunFacts;
+//# sourceMappingURL=run-facts.d.ts.map

package/dist/evals/run-facts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-facts.d.ts","sourceRoot":"","sources":["../../src/evals/run-facts.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAExD,0EAA0E;AAC1E,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,yEAAyE;IACzE,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,oCAAoC;IACpC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,gEAAgE;IAChE,gBAAgB,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;IACtC,+CAA+C;IAC/C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wCAAwC;AACxC,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yEAAyE;IACzE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,WAAW,GAAG,SAAS,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,mEAAmE;AACnE,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,QAAQ,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,YAAY;IAC3B,yEAAyE;IACzE,SAAS,EAAE,gBAAgB,EAAE,CAAC;IAC9B,SAAS,EAAE,gBAAgB,EAAE,CAAC;IAC9B,aAAa,EAAE,oBAAoB,EAAE,CAAC;IACtC,yDAAyD;IACzD,oBAAoB,EAAE,oBAAoB,EAAE,CAAC;IAC7C,mEAAmE;IACnE,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,mEAAmE;IACnE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,4CAA4C;IAC5C,EAAE,EAAE,OAAO,CAAC;IACZ,2EAA2E;IAC3E,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,+CAA+C;IAC/C,aAAa,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,eAAe,EAAE,gBAAgB,EAAE,CAAC;CACrC;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE,GAAG,YAAY,CAgM5E"}

package/dist/evals/run-facts.js

@@ -0,0 +1,187 @@
+/**
+ * Typed facts derived from a captured session event stream.
+ *
+ * Assertions never walk raw events: they read this normalized view, which
+ * tracks each tool call's lifecycle (requested → result / approval decision),
+ * subagent delegations, and unanswered human-in-the-loop (HITL) approvals.
+ */
+/**
+ * Collapse a session event list into {@link EvalRunFacts}.
+ *
+ * Tool calls are keyed by `callId`, so a result event updates the call that
+ * `actions.requested` created rather than appending a duplicate. Result-only
+ * calls (no matching request) are still recorded, which keeps assertions honest
+ * against transports that drop the request event.
+ */
+export function deriveRunFacts(events) {
+    var _a, _b, _c, _d;
+    const toolCalls = [];
+    const byCallId = new Map();
+    const subagents = [];
+    const subagentsByCallId = new Map();
+    const inputRequests = [];
+    const inputRequestsByCallId = new Map();
+    const assistantMessages = [];
+    const turnIds = new Set();
+    const failedTurnIds = new Set();
+    let failureMessage;
+    let sessionFailed = false;
+    const ensureCall = (callId, toolName, turnId, parentCallId) => {
+        const existing = byCallId.get(callId);
+        if (existing !== undefined) {
+            return existing;
+        }
+        const fact = {
+            callId,
+            toolName,
+            status: "pending",
+            parentCallId,
+            turnId,
+            index: toolCalls.length,
+        };
+        toolCalls.push(fact);
+        byCallId.set(callId, fact);
+        return fact;
+    };
+    for (const event of events) {
+        if (event.turnId !== undefined) {
+            turnIds.add(event.turnId);
+        }
+        switch (event.type) {
+            case "actions.requested": {
+                for (const call of event.data.calls) {
+                    const fact = ensureCall(call.callId, call.toolName, event.turnId, (_a = call.parentCallId) !== null && _a !== void 0 ? _a : event.data.parentCallId);
+                    fact.input = call.args;
+                }
+                break;
+            }
+            case "action.result": {
+                const fact = ensureCall(event.data.callId, event.data.toolName, event.turnId, event.data.parentCallId);
+                fact.toolName = event.data.toolName;
+                fact.output = event.data.output;
+                // A denied approval is a distinct outcome from a tool that ran and
+                // errored, so do not let the synthetic error result overwrite it.
+                if (fact.status !== "rejected") {
+                    fact.status = event.data.isError ? "failed" : "completed";
+                }
+                const subagent = subagentsByCallId.get(event.data.callId);
+                if (subagent !== undefined) {
+                    subagent.output = event.data.output;
+                    subagent.status = "completed";
+                }
+                break;
+            }
+            case "action.approval_requested": {
+                const request = {
+                    callId: event.data.callId,
+                    toolName: event.data.toolName,
+                    args: event.data.args,
+                    resolved: false,
+                };
+                inputRequests.push(request);
+                inputRequestsByCallId.set(event.data.callId, request);
+                break;
+            }
+            case "action.approval_resolved": {
+                const request = inputRequestsByCallId.get(event.data.callId);
+                if (request !== undefined) {
+                    request.resolved = true;
+                    request.decision = event.data.decision;
+                }
+                const fact = ensureCall(event.data.callId, event.data.toolName, event.turnId, event.data.parentCallId);
+                fact.approvalDecision = event.data.decision;
+                if (event.data.decision === "deny") {
+                    fact.status = "rejected";
+                }
+                break;
+            }
+            case "subagent.called": {
+                const existing = subagentsByCallId.get(event.data.callId);
+                if (existing === undefined) {
+                    const fact = {
+                        callId: event.data.callId,
+                        name: event.data.name,
+                        description: event.data.description,
+                        status: "pending",
+                        turnId: event.turnId,
+                        index: subagents.length,
+                    };
+                    subagents.push(fact);
+                    subagentsByCallId.set(event.data.callId, fact);
+                }
+                else {
+                    existing.name = (_b = event.data.name) !== null && _b !== void 0 ? _b : existing.name;
+                    existing.description = (_c = event.data.description) !== null && _c !== void 0 ? _c : existing.description;
+                }
+                break;
+            }
+            case "subagent.completed": {
+                const fact = subagentsByCallId.get(event.data.callId);
+                if (fact === undefined) {
+                    const created = {
+                        callId: event.data.callId,
+                        name: event.data.name,
+                        status: "completed",
+                        turnId: event.turnId,
+                        index: subagents.length,
+                    };
+                    subagents.push(created);
+                    subagentsByCallId.set(event.data.callId, created);
+                }
+                else {
+                    fact.status = "completed";
+                    fact.name = (_d = fact.name) !== null && _d !== void 0 ? _d : event.data.name;
+                }
+                break;
+            }
+            case "message.completed": {
+                // Nested subagent chatter is not part of the agent's reply.
+                if (event.data.parentCallId === undefined && event.data.text !== "") {
+                    assistantMessages.push(event.data.text);
+                }
+                break;
+            }
+            case "turn.completed": {
+                if (event.data.result !== undefined &&
+                    event.data.result !== "" &&
+                    !assistantMessages.includes(event.data.result)) {
+                    assistantMessages.push(event.data.result);
+                }
+                break;
+            }
+            case "turn.failed":
+            case "session.failed": {
+                if (event.turnId !== undefined) {
+                    failedTurnIds.add(event.turnId);
+                }
+                else {
+                    // A session-scoped failure carries no turn id, so it would otherwise
+                    // leave `turnsFailed` at 0 and read as a healthy run.
+                    sessionFailed = true;
+                }
+                failureMessage = failureMessage !== null && failureMessage !== void 0 ? failureMessage : event.data.message;
+                break;
+            }
+            default:
+                break;
+        }
+    }
+    const pendingInputRequests = inputRequests.filter((r) => !r.resolved);
+    // `turn.*` events all carry a turnId; session-scoped events do not, so the
+    // turn count is the number of distinct turn ids seen.
+    const turns = turnIds.size;
+    return {
+        toolCalls,
+        subagents,
+        inputRequests,
+        pendingInputRequests,
+        parked: pendingInputRequests.length > 0,
+        turns,
+        turnsFailed: failedTurnIds.size,
+        failureMessage,
+        ok: failedTurnIds.size === 0 && !sessionFailed && turns > 0,
+        assistantMessages,
+        assistantText: assistantMessages.join("\n"),
+        failedToolCalls: toolCalls.filter((c) => c.status === "failed"),
+    };
+}