@cursor/july 0.1.16 → 0.1.17

package/src/evals/results.ts

@@ -0,0 +1,132 @@
+/**
+ * Result and reporter types shared by the runner, the CLI, and reporters.
+ *
+ * Pure types only: reporters that touch the filesystem live in
+ * `@cursor/july/evals/reporters`, so the authoring surface stays importable
+ * from anywhere.
+ */
+
+import type { EvalAssertionResult, EvalVerdict } from "./assertions.js";
+
+/** One finished eval case. */
+export interface EvalRunResult {
+  id: string;
+  path: string;
+  description?: string;
+  /**
+   * No hard gate failed. Unchanged from before severity existed: a soft
+   * threshold miss (`verdict: "scored"`) leaves this `true` and is fatal only
+   * under `--strict`.
+   */
+  ok: boolean;
+  /** Full grade, including `scored` and `skipped`. */
+  verdict?: EvalVerdict;
+  /** Reason passed to `t.skip(...)`. */
+  skipReason?: string;
+  assertions: EvalAssertionResult[];
+  sessionId?: string;
+  /** Messages passed to `t.send` (eval inputs). */
+  inputs: string[];
+  finalText?: string;
+  tools?: string[];
+  /** Tool calls with args/results from the trajectory. */
+  toolCalls?: Array<{
+    toolName: string;
+    args?: unknown;
+    output?: unknown;
+    isError: boolean;
+  }>;
+  error?: string;
+  logs: string[];
+  /** Structured scores from `t.metric(...)`. */
+  metrics: Record<string, string | number>;
+  /** Author-supplied metadata from `defineEval({ metadata })`. */
+  metadata?: Record<string, unknown>;
+  tags?: string[];
+  durationMs: number;
+}
+
+/** Aggregate outcome of one `eval` invocation. */
+export interface EvalRunSummary {
+  total: number;
+  passed: number;
+  failed: number;
+  /** Cases that only missed a soft threshold. */
+  scored: number;
+  skipped: number;
+  /** Whether soft threshold misses were treated as failures. */
+  strict: boolean;
+  startedAt: string;
+  finishedAt: string;
+  durationMs: number;
+  results: readonly EvalRunResult[];
+}
+
+/** One discovered case, as handed to {@link EvalReporter.onRunStart}. */
+export interface EvalReporterEvalInfo {
+  id: string;
+  fileId: string;
+  description?: string;
+  tags?: string[];
+}
+
+/** Where the run executed. */
+export interface EvalReporterTarget {
+  baseUrl: string;
+  /** `local` boots an ephemeral server; the others target a running one. */
+  mode: "local" | "url" | "prod";
+}
+
+/**
+ * A destination for eval results.
+ *
+ * The runner grades everything itself; a reporter only ships results out. All
+ * three hooks are optional and may return a promise for async work such as an
+ * upload or a file write.
+ *
+ * ```ts
+ * // evals/evals.config.ts
+ * export default defineEvalConfig({
+ *   maxConcurrency: 20,
+ *   reporters: [JUnit({ filePath: ".agent-serve/junit.xml" })],
+ * });
+ * ```
+ */
+export interface EvalReporter {
+  /** Fires once before any case runs. */
+  onRunStart?(
+    evals: readonly EvalReporterEvalInfo[],
+    target: EvalReporterTarget
+  ): void | Promise<void>;
+  /** Fires after each observed case, with its assertions and verdict. */
+  onEvalComplete?(result: EvalRunResult): void | Promise<void>;
+  /** Fires once with the aggregated summary. */
+  onRunComplete?(summary: EvalRunSummary): void | Promise<void>;
+}
+
+/** Tally verdicts for {@link EvalRunSummary}. */
+export function summarizeEvalResults(
+  results: readonly EvalRunResult[],
+  options: { strict: boolean; startedAt: string; finishedAt: string }
+): EvalRunSummary {
+  const verdictOf = (result: EvalRunResult): EvalVerdict =>
+    result.verdict ?? (result.ok ? "passed" : "failed");
+  const counts = { passed: 0, failed: 0, scored: 0, skipped: 0 };
+  for (const result of results) {
+    counts[verdictOf(result)]++;
+  }
+  return {
+    total: results.length,
+    passed: counts.passed,
+    failed: counts.failed,
+    scored: counts.scored,
+    skipped: counts.skipped,
+    strict: options.strict,
+    startedAt: options.startedAt,
+    finishedAt: options.finishedAt,
+    durationMs:
+      new Date(options.finishedAt).getTime() -
+      new Date(options.startedAt).getTime(),
+    results,
+  };
+}

package/src/evals/run-facts.ts

@@ -0,0 +1,274 @@
+/**
+ * Typed facts derived from a captured session event stream.
+ *
+ * Assertions never walk raw events: they read this normalized view, which
+ * tracks each tool call's lifecycle (requested → result / approval decision),
+ * subagent delegations, and unanswered human-in-the-loop (HITL) approvals.
+ */
+
+import type { SessionEvent } from "../types.js";
+import type { EvalToolCallStatus } from "./matchers.js";
+
+/** One tool call, in request order, with its resolved lifecycle state. */
+export interface EvalToolCallFact {
+  callId: string;
+  toolName: string;
+  /** Arguments from `actions.requested` (absent for result-only calls). */
+  input?: unknown;
+  /** Payload from `action.result`. */
+  output?: unknown;
+  status: EvalToolCallStatus;
+  /** Approval outcome when the tool required a human decision. */
+  approvalDecision?: "approve" | "deny";
+  /** Set when the call ran inside a subagent. */
+  parentCallId?: string;
+  turnId?: string;
+  /** 0-based position in request order across the whole run. */
+  index: number;
+}
+
+/** One subagent delegation (`task`). */
+export interface EvalSubagentFact {
+  callId: string;
+  name?: string;
+  description?: string;
+  /** The delegating `task` tool call's result payload, once it settled. */
+  output?: unknown;
+  status: "completed" | "pending";
+  turnId?: string;
+  index: number;
+}
+
+/** One human approval request raised by a `needsApproval` tool. */
+export interface EvalInputRequestFact {
+  callId: string;
+  toolName: string;
+  args?: unknown;
+  resolved: boolean;
+  decision?: "approve" | "deny";
+}
+
+export interface EvalRunFacts {
+  /** Every tool call in request order, including nested subagent calls. */
+  toolCalls: EvalToolCallFact[];
+  subagents: EvalSubagentFact[];
+  inputRequests: EvalInputRequestFact[];
+  /** Approval requests still awaiting a human decision. */
+  pendingInputRequests: EvalInputRequestFact[];
+  /** True when the run stopped on an unanswered approval request. */
+  parked: boolean;
+  turns: number;
+  turnsFailed: number;
+  /** First failure message from `turn.failed` / `session.failed`. */
+  failureMessage?: string;
+  /** No failed turn and at least one turn. */
+  ok: boolean;
+  /** Every finished assistant message, in order (excludes subagent text). */
+  assistantMessages: string[];
+  /** All assistant text joined with newlines. */
+  assistantText: string;
+  /** Tool calls that reported `isError`. */
+  failedToolCalls: EvalToolCallFact[];
+}
+
+/**
+ * Collapse a session event list into {@link EvalRunFacts}.
+ *
+ * Tool calls are keyed by `callId`, so a result event updates the call that
+ * `actions.requested` created rather than appending a duplicate. Result-only
+ * calls (no matching request) are still recorded, which keeps assertions honest
+ * against transports that drop the request event.
+ */
+export function deriveRunFacts(events: readonly SessionEvent[]): EvalRunFacts {
+  const toolCalls: EvalToolCallFact[] = [];
+  const byCallId = new Map<string, EvalToolCallFact>();
+  const subagents: EvalSubagentFact[] = [];
+  const subagentsByCallId = new Map<string, EvalSubagentFact>();
+  const inputRequests: EvalInputRequestFact[] = [];
+  const inputRequestsByCallId = new Map<string, EvalInputRequestFact>();
+  const assistantMessages: string[] = [];
+  const turnIds = new Set<string>();
+  const failedTurnIds = new Set<string>();
+  let failureMessage: string | undefined;
+  let sessionFailed = false;
+
+  const ensureCall = (
+    callId: string,
+    toolName: string,
+    turnId: string | undefined,
+    parentCallId: string | undefined
+  ): EvalToolCallFact => {
+    const existing = byCallId.get(callId);
+    if (existing !== undefined) {
+      return existing;
+    }
+    const fact: EvalToolCallFact = {
+      callId,
+      toolName,
+      status: "pending",
+      parentCallId,
+      turnId,
+      index: toolCalls.length,
+    };
+    toolCalls.push(fact);
+    byCallId.set(callId, fact);
+    return fact;
+  };
+
+  for (const event of events) {
+    if (event.turnId !== undefined) {
+      turnIds.add(event.turnId);
+    }
+    switch (event.type) {
+      case "actions.requested": {
+        for (const call of event.data.calls) {
+          const fact = ensureCall(
+            call.callId,
+            call.toolName,
+            event.turnId,
+            call.parentCallId ?? event.data.parentCallId
+          );
+          fact.input = call.args;
+        }
+        break;
+      }
+      case "action.result": {
+        const fact = ensureCall(
+          event.data.callId,
+          event.data.toolName,
+          event.turnId,
+          event.data.parentCallId
+        );
+        fact.toolName = event.data.toolName;
+        fact.output = event.data.output;
+        // A denied approval is a distinct outcome from a tool that ran and
+        // errored, so do not let the synthetic error result overwrite it.
+        if (fact.status !== "rejected") {
+          fact.status = event.data.isError ? "failed" : "completed";
+        }
+        const subagent = subagentsByCallId.get(event.data.callId);
+        if (subagent !== undefined) {
+          subagent.output = event.data.output;
+          subagent.status = "completed";
+        }
+        break;
+      }
+      case "action.approval_requested": {
+        const request: EvalInputRequestFact = {
+          callId: event.data.callId,
+          toolName: event.data.toolName,
+          args: event.data.args,
+          resolved: false,
+        };
+        inputRequests.push(request);
+        inputRequestsByCallId.set(event.data.callId, request);
+        break;
+      }
+      case "action.approval_resolved": {
+        const request = inputRequestsByCallId.get(event.data.callId);
+        if (request !== undefined) {
+          request.resolved = true;
+          request.decision = event.data.decision;
+        }
+        const fact = ensureCall(
+          event.data.callId,
+          event.data.toolName,
+          event.turnId,
+          event.data.parentCallId
+        );
+        fact.approvalDecision = event.data.decision;
+        if (event.data.decision === "deny") {
+          fact.status = "rejected";
+        }
+        break;
+      }
+      case "subagent.called": {
+        const existing = subagentsByCallId.get(event.data.callId);
+        if (existing === undefined) {
+          const fact: EvalSubagentFact = {
+            callId: event.data.callId,
+            name: event.data.name,
+            description: event.data.description,
+            status: "pending",
+            turnId: event.turnId,
+            index: subagents.length,
+          };
+          subagents.push(fact);
+          subagentsByCallId.set(event.data.callId, fact);
+        } else {
+          existing.name = event.data.name ?? existing.name;
+          existing.description = event.data.description ?? existing.description;
+        }
+        break;
+      }
+      case "subagent.completed": {
+        const fact = subagentsByCallId.get(event.data.callId);
+        if (fact === undefined) {
+          const created: EvalSubagentFact = {
+            callId: event.data.callId,
+            name: event.data.name,
+            status: "completed",
+            turnId: event.turnId,
+            index: subagents.length,
+          };
+          subagents.push(created);
+          subagentsByCallId.set(event.data.callId, created);
+        } else {
+          fact.status = "completed";
+          fact.name = fact.name ?? event.data.name;
+        }
+        break;
+      }
+      case "message.completed": {
+        // Nested subagent chatter is not part of the agent's reply.
+        if (event.data.parentCallId === undefined && event.data.text !== "") {
+          assistantMessages.push(event.data.text);
+        }
+        break;
+      }
+      case "turn.completed": {
+        if (
+          event.data.result !== undefined &&
+          event.data.result !== "" &&
+          !assistantMessages.includes(event.data.result)
+        ) {
+          assistantMessages.push(event.data.result);
+        }
+        break;
+      }
+      case "turn.failed":
+      case "session.failed": {
+        if (event.turnId !== undefined) {
+          failedTurnIds.add(event.turnId);
+        } else {
+          // A session-scoped failure carries no turn id, so it would otherwise
+          // leave `turnsFailed` at 0 and read as a healthy run.
+          sessionFailed = true;
+        }
+        failureMessage = failureMessage ?? event.data.message;
+        break;
+      }
+      default:
+        break;
+    }
+  }
+
+  const pendingInputRequests = inputRequests.filter((r) => !r.resolved);
+  // `turn.*` events all carry a turnId; session-scoped events do not, so the
+  // turn count is the number of distinct turn ids seen.
+  const turns = turnIds.size;
+  return {
+    toolCalls,
+    subagents,
+    inputRequests,
+    pendingInputRequests,
+    parked: pendingInputRequests.length > 0,
+    turns,
+    turnsFailed: failedTurnIds.size,
+    failureMessage,
+    ok: failedTurnIds.size === 0 && !sessionFailed && turns > 0,
+    assistantMessages,
+    assistantText: assistantMessages.join("\n"),
+    failedToolCalls: toolCalls.filter((c) => c.status === "failed"),
+  };
+}