@cursor/bdk 0.2.3 → 0.2.5

package/src/evals/run-facts.ts

@@ -1,306 +0,0 @@
-/**
- * Typed facts derived from a captured session event stream.
- *
- * Assertions never walk raw events: they read this normalized view, which
- * tracks each tool call's lifecycle (requested → result / approval decision),
- * subagent delegations, and unanswered human-in-the-loop (HITL) approvals.
- */
-
-import type { SessionEvent } from "../types.js";
-import type { EvalToolCallStatus } from "./matchers.js";
-
-/** One tool call, in request order, with its resolved lifecycle state. */
-export interface EvalToolCallFact {
-  callId: string;
-  toolName: string;
-  /** Arguments from `actions.requested` (absent for result-only calls). */
-  input?: unknown;
-  /** Payload from `action.result`. */
-  output?: unknown;
-  status: EvalToolCallStatus;
-  /** Approval outcome when the tool required a human decision. */
-  approvalDecision?: "approve" | "deny";
-  /** Set when the call ran inside a subagent. */
-  parentCallId?: string;
-  turnId?: string;
-  /** 0-based position in request order across the whole run. */
-  index: number;
-}
-
-/** One subagent delegation (`task`). */
-export interface EvalSubagentFact {
-  callId: string;
-  name?: string;
-  description?: string;
-  /** The delegating `task` tool call's result payload, once it settled. */
-  output?: unknown;
-  status: "completed" | "pending";
-  turnId?: string;
-  index: number;
-}
-
-/** One human approval request raised by a `needsApproval` tool. */
-export interface EvalInputRequestFact {
-  callId: string;
-  toolName: string;
-  args?: unknown;
-  resolved: boolean;
-  decision?: "approve" | "deny";
-}
-
-export interface EvalRunFacts {
-  /** Every tool call in request order, including nested subagent calls. */
-  toolCalls: EvalToolCallFact[];
-  subagents: EvalSubagentFact[];
-  inputRequests: EvalInputRequestFact[];
-  /** Approval requests still awaiting a human decision. */
-  pendingInputRequests: EvalInputRequestFact[];
-  /** True when the run stopped on an unanswered approval request. */
-  parked: boolean;
-  turns: number;
-  turnsFailed: number;
-  /** First failure message from `turn.failed` / `session.failed`. */
-  failureMessage?: string;
-  /** No failed turn and at least one turn. */
-  ok: boolean;
-  /** Every finished assistant message, in order (excludes subagent text). */
-  assistantMessages: string[];
-  /** All assistant text joined with newlines. */
-  assistantText: string;
-  /** Tool calls that reported `isError`. */
-  failedToolCalls: EvalToolCallFact[];
-}
-
-/**
- * Collapse a session event list into {@link EvalRunFacts}.
- *
- * Tool calls are keyed by `callId`, so a result event updates the call that
- * `actions.requested` created rather than appending a duplicate. Result-only
- * calls (no matching request) are still recorded, which keeps assertions honest
- * against transports that drop the request event.
- */
-export function deriveRunFacts(events: readonly SessionEvent[]): EvalRunFacts {
-  const toolCalls: EvalToolCallFact[] = [];
-  const byCallId = new Map<string, EvalToolCallFact>();
-  const subagents: EvalSubagentFact[] = [];
-  const subagentsByCallId = new Map<string, EvalSubagentFact>();
-  const inputRequests: EvalInputRequestFact[] = [];
-  const inputRequestsByCallId = new Map<string, EvalInputRequestFact>();
-  const assistantMessages: string[] = [];
-  const turnIds = new Set<string>();
-  const failedTurnIds = new Set<string>();
-  let failureMessage: string | undefined;
-  let sessionFailed = false;
-
-  const ensureCall = (
-    callId: string,
-    toolName: string,
-    turnId: string | undefined,
-    parentCallId: string | undefined,
-  ): EvalToolCallFact => {
-    const existing = byCallId.get(callId);
-    if (existing !== undefined) {
-      return existing;
-    }
-    const fact: EvalToolCallFact = {
-      callId,
-      toolName,
-      status: "pending",
-      parentCallId,
-      turnId,
-      index: toolCalls.length,
-    };
-    toolCalls.push(fact);
-    byCallId.set(callId, fact);
-    return fact;
-  };
-
-  for (const event of events) {
-    if (event.turnId !== undefined) {
-      turnIds.add(event.turnId);
-    }
-    switch (event.type) {
-      case "actions.requested": {
-        for (const call of event.data.calls) {
-          const fact = ensureCall(
-            call.callId,
-            call.toolName,
-            event.turnId,
-            call.parentCallId ?? event.data.parentCallId,
-          );
-          fact.input = call.args;
-        }
-        break;
-      }
-      case "action.result": {
-        const fact = ensureCall(
-          event.data.callId,
-          event.data.toolName,
-          event.turnId,
-          event.data.parentCallId,
-        );
-        fact.toolName = event.data.toolName;
-        fact.output = event.data.output;
-        // A denied approval is a distinct outcome from a tool that ran and
-        // errored, so do not let the synthetic error result overwrite it.
-        if (fact.status !== "rejected") {
-          fact.status = event.data.isError ? "failed" : "completed";
-        }
-        const subagent = subagentsByCallId.get(event.data.callId);
-        if (subagent !== undefined) {
-          subagent.output = event.data.output;
-          subagent.status = "completed";
-        }
-        break;
-      }
-      case "action.approval_requested": {
-        const request: EvalInputRequestFact = {
-          callId: event.data.callId,
-          toolName: event.data.toolName,
-          args: event.data.args,
-          resolved: false,
-        };
-        inputRequests.push(request);
-        inputRequestsByCallId.set(event.data.callId, request);
-        break;
-      }
-      case "action.approval_resolved": {
-        const request = inputRequestsByCallId.get(event.data.callId);
-        if (request !== undefined) {
-          request.resolved = true;
-          request.decision = event.data.decision;
-        }
-        const fact = ensureCall(
-          event.data.callId,
-          event.data.toolName,
-          event.turnId,
-          event.data.parentCallId,
-        );
-        fact.approvalDecision = event.data.decision;
-        if (event.data.decision === "deny") {
-          fact.status = "rejected";
-        }
-        break;
-      }
-      case "action.auth_requested": {
-        const request: EvalInputRequestFact = {
-          callId: event.data.callId,
-          toolName: event.data.toolName,
-          args: {
-            connection: event.data.connection,
-            authUrl: event.data.authUrl,
-          },
-          resolved: false,
-        };
-        inputRequests.push(request);
-        inputRequestsByCallId.set(event.data.callId, request);
-        break;
-      }
-      case "action.auth_resolved": {
-        const request = inputRequestsByCallId.get(event.data.callId);
-        if (request !== undefined) {
-          request.resolved = true;
-          request.decision = event.data.decision;
-        }
-        const fact = ensureCall(
-          event.data.callId,
-          event.data.toolName,
-          event.turnId,
-          event.data.parentCallId,
-        );
-        fact.approvalDecision = event.data.decision;
-        if (event.data.decision === "deny") {
-          fact.status = "rejected";
-        }
-        break;
-      }
-      case "subagent.called": {
-        const existing = subagentsByCallId.get(event.data.callId);
-        if (existing === undefined) {
-          const fact: EvalSubagentFact = {
-            callId: event.data.callId,
-            name: event.data.name,
-            description: event.data.description,
-            status: "pending",
-            turnId: event.turnId,
-            index: subagents.length,
-          };
-          subagents.push(fact);
-          subagentsByCallId.set(event.data.callId, fact);
-        } else {
-          existing.name = event.data.name ?? existing.name;
-          existing.description = event.data.description ?? existing.description;
-        }
-        break;
-      }
-      case "subagent.completed": {
-        const fact = subagentsByCallId.get(event.data.callId);
-        if (fact === undefined) {
-          const created: EvalSubagentFact = {
-            callId: event.data.callId,
-            name: event.data.name,
-            status: "completed",
-            turnId: event.turnId,
-            index: subagents.length,
-          };
-          subagents.push(created);
-          subagentsByCallId.set(event.data.callId, created);
-        } else {
-          fact.status = "completed";
-          fact.name = fact.name ?? event.data.name;
-        }
-        break;
-      }
-      case "message.completed": {
-        // Nested subagent chatter is not part of the agent's reply.
-        if (event.data.parentCallId === undefined && event.data.text !== "") {
-          assistantMessages.push(event.data.text);
-        }
-        break;
-      }
-      case "turn.completed": {
-        if (
-          event.data.result !== undefined &&
-          event.data.result !== "" &&
-          !assistantMessages.includes(event.data.result)
-        ) {
-          assistantMessages.push(event.data.result);
-        }
-        break;
-      }
-      case "turn.failed":
-      case "session.failed": {
-        if (event.turnId !== undefined) {
-          failedTurnIds.add(event.turnId);
-        } else {
-          // A session-scoped failure carries no turn id, so it would otherwise
-          // leave `turnsFailed` at 0 and read as a healthy run.
-          sessionFailed = true;
-        }
-        failureMessage = failureMessage ?? event.data.message;
-        break;
-      }
-      default:
-        break;
-    }
-  }
-
-  const pendingInputRequests = inputRequests.filter(r => !r.resolved);
-  // `turn.*` events all carry a turnId; session-scoped events do not, so the
-  // turn count is the number of distinct turn ids seen.
-  const turns = turnIds.size;
-  return {
-    toolCalls,
-    subagents,
-    inputRequests,
-    pendingInputRequests,
-    parked: pendingInputRequests.length > 0,
-    turns,
-    turnsFailed: failedTurnIds.size,
-    failureMessage,
-    ok: failedTurnIds.size === 0 && !sessionFailed && turns > 0,
-    assistantMessages,
-    assistantText: assistantMessages.join("\n"),
-    failedToolCalls: toolCalls.filter(c => c.status === "failed"),
-  };
-}