@cursor/bdk 0.2.1

package/src/internal/evals-client.ts

@@ -0,0 +1,475 @@
+/**
+ * HTTP client for playground eval batches (`/v1/dev/evals/runs`).
+ *
+ * Used by `agent-sdk eval --prod` (or `--url`) so the batch runs on the serve
+ * process and is mirrored to `defineStorage` when an `evals` table is
+ * configured. Without those flags, eval keeps the in-process ephemeral path
+ * in `cli-ax.ts`.
+ */
+
+import type { EvalRunSnapshot } from "../evals.js";
+import { expandSummarySuiteSelection, isEvalSummaryCase, matchesEvalIdFilter } from "../evals.js";
+import type { EvalRunResult } from "./eval-runner.js";
+import { mergeRequestHeaders, type RequestAuth } from "./request-headers.js";
+
+const DEFAULT_POLL_MS = 1_000;
+
+export type RemoteEvalErrorCode =
+  | "no_matching_evals"
+  | "eval_run_in_progress"
+  | "eval_run_not_running"
+  | "unknown_eval_run"
+  | "start_failed"
+  | "poll_failed"
+  | "cancel_failed"
+  | "aborted";
+
+export class RemoteEvalError extends Error {
+  constructor(
+    message: string,
+    readonly status?: number,
+    readonly code?: RemoteEvalErrorCode,
+    readonly activeRunId?: string,
+  ) {
+    super(message);
+    this.name = "RemoteEvalError";
+  }
+}
+
+export interface RemoteEvalListItem {
+  id: string;
+  fileId: string;
+  description?: string;
+  tags?: string[];
+}
+
+export interface RunRemoteEvalsOptions extends RequestAuth {
+  baseUrl: string;
+  filterIds?: string[];
+  tags?: string[];
+  /** Per-case timeout override forwarded to the serve process. */
+  timeoutMs?: number;
+  /** Ask the serve harness to emit verbose case logs (server-side). */
+  verbose?: boolean;
+  fetchImpl?: typeof fetch;
+  /** Poll interval while the batch is `running`. */
+  pollMs?: number;
+  /** Abort the wait (not the remote batch). */
+  signal?: AbortSignal;
+  /** Called each time the polled snapshot changes. */
+  onSnapshot?: (snap: EvalRunSnapshot) => void;
+}
+
+/**
+ * Start a batch on the target serve via `POST /v1/dev/evals/runs`, poll until
+ * terminal, and return CLI-shaped case results.
+ */
+export async function runRemoteEvals(
+  options: RunRemoteEvalsOptions,
+): Promise<{ snapshot: EvalRunSnapshot; results: EvalRunResult[] }> {
+  const baseUrl = options.baseUrl.replace(/\/$/, "");
+  const fetchImpl = options.fetchImpl ?? fetch;
+  const auth: RequestAuth = {
+    bearerToken: options.bearerToken,
+    headers: options.headers,
+  };
+
+  const started = await startRemoteEvalRun({
+    baseUrl,
+    auth,
+    fetchImpl,
+    filterIds: options.filterIds,
+    tags: options.tags,
+    timeoutMs: options.timeoutMs,
+    verbose: options.verbose,
+    signal: options.signal,
+  });
+  options.onSnapshot?.(started);
+
+  const snapshot = await pollRemoteEvalRun({
+    baseUrl,
+    runId: started.runId,
+    auth,
+    fetchImpl,
+    pollMs: options.pollMs ?? DEFAULT_POLL_MS,
+    signal: options.signal,
+    onSnapshot: options.onSnapshot,
+  });
+
+  return { snapshot, results: snapshotToEvalRunResults(snapshot) };
+}
+
+/** List datapoints from `GET /v1/dev/evals` (same surface a remote run uses). */
+export async function listRemoteEvals(args: {
+  baseUrl: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  filterIds?: string[];
+  tags?: string[];
+  signal?: AbortSignal;
+}): Promise<RemoteEvalListItem[]> {
+  const baseUrl = args.baseUrl.replace(/\/$/, "");
+  const fetchImpl = args.fetchImpl ?? fetch;
+  let resp: Response;
+  try {
+    resp = await fetchImpl(`${baseUrl}/v1/dev/evals`, {
+      headers: mergeRequestHeaders(args.auth),
+      signal: args.signal,
+    });
+  } catch (error) {
+    throw new RemoteEvalError(
+      error instanceof Error ? error.message : "Could not list evals.",
+      undefined,
+      "start_failed",
+    );
+  }
+  if (!resp.ok) {
+    throw new RemoteEvalError(
+      `Could not list evals (HTTP ${resp.status}).`,
+      resp.status,
+      "start_failed",
+    );
+  }
+  const body = (await resp.json()) as {
+    evals?: Array<{
+      id?: unknown;
+      fileId?: unknown;
+      description?: unknown;
+      tags?: unknown;
+    }>;
+  };
+  const listed: RemoteEvalListItem[] = [];
+  for (const row of body.evals ?? []) {
+    if (typeof row.id !== "string" || row.id === "") {
+      continue;
+    }
+    if (typeof row.fileId !== "string" || row.fileId === "") {
+      continue;
+    }
+    const tags = Array.isArray(row.tags)
+      ? row.tags.filter((t): t is string => typeof t === "string")
+      : undefined;
+    listed.push({
+      id: row.id,
+      fileId: row.fileId,
+      ...(typeof row.description === "string" ? { description: row.description } : {}),
+      ...(tags === undefined || tags.length === 0 ? {} : { tags }),
+    });
+  }
+
+  let selected = listed;
+  if (args.filterIds !== undefined && args.filterIds.length > 0) {
+    selected = selected.filter(e => args.filterIds!.some(f => matchesEvalIdFilter(e.id, f)));
+  }
+  if (args.tags !== undefined && args.tags.length > 0) {
+    selected = selected.filter(e => (e.tags ?? []).some(t => args.tags!.includes(t)));
+  }
+  // Match server-side filterDiscoveredEvals: summary → whole file.
+  return expandSummarySuiteSelection(listed, selected, e => isEvalSummaryCase(e.tags));
+}
+
+export async function startRemoteEvalRun(args: {
+  baseUrl: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  filterIds?: string[];
+  tags?: string[];
+  timeoutMs?: number;
+  verbose?: boolean;
+  signal?: AbortSignal;
+}): Promise<EvalRunSnapshot> {
+  const baseUrl = args.baseUrl.replace(/\/$/, "");
+  const fetchImpl = args.fetchImpl ?? fetch;
+  const body: {
+    filterIds?: string[];
+    tags?: string[];
+    timeoutMs?: number;
+    verbose?: boolean;
+  } = {};
+  if (args.filterIds !== undefined && args.filterIds.length > 0) {
+    body.filterIds = args.filterIds;
+  }
+  if (args.tags !== undefined && args.tags.length > 0) {
+    body.tags = args.tags;
+  }
+  if (args.timeoutMs !== undefined) {
+    body.timeoutMs = args.timeoutMs;
+  }
+  if (args.verbose === true) {
+    body.verbose = true;
+  }
+
+  let resp: Response;
+  try {
+    resp = await fetchImpl(`${baseUrl}/v1/dev/evals/runs`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        ...mergeRequestHeaders(args.auth),
+      },
+      body: JSON.stringify(body),
+      signal: args.signal,
+    });
+  } catch (error) {
+    throw new RemoteEvalError(
+      error instanceof Error ? error.message : "Could not start eval run.",
+      undefined,
+      "start_failed",
+    );
+  }
+
+  const json = (await resp.json().catch(() => ({}))) as {
+    ok?: boolean;
+    error?: string;
+    message?: string;
+    activeRunId?: string;
+    runId?: string;
+  };
+
+  if (json.error === "no_matching_evals") {
+    throw new RemoteEvalError("No matching evals found.", 404, "no_matching_evals");
+  }
+  if (resp.status === 409 || json.error === "eval_run_in_progress") {
+    throw new RemoteEvalError(
+      json.activeRunId !== undefined && json.activeRunId !== ""
+        ? `An eval run is already in progress (${json.activeRunId}).`
+        : "An eval run is already in progress.",
+      409,
+      "eval_run_in_progress",
+      typeof json.activeRunId === "string" ? json.activeRunId : undefined,
+    );
+  }
+  if (!resp.ok) {
+    const detail =
+      typeof json.message === "string" && json.message !== ""
+        ? json.message
+        : typeof json.error === "string" && json.error !== ""
+          ? json.error
+          : `HTTP ${resp.status}`;
+    throw new RemoteEvalError(`Could not start eval run: ${detail}`, resp.status, "start_failed");
+  }
+  if (typeof json.runId !== "string" || json.runId === "") {
+    throw new RemoteEvalError("Eval start response missing runId.", undefined, "start_failed");
+  }
+  return json as EvalRunSnapshot;
+}
+
+/** Cancel a running batch via `POST /v1/dev/evals/runs/:runId/cancel`. */
+export async function cancelRemoteEvalRun(args: {
+  baseUrl: string;
+  runId: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  signal?: AbortSignal;
+}): Promise<EvalRunSnapshot> {
+  const baseUrl = args.baseUrl.replace(/\/$/, "");
+  const fetchImpl = args.fetchImpl ?? fetch;
+  let resp: Response;
+  try {
+    resp = await fetchImpl(
+      `${baseUrl}/v1/dev/evals/runs/${encodeURIComponent(args.runId)}/cancel`,
+      {
+        method: "POST",
+        headers: mergeRequestHeaders(args.auth),
+        signal: args.signal,
+      },
+    );
+  } catch (error) {
+    throw new RemoteEvalError(
+      error instanceof Error ? error.message : "Could not cancel eval run.",
+      undefined,
+      "cancel_failed",
+    );
+  }
+
+  const json = (await resp.json().catch(() => ({}))) as {
+    ok?: boolean;
+    error?: string;
+    message?: string;
+    status?: string;
+    runId?: string;
+  };
+
+  if (resp.status === 404 || json.error === "unknown_eval_run") {
+    throw new RemoteEvalError(`Unknown eval run ${args.runId}.`, 404, "unknown_eval_run");
+  }
+  if (resp.status === 409 || json.error === "eval_run_not_running") {
+    const status = typeof json.status === "string" && json.status !== "" ? json.status : "unknown";
+    throw new RemoteEvalError(
+      typeof json.message === "string" && json.message !== ""
+        ? json.message
+        : `Eval run ${args.runId} is not running (status: ${status}).`,
+      409,
+      "eval_run_not_running",
+    );
+  }
+  if (!resp.ok) {
+    const detail =
+      typeof json.message === "string" && json.message !== ""
+        ? json.message
+        : typeof json.error === "string" && json.error !== ""
+          ? json.error
+          : `HTTP ${resp.status}`;
+    throw new RemoteEvalError(`Could not cancel eval run: ${detail}`, resp.status, "cancel_failed");
+  }
+  if (typeof json.runId !== "string" || json.runId === "") {
+    throw new RemoteEvalError("Eval cancel response missing runId.", undefined, "cancel_failed");
+  }
+  return json as EvalRunSnapshot;
+}
+
+/** One-shot `GET /v1/dev/evals/runs/:runId` (no polling). */
+export async function getRemoteEvalRun(args: {
+  baseUrl: string;
+  runId: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  signal?: AbortSignal;
+}): Promise<EvalRunSnapshot> {
+  const baseUrl = args.baseUrl.replace(/\/$/, "");
+  const fetchImpl = args.fetchImpl ?? fetch;
+  let resp: Response;
+  try {
+    resp = await fetchImpl(`${baseUrl}/v1/dev/evals/runs/${encodeURIComponent(args.runId)}`, {
+      headers: mergeRequestHeaders(args.auth),
+      signal: args.signal,
+    });
+  } catch (error) {
+    throw new RemoteEvalError(
+      error instanceof Error ? error.message : "Could not fetch eval run.",
+      undefined,
+      "poll_failed",
+    );
+  }
+
+  if (resp.status === 404) {
+    throw new RemoteEvalError(`Unknown eval run ${args.runId}.`, 404, "unknown_eval_run");
+  }
+  if (!resp.ok) {
+    throw new RemoteEvalError(
+      `Could not fetch eval run (HTTP ${resp.status}).`,
+      resp.status,
+      "poll_failed",
+    );
+  }
+  return (await resp.json()) as EvalRunSnapshot;
+}
+
+/** Recent batches from `GET /v1/dev/evals/runs`. */
+export async function listRemoteEvalRuns(args: {
+  baseUrl: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  signal?: AbortSignal;
+}): Promise<{ runs: EvalRunSnapshot[]; activeRunId?: string }> {
+  const baseUrl = args.baseUrl.replace(/\/$/, "");
+  const fetchImpl = args.fetchImpl ?? fetch;
+  let resp: Response;
+  try {
+    resp = await fetchImpl(`${baseUrl}/v1/dev/evals/runs`, {
+      headers: mergeRequestHeaders(args.auth),
+      signal: args.signal,
+    });
+  } catch (error) {
+    throw new RemoteEvalError(
+      error instanceof Error ? error.message : "Could not list eval runs.",
+      undefined,
+      "poll_failed",
+    );
+  }
+  if (!resp.ok) {
+    throw new RemoteEvalError(
+      `Could not list eval runs (HTTP ${resp.status}).`,
+      resp.status,
+      "poll_failed",
+    );
+  }
+  const body = (await resp.json()) as {
+    runs?: unknown;
+    activeRunId?: unknown;
+  };
+  const runs = Array.isArray(body.runs)
+    ? (body.runs.filter(
+        (r): r is EvalRunSnapshot =>
+          r !== null && typeof r === "object" && typeof (r as EvalRunSnapshot).runId === "string",
+      ) as EvalRunSnapshot[])
+    : [];
+  return {
+    runs,
+    ...(typeof body.activeRunId === "string" && body.activeRunId !== ""
+      ? { activeRunId: body.activeRunId }
+      : {}),
+  };
+}
+
+export async function pollRemoteEvalRun(args: {
+  baseUrl: string;
+  runId: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  pollMs?: number;
+  signal?: AbortSignal;
+  onSnapshot?: (snap: EvalRunSnapshot) => void;
+}): Promise<EvalRunSnapshot> {
+  const pollMs = args.pollMs ?? DEFAULT_POLL_MS;
+  let lastJson = "";
+
+  for (;;) {
+    if (args.signal?.aborted === true) {
+      throw new RemoteEvalError("Eval wait aborted.", undefined, "aborted");
+    }
+
+    const snap = await getRemoteEvalRun(args);
+    const serialized = JSON.stringify(snap);
+    if (serialized !== lastJson) {
+      lastJson = serialized;
+      args.onSnapshot?.(snap);
+    }
+    if (snap.status !== "running") {
+      return snap;
+    }
+    await sleep(pollMs, args.signal);
+  }
+}
+
+export function snapshotToEvalRunResults(snapshot: EvalRunSnapshot): EvalRunResult[] {
+  return snapshot.cases.map(c => ({
+    id: c.id,
+    path: c.fileId,
+    ...(c.description === undefined ? {} : { description: c.description }),
+    ok: c.ok === true,
+    // Older serve processes send no verdict; infer it from `ok` so a remote
+    // batch still prints and reports consistently.
+    verdict: c.verdict ?? (c.ok === true ? "passed" : "failed"),
+    ...(c.skipReason === undefined ? {} : { skipReason: c.skipReason }),
+    ...(c.metadata === undefined ? {} : { metadata: c.metadata }),
+    assertions: c.assertions ?? [],
+    ...(c.sessionId === undefined ? {} : { sessionId: c.sessionId }),
+    inputs: c.inputs ?? [],
+    ...(c.finalText === undefined ? {} : { finalText: c.finalText }),
+    ...(c.tools === undefined ? {} : { tools: c.tools }),
+    ...(c.toolCalls === undefined ? {} : { toolCalls: c.toolCalls }),
+    ...(c.error === undefined ? {} : { error: c.error }),
+    logs: c.logs ?? [],
+    metrics: c.metrics ?? {},
+    durationMs: c.durationMs ?? 0,
+  }));
+}
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted === true) {
+      reject(new RemoteEvalError("Eval wait aborted.", undefined, "aborted"));
+      return;
+    }
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    const onAbort = (): void => {
+      clearTimeout(timer);
+      reject(new RemoteEvalError("Eval wait aborted.", undefined, "aborted"));
+    };
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}