@cursor/bdk 0.2.3 → 0.2.4

package/src/internal/evals-client.ts

@@ -1,475 +0,0 @@
-/**
- * HTTP client for playground eval batches (`/v1/dev/evals/runs`).
- *
- * Used by `agent-sdk eval --prod` (or `--url`) so the batch runs on the serve
- * process and is mirrored to `defineStorage` when an `evals` table is
- * configured. Without those flags, eval keeps the in-process ephemeral path
- * in `cli-ax.ts`.
- */
-
-import type { EvalRunSnapshot } from "../evals.js";
-import { expandSummarySuiteSelection, isEvalSummaryCase, matchesEvalIdFilter } from "../evals.js";
-import type { EvalRunResult } from "./eval-runner.js";
-import { mergeRequestHeaders, type RequestAuth } from "./request-headers.js";
-
-const DEFAULT_POLL_MS = 1_000;
-
-export type RemoteEvalErrorCode =
-  | "no_matching_evals"
-  | "eval_run_in_progress"
-  | "eval_run_not_running"
-  | "unknown_eval_run"
-  | "start_failed"
-  | "poll_failed"
-  | "cancel_failed"
-  | "aborted";
-
-export class RemoteEvalError extends Error {
-  constructor(
-    message: string,
-    readonly status?: number,
-    readonly code?: RemoteEvalErrorCode,
-    readonly activeRunId?: string,
-  ) {
-    super(message);
-    this.name = "RemoteEvalError";
-  }
-}
-
-export interface RemoteEvalListItem {
-  id: string;
-  fileId: string;
-  description?: string;
-  tags?: string[];
-}
-
-export interface RunRemoteEvalsOptions extends RequestAuth {
-  baseUrl: string;
-  filterIds?: string[];
-  tags?: string[];
-  /** Per-case timeout override forwarded to the serve process. */
-  timeoutMs?: number;
-  /** Ask the serve harness to emit verbose case logs (server-side). */
-  verbose?: boolean;
-  fetchImpl?: typeof fetch;
-  /** Poll interval while the batch is `running`. */
-  pollMs?: number;
-  /** Abort the wait (not the remote batch). */
-  signal?: AbortSignal;
-  /** Called each time the polled snapshot changes. */
-  onSnapshot?: (snap: EvalRunSnapshot) => void;
-}
-
-/**
- * Start a batch on the target serve via `POST /v1/dev/evals/runs`, poll until
- * terminal, and return CLI-shaped case results.
- */
-export async function runRemoteEvals(
-  options: RunRemoteEvalsOptions,
-): Promise<{ snapshot: EvalRunSnapshot; results: EvalRunResult[] }> {
-  const baseUrl = options.baseUrl.replace(/\/$/, "");
-  const fetchImpl = options.fetchImpl ?? fetch;
-  const auth: RequestAuth = {
-    bearerToken: options.bearerToken,
-    headers: options.headers,
-  };
-
-  const started = await startRemoteEvalRun({
-    baseUrl,
-    auth,
-    fetchImpl,
-    filterIds: options.filterIds,
-    tags: options.tags,
-    timeoutMs: options.timeoutMs,
-    verbose: options.verbose,
-    signal: options.signal,
-  });
-  options.onSnapshot?.(started);
-
-  const snapshot = await pollRemoteEvalRun({
-    baseUrl,
-    runId: started.runId,
-    auth,
-    fetchImpl,
-    pollMs: options.pollMs ?? DEFAULT_POLL_MS,
-    signal: options.signal,
-    onSnapshot: options.onSnapshot,
-  });
-
-  return { snapshot, results: snapshotToEvalRunResults(snapshot) };
-}
-
-/** List datapoints from `GET /v1/dev/evals` (same surface a remote run uses). */
-export async function listRemoteEvals(args: {
-  baseUrl: string;
-  auth?: RequestAuth;
-  fetchImpl?: typeof fetch;
-  filterIds?: string[];
-  tags?: string[];
-  signal?: AbortSignal;
-}): Promise<RemoteEvalListItem[]> {
-  const baseUrl = args.baseUrl.replace(/\/$/, "");
-  const fetchImpl = args.fetchImpl ?? fetch;
-  let resp: Response;
-  try {
-    resp = await fetchImpl(`${baseUrl}/v1/dev/evals`, {
-      headers: mergeRequestHeaders(args.auth),
-      signal: args.signal,
-    });
-  } catch (error) {
-    throw new RemoteEvalError(
-      error instanceof Error ? error.message : "Could not list evals.",
-      undefined,
-      "start_failed",
-    );
-  }
-  if (!resp.ok) {
-    throw new RemoteEvalError(
-      `Could not list evals (HTTP ${resp.status}).`,
-      resp.status,
-      "start_failed",
-    );
-  }
-  const body = (await resp.json()) as {
-    evals?: Array<{
-      id?: unknown;
-      fileId?: unknown;
-      description?: unknown;
-      tags?: unknown;
-    }>;
-  };
-  const listed: RemoteEvalListItem[] = [];
-  for (const row of body.evals ?? []) {
-    if (typeof row.id !== "string" || row.id === "") {
-      continue;
-    }
-    if (typeof row.fileId !== "string" || row.fileId === "") {
-      continue;
-    }
-    const tags = Array.isArray(row.tags)
-      ? row.tags.filter((t): t is string => typeof t === "string")
-      : undefined;
-    listed.push({
-      id: row.id,
-      fileId: row.fileId,
-      ...(typeof row.description === "string" ? { description: row.description } : {}),
-      ...(tags === undefined || tags.length === 0 ? {} : { tags }),
-    });
-  }
-
-  let selected = listed;
-  if (args.filterIds !== undefined && args.filterIds.length > 0) {
-    selected = selected.filter(e => args.filterIds!.some(f => matchesEvalIdFilter(e.id, f)));
-  }
-  if (args.tags !== undefined && args.tags.length > 0) {
-    selected = selected.filter(e => (e.tags ?? []).some(t => args.tags!.includes(t)));
-  }
-  // Match server-side filterDiscoveredEvals: summary → whole file.
-  return expandSummarySuiteSelection(listed, selected, e => isEvalSummaryCase(e.tags));
-}
-
-export async function startRemoteEvalRun(args: {
-  baseUrl: string;
-  auth?: RequestAuth;
-  fetchImpl?: typeof fetch;
-  filterIds?: string[];
-  tags?: string[];
-  timeoutMs?: number;
-  verbose?: boolean;
-  signal?: AbortSignal;
-}): Promise<EvalRunSnapshot> {
-  const baseUrl = args.baseUrl.replace(/\/$/, "");
-  const fetchImpl = args.fetchImpl ?? fetch;
-  const body: {
-    filterIds?: string[];
-    tags?: string[];
-    timeoutMs?: number;
-    verbose?: boolean;
-  } = {};
-  if (args.filterIds !== undefined && args.filterIds.length > 0) {
-    body.filterIds = args.filterIds;
-  }
-  if (args.tags !== undefined && args.tags.length > 0) {
-    body.tags = args.tags;
-  }
-  if (args.timeoutMs !== undefined) {
-    body.timeoutMs = args.timeoutMs;
-  }
-  if (args.verbose === true) {
-    body.verbose = true;
-  }
-
-  let resp: Response;
-  try {
-    resp = await fetchImpl(`${baseUrl}/v1/dev/evals/runs`, {
-      method: "POST",
-      headers: {
-        "content-type": "application/json",
-        ...mergeRequestHeaders(args.auth),
-      },
-      body: JSON.stringify(body),
-      signal: args.signal,
-    });
-  } catch (error) {
-    throw new RemoteEvalError(
-      error instanceof Error ? error.message : "Could not start eval run.",
-      undefined,
-      "start_failed",
-    );
-  }
-
-  const json = (await resp.json().catch(() => ({}))) as {
-    ok?: boolean;
-    error?: string;
-    message?: string;
-    activeRunId?: string;
-    runId?: string;
-  };
-
-  if (json.error === "no_matching_evals") {
-    throw new RemoteEvalError("No matching evals found.", 404, "no_matching_evals");
-  }
-  if (resp.status === 409 || json.error === "eval_run_in_progress") {
-    throw new RemoteEvalError(
-      json.activeRunId !== undefined && json.activeRunId !== ""
-        ? `An eval run is already in progress (${json.activeRunId}).`
-        : "An eval run is already in progress.",
-      409,
-      "eval_run_in_progress",
-      typeof json.activeRunId === "string" ? json.activeRunId : undefined,
-    );
-  }
-  if (!resp.ok) {
-    const detail =
-      typeof json.message === "string" && json.message !== ""
-        ? json.message
-        : typeof json.error === "string" && json.error !== ""
-          ? json.error
-          : `HTTP ${resp.status}`;
-    throw new RemoteEvalError(`Could not start eval run: ${detail}`, resp.status, "start_failed");
-  }
-  if (typeof json.runId !== "string" || json.runId === "") {
-    throw new RemoteEvalError("Eval start response missing runId.", undefined, "start_failed");
-  }
-  return json as EvalRunSnapshot;
-}
-
-/** Cancel a running batch via `POST /v1/dev/evals/runs/:runId/cancel`. */
-export async function cancelRemoteEvalRun(args: {
-  baseUrl: string;
-  runId: string;
-  auth?: RequestAuth;
-  fetchImpl?: typeof fetch;
-  signal?: AbortSignal;
-}): Promise<EvalRunSnapshot> {
-  const baseUrl = args.baseUrl.replace(/\/$/, "");
-  const fetchImpl = args.fetchImpl ?? fetch;
-  let resp: Response;
-  try {
-    resp = await fetchImpl(
-      `${baseUrl}/v1/dev/evals/runs/${encodeURIComponent(args.runId)}/cancel`,
-      {
-        method: "POST",
-        headers: mergeRequestHeaders(args.auth),
-        signal: args.signal,
-      },
-    );
-  } catch (error) {
-    throw new RemoteEvalError(
-      error instanceof Error ? error.message : "Could not cancel eval run.",
-      undefined,
-      "cancel_failed",
-    );
-  }
-
-  const json = (await resp.json().catch(() => ({}))) as {
-    ok?: boolean;
-    error?: string;
-    message?: string;
-    status?: string;
-    runId?: string;
-  };
-
-  if (resp.status === 404 || json.error === "unknown_eval_run") {
-    throw new RemoteEvalError(`Unknown eval run ${args.runId}.`, 404, "unknown_eval_run");
-  }
-  if (resp.status === 409 || json.error === "eval_run_not_running") {
-    const status = typeof json.status === "string" && json.status !== "" ? json.status : "unknown";
-    throw new RemoteEvalError(
-      typeof json.message === "string" && json.message !== ""
-        ? json.message
-        : `Eval run ${args.runId} is not running (status: ${status}).`,
-      409,
-      "eval_run_not_running",
-    );
-  }
-  if (!resp.ok) {
-    const detail =
-      typeof json.message === "string" && json.message !== ""
-        ? json.message
-        : typeof json.error === "string" && json.error !== ""
-          ? json.error
-          : `HTTP ${resp.status}`;
-    throw new RemoteEvalError(`Could not cancel eval run: ${detail}`, resp.status, "cancel_failed");
-  }
-  if (typeof json.runId !== "string" || json.runId === "") {
-    throw new RemoteEvalError("Eval cancel response missing runId.", undefined, "cancel_failed");
-  }
-  return json as EvalRunSnapshot;
-}
-
-/** One-shot `GET /v1/dev/evals/runs/:runId` (no polling). */
-export async function getRemoteEvalRun(args: {
-  baseUrl: string;
-  runId: string;
-  auth?: RequestAuth;
-  fetchImpl?: typeof fetch;
-  signal?: AbortSignal;
-}): Promise<EvalRunSnapshot> {
-  const baseUrl = args.baseUrl.replace(/\/$/, "");
-  const fetchImpl = args.fetchImpl ?? fetch;
-  let resp: Response;
-  try {
-    resp = await fetchImpl(`${baseUrl}/v1/dev/evals/runs/${encodeURIComponent(args.runId)}`, {
-      headers: mergeRequestHeaders(args.auth),
-      signal: args.signal,
-    });
-  } catch (error) {
-    throw new RemoteEvalError(
-      error instanceof Error ? error.message : "Could not fetch eval run.",
-      undefined,
-      "poll_failed",
-    );
-  }
-
-  if (resp.status === 404) {
-    throw new RemoteEvalError(`Unknown eval run ${args.runId}.`, 404, "unknown_eval_run");
-  }
-  if (!resp.ok) {
-    throw new RemoteEvalError(
-      `Could not fetch eval run (HTTP ${resp.status}).`,
-      resp.status,
-      "poll_failed",
-    );
-  }
-  return (await resp.json()) as EvalRunSnapshot;
-}
-
-/** Recent batches from `GET /v1/dev/evals/runs`. */
-export async function listRemoteEvalRuns(args: {
-  baseUrl: string;
-  auth?: RequestAuth;
-  fetchImpl?: typeof fetch;
-  signal?: AbortSignal;
-}): Promise<{ runs: EvalRunSnapshot[]; activeRunId?: string }> {
-  const baseUrl = args.baseUrl.replace(/\/$/, "");
-  const fetchImpl = args.fetchImpl ?? fetch;
-  let resp: Response;
-  try {
-    resp = await fetchImpl(`${baseUrl}/v1/dev/evals/runs`, {
-      headers: mergeRequestHeaders(args.auth),
-      signal: args.signal,
-    });
-  } catch (error) {
-    throw new RemoteEvalError(
-      error instanceof Error ? error.message : "Could not list eval runs.",
-      undefined,
-      "poll_failed",
-    );
-  }
-  if (!resp.ok) {
-    throw new RemoteEvalError(
-      `Could not list eval runs (HTTP ${resp.status}).`,
-      resp.status,
-      "poll_failed",
-    );
-  }
-  const body = (await resp.json()) as {
-    runs?: unknown;
-    activeRunId?: unknown;
-  };
-  const runs = Array.isArray(body.runs)
-    ? (body.runs.filter(
-        (r): r is EvalRunSnapshot =>
-          r !== null && typeof r === "object" && typeof (r as EvalRunSnapshot).runId === "string",
-      ) as EvalRunSnapshot[])
-    : [];
-  return {
-    runs,
-    ...(typeof body.activeRunId === "string" && body.activeRunId !== ""
-      ? { activeRunId: body.activeRunId }
-      : {}),
-  };
-}
-
-export async function pollRemoteEvalRun(args: {
-  baseUrl: string;
-  runId: string;
-  auth?: RequestAuth;
-  fetchImpl?: typeof fetch;
-  pollMs?: number;
-  signal?: AbortSignal;
-  onSnapshot?: (snap: EvalRunSnapshot) => void;
-}): Promise<EvalRunSnapshot> {
-  const pollMs = args.pollMs ?? DEFAULT_POLL_MS;
-  let lastJson = "";
-
-  for (;;) {
-    if (args.signal?.aborted === true) {
-      throw new RemoteEvalError("Eval wait aborted.", undefined, "aborted");
-    }
-
-    const snap = await getRemoteEvalRun(args);
-    const serialized = JSON.stringify(snap);
-    if (serialized !== lastJson) {
-      lastJson = serialized;
-      args.onSnapshot?.(snap);
-    }
-    if (snap.status !== "running") {
-      return snap;
-    }
-    await sleep(pollMs, args.signal);
-  }
-}
-
-export function snapshotToEvalRunResults(snapshot: EvalRunSnapshot): EvalRunResult[] {
-  return snapshot.cases.map(c => ({
-    id: c.id,
-    path: c.fileId,
-    ...(c.description === undefined ? {} : { description: c.description }),
-    ok: c.ok === true,
-    // Older serve processes send no verdict; infer it from `ok` so a remote
-    // batch still prints and reports consistently.
-    verdict: c.verdict ?? (c.ok === true ? "passed" : "failed"),
-    ...(c.skipReason === undefined ? {} : { skipReason: c.skipReason }),
-    ...(c.metadata === undefined ? {} : { metadata: c.metadata }),
-    assertions: c.assertions ?? [],
-    ...(c.sessionId === undefined ? {} : { sessionId: c.sessionId }),
-    inputs: c.inputs ?? [],
-    ...(c.finalText === undefined ? {} : { finalText: c.finalText }),
-    ...(c.tools === undefined ? {} : { tools: c.tools }),
-    ...(c.toolCalls === undefined ? {} : { toolCalls: c.toolCalls }),
-    ...(c.error === undefined ? {} : { error: c.error }),
-    logs: c.logs ?? [],
-    metrics: c.metrics ?? {},
-    durationMs: c.durationMs ?? 0,
-  }));
-}
-
-function sleep(ms: number, signal?: AbortSignal): Promise<void> {
-  return new Promise((resolve, reject) => {
-    if (signal?.aborted === true) {
-      reject(new RemoteEvalError("Eval wait aborted.", undefined, "aborted"));
-      return;
-    }
-    const timer = setTimeout(() => {
-      signal?.removeEventListener("abort", onAbort);
-      resolve();
-    }, ms);
-    const onAbort = (): void => {
-      clearTimeout(timer);
-      reject(new RemoteEvalError("Eval wait aborted.", undefined, "aborted"));
-    };
-    signal?.addEventListener("abort", onAbort, { once: true });
-  });
-}