@cursor/july 0.1.5 → 0.1.7

package/src/internal/evals-client.ts

@@ -0,0 +1,431 @@
+/**
+ * HTTP client for playground eval batches (`/v1/dev/evals/runs`).
+ *
+ * Used by `agentkit eval --prod` (or `--url`) so the batch runs on the serve
+ * process and is mirrored to `defineStorage` when an `evals` table is
+ * configured. Without those flags, eval keeps the in-process ephemeral path
+ * in `cli-ax.ts`.
+ */
+
+import type { EvalRunSnapshot } from "../evals.js";
+import { matchesEvalIdFilter } from "../evals.js";
+import type { EvalRunResult } from "./eval-runner.js";
+import { mergeRequestHeaders, type RequestAuth } from "./request-headers.js";
+
+const DEFAULT_POLL_MS = 1_000;
+
+export type RemoteEvalErrorCode =
+  | "no_matching_evals"
+  | "eval_run_in_progress"
+  | "unknown_eval_run"
+  | "start_failed"
+  | "poll_failed"
+  | "aborted";
+
+export class RemoteEvalError extends Error {
+  constructor(
+    message: string,
+    readonly status?: number,
+    readonly code?: RemoteEvalErrorCode,
+    readonly activeRunId?: string
+  ) {
+    super(message);
+    this.name = "RemoteEvalError";
+  }
+}
+
+export interface RemoteEvalListItem {
+  id: string;
+  fileId: string;
+  description?: string;
+  tags?: string[];
+}
+
+export interface RunRemoteEvalsOptions extends RequestAuth {
+  baseUrl: string;
+  filterIds?: string[];
+  tags?: string[];
+  /** Per-case timeout override forwarded to the serve process. */
+  timeoutMs?: number;
+  /** Ask the serve harness to emit verbose case logs (server-side). */
+  verbose?: boolean;
+  fetchImpl?: typeof fetch;
+  /** Poll interval while the batch is `running`. */
+  pollMs?: number;
+  /** Abort the wait (not the remote batch). */
+  signal?: AbortSignal;
+  /** Called each time the polled snapshot changes. */
+  onSnapshot?: (snap: EvalRunSnapshot) => void;
+}
+
+/**
+ * Start a batch on the target serve via `POST /v1/dev/evals/runs`, poll until
+ * terminal, and return CLI-shaped case results.
+ */
+export async function runRemoteEvals(
+  options: RunRemoteEvalsOptions
+): Promise<{ snapshot: EvalRunSnapshot; results: EvalRunResult[] }> {
+  const baseUrl = options.baseUrl.replace(/\/$/, "");
+  const fetchImpl = options.fetchImpl ?? fetch;
+  const auth: RequestAuth = {
+    bearerToken: options.bearerToken,
+    headers: options.headers,
+  };
+
+  const started = await startRemoteEvalRun({
+    baseUrl,
+    auth,
+    fetchImpl,
+    filterIds: options.filterIds,
+    tags: options.tags,
+    timeoutMs: options.timeoutMs,
+    verbose: options.verbose,
+    signal: options.signal,
+  });
+  options.onSnapshot?.(started);
+
+  const snapshot = await pollRemoteEvalRun({
+    baseUrl,
+    runId: started.runId,
+    auth,
+    fetchImpl,
+    pollMs: options.pollMs ?? DEFAULT_POLL_MS,
+    signal: options.signal,
+    onSnapshot: options.onSnapshot,
+  });
+
+  return { snapshot, results: snapshotToEvalRunResults(snapshot) };
+}
+
+/** List datapoints from `GET /v1/dev/evals` (same surface a remote run uses). */
+export async function listRemoteEvals(args: {
+  baseUrl: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  filterIds?: string[];
+  tags?: string[];
+  signal?: AbortSignal;
+}): Promise<RemoteEvalListItem[]> {
+  const baseUrl = args.baseUrl.replace(/\/$/, "");
+  const fetchImpl = args.fetchImpl ?? fetch;
+  let resp: Response;
+  try {
+    resp = await fetchImpl(`${baseUrl}/v1/dev/evals`, {
+      headers: mergeRequestHeaders(args.auth),
+      signal: args.signal,
+    });
+  } catch (error) {
+    throw new RemoteEvalError(
+      error instanceof Error ? error.message : "Could not list evals.",
+      undefined,
+      "start_failed"
+    );
+  }
+  if (!resp.ok) {
+    throw new RemoteEvalError(
+      `Could not list evals (HTTP ${resp.status}).`,
+      resp.status,
+      "start_failed"
+    );
+  }
+  const body = (await resp.json()) as {
+    evals?: Array<{
+      id?: unknown;
+      fileId?: unknown;
+      description?: unknown;
+      tags?: unknown;
+    }>;
+  };
+  const listed: RemoteEvalListItem[] = [];
+  for (const row of body.evals ?? []) {
+    if (typeof row.id !== "string" || row.id === "") {
+      continue;
+    }
+    if (typeof row.fileId !== "string" || row.fileId === "") {
+      continue;
+    }
+    const tags = Array.isArray(row.tags)
+      ? row.tags.filter((t): t is string => typeof t === "string")
+      : undefined;
+    listed.push({
+      id: row.id,
+      fileId: row.fileId,
+      ...(typeof row.description === "string"
+        ? { description: row.description }
+        : {}),
+      ...(tags === undefined || tags.length === 0 ? {} : { tags }),
+    });
+  }
+
+  let selected = listed;
+  if (args.filterIds !== undefined && args.filterIds.length > 0) {
+    selected = selected.filter((e) =>
+      args.filterIds!.some((f) => matchesEvalIdFilter(e.id, f))
+    );
+  }
+  if (args.tags !== undefined && args.tags.length > 0) {
+    selected = selected.filter((e) =>
+      (e.tags ?? []).some((t) => args.tags!.includes(t))
+    );
+  }
+  return selected;
+}
+
+export async function startRemoteEvalRun(args: {
+  baseUrl: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  filterIds?: string[];
+  tags?: string[];
+  timeoutMs?: number;
+  verbose?: boolean;
+  signal?: AbortSignal;
+}): Promise<EvalRunSnapshot> {
+  const baseUrl = args.baseUrl.replace(/\/$/, "");
+  const fetchImpl = args.fetchImpl ?? fetch;
+  const body: {
+    filterIds?: string[];
+    tags?: string[];
+    timeoutMs?: number;
+    verbose?: boolean;
+  } = {};
+  if (args.filterIds !== undefined && args.filterIds.length > 0) {
+    body.filterIds = args.filterIds;
+  }
+  if (args.tags !== undefined && args.tags.length > 0) {
+    body.tags = args.tags;
+  }
+  if (args.timeoutMs !== undefined) {
+    body.timeoutMs = args.timeoutMs;
+  }
+  if (args.verbose === true) {
+    body.verbose = true;
+  }
+
+  let resp: Response;
+  try {
+    resp = await fetchImpl(`${baseUrl}/v1/dev/evals/runs`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        ...mergeRequestHeaders(args.auth),
+      },
+      body: JSON.stringify(body),
+      signal: args.signal,
+    });
+  } catch (error) {
+    throw new RemoteEvalError(
+      error instanceof Error ? error.message : "Could not start eval run.",
+      undefined,
+      "start_failed"
+    );
+  }
+
+  const json = (await resp.json().catch(() => ({}))) as {
+    ok?: boolean;
+    error?: string;
+    message?: string;
+    activeRunId?: string;
+    runId?: string;
+  };
+
+  if (json.error === "no_matching_evals") {
+    throw new RemoteEvalError(
+      "No matching evals found.",
+      404,
+      "no_matching_evals"
+    );
+  }
+  if (resp.status === 409 || json.error === "eval_run_in_progress") {
+    throw new RemoteEvalError(
+      json.activeRunId !== undefined && json.activeRunId !== ""
+        ? `An eval run is already in progress (${json.activeRunId}).`
+        : "An eval run is already in progress.",
+      409,
+      "eval_run_in_progress",
+      typeof json.activeRunId === "string" ? json.activeRunId : undefined
+    );
+  }
+  if (!resp.ok) {
+    const detail =
+      typeof json.message === "string" && json.message !== ""
+        ? json.message
+        : typeof json.error === "string" && json.error !== ""
+          ? json.error
+          : `HTTP ${resp.status}`;
+    throw new RemoteEvalError(
+      `Could not start eval run: ${detail}`,
+      resp.status,
+      "start_failed"
+    );
+  }
+  if (typeof json.runId !== "string" || json.runId === "") {
+    throw new RemoteEvalError(
+      "Eval start response missing runId.",
+      undefined,
+      "start_failed"
+    );
+  }
+  return json as EvalRunSnapshot;
+}
+
+/** One-shot `GET /v1/dev/evals/runs/:runId` (no polling). */
+export async function getRemoteEvalRun(args: {
+  baseUrl: string;
+  runId: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  signal?: AbortSignal;
+}): Promise<EvalRunSnapshot> {
+  const baseUrl = args.baseUrl.replace(/\/$/, "");
+  const fetchImpl = args.fetchImpl ?? fetch;
+  let resp: Response;
+  try {
+    resp = await fetchImpl(
+      `${baseUrl}/v1/dev/evals/runs/${encodeURIComponent(args.runId)}`,
+      {
+        headers: mergeRequestHeaders(args.auth),
+        signal: args.signal,
+      }
+    );
+  } catch (error) {
+    throw new RemoteEvalError(
+      error instanceof Error ? error.message : "Could not fetch eval run.",
+      undefined,
+      "poll_failed"
+    );
+  }
+
+  if (resp.status === 404) {
+    throw new RemoteEvalError(
+      `Unknown eval run ${args.runId}.`,
+      404,
+      "unknown_eval_run"
+    );
+  }
+  if (!resp.ok) {
+    throw new RemoteEvalError(
+      `Could not fetch eval run (HTTP ${resp.status}).`,
+      resp.status,
+      "poll_failed"
+    );
+  }
+  return (await resp.json()) as EvalRunSnapshot;
+}
+
+/** Recent batches from `GET /v1/dev/evals/runs`. */
+export async function listRemoteEvalRuns(args: {
+  baseUrl: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  signal?: AbortSignal;
+}): Promise<{ runs: EvalRunSnapshot[]; activeRunId?: string }> {
+  const baseUrl = args.baseUrl.replace(/\/$/, "");
+  const fetchImpl = args.fetchImpl ?? fetch;
+  let resp: Response;
+  try {
+    resp = await fetchImpl(`${baseUrl}/v1/dev/evals/runs`, {
+      headers: mergeRequestHeaders(args.auth),
+      signal: args.signal,
+    });
+  } catch (error) {
+    throw new RemoteEvalError(
+      error instanceof Error ? error.message : "Could not list eval runs.",
+      undefined,
+      "poll_failed"
+    );
+  }
+  if (!resp.ok) {
+    throw new RemoteEvalError(
+      `Could not list eval runs (HTTP ${resp.status}).`,
+      resp.status,
+      "poll_failed"
+    );
+  }
+  const body = (await resp.json()) as {
+    runs?: unknown;
+    activeRunId?: unknown;
+  };
+  const runs = Array.isArray(body.runs)
+    ? (body.runs.filter(
+        (r): r is EvalRunSnapshot =>
+          r !== null &&
+          typeof r === "object" &&
+          typeof (r as EvalRunSnapshot).runId === "string"
+      ) as EvalRunSnapshot[])
+    : [];
+  return {
+    runs,
+    ...(typeof body.activeRunId === "string" && body.activeRunId !== ""
+      ? { activeRunId: body.activeRunId }
+      : {}),
+  };
+}
+
+export async function pollRemoteEvalRun(args: {
+  baseUrl: string;
+  runId: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  pollMs?: number;
+  signal?: AbortSignal;
+  onSnapshot?: (snap: EvalRunSnapshot) => void;
+}): Promise<EvalRunSnapshot> {
+  const pollMs = args.pollMs ?? DEFAULT_POLL_MS;
+  let lastJson = "";
+
+  for (;;) {
+    if (args.signal?.aborted === true) {
+      throw new RemoteEvalError("Eval wait aborted.", undefined, "aborted");
+    }
+
+    const snap = await getRemoteEvalRun(args);
+    const serialized = JSON.stringify(snap);
+    if (serialized !== lastJson) {
+      lastJson = serialized;
+      args.onSnapshot?.(snap);
+    }
+    if (snap.status !== "running") {
+      return snap;
+    }
+    await sleep(pollMs, args.signal);
+  }
+}
+
+export function snapshotToEvalRunResults(
+  snapshot: EvalRunSnapshot
+): EvalRunResult[] {
+  return snapshot.cases.map((c) => ({
+    id: c.id,
+    path: c.fileId,
+    ...(c.description === undefined ? {} : { description: c.description }),
+    ok: c.ok === true,
+    assertions: c.assertions ?? [],
+    ...(c.sessionId === undefined ? {} : { sessionId: c.sessionId }),
+    inputs: c.inputs ?? [],
+    ...(c.finalText === undefined ? {} : { finalText: c.finalText }),
+    ...(c.tools === undefined ? {} : { tools: c.tools }),
+    ...(c.toolCalls === undefined ? {} : { toolCalls: c.toolCalls }),
+    ...(c.error === undefined ? {} : { error: c.error }),
+    logs: c.logs ?? [],
+    durationMs: c.durationMs ?? 0,
+  }));
+}
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted === true) {
+      reject(new RemoteEvalError("Eval wait aborted.", undefined, "aborted"));
+      return;
+    }
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    const onAbort = (): void => {
+      clearTimeout(timer);
+      reject(new RemoteEvalError("Eval wait aborted.", undefined, "aborted"));
+    };
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}

package/src/internal/init-project.ts

@@ -441,6 +441,7 @@ Evals:
   Required: evals/evals.config.ts with defineEvalConfig({ maxConcurrency })
     (hard-capped at 200 due to model provider request limits)
   Run: ${CLI_COMMAND_NAME} eval [--json] [evalId...]
+    (pass --prod to run against the hosted deployment)
 
 See AGENTS.md in the ${PACKAGE_NAME} package for the full loop.
 `);

package/src/internal/playground-proxy.ts

@@ -14,9 +14,9 @@
  * request-targets are rejected so injected headers never leave the configured
  * upstream.
  *
- * When `refreshUpstream` is provided (hosted `--prod`), credentials are
- * re-minted shortly before `expiresAt` and once on upstream 401 so a long
- * playground session does not require restarting the CLI.
+ * When `refreshUpstream` is provided (e.g. short-lived upstream tokens),
+ * credentials are re-minted shortly before `expiresAt` and once on upstream
+ * 401 so a long playground session does not require restarting the CLI.
  */
 
 import { randomBytes } from "node:crypto";
@@ -84,8 +84,8 @@ export interface StartPlaygroundProxyArgs {
   /** Initial engineAccess expiry (ISO); enables proactive refresh. */
   expiresAt?: string;
   /**
-   * Re-mint upstream URL + headers (hosted `--prod`). Single-flighted across
-   * concurrent browser requests; called near expiry and once on 401.
+   * Re-mint upstream URL + headers. Single-flighted across concurrent browser
+   * requests; called near expiry and once on 401.
    */
   refreshUpstream?: () => Promise<PlaygroundUpstreamAccess>;
   /** Bind port; `0` = ephemeral. */

package/src/internal/resolve-prod-target.ts

@@ -48,6 +48,29 @@ export interface ResolvedProdTarget {
   expiresAt: string;
 }
 
+/**
+ * Thrown when a hosted deployment cannot be resolved. `notFound` is true
+ * when the deployment does not exist, so callers can suggest deploying
+ * (or running locally) instead of surfacing a bare API error.
+ */
+export class ProdTargetError extends Error {
+  constructor(
+    message: string,
+    readonly options: { slug: string; notFound?: boolean } = { slug: "" }
+  ) {
+    super(message);
+    this.name = "ProdTargetError";
+  }
+
+  get slug(): string {
+    return this.options.slug;
+  }
+
+  get notFound(): boolean {
+    return this.options.notFound === true;
+  }
+}
+
 /**
  * Look up a running deployment and return its engineAccess URL/headers.
  * Throws {@link Error} with a CLI-ready message on failure.
@@ -65,8 +88,10 @@ export async function resolveProdTarget(
     err,
   });
   if (typeof auth === "number") {
+    // Not "--prod": `eval` resolves the hosted deployment by default,
+    // without any flag.
     throw new Error(
-      `Could not resolve Cursor credentials/team for --prod (see above).`
+      `Could not resolve Cursor credentials/team for the hosted deployment (see above).`
     );
   }
 
@@ -83,8 +108,9 @@ export async function resolveProdTarget(
       });
     } catch (error) {
       if (error instanceof DeployApiError) {
-        throw new Error(
-          `Deployment "${slug}" (team ${auth.teamId}): ${error.message}`
+        throw new ProdTargetError(
+          `Deployment "${slug}" (team ${auth.teamId}): ${error.message}`,
+          { slug, notFound: error.status === 404 }
         );
       }
       throw error;
@@ -143,6 +169,78 @@ export async function resolveProdTarget(
   };
 }
 
+export interface ResolvedProdAliasPlayground {
+  /** Hosted playground URL on the API alias (SPA owns alias-token sign-in). */
+  playgroundUrl: string;
+  slug: string;
+}
+
+/**
+ * Look up a running deployment and return its stable alias playground URL.
+ * Does not mint `engineAccess` — the browser signs in with the alias token.
+ * Throws {@link Error} with a CLI-ready message on failure.
+ */
+export async function resolveProdAliasPlaygroundUrl(
+  options: ResolveProdTargetOptions & { sessionId?: string }
+): Promise<ResolvedProdAliasPlayground> {
+  const err =
+    options.err ?? ((text: string) => void process.stderr.write(text));
+  const slug = resolveProdSlug(options);
+  const auth = await resolveDeployAuth({
+    team: options.team,
+    apiKey: options.apiKey,
+    fetchImpl: options.fetchImpl,
+    err,
+  });
+  if (typeof auth === "number") {
+    throw new Error(
+      `Could not resolve Cursor credentials/team for the hosted deployment (see above).`
+    );
+  }
+
+  let detail: DeploymentDetail;
+  try {
+    detail = await auth.client.getDeployment({
+      teamId: auth.teamId,
+      slug,
+    });
+  } catch (error) {
+    if (error instanceof DeployApiError) {
+      throw new Error(
+        `Deployment "${slug}" (team ${auth.teamId}): ${error.message}`
+      );
+    }
+    throw error;
+  }
+
+  const statusOk = detail.status === "running" || detail.status === "degraded";
+  if (!statusOk) {
+    const lastError =
+      detail.lastError === null || detail.lastError === undefined
+        ? ""
+        : ` ${sanitizeCustomerFacingDetail(detail.lastError)}`;
+    throw new Error(
+      `Deployment "${slug}" is ${detail.status}, not running.${lastError}\n` +
+        `Deploy or wait: \`${CLI} deploy --slug ${slug}\` / \`${CLI} deployment ${slug}\``
+    );
+  }
+
+  const aliasBase = auth.client.aliasUrl(detail.engineAlias).replace(/\/$/, "");
+  const base = `${aliasBase}/playground`;
+  const sessionId =
+    options.sessionId !== undefined && options.sessionId.trim() !== ""
+      ? options.sessionId.trim()
+      : undefined;
+  const playgroundUrl =
+    sessionId === undefined
+      ? base
+      : `${base}?sessionId=${encodeURIComponent(sessionId)}`;
+
+  err(`${stderrPalette().dim(`Using hosted deployment "${slug}".`)}\n`);
+
+  return { playgroundUrl, slug };
+}
+
 /** Strip obvious hosting internals from customer-facing error fragments. */
 function sanitizeCustomerFacingDetail(message: string): string {
   return message

package/src/internal/server.ts

@@ -362,9 +362,8 @@ export async function startServer(
     const evalRuns = new EvalRunStore(
       project.rootDir,
       logger,
-      // defineStorage fallback for playground eval history; explicit
-      // `evals.config.ts` persistRuns still wins inside the store.
-      engine.storage?.asEvalRunPersistence()
+      // Playground eval history mirrors onto the defineStorage evals table.
+      engine.storage?.evalRuns()
     );
     // Eval history restores with the same startup step as sessions (reads
     // also hydrate lazily, but doing it here surfaces store problems at
@@ -738,6 +737,9 @@ export async function startServer(
       for (const runtime of runtimes) {
         runtime.scheduleRunner.stop();
         runtime.reminderRunner.stop();
+        // Before the engines close their storage coordinators, so the
+        // interrupted-batch write can still reach the sink.
+        runtime.evalRuns.close();
       }
       for (const stop of channelStopHooks) {
         try {
@@ -1080,6 +1082,8 @@ function mountAgentRoutes(
       const body = await readJsonObject(request);
       const filterIds = asStringArray(body?.filterIds);
       const tags = asStringArray(body?.tags);
+      const timeoutMs = asPositiveInt(body?.timeoutMs);
+      const verbose = body?.verbose === true;
       const authHeader = request.headers.get("authorization");
       const bearerToken = authHeader?.toLowerCase().startsWith("bearer ")
         ? authHeader.slice("bearer ".length).trim()
@@ -1091,6 +1095,8 @@ function mountAgentRoutes(
           ...(bearerToken === undefined || bearerToken === ""
             ? {}
             : { bearerToken }),
+          ...(timeoutMs === undefined ? {} : { timeoutMs }),
+          ...(verbose ? { verbose: true } : {}),
         });
         return jsonResponse(snap, { status: 202 });
       } catch (error) {
@@ -1894,6 +1900,14 @@ function asStringArray(value: unknown): string[] | undefined {
   return out.length === 0 ? undefined : out;
 }
 
+function asPositiveInt(value: unknown): number | undefined {
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    return undefined;
+  }
+  const n = Math.floor(value);
+  return n > 0 ? n : undefined;
+}
+
 async function assertReminderReadable(input: {
   engine: SessionEngine;
   reminder: ReminderInfo;