@cursor/july 0.1.6 → 0.1.7

package/src/internal/cli-ax.ts

@@ -10,6 +10,7 @@
 import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
+import type { EvalRunSnapshot } from "../evals.js";
 import { serve } from "../index.js";
 import type { AgentServeHandle } from "../types.js";
 import { lookupSessionContinuation, runChat } from "./chat-client.js";
@@ -23,6 +24,14 @@ import {
   filterDiscoveredEvals,
   runDiscoveredEvals,
 } from "./eval-runner.js";
+import {
+  getRemoteEvalRun,
+  listRemoteEvalRuns,
+  listRemoteEvals,
+  RemoteEvalError,
+  runRemoteEvals,
+  startRemoteEvalRun,
+} from "./evals-client.js";
 import {
   formatInitNextSteps,
   formatInitScaffoldSummary,
@@ -95,17 +104,18 @@ export interface AxCliOptions {
   url?: string;
   /**
    * Route chat/resume/run/call/eval/logs/sessions at the team's hosted
-   * deployment
-   * instead of a local ephemeral server. Mutually exclusive with `--url`.
-   * `eval` targets the deployment by default, so there `--prod` is just
-   * being explicit.
+   * deployment instead of a local ephemeral server. Mutually exclusive with
+   * `--url`. For `eval`, this runs a server-side batch via
+   * `/v1/dev/evals/runs` (persisted when `defineStorage` has an `evals` table);
+   * without `--prod`, eval defaults to an ephemeral local harness.
    */
   prod?: boolean;
   /**
-   * `eval`: run against an ephemeral local server instead of the hosted
-   * deployment (the eval default). Mutually exclusive with `--url`/`--prod`.
+   * `eval` (`--prod` / `--url` only): start the batch via
+   * `POST /v1/dev/evals/runs` and return the `runId` without polling.
+   * Mutually exclusive with `--list`.
    */
-  local?: boolean;
+  noWait?: boolean;
   /** Cursor team id for `--prod` (defaults to the account's team). */
   team?: string;
   /** Cursor API key override for `--prod` management calls. */
@@ -667,28 +677,27 @@ export async function cmdTrajectory(options: AxCliOptions): Promise<number> {
 }
 
 /**
- * Which surface `eval` should target. Deployment is the default; `--local`
- * opts into an ephemeral local server, `--url` into a running one.
+ * Which surface `eval` should target. Local (ephemeral) is the default;
+ * `--prod` uses the hosted deployment, `--url` a running server.
  * Exported for tests.
  */
 export function resolveEvalTargetMode(
-  options: Pick<AxCliOptions, "local" | "prod" | "url">
+  options: Pick<AxCliOptions, "prod" | "url">
 ): "local" | "url" | "deployment" | { error: string } {
   const flags = [
-    options.local === true ? "--local" : undefined,
     options.prod === true ? "--prod" : undefined,
     options.url !== undefined ? "--url" : undefined,
   ].filter((flag) => flag !== undefined);
   if (flags.length > 1) {
     return { error: `${flags.join(" and ")} are mutually exclusive` };
   }
-  if (options.local === true) {
-    return "local";
-  }
   if (options.url !== undefined) {
     return "url";
   }
-  return "deployment";
+  if (options.prod === true) {
+    return "deployment";
+  }
+  return "local";
 }
 
 /** Resolve the eval target; prints a CLI-ready error and returns an exit code on failure. */
@@ -723,9 +732,8 @@ async function resolveEvalTarget(
       // genuinely missing deployment.
       process.stderr.write(
         `${sanitizeCustomerError(error.message)}\n` +
-          `Evals run on the hosted deployment by default.\n` +
           `  deploy it first:  ${CLI_COMMAND_NAME} deploy --slug ${error.slug}\n` +
-          `  or run locally:   ${CLI_COMMAND_NAME} eval --local --dir ${options.dir}\n`
+          `  or omit --prod:   ${CLI_COMMAND_NAME} eval --dir ${options.dir}\n`
       );
       return 1;
     }
@@ -733,7 +741,7 @@ async function resolveEvalTarget(
       `${sanitizeCustomerError(
         error instanceof Error ? error.message : String(error)
       )}\n` +
-        `Evals run on the hosted deployment by default; pass --local to run on an ephemeral local server.\n`
+        `Pass --prod for the hosted deployment, or omit it to run on an ephemeral local server.\n`
     );
     return 1;
   }
@@ -741,43 +749,76 @@ async function resolveEvalTarget(
 
 export async function cmdEval(options: AxCliOptions): Promise<number> {
   const projectRoot = resolve(options.cwd ?? process.cwd(), options.dir);
+  const mode = resolveEvalTargetMode(options);
+  if (typeof mode === "object") {
+    process.stderr.write(`${mode.error}\n`);
+    return 2;
+  }
+  if (options.noWait === true) {
+    if (options.list === true) {
+      process.stderr.write("--no-wait and --list are mutually exclusive\n");
+      return 2;
+    }
+    if (mode === "local") {
+      process.stderr.write(
+        "--no-wait requires --prod or --url (not the default local target)\n"
+      );
+      return 2;
+    }
+  }
+
+  const filterIds =
+    options.evalIds === undefined || options.evalIds.length === 0
+      ? undefined
+      : options.evalIds;
+  const tags =
+    options.tags === undefined || options.tags.length === 0
+      ? undefined
+      : options.tags;
 
   if (options.list) {
-    const { evals } = await discoverEvals(projectRoot);
-    const listed = filterDiscoveredEvals(evals, {
-      filterIds: options.evalIds,
-      tags: options.tags,
-    });
-    if (options.json) {
-      console.log(
-        JSON.stringify(
-          listed.map((e) => ({
-            id: e.id,
-            fileId: e.fileId,
-            path: e.path,
-            description: e.definition.description,
-            tags: e.definition.tags ?? [],
-          })),
-          null,
-          2
-        )
+    // List the same surface a run will execute: local filesystem by default,
+    // GET /v1/dev/evals on --prod / --url.
+    if (mode === "local") {
+      return listLocalEvals(projectRoot, options, filterIds, tags);
+    }
+    const target = await resolveEvalTarget(options);
+    if (typeof target === "number") {
+      return target;
+    }
+    try {
+      const listed = await listRemoteEvals({
+        baseUrl: target.agentUrl,
+        auth: {
+          bearerToken: options.bearerToken,
+          headers: target.headers,
+        },
+        fetchImpl: options.fetchImpl,
+        filterIds,
+        tags,
+      });
+      return printListedEvals(
+        listed.map((e) => ({
+          id: e.id,
+          fileId: e.fileId,
+          description: e.description,
+          tags: e.tags ?? [],
+        })),
+        options,
+        `No evals on ${target.agentUrl}`
       );
-    } else if (listed.length === 0) {
-      process.stdout.write(`No evals under ${projectRoot}/evals\n`);
-    } else {
-      for (const e of listed) {
-        const tags =
-          e.definition.tags && e.definition.tags.length > 0
-            ? `  [${e.definition.tags.join(", ")}]`
-            : "";
-        const desc =
-          e.definition.description !== undefined
-            ? ` — ${e.definition.description}`
-            : "";
-        process.stdout.write(`${e.id}${tags}${desc}\n`);
+    } catch (error) {
+      process.stderr.write(
+        `${sanitizeCustomerError(
+          error instanceof Error ? error.message : String(error)
+        )}\n`
+      );
+      return 1;
+    } finally {
+      if (target.close !== undefined) {
+        await target.close();
       }
     }
-    return 0;
   }
 
   const target = await resolveEvalTarget(options);
@@ -785,52 +826,402 @@ export async function cmdEval(options: AxCliOptions): Promise<number> {
     return target;
   }
   const stream = shouldStreamProgress(options);
-  const progress = stream ? createStreamProgress() : undefined;
+
   try {
     if (options.verbose || stream) {
       process.stderr.write(`eval target: ${target.agentUrl}\n`);
     }
-    const results = await runDiscoveredEvals({
-      projectRoot,
+
+    // Default local: ephemeral in-process harness against local eval sources.
+    // --prod / --url: run on the serve process via /v1/dev/evals/runs so the
+    // batch lands in playground history and defineStorage evals (when set).
+    if (mode === "local") {
+      const progress = stream ? createStreamProgress() : undefined;
+      try {
+        const results = await runDiscoveredEvals({
+          projectRoot,
+          baseUrl: target.agentUrl,
+          filterIds,
+          tags,
+          timeoutMs: options.timeoutMs,
+          bearerToken: options.bearerToken,
+          headers: target.headers,
+          verbose: Boolean(options.verbose) || stream,
+          onEvent:
+            progress === undefined ? undefined : (e) => progress.onEvent(e),
+        });
+        if (results.length === 0) {
+          process.stderr.write("No matching evals found.\n");
+          return 2;
+        }
+        return printEvalResults(results, options);
+      } finally {
+        progress?.close();
+      }
+    }
+
+    try {
+      if (options.noWait === true) {
+        const started = await startRemoteEvalRun({
+          baseUrl: target.agentUrl,
+          filterIds,
+          tags,
+          timeoutMs: options.timeoutMs,
+          verbose: Boolean(options.verbose),
+          auth: {
+            bearerToken: options.bearerToken,
+            headers: target.headers,
+          },
+          fetchImpl: options.fetchImpl,
+        });
+        return printEvalAccepted(started, options, target.agentUrl);
+      }
+
+      const progress = createRemoteEvalProgress(
+        options.verbose === true || stream
+      );
+      const { snapshot, results } = await runRemoteEvals({
+        baseUrl: target.agentUrl,
+        filterIds,
+        tags,
+        timeoutMs: options.timeoutMs,
+        verbose: Boolean(options.verbose) || stream,
+        bearerToken: options.bearerToken,
+        headers: target.headers,
+        fetchImpl: options.fetchImpl,
+        onSnapshot: progress === undefined ? undefined : progress.onSnapshot,
+      });
+      if (results.length === 0 && snapshot.status !== "failed") {
+        process.stderr.write("No matching evals found.\n");
+        return 2;
+      }
+      return printEvalResults(results, options, {
+        runId: snapshot.runId,
+        status: snapshot.status,
+        error: snapshot.error,
+      });
+    } catch (error) {
+      if (
+        error instanceof RemoteEvalError &&
+        error.code === "no_matching_evals"
+      ) {
+        process.stderr.write(`${error.message}\n`);
+        return 2;
+      }
+      process.stderr.write(
+        `${sanitizeCustomerError(
+          error instanceof Error ? error.message : String(error)
+        )}\n`
+      );
+      return 1;
+    }
+  } finally {
+    if (target.close !== undefined) {
+      await target.close();
+    }
+  }
+}
+
+/**
+ * Flags to re-target the same surface from an `eval --no-wait` hint.
+ * Exported for tests.
+ */
+export function formatEvalStatusFlags(
+  options: Pick<AxCliOptions, "prod" | "url" | "slug" | "team">
+): string {
+  const parts: string[] = [];
+  if (options.url !== undefined) {
+    parts.push(`--url ${options.url.replace(/\/$/, "")}`);
+  } else {
+    parts.push("--prod");
+    if (options.slug !== undefined && options.slug !== "") {
+      parts.push(`--slug ${options.slug}`);
+    }
+  }
+  if (options.team !== undefined && options.team !== "") {
+    parts.push(`--team ${options.team}`);
+  }
+  return parts.length === 0 ? "" : ` ${parts.join(" ")}`;
+}
+
+/** `--no-wait`: print the accepted batch id (and how to poll) then exit. */
+function printEvalAccepted(
+  started: EvalRunSnapshot,
+  options: AxCliOptions,
+  agentUrl: string
+): number {
+  const pollPath = `${agentUrl.replace(/\/$/, "")}/v1/dev/evals/runs/${started.runId}`;
+  const statusFlags = formatEvalStatusFlags(options);
+  if (options.json) {
+    console.log(
+      JSON.stringify(
+        {
+          ok: true,
+          accepted: true,
+          runId: started.runId,
+          status: started.status,
+          summary: started.summary,
+          pollUrl: pollPath,
+          durableRuns: started.config.durableRuns === true,
+        },
+        null,
+        2
+      )
+    );
+  } else {
+    const durable = started.config.durableRuns === true ? " (persisted)" : "";
+    process.stdout.write(
+      `Accepted eval run ${started.runId}${durable} — ${started.summary.total} case(s)\n`
+    );
+    process.stdout.write(
+      `Status: ${CLI_COMMAND_NAME} eval status ${started.runId}${statusFlags}\n`
+    );
+    process.stdout.write(`Poll: ${pollPath}\n`);
+  }
+  return 0;
+}
+
+/**
+ * `eval status [runId]` — fetch one batch (or list recent) on the deployment /
+ * `--url` target. Exit `0` when the batch is completed with all cases passing,
+ * `1` when failed / any case failed / unknown, `2` for usage errors, `3` when
+ * the batch is still running (for poll loops after `--no-wait`).
+ */
+export async function cmdEvalStatus(
+  runId: string | undefined,
+  options: AxCliOptions
+): Promise<number> {
+  const mode = resolveEvalTargetMode(options);
+  if (typeof mode === "object") {
+    process.stderr.write(`${mode.error}\n`);
+    return 2;
+  }
+  if (mode === "local") {
+    process.stderr.write(
+      "eval status requires --prod or --url (not the default local target)\n"
+    );
+    return 2;
+  }
+  if (options.list === true || options.noWait === true) {
+    process.stderr.write("eval status does not accept --list or --no-wait\n");
+    return 2;
+  }
+
+  const target = await resolveEvalTarget(options);
+  if (typeof target === "number") {
+    return target;
+  }
+  const auth = {
+    bearerToken: options.bearerToken,
+    headers: target.headers,
+  };
+
+  try {
+    if (runId === undefined || runId === "") {
+      const { runs, activeRunId } = await listRemoteEvalRuns({
+        baseUrl: target.agentUrl,
+        auth,
+        fetchImpl: options.fetchImpl,
+      });
+      if (options.json) {
+        console.log(JSON.stringify({ runs, activeRunId }, null, 2));
+      } else if (runs.length === 0) {
+        process.stdout.write("No eval runs on this agent.\n");
+      } else {
+        if (activeRunId !== undefined) {
+          process.stdout.write(`active: ${activeRunId}\n`);
+        }
+        for (const run of runs) {
+          process.stdout.write(`${formatEvalRunSummaryLine(run)}\n`);
+        }
+      }
+      return 0;
+    }
+
+    const snap = await getRemoteEvalRun({
       baseUrl: target.agentUrl,
-      filterIds:
-        options.evalIds === undefined || options.evalIds.length === 0
-          ? undefined
-          : options.evalIds,
-      tags:
-        options.tags === undefined || options.tags.length === 0
-          ? undefined
-          : options.tags,
-      timeoutMs: options.timeoutMs,
-      bearerToken: options.bearerToken,
-      headers: target.headers,
-      verbose: Boolean(options.verbose) || stream,
-      onEvent: progress === undefined ? undefined : (e) => progress.onEvent(e),
+      runId,
+      auth,
+      fetchImpl: options.fetchImpl,
     });
-
-    if (results.length === 0) {
-      process.stderr.write("No matching evals found.\n");
-      return 2;
+    if (options.json) {
+      console.log(JSON.stringify(snap, null, 2));
+    } else {
+      process.stdout.write(`${formatEvalRunSummaryLine(snap)}\n`);
+      if (snap.error !== undefined && snap.error !== "") {
+        process.stdout.write(`error: ${snap.error}\n`);
+      }
+      for (const c of snap.cases) {
+        const mark =
+          c.status !== "done" ? c.status : c.ok === true ? "PASS" : "FAIL";
+        const dur =
+          c.durationMs === undefined
+            ? ""
+            : ` (${(c.durationMs / 1000).toFixed(1)}s)`;
+        process.stdout.write(`  ${mark}  ${c.id}${dur}\n`);
+        if (c.error !== undefined && c.error !== "") {
+          process.stdout.write(`    error: ${c.error}\n`);
+        }
+      }
+    }
+    if (snap.status === "running") {
+      return 3;
     }
-    return printEvalResults(results, options);
+    const failed =
+      snap.status === "failed" || snap.cases.some((c) => c.ok !== true);
+    return failed ? 1 : 0;
+  } catch (error) {
+    process.stderr.write(
+      `${sanitizeCustomerError(
+        error instanceof Error ? error.message : String(error)
+      )}\n`
+    );
+    return 1;
   } finally {
-    progress?.close();
     if (target.close !== undefined) {
       await target.close();
     }
   }
 }
 
+function formatEvalRunSummaryLine(run: EvalRunSnapshot): string {
+  const { passed, failed, total, done } = run.summary;
+  const durable = run.config.durableRuns === true ? " persisted" : "";
+  return `${run.runId}  ${run.status}  ${done}/${total} done (${passed} passed, ${failed} failed)${durable}`;
+}
+
+async function listLocalEvals(
+  projectRoot: string,
+  options: AxCliOptions,
+  filterIds: string[] | undefined,
+  tags: string[] | undefined
+): Promise<number> {
+  const { evals } = await discoverEvals(projectRoot);
+  const listed = filterDiscoveredEvals(evals, { filterIds, tags });
+  return printListedEvals(
+    listed.map((e) => ({
+      id: e.id,
+      fileId: e.fileId,
+      path: e.path,
+      description: e.definition.description,
+      tags: e.definition.tags ?? [],
+    })),
+    options,
+    `No evals under ${projectRoot}/evals`
+  );
+}
+
+function printListedEvals(
+  listed: Array<{
+    id: string;
+    fileId: string;
+    path?: string;
+    description?: string;
+    tags: string[];
+  }>,
+  options: AxCliOptions,
+  emptyMessage: string
+): number {
+  if (options.json) {
+    console.log(JSON.stringify(listed, null, 2));
+  } else if (listed.length === 0) {
+    process.stdout.write(`${emptyMessage}\n`);
+  } else {
+    for (const e of listed) {
+      const tagSuffix = e.tags.length > 0 ? `  [${e.tags.join(", ")}]` : "";
+      const desc = e.description !== undefined ? ` — ${e.description}` : "";
+      process.stdout.write(`${e.id}${tagSuffix}${desc}\n`);
+    }
+  }
+  return 0;
+}
+
+/**
+ * Remote batches don't stream turn events to the CLI; print run/case progress
+ * (and case logs once they appear on the snapshot) instead.
+ */
+function createRemoteEvalProgress(
+  enabled: boolean
+): { onSnapshot: (snap: EvalRunSnapshot) => void } | undefined {
+  if (!enabled) {
+    return undefined;
+  }
+  let announced = false;
+  let lastSummaryKey = "";
+  const seenCaseKeys = new Set<string>();
+  return {
+    onSnapshot(snap) {
+      if (!announced) {
+        announced = true;
+        const durable = snap.config.durableRuns === true ? " (persisted)" : "";
+        process.stderr.write(`eval run: ${snap.runId}${durable}\n`);
+      }
+      const summaryKey = `${snap.status}:${snap.summary.done}:${snap.summary.passed}:${snap.summary.failed}`;
+      if (summaryKey !== lastSummaryKey) {
+        lastSummaryKey = summaryKey;
+        const { done, total, passed, failed } = snap.summary;
+        process.stderr.write(
+          `eval progress: ${done}/${total} done (${passed} passed, ${failed} failed) [${snap.status}]\n`
+        );
+      }
+      for (const c of snap.cases) {
+        if (c.status !== "done") {
+          continue;
+        }
+        const caseKey = `${c.id}:${c.ok === true ? "pass" : "fail"}:${c.durationMs ?? 0}`;
+        if (seenCaseKeys.has(caseKey)) {
+          continue;
+        }
+        seenCaseKeys.add(caseKey);
+        process.stderr.write(
+          `  eval ${c.id}: ${c.ok === true ? "PASS" : "FAIL"}${
+            c.error !== undefined ? ` — ${c.error}` : ""
+          }\n`
+        );
+        for (const line of c.logs ?? []) {
+          process.stderr.write(`    ${line}\n`);
+        }
+      }
+    },
+  };
+}
+
 function printEvalResults(
   results: EvalRunResult[],
-  options: AxCliOptions
+  options: AxCliOptions,
+  batch?: {
+    runId?: string;
+    status?: EvalRunSnapshot["status"];
+    error?: string;
+  }
 ): number {
   const passed = results.filter((r) => r.ok).length;
   const failed = results.length - passed;
+  const batchFailed = batch?.status === "failed";
+  const ok = failed === 0 && !batchFailed;
+
+  if (batch?.error !== undefined && batch.error !== "" && !options.json) {
+    // Always surface harness-level failures (not only --verbose).
+    process.stderr.write(`eval run error: ${batch.error}\n`);
+  }
 
   if (options.json) {
     console.log(
-      JSON.stringify({ ok: failed === 0, passed, failed, results }, null, 2)
+      JSON.stringify(
+        {
+          ok,
+          passed,
+          failed,
+          results,
+          ...(batch?.runId === undefined ? {} : { runId: batch.runId }),
+          ...(batch?.status === undefined ? {} : { status: batch.status }),
+          ...(batch?.error === undefined || batch.error === ""
+            ? {}
+            : { error: batch.error }),
+        },
+        null,
+        2
+      )
     );
   } else {
     for (const r of results) {
@@ -861,7 +1252,7 @@ function printEvalResults(
       `\n${passed} passed, ${failed} failed, ${results.length} total\n`
     );
   }
-  return failed === 0 ? 0 : 1;
+  return ok ? 0 : 1;
 }
 
 /**

package/src/internal/eval-run-store.ts

@@ -47,6 +47,8 @@ export interface StartEvalRunInput {
   tags?: string[];
   bearerToken?: string;
   verbose?: boolean;
+  /** Per-case timeout override (same semantics as CLI `--timeout-ms`). */
+  timeoutMs?: number;
 }
 
 export type EvalListConfig = {
@@ -190,6 +192,7 @@ export class EvalRunStore {
         ? {}
         : { bearerToken: input.bearerToken }),
       ...(input.verbose === undefined ? {} : { verbose: input.verbose }),
+      ...(input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs }),
     });
 
     return snapshot;
@@ -316,6 +319,7 @@ export class EvalRunStore {
       config: Partial<EvalConfigFile>;
       bearerToken?: string;
       verbose?: boolean;
+      timeoutMs?: number;
     }
   ): Promise<void> {
     if (this.runs.get(runId) === undefined) {
@@ -330,6 +334,7 @@ export class EvalRunStore {
           ? {}
           : { bearerToken: opts.bearerToken }),
         ...(opts.verbose === undefined ? {} : { verbose: opts.verbose }),
+        ...(opts.timeoutMs === undefined ? {} : { timeoutMs: opts.timeoutMs }),
         onCaseStart: (id) => {
           const run = this.runs.get(runId);
           if (run === undefined) {