@cursor/july 0.1.5 → 0.1.6

package/src/internal/cli-ax.ts

@@ -35,7 +35,12 @@ import { offerInstallCursorSkills } from "./install-cursor-skills.js";
 import { runLogsTail, sanitizeCustomerError } from "./logs-client.js";
 import { openBrowser } from "./open-browser.js";
 import { startPlaygroundProxy } from "./playground-proxy.js";
-import { resolveProdSlug, resolveProdTarget } from "./resolve-prod-target.js";
+import {
+  ProdTargetError,
+  resolveProdAliasPlaygroundUrl,
+  resolveProdSlug,
+  resolveProdTarget,
+} from "./resolve-prod-target.js";
 import { runSession } from "./run-client.js";
 import {
   fetchSessionEvents,
@@ -92,8 +97,15 @@ export interface AxCliOptions {
    * Route chat/resume/run/call/eval/logs/sessions at the team's hosted
    * deployment
    * instead of a local ephemeral server. Mutually exclusive with `--url`.
+   * `eval` targets the deployment by default, so there `--prod` is just
+   * being explicit.
    */
   prod?: boolean;
+  /**
+   * `eval`: run against an ephemeral local server instead of the hosted
+   * deployment (the eval default). Mutually exclusive with `--url`/`--prod`.
+   */
+  local?: boolean;
   /** Cursor team id for `--prod` (defaults to the account's team). */
   team?: string;
   /** Cursor API key override for `--prod` management calls. */
@@ -654,6 +666,79 @@ export async function cmdTrajectory(options: AxCliOptions): Promise<number> {
   return trajectory.ok ? 0 : 1;
 }
 
+/**
+ * Which surface `eval` should target. Deployment is the default; `--local`
+ * opts into an ephemeral local server, `--url` into a running one.
+ * Exported for tests.
+ */
+export function resolveEvalTargetMode(
+  options: Pick<AxCliOptions, "local" | "prod" | "url">
+): "local" | "url" | "deployment" | { error: string } {
+  const flags = [
+    options.local === true ? "--local" : undefined,
+    options.prod === true ? "--prod" : undefined,
+    options.url !== undefined ? "--url" : undefined,
+  ].filter((flag) => flag !== undefined);
+  if (flags.length > 1) {
+    return { error: `${flags.join(" and ")} are mutually exclusive` };
+  }
+  if (options.local === true) {
+    return "local";
+  }
+  if (options.url !== undefined) {
+    return "url";
+  }
+  return "deployment";
+}
+
+/** Resolve the eval target; prints a CLI-ready error and returns an exit code on failure. */
+async function resolveEvalTarget(
+  options: AxCliOptions
+): Promise<ResolvedAgentTarget | number> {
+  const mode = resolveEvalTargetMode(options);
+  if (typeof mode === "object") {
+    process.stderr.write(`${mode.error}\n`);
+    return 2;
+  }
+  if (mode === "url") {
+    return { agentUrl: options.url!.replace(/\/$/, "") };
+  }
+  if (mode === "local") {
+    return startEphemeral(options);
+  }
+  try {
+    const prod = await resolveProdTarget({
+      dir: options.dir,
+      cwd: options.cwd,
+      slug: options.slug,
+      team: options.team,
+      apiKey: options.apiKey,
+      fetchImpl: options.fetchImpl,
+    });
+    return { agentUrl: prod.agentUrl, headers: prod.headers };
+  } catch (error) {
+    if (error instanceof ProdTargetError && error.notFound) {
+      // Keep the API error text: a 404 can also mean a wrong API host or a
+      // disabled feature gate, and the message distinguishes those from a
+      // genuinely missing deployment.
+      process.stderr.write(
+        `${sanitizeCustomerError(error.message)}\n` +
+          `Evals run on the hosted deployment by default.\n` +
+          `  deploy it first:  ${CLI_COMMAND_NAME} deploy --slug ${error.slug}\n` +
+          `  or run locally:   ${CLI_COMMAND_NAME} eval --local --dir ${options.dir}\n`
+      );
+      return 1;
+    }
+    process.stderr.write(
+      `${sanitizeCustomerError(
+        error instanceof Error ? error.message : String(error)
+      )}\n` +
+        `Evals run on the hosted deployment by default; pass --local to run on an ephemeral local server.\n`
+    );
+    return 1;
+  }
+}
+
 export async function cmdEval(options: AxCliOptions): Promise<number> {
   const projectRoot = resolve(options.cwd ?? process.cwd(), options.dir);
 
@@ -695,7 +780,10 @@ export async function cmdEval(options: AxCliOptions): Promise<number> {
     return 0;
   }
 
-  const target = await resolveTarget(options);
+  const target = await resolveEvalTarget(options);
+  if (typeof target === "number") {
+    return target;
+  }
   const stream = shouldStreamProgress(options);
   const progress = stream ? createStreamProgress() : undefined;
   try {
@@ -991,9 +1079,9 @@ export async function cmdSession(
 /**
  * `playground` — open the web UI against local / `--url` / `--prod`.
  *
- * Hosted engines need short-lived access headers a browser cannot set, so
- * `--prod` (and `--bearer-token`) run a loopback reverse proxy that injects
- * them. Ctrl-C stops the proxy.
+ * `--prod` opens the stable hosted alias playground URL; the SPA collects
+ * the alias token. `--bearer-token` (and other injected headers) still run a
+ * loopback reverse proxy — Ctrl-C stops that proxy.
  */
 export async function cmdPlayground(options: AxCliOptions): Promise<number> {
   if (options.prod === true && options.url !== undefined) {
@@ -1006,17 +1094,18 @@ export async function cmdPlayground(options: AxCliOptions): Promise<number> {
       ? options.sessionId.trim()
       : undefined;
 
-  // Hosted: mint engineAccess and refresh it for the life of the proxy.
+  // Hosted: open the alias playground; SPA owns alias-token sign-in.
   if (options.prod === true) {
-    let initial: Awaited<ReturnType<typeof resolveProdTarget>>;
+    let resolved: Awaited<ReturnType<typeof resolveProdAliasPlaygroundUrl>>;
     try {
-      initial = await resolveProdTarget({
+      resolved = await resolveProdAliasPlaygroundUrl({
         dir: options.dir,
         cwd: options.cwd,
         slug: options.slug,
         team: options.team,
         apiKey: options.apiKey,
         fetchImpl: options.fetchImpl,
+        sessionId,
       });
     } catch (error) {
       process.stderr.write(
@@ -1027,44 +1116,11 @@ export async function cmdPlayground(options: AxCliOptions): Promise<number> {
       return 1;
     }
 
-    let proxy: Awaited<ReturnType<typeof startPlaygroundProxy>>;
-    try {
-      proxy = await startPlaygroundProxy({
-        upstreamUrl: initial.agentUrl,
-        headers: initial.headers,
-        expiresAt: initial.expiresAt,
-        refreshUpstream: async () => {
-          const next = await resolveProdTarget({
-            dir: options.dir,
-            cwd: options.cwd,
-            slug: options.slug ?? initial.slug,
-            team: options.team,
-            apiKey: options.apiKey,
-            fetchImpl: options.fetchImpl,
-            err: () => {},
-          });
-          return {
-            upstreamUrl: next.agentUrl,
-            headers: next.headers,
-            expiresAt: next.expiresAt,
-          };
-        },
-      });
-    } catch (error) {
-      process.stderr.write(
-        `${sanitizeCustomerError(
-          error instanceof Error ? error.message : String(error)
-        )}\n`
-      );
-      return 1;
+    process.stdout.write(`${resolved.playgroundUrl}\n`);
+    if (options.print !== true) {
+      openBrowser(resolved.playgroundUrl);
     }
-
-    return await runPlaygroundProxySession({
-      proxy,
-      sessionId,
-      print: options.print === true,
-      signal: options.signal,
-    });
+    return 0;
   }
 
   const target = await resolveSessionTarget(options);

package/src/internal/cli-deploy.ts

@@ -675,7 +675,7 @@ async function deployOne(
       `  kind:        ${detail.deploymentKind}\n` +
       `  alias:       ${aliasUrl}\n` +
       `  playground:  ${aliasUrl}/playground (sign in with the alias token)\n` +
-      `               or: ${CLI} playground --prod --slug ${slug} --team ${ctx.teamId}\n`
+      `               open: ${CLI} playground --prod --slug ${slug} --team ${ctx.teamId}\n`
   );
   return 0;
 }

package/src/internal/discovery.ts

@@ -1611,8 +1611,8 @@ export function projectInfo(project: AgentProject): AgentProjectInfo {
             maxPlaygroundSessions: resolveABMaxPlaygroundSessions(
               project.abConfig?.maxPlaygroundSessions
             ),
-            durableSamples: project.abConfig?.persistSamples !== undefined,
-            durableSnapshots: project.abConfig?.persistSnapshots !== undefined,
+            durableSamples: project.storage?.abs !== undefined,
+            durableSnapshots: project.storage?.abs?.putSnapshot !== undefined,
           },
         }),
     storage:

package/src/internal/eval-run-store.ts

@@ -1,8 +1,9 @@
 /**
  * Store for playground-triggered eval batches.
  *
- * Default: process memory only, capped by `maxPlaygroundRuns` (see `defineEvalConfig`).
- * Optional durable `persistRuns` from `evals/evals.config.ts` survives restart.
+ * Default: process memory only, capped by `maxPlaygroundRuns` (see
+ * `defineEvalConfig`). When `agent/storage.ts` (`defineStorage`) declares an
+ * `evals` table, runs are mirrored there and survive restart.
  * Runs against the live mount's loopback URL so case sessions appear in
  * the playground session list.
  */
@@ -11,7 +12,6 @@ import { randomUUID } from "node:crypto";
 import {
   type EvalCaseSnapshot,
   type EvalConfigFile,
-  type EvalRunPersistence,
   type EvalRunSnapshot,
   resolveEvalMaxPlaygroundRuns,
 } from "../evals.js";
@@ -30,6 +30,18 @@ export type {
   EvalRunSnapshot,
 } from "../evals.js";
 
+/**
+ * Durable mirror for playground eval batches — the `StorageCoordinator`
+ * adapter over the `defineStorage` `evals` table. Writes are
+ * fire-and-forget (the coordinator serializes and isolates failures);
+ * `list` is the serve-start hydrate read.
+ */
+export interface EvalRunStorage {
+  save(run: EvalRunSnapshot): void;
+  delete(runId: string): void;
+  list(): Promise<EvalRunSnapshot[]>;
+}
+
 export interface StartEvalRunInput {
   filterIds?: string[];
   tags?: string[];
@@ -41,7 +53,7 @@ export type EvalListConfig = {
   maxConcurrency?: number;
   timeoutMs?: number;
   maxPlaygroundRuns: number;
-  /** True when `evals.config.ts` set `persistRuns`. */
+  /** True when `agent/storage.ts` declares an `evals` table. */
   durableRuns: boolean;
 };
 
@@ -55,28 +67,16 @@ export class EvalRunStore {
   private activeRunId: string | undefined;
   private hydratePromise: Promise<void> | undefined;
   private hydrated = false;
-  /** Per-run persist chain so intermediate saves cannot overwrite a later final write. */
-  private readonly persistChains = new Map<string, Promise<void>>();
   /** Filled after listen — e.g. `http://127.0.0.1:3000/weather-agent`. */
   loopbackUrl: string | undefined;
 
   constructor(
     private readonly projectRoot: string,
     private readonly logger: (line: string) => void,
-    /**
-     * Project-wide sink from `agent/storage.ts` (`defineStorage`),
-     * used when `evals.config.ts` sets no `persistRuns` of its own.
-     */
-    private readonly fallbackPersistence?: EvalRunPersistence
+    /** `defineStorage` evals-table adapter; undefined ⇒ memory only. */
+    private readonly storage?: EvalRunStorage | undefined
   ) {}
 
-  /** `evals.config.ts` `persistRuns` wins; `defineStorage` is fallback. */
-  private resolvePersist(
-    config: Partial<EvalConfigFile>
-  ): EvalRunPersistence | undefined {
-    return config.persistRuns ?? this.fallbackPersistence;
-  }
-
   async list(): Promise<{
     evals: Array<{
       id: string;
@@ -97,10 +97,7 @@ export class EvalRunStore {
           : { description: e.definition.description }),
         ...(e.definition.tags === undefined ? {} : { tags: e.definition.tags }),
       })),
-      config: snapshotListConfig(
-        config,
-        this.resolvePersist(config) !== undefined
-      ),
+      config: snapshotListConfig(config, this.storage !== undefined),
     };
   }
 
@@ -175,15 +172,12 @@ export class EvalRunStore {
         done: 0,
       },
       cases,
-      config: snapshotRunConfig(
-        config,
-        this.resolvePersist(config) !== undefined
-      ),
+      config: snapshotRunConfig(config, this.storage !== undefined),
     };
     this.runs.set(runId, snapshot);
     this.activeRunId = runId;
-    await this.pruneOldRuns(config);
-    await this.enqueuePersist(snapshot, this.resolvePersist(config));
+    this.pruneOldRuns(config);
+    this.persist(snapshot);
     this.logger(
       `[agentkit] eval run ${runId}: starting ${cases.length} case(s)`
     );
@@ -208,39 +202,28 @@ export class EvalRunStore {
   }
 
   /**
-   * Hydrate playground eval history from durable persistence
-   * (`evals.config.ts` `persistRuns`, else the `defineStorage`
-   * fallback). Called eagerly from the serve startup restore step —
-   * alongside session restore — and lazily as a safety net by every read.
-   * Idempotent; concurrent callers share one pass.
+   * Hydrate playground eval history from the `defineStorage` evals table.
+   * Called eagerly from the serve startup restore step — alongside session
+   * restore — and lazily as a safety net by every read. Idempotent;
+   * concurrent callers share one pass.
    */
   async hydrate(): Promise<void> {
     if (this.hydrated) {
       return;
     }
     if (this.hydratePromise === undefined) {
-      this.hydratePromise = this.hydrateFromPersistence();
+      this.hydratePromise = this.hydrateFromStorage();
     }
     await this.hydratePromise;
   }
 
-  private async hydrateFromPersistence(): Promise<void> {
+  private async hydrateFromStorage(): Promise<void> {
     try {
-      const { config } = await discoverEvals(this.projectRoot);
-      const persist = this.resolvePersist(config);
-      if (persist === undefined) {
-        this.hydrated = true;
-        return;
-      }
-      const loaded = await persist.load({ projectRoot: this.projectRoot });
-      if (!Array.isArray(loaded)) {
-        this.logger(
-          "[agentkit] eval persistRuns.load() did not return an array; ignoring"
-        );
+      if (this.storage === undefined) {
         this.hydrated = true;
         return;
       }
-      const repaired: EvalRunSnapshot[] = [];
+      const loaded = await this.storage.list();
       for (const run of loaded) {
         if (
           run === null ||
@@ -253,16 +236,26 @@ export class EvalRunStore {
         // Incomplete batches from a killed process should not block a new run.
         if (run.status === "running") {
           markInterruptedRun(run);
-          repaired.push(run);
+          this.persist(run);
         }
         this.runs.set(run.runId, run);
       }
-      for (const run of repaired) {
-        await this.enqueuePersist(run, persist);
+      // Prune with the defaults even when eval discovery fails — loaded
+      // history must not stay unpruned for the process lifetime just
+      // because an eval file has a syntax error at boot.
+      let config: Partial<EvalConfigFile> = {};
+      try {
+        config = (await discoverEvals(this.projectRoot)).config;
+      } catch (error) {
+        this.logger(
+          `[agentkit] eval discovery failed during hydrate (pruning with defaults): ${
+            error instanceof Error ? error.message : String(error)
+          }`
+        );
       }
-      await this.pruneOldRuns(config);
+      this.pruneOldRuns(config);
       this.logger(
-        `[agentkit] hydrated ${this.runs.size} playground eval run(s) from persistRuns`
+        `[agentkit] hydrated ${this.runs.size} playground eval run(s) from storage`
       );
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
@@ -274,7 +267,7 @@ export class EvalRunStore {
     }
   }
 
-  private async pruneOldRuns(config: Partial<EvalConfigFile>): Promise<void> {
+  private pruneOldRuns(config: Partial<EvalConfigFile>): void {
     const limit = resolveEvalMaxPlaygroundRuns(config.maxPlaygroundRuns);
     const ordered = this.sortedRuns();
     if (ordered.length <= limit) {
@@ -285,58 +278,34 @@ export class EvalRunStore {
         continue;
       }
       this.runs.delete(stale.runId);
-      await this.persistDelete(stale.runId, this.resolvePersist(config));
-    }
-  }
-
-  /**
-   * Queue a durable save for `run`. Snapshots are cloned at enqueue time and
-   * writes for the same runId run strictly in order so an intermediate
-   * fire-and-forget save cannot finish after (and overwrite) a later final save.
-   */
-  private enqueuePersist(
-    run: EvalRunSnapshot,
-    persist: EvalRunPersistence | undefined
-  ): Promise<void> {
-    if (persist === undefined) {
-      return Promise.resolve();
+      this.storage?.delete(stale.runId);
     }
-    const snapshot = structuredClone(run);
-    const prev = this.persistChains.get(run.runId) ?? Promise.resolve();
-    const next = prev
-      .catch(() => undefined)
-      .then(() => this.persistSave(snapshot, persist));
-    this.persistChains.set(run.runId, next);
-    return next;
   }
 
-  private async persistSave(
-    run: EvalRunSnapshot,
-    persist: EvalRunPersistence
-  ): Promise<void> {
-    try {
-      await persist.save(run, { projectRoot: this.projectRoot });
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      this.logger(
-        `[agentkit] persistRuns.save(${run.runId}) failed: ${message}`
-      );
-    }
+  /** Mirror the run onto durable storage (coordinator clones + serializes). */
+  private persist(run: EvalRunSnapshot): void {
+    this.storage?.save(run);
   }
 
-  private async persistDelete(
-    runId: string,
-    persist: EvalRunPersistence | undefined
-  ): Promise<void> {
-    if (persist === undefined) {
+  /**
+   * Serve is draining: durably mark any in-flight batch as interrupted
+   * while the storage queue can still deliver. Late `execute` writes are
+   * dropped by the closed coordinator, so without this a batch that was
+   * running at shutdown would look "running" (or silently lose its final
+   * completion write) until the next hydrate repairs it.
+   */
+  close(): void {
+    const activeRunId = this.activeRunId;
+    if (activeRunId === undefined) {
       return;
     }
-    try {
-      await persist.delete(runId, { projectRoot: this.projectRoot });
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      this.logger(`[agentkit] persistRuns.delete(${runId}) failed: ${message}`);
+    const run = this.runs.get(activeRunId);
+    if (run === undefined || run.status !== "running") {
+      return;
     }
+    markInterruptedRun(run);
+    this.persist(run);
+    this.activeRunId = undefined;
   }
 
   private async execute(
@@ -408,7 +377,7 @@ export class EvalRunStore {
           this.applyCaseResult(runId, result);
           const run = this.runs.get(runId);
           if (run !== undefined) {
-            void this.enqueuePersist(run, this.resolvePersist(opts.config));
+            this.persist(run);
           }
         },
       });
@@ -418,7 +387,7 @@ export class EvalRunStore {
       }
       run.status = "completed";
       run.finishedAt = new Date().toISOString();
-      await this.enqueuePersist(run, this.resolvePersist(opts.config));
+      this.persist(run);
       this.logger(
         `[agentkit] eval run ${runId}: done — ${run.summary.passed} passed, ${run.summary.failed} failed`
       );
@@ -430,7 +399,7 @@ export class EvalRunStore {
       run.status = "failed";
       run.finishedAt = new Date().toISOString();
       run.error = error instanceof Error ? error.message : String(error);
-      await this.enqueuePersist(run, this.resolvePersist(opts.config));
+      this.persist(run);
       this.logger(`[agentkit] eval run ${runId}: failed — ${run.error}`);
     } finally {
       if (this.activeRunId === runId) {
@@ -489,6 +458,23 @@ function markInterruptedRun(run: EvalRunSnapshot): void {
       c.error = c.error ?? INTERRUPTED_CASE_ERROR;
     }
   }
+  // Reconcile the summary with the repaired case rows so a persisted
+  // interrupted batch never claims pre-interruption pass counts.
+  let passed = 0;
+  let failed = 0;
+  for (const c of run.cases) {
+    if (c.ok === true) {
+      passed += 1;
+    } else {
+      failed += 1;
+    }
+  }
+  run.summary = {
+    passed,
+    failed,
+    total: run.cases.length,
+    done: run.cases.length,
+  };
 }
 
 function snapshotListConfig(

package/src/internal/init-project.ts

@@ -433,14 +433,15 @@ Recommended loop once the project exists:
   1. Edit agent/ + tools under agent/tools/
   2. ${CLI_COMMAND_NAME} info --dir . --json
   3. ${CLI_COMMAND_NAME} run --dir . --message "..."
-  4. ${CLI_COMMAND_NAME} eval --dir .
+  4. ${CLI_COMMAND_NAME} eval --local --dir .
 
 Evals:
   Create evals/**/*.eval.ts using defineEval from ${PACKAGE_NAME}/evals
   Case id is the path: evals/weather/nyc.eval.ts → weather/nyc
   Required: evals/evals.config.ts with defineEvalConfig({ maxConcurrency })
     (hard-capped at 200 due to model provider request limits)
-  Run: ${CLI_COMMAND_NAME} eval [--json] [evalId...]
+  Run: ${CLI_COMMAND_NAME} eval --local [--json] [evalId...]
+    (omit --local to run against the hosted deployment)
 
 See AGENTS.md in the ${PACKAGE_NAME} package for the full loop.
 `);

package/src/internal/playground-proxy.ts

@@ -14,9 +14,9 @@
  * request-targets are rejected so injected headers never leave the configured
  * upstream.
  *
- * When `refreshUpstream` is provided (hosted `--prod`), credentials are
- * re-minted shortly before `expiresAt` and once on upstream 401 so a long
- * playground session does not require restarting the CLI.
+ * When `refreshUpstream` is provided (e.g. short-lived upstream tokens),
+ * credentials are re-minted shortly before `expiresAt` and once on upstream
+ * 401 so a long playground session does not require restarting the CLI.
  */
 
 import { randomBytes } from "node:crypto";
@@ -84,8 +84,8 @@ export interface StartPlaygroundProxyArgs {
   /** Initial engineAccess expiry (ISO); enables proactive refresh. */
   expiresAt?: string;
   /**
-   * Re-mint upstream URL + headers (hosted `--prod`). Single-flighted across
-   * concurrent browser requests; called near expiry and once on 401.
+   * Re-mint upstream URL + headers. Single-flighted across concurrent browser
+   * requests; called near expiry and once on 401.
    */
   refreshUpstream?: () => Promise<PlaygroundUpstreamAccess>;
   /** Bind port; `0` = ephemeral. */