@cursor/july 0.1.4 → 0.1.6

package/src/internal/eval-run-store.ts

@@ -1,8 +1,9 @@
 /**
  * Store for playground-triggered eval batches.
  *
- * Default: process memory only, capped by `maxPlaygroundRuns` (see `defineEvalConfig`).
- * Optional durable `persistRuns` from `evals/evals.config.ts` survives restart.
+ * Default: process memory only, capped by `maxPlaygroundRuns` (see
+ * `defineEvalConfig`). When `agent/storage.ts` (`defineStorage`) declares an
+ * `evals` table, runs are mirrored there and survive restart.
  * Runs against the live mount's loopback URL so case sessions appear in
  * the playground session list.
  */
@@ -11,7 +12,6 @@ import { randomUUID } from "node:crypto";
 import {
   type EvalCaseSnapshot,
   type EvalConfigFile,
-  type EvalRunPersistence,
   type EvalRunSnapshot,
   resolveEvalMaxPlaygroundRuns,
 } from "../evals.js";
@@ -30,6 +30,18 @@ export type {
   EvalRunSnapshot,
 } from "../evals.js";
 
+/**
+ * Durable mirror for playground eval batches — the `StorageCoordinator`
+ * adapter over the `defineStorage` `evals` table. Writes are
+ * fire-and-forget (the coordinator serializes and isolates failures);
+ * `list` is the serve-start hydrate read.
+ */
+export interface EvalRunStorage {
+  save(run: EvalRunSnapshot): void;
+  delete(runId: string): void;
+  list(): Promise<EvalRunSnapshot[]>;
+}
+
 export interface StartEvalRunInput {
   filterIds?: string[];
   tags?: string[];
@@ -41,7 +53,7 @@ export type EvalListConfig = {
   maxConcurrency?: number;
   timeoutMs?: number;
   maxPlaygroundRuns: number;
-  /** True when `evals.config.ts` set `persistRuns`. */
+  /** True when `agent/storage.ts` declares an `evals` table. */
   durableRuns: boolean;
 };
 
@@ -55,28 +67,16 @@ export class EvalRunStore {
   private activeRunId: string | undefined;
   private hydratePromise: Promise<void> | undefined;
   private hydrated = false;
-  /** Per-run persist chain so intermediate saves cannot overwrite a later final write. */
-  private readonly persistChains = new Map<string, Promise<void>>();
   /** Filled after listen — e.g. `http://127.0.0.1:3000/weather-agent`. */
   loopbackUrl: string | undefined;
 
   constructor(
     private readonly projectRoot: string,
     private readonly logger: (line: string) => void,
-    /**
-     * Project-wide sink from `agent/storage.ts` (`defineStorage`),
-     * used when `evals.config.ts` sets no `persistRuns` of its own.
-     */
-    private readonly fallbackPersistence?: EvalRunPersistence
+    /** `defineStorage` evals-table adapter; undefined ⇒ memory only. */
+    private readonly storage?: EvalRunStorage | undefined
   ) {}
 
-  /** `evals.config.ts` `persistRuns` wins; `defineStorage` is fallback. */
-  private resolvePersist(
-    config: Partial<EvalConfigFile>
-  ): EvalRunPersistence | undefined {
-    return config.persistRuns ?? this.fallbackPersistence;
-  }
-
   async list(): Promise<{
     evals: Array<{
       id: string;
@@ -97,10 +97,7 @@ export class EvalRunStore {
           : { description: e.definition.description }),
         ...(e.definition.tags === undefined ? {} : { tags: e.definition.tags }),
       })),
-      config: snapshotListConfig(
-        config,
-        this.resolvePersist(config) !== undefined
-      ),
+      config: snapshotListConfig(config, this.storage !== undefined),
     };
   }
 
@@ -175,15 +172,12 @@ export class EvalRunStore {
         done: 0,
       },
       cases,
-      config: snapshotRunConfig(
-        config,
-        this.resolvePersist(config) !== undefined
-      ),
+      config: snapshotRunConfig(config, this.storage !== undefined),
     };
     this.runs.set(runId, snapshot);
     this.activeRunId = runId;
-    await this.pruneOldRuns(config);
-    await this.enqueuePersist(snapshot, this.resolvePersist(config));
+    this.pruneOldRuns(config);
+    this.persist(snapshot);
     this.logger(
       `[agentkit] eval run ${runId}: starting ${cases.length} case(s)`
     );
@@ -208,39 +202,28 @@ export class EvalRunStore {
   }
 
   /**
-   * Hydrate playground eval history from durable persistence
-   * (`evals.config.ts` `persistRuns`, else the `defineStorage`
-   * fallback). Called eagerly from the serve startup restore step —
-   * alongside session restore — and lazily as a safety net by every read.
-   * Idempotent; concurrent callers share one pass.
+   * Hydrate playground eval history from the `defineStorage` evals table.
+   * Called eagerly from the serve startup restore step — alongside session
+   * restore — and lazily as a safety net by every read. Idempotent;
+   * concurrent callers share one pass.
    */
   async hydrate(): Promise<void> {
     if (this.hydrated) {
       return;
     }
     if (this.hydratePromise === undefined) {
-      this.hydratePromise = this.hydrateFromPersistence();
+      this.hydratePromise = this.hydrateFromStorage();
     }
     await this.hydratePromise;
   }
 
-  private async hydrateFromPersistence(): Promise<void> {
+  private async hydrateFromStorage(): Promise<void> {
     try {
-      const { config } = await discoverEvals(this.projectRoot);
-      const persist = this.resolvePersist(config);
-      if (persist === undefined) {
-        this.hydrated = true;
-        return;
-      }
-      const loaded = await persist.load({ projectRoot: this.projectRoot });
-      if (!Array.isArray(loaded)) {
-        this.logger(
-          "[agentkit] eval persistRuns.load() did not return an array; ignoring"
-        );
+      if (this.storage === undefined) {
         this.hydrated = true;
         return;
       }
-      const repaired: EvalRunSnapshot[] = [];
+      const loaded = await this.storage.list();
       for (const run of loaded) {
         if (
           run === null ||
@@ -253,16 +236,26 @@ export class EvalRunStore {
         // Incomplete batches from a killed process should not block a new run.
         if (run.status === "running") {
           markInterruptedRun(run);
-          repaired.push(run);
+          this.persist(run);
         }
         this.runs.set(run.runId, run);
       }
-      for (const run of repaired) {
-        await this.enqueuePersist(run, persist);
+      // Prune with the defaults even when eval discovery fails — loaded
+      // history must not stay unpruned for the process lifetime just
+      // because an eval file has a syntax error at boot.
+      let config: Partial<EvalConfigFile> = {};
+      try {
+        config = (await discoverEvals(this.projectRoot)).config;
+      } catch (error) {
+        this.logger(
+          `[agentkit] eval discovery failed during hydrate (pruning with defaults): ${
+            error instanceof Error ? error.message : String(error)
+          }`
+        );
       }
-      await this.pruneOldRuns(config);
+      this.pruneOldRuns(config);
       this.logger(
-        `[agentkit] hydrated ${this.runs.size} playground eval run(s) from persistRuns`
+        `[agentkit] hydrated ${this.runs.size} playground eval run(s) from storage`
       );
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
@@ -274,7 +267,7 @@ export class EvalRunStore {
     }
   }
 
-  private async pruneOldRuns(config: Partial<EvalConfigFile>): Promise<void> {
+  private pruneOldRuns(config: Partial<EvalConfigFile>): void {
     const limit = resolveEvalMaxPlaygroundRuns(config.maxPlaygroundRuns);
     const ordered = this.sortedRuns();
     if (ordered.length <= limit) {
@@ -285,58 +278,34 @@ export class EvalRunStore {
         continue;
       }
       this.runs.delete(stale.runId);
-      await this.persistDelete(stale.runId, this.resolvePersist(config));
-    }
-  }
-
-  /**
-   * Queue a durable save for `run`. Snapshots are cloned at enqueue time and
-   * writes for the same runId run strictly in order so an intermediate
-   * fire-and-forget save cannot finish after (and overwrite) a later final save.
-   */
-  private enqueuePersist(
-    run: EvalRunSnapshot,
-    persist: EvalRunPersistence | undefined
-  ): Promise<void> {
-    if (persist === undefined) {
-      return Promise.resolve();
+      this.storage?.delete(stale.runId);
     }
-    const snapshot = structuredClone(run);
-    const prev = this.persistChains.get(run.runId) ?? Promise.resolve();
-    const next = prev
-      .catch(() => undefined)
-      .then(() => this.persistSave(snapshot, persist));
-    this.persistChains.set(run.runId, next);
-    return next;
   }
 
-  private async persistSave(
-    run: EvalRunSnapshot,
-    persist: EvalRunPersistence
-  ): Promise<void> {
-    try {
-      await persist.save(run, { projectRoot: this.projectRoot });
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      this.logger(
-        `[agentkit] persistRuns.save(${run.runId}) failed: ${message}`
-      );
-    }
+  /** Mirror the run onto durable storage (coordinator clones + serializes). */
+  private persist(run: EvalRunSnapshot): void {
+    this.storage?.save(run);
   }
 
-  private async persistDelete(
-    runId: string,
-    persist: EvalRunPersistence | undefined
-  ): Promise<void> {
-    if (persist === undefined) {
+  /**
+   * Serve is draining: durably mark any in-flight batch as interrupted
+   * while the storage queue can still deliver. Late `execute` writes are
+   * dropped by the closed coordinator, so without this a batch that was
+   * running at shutdown would look "running" (or silently lose its final
+   * completion write) until the next hydrate repairs it.
+   */
+  close(): void {
+    const activeRunId = this.activeRunId;
+    if (activeRunId === undefined) {
       return;
     }
-    try {
-      await persist.delete(runId, { projectRoot: this.projectRoot });
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      this.logger(`[agentkit] persistRuns.delete(${runId}) failed: ${message}`);
+    const run = this.runs.get(activeRunId);
+    if (run === undefined || run.status !== "running") {
+      return;
     }
+    markInterruptedRun(run);
+    this.persist(run);
+    this.activeRunId = undefined;
   }
 
   private async execute(
@@ -408,7 +377,7 @@ export class EvalRunStore {
           this.applyCaseResult(runId, result);
           const run = this.runs.get(runId);
           if (run !== undefined) {
-            void this.enqueuePersist(run, this.resolvePersist(opts.config));
+            this.persist(run);
           }
         },
       });
@@ -418,7 +387,7 @@ export class EvalRunStore {
       }
       run.status = "completed";
       run.finishedAt = new Date().toISOString();
-      await this.enqueuePersist(run, this.resolvePersist(opts.config));
+      this.persist(run);
       this.logger(
         `[agentkit] eval run ${runId}: done — ${run.summary.passed} passed, ${run.summary.failed} failed`
       );
@@ -430,7 +399,7 @@ export class EvalRunStore {
       run.status = "failed";
       run.finishedAt = new Date().toISOString();
       run.error = error instanceof Error ? error.message : String(error);
-      await this.enqueuePersist(run, this.resolvePersist(opts.config));
+      this.persist(run);
       this.logger(`[agentkit] eval run ${runId}: failed — ${run.error}`);
     } finally {
       if (this.activeRunId === runId) {
@@ -489,6 +458,23 @@ function markInterruptedRun(run: EvalRunSnapshot): void {
       c.error = c.error ?? INTERRUPTED_CASE_ERROR;
     }
   }
+  // Reconcile the summary with the repaired case rows so a persisted
+  // interrupted batch never claims pre-interruption pass counts.
+  let passed = 0;
+  let failed = 0;
+  for (const c of run.cases) {
+    if (c.ok === true) {
+      passed += 1;
+    } else {
+      failed += 1;
+    }
+  }
+  run.summary = {
+    passed,
+    failed,
+    total: run.cases.length,
+    done: run.cases.length,
+  };
 }
 
 function snapshotListConfig(

package/src/internal/init-project.ts

@@ -433,14 +433,15 @@ Recommended loop once the project exists:
   1. Edit agent/ + tools under agent/tools/
   2. ${CLI_COMMAND_NAME} info --dir . --json
   3. ${CLI_COMMAND_NAME} run --dir . --message "..."
-  4. ${CLI_COMMAND_NAME} eval --dir .
+  4. ${CLI_COMMAND_NAME} eval --local --dir .
 
 Evals:
   Create evals/**/*.eval.ts using defineEval from ${PACKAGE_NAME}/evals
   Case id is the path: evals/weather/nyc.eval.ts → weather/nyc
   Required: evals/evals.config.ts with defineEvalConfig({ maxConcurrency })
     (hard-capped at 200 due to model provider request limits)
-  Run: ${CLI_COMMAND_NAME} eval [--json] [evalId...]
+  Run: ${CLI_COMMAND_NAME} eval --local [--json] [evalId...]
+    (omit --local to run against the hosted deployment)
 
 See AGENTS.md in the ${PACKAGE_NAME} package for the full loop.
 `);

package/src/internal/playground-proxy.ts

@@ -14,9 +14,9 @@
  * request-targets are rejected so injected headers never leave the configured
  * upstream.
  *
- * When `refreshUpstream` is provided (hosted `--prod`), credentials are
- * re-minted shortly before `expiresAt` and once on upstream 401 so a long
- * playground session does not require restarting the CLI.
+ * When `refreshUpstream` is provided (e.g. short-lived upstream tokens),
+ * credentials are re-minted shortly before `expiresAt` and once on upstream
+ * 401 so a long playground session does not require restarting the CLI.
  */
 
 import { randomBytes } from "node:crypto";
@@ -84,8 +84,8 @@ export interface StartPlaygroundProxyArgs {
   /** Initial engineAccess expiry (ISO); enables proactive refresh. */
   expiresAt?: string;
   /**
-   * Re-mint upstream URL + headers (hosted `--prod`). Single-flighted across
-   * concurrent browser requests; called near expiry and once on 401.
+   * Re-mint upstream URL + headers. Single-flighted across concurrent browser
+   * requests; called near expiry and once on 401.
    */
   refreshUpstream?: () => Promise<PlaygroundUpstreamAccess>;
   /** Bind port; `0` = ephemeral. */

package/src/internal/resolve-prod-target.ts

@@ -48,6 +48,29 @@ export interface ResolvedProdTarget {
   expiresAt: string;
 }
 
+/**
+ * Thrown when a hosted deployment cannot be resolved. `notFound` is true
+ * when the deployment does not exist, so callers can suggest deploying
+ * (or running locally) instead of surfacing a bare API error.
+ */
+export class ProdTargetError extends Error {
+  constructor(
+    message: string,
+    readonly options: { slug: string; notFound?: boolean } = { slug: "" }
+  ) {
+    super(message);
+    this.name = "ProdTargetError";
+  }
+
+  get slug(): string {
+    return this.options.slug;
+  }
+
+  get notFound(): boolean {
+    return this.options.notFound === true;
+  }
+}
+
 /**
  * Look up a running deployment and return its engineAccess URL/headers.
  * Throws {@link Error} with a CLI-ready message on failure.
@@ -65,8 +88,10 @@ export async function resolveProdTarget(
     err,
   });
   if (typeof auth === "number") {
+    // Not "--prod": `eval` resolves the hosted deployment by default,
+    // without any flag.
     throw new Error(
-      `Could not resolve Cursor credentials/team for --prod (see above).`
+      `Could not resolve Cursor credentials/team for the hosted deployment (see above).`
     );
   }
 
@@ -83,8 +108,9 @@ export async function resolveProdTarget(
       });
     } catch (error) {
       if (error instanceof DeployApiError) {
-        throw new Error(
-          `Deployment "${slug}" (team ${auth.teamId}): ${error.message}`
+        throw new ProdTargetError(
+          `Deployment "${slug}" (team ${auth.teamId}): ${error.message}`,
+          { slug, notFound: error.status === 404 }
         );
       }
       throw error;
@@ -143,6 +169,78 @@ export async function resolveProdTarget(
   };
 }
 
+export interface ResolvedProdAliasPlayground {
+  /** Hosted playground URL on the API alias (SPA owns alias-token sign-in). */
+  playgroundUrl: string;
+  slug: string;
+}
+
+/**
+ * Look up a running deployment and return its stable alias playground URL.
+ * Does not mint `engineAccess` — the browser signs in with the alias token.
+ * Throws {@link Error} with a CLI-ready message on failure.
+ */
+export async function resolveProdAliasPlaygroundUrl(
+  options: ResolveProdTargetOptions & { sessionId?: string }
+): Promise<ResolvedProdAliasPlayground> {
+  const err =
+    options.err ?? ((text: string) => void process.stderr.write(text));
+  const slug = resolveProdSlug(options);
+  const auth = await resolveDeployAuth({
+    team: options.team,
+    apiKey: options.apiKey,
+    fetchImpl: options.fetchImpl,
+    err,
+  });
+  if (typeof auth === "number") {
+    throw new Error(
+      `Could not resolve Cursor credentials/team for the hosted deployment (see above).`
+    );
+  }
+
+  let detail: DeploymentDetail;
+  try {
+    detail = await auth.client.getDeployment({
+      teamId: auth.teamId,
+      slug,
+    });
+  } catch (error) {
+    if (error instanceof DeployApiError) {
+      throw new Error(
+        `Deployment "${slug}" (team ${auth.teamId}): ${error.message}`
+      );
+    }
+    throw error;
+  }
+
+  const statusOk = detail.status === "running" || detail.status === "degraded";
+  if (!statusOk) {
+    const lastError =
+      detail.lastError === null || detail.lastError === undefined
+        ? ""
+        : ` ${sanitizeCustomerFacingDetail(detail.lastError)}`;
+    throw new Error(
+      `Deployment "${slug}" is ${detail.status}, not running.${lastError}\n` +
+        `Deploy or wait: \`${CLI} deploy --slug ${slug}\` / \`${CLI} deployment ${slug}\``
+    );
+  }
+
+  const aliasBase = auth.client.aliasUrl(detail.engineAlias).replace(/\/$/, "");
+  const base = `${aliasBase}/playground`;
+  const sessionId =
+    options.sessionId !== undefined && options.sessionId.trim() !== ""
+      ? options.sessionId.trim()
+      : undefined;
+  const playgroundUrl =
+    sessionId === undefined
+      ? base
+      : `${base}?sessionId=${encodeURIComponent(sessionId)}`;
+
+  err(`${stderrPalette().dim(`Using hosted deployment "${slug}".`)}\n`);
+
+  return { playgroundUrl, slug };
+}
+
 /** Strip obvious hosting internals from customer-facing error fragments. */
 function sanitizeCustomerFacingDetail(message: string): string {
   return message

package/src/internal/server.ts

@@ -362,9 +362,8 @@ export async function startServer(
     const evalRuns = new EvalRunStore(
       project.rootDir,
       logger,
-      // defineStorage fallback for playground eval history; explicit
-      // `evals.config.ts` persistRuns still wins inside the store.
-      engine.storage?.asEvalRunPersistence()
+      // Playground eval history mirrors onto the defineStorage evals table.
+      engine.storage?.evalRuns()
     );
     // Eval history restores with the same startup step as sessions (reads
     // also hydrate lazily, but doing it here surfaces store problems at
@@ -738,6 +737,9 @@ export async function startServer(
       for (const runtime of runtimes) {
         runtime.scheduleRunner.stop();
         runtime.reminderRunner.stop();
+        // Before the engines close their storage coordinators, so the
+        // interrupted-batch write can still reach the sink.
+        runtime.evalRuns.close();
       }
       for (const stop of channelStopHooks) {
         try {

package/src/internal/session-engine.ts

@@ -204,8 +204,7 @@ export class SessionEngine {
    * Runtime for `agent/storage.ts` (`defineStorage`), when
    * authored. All durable state changes funnel through it (session
    * records, event chunks, A/B samples/snapshots); the eval run store
-   * attaches via {@link StorageCoordinator.asEvalRunPersistence} at
-   * serve start.
+   * attaches via {@link StorageCoordinator.evalRuns} at serve start.
    */
   readonly storage: StorageCoordinator | undefined;
   /** In-flight lazy restores, deduped per channel + continuation key. */
@@ -261,18 +260,12 @@ export class SessionEngine {
             projectRoot: options.project.rootDir,
             logger: this.logger,
           });
-    // Domain-specific hooks win; the defineStorage sink is the fallback.
-    const persistSamples =
-      options.project.abConfig?.persistSamples ??
-      this.storage?.asABSamplePersistence();
     this.abCollector = new ABCollector(
       options.project.abs,
       this.logger,
       async (sessionId) => (await this.logs.get(sessionId)).snapshot(0),
-      {
-        projectRoot: options.project.rootDir,
-        persistSamples,
-      }
+      // Samples flow to the defineStorage `abs` table (no-op without one).
+      { persistSample: (sample) => this.storage?.abSample(sample) }
     );
     // Peer and Cursor-account transports are symbolic until the serve host
     // resolves them; the host-side MCP registry starts with the concrete
@@ -1592,6 +1585,7 @@ export class SessionEngine {
         return restoredSnapshot;
       }
     }
+    const abTable = this.project.storage?.abs;
     const snapshot = await buildABSnapshot({
       experiments: this.project.abs,
       agentName: this.project.name,
@@ -1602,27 +1596,13 @@ export class SessionEngine {
       logger: this.logger,
       config: {
         maxPlaygroundSessions,
-        durableSamples: abConfig?.persistSamples !== undefined,
-        durableSnapshots: abConfig?.persistSnapshots !== undefined,
+        durableSamples: abTable !== undefined,
+        durableSnapshots: abTable?.putSnapshot !== undefined,
       },
     });
-    // Side effect on the read path: playground polls GET /v1/abs ~every 4s.
-    // Authors who set persistSnapshots accept that write cadence.
-    const persistSnapshots = abConfig?.persistSnapshots;
-    if (persistSnapshots !== undefined) {
-      try {
-        await persistSnapshots.save(snapshot, {
-          projectRoot: this.project.rootDir,
-        });
-      } catch (error) {
-        this.logger(
-          `[agentkit] ab persistSnapshots.save threw: ${describeError(error)}`
-        );
-      }
-    } else {
-      // defineStorage fallback (throttled inside the coordinator).
-      this.storage?.abSnapshot(snapshot);
-    }
+    // Side effect on the read path: playground polls GET /v1/abs ~every 4s;
+    // the coordinator throttles the actual table write.
+    this.storage?.abSnapshot(snapshot);
     return snapshot;
   }