@cursor/july 0.1.12 → 0.1.15

package/src/internal/eval-runner.ts

@@ -63,6 +63,8 @@ export interface EvalRunResult {
   }>;
   error?: string;
   logs: string[];
+  /** Structured scores from {@link import("../evals.js").EvalTestContext.metric}. */
+  metrics: Record<string, string | number>;
   durationMs: number;
 }
 
@@ -95,6 +97,8 @@ export interface RunEvalsOptions extends EvalSelectionOptions {
     evals: readonly DiscoveredEval[];
     config: Partial<EvalConfigFile>;
   };
+  /** When aborted, stop starting new cases and cancel in-flight turns. */
+  signal?: AbortSignal;
 }
 
 async function defaultImport(
@@ -233,24 +237,86 @@ export async function runDiscoveredEvals(
   const maxConcurrency = resolveEvalMaxConcurrency(config);
   const defaultTimeoutMs = options.timeoutMs ?? config.timeoutMs ?? 180_000;
 
-  return mapPool(selected, maxConcurrency, async (discovered) => {
-    if (options.verbose) {
-      process.stderr.write(`  eval ${discovered.id}…\n`);
+  // Iterations of the same authored datapoint share one concurrency slot and
+  // run sequentially. maxConcurrency is therefore the max number of distinct
+  // datapoints in flight, not datapoints × iterations.
+  const groups = groupEvalsForIterationSchedule(selected);
+  const nested = await mapPool(
+    groups,
+    maxConcurrency,
+    async (group) => {
+      const results: EvalRunResult[] = [];
+      for (const discovered of group) {
+        if (options.signal?.aborted) {
+          break;
+        }
+        if (options.verbose) {
+          process.stderr.write(`  eval ${discovered.id}…\n`);
+        }
+        options.onCaseStart?.(discovered.id);
+        const result = await runOneEval(discovered, {
+          baseUrl: options.baseUrl,
+          defaultTimeoutMs,
+          bearerToken: options.bearerToken,
+          headers: options.headers,
+          verbose: Boolean(options.verbose),
+          signal: options.signal,
+          onEvent: (event) => {
+            options.onCaseEvent?.(discovered.id, event);
+            options.onEvent?.(event);
+          },
+        });
+        options.onCaseDone?.(result);
+        results.push(result);
+      }
+      return results;
+    },
+    { signal: options.signal }
+  );
+  return nested.flat();
+}
+
+/**
+ * Schedule key for {@link groupEvalsForIterationSchedule}: iteration siblings
+ * (`weather/nyc/1`…`/3`) share a key; single-iteration ids keep their full id.
+ */
+export function iterationScheduleKey(discovered: DiscoveredEval): string {
+  const iterations = discovered.definition.iterations ?? 1;
+  const iteration = discovered.definition.iteration ?? 1;
+  if (iterations <= 1) {
+    return discovered.id;
+  }
+  const suffix = `/${iteration}`;
+  if (discovered.id.endsWith(suffix)) {
+    return discovered.id.slice(0, -suffix.length);
+  }
+  return discovered.id;
+}
+
+/**
+ * Group expanded iteration cases so each authored datapoint's iterations run
+ * back-to-back under one {@link mapPool} slot (sorted by iteration index).
+ */
+export function groupEvalsForIterationSchedule(
+  evals: readonly DiscoveredEval[]
+): DiscoveredEval[][] {
+  const groups = new Map<string, DiscoveredEval[]>();
+  const order: string[] = [];
+  for (const discovered of evals) {
+    const key = iterationScheduleKey(discovered);
+    let group = groups.get(key);
+    if (group === undefined) {
+      group = [];
+      groups.set(key, group);
+      order.push(key);
     }
-    options.onCaseStart?.(discovered.id);
-    const result = await runOneEval(discovered, {
-      baseUrl: options.baseUrl,
-      defaultTimeoutMs,
-      bearerToken: options.bearerToken,
-      headers: options.headers,
-      verbose: Boolean(options.verbose),
-      onEvent: (event) => {
-        options.onCaseEvent?.(discovered.id, event);
-        options.onEvent?.(event);
-      },
-    });
-    options.onCaseDone?.(result);
-    return result;
+    group.push(discovered);
+  }
+  return order.map((key) => {
+    const group = groups.get(key)!;
+    return group.sort(
+      (a, b) => (a.definition.iteration ?? 1) - (b.definition.iteration ?? 1)
+    );
   });
 }
 
@@ -274,9 +340,11 @@ async function runOneEval(
     headers?: Record<string, string>;
     verbose: boolean;
     onEvent?: (event: SessionEvent) => void;
+    signal?: AbortSignal;
   }
 ): Promise<EvalRunResult> {
   const logs: string[] = [];
+  const metrics: Record<string, string | number> = {};
   const assertions: EvalAssertionResult[] = [];
   const def = discovered.definition;
   const timeoutMs = def.timeoutMs ?? opts.defaultTimeoutMs;
@@ -288,6 +356,25 @@ async function runOneEval(
   const inputs: string[] = [];
   let reply: string | null = null;
 
+  if (opts.signal?.aborted) {
+    return {
+      id: discovered.id,
+      path: discovered.path,
+      ...(def.description === undefined
+        ? {}
+        : { description: def.description }),
+      ok: false,
+      assertions,
+      error: "cancelled",
+      logs,
+      metrics,
+      inputs,
+      toolCalls: [],
+      tools: [],
+      durationMs: Date.now() - startedAt,
+    };
+  }
+
   const record = (name: string, passed: boolean, detail?: string): void => {
     assertions.push({
       name,
@@ -321,6 +408,12 @@ async function runOneEval(
     get events() {
       return allEvents;
     },
+    get iteration() {
+      return def.iteration ?? 1;
+    },
+    get iterations() {
+      return def.iterations ?? 1;
+    },
     async send(
       message: string,
       options: EvalSendOptions = {}
@@ -337,7 +430,9 @@ async function runOneEval(
         headers: opts.headers,
         workspaceDir: options.workspaceDir,
         workspaceFiles: options.workspaceFiles,
+        cloud: options.cloud,
         timeoutMs,
+        signal: opts.signal,
         onEvent: opts.onEvent,
       });
       sessionId = result.sessionId;
@@ -400,6 +495,16 @@ async function runOneEval(
         process.stderr.write(`    ${message}\n`);
       }
     },
+    metric(name: string, value: string | number) {
+      const key = name.trim();
+      if (key === "") {
+        return;
+      }
+      metrics[key] = value;
+      if (opts.verbose) {
+        process.stderr.write(`    metric ${key}=${String(value)}\n`);
+      }
+    },
   };
 
   try {
@@ -419,6 +524,7 @@ async function runOneEval(
       assertions,
       error: message,
       logs,
+      metrics,
       inputs,
       toolCalls: toolCallIo(),
       ...(sessionId === undefined ? {} : { sessionId }),
@@ -441,6 +547,7 @@ async function runOneEval(
     ok,
     assertions,
     logs,
+    metrics,
     inputs,
     toolCalls: toolCallIo(),
     ...(sessionId === undefined ? {} : { sessionId }),
@@ -472,9 +579,13 @@ function matchExpectation(
     }
     case "equals": {
       const passed = deepEqual(value, assertion.value);
+      // Always attach a short detail so values like recall `40.0% (4/10)`
+      // show up on PASS lines / playground, not only on failure.
+      const rendered =
+        typeof value === "string" ? value : JSON.stringify(value);
       return {
         passed,
-        ...(passed ? {} : { detail: `got ${JSON.stringify(value)}` }),
+        detail: passed ? rendered : `got ${rendered}`,
       };
     }
     case "satisfies": {

package/src/internal/evals-client.ts

@@ -17,9 +17,11 @@ const DEFAULT_POLL_MS = 1_000;
 export type RemoteEvalErrorCode =
   | "no_matching_evals"
   | "eval_run_in_progress"
+  | "eval_run_not_running"
   | "unknown_eval_run"
   | "start_failed"
   | "poll_failed"
+  | "cancel_failed"
   | "aborted";
 
 export class RemoteEvalError extends Error {
@@ -269,6 +271,85 @@ export async function startRemoteEvalRun(args: {
   return json as EvalRunSnapshot;
 }
 
+/** Cancel a running batch via `POST /v1/dev/evals/runs/:runId/cancel`. */
+export async function cancelRemoteEvalRun(args: {
+  baseUrl: string;
+  runId: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  signal?: AbortSignal;
+}): Promise<EvalRunSnapshot> {
+  const baseUrl = args.baseUrl.replace(/\/$/, "");
+  const fetchImpl = args.fetchImpl ?? fetch;
+  let resp: Response;
+  try {
+    resp = await fetchImpl(
+      `${baseUrl}/v1/dev/evals/runs/${encodeURIComponent(args.runId)}/cancel`,
+      {
+        method: "POST",
+        headers: mergeRequestHeaders(args.auth),
+        signal: args.signal,
+      }
+    );
+  } catch (error) {
+    throw new RemoteEvalError(
+      error instanceof Error ? error.message : "Could not cancel eval run.",
+      undefined,
+      "cancel_failed"
+    );
+  }
+
+  const json = (await resp.json().catch(() => ({}))) as {
+    ok?: boolean;
+    error?: string;
+    message?: string;
+    status?: string;
+    runId?: string;
+  };
+
+  if (resp.status === 404 || json.error === "unknown_eval_run") {
+    throw new RemoteEvalError(
+      `Unknown eval run ${args.runId}.`,
+      404,
+      "unknown_eval_run"
+    );
+  }
+  if (resp.status === 409 || json.error === "eval_run_not_running") {
+    const status =
+      typeof json.status === "string" && json.status !== ""
+        ? json.status
+        : "unknown";
+    throw new RemoteEvalError(
+      typeof json.message === "string" && json.message !== ""
+        ? json.message
+        : `Eval run ${args.runId} is not running (status: ${status}).`,
+      409,
+      "eval_run_not_running"
+    );
+  }
+  if (!resp.ok) {
+    const detail =
+      typeof json.message === "string" && json.message !== ""
+        ? json.message
+        : typeof json.error === "string" && json.error !== ""
+          ? json.error
+          : `HTTP ${resp.status}`;
+    throw new RemoteEvalError(
+      `Could not cancel eval run: ${detail}`,
+      resp.status,
+      "cancel_failed"
+    );
+  }
+  if (typeof json.runId !== "string" || json.runId === "") {
+    throw new RemoteEvalError(
+      "Eval cancel response missing runId.",
+      undefined,
+      "cancel_failed"
+    );
+  }
+  return json as EvalRunSnapshot;
+}
+
 /** One-shot `GET /v1/dev/evals/runs/:runId` (no polling). */
 export async function getRemoteEvalRun(args: {
   baseUrl: string;
@@ -408,6 +489,7 @@ export function snapshotToEvalRunResults(
     ...(c.toolCalls === undefined ? {} : { toolCalls: c.toolCalls }),
     ...(c.error === undefined ? {} : { error: c.error }),
     logs: c.logs ?? [],
+    metrics: c.metrics ?? {},
     durationMs: c.durationMs ?? 0,
   }));
 }

package/src/internal/http-channel.ts

@@ -12,6 +12,7 @@
  */
 
 import type {
+  AgentCloudOptions,
   AuthContext,
   AuthPolicy,
   HttpChannelDefinition,
@@ -100,12 +101,16 @@ export function buildHttpChannelRoutes(args: {
       }
       const message = await transformMessage(body.message, ctx.auth);
       const workspaceFiles = readWorkspaceFiles(body.workspaceFiles);
+      // Per-session cloud attach (create only). Eval harness uses this to pin
+      // fixture repos without putting them on the agent's default cloud.repos.
+      const cloud = readCloudOptions(body.cloud);
       const session = await engine.send(HTTP_CHANNEL_ID, message, {
         auth: ctx.auth,
         ...(body.state === undefined ? {} : { state: body.state as JsonValue }),
         ...(typeof body.title === "string" ? { title: body.title } : {}),
         ...(body.purpose === "eval" ? { purpose: "eval" as const } : {}),
         ...(workspaceFiles === undefined ? {} : { workspaceFiles }),
+        ...(cloud === undefined ? {} : { cloud }),
       });
       return jsonResponse(
         {
@@ -129,7 +134,7 @@ export function buildHttpChannelRoutes(args: {
     auth,
     handler: async (request, ctx) => {
       const sessionId = ctx.params.sessionId ?? "";
-      const record = await engine.sessions.get(sessionId);
+      const record = await engine.getOrRestoreSession(sessionId);
       if (record === undefined) {
         return errorResponse(
           404,
@@ -230,7 +235,9 @@ export function buildHttpChannelRoutes(args: {
       // custom channels or schedules are observable here too when the caller
       // matches the session's stored principal (or `--dev` relax). Follow-ups
       // use the same ownership gate plus the session's continuation token.
-      const record = await engine.sessions.get(sessionId);
+      // Lazy-restore by id so playground Open-trace works after redeploy
+      // when the session is durable but not in the local state root yet.
+      const record = await engine.getOrRestoreSession(sessionId);
       if (record === undefined) {
         return errorResponse(
           404,
@@ -276,7 +283,7 @@ export function buildHttpChannelRoutes(args: {
     auth,
     handler: async (request, ctx) => {
       const sessionId = ctx.params.sessionId ?? "";
-      const record = await engine.sessions.get(sessionId);
+      const record = await engine.getOrRestoreSession(sessionId);
       if (record === undefined) {
         return errorResponse(
           404,
@@ -323,7 +330,7 @@ export function buildHttpChannelRoutes(args: {
     auth,
     handler: async (_request, ctx) => {
       const sessionId = ctx.params.sessionId ?? "";
-      const record = await engine.sessions.get(sessionId);
+      const record = await engine.getOrRestoreSession(sessionId);
       if (record === undefined) {
         return errorResponse(
           404,
@@ -351,7 +358,7 @@ export function buildHttpChannelRoutes(args: {
     handler: async (request, ctx) => {
       const sessionId = ctx.params.sessionId ?? "";
       const callId = ctx.params.callId ?? "";
-      const record = await engine.sessions.get(sessionId);
+      const record = await engine.getOrRestoreSession(sessionId);
       if (record === undefined) {
         return errorResponse(
           404,
@@ -420,7 +427,7 @@ export function buildHttpChannelRoutes(args: {
     auth,
     handler: async (_request, ctx) => {
       const sessionId = ctx.params.sessionId ?? "";
-      const record = await engine.sessions.get(sessionId);
+      const record = await engine.getOrRestoreSession(sessionId);
       if (record === undefined) {
         return errorResponse(
           404,
@@ -527,6 +534,80 @@ function readWorkspaceFiles(
   return Object.keys(files).length === 0 ? undefined : files;
 }
 
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/** Best-effort parse of {@link AgentCloudOptions} from a JSON body field. */
+function readCloudOptions(value: unknown): AgentCloudOptions | undefined {
+  if (!isRecord(value)) {
+    return undefined;
+  }
+  const cloud: AgentCloudOptions = {};
+
+  if (isRecord(value.env) && typeof value.env.type === "string") {
+    const name =
+      typeof value.env.name === "string" ? value.env.name : undefined;
+    if (value.env.type === "cloud") {
+      cloud.env =
+        name === undefined ? { type: "cloud" } : { type: "cloud", name };
+    } else if (value.env.type === "pool") {
+      cloud.env =
+        name === undefined ? { type: "pool" } : { type: "pool", name };
+    } else if (value.env.type === "machine") {
+      cloud.env =
+        name === undefined ? { type: "machine" } : { type: "machine", name };
+    }
+  }
+
+  if (Array.isArray(value.repos)) {
+    const repos: NonNullable<AgentCloudOptions["repos"]> = [];
+    for (const entry of value.repos) {
+      if (
+        !isRecord(entry) ||
+        typeof entry.url !== "string" ||
+        entry.url === ""
+      ) {
+        continue;
+      }
+      repos.push({
+        url: entry.url,
+        ...(typeof entry.startingRef === "string"
+          ? { startingRef: entry.startingRef }
+          : {}),
+        ...(typeof entry.prUrl === "string" ? { prUrl: entry.prUrl } : {}),
+      });
+    }
+    if (repos.length > 0) {
+      cloud.repos = repos;
+    }
+  }
+
+  if (typeof value.workOnCurrentBranch === "boolean") {
+    cloud.workOnCurrentBranch = value.workOnCurrentBranch;
+  }
+  if (typeof value.autoCreatePR === "boolean") {
+    cloud.autoCreatePR = value.autoCreatePR;
+  }
+  if (typeof value.skipReviewerRequest === "boolean") {
+    cloud.skipReviewerRequest = value.skipReviewerRequest;
+  }
+
+  if (isRecord(value.envVars)) {
+    const envVars: Record<string, string> = {};
+    for (const [key, envValue] of Object.entries(value.envVars)) {
+      if (typeof key === "string" && typeof envValue === "string") {
+        envVars[key] = envValue;
+      }
+    }
+    if (Object.keys(envVars).length > 0) {
+      cloud.envVars = envVars;
+    }
+  }
+
+  return Object.keys(cloud).length === 0 ? undefined : cloud;
+}
+
 export async function readJsonObject(
   request: Request
 ): Promise<Record<string, unknown> | undefined> {

package/src/internal/map-pool.ts

@@ -1,16 +1,23 @@
 /**
  * Run `fn` over `items` with at most `concurrency` in flight; preserve order
  * of results (and of side-effect completion relative to index assignment).
+ *
+ * When `signal` aborts, workers stop claiming new items; in-flight `fn` calls
+ * are responsible for honoring the same signal.
  */
 export async function mapPool<T, R>(
   items: readonly T[],
   concurrency: number,
-  fn: (item: T, index: number) => Promise<R>
+  fn: (item: T, index: number) => Promise<R>,
+  options?: { signal?: AbortSignal }
 ): Promise<R[]> {
   const results = new Array<R>(items.length);
   let next = 0;
   async function worker(): Promise<void> {
     for (;;) {
+      if (options?.signal?.aborted) {
+        return;
+      }
       const index = next;
       next += 1;
       if (index >= items.length) {

package/src/internal/mcp-host.ts

@@ -52,17 +52,18 @@ export class McpHost implements HostMcpRegistry {
   }
 
   async listTools(connectionName: string): Promise<HostMcpToolInfo[]> {
-    const { client } = await this.getClient(connectionName);
-    const listed = await client.listTools();
-    return listed.tools.map((tool) => ({
-      name: tool.name,
-      ...(tool.description === undefined
-        ? {}
-        : { description: tool.description }),
-      ...(tool.inputSchema === undefined
-        ? {}
-        : { inputSchema: tool.inputSchema as JsonObject }),
-    }));
+    return this.withClient(connectionName, async (client) => {
+      const listed = await client.listTools();
+      return listed.tools.map((tool) => ({
+        name: tool.name,
+        ...(tool.description === undefined
+          ? {}
+          : { description: tool.description }),
+        ...(tool.inputSchema === undefined
+          ? {}
+          : { inputSchema: tool.inputSchema as JsonObject }),
+      }));
+    });
   }
 
   async callTool(
@@ -70,12 +71,13 @@ export class McpHost implements HostMcpRegistry {
     toolName: string,
     args: Record<string, unknown> = {}
   ): Promise<HostMcpCallResult> {
-    const { client } = await this.getClient(connectionName);
-    const result = await client.callTool({
-      name: toolName,
-      arguments: args,
+    return this.withClient(connectionName, async (client) => {
+      const result = await client.callTool({
+        name: toolName,
+        arguments: args,
+      });
+      return normalizeCallResult(result);
     });
-    return normalizeCallResult(result);
   }
 
   async close(): Promise<void> {
@@ -90,6 +92,32 @@ export class McpHost implements HostMcpRegistry {
     );
   }
 
+  /**
+   * Run against a cached host MCP client, reconnecting once when the remote
+   * Streamable HTTP session is gone (`session_terminated` / HTTP 404).
+   *
+   * Anytool (and the MCP spec) return 404 for unknown/terminated
+   * `Mcp-Session-Id`s so clients can re-initialize. The SDK transport keeps
+   * the dead session id, so without this invalidation every later host.mcp
+   * call fails until the serve process restarts.
+   */
+  private async withClient<T>(
+    name: string,
+    run: (client: Client) => Promise<T>
+  ): Promise<T> {
+    const opened = await this.getClient(name);
+    try {
+      return await run(opened.client);
+    } catch (error) {
+      if (!isStaleMcpSessionError(error)) {
+        throw error;
+      }
+      await this.invalidateClient(name, opened);
+      const reopened = await this.getClient(name);
+      return await run(reopened.client);
+    }
+  }
+
   private getClient(name: string): Promise<OpenedConnection> {
     let pending = this.clients.get(name);
     if (pending === undefined) {
@@ -102,6 +130,23 @@ export class McpHost implements HostMcpRegistry {
     return pending;
   }
 
+  private async invalidateClient(
+    name: string,
+    opened: OpenedConnection
+  ): Promise<void> {
+    const pending = this.clients.get(name);
+    if (pending !== undefined) {
+      try {
+        if ((await pending) === opened) {
+          this.clients.delete(name);
+        }
+      } catch {
+        this.clients.delete(name);
+      }
+    }
+    await opened.close();
+  }
+
   private async connect(name: string): Promise<OpenedConnection> {
     const connection = this.byName.get(name);
     if (connection === undefined) {
@@ -114,6 +159,19 @@ export class McpHost implements HostMcpRegistry {
   }
 }
 
+/** Exported for unit tests. */
+export function isStaleMcpSessionError(error: unknown): boolean {
+  if (!(error instanceof Error)) {
+    return false;
+  }
+  if (error.message.includes("session_terminated")) {
+    return true;
+  }
+  // StreamableHTTPError from the MCP SDK carries the HTTP status as `.code`.
+  const code = (error as { code?: unknown }).code;
+  return code === 404;
+}
+
 async function openConnection(
   connection: DiscoveredConnection
 ): Promise<OpenedConnection> {