@cursor/july 0.1.15 → 0.1.17

package/src/internal/deploy-client.ts

@@ -529,6 +529,16 @@ export class AgentServeDeployClient {
     return `${this.backendUrl}${engineAlias}`;
   }
 
+  /**
+   * Stable remote MCP gateway URL for a deployment, on the user-API-key
+   * surface (headless / CLI, `agentkit login` credential). The gateway
+   * mints/refreshes engineAccess server-side and proxies `/v1/mcp` to the
+   * pod, so a signed-in client never sees a pod hostname or port token.
+   */
+  mcpGatewayUrl(teamId: number, slug: string): string {
+    return `${this.backendUrl}/internal/agent-serve/mcp/${teamId}/${encodeURIComponent(slug)}`;
+  }
+
   private async request(args: {
     method: "GET" | "POST" | "PUT" | "DELETE";
     path: string;

package/src/internal/discovery.ts

@@ -29,6 +29,7 @@ import {
   type ChannelDefinition,
   type ConnectionDefinition,
   type ConnectionTransport,
+  cursorAccountServersHasMixedWildcard,
   DEFAULT_MODEL,
   type Diagnostic,
   type DiscoveredAB,
@@ -43,6 +44,7 @@ import {
   type InstructionsDefinition,
   MAX_RUNNING_TURNS_LIMIT,
   type ModelSetting,
+  normalizeCursorAccountServers,
   type ResolvedAgent,
   type ScheduleDefinition,
   type SeedFile,
@@ -368,11 +370,23 @@ async function loadAgent(
   // `send({ cloud })` merges it under cloudOverride for handoff sessions.
   const cloud = config?.cloud;
   const localCwd = resolveAgentCwd(ctx, config?.local?.cwd);
+  const localWorkspaceDir = resolveAgentCwd(ctx, config?.local?.workspaceDir);
+  const localSandbox = config?.local?.sandbox === true;
+  if (localCwd !== undefined && localWorkspaceDir !== undefined) {
+    ctx.diagnostics.push({
+      severity: "warning",
+      path: displayPath(ctx, join(agentDir, "agent.ts")),
+      message:
+        "local.workspaceDir pins every session to one directory, so local.cwd is unused. Drop one of them.",
+    });
+  }
   const local =
-    localCwd === undefined
+    localCwd === undefined && localWorkspaceDir === undefined && !localSandbox
       ? undefined
       : {
           cwd: localCwd,
+          workspaceDir: localWorkspaceDir,
+          sandbox: localSandbox,
         };
   const hosting = resolveHosting(ctx, agentDir, config?.hosting);
   const concurrency = resolveConcurrency(ctx, agentDir, config?.concurrency);
@@ -889,11 +903,10 @@ function connectionTransport(
   connection: ConnectionDefinition
 ): ConnectionTransport | undefined {
   if ("cursorAccount" in connection && connection.cursorAccount === true) {
+    const servers = normalizeCursorAccountServers(connection.servers);
     return {
       cursorAccount: true,
-      ...(connection.servers === undefined
-        ? {}
-        : { servers: connection.servers }),
+      ...(servers === undefined ? {} : { servers }),
     };
   }
   if ("agent" in connection && !isBlank(connection.agent)) {
@@ -1035,6 +1048,19 @@ async function loadConnections(
       });
       continue;
     }
+    if (
+      "cursorAccount" in connection &&
+      connection.cursorAccount === true &&
+      cursorAccountServersHasMixedWildcard(connection.servers)
+    ) {
+      ctx.diagnostics.push({
+        severity: "error",
+        path: displayPath(ctx, entryPath),
+        message:
+          'Cursor account MCP connection `servers` cannot mix `"*"` with connector names. Use `"*"` / `["*"]` (or omit `servers`) for every connected server, or list only concrete connector names.',
+      });
+      continue;
+    }
     const transport = connectionTransport(connection);
     if (transport === undefined) {
       ctx.diagnostics.push({
@@ -1055,7 +1081,7 @@ async function loadConnections(
         severity: "error",
         path: displayPath(ctx, entryPath),
         message:
-          "Cursor account MCP connection `servers` must be a non-empty list of connector names (or omitted to expose every connected server).",
+          'Cursor account MCP connection `servers` must be a non-empty list of connector names, `"*"` / `["*"]` for every connected server, or omitted for the same all-connectors behavior.',
       });
       continue;
     }

package/src/internal/distribution.ts

@@ -102,6 +102,8 @@ export const AUTHORING_ENTRY_FILES: ReadonlyArray<
   ["hooks", "hooks.js"],
   ["prompt", "prompt.js"],
   ["evals", "evals.js"],
+  ["evals/reporters", "evals/reporters.js"],
+  ["evals/loaders", "evals/loaders.js"],
   ["ab", "ab.js"],
   ["storage", "storage.js"],
   ["storage/file-kv", "storage-backends/file-kv.js"],

package/src/internal/eval-judge-model.ts

@@ -0,0 +1,148 @@
+/**
+ * The model call behind `t.judge.*`.
+ *
+ * Grading is completion-only in intent: the judge reads a prompt and returns a
+ * choice. It is nonetheless a real SDK session, and its prompt embeds output
+ * from the agent under test — attacker-controlled text in exactly the way a
+ * prompt injection needs. A reply that talks the judge into running a tool would
+ * do so on the machine running the eval CLI, under the runner's credentials.
+ *
+ * The session is therefore confined on every axis the SDK exposes:
+ *
+ * - `sandboxOptions.enabled` so any tool call the judge is talked into is
+ *   confined rather than run directly against the host.
+ * - `mcpServers: {}` so no MCP surface is reachable.
+ * - `settingSources: []` plus an empty temp `cwd`, so there is no repo to act on
+ *   and no ambient `AGENTS.md` / `.cursor` rule can steer a verdict.
+ *
+ * The prompt side is hardened separately: `evals/judge.ts` fences untrusted
+ * content behind a per-call boundary. Both halves matter — this one limits blast
+ * radius, that one removes the instruction channel.
+ *
+ * Grading also declines outright when the credential's backend and the SDK's
+ * backend disagree, rather than sending a key somewhere it was not minted for
+ * (see {@link assertJudgeBackendMatch}).
+ */
+
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { EvalJudgeCaller } from "../evals/judge.js";
+import { EvalJudgeUnavailableError } from "../evals/judge.js";
+import type { ModelSetting } from "../types.js";
+import {
+  DEFAULT_CURSOR_BACKEND_URL,
+  resolveApiKey,
+  SIGN_IN_HINT,
+} from "./cursor/credentials.js";
+
+/** Env override for the default judge model (below `evals.config.ts`). */
+export const EVAL_JUDGE_MODEL_ENV = "AGENTKIT_EVAL_JUDGE_MODEL";
+
+/**
+ * Backend the Cursor SDK will actually talk to.
+ *
+ * `Agent.create` takes a credential but no backend, so the SDK picks its own
+ * host from `CURSOR_API_BASE_URL`. Kept next to {@link assertJudgeBackendMatch}
+ * so the two stay in step if the SDK's resolution changes.
+ */
+function sdkBackendUrl(): string {
+  const configured = process.env.CURSOR_API_BASE_URL?.trim();
+  return trimTrailingSlash(
+    configured === undefined || configured === ""
+      ? DEFAULT_CURSOR_BACKEND_URL
+      : configured
+  );
+}
+
+function trimTrailingSlash(url: string): string {
+  return url.replace(/\/+$/, "");
+}
+
+/**
+ * Refuse to grade when the judge credential belongs to a different backend than
+ * the SDK will send it to.
+ *
+ * `resolveApiKey` resolves against `AGENT_SERVE_CURSOR_BACKEND_URL`, which the
+ * SDK does not read. When only that override is set, the key and the judge
+ * prompt — which embeds agent output — would go to the default host instead of
+ * the intended one. There is no backend field on `AgentOptions` to reconcile
+ * them, so the honest move is to decline rather than send a credential
+ * somewhere it was not minted for.
+ */
+export function assertJudgeBackendMatch(
+  credentialBackendUrl: string,
+  sdkBackend: string = sdkBackendUrl()
+): void {
+  const credentialBackend = trimTrailingSlash(credentialBackendUrl);
+  if (credentialBackend === sdkBackend) {
+    return;
+  }
+  throw new EvalJudgeUnavailableError(
+    `eval judge credential is scoped to ${credentialBackend}, but the Cursor SDK would send it to ${sdkBackend}. ` +
+      "Set CURSOR_API_BASE_URL to the same backend (the SDK does not read AGENT_SERVE_CURSOR_BACKEND_URL)."
+  );
+}
+
+/**
+ * Resolve the judge model, innermost first: per-call → per-eval →
+ * `evals.config.ts` → {@link EVAL_JUDGE_MODEL_ENV}.
+ */
+export function resolveJudgeModel(layers: {
+  call?: ModelSetting;
+  evalLevel?: ModelSetting;
+  config?: ModelSetting;
+  env?: string | undefined;
+}): ModelSetting | undefined {
+  const envModel = layers.env?.trim();
+  return (
+    layers.call ??
+    layers.evalLevel ??
+    layers.config ??
+    (envModel === undefined || envModel === "" ? undefined : envModel)
+  );
+}
+
+/**
+ * Ask the judge model for a grade.
+ *
+ * Throws {@link EvalJudgeUnavailableError} when no credentials are available so
+ * the runner can skip the eval visibly instead of reporting a spurious failure.
+ */
+export const callJudgeModel: EvalJudgeCaller = async ({ prompt, model }) => {
+  const resolved = await resolveApiKey();
+  if (resolved === undefined) {
+    throw new EvalJudgeUnavailableError(
+      `no Cursor credentials for the eval judge. ${SIGN_IN_HINT}`
+    );
+  }
+  assertJudgeBackendMatch(resolved.backendUrl);
+  // Imported lazily so a tree of fully deterministic evals never loads the SDK.
+  const { Agent } = await import("@cursor/sdk");
+  const cwd = await mkdtemp(join(tmpdir(), "agentkit-eval-judge-"));
+  const agent = await Agent.create({
+    apiKey: resolved.apiKey,
+    model: typeof model === "string" ? { id: model } : model,
+    // See the module comment: grading embeds untrusted agent output, so the
+    // session gets no MCP surface, no ambient settings, an empty working
+    // directory, and a sandbox around anything it does attempt.
+    mcpServers: {},
+    local: {
+      cwd,
+      settingSources: [],
+      sandboxOptions: { enabled: true },
+    },
+  });
+  try {
+    const run = await agent.send(prompt);
+    const result = await run.wait();
+    if (result.status === "error" || result.status === "cancelled") {
+      const detail = result.error?.message ?? result.result ?? result.status;
+      throw new Error(`eval judge run failed: ${detail}`);
+    }
+    return result.result ?? "";
+  } finally {
+    agent.close();
+    await rm(cwd, { recursive: true, force: true }).catch(() => undefined);
+  }
+};

package/src/internal/eval-run-store.ts

@@ -173,6 +173,8 @@ export class EvalRunStore {
       summary: {
         passed: 0,
         failed: 0,
+        scored: 0,
+        skipped: 0,
         total: cases.length,
         done: 0,
       },
@@ -474,6 +476,9 @@ export class EvalRunStore {
     const alreadyCounted = c.status === "done";
     c.status = "done";
     c.ok = result.ok;
+    c.verdict = result.verdict;
+    c.skipReason = result.skipReason;
+    c.metadata = result.metadata;
     c.assertions = result.assertions;
     c.durationMs = result.durationMs;
     c.logs = result.logs;
@@ -502,11 +507,8 @@ export class EvalRunStore {
       return;
     }
     run.summary.done += 1;
-    if (result.ok) {
-      run.summary.passed += 1;
-    } else {
-      run.summary.failed += 1;
-    }
+    const verdict = result.verdict ?? (result.ok ? "passed" : "failed");
+    run.summary[verdict] = (run.summary[verdict] ?? 0) + 1;
   }
 }
 
@@ -539,18 +541,12 @@ function markCancelledRun(run: EvalRunSnapshot): void {
 }
 
 function reconcileRunSummary(run: EvalRunSnapshot): void {
-  let passed = 0;
-  let failed = 0;
+  const counts = { passed: 0, failed: 0, scored: 0, skipped: 0 };
   for (const c of run.cases) {
-    if (c.ok === true) {
-      passed += 1;
-    } else {
-      failed += 1;
-    }
+    counts[c.verdict ?? (c.ok === true ? "passed" : "failed")] += 1;
   }
   run.summary = {
-    passed,
-    failed,
+    ...counts,
     total: run.cases.length,
     done: run.cases.length,
   };