@cursor/july 0.1.16 → 0.1.18

package/src/internal/cli-ax.ts

@@ -10,7 +10,13 @@
 import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
-import type { EvalRunSnapshot } from "../evals.js";
+import { Artifacts, JUnit } from "../evals/reporters.js";
+import type {
+  EvalAssertionResult,
+  EvalReporter,
+  EvalRunSnapshot,
+  EvalVerdict,
+} from "../evals.js";
 import { serve } from "../index.js";
 import type { AgentServeHandle } from "../types.js";
 import { lookupSessionContinuation, runChat } from "./chat-client.js";
@@ -96,6 +102,19 @@ export interface AxCliOptions {
   tags?: string[];
   /** Eval ids / path prefixes. */
   evalIds?: string[];
+  /** `eval`: a soft threshold miss (`scored`) also fails the exit code. */
+  strict?: boolean;
+  /** `eval`: write JUnit XML to this path. */
+  junit?: string;
+  /**
+   * `eval`: run-artifact directory. Defaults to a timestamped directory under
+   * `<dir>/.agent-serve/evals/`; `false` disables artifacts entirely.
+   */
+  artifacts?: string | false;
+  /** `eval`: skip reporters declared in `evals.config.ts` and eval files. */
+  skipReport?: boolean;
+  /** `eval`: override `evals.config.ts` `maxConcurrency`. */
+  maxConcurrency?: number;
   /** Scaffold target directory for `init`. */
   initTarget?: string;
   /** Tool name for `call`. */
@@ -839,6 +858,7 @@ export async function cmdEval(options: AxCliOptions): Promise<number> {
     // batch lands in playground history and defineStorage evals (when set).
     if (mode === "local") {
       const progress = stream ? createStreamProgress() : undefined;
+      const artifactsDir = resolveEvalArtifactsDir(projectRoot, options);
       try {
         const results = await runDiscoveredEvals({
           projectRoot,
@@ -846,6 +866,11 @@ export async function cmdEval(options: AxCliOptions): Promise<number> {
           filterIds,
           tags,
           timeoutMs: options.timeoutMs,
+          maxConcurrency: options.maxConcurrency,
+          strict: options.strict === true,
+          skipReport: options.skipReport === true,
+          reporters: buildCliEvalReporters(options, artifactsDir),
+          target: { baseUrl: target.agentUrl, mode: "local" },
           bearerToken: options.bearerToken,
           headers: target.headers,
           verbose: Boolean(options.verbose) || stream,
@@ -856,7 +881,7 @@ export async function cmdEval(options: AxCliOptions): Promise<number> {
           process.stderr.write("No matching evals found.\n");
           return 2;
         }
-        return printEvalResults(results, options);
+        return printEvalResults(results, options, undefined, artifactsDir);
       } finally {
         progress?.close();
       }
@@ -1359,6 +1384,53 @@ function createRemoteEvalProgress(
   };
 }
 
+/**
+ * Where a local run writes its artifacts.
+ *
+ * Defaults to a timestamped directory under the project's gitignored
+ * `.agent-serve/evals/`, so the full story of a failure is always on disk
+ * without the eval author opting in. `--no-artifacts` turns it off.
+ */
+export function resolveEvalArtifactsDir(
+  projectRoot: string,
+  options: Pick<AxCliOptions, "artifacts">
+): string | undefined {
+  if (options.artifacts === false) {
+    return undefined;
+  }
+  if (typeof options.artifacts === "string" && options.artifacts !== "") {
+    return resolve(options.artifacts);
+  }
+  const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+  return join(projectRoot, ".agent-serve", "evals", stamp);
+}
+
+/** JUnit / artifacts reporters requested via CLI flags. */
+function buildCliEvalReporters(
+  options: AxCliOptions,
+  artifactsDir: string | undefined
+): EvalReporter[] {
+  const reporters: EvalReporter[] = [];
+  if (artifactsDir !== undefined) {
+    reporters.push(Artifacts({ dir: artifactsDir }));
+  }
+  if (options.junit !== undefined && options.junit !== "") {
+    reporters.push(JUnit({ filePath: options.junit }));
+  }
+  return reporters;
+}
+
+function evalVerdictOf(result: EvalRunResult): EvalVerdict {
+  return result.verdict ?? (result.ok ? "passed" : "failed");
+}
+
+const EVAL_VERDICT_LABEL: Record<EvalVerdict, string> = {
+  passed: "PASS",
+  failed: "FAIL",
+  scored: "SCORED",
+  skipped: "SKIP",
+};
+
 function printEvalResults(
   results: EvalRunResult[],
   options: AxCliOptions,
@@ -1366,12 +1438,17 @@ function printEvalResults(
     runId?: string;
     status?: EvalRunSnapshot["status"];
     error?: string;
-  }
+  },
+  artifactsDir?: string
 ): number {
-  const passed = results.filter((r) => r.ok).length;
-  const failed = results.length - passed;
+  const strict = options.strict === true;
+  const counts = { passed: 0, failed: 0, scored: 0, skipped: 0 };
+  for (const result of results) {
+    counts[evalVerdictOf(result)]++;
+  }
   const batchFailed = batch?.status === "failed";
-  const ok = failed === 0 && !batchFailed;
+  const ok =
+    counts.failed === 0 && !(strict && counts.scored > 0) && !batchFailed;
 
   if (batch?.error !== undefined && batch.error !== "" && !options.json) {
     // Always surface harness-level failures (not only --verbose).
@@ -1383,9 +1460,13 @@ function printEvalResults(
       JSON.stringify(
         {
           ok,
-          passed,
-          failed,
+          passed: counts.passed,
+          failed: counts.failed,
+          scored: counts.scored,
+          skipped: counts.skipped,
+          strict,
           results,
+          ...(artifactsDir === undefined ? {} : { artifactsDir }),
           ...(batch?.runId === undefined ? {} : { runId: batch.runId }),
           ...(batch?.status === undefined ? {} : { status: batch.status }),
           ...(batch?.error === undefined || batch.error === ""
@@ -1398,19 +1479,19 @@ function printEvalResults(
     );
   } else {
     for (const r of results) {
+      const verdict = evalVerdictOf(r);
       process.stdout.write(
-        `${r.ok ? "PASS" : "FAIL"}  ${r.id} (${(r.durationMs / 1000).toFixed(1)}s)`
+        `${EVAL_VERDICT_LABEL[verdict]}  ${r.id} (${(r.durationMs / 1000).toFixed(1)}s)`
       );
       if (r.description !== undefined) {
         process.stdout.write(` — ${r.description}`);
       }
       process.stdout.write("\n");
+      if (r.skipReason !== undefined) {
+        process.stdout.write(`  skipped: ${r.skipReason}\n`);
+      }
       for (const a of r.assertions) {
-        process.stdout.write(
-          `  ${a.passed ? "✓" : "✗"} ${a.name}${
-            a.detail !== undefined ? ` (${a.detail})` : ""
-          }\n`
-        );
+        process.stdout.write(`  ${formatEvalAssertionLine(a)}\n`);
       }
       for (const [name, value] of Object.entries(r.metrics ?? {})) {
         process.stdout.write(`  metric ${name}=${String(value)}\n`);
@@ -1427,13 +1508,51 @@ function printEvalResults(
         );
       }
     }
+    const extra: string[] = [];
+    if (counts.scored > 0) {
+      extra.push(
+        `${counts.scored} scored${strict ? " (fatal: --strict)" : ""}`
+      );
+    }
+    if (counts.skipped > 0) {
+      extra.push(`${counts.skipped} skipped`);
+    }
     process.stdout.write(
-      `\n${passed} passed, ${failed} failed, ${results.length} total\n`
+      `\n${counts.passed} passed, ${counts.failed} failed${
+        extra.length === 0 ? "" : `, ${extra.join(", ")}`
+      }, ${results.length} total\n`
     );
+    if (artifactsDir !== undefined) {
+      process.stdout.write(`artifacts: ${artifactsDir}\n`);
+    }
   }
   return ok ? 0 : 1;
 }
 
+/**
+ * One assertion line. Soft assertions read `~` rather than `✗` when they miss,
+ * so a tracked score is visibly different from a failed gate.
+ */
+export function formatEvalAssertionLine(
+  assertion: EvalAssertionResult
+): string {
+  const soft = (assertion.severity ?? "gate") === "soft";
+  const mark = assertion.passed ? "✓" : soft ? "~" : "✗";
+  const parts: string[] = [];
+  if (assertion.score !== undefined) {
+    parts.push(
+      `score=${assertion.score.toFixed(3)}${
+        assertion.threshold === undefined ? "" : ` >= ${assertion.threshold}`
+      }`
+    );
+  }
+  if (assertion.detail !== undefined) {
+    parts.push(assertion.detail);
+  }
+  const suffix = parts.length === 0 ? "" : ` (${parts.join(", ")})`;
+  return `${mark} ${assertion.name}${soft ? " [soft]" : ""}${suffix}`;
+}
+
 /**
  * `logs` — follow deploy progress (`--prod`) then runtime agent logs.
  * Local / `--url` tails `/v1/logs` only.

package/src/internal/cli-mcp.ts

@@ -29,6 +29,8 @@
 import { mkdir, readFile, writeFile } from "node:fs/promises";
 import { homedir } from "node:os";
 import { dirname, join } from "node:path";
+import { resolveDeployAuth } from "./cli-deploy.js";
+import { DeployApiError } from "./deploy-client.js";
 import { CLI_COMMAND_NAME as CLI, LOG_PREFIX } from "./distribution.js";
 import {
   ProdTargetError,
@@ -77,9 +79,16 @@ export interface McpCliOptions {
   print?: boolean;
   /** `install`: machine-readable output. */
   json?: boolean;
+  /**
+   * `install --prod`: write a REMOTE HTTP entry pointing at the stable Cursor
+   * MCP gateway (`<api>/…/mcp/<team>/<slug>`) instead of the local stdio
+   * proxy. For remote-only MCP hosts (Glass) that cannot spawn stdio.
+   */
+  remote?: boolean;
   /** Injected seams for tests. */
   fetchImpl?: typeof fetch;
   resolveProdTargetImpl?: typeof resolveProdTarget;
+  resolveDeployAuthImpl?: typeof resolveDeployAuth;
   stdin?: NodeJS.ReadableStream;
   stdout?: { write(chunk: string): unknown };
   err?: (text: string) => void;
@@ -490,10 +499,9 @@ function requestIdsOf(line: string): Array<string | number> {
 // install: write the entry into ~/.cursor/mcp.json
 // ============================================================================
 
-interface McpJsonEntry {
-  command: string;
-  args: string[];
-}
+type McpJsonEntry =
+  | { command: string; args: string[] }
+  | { url: string; headers?: Record<string, string> };
 
 export async function cmdMcpInstall(options: McpCliOptions): Promise<number> {
   const err = options.err ?? defaultErr;
@@ -502,9 +510,16 @@ export async function cmdMcpInstall(options: McpCliOptions): Promise<number> {
     err("--prod and --url are mutually exclusive\n");
     return 1;
   }
+  if (options.remote === true) {
+    if (options.prod !== true) {
+      err("--remote requires --prod (the hosted deployment MCP gateway)\n");
+      return 1;
+    }
+    return cmdMcpInstallRemote(options);
+  }
 
   let name: string;
-  let entry: McpJsonEntry;
+  let entry: { command: string; args: string[] };
   if (options.prod === true) {
     let slug: string;
     try {
@@ -598,6 +613,104 @@ export async function cmdMcpInstall(options: McpCliOptions): Promise<number> {
   return 0;
 }
 
+/**
+ * `mcp install --prod --remote` — write a REMOTE HTTP entry pointing at the
+ * stable Cursor MCP gateway for the hosted deployment, for MCP hosts that
+ * cannot spawn the stdio proxy (Glass / Sand). The gateway mints/refreshes
+ * engineAccess server-side, so the entry never carries a pod hostname or a
+ * short-lived port token — only the durable, stable gateway URL.
+ *
+ * Auth: this headless/CLI entry sends the `agentkit login` API key as a
+ * bearer against the user-API-key gateway surface. That key is written to
+ * `mcp.json` in plain text (flagged below); a signed-in Cursor client
+ * (Glass) can instead use the session gateway surface with no header at all
+ * once first-party session attachment ships (see PR follow-ups).
+ */
+async function cmdMcpInstallRemote(options: McpCliOptions): Promise<number> {
+  const err = options.err ?? defaultErr;
+  const stdout = options.stdout ?? process.stdout;
+
+  let slug: string;
+  try {
+    slug = resolveProdSlug({
+      dir: options.dir,
+      cwd: options.cwd,
+      slug: options.slug,
+    });
+  } catch (error) {
+    err(`${describeError(error)}\n`);
+    return 1;
+  }
+
+  const resolveAuthImpl = options.resolveDeployAuthImpl ?? resolveDeployAuth;
+  const auth = await resolveAuthImpl({
+    team: options.team,
+    apiKey: options.apiKey,
+    fetchImpl: options.fetchImpl,
+    err,
+  });
+  if (typeof auth === "number") {
+    return auth;
+  }
+
+  // Verify the deployment resolves before writing config that cannot work.
+  // This read is admin-tier, but calling the gateway only needs team
+  // membership — so a 403 means "cannot verify", not "cannot use": warn and
+  // still write the entry rather than locking members out of their own agent.
+  try {
+    await auth.client.getDeployment({ teamId: auth.teamId, slug });
+  } catch (error) {
+    if (error instanceof DeployApiError && error.status === 403) {
+      err(
+        `Could not verify deployment "${slug}" (reading deployments needs team-admin); writing the entry anyway.\n`
+      );
+    } else {
+      err(`${describeError(error)}\n`);
+      if (error instanceof DeployApiError && error.status === 404) {
+        err(`Deploy it first: ${CLI} deploy --slug ${slug}\n`);
+      }
+      return 1;
+    }
+  }
+
+  const name = resolveServerName(options.name, slug);
+  const url = auth.client.mcpGatewayUrl(auth.teamId, slug);
+  const entry: { url: string; headers?: Record<string, string> } = {
+    url,
+    headers: { Authorization: `Bearer ${auth.apiKey}` },
+  };
+
+  if (options.print === true) {
+    stdout.write(
+      `${JSON.stringify({ mcpServers: { [name]: entry } }, null, 2)}\n`
+    );
+    return 0;
+  }
+
+  const path = cursorMcpJsonPath();
+  try {
+    await upsertMcpJsonEntry(path, name, entry);
+  } catch (error) {
+    err(`${describeError(error)}\n`);
+    return 1;
+  }
+
+  err(
+    "note: the API key is written to mcp.json in plain text; it is a durable " +
+      "credential for a remote MCP host that cannot spawn the stdio proxy.\n"
+  );
+  if (options.json === true) {
+    stdout.write(`${JSON.stringify({ ok: true, path, name, entry })}\n`);
+    return 0;
+  }
+  stdout.write(
+    `Installed remote MCP server "${name}" in ${path}\n` +
+      `  ${url}\n` +
+      "Cursor watches this file; the server appears without a restart.\n"
+  );
+  return 0;
+}
+
 /** `~/.cursor/mcp.json` — the user-level Cursor MCP config. */
 export function cursorMcpJsonPath(
   home: string = process.env.HOME ?? homedir()