@cursor/july 0.1.6 → 0.1.7

package/src/internal/evals-client.ts

@@ -0,0 +1,431 @@
+/**
+ * HTTP client for playground eval batches (`/v1/dev/evals/runs`).
+ *
+ * Used by `agentkit eval --prod` (or `--url`) so the batch runs on the serve
+ * process and is mirrored to `defineStorage` when an `evals` table is
+ * configured. Without those flags, eval keeps the in-process ephemeral path
+ * in `cli-ax.ts`.
+ */
+
+import type { EvalRunSnapshot } from "../evals.js";
+import { matchesEvalIdFilter } from "../evals.js";
+import type { EvalRunResult } from "./eval-runner.js";
+import { mergeRequestHeaders, type RequestAuth } from "./request-headers.js";
+
+const DEFAULT_POLL_MS = 1_000;
+
+export type RemoteEvalErrorCode =
+  | "no_matching_evals"
+  | "eval_run_in_progress"
+  | "unknown_eval_run"
+  | "start_failed"
+  | "poll_failed"
+  | "aborted";
+
+export class RemoteEvalError extends Error {
+  constructor(
+    message: string,
+    readonly status?: number,
+    readonly code?: RemoteEvalErrorCode,
+    readonly activeRunId?: string
+  ) {
+    super(message);
+    this.name = "RemoteEvalError";
+  }
+}
+
+export interface RemoteEvalListItem {
+  id: string;
+  fileId: string;
+  description?: string;
+  tags?: string[];
+}
+
+export interface RunRemoteEvalsOptions extends RequestAuth {
+  baseUrl: string;
+  filterIds?: string[];
+  tags?: string[];
+  /** Per-case timeout override forwarded to the serve process. */
+  timeoutMs?: number;
+  /** Ask the serve harness to emit verbose case logs (server-side). */
+  verbose?: boolean;
+  fetchImpl?: typeof fetch;
+  /** Poll interval while the batch is `running`. */
+  pollMs?: number;
+  /** Abort the wait (not the remote batch). */
+  signal?: AbortSignal;
+  /** Called each time the polled snapshot changes. */
+  onSnapshot?: (snap: EvalRunSnapshot) => void;
+}
+
+/**
+ * Start a batch on the target serve via `POST /v1/dev/evals/runs`, poll until
+ * terminal, and return CLI-shaped case results.
+ */
+export async function runRemoteEvals(
+  options: RunRemoteEvalsOptions
+): Promise<{ snapshot: EvalRunSnapshot; results: EvalRunResult[] }> {
+  const baseUrl = options.baseUrl.replace(/\/$/, "");
+  const fetchImpl = options.fetchImpl ?? fetch;
+  const auth: RequestAuth = {
+    bearerToken: options.bearerToken,
+    headers: options.headers,
+  };
+
+  const started = await startRemoteEvalRun({
+    baseUrl,
+    auth,
+    fetchImpl,
+    filterIds: options.filterIds,
+    tags: options.tags,
+    timeoutMs: options.timeoutMs,
+    verbose: options.verbose,
+    signal: options.signal,
+  });
+  options.onSnapshot?.(started);
+
+  const snapshot = await pollRemoteEvalRun({
+    baseUrl,
+    runId: started.runId,
+    auth,
+    fetchImpl,
+    pollMs: options.pollMs ?? DEFAULT_POLL_MS,
+    signal: options.signal,
+    onSnapshot: options.onSnapshot,
+  });
+
+  return { snapshot, results: snapshotToEvalRunResults(snapshot) };
+}
+
+/** List datapoints from `GET /v1/dev/evals` (same surface a remote run uses). */
+export async function listRemoteEvals(args: {
+  baseUrl: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  filterIds?: string[];
+  tags?: string[];
+  signal?: AbortSignal;
+}): Promise<RemoteEvalListItem[]> {
+  const baseUrl = args.baseUrl.replace(/\/$/, "");
+  const fetchImpl = args.fetchImpl ?? fetch;
+  let resp: Response;
+  try {
+    resp = await fetchImpl(`${baseUrl}/v1/dev/evals`, {
+      headers: mergeRequestHeaders(args.auth),
+      signal: args.signal,
+    });
+  } catch (error) {
+    throw new RemoteEvalError(
+      error instanceof Error ? error.message : "Could not list evals.",
+      undefined,
+      "start_failed"
+    );
+  }
+  if (!resp.ok) {
+    throw new RemoteEvalError(
+      `Could not list evals (HTTP ${resp.status}).`,
+      resp.status,
+      "start_failed"
+    );
+  }
+  const body = (await resp.json()) as {
+    evals?: Array<{
+      id?: unknown;
+      fileId?: unknown;
+      description?: unknown;
+      tags?: unknown;
+    }>;
+  };
+  const listed: RemoteEvalListItem[] = [];
+  for (const row of body.evals ?? []) {
+    if (typeof row.id !== "string" || row.id === "") {
+      continue;
+    }
+    if (typeof row.fileId !== "string" || row.fileId === "") {
+      continue;
+    }
+    const tags = Array.isArray(row.tags)
+      ? row.tags.filter((t): t is string => typeof t === "string")
+      : undefined;
+    listed.push({
+      id: row.id,
+      fileId: row.fileId,
+      ...(typeof row.description === "string"
+        ? { description: row.description }
+        : {}),
+      ...(tags === undefined || tags.length === 0 ? {} : { tags }),
+    });
+  }
+
+  let selected = listed;
+  if (args.filterIds !== undefined && args.filterIds.length > 0) {
+    selected = selected.filter((e) =>
+      args.filterIds!.some((f) => matchesEvalIdFilter(e.id, f))
+    );
+  }
+  if (args.tags !== undefined && args.tags.length > 0) {
+    selected = selected.filter((e) =>
+      (e.tags ?? []).some((t) => args.tags!.includes(t))
+    );
+  }
+  return selected;
+}
+
+export async function startRemoteEvalRun(args: {
+  baseUrl: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  filterIds?: string[];
+  tags?: string[];
+  timeoutMs?: number;
+  verbose?: boolean;
+  signal?: AbortSignal;
+}): Promise<EvalRunSnapshot> {
+  const baseUrl = args.baseUrl.replace(/\/$/, "");
+  const fetchImpl = args.fetchImpl ?? fetch;
+  const body: {
+    filterIds?: string[];
+    tags?: string[];
+    timeoutMs?: number;
+    verbose?: boolean;
+  } = {};
+  if (args.filterIds !== undefined && args.filterIds.length > 0) {
+    body.filterIds = args.filterIds;
+  }
+  if (args.tags !== undefined && args.tags.length > 0) {
+    body.tags = args.tags;
+  }
+  if (args.timeoutMs !== undefined) {
+    body.timeoutMs = args.timeoutMs;
+  }
+  if (args.verbose === true) {
+    body.verbose = true;
+  }
+
+  let resp: Response;
+  try {
+    resp = await fetchImpl(`${baseUrl}/v1/dev/evals/runs`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        ...mergeRequestHeaders(args.auth),
+      },
+      body: JSON.stringify(body),
+      signal: args.signal,
+    });
+  } catch (error) {
+    throw new RemoteEvalError(
+      error instanceof Error ? error.message : "Could not start eval run.",
+      undefined,
+      "start_failed"
+    );
+  }
+
+  const json = (await resp.json().catch(() => ({}))) as {
+    ok?: boolean;
+    error?: string;
+    message?: string;
+    activeRunId?: string;
+    runId?: string;
+  };
+
+  if (json.error === "no_matching_evals") {
+    throw new RemoteEvalError(
+      "No matching evals found.",
+      404,
+      "no_matching_evals"
+    );
+  }
+  if (resp.status === 409 || json.error === "eval_run_in_progress") {
+    throw new RemoteEvalError(
+      json.activeRunId !== undefined && json.activeRunId !== ""
+        ? `An eval run is already in progress (${json.activeRunId}).`
+        : "An eval run is already in progress.",
+      409,
+      "eval_run_in_progress",
+      typeof json.activeRunId === "string" ? json.activeRunId : undefined
+    );
+  }
+  if (!resp.ok) {
+    const detail =
+      typeof json.message === "string" && json.message !== ""
+        ? json.message
+        : typeof json.error === "string" && json.error !== ""
+          ? json.error
+          : `HTTP ${resp.status}`;
+    throw new RemoteEvalError(
+      `Could not start eval run: ${detail}`,
+      resp.status,
+      "start_failed"
+    );
+  }
+  if (typeof json.runId !== "string" || json.runId === "") {
+    throw new RemoteEvalError(
+      "Eval start response missing runId.",
+      undefined,
+      "start_failed"
+    );
+  }
+  return json as EvalRunSnapshot;
+}
+
+/** One-shot `GET /v1/dev/evals/runs/:runId` (no polling). */
+export async function getRemoteEvalRun(args: {
+  baseUrl: string;
+  runId: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  signal?: AbortSignal;
+}): Promise<EvalRunSnapshot> {
+  const baseUrl = args.baseUrl.replace(/\/$/, "");
+  const fetchImpl = args.fetchImpl ?? fetch;
+  let resp: Response;
+  try {
+    resp = await fetchImpl(
+      `${baseUrl}/v1/dev/evals/runs/${encodeURIComponent(args.runId)}`,
+      {
+        headers: mergeRequestHeaders(args.auth),
+        signal: args.signal,
+      }
+    );
+  } catch (error) {
+    throw new RemoteEvalError(
+      error instanceof Error ? error.message : "Could not fetch eval run.",
+      undefined,
+      "poll_failed"
+    );
+  }
+
+  if (resp.status === 404) {
+    throw new RemoteEvalError(
+      `Unknown eval run ${args.runId}.`,
+      404,
+      "unknown_eval_run"
+    );
+  }
+  if (!resp.ok) {
+    throw new RemoteEvalError(
+      `Could not fetch eval run (HTTP ${resp.status}).`,
+      resp.status,
+      "poll_failed"
+    );
+  }
+  return (await resp.json()) as EvalRunSnapshot;
+}
+
+/** Recent batches from `GET /v1/dev/evals/runs`. */
+export async function listRemoteEvalRuns(args: {
+  baseUrl: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  signal?: AbortSignal;
+}): Promise<{ runs: EvalRunSnapshot[]; activeRunId?: string }> {
+  const baseUrl = args.baseUrl.replace(/\/$/, "");
+  const fetchImpl = args.fetchImpl ?? fetch;
+  let resp: Response;
+  try {
+    resp = await fetchImpl(`${baseUrl}/v1/dev/evals/runs`, {
+      headers: mergeRequestHeaders(args.auth),
+      signal: args.signal,
+    });
+  } catch (error) {
+    throw new RemoteEvalError(
+      error instanceof Error ? error.message : "Could not list eval runs.",
+      undefined,
+      "poll_failed"
+    );
+  }
+  if (!resp.ok) {
+    throw new RemoteEvalError(
+      `Could not list eval runs (HTTP ${resp.status}).`,
+      resp.status,
+      "poll_failed"
+    );
+  }
+  const body = (await resp.json()) as {
+    runs?: unknown;
+    activeRunId?: unknown;
+  };
+  const runs = Array.isArray(body.runs)
+    ? (body.runs.filter(
+        (r): r is EvalRunSnapshot =>
+          r !== null &&
+          typeof r === "object" &&
+          typeof (r as EvalRunSnapshot).runId === "string"
+      ) as EvalRunSnapshot[])
+    : [];
+  return {
+    runs,
+    ...(typeof body.activeRunId === "string" && body.activeRunId !== ""
+      ? { activeRunId: body.activeRunId }
+      : {}),
+  };
+}
+
+export async function pollRemoteEvalRun(args: {
+  baseUrl: string;
+  runId: string;
+  auth?: RequestAuth;
+  fetchImpl?: typeof fetch;
+  pollMs?: number;
+  signal?: AbortSignal;
+  onSnapshot?: (snap: EvalRunSnapshot) => void;
+}): Promise<EvalRunSnapshot> {
+  const pollMs = args.pollMs ?? DEFAULT_POLL_MS;
+  let lastJson = "";
+
+  for (;;) {
+    if (args.signal?.aborted === true) {
+      throw new RemoteEvalError("Eval wait aborted.", undefined, "aborted");
+    }
+
+    const snap = await getRemoteEvalRun(args);
+    const serialized = JSON.stringify(snap);
+    if (serialized !== lastJson) {
+      lastJson = serialized;
+      args.onSnapshot?.(snap);
+    }
+    if (snap.status !== "running") {
+      return snap;
+    }
+    await sleep(pollMs, args.signal);
+  }
+}
+
+export function snapshotToEvalRunResults(
+  snapshot: EvalRunSnapshot
+): EvalRunResult[] {
+  return snapshot.cases.map((c) => ({
+    id: c.id,
+    path: c.fileId,
+    ...(c.description === undefined ? {} : { description: c.description }),
+    ok: c.ok === true,
+    assertions: c.assertions ?? [],
+    ...(c.sessionId === undefined ? {} : { sessionId: c.sessionId }),
+    inputs: c.inputs ?? [],
+    ...(c.finalText === undefined ? {} : { finalText: c.finalText }),
+    ...(c.tools === undefined ? {} : { tools: c.tools }),
+    ...(c.toolCalls === undefined ? {} : { toolCalls: c.toolCalls }),
+    ...(c.error === undefined ? {} : { error: c.error }),
+    logs: c.logs ?? [],
+    durationMs: c.durationMs ?? 0,
+  }));
+}
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted === true) {
+      reject(new RemoteEvalError("Eval wait aborted.", undefined, "aborted"));
+      return;
+    }
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    const onAbort = (): void => {
+      clearTimeout(timer);
+      reject(new RemoteEvalError("Eval wait aborted.", undefined, "aborted"));
+    };
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}

package/src/internal/init-project.ts

@@ -433,15 +433,15 @@ Recommended loop once the project exists:
   1. Edit agent/ + tools under agent/tools/
   2. ${CLI_COMMAND_NAME} info --dir . --json
   3. ${CLI_COMMAND_NAME} run --dir . --message "..."
-  4. ${CLI_COMMAND_NAME} eval --local --dir .
+  4. ${CLI_COMMAND_NAME} eval --dir .
 
 Evals:
   Create evals/**/*.eval.ts using defineEval from ${PACKAGE_NAME}/evals
   Case id is the path: evals/weather/nyc.eval.ts → weather/nyc
   Required: evals/evals.config.ts with defineEvalConfig({ maxConcurrency })
     (hard-capped at 200 due to model provider request limits)
-  Run: ${CLI_COMMAND_NAME} eval --local [--json] [evalId...]
-    (omit --local to run against the hosted deployment)
+  Run: ${CLI_COMMAND_NAME} eval [--json] [evalId...]
+    (pass --prod to run against the hosted deployment)
 
 See AGENTS.md in the ${PACKAGE_NAME} package for the full loop.
 `);

package/src/internal/server.ts

@@ -1082,6 +1082,8 @@ function mountAgentRoutes(
       const body = await readJsonObject(request);
       const filterIds = asStringArray(body?.filterIds);
       const tags = asStringArray(body?.tags);
+      const timeoutMs = asPositiveInt(body?.timeoutMs);
+      const verbose = body?.verbose === true;
       const authHeader = request.headers.get("authorization");
       const bearerToken = authHeader?.toLowerCase().startsWith("bearer ")
         ? authHeader.slice("bearer ".length).trim()
@@ -1093,6 +1095,8 @@ function mountAgentRoutes(
           ...(bearerToken === undefined || bearerToken === ""
             ? {}
             : { bearerToken }),
+          ...(timeoutMs === undefined ? {} : { timeoutMs }),
+          ...(verbose ? { verbose: true } : {}),
         });
         return jsonResponse(snap, { status: 202 });
       } catch (error) {
@@ -1896,6 +1900,14 @@ function asStringArray(value: unknown): string[] | undefined {
   return out.length === 0 ? undefined : out;
 }
 
+function asPositiveInt(value: unknown): number | undefined {
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    return undefined;
+  }
+  const n = Math.floor(value);
+  return n > 0 ? n : undefined;
+}
+
 async function assertReminderReadable(input: {
   engine: SessionEngine;
   reminder: ReminderInfo;