@cursor/july 0.1.12 → 0.1.15

package/src/channels/slack/api.ts

@@ -331,10 +331,16 @@ export async function postMessage(args: {
   text: string;
   threadTs?: string;
   blocks?: unknown[];
-}): Promise<{ ts: string }> {
+  /**
+   * Skip `markdown_text` and post with plain `text` only (Deploy Buddy /
+   * host-report pattern). Prefer this for large structured bodies — Slack
+   * `markdown_text` hard-caps at 12k while `text` allows ~40k.
+   */
+  plainText?: boolean;
+}): Promise<{ ts: string; channel: string }> {
   // Slack treats markdown_text as mutually exclusive with text and blocks
   // (error: markdown_text_conflict). Prefer markdown when we have no
-  // blocks; with Block Kit, use text as body/fallback.
+  // blocks; with Block Kit or plainText, use text as body/fallback.
   const base: Record<string, unknown> = {
     channel: args.channel,
     unfurl_links: false,
@@ -344,16 +350,17 @@ export async function postMessage(args: {
     base.thread_ts = args.threadTs;
   }
   const hasBlocks = args.blocks !== undefined && args.blocks.length > 0;
+  const usePlainText = args.plainText === true || hasBlocks;
 
   let response: Record<string, unknown>;
-  if (hasBlocks) {
+  if (usePlainText) {
     response = await callSlackApi({
       botToken: args.botToken,
       operation: "chat.postMessage",
       body: {
         ...base,
         text: args.text,
-        blocks: args.blocks,
+        ...(hasBlocks ? { blocks: args.blocks } : {}),
       },
     });
   } else {
@@ -366,11 +373,14 @@ export async function postMessage(args: {
       },
     });
     // Older workspaces / scopes may not accept markdown_text — retry with text.
+    // Also fall back when markdown_text hits its 12k limit (msg_too_long) while
+    // plain text still allows up to ~40k.
     if (
       response.ok !== true &&
       (response.error === "markdown_text_conflict" ||
         response.error === "invalid_arguments" ||
         response.error === "invalid_blocks" ||
+        response.error === "msg_too_long" ||
         SLACK_UNSUPPORTED_ERRORS.has(String(response.error ?? "")))
     ) {
       response = await callSlackApi({
@@ -388,8 +398,43 @@ export async function postMessage(args: {
       `Slack chat.postMessage failed: ${String(response.error ?? "unknown_error")}`
     );
   }
+  // Slack requires thread_ts as a string. A numeric ts (JSON number) silently
+  // drops threading server-side — never coerce via Number.
   const ts = typeof response.ts === "string" ? response.ts : "";
-  return { ts };
+  if (ts === "") {
+    throw new Error(
+      `Slack chat.postMessage ok but missing string ts (got ${typeof response.ts})`
+    );
+  }
+  // When we asked to thread, confirm Slack honored it — but only when the
+  // response includes a thread_ts. Some proxy / customize-username payloads
+  // omit `message.thread_ts` even for successful threaded posts; treating
+  // that as failure retries (or deletes) replies and leaves empty threads.
+  const requestedThreadTs = args.threadTs;
+  if (requestedThreadTs !== undefined && requestedThreadTs !== "") {
+    const message = response.message;
+    if (
+      typeof message === "object" &&
+      message !== null &&
+      !Array.isArray(message)
+    ) {
+      const applied =
+        typeof (message as { thread_ts?: unknown }).thread_ts === "string"
+          ? (message as { thread_ts: string }).thread_ts
+          : "";
+      if (applied !== "" && applied !== requestedThreadTs) {
+        throw new Error(
+          `Slack chat.postMessage ignored thread_ts (requested ${requestedThreadTs}, got ${applied})`
+        );
+      }
+    }
+  }
+  // Slack returns the resolved channel ID (C…) even when the request used a name.
+  const channel =
+    typeof response.channel === "string" && response.channel !== ""
+      ? response.channel
+      : args.channel;
+  return { ts, channel };
 }
 
 /** Replace an existing message (e.g. strip Approve/Deny after a decision). */
@@ -399,20 +444,22 @@ export async function updateMessage(args: {
   ts: string;
   text: string;
   blocks?: unknown[];
+  /** Skip `markdown_text` and update with plain `text` only. */
+  plainText?: boolean;
 }): Promise<SoftSlackResult> {
   const base: Record<string, unknown> = {
     channel: args.channel,
     ts: args.ts,
   };
-  if (args.blocks !== undefined) {
-    // Blocks force the legacy text path (same as postMessage).
+  if (args.blocks !== undefined || args.plainText === true) {
+    // Blocks / plainText force the legacy text path (same as postMessage).
     return callSlackApiSoft({
       botToken: args.botToken,
       operation: "chat.update",
       body: {
         ...base,
         text: args.text,
-        blocks: args.blocks,
+        ...(args.blocks !== undefined ? { blocks: args.blocks } : {}),
       },
     });
   }
@@ -425,11 +472,13 @@ export async function updateMessage(args: {
     },
   });
   // Older workspaces / scopes may not accept markdown_text — retry with text.
+  // Also fall back when markdown_text hits its 12k limit (msg_too_long).
   if (
     !result.ok &&
     (result.error === "markdown_text_conflict" ||
       result.error === "invalid_arguments" ||
       result.error === "invalid_blocks" ||
+      result.error === "msg_too_long" ||
       SLACK_UNSUPPORTED_ERRORS.has(String(result.error ?? "")))
   ) {
     result = await callSlackApiSoft({

package/src/channels/slack/eval-directive.ts

@@ -26,6 +26,7 @@ export type SlackEvalDirective =
   | { kind: "help" }
   | { kind: "list" }
   | { kind: "status" }
+  | { kind: "cancel"; runId: string }
   | { kind: "run"; filterIds?: string[]; tags?: string[] };
 
 function escapeRegExp(value: string): string {
@@ -89,6 +90,13 @@ export function parseSlackEvalDirective(
   if (rest.length === 1 && head === "status") {
     return { kind: "status" };
   }
+  if (head === "cancel") {
+    const runId = rest[1];
+    if (rest.length !== 2 || runId === undefined || runId.length === 0) {
+      return { kind: "help" };
+    }
+    return { kind: "cancel", runId };
+  }
 
   const filterIds: string[] = [];
   const tags: string[] = [];
@@ -152,6 +160,7 @@ export function formatSlackEvalHelp(): string {
     "• `eval --tag <tag>` — run evals with a tag (`-t` also works)",
     "• `eval list` — list available evals",
     "• `eval status` — show the active or latest eval run",
+    "• `eval cancel <evalId>` — cancel a running eval batch",
     "• `eval help` — this message",
   ].join("\n");
 }
@@ -291,6 +300,20 @@ export async function handleSlackEvalDirective(args: {
     return;
   }
 
+  if (directive.kind === "cancel") {
+    try {
+      const run = await evals.cancel(directive.runId);
+      await safePost(
+        binding,
+        `Cancelled Eval ID \`${run.runId}\`.\n${formatSlackEvalRunSummary(run)}`
+      );
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      await safePost(binding, `Failed to cancel eval: ${message}`);
+    }
+    return;
+  }
+
   await safeTyping(binding, "Starting evals…");
   let run: EvalRunSnapshot;
   try {
@@ -323,7 +346,7 @@ export async function handleSlackEvalDirective(args: {
 
   await safePost(
     binding,
-    `Started eval run \`${run.runId}\` (${run.summary.total} case(s)).`
+    `Started Eval ID \`${run.runId}\` (${run.summary.total} case(s)). Cancel with \`eval cancel ${run.runId}\`.`
   );
 
   while (run.status === "running") {

package/src/evals.ts

@@ -48,7 +48,7 @@
 
 import { normalizePositiveInt } from "./internal/bounded-int.js";
 import { brandDefinition } from "./internal/brand.js";
-import type { SessionEvent } from "./types.js";
+import type { AgentCloudOptions, SessionEvent } from "./types.js";
 
 export type Expectation =
   | { kind: "includes"; value: string | RegExp }
@@ -93,6 +93,12 @@ export interface EvalSendOptions {
   workspaceDir?: string;
   /** Files seeded into the session workspace before the first turn. */
   workspaceFiles?: Record<string, string>;
+  /**
+   * Per-session cloud options merged over the agent's static `cloud` config
+   * (create only). Use this to attach a pinned fixture repo for cloud evals
+   * without putting it on the agent's default `cloud.repos`.
+   */
+  cloud?: AgentCloudOptions;
 }
 
 export interface EvalTestContext {
@@ -103,6 +109,14 @@ export interface EvalTestContext {
   readonly sessionId: string | undefined;
   /** All events captured so far across turns. */
   readonly events: SessionEvent[];
+  /**
+   * 1-based index of this run when {@link EvalConfig.iterations} /
+   * {@link EvalDatapoint.iterations} expands a datapoint. Always `1` when
+   * iterations is unset or `1`.
+   */
+  readonly iteration: number;
+  /** Total iterations for this datapoint (default `1`). */
+  readonly iterations: number;
   /** Gate: run did not fail. */
   succeeded(): void;
   /** Gate: at least one completed call to `name`. */
@@ -115,8 +129,17 @@ export interface EvalTestContext {
   check(value: unknown, assertion: Expectation): void;
   /** Debug line (surfaced under `agent-serve eval --verbose`). */
   log(message: string): void;
+  /**
+   * Structured score for the playground (and CLI). Values appear on the
+   * case card; agents can further customize rendering via
+   * `agent/playground/evals.tsx`.
+   */
+  metric(name: string, value: string | number): void;
 }
 
+/** Hard ceiling for {@link EvalConfig.iterations} / case `iterations`. */
+export const EVAL_MAX_ITERATIONS = 100;
+
 /** One runnable datapoint inside a multi-case {@link defineEval} file. */
 export interface EvalDatapoint {
   /**
@@ -128,6 +151,13 @@ export interface EvalDatapoint {
   description?: string;
   tags?: string[];
   timeoutMs?: number;
+  /**
+   * How many times to run this datapoint (default `1`, or the file-level
+   * {@link EvalConfig.iterations}). When greater than 1, discovery expands
+   * to `<fileId>/<id>/1` … `<fileId>/<id>/<n>` so each iteration is its own
+   * concurrent case.
+   */
+  iterations?: number;
   test: (t: EvalTestContext) => Promise<void>;
 }
 
@@ -139,6 +169,11 @@ export interface EvalConfig {
   description?: string;
   tags?: string[];
   timeoutMs?: number;
+  /**
+   * Default iterations for every datapoint in this file (default `1`).
+   * Case-level {@link EvalDatapoint.iterations} overrides this.
+   */
+  iterations?: number;
   /** Single-datapoint shorthand. Case id = file path under `evals/`. */
   test?: (t: EvalTestContext) => Promise<void>;
   /** Multiple datapoints in this file. Each id = `<fileId>/<case.id>`. */
@@ -150,6 +185,10 @@ export interface EvalCaseDefinition {
   description?: string;
   tags?: string[];
   timeoutMs?: number;
+  /** Total iterations configured for the authored datapoint. */
+  iterations?: number;
+  /** 1-based iteration index for this expanded runnable case. */
+  iteration?: number;
   test: (t: EvalTestContext) => Promise<void>;
   readonly __agentServe: "eval";
 }
@@ -161,6 +200,13 @@ export type EvalDefinition = EvalConfig & { readonly __agentServe: "eval" };
  * `evals/weather.eval.ts` → `weather`. With a single `test`, that is also
  * the case id. With `cases`, each datapoint is `<fileId>/<case.id>`.
  */
+export function normalizeEvalIterations(value: unknown): number {
+  return normalizePositiveInt(value, {
+    label: "defineEval: iterations",
+    max: EVAL_MAX_ITERATIONS,
+  });
+}
+
 export function defineEval(config: EvalConfig): EvalDefinition {
   const hasTest = typeof config.test === "function";
   const hasCases = config.cases !== undefined;
@@ -170,6 +216,9 @@ export function defineEval(config: EvalConfig): EvalDefinition {
   if (!hasTest && !hasCases) {
     throw new Error("defineEval requires test(t) or a non-empty cases array");
   }
+  if (config.iterations !== undefined) {
+    normalizeEvalIterations(config.iterations);
+  }
   if (hasCases) {
     if (!Array.isArray(config.cases) || config.cases.length === 0) {
       throw new Error("defineEval: cases must be a non-empty array");
@@ -188,6 +237,9 @@ export function defineEval(config: EvalConfig): EvalDefinition {
           `defineEval: cases[${JSON.stringify(datapoint.id)}] requires test(t)`
         );
       }
+      if (datapoint.iterations !== undefined) {
+        normalizeEvalIterations(datapoint.iterations);
+      }
     }
   }
   return brandDefinition("eval", config);
@@ -207,44 +259,75 @@ function validateDatapointId(id: unknown): asserts id is string {
 /**
  * Expand a file-level definition into runnable datapoints.
  * Exported for tests and tooling.
+ *
+ * When `iterations` > 1, each authored datapoint expands to
+ * `<fileId>/<case.id>/1` … `<fileId>/<case.id>/<n>` (single-test files
+ * expand to `<fileId>/1` … `<fileId>/<n>`).
  */
 export function expandEvalDefinition(
   fileId: string,
   definition: EvalDefinition
 ): Array<{ id: string; definition: EvalCaseDefinition }> {
   if (definition.cases !== undefined) {
-    return definition.cases.map((datapoint) => {
+    return definition.cases.flatMap((datapoint) => {
       const description = datapoint.description ?? definition.description;
       const timeoutMs = datapoint.timeoutMs ?? definition.timeoutMs;
-      return {
-        id: `${fileId}/${datapoint.id}`,
-        definition: brandDefinition("eval", {
-          ...(description !== undefined ? { description } : {}),
-          tags: datapoint.tags ?? definition.tags,
-          ...(timeoutMs !== undefined ? { timeoutMs } : {}),
-          test: datapoint.test,
-        }),
-      };
+      const iterations = normalizeEvalIterations(
+        datapoint.iterations ?? definition.iterations ?? 1
+      );
+      return expandIterations({
+        baseId: `${fileId}/${datapoint.id}`,
+        description,
+        tags: datapoint.tags ?? definition.tags,
+        timeoutMs,
+        iterations,
+        test: datapoint.test,
+      });
     });
   }
   if (typeof definition.test !== "function") {
     throw new Error(`defineEval(${fileId}): missing test(t)`);
   }
-  return [
-    {
-      id: fileId,
+  const iterations = normalizeEvalIterations(definition.iterations ?? 1);
+  return expandIterations({
+    baseId: fileId,
+    description: definition.description,
+    tags: definition.tags,
+    timeoutMs: definition.timeoutMs,
+    iterations,
+    test: definition.test,
+  });
+}
+
+function expandIterations(args: {
+  baseId: string;
+  description?: string;
+  tags?: string[];
+  timeoutMs?: number;
+  iterations: number;
+  test: (t: EvalTestContext) => Promise<void>;
+}): Array<{ id: string; definition: EvalCaseDefinition }> {
+  const out: Array<{ id: string; definition: EvalCaseDefinition }> = [];
+  for (let iteration = 1; iteration <= args.iterations; iteration++) {
+    const id =
+      args.iterations === 1 ? args.baseId : `${args.baseId}/${iteration}`;
+    const description =
+      args.iterations === 1 || args.description === undefined
+        ? args.description
+        : `${args.description} (iteration ${iteration}/${args.iterations})`;
+    out.push({
+      id,
       definition: brandDefinition("eval", {
-        ...(definition.description !== undefined
-          ? { description: definition.description }
-          : {}),
-        ...(definition.tags !== undefined ? { tags: definition.tags } : {}),
-        ...(definition.timeoutMs !== undefined
-          ? { timeoutMs: definition.timeoutMs }
-          : {}),
-        test: definition.test,
+        ...(description !== undefined ? { description } : {}),
+        ...(args.tags !== undefined ? { tags: args.tags } : {}),
+        ...(args.timeoutMs !== undefined ? { timeoutMs: args.timeoutMs } : {}),
+        iterations: args.iterations,
+        iteration,
+        test: args.test,
       }),
-    },
-  ];
+    });
+  }
+  return out;
 }
 
 /**
@@ -272,7 +355,7 @@ export const EVAL_DEFAULT_MAX_PLAYGROUND_RUNS = 20;
 export const EVAL_MAX_PLAYGROUND_RUNS = 500;
 
 export type EvalCaseRunStatus = "pending" | "running" | "done";
-export type EvalBatchStatus = "running" | "completed" | "failed";
+export type EvalBatchStatus = "running" | "completed" | "failed" | "cancelled";
 
 /** One case inside a playground / HTTP eval batch snapshot. */
 export interface EvalCaseSnapshot {
@@ -287,6 +370,8 @@ export interface EvalCaseSnapshot {
   durationMs?: number;
   error?: string;
   logs?: string[];
+  /** Structured scores from {@link EvalTestContext.metric}. */
+  metrics?: Record<string, string | number>;
   inputs?: string[];
   finalText?: string;
   tools?: string[];
@@ -356,6 +441,8 @@ export interface HostEvalsApi {
     activeRunId?: string;
   }>;
   start(input?: HostStartEvalRunInput): Promise<EvalRunSnapshot>;
+  /** Cancel a running batch by Eval ID (`runId`). */
+  cancel(runId: string): Promise<EvalRunSnapshot>;
 }
 
 export interface EvalConfigFile {