@cursor/july 0.1.10 → 0.1.12

package/src/channels/slack/eval-directive.ts

@@ -0,0 +1,348 @@
+/**
+ * Built-in Slack `eval` directive — run playground evals from a mention/DM
+ * without per-agent custom handlers (parity with CLI / playground runs).
+ *
+ * Examples (after stripping @mentions / cursorAccount agent name):
+ *   eval
+ *   eval triage/open-incidents smoke
+ *   eval --tag oncall
+ *   eval list
+ *   eval status
+ *   eval help
+ */
+
+import type { EvalRunSnapshot, HostEvalsApi } from "../../evals.js";
+import {
+  EvalRunConflictError,
+  EvalRunEmptyError,
+} from "../../internal/eval-run-store.js";
+import { stripLeadingSlackMentions } from "./stop.js";
+import type { SlackBinding } from "./types.js";
+
+const EVAL_POLL_MS = 2_000;
+const TYPING_STATUS_MAX = 50;
+
+export type SlackEvalDirective =
+  | { kind: "help" }
+  | { kind: "list" }
+  | { kind: "status" }
+  | { kind: "run"; filterIds?: string[]; tags?: string[] };
+
+function escapeRegExp(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * Normalize inbound Slack text for directive parsing: strip leading
+ * `<@U…>` mentions, leading `@U…` markdown mentions, and an optional
+ * cursorAccount agentName prefix (`OncallBuddy eval …`).
+ */
+export function normalizeSlackEvalDirectiveBody(
+  text: string,
+  options?: { agentName?: string }
+): string {
+  let body = stripLeadingSlackMentions(text);
+  // markdown form from slackTextToMarkdown / cursorAccount envelopes
+  while (/^@[A-Za-z0-9._-]+\s+/.test(body)) {
+    body = body.replace(/^@[A-Za-z0-9._-]+\s+/, "").trim();
+  }
+  const agentName = options?.agentName?.trim();
+  if (agentName !== undefined && agentName.length > 0) {
+    const prefix = new RegExp(`^${escapeRegExp(agentName)}\\s+`, "i");
+    body = body.replace(prefix, "").trim();
+  }
+  return body;
+}
+
+/**
+ * Parse a Slack eval directive. Returns null when the message is not an
+ * eval command (falls through to the normal agent handler).
+ */
+export function parseSlackEvalDirective(
+  text: string,
+  options?: { agentName?: string }
+): SlackEvalDirective | null {
+  const body = normalizeSlackEvalDirectiveBody(text, options);
+  if (body.length === 0) {
+    return null;
+  }
+  const tokens = body.split(/\s+/).filter((t) => t.length > 0);
+  if (tokens.length === 0 || !/^eval$/i.test(tokens[0] ?? "")) {
+    return null;
+  }
+
+  const rest = tokens.slice(1);
+  if (rest.length === 0) {
+    return { kind: "run" };
+  }
+
+  const head = (rest[0] ?? "").toLowerCase();
+  if (
+    rest.length === 1 &&
+    (head === "help" || head === "-h" || head === "--help")
+  ) {
+    return { kind: "help" };
+  }
+  if (rest.length === 1 && (head === "list" || head === "ls")) {
+    return { kind: "list" };
+  }
+  if (rest.length === 1 && head === "status") {
+    return { kind: "status" };
+  }
+
+  const filterIds: string[] = [];
+  const tags: string[] = [];
+  for (let i = 0; i < rest.length; i++) {
+    const tok = rest[i] ?? "";
+    if (tok === "--tag" || tok === "-t") {
+      const value = rest[i + 1];
+      if (value === undefined || value.length === 0) {
+        return { kind: "help" };
+      }
+      tags.push(value);
+      i += 1;
+      continue;
+    }
+    if (tok.startsWith("--tag=")) {
+      const value = tok.slice("--tag=".length);
+      if (value.length === 0) {
+        return { kind: "help" };
+      }
+      tags.push(value);
+      continue;
+    }
+    if (tok.startsWith("-t=") && tok.length > 3) {
+      tags.push(tok.slice(3));
+      continue;
+    }
+    if (tok.startsWith("-")) {
+      // Unknown flag — surface help rather than treating as an eval id.
+      return { kind: "help" };
+    }
+    filterIds.push(tok);
+  }
+
+  const run: SlackEvalDirective = { kind: "run" };
+  if (filterIds.length > 0) {
+    run.filterIds = filterIds;
+  }
+  if (tags.length > 0) {
+    run.tags = tags;
+  }
+  return run;
+}
+
+/** True when either raw Slack text or markdown form is an eval directive. */
+export function isSlackEvalDirective(
+  text: string,
+  markdown: string,
+  options?: { agentName?: string }
+): boolean {
+  return (
+    parseSlackEvalDirective(text, options) !== null ||
+    parseSlackEvalDirective(markdown, options) !== null
+  );
+}
+
+export function formatSlackEvalHelp(): string {
+  return [
+    "*Eval commands* (same batches as the playground / `agentkit eval --prod`):",
+    "• `eval` — run all evals",
+    "• `eval <id> [id…]` — run specific eval ids",
+    "• `eval --tag <tag>` — run evals with a tag (`-t` also works)",
+    "• `eval list` — list available evals",
+    "• `eval status` — show the active or latest eval run",
+    "• `eval help` — this message",
+  ].join("\n");
+}
+
+export function formatSlackEvalList(
+  evals: Array<{
+    id: string;
+    description?: string;
+    tags?: string[];
+  }>
+): string {
+  if (evals.length === 0) {
+    return "No evals found under `evals/`.";
+  }
+  const lines = [`*${evals.length} eval(s):*`];
+  for (const item of evals) {
+    const tags =
+      item.tags === undefined || item.tags.length === 0
+        ? ""
+        : ` _[${item.tags.join(", ")}]_`;
+    const desc =
+      item.description === undefined || item.description === ""
+        ? ""
+        : ` — ${item.description}`;
+    lines.push(`• \`${item.id}\`${tags}${desc}`);
+  }
+  return lines.join("\n");
+}
+
+export function formatSlackEvalRunSummary(run: EvalRunSnapshot): string {
+  const { summary } = run;
+  const statusLabel =
+    run.status === "completed"
+      ? "completed"
+      : run.status === "failed"
+        ? "failed"
+        : "running";
+  const lines = [
+    `*Eval run* \`${run.runId}\` — *${statusLabel}*`,
+    `• ${summary.passed} passed, ${summary.failed} failed, ${summary.total} total` +
+      (summary.done < summary.total
+        ? ` (${summary.done}/${summary.total} done)`
+        : ""),
+  ];
+  if (run.error !== undefined && run.error !== "") {
+    lines.push(`• error: ${run.error}`);
+  }
+  for (const c of run.cases) {
+    const mark =
+      c.status !== "done"
+        ? c.status === "running"
+          ? "RUN "
+          : "….  "
+        : c.ok === true
+          ? "PASS"
+          : "FAIL";
+    const dur =
+      c.durationMs === undefined
+        ? ""
+        : ` (${(c.durationMs / 1000).toFixed(1)}s)`;
+    const err =
+      c.ok === false && c.error !== undefined && c.error !== ""
+        ? ` — ${c.error}`
+        : "";
+    lines.push(`\`${mark}\` \`${c.id}\`${dur}${err}`);
+  }
+  return lines.join("\n");
+}
+
+function truncateTyping(status: string): string {
+  if (status.length <= TYPING_STATUS_MAX) {
+    return status;
+  }
+  return `${status.slice(0, TYPING_STATUS_MAX - 1)}…`;
+}
+
+async function sleep(ms: number): Promise<void> {
+  await new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+
+async function safeTyping(
+  binding: SlackBinding,
+  status: string
+): Promise<void> {
+  try {
+    await binding.thread.startTyping(truncateTyping(status));
+  } catch {
+    // best-effort status
+  }
+}
+
+async function safePost(binding: SlackBinding, text: string): Promise<void> {
+  try {
+    await binding.thread.post(text);
+  } catch {
+    // best-effort reply
+  }
+}
+
+/**
+ * Handle a parsed Slack eval directive against {@link HostEvalsApi}.
+ * Posts results in-thread; does not start an agent turn.
+ */
+export async function handleSlackEvalDirective(args: {
+  directive: SlackEvalDirective;
+  evals: HostEvalsApi;
+  binding: SlackBinding;
+  /** Override poll interval (tests). */
+  pollMs?: number;
+}): Promise<void> {
+  const { directive, evals, binding } = args;
+  const pollMs = args.pollMs ?? EVAL_POLL_MS;
+
+  if (directive.kind === "help") {
+    await safePost(binding, formatSlackEvalHelp());
+    return;
+  }
+
+  if (directive.kind === "list") {
+    const listed = await evals.list();
+    await safePost(binding, formatSlackEvalList(listed.evals));
+    return;
+  }
+
+  if (directive.kind === "status") {
+    const { runs, activeRunId } = await evals.listRunsWithActive();
+    const run =
+      (activeRunId === undefined ? undefined : await evals.get(activeRunId)) ??
+      runs[0];
+    if (run === undefined) {
+      await safePost(binding, "No eval runs yet. Try `eval list` then `eval`.");
+      return;
+    }
+    await safePost(binding, formatSlackEvalRunSummary(run));
+    return;
+  }
+
+  await safeTyping(binding, "Starting evals…");
+  let run: EvalRunSnapshot;
+  try {
+    run = await evals.start({
+      filterIds: directive.filterIds,
+      tags: directive.tags,
+    });
+  } catch (error) {
+    if (error instanceof EvalRunConflictError) {
+      const active = await evals.get(error.activeRunId);
+      await safePost(
+        binding,
+        active === undefined
+          ? `An eval run is already in progress (\`${error.activeRunId}\`).`
+          : `An eval run is already in progress:\n${formatSlackEvalRunSummary(active)}`
+      );
+      return;
+    }
+    if (error instanceof EvalRunEmptyError) {
+      await safePost(
+        binding,
+        "No matching evals. Try `eval list` or `eval help`."
+      );
+      return;
+    }
+    const message = error instanceof Error ? error.message : String(error);
+    await safePost(binding, `Failed to start evals: ${message}`);
+    return;
+  }
+
+  await safePost(
+    binding,
+    `Started eval run \`${run.runId}\` (${run.summary.total} case(s)).`
+  );
+
+  while (run.status === "running") {
+    await safeTyping(
+      binding,
+      `Evals ${run.summary.done}/${run.summary.total}…`
+    );
+    await sleep(pollMs);
+    const next = await evals.get(run.runId);
+    if (next === undefined) {
+      await safePost(
+        binding,
+        `Eval run \`${run.runId}\` disappeared before completion.`
+      );
+      return;
+    }
+    run = next;
+  }
+
+  await safePost(binding, formatSlackEvalRunSummary(run));
+  await safeTyping(binding, "Evals done.");
+}

package/src/channels/slack/index.ts

@@ -29,6 +29,7 @@ export * from "./cursor-account.js";
 export * from "./defaults.js";
 export * from "./dispatch.js";
 export * from "./doctor.js";
+export * from "./eval-directive.js";
 export * from "./event-id-dedupe.js";
 export * from "./inbound.js";
 export * from "./init.js";

package/src/channels/slack/slack-channel.ts

@@ -138,6 +138,7 @@ export function slackChannel<const Name extends string = string>(
   const engagement = options.engagement ?? {};
   const mentionsEnabled = engagement.mentions !== false;
   const directMessagesEnabled = engagement.directMessages !== false;
+  const evalCommandsEnabled = options.evalCommands !== false;
   const channelPosts = engagement.channelPosts;
   if (channelPosts !== undefined) {
     const entries = channelPosts.allow.filter((entry) => entry.trim() !== "");
@@ -204,6 +205,7 @@ export function slackChannel<const Name extends string = string>(
       log: getLog(),
       mentionsEnabled,
       directMessagesEnabled,
+      evalCommandsEnabled,
       ...(args.hasContinuationSession === undefined
         ? {}
         : { hasContinuationSession: args.hasContinuationSession }),
@@ -342,6 +344,7 @@ export function slackChannel<const Name extends string = string>(
                   credentials,
                   host: args.host,
                   log: getLog(),
+                  evalCommandsEnabled,
                   ...(args.hasContinuationSession === undefined
                     ? {}
                     : {

package/src/channels/slack/types.ts

@@ -382,6 +382,13 @@ export interface SlackChannelOptions {
    * entry at 50 chars). Defaults to Thinking… / Working on it… / Almost there….
    */
   loadingMessages?: string[];
+  /**
+   * Built-in Slack `eval` / `eval list` / `eval status` directive that runs
+   * playground eval batches (`host.evals` / `/v1/dev/evals/runs`) without an
+   * agent turn. Defaults to `true`. Set `false` to leave `eval …` for the
+   * model handler instead.
+   */
+  evalCommands?: boolean;
   /** Optional teardown (e.g. stop example-local debouncers). Runs after Socket Mode stops. */
   onStop?: () => void | Promise<void>;
 }

package/src/connections.ts

@@ -2,11 +2,9 @@
  * `defineConnection` — authored at `agent/mcp-connections/<name>.ts`.
  *
  * A connection gives tools from an external MCP server. The filename
- * becomes the connection (server) name. Every connection is always
- * available in three places:
- * - the Cursor agent (local harness or cloud), via SDK `mcpServers`
- * - the agent-serve host, via `ctx.host.mcp` on in-process tools
- * - channel / schedule handlers, via `args.host.mcp`
+ * becomes the connection (server) name. Available to the agent (SDK
+ * `mcpServers`), host (`ctx.host.mcp`), and channel/schedule handlers.
+ * Pass `hostOnly: true` to keep it off the model (still on `host.mcp`).
  *
  * ```ts
  * import { defineConnection } from "@anysphere/agent-serve/connections";
@@ -15,6 +13,13 @@
  *   url: "https://mcp.linear.app/mcp",
  *   headers: { authorization: `Bearer ${process.env.LINEAR_TOKEN}` },
  * });
+ *
+ * // privileged connector, model cannot call directly:
+ * export default defineConnection({
+ *   cursorAccount: true,
+ *   servers: ["anytool"],
+ *   hostOnly: true,
+ * });
  * ```
  *
  * A **Cursor account MCP connection** exposes the MCP connectors the operator

package/src/evals.ts

@@ -315,6 +315,49 @@ export interface EvalRunSnapshot {
   };
 }
 
+/** One discovered eval datapoint as returned by {@link HostEvalsApi.list}. */
+export interface HostEvalListItem {
+  id: string;
+  fileId: string;
+  description?: string;
+  tags?: string[];
+}
+
+/** Config slice returned with {@link HostEvalsApi.list}. */
+export interface HostEvalListConfig {
+  maxConcurrency?: number;
+  timeoutMs?: number;
+  maxPlaygroundRuns: number;
+  /** True when `agent/storage.ts` declares an `evals` table. */
+  durableRuns: boolean;
+}
+
+/** Input for {@link HostEvalsApi.start} (same filters as `POST /v1/dev/evals/runs`). */
+export interface HostStartEvalRunInput {
+  filterIds?: string[];
+  tags?: string[];
+  /** Per-case timeout override (same semantics as CLI `--timeout-ms`). */
+  timeoutMs?: number;
+}
+
+/**
+ * Host-facing playground eval batches — the same surface as
+ * `/v1/dev/evals*` and `agentkit eval --prod`. Attached to
+ * {@link import("./types.js").HostContext.evals} by `serve`.
+ */
+export interface HostEvalsApi {
+  list(): Promise<{
+    evals: HostEvalListItem[];
+    config: HostEvalListConfig;
+  }>;
+  get(runId: string): Promise<EvalRunSnapshot | undefined>;
+  listRunsWithActive(): Promise<{
+    runs: EvalRunSnapshot[];
+    activeRunId?: string;
+  }>;
+  start(input?: HostStartEvalRunInput): Promise<EvalRunSnapshot>;
+}
+
 export interface EvalConfigFile {
   timeoutMs?: number;
   /**

package/src/index.ts

@@ -89,6 +89,8 @@ export {
   UnknownSessionError,
   UnknownToolError,
 } from "./internal/session-engine.js";
+export type { PromptTag } from "./prompt.js";
+export { prompt } from "./prompt.js";
 export type {
   ReminderBusyPolicy,
   ReminderCreateInput,
@@ -127,6 +129,7 @@ export {
   isCursorAccountTransport,
   isPeerConnectionTransport,
   isSymbolicConnectionTransport,
+  MAX_RUNNING_TURNS_LIMIT,
 } from "./types.js";
 
 /**