@cursor/july 0.1.16 → 0.1.18

package/src/evals.ts

@@ -44,48 +44,116 @@
  *   ],
  * });
  * ```
+ *
+ * ## Severity: gates and scores
+ *
+ * Every assertion returns a chainable handle, so severity rides on the
+ * assertion instead of a separate thresholds map. Gates are hard (a miss fails
+ * the eval and `eval` exits non-zero); soft assertions are tracked scores that
+ * mark the eval `scored` and only fail the build under `--strict`:
+ *
+ * ```ts
+ * t.succeeded();                                        // gate (default)
+ * t.calledTool("get_weather").soft();                   // tracked, no gate
+ * t.check(t.reply, similarity("Sunny, 72F")).atLeast(0.8);
+ * t.judge.closedQA("cites a source").atLeast(0.6);      // LLM-graded
+ * ```
+ *
+ * Matchers narrow tool assertions (`{ input, output, status, count }` accept a
+ * literal, a RegExp, or a predicate), and each turn returned by `t.send(...)`
+ * carries the same vocabulary scoped to that turn.
  */
 
+import type {
+  EvalAssertionResult,
+  EvalAssertions,
+  EvalVerdict,
+} from "./evals/assertions.js";
+import type { Expectation } from "./evals/expect.js";
+import type { EvalJudge, EvalJudgeConfig } from "./evals/judge.js";
+import type { EvalReporter } from "./evals/results.js";
 import { normalizePositiveInt } from "./internal/bounded-int.js";
 import { brandDefinition } from "./internal/brand.js";
 import type { AgentCloudOptions, SessionEvent } from "./types.js";
 
-export type Expectation =
-  | { kind: "includes"; value: string | RegExp }
-  | { kind: "equals"; value: unknown }
-  | {
-      kind: "satisfies";
-      predicate: (value: unknown) => boolean;
-      label: string;
-    };
-
-export function includes(value: string | RegExp): Expectation {
-  return { kind: "includes", value };
-}
-
-export function equals(value: unknown): Expectation {
-  return { kind: "equals", value };
-}
-
-export function satisfies(
-  predicate: (value: unknown) => boolean,
-  label: string
-): Expectation {
-  return { kind: "satisfies", predicate, label };
-}
+// Assertion severity, the scoped assertion vocabulary, value assertion
+// builders, the judge surface, and result/reporter types are all part of the
+// authoring surface: one import (`@cursor/july/evals`) covers a whole eval file.
+export type {
+  EvalAssertionHandle,
+  EvalAssertionResult,
+  EvalAssertionSeverity,
+  EvalAssertions,
+  EvalVerdict,
+} from "./evals/assertions.js";
+export { computeVerdict, verdictFailsRun } from "./evals/assertions.js";
+export type { EvalSchemaLike, Expectation } from "./evals/expect.js";
+export {
+  equals,
+  includes,
+  matches,
+  normalizedSimilarity,
+  satisfies,
+  similarity,
+} from "./evals/expect.js";
+export type {
+  EvalJudge,
+  EvalJudgeCallOptions,
+  EvalJudgeConfig,
+  EvalJudgeGraderName,
+} from "./evals/judge.js";
+// A custom judge built on `t.judge.model` embeds the same attacker-controlled
+// agent output the built-in graders do, so it needs the same two protections.
+export {
+  EVAL_JUDGE_INJECTION_GUARD,
+  fenceUntrusted,
+} from "./evals/judge.js";
+export type {
+  EvalCountMatcher,
+  EvalEventMatcher,
+  EvalEventOrderMatcher,
+  EvalMatcher,
+  EvalSubagentMatcher,
+  EvalToolCallMatcher,
+  EvalToolCallStatus,
+} from "./evals/matchers.js";
+export type {
+  EvalReporter,
+  EvalReporterEvalInfo,
+  EvalReporterTarget,
+  EvalRunResult,
+  EvalRunSummary,
+} from "./evals/results.js";
+export type {
+  EvalInputRequestFact,
+  EvalRunFacts,
+  EvalSubagentFact,
+  EvalToolCallFact,
+} from "./evals/run-facts.js";
+export { deriveRunFacts } from "./evals/run-facts.js";
 
-export interface EvalAssertionResult {
-  name: string;
-  passed: boolean;
-  detail?: string;
-}
-
-export interface EvalTurnResult {
+/**
+ * One settled turn. Carries the same assertion vocabulary as `t`, narrowed to
+ * this turn's events, so an intermediate turn can be graded before the next
+ * `send` overwrites `t.reply`:
+ *
+ * ```ts
+ * const draft = await t.send("Draft the summary.");
+ * draft.calledTool("read_file");
+ * t.check(draft.message, includes("Summary"));
+ * ```
+ */
+export interface EvalTurnResult extends EvalAssertions {
   message: string;
   sessionId: string;
   events: SessionEvent[];
+  /** Tool names requested during this turn, in order. */
   toolCalls: string[];
   ok: boolean;
+  /** 1-based turn index within the case. */
+  index: number;
+  /** Throws when this turn failed. Only needed when later steps depend on it. */
+  expectOk(): EvalTurnResult;
 }
 
 export interface EvalSendOptions {
@@ -101,7 +169,15 @@ export interface EvalSendOptions {
   cloud?: AgentCloudOptions;
 }
 
-export interface EvalTestContext {
+/**
+ * The eval driver and assertion surface.
+ *
+ * `t` both drives the agent and grades it. Assertions on `t` read the whole run
+ * once `test` finishes; assertions on the turn returned by `t.send(...)` read
+ * only that turn. Recorded assertions never throw, so one run reports every
+ * failure rather than dying on the first.
+ */
+export interface EvalTestContext extends EvalAssertions {
   /** Send a turn and wait until the session parks (or fails). */
   send(message: string, options?: EvalSendOptions): Promise<EvalTurnResult>;
   /** Last assistant text, or null before the first send. */
@@ -109,6 +185,8 @@ export interface EvalTestContext {
   readonly sessionId: string | undefined;
   /** All events captured so far across turns. */
   readonly events: SessionEvent[];
+  /** Turns settled so far, oldest first. */
+  readonly turns: readonly EvalTurnResult[];
   /**
    * 1-based index of this run when {@link EvalConfig.iterations} /
    * {@link EvalDatapoint.iterations} expands a datapoint. Always `1` when
@@ -117,16 +195,25 @@ export interface EvalTestContext {
   readonly iteration: number;
   /** Total iterations for this datapoint (default `1`). */
   readonly iterations: number;
-  /** Gate: run did not fail. */
-  succeeded(): void;
-  /** Gate: at least one completed call to `name`. */
-  calledTool(name: string): void;
-  /** Gate: no call to `name`. */
-  notCalledTool(name: string): void;
-  /** Gate: joined assistant text matches. */
-  messageIncludes(token: string | RegExp): void;
-  /** Gate a value with an {@link Expectation} builder. */
-  check(value: unknown, assertion: Expectation): void;
+  /** Aborts when the case hits its timeout. */
+  readonly signal: AbortSignal;
+  /**
+   * LLM-as-judge assertions. Soft by default (tracked, never fatal); add a bar
+   * with `.atLeast(0.7)` or promote with `.gate(0.8)`. Requires a judge model
+   * from `defineEvalConfig({ judge })`, `defineEval({ judge })`, or the call.
+   */
+  readonly judge: EvalJudge;
+  /**
+   * Record a gate that must pass before the script can safely continue.
+   * Returns `value` when it passes and stops the test body when it does not,
+   * without adding a duplicate execution error.
+   */
+  require<T>(value: T, assertion: Expectation): Promise<T>;
+  /**
+   * End the case as skipped. Skipped cases are reported separately and never
+   * change the exit code. Call it before sending messages.
+   */
+  skip(reason: string): never;
   /** Debug line (surfaced under `agent-serve eval --verbose`). */
   log(message: string): void;
   /**
@@ -158,6 +245,12 @@ export interface EvalDatapoint {
    * concurrent case.
    */
   iterations?: number;
+  /** Free-form data carried onto the result and every reporter. */
+  metadata?: Record<string, unknown>;
+  /** Judge model for this datapoint (overrides `evals.config.ts`). */
+  judge?: EvalJudgeConfig;
+  /** Destinations for just this datapoint, in addition to config reporters. */
+  reporters?: EvalReporter[];
   test: (t: EvalTestContext) => Promise<void>;
 }
 
@@ -174,6 +267,12 @@ export interface EvalConfig {
    * Case-level {@link EvalDatapoint.iterations} overrides this.
    */
   iterations?: number;
+  /** Free-form data carried onto every result in this file and to reporters. */
+  metadata?: Record<string, unknown>;
+  /** Judge model for this file (overrides `evals.config.ts`). */
+  judge?: EvalJudgeConfig;
+  /** Destinations for this file, in addition to config reporters. */
+  reporters?: EvalReporter[];
   /** Single-datapoint shorthand. Case id = file path under `evals/`. */
   test?: (t: EvalTestContext) => Promise<void>;
   /** Multiple datapoints in this file. Each id = `<fileId>/<case.id>`. */
@@ -189,6 +288,9 @@ export interface EvalCaseDefinition {
   iterations?: number;
   /** 1-based iteration index for this expanded runnable case. */
   iteration?: number;
+  metadata?: Record<string, unknown>;
+  judge?: EvalJudgeConfig;
+  reporters?: EvalReporter[];
   test: (t: EvalTestContext) => Promise<void>;
   readonly __agentServe: "eval";
 }
@@ -275,12 +377,19 @@ export function expandEvalDefinition(
       const iterations = normalizeEvalIterations(
         datapoint.iterations ?? definition.iterations ?? 1
       );
+      const metadata =
+        datapoint.metadata === undefined && definition.metadata === undefined
+          ? undefined
+          : { ...definition.metadata, ...datapoint.metadata };
       return expandIterations({
         baseId: `${fileId}/${datapoint.id}`,
         description,
         tags: datapoint.tags ?? definition.tags,
         timeoutMs,
         iterations,
+        metadata,
+        judge: datapoint.judge ?? definition.judge,
+        reporters: mergeReporters(definition.reporters, datapoint.reporters),
         test: datapoint.test,
       });
     });
@@ -295,16 +404,36 @@ export function expandEvalDefinition(
     tags: definition.tags,
     timeoutMs: definition.timeoutMs,
     iterations,
+    metadata: definition.metadata,
+    judge: definition.judge,
+    reporters: definition.reporters,
     test: definition.test,
   });
 }
 
+/** Union file-level and case-level reporters, keeping each instance once. */
+function mergeReporters(
+  fileLevel: EvalReporter[] | undefined,
+  caseLevel: EvalReporter[] | undefined
+): EvalReporter[] | undefined {
+  if (fileLevel === undefined) {
+    return caseLevel;
+  }
+  if (caseLevel === undefined) {
+    return fileLevel;
+  }
+  return [...new Set([...fileLevel, ...caseLevel])];
+}
+
 function expandIterations(args: {
   baseId: string;
   description?: string;
   tags?: string[];
   timeoutMs?: number;
   iterations: number;
+  metadata?: Record<string, unknown>;
+  judge?: EvalJudgeConfig;
+  reporters?: EvalReporter[];
   test: (t: EvalTestContext) => Promise<void>;
 }): Array<{ id: string; definition: EvalCaseDefinition }> {
   const out: Array<{ id: string; definition: EvalCaseDefinition }> = [];
@@ -321,6 +450,9 @@ function expandIterations(args: {
         ...(description !== undefined ? { description } : {}),
         ...(args.tags !== undefined ? { tags: args.tags } : {}),
         ...(args.timeoutMs !== undefined ? { timeoutMs: args.timeoutMs } : {}),
+        metadata: args.metadata,
+        judge: args.judge,
+        reporters: args.reporters,
         iterations: args.iterations,
         iteration,
         test: args.test,
@@ -409,6 +541,10 @@ export interface EvalCaseSnapshot {
   tags?: string[];
   status: EvalCaseRunStatus;
   ok?: boolean;
+  /** Full grade, including `scored` (soft threshold miss) and `skipped`. */
+  verdict?: EvalVerdict;
+  /** Reason passed to `t.skip(...)`. */
+  skipReason?: string;
   assertions?: EvalAssertionResult[];
   sessionId?: string;
   durationMs?: number;
@@ -416,6 +552,8 @@ export interface EvalCaseSnapshot {
   logs?: string[];
   /** Structured scores from {@link EvalTestContext.metric}. */
   metrics?: Record<string, string | number>;
+  /** Author-supplied metadata from `defineEval({ metadata })`. */
+  metadata?: Record<string, unknown>;
   inputs?: string[];
   finalText?: string;
   tools?: string[];
@@ -434,7 +572,19 @@ export interface EvalRunSnapshot {
   startedAt: string;
   finishedAt?: string;
   error?: string;
-  summary: { passed: number; failed: number; total: number; done: number };
+  summary: {
+    passed: number;
+    failed: number;
+    /**
+     * Cases that only missed a soft threshold (fatal only under `--strict`).
+     * Optional: a batch persisted by an older serve process has no such count.
+     */
+    scored?: number;
+    /** Cases ended by `t.skip(...)`; they never change the exit code. */
+    skipped?: number;
+    total: number;
+    done: number;
+  };
   cases: EvalCaseSnapshot[];
   config: {
     maxConcurrency?: number;
@@ -497,6 +647,17 @@ export interface EvalConfigFile {
    * request limits.
    */
   maxConcurrency: number;
+  /**
+   * Default judge model for `t.judge.*` across this tree. Optional: a tree of
+   * fully deterministic evals never needs one.
+   */
+  judge?: EvalJudgeConfig;
+  /**
+   * Destinations that observe **every** case in the run — the right place for a
+   * shared one, so you do not repeat it in each eval file. `--skip-report`
+   * suppresses them for a local iteration loop.
+   */
+  reporters?: EvalReporter[];
   /**
    * Optional. Max batches kept in the **playground / `/v1/dev/evals*`**
    * history window (in-memory, and pruned from the `defineStorage` evals
@@ -523,6 +684,8 @@ export function defineEvalConfig(config: EvalConfigFile): EvalConfigFile {
     maxPlaygroundRuns: normalizeEvalMaxPlaygroundRuns(
       config.maxPlaygroundRuns ?? EVAL_DEFAULT_MAX_PLAYGROUND_RUNS
     ),
+    judge: config.judge,
+    reporters: config.reporters,
   };
 }
 

package/src/index.ts

@@ -124,12 +124,14 @@ export {
 export { defineTool } from "./tools.js";
 export type * from "./types.js";
 export {
+  cursorAccountServersHasMixedWildcard,
   DEFAULT_MODEL,
   DIRECT_TOOL_CALL_CHANNEL_ID,
   isCursorAccountTransport,
   isPeerConnectionTransport,
   isSymbolicConnectionTransport,
   MAX_RUNNING_TURNS_LIMIT,
+  normalizeCursorAccountServers,
 } from "./types.js";
 
 /**

package/src/internal/builtin-tools/index.ts

@@ -0,0 +1,40 @@
+/**
+ * Built-in tools (`AgentConfig.builtinTools`): framework-provided
+ * model-facing tools, materialized as ordinary server tools at discovery
+ * time so turns, direct calls, `info`, and the playground treat them
+ * exactly like authored tools.
+ */
+
+import type {
+  BuiltinToolsConfig,
+  DiscoveredTool,
+  ToolDefinition,
+} from "../../types.js";
+import { toolInputJsonSchema } from "../json-schema.js";
+import { reminderTools } from "./reminders.js";
+
+/** Resolve the enabled built-in tools, in stable catalog order. */
+export function builtinTools(
+  config: BuiltinToolsConfig | undefined
+): DiscoveredTool[] {
+  const tools: DiscoveredTool[] = [];
+  if (config?.reminders === true) {
+    tools.push(...toDiscovered(reminderTools));
+  }
+  return tools;
+}
+
+function toDiscovered(
+  byName: Record<string, ToolDefinition>
+): DiscoveredTool[] {
+  return Object.entries(byName).map(([name, definition]) => ({
+    name,
+    description: definition.description,
+    inputSchema: toolInputJsonSchema(definition.inputSchema),
+    execution: "server",
+    needsApproval:
+      definition.needsApproval === true ||
+      typeof definition.needsApproval === "function",
+    definition,
+  }));
+}

package/src/internal/builtin-tools/reminders.ts

@@ -0,0 +1,158 @@
+/**
+ * Built-in reminder tools (`builtinTools: { reminders: true }`): durable
+ * wakes bound to the current conversation over `host.reminders`. The
+ * framework's reminder fire prompt tells the model to call
+ * `reminders_cancel` by name, so these tool names must not change.
+ */
+
+import { z } from "zod";
+import { defineTool } from "../../tools.js";
+import type {
+  ReminderHostApi,
+  ReminderInfo,
+  ToolContext,
+  ToolDefinition,
+} from "../../types.js";
+import { parseDuration } from "../duration.js";
+
+/** Floor for model-armed recurring wakes: each fire spends a model turn. */
+const MIN_EVERY_MS = 60_000;
+
+function api(ctx: ToolContext): ReminderHostApi {
+  const reminders = ctx.host.reminders;
+  if (reminders === undefined) {
+    throw new Error("Reminders are disabled on this host.");
+  }
+  return reminders;
+}
+
+function conversation(ctx: ToolContext): {
+  channelId: string;
+  continuationToken: string;
+} {
+  const token = ctx.session.continuationKey;
+  if (token == null) {
+    throw new Error(
+      "This session is not bound to a followable conversation; reminders cannot wake it."
+    );
+  }
+  return { channelId: ctx.session.channelId, continuationToken: token };
+}
+
+/** Reminders visible to this conversation (tokens are channel-local). */
+async function listMine(
+  ctx: ToolContext,
+  status?: ReminderInfo["status"]
+): Promise<ReminderInfo[]> {
+  const conv = conversation(ctx);
+  const all = await api(ctx).list({
+    status,
+    continuationToken: conv.continuationToken,
+  });
+  return all.filter((r) => r.channelId === conv.channelId);
+}
+
+function toResult(r: ReminderInfo) {
+  return {
+    reminderId: r.id,
+    purpose: r.purpose,
+    schedule: `${r.scheduleKind} ${r.scheduleValue}`,
+    nextFireAt: r.nextFireAt,
+    until: r.untilPrompt,
+  };
+}
+
+const create = defineTool({
+  description:
+    "Set a reminder for yourself in this conversation. When it fires, your prompt returns here as a follow-up turn. Use it to re-check state that needs time to settle. One-shot (delay) or recurring (every).",
+  inputSchema: z.object({
+    prompt: z
+      .string()
+      .min(1)
+      .describe(
+        "What to do when you wake. Keep it generic ('re-check CI and report only if the status changed') and re-read live state on wake instead of replaying stale details."
+      ),
+    delay: z
+      .string()
+      .optional()
+      .describe(
+        'One-shot: wake once after this duration, e.g. "20m" or "2h". Provide exactly one of delay or every.'
+      ),
+    every: z
+      .string()
+      .optional()
+      .describe(
+        'Recurring: wake at this interval, e.g. "30m", until cancelled or the stop condition is met. Provide exactly one of delay or every.'
+      ),
+    purpose: z
+      .string()
+      .min(1)
+      .max(64)
+      .optional()
+      .describe(
+        'Short slug naming the reminder, e.g. "ci_recheck". Defaults to "follow_up".'
+      ),
+    until: z
+      .string()
+      .optional()
+      .describe(
+        "Recurring only: plain-language stop condition you will honor by calling reminders_cancel, e.g. 'Cancel once CI is green or the PR is merged.'"
+      ),
+  }),
+  async execute(input, ctx) {
+    if ((input.delay !== undefined) === (input.every !== undefined)) {
+      throw new Error("Provide exactly one of delay or every.");
+    }
+    if (
+      input.every !== undefined &&
+      parseDuration(input.every) < MIN_EVERY_MS
+    ) {
+      throw new Error('Recurring reminders must use every >= "1m".');
+    }
+    const conv = conversation(ctx);
+    const reminder = await api(ctx).create({
+      purpose: input.purpose ?? "follow_up",
+      channelId: conv.channelId,
+      continuationToken: conv.continuationToken,
+      prompt: input.prompt,
+      delay: input.delay,
+      every: input.every,
+      until: input.until,
+    });
+    return toResult(reminder);
+  },
+});
+
+const list = defineTool({
+  description:
+    "List the armed reminders bound to this conversation. Check before arming duplicates.",
+  inputSchema: z.object({}),
+  async execute(_input, ctx) {
+    return { reminders: (await listMine(ctx, "armed")).map(toResult) };
+  },
+});
+
+const cancel = defineTool({
+  description:
+    "Cancel one of this conversation's reminders by id, the moment its stop condition is met or it is no longer needed.",
+  inputSchema: z.object({
+    id: z
+      .string()
+      .min(1)
+      .describe("Reminder id from reminders_create or reminders_list."),
+  }),
+  async execute(input, ctx) {
+    const mine = (await listMine(ctx)).find((r) => r.id === input.id);
+    if (mine === undefined) {
+      throw new Error(`No reminder "${input.id}" in this conversation.`);
+    }
+    const cancelled = await api(ctx).cancel(input.id);
+    return { reminderId: input.id, status: cancelled?.status ?? "cancelled" };
+  },
+});
+
+export const reminderTools: Record<string, ToolDefinition> = {
+  reminders_create: create,
+  reminders_list: list,
+  reminders_cancel: cancel,
+};