@cursor/july 0.1.1

package/src/evals.ts

@@ -0,0 +1,501 @@
+/**
+ * Filesystem-first evals for agent-serve projects.
+ *
+ * Author cases under evals/ as `*.eval.ts` files with {@link defineEval}.
+ * One file may hold a single datapoint (`test`) or many (`cases`). Drive the
+ * agent with `t.send(...)` and assert with `t.succeeded()`, `t.calledTool()`,
+ * `t.messageIncludes(...)`, or `t.check(value, includes(...))`.
+ *
+ * ```ts
+ * import { defineEval, includes } from "@anysphere/agent-serve/evals";
+ *
+ * // One datapoint — id is the file path (evals/weather/nyc.eval.ts → weather/nyc)
+ * export default defineEval({
+ *   description: "Weather tool is used for temperature questions.",
+ *   async test(t) {
+ *     await t.send("What's the temperature in NYC?");
+ *     t.succeeded();
+ *     t.calledTool("get_weather");
+ *     t.check(t.reply, includes(/°|F|C/));
+ *   },
+ * });
+ *
+ * // Many datapoints in one file — ids are <fileId>/<case.id>
+ * // evals/weather.eval.ts → weather/nyc, weather/paris
+ * export default defineEval({
+ *   tags: ["smoke", "weather"],
+ *   cases: [
+ *     {
+ *       id: "nyc",
+ *       async test(t) {
+ *         await t.send("What's the temperature in NYC?");
+ *         t.succeeded();
+ *         t.calledTool("get_weather");
+ *       },
+ *     },
+ *     {
+ *       id: "paris",
+ *       async test(t) {
+ *         await t.send("What's the weather in Paris?");
+ *         t.succeeded();
+ *         t.calledTool("get_weather");
+ *       },
+ *     },
+ *   ],
+ * });
+ * ```
+ */
+
+import { normalizePositiveInt } from "./internal/bounded-int.js";
+import { brandDefinition } from "./internal/brand.js";
+import {
+  assertNonEmptyDir,
+  deleteFileInDir,
+  readJsonDir,
+  resolveProjectRelativeDir,
+  writeJsonFile,
+} from "./internal/json-dir-store.js";
+import type { SessionEvent } from "./types.js";
+
+export type Expectation =
+  | { kind: "includes"; value: string | RegExp }
+  | { kind: "equals"; value: unknown }
+  | {
+      kind: "satisfies";
+      predicate: (value: unknown) => boolean;
+      label: string;
+    };
+
+export function includes(value: string | RegExp): Expectation {
+  return { kind: "includes", value };
+}
+
+export function equals(value: unknown): Expectation {
+  return { kind: "equals", value };
+}
+
+export function satisfies(
+  predicate: (value: unknown) => boolean,
+  label: string
+): Expectation {
+  return { kind: "satisfies", predicate, label };
+}
+
+export interface EvalAssertionResult {
+  name: string;
+  passed: boolean;
+  detail?: string;
+}
+
+export interface EvalTurnResult {
+  message: string;
+  sessionId: string;
+  events: SessionEvent[];
+  toolCalls: string[];
+  ok: boolean;
+}
+
+export interface EvalSendOptions {
+  /** Absolute harness `cwd` for a new session (`SendMessageOptions.workspaceDir`). */
+  workspaceDir?: string;
+  /** Files seeded into the session workspace before the first turn. */
+  workspaceFiles?: Record<string, string>;
+}
+
+export interface EvalTestContext {
+  /** Send a turn and wait until the session parks (or fails). */
+  send(message: string, options?: EvalSendOptions): Promise<EvalTurnResult>;
+  /** Last assistant text, or null before the first send. */
+  readonly reply: string | null;
+  readonly sessionId: string | undefined;
+  /** All events captured so far across turns. */
+  readonly events: SessionEvent[];
+  /** Gate: run did not fail. */
+  succeeded(): void;
+  /** Gate: at least one completed call to `name`. */
+  calledTool(name: string): void;
+  /** Gate: no call to `name`. */
+  notCalledTool(name: string): void;
+  /** Gate: joined assistant text matches. */
+  messageIncludes(token: string | RegExp): void;
+  /** Gate a value with an {@link Expectation} builder. */
+  check(value: unknown, assertion: Expectation): void;
+  /** Debug line (surfaced under `agent-serve eval --verbose`). */
+  log(message: string): void;
+}
+
+/** One runnable datapoint inside a multi-case {@link defineEval} file. */
+export interface EvalDatapoint {
+  /**
+   * Path segment under the file id. Must be a single segment (no `/`).
+   * Full case id = `<fileId>/<id>` (e.g. file `weather.eval.ts` + `nyc` →
+   * `weather/nyc`).
+   */
+  id: string;
+  description?: string;
+  tags?: string[];
+  timeoutMs?: number;
+  test: (t: EvalTestContext) => Promise<void>;
+}
+
+/**
+ * Eval file config. Provide either a single {@link test} or a {@link cases}
+ * array — not both.
+ */
+export interface EvalConfig {
+  description?: string;
+  tags?: string[];
+  timeoutMs?: number;
+  /** Single-datapoint shorthand. Case id = file path under `evals/`. */
+  test?: (t: EvalTestContext) => Promise<void>;
+  /** Multiple datapoints in this file. Each id = `<fileId>/<case.id>`. */
+  cases?: EvalDatapoint[];
+}
+
+/** Normalized single datapoint used by the runner after discovery. */
+export interface EvalCaseDefinition {
+  description?: string;
+  tags?: string[];
+  timeoutMs?: number;
+  test: (t: EvalTestContext) => Promise<void>;
+  readonly __agentServe: "eval";
+}
+
+export type EvalDefinition = EvalConfig & { readonly __agentServe: "eval" };
+
+/**
+ * Author an eval file. The path under `evals/` is the file id:
+ * `evals/weather.eval.ts` → `weather`. With a single `test`, that is also
+ * the case id. With `cases`, each datapoint is `<fileId>/<case.id>`.
+ */
+export function defineEval(config: EvalConfig): EvalDefinition {
+  const hasTest = typeof config.test === "function";
+  const hasCases = config.cases !== undefined;
+  if (hasTest && hasCases) {
+    throw new Error("defineEval: set either test or cases, not both");
+  }
+  if (!hasTest && !hasCases) {
+    throw new Error("defineEval requires test(t) or a non-empty cases array");
+  }
+  if (hasCases) {
+    if (!Array.isArray(config.cases) || config.cases.length === 0) {
+      throw new Error("defineEval: cases must be a non-empty array");
+    }
+    const seen = new Set<string>();
+    for (const datapoint of config.cases) {
+      validateDatapointId(datapoint.id);
+      if (seen.has(datapoint.id)) {
+        throw new Error(
+          `defineEval: duplicate case id ${JSON.stringify(datapoint.id)}`
+        );
+      }
+      seen.add(datapoint.id);
+      if (typeof datapoint.test !== "function") {
+        throw new Error(
+          `defineEval: cases[${JSON.stringify(datapoint.id)}] requires test(t)`
+        );
+      }
+    }
+  }
+  return brandDefinition("eval", config);
+}
+
+function validateDatapointId(id: unknown): asserts id is string {
+  if (typeof id !== "string" || id.length === 0) {
+    throw new Error("defineEval: each case needs a non-empty string id");
+  }
+  if (id.includes("/") || id.includes("\\") || id.includes("..")) {
+    throw new Error(
+      `defineEval: case id must be a single path segment (got ${JSON.stringify(id)})`
+    );
+  }
+}
+
+/**
+ * Expand a file-level definition into runnable datapoints.
+ * Exported for tests and tooling.
+ */
+export function expandEvalDefinition(
+  fileId: string,
+  definition: EvalDefinition
+): Array<{ id: string; definition: EvalCaseDefinition }> {
+  if (definition.cases !== undefined) {
+    return definition.cases.map((datapoint) => {
+      const description = datapoint.description ?? definition.description;
+      const timeoutMs = datapoint.timeoutMs ?? definition.timeoutMs;
+      return {
+        id: `${fileId}/${datapoint.id}`,
+        definition: brandDefinition("eval", {
+          ...(description !== undefined ? { description } : {}),
+          tags: datapoint.tags ?? definition.tags,
+          ...(timeoutMs !== undefined ? { timeoutMs } : {}),
+          test: datapoint.test,
+        }),
+      };
+    });
+  }
+  if (typeof definition.test !== "function") {
+    throw new Error(`defineEval(${fileId}): missing test(t)`);
+  }
+  return [
+    {
+      id: fileId,
+      definition: brandDefinition("eval", {
+        ...(definition.description !== undefined
+          ? { description: definition.description }
+          : {}),
+        ...(definition.tags !== undefined ? { tags: definition.tags } : {}),
+        ...(definition.timeoutMs !== undefined
+          ? { timeoutMs: definition.timeoutMs }
+          : {}),
+        test: definition.test,
+      }),
+    },
+  ];
+}
+
+/**
+ * Whether a discovered case id matches a CLI filter.
+ * - exact datapoint: `weather/nyc`
+ * - whole file / prefix: `weather` selects `weather`, `weather/nyc`, …
+ * - multiple filters: OR
+ */
+export function matchesEvalIdFilter(caseId: string, filter: string): boolean {
+  return caseId === filter || caseId.startsWith(`${filter}/`);
+}
+
+/**
+ * Hard ceiling for concurrent eval cases.
+ *
+ * Each case issues model provider requests; unbounded parallelism would
+ * overwhelm provider rate limits and exhaust local API quota.
+ */
+export const EVAL_MAX_CONCURRENCY_LIMIT = 200;
+
+/** Default playground batches kept in the serve process when `maxPlaygroundRuns` is omitted. */
+export const EVAL_DEFAULT_MAX_PLAYGROUND_RUNS = 20;
+
+/** Hard ceiling for {@link EvalConfigFile.maxPlaygroundRuns}. */
+export const EVAL_MAX_PLAYGROUND_RUNS = 500;
+
+export type EvalCaseRunStatus = "pending" | "running" | "done";
+export type EvalBatchStatus = "running" | "completed" | "failed";
+
+/** One case inside a playground / HTTP eval batch snapshot. */
+export interface EvalCaseSnapshot {
+  id: string;
+  fileId: string;
+  description?: string;
+  tags?: string[];
+  status: EvalCaseRunStatus;
+  ok?: boolean;
+  assertions?: EvalAssertionResult[];
+  sessionId?: string;
+  durationMs?: number;
+  error?: string;
+  logs?: string[];
+  inputs?: string[];
+  finalText?: string;
+  tools?: string[];
+  toolCalls?: Array<{
+    toolName: string;
+    args?: unknown;
+    output?: unknown;
+    isError: boolean;
+  }>;
+}
+
+/** Playground / HTTP snapshot of one eval batch (`POST /v1/dev/evals/runs`). */
+export interface EvalRunSnapshot {
+  runId: string;
+  status: EvalBatchStatus;
+  startedAt: string;
+  finishedAt?: string;
+  error?: string;
+  summary: { passed: number; failed: number; total: number; done: number };
+  cases: EvalCaseSnapshot[];
+  config: {
+    maxConcurrency?: number;
+    timeoutMs?: number;
+    maxPlaygroundRuns?: number;
+    durableRuns?: boolean;
+  };
+}
+
+/** Context passed to {@link EvalRunPersistence} hooks. */
+export interface EvalRunPersistContext {
+  /** Absolute agent project root (directory that contains `evals/`). */
+  projectRoot: string;
+}
+
+/**
+ * Durable store for playground eval batches. Without {@link EvalConfigFile.persistRuns},
+ * runs live only in the serve process memory — lost on restart, and capped by
+ * {@link EvalConfigFile.maxPlaygroundRuns} (default {@link EVAL_DEFAULT_MAX_PLAYGROUND_RUNS}).
+ *
+ * Implement your own (S3, DB, …) or use {@link persistEvalRunsToDir}.
+ * `delete` is required so {@link EvalConfigFile.maxPlaygroundRuns} can prune
+ * durable history (otherwise pruned rows reload on the next hydrate).
+ */
+export interface EvalRunPersistence {
+  /** Load saved runs (any order). Called once when the serve process hydrates. */
+  load(
+    ctx: EvalRunPersistContext
+  ): EvalRunSnapshot[] | Promise<EvalRunSnapshot[]>;
+  /**
+   * Persist a full snapshot. Called when a batch starts, after each case
+   * finishes, and when the batch completes or fails.
+   */
+  save(run: EvalRunSnapshot, ctx: EvalRunPersistContext): void | Promise<void>;
+  /** Remove a run that falls outside {@link EvalConfigFile.maxPlaygroundRuns}. */
+  delete(runId: string, ctx: EvalRunPersistContext): void | Promise<void>;
+}
+
+export interface EvalConfigFile {
+  timeoutMs?: number;
+  /**
+   * Max number of eval cases to run in parallel. Required.
+   * Hard-capped at {@link EVAL_MAX_CONCURRENCY_LIMIT} due to model provider
+   * request limits.
+   */
+  maxConcurrency: number;
+  /**
+   * Optional. Max batches kept in the **playground / `/v1/dev/evals*`**
+   * history window (in-memory, and pruned from {@link persistRuns} via
+   * {@link EvalRunPersistence.delete}).
+   * Does not affect `agent-serve eval` CLI runs.
+   *
+   * Default {@link EVAL_DEFAULT_MAX_PLAYGROUND_RUNS}; hard-capped at
+   * {@link EVAL_MAX_PLAYGROUND_RUNS}. Newest batches win.
+   */
+  maxPlaygroundRuns?: number;
+  /**
+   * Optional durable store for playground eval batches. Omit this and runs
+   * are **not** surviving process restart (only the in-memory
+   * {@link maxPlaygroundRuns} window).
+   *
+   * ```ts
+   * import {
+   *   defineEvalConfig,
+   *   persistEvalRunsToDir,
+   * } from "@anysphere/agent-serve/evals";
+   *
+   * export default defineEvalConfig({
+   *   maxConcurrency: 20,
+   *   maxPlaygroundRuns: 50,
+   *   // Survive serve restarts (relative to the agent project root):
+   *   persistRuns: persistEvalRunsToDir(".agent-serve/eval-runs"),
+   * });
+   * ```
+   */
+  persistRuns?: EvalRunPersistence;
+}
+
+/**
+ * Defaults for every eval under `evals/` (`evals/evals.config.ts`).
+ * `maxConcurrency` is required (1–{@link EVAL_MAX_CONCURRENCY_LIMIT}).
+ *
+ * Playground batches default to **in-memory only** (see {@link maxPlaygroundRuns} /
+ * {@link persistRuns}).
+ */
+export function defineEvalConfig(config: EvalConfigFile): EvalConfigFile {
+  const persistRuns = normalizePersistRuns(config.persistRuns);
+  return {
+    ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),
+    maxConcurrency: normalizeEvalMaxConcurrency(config.maxConcurrency),
+    maxPlaygroundRuns: normalizeEvalMaxPlaygroundRuns(
+      config.maxPlaygroundRuns ?? EVAL_DEFAULT_MAX_PLAYGROUND_RUNS
+    ),
+    ...(persistRuns === undefined ? {} : { persistRuns }),
+  };
+}
+
+/**
+ * Validate and enforce the hard concurrency ceiling.
+ */
+export function normalizeEvalMaxConcurrency(value: unknown): number {
+  return normalizePositiveInt(value, {
+    label: "evals.config.ts: maxConcurrency",
+    max: EVAL_MAX_CONCURRENCY_LIMIT,
+    maxMessage: `evals.config.ts: maxConcurrency ${String(value)} exceeds the hard limit of ${EVAL_MAX_CONCURRENCY_LIMIT} (model provider request limits)`,
+  });
+}
+
+/** Validate {@link EvalConfigFile.maxPlaygroundRuns}. */
+export function normalizeEvalMaxPlaygroundRuns(value: unknown): number {
+  return normalizePositiveInt(value, {
+    label: "evals.config.ts: maxPlaygroundRuns",
+    max: EVAL_MAX_PLAYGROUND_RUNS,
+  });
+}
+
+/**
+ * Effective playground history window. Invalid values fall back to
+ * {@link EVAL_DEFAULT_MAX_PLAYGROUND_RUNS} (same as a missing field).
+ */
+export function resolveEvalMaxPlaygroundRuns(value: unknown): number {
+  if (value === undefined) {
+    return EVAL_DEFAULT_MAX_PLAYGROUND_RUNS;
+  }
+  try {
+    return normalizeEvalMaxPlaygroundRuns(value);
+  } catch {
+    return EVAL_DEFAULT_MAX_PLAYGROUND_RUNS;
+  }
+}
+
+function normalizePersistRuns(
+  value: EvalRunPersistence | undefined
+): EvalRunPersistence | undefined {
+  if (value === undefined) {
+    return undefined;
+  }
+  if (
+    typeof value !== "object" ||
+    value === null ||
+    typeof value.load !== "function" ||
+    typeof value.save !== "function" ||
+    typeof value.delete !== "function"
+  ) {
+    throw new Error(
+      "evals.config.ts: persistRuns must define load(ctx), save(run, ctx), and delete(runId, ctx)"
+    );
+  }
+  return value;
+}
+
+/**
+ * Filesystem {@link EvalRunPersistence}: one JSON file per run under `dir`
+ * (relative paths resolve against the agent project root).
+ *
+ * ```ts
+ * export default defineEvalConfig({
+ *   maxConcurrency: 20,
+ *   persistRuns: persistEvalRunsToDir(".agent-serve/eval-runs"),
+ * });
+ * ```
+ */
+export function persistEvalRunsToDir(dir: string): EvalRunPersistence {
+  const relativeDir = assertNonEmptyDir(dir, "persistEvalRunsToDir");
+  const resolveDir = (projectRoot: string): string =>
+    resolveProjectRelativeDir({ dir: relativeDir, projectRoot });
+
+  return {
+    async load(ctx) {
+      return readJsonDir<EvalRunSnapshot>(resolveDir(ctx.projectRoot));
+    },
+    async save(run, ctx) {
+      await writeJsonFile({
+        root: resolveDir(ctx.projectRoot),
+        fileName: `${run.runId}.json`,
+        value: run,
+      });
+    },
+    async delete(runId, ctx) {
+      await deleteFileInDir({
+        root: resolveDir(ctx.projectRoot),
+        fileName: `${runId}.json`,
+      });
+    },
+  };
+}

package/src/hooks.ts

@@ -0,0 +1,33 @@
+/**
+ * `defineHook` — authored at `agent/hooks/<slug>.ts`.
+ *
+ * Hooks subscribe to the session event stream and run side effects after
+ * each event is durably recorded (audit logging, metrics, persisting
+ * transcripts to your own database). Handlers are observe-only.
+ *
+ * ```ts
+ * import { defineHook } from "@anysphere/agent-serve/hooks";
+ *
+ * export default defineHook({
+ *   events: {
+ *     async "turn.completed"(event, ctx) {
+ *       console.log("turn done", ctx.session.id, event.data.usage);
+ *     },
+ *   },
+ * });
+ * ```
+ */
+
+import { brandDefinition } from "./internal/brand.js";
+import type { HookConfig, HookDefinition } from "./types.js";
+
+export type {
+  HookConfig,
+  HookContext,
+  HookDefinition,
+  HookHandler,
+} from "./types.js";
+
+export function defineHook(config: HookConfig): HookDefinition {
+  return brandDefinition("hook", config);
+}