@cursor/bdk 0.2.3 → 0.2.5

package/src/evals.ts

@@ -1,709 +0,0 @@
-/**
- * Filesystem-first evals for agent-serve projects.
- *
- * Author cases under evals/ as `*.eval.ts` files with {@link defineEval}.
- * One file may hold a single datapoint (`test`) or many (`cases`). Drive the
- * agent with `t.send(...)` and assert with `t.succeeded()`, `t.calledTool()`,
- * `t.messageIncludes(...)`, or `t.check(value, includes(...))`.
- *
- * ```ts
- * import { defineEval, includes } from "@cursor/bdk/evals";
- *
- * // One datapoint — id is the file path (evals/weather/nyc.eval.ts → weather/nyc)
- * export default defineEval({
- *   description: "Weather tool is used for temperature questions.",
- *   async test(t) {
- *     await t.send("What's the temperature in NYC?");
- *     t.succeeded();
- *     t.calledTool("get_weather");
- *     t.check(t.reply, includes(/°|F|C/));
- *   },
- * });
- *
- * // Many datapoints in one file — ids are <fileId>/<case.id>
- * // evals/weather.eval.ts → weather/nyc, weather/paris
- * export default defineEval({
- *   tags: ["smoke", "weather"],
- *   cases: [
- *     {
- *       id: "nyc",
- *       async test(t) {
- *         await t.send("What's the temperature in NYC?");
- *         t.succeeded();
- *         t.calledTool("get_weather");
- *       },
- *     },
- *     {
- *       id: "paris",
- *       async test(t) {
- *         await t.send("What's the weather in Paris?");
- *         t.succeeded();
- *         t.calledTool("get_weather");
- *       },
- *     },
- *   ],
- * });
- * ```
- *
- * ## Severity: gates and scores
- *
- * Every assertion returns a chainable handle, so severity rides on the
- * assertion instead of a separate thresholds map. Gates are hard (a miss fails
- * the eval and `eval` exits non-zero); soft assertions are tracked scores that
- * mark the eval `scored` and only fail the build under `--strict`:
- *
- * ```ts
- * t.succeeded();                                        // gate (default)
- * t.calledTool("get_weather").soft();                   // tracked, no gate
- * t.check(t.reply, similarity("Sunny, 72F")).atLeast(0.8);
- * t.judge.closedQA("cites a source").atLeast(0.6);      // LLM-graded
- * ```
- *
- * Matchers narrow tool assertions (`{ input, output, status, count }` accept a
- * literal, a RegExp, or a predicate), and each turn returned by `t.send(...)`
- * carries the same vocabulary scoped to that turn.
- */
-
-import type { EvalAssertionResult, EvalAssertions, EvalVerdict } from "./evals/assertions.js";
-import type { Expectation } from "./evals/expect.js";
-import type { EvalJudge, EvalJudgeConfig } from "./evals/judge.js";
-import type { EvalReporter } from "./evals/results.js";
-import { normalizePositiveInt } from "./internal/bounded-int.js";
-import { brandDefinition } from "./internal/brand.js";
-import type { AgentCloudOptions, SessionEvent } from "./types.js";
-
-// Assertion severity, the scoped assertion vocabulary, value assertion
-// builders, the judge surface, and result/reporter types are all part of the
-// authoring surface: one import (`@cursor/bdk/evals`) covers a whole eval file.
-export type {
-  EvalAssertionHandle,
-  EvalAssertionResult,
-  EvalAssertionSeverity,
-  EvalAssertions,
-  EvalTaggedArtifactFact,
-  EvalVerdict,
-} from "./evals/assertions.js";
-export { computeVerdict, verdictFailsRun } from "./evals/assertions.js";
-export type { EvalSchemaLike, Expectation } from "./evals/expect.js";
-export {
-  equals,
-  includes,
-  matches,
-  normalizedSimilarity,
-  satisfies,
-  similarity,
-} from "./evals/expect.js";
-export type {
-  EvalJudge,
-  EvalJudgeCallOptions,
-  EvalJudgeConfig,
-  EvalJudgeGraderName,
-} from "./evals/judge.js";
-// A custom judge built on `t.judge.model` embeds the same attacker-controlled
-// agent output the built-in graders do, so it needs the same two protections.
-export { EVAL_JUDGE_INJECTION_GUARD, fenceUntrusted } from "./evals/judge.js";
-export type {
-  EvalCountMatcher,
-  EvalEventMatcher,
-  EvalEventOrderMatcher,
-  EvalMatcher,
-  EvalSubagentMatcher,
-  EvalToolCallMatcher,
-  EvalToolCallStatus,
-} from "./evals/matchers.js";
-export type {
-  EvalReporter,
-  EvalReporterEvalInfo,
-  EvalReporterTarget,
-  EvalRunResult,
-  EvalRunSummary,
-} from "./evals/results.js";
-export type {
-  EvalInputRequestFact,
-  EvalRunFacts,
-  EvalSubagentFact,
-  EvalToolCallFact,
-} from "./evals/run-facts.js";
-export { deriveRunFacts } from "./evals/run-facts.js";
-
-/**
- * One settled turn. Carries the same assertion vocabulary as `t`, narrowed to
- * this turn's events, so an intermediate turn can be graded before the next
- * `send` overwrites `t.reply`:
- *
- * ```ts
- * const draft = await t.send("Draft the summary.");
- * draft.calledTool("read_file");
- * t.check(draft.message, includes("Summary"));
- * ```
- */
-export interface EvalTurnResult extends EvalAssertions {
-  message: string;
-  sessionId: string;
-  events: SessionEvent[];
-  /** Tool names requested during this turn, in order. */
-  toolCalls: string[];
-  ok: boolean;
-  /** 1-based turn index within the case. */
-  index: number;
-  /** Throws when this turn failed. Only needed when later steps depend on it. */
-  expectOk(): EvalTurnResult;
-}
-
-export interface EvalSendOptions {
-  /** Absolute harness `cwd` for a new session (`SendMessageOptions.workspaceDir`). */
-  workspaceDir?: string;
-  /** Files seeded into the session workspace before the first turn. */
-  workspaceFiles?: Record<string, string>;
-  /**
-   * Per-session cloud options merged over the agent's static `cloud` config
-   * (create only). Use this to attach a pinned fixture repo for cloud evals
-   * without putting it on the agent's default `cloud.repos`.
-   */
-  cloud?: AgentCloudOptions;
-}
-
-/**
- * The eval driver and assertion surface.
- *
- * `t` both drives the agent and grades it. Assertions on `t` read the whole run
- * once `test` finishes; assertions on the turn returned by `t.send(...)` read
- * only that turn. Recorded assertions never throw, so one run reports every
- * failure rather than dying on the first.
- */
-export interface EvalTestContext extends EvalAssertions {
-  /** Send a turn and wait until the session parks (or fails). */
-  send(message: string, options?: EvalSendOptions): Promise<EvalTurnResult>;
-  /** Last assistant text, or null before the first send. */
-  readonly reply: string | null;
-  readonly sessionId: string | undefined;
-  /** All events captured so far across turns. */
-  readonly events: SessionEvent[];
-  /** Turns settled so far, oldest first. */
-  readonly turns: readonly EvalTurnResult[];
-  /**
-   * 1-based index of this run when {@link EvalConfig.iterations} /
-   * {@link EvalDatapoint.iterations} expands a datapoint. Always `1` when
-   * iterations is unset or `1`.
-   */
-  readonly iteration: number;
-  /** Total iterations for this datapoint (default `1`). */
-  readonly iterations: number;
-  /** Aborts when the case hits its timeout. */
-  readonly signal: AbortSignal;
-  /**
-   * LLM-as-judge assertions. Soft by default (tracked, never fatal); add a bar
-   * with `.atLeast(0.7)` or promote with `.gate(0.8)`. Requires a judge model
-   * from `defineEvalConfig({ judge })`, `defineEval({ judge })`, or the call.
-   */
-  readonly judge: EvalJudge;
-  /**
-   * Record a gate that must pass before the script can safely continue.
-   * Returns `value` when it passes and stops the test body when it does not,
-   * without adding a duplicate execution error.
-   */
-  require<T>(value: T, assertion: Expectation): Promise<T>;
-  /**
-   * End the case as skipped. Skipped cases are reported separately and never
-   * change the exit code. Call it before sending messages.
-   */
-  skip(reason: string): never;
-  /** Debug line (surfaced under `agent-sdk eval --verbose`). */
-  log(message: string): void;
-  /**
-   * Structured score for the playground (and CLI). Values appear on the
-   * case card; agents can further customize rendering via
-   * `agent/playground/evals.tsx`.
-   */
-  metric(name: string, value: string | number): void;
-}
-
-/** Hard ceiling for {@link EvalConfig.iterations} / case `iterations`. */
-export const EVAL_MAX_ITERATIONS = 100;
-
-/** One runnable datapoint inside a multi-case {@link defineEval} file. */
-export interface EvalDatapoint {
-  /**
-   * Path segment under the file id. Must be a single segment (no `/`).
-   * Full case id = `<fileId>/<id>` (e.g. file `weather.eval.ts` + `nyc` →
-   * `weather/nyc`).
-   */
-  id: string;
-  description?: string;
-  tags?: string[];
-  timeoutMs?: number;
-  /**
-   * How many times to run this datapoint (default `1`, or the file-level
-   * {@link EvalConfig.iterations}). When greater than 1, discovery expands
-   * to `<fileId>/<id>/1` … `<fileId>/<id>/<n>` so each iteration is its own
-   * concurrent case.
-   */
-  iterations?: number;
-  /** Free-form data carried onto the result and every reporter. */
-  metadata?: Record<string, unknown>;
-  /** Judge model for this datapoint (overrides `evals.config.ts`). */
-  judge?: EvalJudgeConfig;
-  /** Destinations for just this datapoint, in addition to config reporters. */
-  reporters?: EvalReporter[];
-  test: (t: EvalTestContext) => Promise<void>;
-}
-
-/**
- * Eval file config. Provide either a single {@link test} or a {@link cases}
- * array — not both.
- */
-export interface EvalConfig {
-  description?: string;
-  tags?: string[];
-  timeoutMs?: number;
-  /**
-   * Default iterations for every datapoint in this file (default `1`).
-   * Case-level {@link EvalDatapoint.iterations} overrides this.
-   */
-  iterations?: number;
-  /** Free-form data carried onto every result in this file and to reporters. */
-  metadata?: Record<string, unknown>;
-  /** Judge model for this file (overrides `evals.config.ts`). */
-  judge?: EvalJudgeConfig;
-  /** Destinations for this file, in addition to config reporters. */
-  reporters?: EvalReporter[];
-  /** Single-datapoint shorthand. Case id = file path under `evals/`. */
-  test?: (t: EvalTestContext) => Promise<void>;
-  /** Multiple datapoints in this file. Each id = `<fileId>/<case.id>`. */
-  cases?: EvalDatapoint[];
-}
-
-/** Normalized single datapoint used by the runner after discovery. */
-export interface EvalCaseDefinition {
-  description?: string;
-  tags?: string[];
-  timeoutMs?: number;
-  /** Total iterations configured for the authored datapoint. */
-  iterations?: number;
-  /** 1-based iteration index for this expanded runnable case. */
-  iteration?: number;
-  metadata?: Record<string, unknown>;
-  judge?: EvalJudgeConfig;
-  reporters?: EvalReporter[];
-  test: (t: EvalTestContext) => Promise<void>;
-  readonly __agentServe: "eval";
-}
-
-export type EvalDefinition = EvalConfig & { readonly __agentServe: "eval" };
-
-/**
- * Author an eval file. The path under `evals/` is the file id:
- * `evals/weather.eval.ts` → `weather`. With a single `test`, that is also
- * the case id. With `cases`, each datapoint is `<fileId>/<case.id>`.
- */
-export function normalizeEvalIterations(value: unknown): number {
-  return normalizePositiveInt(value, {
-    label: "defineEval: iterations",
-    max: EVAL_MAX_ITERATIONS,
-  });
-}
-
-export function defineEval(config: EvalConfig): EvalDefinition {
-  const hasTest = typeof config.test === "function";
-  const hasCases = config.cases !== undefined;
-  if (hasTest && hasCases) {
-    throw new Error("defineEval: set either test or cases, not both");
-  }
-  if (!hasTest && !hasCases) {
-    throw new Error("defineEval requires test(t) or a non-empty cases array");
-  }
-  if (config.iterations !== undefined) {
-    normalizeEvalIterations(config.iterations);
-  }
-  if (hasCases) {
-    if (!Array.isArray(config.cases) || config.cases.length === 0) {
-      throw new Error("defineEval: cases must be a non-empty array");
-    }
-    const seen = new Set<string>();
-    for (const datapoint of config.cases) {
-      validateDatapointId(datapoint.id);
-      if (seen.has(datapoint.id)) {
-        throw new Error(`defineEval: duplicate case id ${JSON.stringify(datapoint.id)}`);
-      }
-      seen.add(datapoint.id);
-      if (typeof datapoint.test !== "function") {
-        throw new Error(`defineEval: cases[${JSON.stringify(datapoint.id)}] requires test(t)`);
-      }
-      if (datapoint.iterations !== undefined) {
-        normalizeEvalIterations(datapoint.iterations);
-      }
-    }
-  }
-  return brandDefinition("eval", config);
-}
-
-function validateDatapointId(id: unknown): asserts id is string {
-  if (typeof id !== "string" || id.length === 0) {
-    throw new Error("defineEval: each case needs a non-empty string id");
-  }
-  if (id.includes("/") || id.includes("\\") || id.includes("..")) {
-    throw new Error(
-      `defineEval: case id must be a single path segment (got ${JSON.stringify(id)})`,
-    );
-  }
-}
-
-/**
- * Expand a file-level definition into runnable datapoints.
- * Exported for tests and tooling.
- *
- * When `iterations` > 1, each authored datapoint expands to
- * `<fileId>/<case.id>/1` … `<fileId>/<case.id>/<n>` (single-test files
- * expand to `<fileId>/1` … `<fileId>/<n>`).
- */
-export function expandEvalDefinition(
-  fileId: string,
-  definition: EvalDefinition,
-): Array<{ id: string; definition: EvalCaseDefinition }> {
-  if (definition.cases !== undefined) {
-    return definition.cases.flatMap(datapoint => {
-      const description = datapoint.description ?? definition.description;
-      const timeoutMs = datapoint.timeoutMs ?? definition.timeoutMs;
-      const iterations = normalizeEvalIterations(
-        datapoint.iterations ?? definition.iterations ?? 1,
-      );
-      const metadata =
-        datapoint.metadata === undefined && definition.metadata === undefined
-          ? undefined
-          : { ...definition.metadata, ...datapoint.metadata };
-      return expandIterations({
-        baseId: `${fileId}/${datapoint.id}`,
-        description,
-        tags: datapoint.tags ?? definition.tags,
-        timeoutMs,
-        iterations,
-        metadata,
-        judge: datapoint.judge ?? definition.judge,
-        reporters: mergeReporters(definition.reporters, datapoint.reporters),
-        test: datapoint.test,
-      });
-    });
-  }
-  if (typeof definition.test !== "function") {
-    throw new Error(`defineEval(${fileId}): missing test(t)`);
-  }
-  const iterations = normalizeEvalIterations(definition.iterations ?? 1);
-  return expandIterations({
-    baseId: fileId,
-    description: definition.description,
-    tags: definition.tags,
-    timeoutMs: definition.timeoutMs,
-    iterations,
-    metadata: definition.metadata,
-    judge: definition.judge,
-    reporters: definition.reporters,
-    test: definition.test,
-  });
-}
-
-/** Union file-level and case-level reporters, keeping each instance once. */
-function mergeReporters(
-  fileLevel: EvalReporter[] | undefined,
-  caseLevel: EvalReporter[] | undefined,
-): EvalReporter[] | undefined {
-  if (fileLevel === undefined) {
-    return caseLevel;
-  }
-  if (caseLevel === undefined) {
-    return fileLevel;
-  }
-  return [...new Set([...fileLevel, ...caseLevel])];
-}
-
-function expandIterations(args: {
-  baseId: string;
-  description?: string;
-  tags?: string[];
-  timeoutMs?: number;
-  iterations: number;
-  metadata?: Record<string, unknown>;
-  judge?: EvalJudgeConfig;
-  reporters?: EvalReporter[];
-  test: (t: EvalTestContext) => Promise<void>;
-}): Array<{ id: string; definition: EvalCaseDefinition }> {
-  const out: Array<{ id: string; definition: EvalCaseDefinition }> = [];
-  for (let iteration = 1; iteration <= args.iterations; iteration++) {
-    const id = args.iterations === 1 ? args.baseId : `${args.baseId}/${iteration}`;
-    const description =
-      args.iterations === 1 || args.description === undefined
-        ? args.description
-        : `${args.description} (iteration ${iteration}/${args.iterations})`;
-    out.push({
-      id,
-      definition: brandDefinition("eval", {
-        ...(description !== undefined ? { description } : {}),
-        ...(args.tags !== undefined ? { tags: args.tags } : {}),
-        ...(args.timeoutMs !== undefined ? { timeoutMs: args.timeoutMs } : {}),
-        metadata: args.metadata,
-        judge: args.judge,
-        reporters: args.reporters,
-        iterations: args.iterations,
-        iteration,
-        test: args.test,
-      }),
-    });
-  }
-  return out;
-}
-
-/**
- * Whether a discovered case id matches a CLI filter.
- * - exact datapoint: `weather/nyc`
- * - whole file / prefix: `weather` selects `weather`, `weather/nyc`, …
- * - multiple filters: OR
- */
-export function matchesEvalIdFilter(caseId: string, filter: string): boolean {
-  return caseId === filter || caseId.startsWith(`${filter}/`);
-}
-
-/**
- * Tag for suite-aggregator datapoints that wait on sibling cases from the
- * same eval file (e.g. `deepsec-recall/overall`). Selecting a summary case
- * alone expands the batch to the whole file via
- * {@link expandSummarySuiteSelection}.
- */
-export const EVAL_SUMMARY_TAG = "summary";
-
-/** True when `tags` marks a suite-aggregator case. */
-export function isEvalSummaryCase(tags: readonly string[] | undefined): boolean {
-  return tags?.includes(EVAL_SUMMARY_TAG) ?? false;
-}
-
-/**
- * If `selected` includes any summary-tagged suite aggregator, pull in every
- * other datapoint from the same `fileId` so the aggregator is not run alone.
- * Preserves discovery order from `all`.
- */
-export function expandSummarySuiteSelection<T extends { id: string; fileId: string }>(
-  all: readonly T[],
-  selected: readonly T[],
-  isSummary: (item: T) => boolean,
-): T[] {
-  const fileIdsNeedingSuite = new Set(
-    selected.filter(item => isSummary(item)).map(item => item.fileId),
-  );
-  if (fileIdsNeedingSuite.size === 0) {
-    return [...selected];
-  }
-  const selectedIds = new Set(selected.map(item => item.id));
-  const extras = all.filter(
-    item => fileIdsNeedingSuite.has(item.fileId) && !selectedIds.has(item.id),
-  );
-  if (extras.length === 0) {
-    return [...selected];
-  }
-  const want = new Set([...selectedIds, ...extras.map(item => item.id)]);
-  return all.filter(item => want.has(item.id));
-}
-
-/**
- * Hard ceiling for concurrent eval cases.
- *
- * Each case issues model provider requests; unbounded parallelism would
- * overwhelm provider rate limits and exhaust local API quota.
- */
-export const EVAL_MAX_CONCURRENCY_LIMIT = 200;
-
-/** Default playground batches kept in the serve process when `maxPlaygroundRuns` is omitted. */
-export const EVAL_DEFAULT_MAX_PLAYGROUND_RUNS = 20;
-
-/** Hard ceiling for {@link EvalConfigFile.maxPlaygroundRuns}. */
-export const EVAL_MAX_PLAYGROUND_RUNS = 500;
-
-export type EvalCaseRunStatus = "pending" | "running" | "done";
-export type EvalBatchStatus = "running" | "completed" | "failed" | "cancelled";
-
-/** One case inside a playground / HTTP eval batch snapshot. */
-export interface EvalCaseSnapshot {
-  id: string;
-  fileId: string;
-  description?: string;
-  tags?: string[];
-  status: EvalCaseRunStatus;
-  ok?: boolean;
-  /** Full grade, including `scored` (soft threshold miss) and `skipped`. */
-  verdict?: EvalVerdict;
-  /** Reason passed to `t.skip(...)`. */
-  skipReason?: string;
-  assertions?: EvalAssertionResult[];
-  sessionId?: string;
-  durationMs?: number;
-  error?: string;
-  logs?: string[];
-  /** Structured scores from {@link EvalTestContext.metric}. */
-  metrics?: Record<string, string | number>;
-  /** Author-supplied metadata from `defineEval({ metadata })`. */
-  metadata?: Record<string, unknown>;
-  inputs?: string[];
-  finalText?: string;
-  tools?: string[];
-  toolCalls?: Array<{
-    toolName: string;
-    args?: unknown;
-    output?: unknown;
-    isError: boolean;
-  }>;
-}
-
-/** Playground / HTTP snapshot of one eval batch (`POST /v1/dev/evals/runs`). */
-export interface EvalRunSnapshot {
-  runId: string;
-  status: EvalBatchStatus;
-  startedAt: string;
-  finishedAt?: string;
-  error?: string;
-  summary: {
-    passed: number;
-    failed: number;
-    /**
-     * Cases that only missed a soft threshold (fatal only under `--strict`).
-     * Optional: a batch persisted by an older serve process has no such count.
-     */
-    scored?: number;
-    /** Cases ended by `t.skip(...)`; they never change the exit code. */
-    skipped?: number;
-    total: number;
-    done: number;
-  };
-  cases: EvalCaseSnapshot[];
-  config: {
-    maxConcurrency?: number;
-    timeoutMs?: number;
-    maxPlaygroundRuns?: number;
-    durableRuns?: boolean;
-  };
-}
-
-/** One discovered eval datapoint as returned by {@link HostEvalsApi.list}. */
-export interface HostEvalListItem {
-  id: string;
-  fileId: string;
-  description?: string;
-  tags?: string[];
-}
-
-/** Config slice returned with {@link HostEvalsApi.list}. */
-export interface HostEvalListConfig {
-  maxConcurrency?: number;
-  timeoutMs?: number;
-  maxPlaygroundRuns: number;
-  /** True when `agent/storage.ts` declares an `evals` table. */
-  durableRuns: boolean;
-}
-
-/** Input for {@link HostEvalsApi.start} (same filters as `POST /v1/dev/evals/runs`). */
-export interface HostStartEvalRunInput {
-  filterIds?: string[];
-  tags?: string[];
-  /** Per-case timeout override (same semantics as CLI `--timeout-ms`). */
-  timeoutMs?: number;
-}
-
-/**
- * Host-facing playground eval batches — the same surface as
- * `/v1/dev/evals*` and `agent-sdk eval --prod`. Attached to
- * {@link import("./types.js").HostContext.evals} by `serve`.
- */
-export interface HostEvalsApi {
-  list(): Promise<{
-    evals: HostEvalListItem[];
-    config: HostEvalListConfig;
-  }>;
-  get(runId: string): Promise<EvalRunSnapshot | undefined>;
-  listRunsWithActive(): Promise<{
-    runs: EvalRunSnapshot[];
-    activeRunId?: string;
-  }>;
-  start(input?: HostStartEvalRunInput): Promise<EvalRunSnapshot>;
-  /** Cancel a running batch by Eval ID (`runId`). */
-  cancel(runId: string): Promise<EvalRunSnapshot>;
-}
-
-export interface EvalConfigFile {
-  timeoutMs?: number;
-  /**
-   * Max number of eval cases to run in parallel. Required.
-   * Hard-capped at {@link EVAL_MAX_CONCURRENCY_LIMIT} due to model provider
-   * request limits.
-   */
-  maxConcurrency: number;
-  /**
-   * Default judge model for `t.judge.*` across this tree. Optional: a tree of
-   * fully deterministic evals never needs one.
-   */
-  judge?: EvalJudgeConfig;
-  /**
-   * Destinations that observe **every** case in the run — the right place for a
-   * shared one, so you do not repeat it in each eval file. `--skip-report`
-   * suppresses them for a local iteration loop.
-   */
-  reporters?: EvalReporter[];
-  /**
-   * Optional. Max batches kept in the **playground / `/v1/dev/evals*`**
-   * history window (in-memory, and pruned from the `defineStorage` evals
-   * table when one is configured). Includes server-side batches started by
-   * `agent-sdk eval --prod` or `--url` (not the default local harness).
-   *
-   * Default {@link EVAL_DEFAULT_MAX_PLAYGROUND_RUNS}; hard-capped at
-   * {@link EVAL_MAX_PLAYGROUND_RUNS}. Newest batches win.
-   */
-  maxPlaygroundRuns?: number;
-}
-
-/**
- * Defaults for every eval under `evals/` (`evals/evals.config.ts`).
- * `maxConcurrency` is required (1–{@link EVAL_MAX_CONCURRENCY_LIMIT}).
- *
- * Playground batches default to **in-memory only**; author an `evals`
- * table in `agent/storage.ts` (`defineStorage`) to survive serve restart.
- */
-export function defineEvalConfig(config: EvalConfigFile): EvalConfigFile {
-  return {
-    ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),
-    maxConcurrency: normalizeEvalMaxConcurrency(config.maxConcurrency),
-    maxPlaygroundRuns: normalizeEvalMaxPlaygroundRuns(
-      config.maxPlaygroundRuns ?? EVAL_DEFAULT_MAX_PLAYGROUND_RUNS,
-    ),
-    judge: config.judge,
-    reporters: config.reporters,
-  };
-}
-
-/**
- * Validate and enforce the hard concurrency ceiling.
- */
-export function normalizeEvalMaxConcurrency(value: unknown): number {
-  return normalizePositiveInt(value, {
-    label: "evals.config.ts: maxConcurrency",
-    max: EVAL_MAX_CONCURRENCY_LIMIT,
-    maxMessage: `evals.config.ts: maxConcurrency ${String(value)} exceeds the hard limit of ${EVAL_MAX_CONCURRENCY_LIMIT} (model provider request limits)`,
-  });
-}
-
-/** Validate {@link EvalConfigFile.maxPlaygroundRuns}. */
-export function normalizeEvalMaxPlaygroundRuns(value: unknown): number {
-  return normalizePositiveInt(value, {
-    label: "evals.config.ts: maxPlaygroundRuns",
-    max: EVAL_MAX_PLAYGROUND_RUNS,
-  });
-}
-
-/**
- * Effective playground history window. Invalid values fall back to
- * {@link EVAL_DEFAULT_MAX_PLAYGROUND_RUNS} (same as a missing field).
- */
-export function resolveEvalMaxPlaygroundRuns(value: unknown): number {
-  if (value === undefined) {
-    return EVAL_DEFAULT_MAX_PLAYGROUND_RUNS;
-  }
-  try {
-    return normalizeEvalMaxPlaygroundRuns(value);
-  } catch {
-    return EVAL_DEFAULT_MAX_PLAYGROUND_RUNS;
-  }
-}