@cursor/july 0.1.1

package/src/internal/eval-runner.ts

@@ -0,0 +1,522 @@
+/**
+ * Discover and run filesystem evals under `<project>/evals/`.
+ *
+ * Case identity: single-`test` files use the path
+ * (`evals/weather/nyc.eval.ts` → `weather/nyc`); multi-`cases` files use
+ * `<fileId>/<case.id>` (`evals/weather.eval.ts` + `nyc` → `weather/nyc`).
+ * Drive with `t.send`, assert with `t.succeeded` / `t.calledTool` /
+ * `t.messageIncludes` / `t.check`.
+ */
+
+import type { Dirent } from "node:fs";
+import { readdir } from "node:fs/promises";
+import { join, relative, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import type {
+  EvalAssertionResult,
+  EvalCaseDefinition,
+  EvalConfigFile,
+  EvalDefinition,
+  EvalSendOptions,
+  EvalTestContext,
+  EvalTurnResult,
+  Expectation,
+} from "../evals.js";
+import {
+  EVAL_MAX_CONCURRENCY_LIMIT,
+  expandEvalDefinition,
+  matchesEvalIdFilter,
+  normalizeEvalMaxConcurrency,
+} from "../evals.js";
+import type { SessionEvent } from "../types.js";
+import { isDefinitionOfKind } from "./brand.js";
+import { mapPool } from "./map-pool.js";
+import { RunSessionTimeoutError, runSession } from "./run-client.js";
+import { buildTrajectory } from "./trajectory.js";
+
+export interface DiscoveredEval {
+  /** Runnable datapoint id (`weather/nyc` or file id for single-test files). */
+  id: string;
+  /** File id under `evals/` (`weather` for `evals/weather.eval.ts`). */
+  fileId: string;
+  path: string;
+  definition: EvalCaseDefinition;
+}
+
+export interface EvalRunResult {
+  id: string;
+  path: string;
+  description?: string;
+  ok: boolean;
+  assertions: EvalAssertionResult[];
+  sessionId?: string;
+  /** Messages passed to `t.send` (eval inputs). */
+  inputs: string[];
+  finalText?: string;
+  tools?: string[];
+  /** Tool calls with args/results from the trajectory. */
+  toolCalls?: Array<{
+    toolName: string;
+    args?: unknown;
+    output?: unknown;
+    isError: boolean;
+  }>;
+  error?: string;
+  logs: string[];
+  durationMs: number;
+}
+
+export interface EvalSelectionOptions {
+  filterIds?: string[];
+  tags?: string[];
+}
+
+export interface RunEvalsOptions extends EvalSelectionOptions {
+  projectRoot: string;
+  baseUrl: string;
+  timeoutMs?: number;
+  bearerToken?: string;
+  verbose?: boolean;
+  onEvent?: (event: SessionEvent) => void;
+  /** Fired when a case begins executing (after concurrency slot acquired). */
+  onCaseStart?: (id: string) => void;
+  /** Fired when a case finishes (pass or fail). */
+  onCaseDone?: (result: EvalRunResult) => void;
+  /** Fired for each session event during a case (for live progress / sessionId). */
+  onCaseEvent?: (id: string, event: SessionEvent) => void;
+  importModule?: (absolutePath: string) => Promise<Record<string, unknown>>;
+  /**
+   * Pre-discovered evals + config. When set, skips filesystem discovery and
+   * selection filters (caller already chose the case list).
+   */
+  discovered?: {
+    evals: readonly DiscoveredEval[];
+    config: Partial<EvalConfigFile>;
+  };
+}
+
+async function defaultImport(
+  absolutePath: string
+): Promise<Record<string, unknown>> {
+  return (await import(pathToFileURL(absolutePath).href)) as Record<
+    string,
+    unknown
+  >;
+}
+
+async function listEvalFiles(dir: string): Promise<string[]> {
+  const out: string[] = [];
+  async function walk(current: string): Promise<void> {
+    let entries: Dirent[];
+    try {
+      entries = await readdir(current, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      if (entry.name.startsWith(".")) {
+        continue;
+      }
+      const abs = join(current, entry.name);
+      if (entry.isDirectory()) {
+        if (entry.name === "node_modules") {
+          continue;
+        }
+        await walk(abs);
+      } else if (entry.isFile()) {
+        out.push(abs);
+      }
+    }
+  }
+  await walk(dir);
+  return out.sort();
+}
+
+export async function discoverEvals(
+  projectRoot: string,
+  importModule: (
+    absolutePath: string
+  ) => Promise<Record<string, unknown>> = defaultImport
+): Promise<{
+  evals: DiscoveredEval[];
+  config: Partial<EvalConfigFile>;
+}> {
+  const root = resolve(projectRoot);
+  const evalsDir = join(root, "evals");
+  const files = await listEvalFiles(evalsDir);
+  const evals: DiscoveredEval[] = [];
+  let config: Partial<EvalConfigFile> = {};
+
+  for (const abs of files) {
+    const rel = relative(evalsDir, abs).replace(/\\/g, "/");
+    if (rel === "evals.config.ts" || rel === "evals.config.js") {
+      const mod = await importModule(abs);
+      if (mod.default !== null && typeof mod.default === "object") {
+        config = mod.default as Partial<EvalConfigFile>;
+      }
+      continue;
+    }
+    if (!rel.endsWith(".eval.ts") && !rel.endsWith(".eval.js")) {
+      continue;
+    }
+    const mod = await importModule(abs);
+    const def = mod.default;
+    if (!isDefinitionOfKind(def, "eval")) {
+      throw new Error(`${rel}: default export must be defineEval({ ... })`);
+    }
+    const fileId = rel.replace(/\.eval\.(ts|js)$/, "").replace(/\/index$/, "");
+    try {
+      for (const expanded of expandEvalDefinition(
+        fileId,
+        def as EvalDefinition
+      )) {
+        evals.push({
+          id: expanded.id,
+          fileId,
+          path: abs,
+          definition: expanded.definition,
+        });
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new Error(`${rel}: ${message}`);
+    }
+  }
+
+  return { evals, config };
+}
+
+/** Apply id/tag filters used by CLI, playground, and `runDiscoveredEvals`. */
+export function filterDiscoveredEvals(
+  evals: readonly DiscoveredEval[],
+  options: EvalSelectionOptions = {}
+): DiscoveredEval[] {
+  let selected = evals as DiscoveredEval[];
+  if (options.filterIds !== undefined && options.filterIds.length > 0) {
+    selected = selected.filter((e) =>
+      options.filterIds!.some((f) => matchesEvalIdFilter(e.id, f))
+    );
+  }
+  if (options.tags !== undefined && options.tags.length > 0) {
+    selected = selected.filter((e) =>
+      (e.definition.tags ?? []).some((t) => options.tags!.includes(t))
+    );
+  }
+  return selected;
+}
+
+export async function runDiscoveredEvals(
+  options: RunEvalsOptions
+): Promise<EvalRunResult[]> {
+  let selected: readonly DiscoveredEval[];
+  let config: Partial<EvalConfigFile>;
+
+  if (options.discovered !== undefined) {
+    selected = options.discovered.evals;
+    config = options.discovered.config;
+  } else {
+    const importModule = options.importModule ?? defaultImport;
+    const discovered = await discoverEvals(options.projectRoot, importModule);
+    config = discovered.config;
+    selected = filterDiscoveredEvals(discovered.evals, {
+      filterIds: options.filterIds,
+      tags: options.tags,
+    });
+  }
+
+  if (selected.length === 0) {
+    return [];
+  }
+
+  const maxConcurrency = resolveEvalMaxConcurrency(config);
+  const defaultTimeoutMs = options.timeoutMs ?? config.timeoutMs ?? 180_000;
+
+  return mapPool(selected, maxConcurrency, async (discovered) => {
+    if (options.verbose) {
+      process.stderr.write(`  eval ${discovered.id}…\n`);
+    }
+    options.onCaseStart?.(discovered.id);
+    const result = await runOneEval(discovered, {
+      baseUrl: options.baseUrl,
+      defaultTimeoutMs,
+      bearerToken: options.bearerToken,
+      verbose: Boolean(options.verbose),
+      onEvent: (event) => {
+        options.onCaseEvent?.(discovered.id, event);
+        options.onEvent?.(event);
+      },
+    });
+    options.onCaseDone?.(result);
+    return result;
+  });
+}
+
+function resolveEvalMaxConcurrency(config: Partial<EvalConfigFile>): number {
+  if (config.maxConcurrency === undefined) {
+    throw new Error(
+      `evals/evals.config.ts must define maxConcurrency (1–${EVAL_MAX_CONCURRENCY_LIMIT}; hard limit is due to model provider request limits). Example:\n` +
+        `  import { defineEvalConfig } from "@anysphere/agent-serve/evals";\n` +
+        `  export default defineEvalConfig({ maxConcurrency: 20 });`
+    );
+  }
+  return normalizeEvalMaxConcurrency(config.maxConcurrency);
+}
+
+async function runOneEval(
+  discovered: DiscoveredEval,
+  opts: {
+    baseUrl: string;
+    defaultTimeoutMs: number;
+    bearerToken?: string;
+    verbose: boolean;
+    onEvent?: (event: SessionEvent) => void;
+  }
+): Promise<EvalRunResult> {
+  const logs: string[] = [];
+  const assertions: EvalAssertionResult[] = [];
+  const def = discovered.definition;
+  const timeoutMs = def.timeoutMs ?? opts.defaultTimeoutMs;
+  const startedAt = Date.now();
+
+  let sessionId: string | undefined;
+  let continuationToken: string | undefined;
+  const allEvents: SessionEvent[] = [];
+  const inputs: string[] = [];
+  let reply: string | null = null;
+
+  const record = (name: string, passed: boolean, detail?: string): void => {
+    assertions.push({
+      name,
+      passed,
+      ...(detail === undefined ? {} : { detail }),
+    });
+  };
+
+  const toolsUsed = (): string[] =>
+    buildTrajectory(allEvents).turns.flatMap((t) =>
+      t.toolCalls.map((c) => c.toolName)
+    );
+
+  const toolCallIo = () =>
+    buildTrajectory(allEvents).turns.flatMap((turn) =>
+      turn.toolCalls.map((c) => ({
+        toolName: c.toolName,
+        ...(c.args === undefined ? {} : { args: c.args }),
+        ...(c.output === undefined ? {} : { output: c.output }),
+        isError: c.isError,
+      }))
+    );
+
+  const t: EvalTestContext = {
+    get reply() {
+      return reply;
+    },
+    get sessionId() {
+      return sessionId;
+    },
+    get events() {
+      return allEvents;
+    },
+    async send(
+      message: string,
+      options: EvalSendOptions = {}
+    ): Promise<EvalTurnResult> {
+      inputs.push(message);
+      const result = await runSession({
+        baseUrl: opts.baseUrl,
+        messages: [message],
+        sessionId,
+        continuationToken,
+        // Keep defineEval traffic out of live defineAB enrollment/metrics.
+        purpose: "eval",
+        bearerToken: opts.bearerToken,
+        workspaceDir: options.workspaceDir,
+        workspaceFiles: options.workspaceFiles,
+        timeoutMs,
+        onEvent: opts.onEvent,
+      });
+      sessionId = result.sessionId;
+      continuationToken = result.continuationToken;
+      allEvents.push(...result.events);
+      if (result.trajectory.finalText !== "") {
+        reply = result.trajectory.finalText;
+      }
+      return {
+        message: result.trajectory.finalText,
+        sessionId: result.sessionId,
+        events: result.events,
+        toolCalls: result.trajectory.turns.flatMap((turn) =>
+          turn.toolCalls.map((c) => c.toolName)
+        ),
+        ok: result.ok,
+      };
+    },
+    succeeded() {
+      const traj = buildTrajectory(allEvents);
+      record(
+        "succeeded",
+        traj.ok && traj.turns.length > 0,
+        traj.ok ? undefined : "turn failed"
+      );
+    },
+    calledTool(name: string) {
+      const tools = toolsUsed();
+      record(
+        `calledTool(${name})`,
+        tools.includes(name),
+        tools.includes(name) ? undefined : `got [${tools.join(", ")}]`
+      );
+    },
+    notCalledTool(name: string) {
+      const tools = toolsUsed();
+      record(
+        `notCalledTool(${name})`,
+        !tools.includes(name),
+        tools.includes(name) ? "tool was called" : undefined
+      );
+    },
+    messageIncludes(token: string | RegExp) {
+      const text = reply ?? "";
+      const passed =
+        typeof token === "string" ? text.includes(token) : token.test(text);
+      record(
+        `messageIncludes(${String(token)})`,
+        passed,
+        passed ? undefined : `reply=${JSON.stringify(text.slice(0, 120))}`
+      );
+    },
+    check(value: unknown, assertion: Expectation) {
+      const { passed, detail } = matchExpectation(value, assertion);
+      record(`check(${assertion.kind})`, passed, detail);
+    },
+    log(message: string) {
+      logs.push(message);
+      if (opts.verbose) {
+        process.stderr.write(`    ${message}\n`);
+      }
+    },
+  };
+
+  try {
+    await def.test(t);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (error instanceof RunSessionTimeoutError && sessionId === undefined) {
+      sessionId = error.sessionId;
+    }
+    return {
+      id: discovered.id,
+      path: discovered.path,
+      ...(def.description === undefined
+        ? {}
+        : { description: def.description }),
+      ok: false,
+      assertions,
+      error: message,
+      logs,
+      inputs,
+      toolCalls: toolCallIo(),
+      ...(sessionId === undefined ? {} : { sessionId }),
+      ...(reply === null ? {} : { finalText: reply }),
+      tools: toolsUsed(),
+      durationMs: Date.now() - startedAt,
+    };
+  }
+
+  const traj = buildTrajectory(allEvents);
+  const ok =
+    assertions.length > 0
+      ? assertions.every((a) => a.passed)
+      : traj.ok && traj.turns.length > 0;
+
+  return {
+    id: discovered.id,
+    path: discovered.path,
+    ...(def.description === undefined ? {} : { description: def.description }),
+    ok,
+    assertions,
+    logs,
+    inputs,
+    toolCalls: toolCallIo(),
+    ...(sessionId === undefined ? {} : { sessionId }),
+    finalText: reply ?? traj.finalText,
+    tools: toolsUsed(),
+    durationMs: Date.now() - startedAt,
+  };
+}
+
+function matchExpectation(
+  value: unknown,
+  assertion: Expectation
+): { passed: boolean; detail?: string } {
+  switch (assertion.kind) {
+    case "includes": {
+      const text = value === null || value === undefined ? "" : String(value);
+      const passed =
+        typeof assertion.value === "string"
+          ? text.includes(assertion.value)
+          : assertion.value.test(text);
+      return {
+        passed,
+        ...(passed
+          ? {}
+          : {
+              detail: `value=${JSON.stringify(text.slice(0, 120))}`,
+            }),
+      };
+    }
+    case "equals": {
+      const passed = deepEqual(value, assertion.value);
+      return {
+        passed,
+        ...(passed ? {} : { detail: `got ${JSON.stringify(value)}` }),
+      };
+    }
+    case "satisfies": {
+      const passed = assertion.predicate(value);
+      return {
+        passed,
+        ...(passed ? {} : { detail: assertion.label }),
+      };
+    }
+  }
+}
+
+function deepEqual(a: unknown, b: unknown): boolean {
+  if (Object.is(a, b)) {
+    return true;
+  }
+  if (
+    typeof a !== "object" ||
+    typeof b !== "object" ||
+    a === null ||
+    b === null
+  ) {
+    return false;
+  }
+  if (Array.isArray(a) !== Array.isArray(b)) {
+    return false;
+  }
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) {
+      return false;
+    }
+    return a.every((item, i) => deepEqual(item, b[i]));
+  }
+  const aKeys = Object.keys(a as object);
+  const bKeys = Object.keys(b as object);
+  if (aKeys.length !== bKeys.length) {
+    return false;
+  }
+  for (const key of aKeys) {
+    if (
+      !deepEqual(
+        (a as Record<string, unknown>)[key],
+        (b as Record<string, unknown>)[key]
+      )
+    ) {
+      return false;
+    }
+  }
+  return true;
+}

package/src/internal/event-log.ts

@@ -0,0 +1,175 @@
+/**
+ * Durable, replayable per-session event log. Events are held in memory for
+ * live fan-out and appended to `events.ndjson` for durability; on process
+ * restart the file is hydrated back so streams can replay from any
+ * `startIndex`.
+ */
+
+import { appendFile, mkdir, readFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import type { SessionEvent, SessionEventPayload } from "../types.js";
+
+export type AppendableEvent = SessionEventPayload & { turnId?: string };
+
+export class SessionEventLog {
+  private readonly events: SessionEvent[] = [];
+  private readonly listeners = new Set<(event: SessionEvent) => void>();
+  private persistChain: Promise<void> = Promise.resolve();
+
+  private constructor(
+    private readonly sessionId: string,
+    private readonly filePath: string,
+    private readonly onPersistError: (error: unknown) => void
+  ) {}
+
+  static async open(args: {
+    sessionId: string;
+    filePath: string;
+    onPersistError?: (error: unknown) => void;
+  }): Promise<SessionEventLog> {
+    const log = new SessionEventLog(
+      args.sessionId,
+      args.filePath,
+      args.onPersistError ?? (() => {})
+    );
+    await log.hydrate();
+    return log;
+  }
+
+  private async hydrate(): Promise<void> {
+    let raw: string;
+    try {
+      raw = await readFile(this.filePath, "utf8");
+    } catch {
+      return;
+    }
+    for (const line of raw.split("\n")) {
+      if (line.trim() === "") {
+        continue;
+      }
+      try {
+        this.events.push(JSON.parse(line) as SessionEvent);
+      } catch {
+        // Skip torn trailing writes from a crashed process.
+      }
+    }
+  }
+
+  get count(): number {
+    return this.events.length;
+  }
+
+  /**
+   * Assign the next index, record the event, and fan out to live
+   * subscribers. Ordering is synchronous; file persistence is chained in
+   * the background and awaited by {@link flush}.
+   */
+  append(payload: AppendableEvent): SessionEvent {
+    const { turnId, ...rest } = payload;
+    const event = {
+      index: this.events.length,
+      sessionId: this.sessionId,
+      ...(turnId === undefined ? {} : { turnId }),
+      at: new Date().toISOString(),
+      ...rest,
+    } as SessionEvent;
+    this.events.push(event);
+    for (const listener of [...this.listeners]) {
+      try {
+        listener(event);
+      } catch {
+        // Subscriber errors must not break the log.
+      }
+    }
+    const line = `${JSON.stringify(event)}\n`;
+    this.persistChain = this.persistChain
+      .then(() => appendFile(this.filePath, line, "utf8"))
+      .catch((error) => this.onPersistError(error));
+    return event;
+  }
+
+  /** Wait for pending file writes (used on close). */
+  async flush(): Promise<void> {
+    await this.persistChain;
+  }
+
+  snapshot(startIndex = 0): SessionEvent[] {
+    return startIndex <= 0 ? [...this.events] : this.events.slice(startIndex);
+  }
+
+  subscribe(listener: (event: SessionEvent) => void): () => void {
+    this.listeners.add(listener);
+    return () => {
+      this.listeners.delete(listener);
+    };
+  }
+
+  /**
+   * NDJSON stream: replays recorded events from `startIndex`, then follows
+   * live events until the consumer cancels.
+   */
+  stream(startIndex = 0): ReadableStream<Uint8Array> {
+    const encoder = new TextEncoder();
+    let unsubscribe: (() => void) | undefined;
+    return new ReadableStream<Uint8Array>({
+      start: (controller) => {
+        const push = (event: SessionEvent): void => {
+          try {
+            controller.enqueue(encoder.encode(`${JSON.stringify(event)}\n`));
+          } catch {
+            unsubscribe?.();
+          }
+        };
+        // Synchronous snapshot + subscribe: no events can interleave.
+        for (const event of this.snapshot(startIndex)) {
+          push(event);
+        }
+        unsubscribe = this.subscribe((event) => {
+          if (event.index >= startIndex) {
+            push(event);
+          }
+        });
+      },
+      cancel: () => {
+        unsubscribe?.();
+      },
+    });
+  }
+}
+
+/** Lazily opens one {@link SessionEventLog} per session. */
+export class EventLogStore {
+  private readonly logs = new Map<string, Promise<SessionEventLog>>();
+
+  constructor(
+    private readonly eventFilePathFor: (sessionId: string) => string,
+    private readonly onPersistError: (
+      sessionId: string,
+      error: unknown
+    ) => void = () => {}
+  ) {}
+
+  get(sessionId: string): Promise<SessionEventLog> {
+    let log = this.logs.get(sessionId);
+    if (log === undefined) {
+      const filePath = this.eventFilePathFor(sessionId);
+      log = mkdir(dirname(filePath), { recursive: true }).then(() =>
+        SessionEventLog.open({
+          sessionId,
+          filePath,
+          onPersistError: (error) => this.onPersistError(sessionId, error),
+        })
+      );
+      this.logs.set(sessionId, log);
+    }
+    return log;
+  }
+
+  async flushAll(): Promise<void> {
+    await Promise.all(
+      [...this.logs.values()].map((logPromise) =>
+        logPromise.then((log) => log.flush()).catch(() => {})
+      )
+    );
+  }
+}