@andreprado/agentkit 0.1.0-alpha.2 → 0.1.0-alpha.20

package/src/runtime/evals.ts

@@ -1,13 +1,14 @@
-import { readdir } from "node:fs/promises";
+import { mkdir, readdir, writeFile } from "node:fs/promises";
 import type { Dirent } from "node:fs";
-import { join, relative } from "node:path";
+import { dirname, isAbsolute, join, relative, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
 
 import type { AgentRunResult } from "./chat";
 import { runAgentMessageFromCwd } from "./chat";
 import { findAgentCapsuleRoot, loadAgentCapsule } from "./config";
-import { AgentKitError } from "./errors";
+import { AgentKitError, isAgentKitError } from "./errors";
 import { openCapsuleStore, type StoredToolCall } from "../storage/sqlite";
+import { getConversationTraceFromCwd } from "./traces";
 
 export type EvalRunSummary = {
   root: string;
@@ -24,30 +25,76 @@ export type EvalResult = {
   output: string;
 };
 
-type EvalCase = {
+export type EvalCase = {
   name?: string;
   input?: string;
+  now?: string | Date;
+  turns?: EvalTurn[];
   expect?: EvalExpect;
 };
 
-type EvalExpect = {
+export type EvalTurn =
+  | string
+  | {
+      input: string;
+      expect?: EvalExpect;
+    };
+
+export type EvalExpect = EvalResponseExpectation & {
+  response?: EvalResponseExpectation;
+  tools?: EvalToolsExpectation;
+  tool_call?: ToolExpectation | ToolExpectation[];
+  toolCall?: ToolExpectation | ToolExpectation[];
+  tool_calls?: ToolExpectation | ToolExpectation[];
+  toolCalls?: ToolExpectation | ToolExpectation[];
+  tool_call_count?: number;
+  toolCallCount?: number;
+  tool_call_order?: string[];
+  toolCallOrder?: string[];
+  persisted_tool_call?: ToolExpectation | ToolExpectation[];
+  persistedToolCall?: ToolExpectation | ToolExpectation[];
+  persisted_tool_calls?: ToolExpectation | ToolExpectation[];
+  persistedToolCalls?: ToolExpectation | ToolExpectation[];
+};
+
+export type EvalResponseExpectation = {
   contains?: string | string[];
+  contains_all?: string | string[];
+  containsAll?: string | string[];
+  contains_any?: string | string[];
+  containsAny?: string | string[];
+  case_insensitive_contains?: string | string[];
+  caseInsensitiveContains?: string | string[];
   not_contains?: string | string[];
   notContains?: string | string[];
   regex?: string | string[];
   matches_regex?: string | string[];
   matchesRegex?: string | string[];
-  tool_call?: ToolExpectation | ToolExpectation[];
-  toolCall?: ToolExpectation | ToolExpectation[];
-  tool_calls?: ToolExpectation | ToolExpectation[];
-  toolCalls?: ToolExpectation | ToolExpectation[];
+  not_regex?: string | string[];
+  notRegex?: string | string[];
+  max_length?: number;
+  maxLength?: number;
+};
+
+export type EvalToolsExpectation =
+  | ToolExpectation
+  | ToolExpectation[]
+  | EvalToolsContainerExpectation;
+
+export type EvalToolsContainerExpectation = {
+  persisted?: ToolExpectation | ToolExpectation[];
   persisted_tool_call?: ToolExpectation | ToolExpectation[];
   persistedToolCall?: ToolExpectation | ToolExpectation[];
   persisted_tool_calls?: ToolExpectation | ToolExpectation[];
   persistedToolCalls?: ToolExpectation | ToolExpectation[];
+  called?: string | string[];
+  called_once?: string | string[];
+  calledOnce?: string | string[];
+  count?: number;
+  order?: string[];
 };
 
-type ToolExpectation =
+export type ToolExpectation =
   | string
   | {
       name?: string;
@@ -55,16 +102,29 @@ type ToolExpectation =
       output?: unknown;
       status?: "running" | "completed" | "failed";
       visibility?: "user" | "internal";
+      rendered?: unknown;
     };
 
 type ToolCallSnapshot = {
   name?: unknown;
   input?: unknown;
   output?: unknown;
+  rendered?: unknown;
   status?: unknown;
   visibility?: unknown;
 };
 
+export type EvalFromConversationResult = {
+  root: string;
+  file: string;
+  conversationId: string;
+  turns: number;
+};
+
+export function defineEval<const T extends EvalCase>(evalCase: T): T {
+  return evalCase;
+}
+
 export async function runEvalsFromCwd(cwd: string): Promise<EvalRunSummary> {
   const root = await findAgentCapsuleRoot(cwd);
   const evalFiles = await findEvalFiles(join(root, "evals"));
@@ -76,29 +136,29 @@ export async function runEvalsFromCwd(cwd: string): Promise<EvalRunSummary> {
   const results: EvalResult[] = [];
 
   for (const file of evalFiles) {
-    const evalCase = await loadEvalCase(file);
-    const input = evalCase.input;
-
-    if (typeof input !== "string" || input.trim().length === 0) {
-      throw new AgentKitError("validation_error", `${relative(root, file)} must export an eval with a non-empty input string.`);
+    const evalFile = relative(root, file);
+    let name = evalFile;
+    let failures: string[];
+    let output: string;
+
+    try {
+      const evalCase = await loadEvalCase(file);
+      const run = await runEvalCase(root, evalFile, evalCase);
+
+      name = evalCase.name ?? evalFile;
+      failures = run.failures;
+      output = run.output;
+    } catch (error) {
+      failures = [formatEvalFailure(error)];
+      output = "";
     }
 
-    const run = await runAgentMessageFromCwd(root, {
-      message: input,
-      runtime: {
-        environment: "eval",
-        invocation: "eval",
-      },
-    });
-    const persistedToolCalls = await loadPersistedToolCalls(root, run.runId);
-    const failures = evaluateExpectations(evalCase.expect ?? {}, run, persistedToolCalls);
-
     results.push({
-      name: evalCase.name ?? relative(root, file),
-      file: relative(root, file),
+      name,
+      file: evalFile,
       passed: failures.length === 0,
       failures,
-      output: run.message.content,
+      output,
     });
   }
 
@@ -112,6 +172,171 @@ export async function runEvalsFromCwd(cwd: string): Promise<EvalRunSummary> {
   };
 }
 
+export async function writeEvalFromConversation(
+  cwd: string,
+  input: {
+    conversationId: string;
+    out?: string;
+    name?: string;
+    force?: boolean;
+  },
+): Promise<EvalFromConversationResult> {
+  const root = await findAgentCapsuleRoot(cwd);
+  const trace = await getConversationTraceFromCwd(root, input.conversationId);
+  const turns = replayTurnsFromMessages(trace.messages);
+  const safeTitle = redactEvalText(trace.title ?? trace.id);
+
+  if (turns.length === 0) {
+    throw new AgentKitError(
+      "validation_error",
+      `Conversation "${input.conversationId}" does not contain user/assistant turns that can become an eval.`,
+    );
+  }
+
+  const file = resolveEvalOutputPath(root, input.out, safeTitle);
+  const source = `import { defineEval } from "@andreprado/agentkit";
+
+export default defineEval({
+  name: ${JSON.stringify(input.name ? redactEvalText(input.name) : `replay ${safeTitle}`)},
+  turns: ${formatEvalTurns(turns)},
+});
+`;
+
+  await mkdir(dirname(file), { recursive: true });
+
+  try {
+    await writeFile(file, source, { flag: input.force ? "w" : "wx" });
+  } catch (error) {
+    if (isNodeError(error) && error.code === "EEXIST") {
+      throw new AgentKitError(
+        "validation_error",
+        `${relative(root, file)} already exists. Re-run with --force or choose --out <path>.`,
+      );
+    }
+
+    throw error;
+  }
+
+  return {
+    root,
+    file,
+    conversationId: trace.id,
+    turns: turns.length,
+  };
+}
+
+async function runEvalCase(
+  root: string,
+  file: string,
+  evalCase: EvalCase,
+): Promise<{ failures: string[]; output: string }> {
+  const turns = normalizeEvalTurns(evalCase);
+
+  if (turns.length === 0) {
+    throw new AgentKitError(
+      "validation_error",
+      `${file} must export either a non-empty input string or a non-empty turns array.`,
+    );
+  }
+
+  const conversationId = `eval_${crypto.randomUUID()}`;
+  const failures: string[] = [];
+  let output = "";
+  const now = resolveEvalNow(file, evalCase.now);
+
+  for (const [index, turn] of turns.entries()) {
+    try {
+      const run = await runAgentMessageFromCwd(root, {
+        message: turn.input,
+        conversationId,
+        now,
+        runtime: {
+          environment: "eval",
+          invocation: "eval",
+        },
+      });
+      const persistedToolCalls = await loadPersistedToolCalls(root, run.runId);
+      const turnFailures = evaluateExpectations(turn.expect ?? {}, run, persistedToolCalls);
+
+      for (const failure of turnFailures) {
+        failures.push(turns.length === 1 ? failure : `turn ${index + 1}: ${failure}`);
+      }
+
+      output = run.message.content;
+    } catch (error) {
+      failures.push(turns.length === 1 ? formatEvalFailure(error) : `turn ${index + 1}: ${formatEvalFailure(error)}`);
+      break;
+    }
+  }
+
+  return { failures, output };
+}
+
+function resolveEvalNow(file: string, value: EvalCase["now"]): Date | undefined {
+  if (value === undefined) {
+    return undefined;
+  }
+
+  if (typeof value === "string" && !hasExplicitIsoOffset(value)) {
+    throw new AgentKitError(
+      "validation_error",
+      `${file} now must be an ISO timestamp with an explicit timezone offset, such as "2026-02-04T02:30:00.000Z" or "2026-02-04T02:30:00-05:00".`,
+    );
+  }
+
+  const date = value instanceof Date ? new Date(value.getTime()) : new Date(value);
+
+  if (Number.isNaN(date.getTime())) {
+    throw new AgentKitError("validation_error", `${file} now must be a valid ISO timestamp or Date when provided.`);
+  }
+
+  return date;
+}
+
+function hasExplicitIsoOffset(value: string): boolean {
+  return /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/.test(value.trim());
+}
+
+function normalizeEvalTurns(evalCase: EvalCase): Array<{ input: string; expect?: EvalExpect }> {
+  if (evalCase.turns !== undefined) {
+    if (!Array.isArray(evalCase.turns)) {
+      throw new AgentKitError("validation_error", "eval turns must be an array when provided.");
+    }
+
+    return evalCase.turns.map((turn, index) => normalizeEvalTurn(turn, index));
+  }
+
+  if (typeof evalCase.input === "string" && evalCase.input.trim().length > 0) {
+    return [
+      {
+        input: evalCase.input,
+        expect: evalCase.expect,
+      },
+    ];
+  }
+
+  return [];
+}
+
+function normalizeEvalTurn(turn: EvalTurn, index: number): { input: string; expect?: EvalExpect } {
+  if (typeof turn === "string") {
+    if (turn.trim().length === 0) {
+      throw new AgentKitError("validation_error", `eval turns[${index}] must be non-empty.`);
+    }
+
+    return { input: turn };
+  }
+
+  if (!isRecord(turn) || typeof turn.input !== "string" || turn.input.trim().length === 0) {
+    throw new AgentKitError("validation_error", `eval turns[${index}].input must be a non-empty string.`);
+  }
+
+  return {
+    input: turn.input,
+    ...(turn.expect ? { expect: turn.expect } : {}),
+  };
+}
+
 async function findEvalFiles(directory: string): Promise<string[]> {
   let entries: Dirent[];
 
@@ -172,12 +397,89 @@ function evaluateExpectations(expect: EvalExpect, run: AgentRunResult, persisted
   const failures: string[] = [];
   const content = run.message.content;
 
-  for (const expected of list(expect.contains)) {
+  for (const responseExpectation of responseExpectations(expect)) {
+    failures.push(...evaluateResponseExpectation(responseExpectation, content));
+  }
+
+  const toolAssertions = collectToolAssertions(expect);
+
+  for (const toolExpectation of toolAssertions.persisted) {
+    const match = persistedToolCalls.find((toolCall) => matchesToolExpectation(toolCall, toolExpectation));
+
+    if (!match) {
+      failures.push(`expected persisted tool call ${formatToolExpectation(toolExpectation)}`);
+    }
+  }
+
+  for (const expectedName of toolAssertions.called) {
+    if (!persistedToolCalls.some((toolCall) => toolCall.name === expectedName)) {
+      failures.push(`expected persisted tool call named ${JSON.stringify(expectedName)}`);
+    }
+  }
+
+  for (const expectedName of toolAssertions.calledOnce) {
+    const count = persistedToolCalls.filter((toolCall) => toolCall.name === expectedName).length;
+
+    if (count !== 1) {
+      failures.push(`expected persisted tool call ${JSON.stringify(expectedName)} exactly once, got ${count}`);
+    }
+  }
+
+  for (const expectedCount of toolAssertions.counts) {
+    if (!Number.isInteger(expectedCount) || expectedCount < 0) {
+      failures.push(`expected tool call count to be a non-negative integer, got ${JSON.stringify(expectedCount)}`);
+      continue;
+    }
+
+    if (persistedToolCalls.length !== expectedCount) {
+      failures.push(`expected ${expectedCount} persisted tool call(s), got ${persistedToolCalls.length}`);
+    }
+  }
+
+  for (const expectedOrder of toolAssertions.orders) {
+    if (!namesContainSubsequence(persistedToolNames(persistedToolCalls), expectedOrder)) {
+      failures.push(
+        `expected persisted tool call order ${JSON.stringify(expectedOrder)}, got ${JSON.stringify(
+          persistedToolNames(persistedToolCalls),
+        )}`,
+      );
+    }
+  }
+
+  return failures;
+}
+
+function responseExpectations(expect: EvalExpect): EvalResponseExpectation[] {
+  return [expect, expect.response].filter((value): value is EvalResponseExpectation => value !== undefined);
+}
+
+function evaluateResponseExpectation(expect: EvalResponseExpectation, content: string): string[] {
+  const failures: string[] = [];
+
+  for (const expected of [
+    ...list(expect.contains),
+    ...list(expect.contains_all),
+    ...list(expect.containsAll),
+  ]) {
     if (!content.includes(expected)) {
       failures.push(`expected output to contain ${JSON.stringify(expected)}`);
     }
   }
 
+  const containsAny = [...list(expect.contains_any), ...list(expect.containsAny)];
+
+  if (containsAny.length > 0 && !containsAny.some((expected) => content.includes(expected))) {
+    failures.push(`expected output to contain any of ${JSON.stringify(containsAny)}`);
+  }
+
+  const lowerContent = content.toLowerCase();
+
+  for (const expected of [...list(expect.case_insensitive_contains), ...list(expect.caseInsensitiveContains)]) {
+    if (!lowerContent.includes(expected.toLowerCase())) {
+      failures.push(`expected output to contain ${JSON.stringify(expected)} case-insensitively`);
+    }
+  }
+
   for (const forbidden of [...list(expect.not_contains), ...list(expect.notContains)]) {
     if (content.includes(forbidden)) {
       failures.push(`expected output not to contain ${JSON.stringify(forbidden)}`);
@@ -190,28 +492,140 @@ function evaluateExpectations(expect: EvalExpect, run: AgentRunResult, persisted
     }
   }
 
-  const toolExpectations = [
-    ...toolList(expect.tool_call),
-    ...toolList(expect.toolCall),
-    ...toolList(expect.tool_calls),
-    ...toolList(expect.toolCalls),
-    ...toolList(expect.persisted_tool_call),
-    ...toolList(expect.persistedToolCall),
-    ...toolList(expect.persisted_tool_calls),
-    ...toolList(expect.persistedToolCalls),
-  ];
+  for (const pattern of [...list(expect.not_regex), ...list(expect.notRegex)]) {
+    if (new RegExp(pattern).test(content)) {
+      failures.push(`expected output not to match /${pattern}/`);
+    }
+  }
 
-  for (const toolExpectation of toolExpectations) {
-    const match = persistedToolCalls.find((toolCall) => matchesToolExpectation(toolCall, toolExpectation));
+  for (const expectedMaxLength of [expect.max_length, expect.maxLength]) {
+    if (expectedMaxLength === undefined) {
+      continue;
+    }
 
-    if (!match) {
-      failures.push(`expected persisted tool call ${formatToolExpectation(toolExpectation)}`);
+    if (!Number.isInteger(expectedMaxLength) || expectedMaxLength < 0) {
+      failures.push(`expected response maxLength to be a non-negative integer, got ${JSON.stringify(expectedMaxLength)}`);
+      continue;
+    }
+
+    if (content.length > expectedMaxLength) {
+      failures.push(`expected output length to be <= ${expectedMaxLength}, got ${content.length}`);
     }
   }
 
   return failures;
 }
 
+type ToolAssertions = {
+  persisted: ToolExpectation[];
+  called: string[];
+  calledOnce: string[];
+  counts: number[];
+  orders: string[][];
+};
+
+function collectToolAssertions(expect: EvalExpect): ToolAssertions {
+  const assertions: ToolAssertions = {
+    persisted: [
+      ...toolList(expect.tool_call),
+      ...toolList(expect.toolCall),
+      ...toolList(expect.tool_calls),
+      ...toolList(expect.toolCalls),
+      ...toolList(expect.persisted_tool_call),
+      ...toolList(expect.persistedToolCall),
+      ...toolList(expect.persisted_tool_calls),
+      ...toolList(expect.persistedToolCalls),
+    ],
+    called: [],
+    calledOnce: [],
+    counts: [],
+    orders: [],
+  };
+
+  if (expect.tool_call_count !== undefined) {
+    assertions.counts.push(expect.tool_call_count);
+  }
+
+  if (expect.toolCallCount !== undefined) {
+    assertions.counts.push(expect.toolCallCount);
+  }
+
+  if (expect.tool_call_order !== undefined) {
+    assertions.orders.push(expect.tool_call_order);
+  }
+
+  if (expect.toolCallOrder !== undefined) {
+    assertions.orders.push(expect.toolCallOrder);
+  }
+
+  appendNestedToolAssertions(assertions, expect.tools);
+
+  return assertions;
+}
+
+function appendNestedToolAssertions(assertions: ToolAssertions, tools: EvalToolsExpectation | undefined): void {
+  if (tools === undefined) {
+    return;
+  }
+
+  if (!isToolsContainer(tools)) {
+    assertions.persisted.push(...toolList(tools));
+    return;
+  }
+
+  assertions.persisted.push(
+    ...toolList(tools.persisted),
+    ...toolList(tools.persisted_tool_call),
+    ...toolList(tools.persistedToolCall),
+    ...toolList(tools.persisted_tool_calls),
+    ...toolList(tools.persistedToolCalls),
+  );
+  assertions.called.push(...list(tools.called));
+  assertions.calledOnce.push(...list(tools.called_once), ...list(tools.calledOnce));
+
+  if (tools.count !== undefined) {
+    assertions.counts.push(tools.count);
+  }
+
+  if (tools.order !== undefined) {
+    assertions.orders.push(tools.order);
+  }
+}
+
+function isToolsContainer(value: EvalToolsExpectation): value is EvalToolsContainerExpectation {
+  if (!isRecord(value)) {
+    return false;
+  }
+
+  if (
+    "name" in value ||
+    "input" in value ||
+    "output" in value ||
+    "rendered" in value ||
+    "status" in value ||
+    "visibility" in value
+  ) {
+    return false;
+  }
+
+  if (Object.keys(value).length === 0) {
+    return true;
+  }
+
+  return (
+    "persisted" in value ||
+    "persisted_tool_call" in value ||
+    "persistedToolCall" in value ||
+    "persisted_tool_calls" in value ||
+    "persistedToolCalls" in value ||
+    "called" in value ||
+    "called_once" in value ||
+    "calledOnce" in value ||
+    "count" in value ||
+    "order" in value
+  );
+}
+
 function list(value: string | string[] | undefined): string[] {
   if (value === undefined) {
     return [];
@@ -220,6 +634,30 @@ function list(value: string | string[] | undefined): string[] {
   return Array.isArray(value) ? value : [value];
 }
 
+function persistedToolNames(toolCalls: ToolCallSnapshot[]): string[] {
+  return toolCalls.map((toolCall) => String(toolCall.name ?? ""));
+}
+
+function namesContainSubsequence(actual: string[], expected: string[]): boolean {
+  if (expected.length === 0) {
+    return true;
+  }
+
+  let actualIndex = 0;
+
+  for (const expectedName of expected) {
+    actualIndex = actual.findIndex((actualName, index) => index >= actualIndex && actualName === expectedName);
+
+    if (actualIndex === -1) {
+      return false;
+    }
+
+    actualIndex += 1;
+  }
+
+  return true;
+}
+
 function toolList(value: ToolExpectation | ToolExpectation[] | undefined): ToolExpectation[] {
   if (value === undefined) {
     return [];
@@ -253,6 +691,10 @@ function matchesToolExpectation(toolCall: ToolCallSnapshot, expectation: ToolExp
     return false;
   }
 
+  if ("rendered" in expectation && !jsonContains(toolCall.rendered, expectation.rendered)) {
+    return false;
+  }
+
   return true;
 }
 
@@ -269,11 +711,115 @@ function toolCallSnapshotFromStored(toolCall: StoredToolCall): ToolCallSnapshot
     name: toolCall.toolName,
     input: toolCall.input,
     output: toolCall.output,
+    rendered: toolCall.rendered,
     status: toolCall.status,
     visibility: toolCall.visibility,
   };
 }
 
+function replayTurnsFromMessages(
+  messages: Array<{ role: string; content: string }>,
+): Array<{ input: string; expect?: EvalExpect }> {
+  const turns: Array<{ input: string; expect?: EvalExpect }> = [];
+
+  for (let index = 0; index < messages.length; index += 1) {
+    const message = messages[index];
+
+    if (message.role !== "user") {
+      continue;
+    }
+
+    const nextAssistant = messages.slice(index + 1).find((candidate) => candidate.role === "assistant");
+    const input = redactEvalText(message.content);
+    const contains = nextAssistant ? redactEvalText(nextAssistant.content) : undefined;
+    const redactionPresent = input.includes("[redacted_") || Boolean(contains?.includes("[redacted_"));
+
+    turns.push({
+      input,
+      ...(nextAssistant
+        ? {
+            expect: {
+              response: redactionPresent
+                ? {
+                    notRegex: ["API_KEY|Bearer\\s+|(?:sk|agk|ak|dpat)[_-]|[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}"],
+                  }
+                : {
+                    contains: contains ?? "",
+                  },
+            },
+          }
+        : {}),
+    });
+  }
+
+  return turns;
+}
+
+function redactEvalText(value: string): string {
+  return value
+    .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "[redacted_email]")
+    .replace(
+      /\b[A-Z0-9_]*(?:API[_-]?KEY|TOKEN|SECRET|PASSWORD|PRIVATE[_-]?KEY|CLIENT[_-]?SECRET)\s*=\s*[^\s,;]+/gi,
+      (match) => `${match.slice(0, match.indexOf("=")).trim()}=[redacted_secret]`,
+    )
+    .replace(/\b(?:sk|agk|ak|dpat)[_-][A-Za-z0-9_-]{8,}\b/g, "[redacted_token]")
+    .replace(/\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi, "Bearer [redacted_token]")
+    .replace(/\+?\d[\d\s().-]{7,}\d/g, "[redacted_phone]");
+}
+
+function formatEvalTurns(turns: Array<{ input: string; expect?: EvalExpect }>): string {
+  return JSON.stringify(turns, null, 4)
+    .split("\n")
+    .map((line, index) => (index === 0 ? line : `  ${line}`))
+    .join("\n");
+}
+
+function slugify(value: string): string {
+  const slug = value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 48);
+
+  return slug || "conversation";
+}
+
+function resolveEvalOutputPath(root: string, out: string | undefined, title: string): string {
+  const outputPath = out ?? join("evals", `replay-${slugify(title)}.eval.ts`);
+
+  if (isAbsolute(outputPath)) {
+    throw new AgentKitError("validation_error", "eval --out must be a relative path inside the Agent Capsule.");
+  }
+
+  const file = resolve(root, outputPath);
+  const relativePath = relative(root, file);
+
+  if (relativePath === "" || relativePath.startsWith("..") || isAbsolute(relativePath)) {
+    throw new AgentKitError("validation_error", "eval --out must stay inside the Agent Capsule.");
+  }
+
+  if (!/\.(eval|spec)\.[cm]?[tj]s$/.test(file)) {
+    throw new AgentKitError(
+      "validation_error",
+      "eval --out must end with .eval.ts, .eval.js, .spec.ts, or another AgentKit eval/spec extension.",
+    );
+  }
+
+  return file;
+}
+
+function formatEvalFailure(error: unknown): string {
+  if (isAgentKitError(error)) {
+    return `${error.code}: ${error.message}`;
+  }
+
+  if (error instanceof Error) {
+    return `runtime_error: ${error.message}`;
+  }
+
+  return `runtime_error: ${String(error)}`;
+}
+
 function jsonContains(actual: unknown, expected: unknown): boolean {
   if (Array.isArray(expected)) {
     return jsonEqual(actual, expected);