@andreprado/agentkit 0.1.0-alpha.9 → 0.1.1

package/src/runtime/evals.ts

@@ -1,12 +1,13 @@
-import { mkdir, readdir, stat, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
 import type { Dirent } from "node:fs";
-import { dirname, join, relative, resolve } from "node:path";
+import { tmpdir } from "node:os";
+import { dirname, isAbsolute, join, relative, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
 
 import type { AgentRunResult } from "./chat";
 import { runAgentMessageFromCwd } from "./chat";
 import { findAgentCapsuleRoot, loadAgentCapsule } from "./config";
-import { AgentKitError } from "./errors";
+import { AgentKitError, isAgentKitError } from "./errors";
 import { openCapsuleStore, type StoredToolCall } from "../storage/sqlite";
 import { getConversationTraceFromCwd } from "./traces";
 
@@ -25,38 +26,76 @@ export type EvalResult = {
   output: string;
 };
 
-type EvalCase = {
+export type EvalCase = {
   name?: string;
   input?: string;
+  now?: string | Date;
   turns?: EvalTurn[];
   expect?: EvalExpect;
 };
 
-type EvalTurn =
+export type EvalTurn =
   | string
   | {
       input: string;
       expect?: EvalExpect;
     };
 
-type EvalExpect = {
+export type EvalExpect = EvalResponseExpectation & {
+  response?: EvalResponseExpectation;
+  tools?: EvalToolsExpectation;
+  tool_call?: ToolExpectation | ToolExpectation[];
+  toolCall?: ToolExpectation | ToolExpectation[];
+  tool_calls?: ToolExpectation | ToolExpectation[];
+  toolCalls?: ToolExpectation | ToolExpectation[];
+  tool_call_count?: number;
+  toolCallCount?: number;
+  tool_call_order?: string[];
+  toolCallOrder?: string[];
+  persisted_tool_call?: ToolExpectation | ToolExpectation[];
+  persistedToolCall?: ToolExpectation | ToolExpectation[];
+  persisted_tool_calls?: ToolExpectation | ToolExpectation[];
+  persistedToolCalls?: ToolExpectation | ToolExpectation[];
+};
+
+export type EvalResponseExpectation = {
   contains?: string | string[];
+  contains_all?: string | string[];
+  containsAll?: string | string[];
+  contains_any?: string | string[];
+  containsAny?: string | string[];
+  case_insensitive_contains?: string | string[];
+  caseInsensitiveContains?: string | string[];
   not_contains?: string | string[];
   notContains?: string | string[];
   regex?: string | string[];
   matches_regex?: string | string[];
   matchesRegex?: string | string[];
-  tool_call?: ToolExpectation | ToolExpectation[];
-  toolCall?: ToolExpectation | ToolExpectation[];
-  tool_calls?: ToolExpectation | ToolExpectation[];
-  toolCalls?: ToolExpectation | ToolExpectation[];
+  not_regex?: string | string[];
+  notRegex?: string | string[];
+  max_length?: number;
+  maxLength?: number;
+};
+
+export type EvalToolsExpectation =
+  | ToolExpectation
+  | ToolExpectation[]
+  | EvalToolsContainerExpectation;
+
+export type EvalToolsContainerExpectation = {
+  persisted?: ToolExpectation | ToolExpectation[];
   persisted_tool_call?: ToolExpectation | ToolExpectation[];
   persistedToolCall?: ToolExpectation | ToolExpectation[];
   persisted_tool_calls?: ToolExpectation | ToolExpectation[];
   persistedToolCalls?: ToolExpectation | ToolExpectation[];
+  called?: string | string[];
+  called_once?: string | string[];
+  calledOnce?: string | string[];
+  count?: number;
+  order?: string[];
 };
 
-type ToolExpectation =
+export type ToolExpectation =
   | string
   | {
       name?: string;
@@ -83,6 +122,10 @@ export type EvalFromConversationResult = {
   turns: number;
 };
 
+export function defineEval<const T extends EvalCase>(evalCase: T): T {
+  return evalCase;
+}
+
 export async function runEvalsFromCwd(cwd: string): Promise<EvalRunSummary> {
   const root = await findAgentCapsuleRoot(cwd);
   const evalFiles = await findEvalFiles(join(root, "evals"));
@@ -94,15 +137,29 @@ export async function runEvalsFromCwd(cwd: string): Promise<EvalRunSummary> {
   const results: EvalResult[] = [];
 
   for (const file of evalFiles) {
-    const evalCase = await loadEvalCase(file);
-    const run = await runEvalCase(root, relative(root, file), evalCase);
+    const evalFile = relative(root, file);
+    let name = evalFile;
+    let failures: string[];
+    let output: string;
+
+    try {
+      const evalCase = await loadEvalCase(file);
+      const run = await runEvalCase(root, evalFile, evalCase);
+
+      name = evalCase.name ?? evalFile;
+      failures = run.failures;
+      output = run.output;
+    } catch (error) {
+      failures = [formatEvalFailure(error)];
+      output = "";
+    }
 
     results.push({
-      name: evalCase.name ?? relative(root, file),
-      file: relative(root, file),
-      passed: run.failures.length === 0,
-      failures: run.failures,
-      output: run.output,
+      name,
+      file: evalFile,
+      passed: failures.length === 0,
+      failures,
+      output,
     });
   }
 
@@ -128,6 +185,7 @@ export async function writeEvalFromConversation(
   const root = await findAgentCapsuleRoot(cwd);
   const trace = await getConversationTraceFromCwd(root, input.conversationId);
   const turns = replayTurnsFromMessages(trace.messages);
+  const safeTitle = redactEvalText(trace.title ?? trace.id);
 
   if (turns.length === 0) {
     throw new AgentKitError(
@@ -136,24 +194,29 @@ export async function writeEvalFromConversation(
     );
   }
 
-  const file = resolve(root, input.out ?? join("evals", `replay-${slugify(trace.title ?? trace.id)}.eval.ts`));
+  const file = resolveEvalOutputPath(root, input.out, safeTitle);
+  const source = `import { defineEval } from "@andreprado/agentkit";
 
-  if (!input.force && await pathExists(file)) {
-    throw new AgentKitError(
-      "validation_error",
-      `${relative(root, file)} already exists. Re-run with --force or choose --out <path>.`,
-    );
-  }
+export default defineEval({
+  name: ${JSON.stringify(input.name ? redactEvalText(input.name) : `replay ${safeTitle}`)},
+  turns: ${formatEvalTurns(turns)},
+});
+`;
 
   await mkdir(dirname(file), { recursive: true });
-  await writeFile(
-    file,
-    `export default {
-  name: ${JSON.stringify(input.name ?? `replay ${trace.title ?? trace.id}`)},
-  turns: ${formatEvalTurns(turns)},
-};
-`,
-  );
+
+  try {
+    await writeFile(file, source, { flag: input.force ? "w" : "wx" });
+  } catch (error) {
+    if (isNodeError(error) && error.code === "EEXIST") {
+      throw new AgentKitError(
+        "validation_error",
+        `${relative(root, file)} already exists. Re-run with --force or choose --out <path>.`,
+      );
+    }
+
+    throw error;
+  }
 
   return {
     root,
@@ -180,29 +243,75 @@ async function runEvalCase(
   const conversationId = `eval_${crypto.randomUUID()}`;
   const failures: string[] = [];
   let output = "";
+  const now = resolveEvalNow(file, evalCase.now);
+  const evalStore = await createEvalStore();
 
-  for (const [index, turn] of turns.entries()) {
-    const run = await runAgentMessageFromCwd(root, {
-      message: turn.input,
-      conversationId,
-      runtime: {
-        environment: "eval",
-        invocation: "eval",
-      },
-    });
-    const persistedToolCalls = await loadPersistedToolCalls(root, run.runId);
-    const turnFailures = evaluateExpectations(turn.expect ?? {}, run, persistedToolCalls);
-
-    for (const failure of turnFailures) {
-      failures.push(turns.length === 1 ? failure : `turn ${index + 1}: ${failure}`);
+  try {
+    for (const [index, turn] of turns.entries()) {
+      try {
+        const run = await runAgentMessageFromCwd(root, {
+          message: turn.input,
+          conversationId,
+          now,
+          localStoragePath: evalStore.databasePath,
+          runtime: {
+            environment: "eval",
+            invocation: "eval",
+          },
+        });
+        const persistedToolCalls = await loadPersistedToolCalls(root, run.runId, evalStore.databasePath);
+        const turnFailures = evaluateExpectations(turn.expect ?? {}, run, persistedToolCalls);
+
+        for (const failure of turnFailures) {
+          failures.push(turns.length === 1 ? failure : `turn ${index + 1}: ${failure}`);
+        }
+
+        output = run.message.content;
+      } catch (error) {
+        failures.push(turns.length === 1 ? formatEvalFailure(error) : `turn ${index + 1}: ${formatEvalFailure(error)}`);
+        break;
+      }
     }
-
-    output = run.message.content;
+  } finally {
+    await rm(evalStore.directory, { recursive: true, force: true });
   }
 
   return { failures, output };
 }
 
+async function createEvalStore(): Promise<{ directory: string; databasePath: string }> {
+  const directory = await mkdtemp(join(tmpdir(), "agentkit-eval-"));
+  return {
+    directory,
+    databasePath: join(directory, "agentkit.db"),
+  };
+}
+
+function resolveEvalNow(file: string, value: EvalCase["now"]): Date | undefined {
+  if (value === undefined) {
+    return undefined;
+  }
+
+  if (typeof value === "string" && !hasExplicitIsoOffset(value)) {
+    throw new AgentKitError(
+      "validation_error",
+      `${file} now must be an ISO timestamp with an explicit timezone offset, such as "2026-02-04T02:30:00.000Z" or "2026-02-04T02:30:00-05:00".`,
+    );
+  }
+
+  const date = value instanceof Date ? new Date(value.getTime()) : new Date(value);
+
+  if (Number.isNaN(date.getTime())) {
+    throw new AgentKitError("validation_error", `${file} now must be a valid ISO timestamp or Date when provided.`);
+  }
+
+  return date;
+}
+
+function hasExplicitIsoOffset(value: string): boolean {
+  return /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/.test(value.trim());
+}
+
 function normalizeEvalTurns(evalCase: EvalCase): Array<{ input: string; expect?: EvalExpect }> {
   if (evalCase.turns !== undefined) {
     if (!Array.isArray(evalCase.turns)) {
@@ -288,9 +397,9 @@ async function loadEvalCase(file: string): Promise<EvalCase> {
   return module.default;
 }
 
-async function loadPersistedToolCalls(root: string, runId: string): Promise<ToolCallSnapshot[]> {
+async function loadPersistedToolCalls(root: string, runId: string, localStoragePath?: string): Promise<ToolCallSnapshot[]> {
   const capsule = await loadAgentCapsule(root);
-  const store = await openCapsuleStore(capsule);
+  const store = await openCapsuleStore(localStoragePath ? { ...capsule, storagePath: localStoragePath } : capsule);
 
   try {
     return store.listToolCallsForRun(runId).map(toolCallSnapshotFromStored);
@@ -303,12 +412,89 @@ function evaluateExpectations(expect: EvalExpect, run: AgentRunResult, persisted
   const failures: string[] = [];
   const content = run.message.content;
 
-  for (const expected of list(expect.contains)) {
+  for (const responseExpectation of responseExpectations(expect)) {
+    failures.push(...evaluateResponseExpectation(responseExpectation, content));
+  }
+
+  const toolAssertions = collectToolAssertions(expect);
+
+  for (const toolExpectation of toolAssertions.persisted) {
+    const match = persistedToolCalls.find((toolCall) => matchesToolExpectation(toolCall, toolExpectation));
+
+    if (!match) {
+      failures.push(`expected persisted tool call ${formatToolExpectation(toolExpectation)}`);
+    }
+  }
+
+  for (const expectedName of toolAssertions.called) {
+    if (!persistedToolCalls.some((toolCall) => toolCall.name === expectedName)) {
+      failures.push(`expected persisted tool call named ${JSON.stringify(expectedName)}`);
+    }
+  }
+
+  for (const expectedName of toolAssertions.calledOnce) {
+    const count = persistedToolCalls.filter((toolCall) => toolCall.name === expectedName).length;
+
+    if (count !== 1) {
+      failures.push(`expected persisted tool call ${JSON.stringify(expectedName)} exactly once, got ${count}`);
+    }
+  }
+
+  for (const expectedCount of toolAssertions.counts) {
+    if (!Number.isInteger(expectedCount) || expectedCount < 0) {
+      failures.push(`expected tool call count to be a non-negative integer, got ${JSON.stringify(expectedCount)}`);
+      continue;
+    }
+
+    if (persistedToolCalls.length !== expectedCount) {
+      failures.push(`expected ${expectedCount} persisted tool call(s), got ${persistedToolCalls.length}`);
+    }
+  }
+
+  for (const expectedOrder of toolAssertions.orders) {
+    if (!namesContainSubsequence(persistedToolNames(persistedToolCalls), expectedOrder)) {
+      failures.push(
+        `expected persisted tool call order ${JSON.stringify(expectedOrder)}, got ${JSON.stringify(
+          persistedToolNames(persistedToolCalls),
+        )}`,
+      );
+    }
+  }
+
+  return failures;
+}
+
+function responseExpectations(expect: EvalExpect): EvalResponseExpectation[] {
+  return [expect, expect.response].filter((value): value is EvalResponseExpectation => value !== undefined);
+}
+
+function evaluateResponseExpectation(expect: EvalResponseExpectation, content: string): string[] {
+  const failures: string[] = [];
+
+  for (const expected of [
+    ...list(expect.contains),
+    ...list(expect.contains_all),
+    ...list(expect.containsAll),
+  ]) {
     if (!content.includes(expected)) {
       failures.push(`expected output to contain ${JSON.stringify(expected)}`);
     }
   }
 
+  const containsAny = [...list(expect.contains_any), ...list(expect.containsAny)];
+
+  if (containsAny.length > 0 && !containsAny.some((expected) => content.includes(expected))) {
+    failures.push(`expected output to contain any of ${JSON.stringify(containsAny)}`);
+  }
+
+  const lowerContent = content.toLowerCase();
+
+  for (const expected of [...list(expect.case_insensitive_contains), ...list(expect.caseInsensitiveContains)]) {
+    if (!lowerContent.includes(expected.toLowerCase())) {
+      failures.push(`expected output to contain ${JSON.stringify(expected)} case-insensitively`);
+    }
+  }
+
   for (const forbidden of [...list(expect.not_contains), ...list(expect.notContains)]) {
     if (content.includes(forbidden)) {
       failures.push(`expected output not to contain ${JSON.stringify(forbidden)}`);
@@ -321,28 +507,140 @@ function evaluateExpectations(expect: EvalExpect, run: AgentRunResult, persisted
     }
   }
 
-  const toolExpectations = [
-    ...toolList(expect.tool_call),
-    ...toolList(expect.toolCall),
-    ...toolList(expect.tool_calls),
-    ...toolList(expect.toolCalls),
-    ...toolList(expect.persisted_tool_call),
-    ...toolList(expect.persistedToolCall),
-    ...toolList(expect.persisted_tool_calls),
-    ...toolList(expect.persistedToolCalls),
-  ];
+  for (const pattern of [...list(expect.not_regex), ...list(expect.notRegex)]) {
+    if (new RegExp(pattern).test(content)) {
+      failures.push(`expected output not to match /${pattern}/`);
+    }
+  }
 
-  for (const toolExpectation of toolExpectations) {
-    const match = persistedToolCalls.find((toolCall) => matchesToolExpectation(toolCall, toolExpectation));
+  for (const expectedMaxLength of [expect.max_length, expect.maxLength]) {
+    if (expectedMaxLength === undefined) {
+      continue;
+    }
 
-    if (!match) {
-      failures.push(`expected persisted tool call ${formatToolExpectation(toolExpectation)}`);
+    if (!Number.isInteger(expectedMaxLength) || expectedMaxLength < 0) {
+      failures.push(`expected response maxLength to be a non-negative integer, got ${JSON.stringify(expectedMaxLength)}`);
+      continue;
+    }
+
+    if (content.length > expectedMaxLength) {
+      failures.push(`expected output length to be <= ${expectedMaxLength}, got ${content.length}`);
     }
   }
 
   return failures;
 }
 
+type ToolAssertions = {
+  persisted: ToolExpectation[];
+  called: string[];
+  calledOnce: string[];
+  counts: number[];
+  orders: string[][];
+};
+
+function collectToolAssertions(expect: EvalExpect): ToolAssertions {
+  const assertions: ToolAssertions = {
+    persisted: [
+      ...toolList(expect.tool_call),
+      ...toolList(expect.toolCall),
+      ...toolList(expect.tool_calls),
+      ...toolList(expect.toolCalls),
+      ...toolList(expect.persisted_tool_call),
+      ...toolList(expect.persistedToolCall),
+      ...toolList(expect.persisted_tool_calls),
+      ...toolList(expect.persistedToolCalls),
+    ],
+    called: [],
+    calledOnce: [],
+    counts: [],
+    orders: [],
+  };
+
+  if (expect.tool_call_count !== undefined) {
+    assertions.counts.push(expect.tool_call_count);
+  }
+
+  if (expect.toolCallCount !== undefined) {
+    assertions.counts.push(expect.toolCallCount);
+  }
+
+  if (expect.tool_call_order !== undefined) {
+    assertions.orders.push(expect.tool_call_order);
+  }
+
+  if (expect.toolCallOrder !== undefined) {
+    assertions.orders.push(expect.toolCallOrder);
+  }
+
+  appendNestedToolAssertions(assertions, expect.tools);
+
+  return assertions;
+}
+
+function appendNestedToolAssertions(assertions: ToolAssertions, tools: EvalToolsExpectation | undefined): void {
+  if (tools === undefined) {
+    return;
+  }
+
+  if (!isToolsContainer(tools)) {
+    assertions.persisted.push(...toolList(tools));
+    return;
+  }
+
+  assertions.persisted.push(
+    ...toolList(tools.persisted),
+    ...toolList(tools.persisted_tool_call),
+    ...toolList(tools.persistedToolCall),
+    ...toolList(tools.persisted_tool_calls),
+    ...toolList(tools.persistedToolCalls),
+  );
+  assertions.called.push(...list(tools.called));
+  assertions.calledOnce.push(...list(tools.called_once), ...list(tools.calledOnce));
+
+  if (tools.count !== undefined) {
+    assertions.counts.push(tools.count);
+  }
+
+  if (tools.order !== undefined) {
+    assertions.orders.push(tools.order);
+  }
+}
+
+function isToolsContainer(value: EvalToolsExpectation): value is EvalToolsContainerExpectation {
+  if (!isRecord(value)) {
+    return false;
+  }
+
+  if (
+    "name" in value ||
+    "input" in value ||
+    "output" in value ||
+    "rendered" in value ||
+    "status" in value ||
+    "visibility" in value
+  ) {
+    return false;
+  }
+
+  if (Object.keys(value).length === 0) {
+    return true;
+  }
+
+  return (
+    "persisted" in value ||
+    "persisted_tool_call" in value ||
+    "persistedToolCall" in value ||
+    "persisted_tool_calls" in value ||
+    "persistedToolCalls" in value ||
+    "called" in value ||
+    "called_once" in value ||
+    "calledOnce" in value ||
+    "count" in value ||
+    "order" in value
+  );
+}
+
 function list(value: string | string[] | undefined): string[] {
   if (value === undefined) {
     return [];
@@ -351,6 +649,30 @@ function list(value: string | string[] | undefined): string[] {
   return Array.isArray(value) ? value : [value];
 }
 
+function persistedToolNames(toolCalls: ToolCallSnapshot[]): string[] {
+  return toolCalls.map((toolCall) => String(toolCall.name ?? ""));
+}
+
+function namesContainSubsequence(actual: string[], expected: string[]): boolean {
+  if (expected.length === 0) {
+    return true;
+  }
+
+  let actualIndex = 0;
+
+  for (const expectedName of expected) {
+    actualIndex = actual.findIndex((actualName, index) => index >= actualIndex && actualName === expectedName);
+
+    if (actualIndex === -1) {
+      return false;
+    }
+
+    actualIndex += 1;
+  }
+
+  return true;
+}
+
 function toolList(value: ToolExpectation | ToolExpectation[] | undefined): ToolExpectation[] {
   if (value === undefined) {
     return [];
@@ -423,12 +745,22 @@ function replayTurnsFromMessages(
     }
 
     const nextAssistant = messages.slice(index + 1).find((candidate) => candidate.role === "assistant");
+    const input = redactEvalText(message.content);
+    const contains = nextAssistant ? redactEvalText(nextAssistant.content) : undefined;
+    const redactionPresent = input.includes("[redacted_") || Boolean(contains?.includes("[redacted_"));
+
     turns.push({
-      input: message.content,
+      input,
       ...(nextAssistant
         ? {
             expect: {
-              contains: nextAssistant.content,
+              response: redactionPresent
+                ? {
+                    notRegex: ["API_KEY|Bearer\\s+|(?:sk|agk|ak|dpat)[_-]|[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}"],
+                  }
+                : {
+                    contains: contains ?? "",
+                  },
             },
           }
         : {}),
@@ -438,6 +770,18 @@ function replayTurnsFromMessages(
   return turns;
 }
 
+function redactEvalText(value: string): string {
+  return value
+    .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "[redacted_email]")
+    .replace(
+      /\b[A-Z0-9_]*(?:API[_-]?KEY|TOKEN|SECRET|PASSWORD|PRIVATE[_-]?KEY|CLIENT[_-]?SECRET)\s*=\s*[^\s,;]+/gi,
+      (match) => `${match.slice(0, match.indexOf("=")).trim()}=[redacted_secret]`,
+    )
+    .replace(/\b(?:sk|agk|ak|dpat)[_-][A-Za-z0-9_-]{8,}\b/g, "[redacted_token]")
+    .replace(/\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi, "Bearer [redacted_token]")
+    .replace(/\+?\d[\d\s().-]{7,}\d/g, "[redacted_phone]");
+}
+
 function formatEvalTurns(turns: Array<{ input: string; expect?: EvalExpect }>): string {
   return JSON.stringify(turns, null, 4)
     .split("\n")
@@ -455,13 +799,40 @@ function slugify(value: string): string {
   return slug || "conversation";
 }
 
-async function pathExists(path: string): Promise<boolean> {
-  try {
-    await stat(path);
-    return true;
-  } catch {
-    return false;
+function resolveEvalOutputPath(root: string, out: string | undefined, title: string): string {
+  const outputPath = out ?? join("evals", `replay-${slugify(title)}.eval.ts`);
+
+  if (isAbsolute(outputPath)) {
+    throw new AgentKitError("validation_error", "eval --out must be a relative path inside the Agent Capsule.");
+  }
+
+  const file = resolve(root, outputPath);
+  const relativePath = relative(root, file);
+
+  if (relativePath === "" || relativePath.startsWith("..") || isAbsolute(relativePath)) {
+    throw new AgentKitError("validation_error", "eval --out must stay inside the Agent Capsule.");
   }
+
+  if (!/\.(eval|spec)\.[cm]?[tj]s$/.test(file)) {
+    throw new AgentKitError(
+      "validation_error",
+      "eval --out must end with .eval.ts, .eval.js, .spec.ts, or another AgentKit eval/spec extension.",
+    );
+  }
+
+  return file;
+}
+
+function formatEvalFailure(error: unknown): string {
+  if (isAgentKitError(error)) {
+    return `${error.code}: ${error.message}`;
+  }
+
+  if (error instanceof Error) {
+    return `runtime_error: ${error.message}`;
+  }
+
+  return `runtime_error: ${String(error)}`;
 }
 
 function jsonContains(actual: unknown, expected: unknown): boolean {