@andreprado/agentkit 0.1.0-alpha.14 → 0.1.0-alpha.16

package/src/runtime/evals.ts

@@ -1,12 +1,12 @@
-import { mkdir, readdir, stat, writeFile } from "node:fs/promises";
+import { mkdir, readdir, writeFile } from "node:fs/promises";
 import type { Dirent } from "node:fs";
-import { dirname, join, relative, resolve } from "node:path";
+import { dirname, isAbsolute, join, relative, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
 
 import type { AgentRunResult } from "./chat";
 import { runAgentMessageFromCwd } from "./chat";
 import { findAgentCapsuleRoot, loadAgentCapsule } from "./config";
-import { AgentKitError } from "./errors";
+import { AgentKitError, isAgentKitError } from "./errors";
 import { openCapsuleStore, type StoredToolCall } from "../storage/sqlite";
 import { getConversationTraceFromCwd } from "./traces";
 
@@ -25,38 +25,76 @@ export type EvalResult = {
   output: string;
 };
 
-type EvalCase = {
+export type EvalCase = {
   name?: string;
   input?: string;
+  now?: string | Date;
   turns?: EvalTurn[];
   expect?: EvalExpect;
 };
 
-type EvalTurn =
+export type EvalTurn =
   | string
   | {
       input: string;
       expect?: EvalExpect;
     };
 
-type EvalExpect = {
+export type EvalExpect = EvalResponseExpectation & {
+  response?: EvalResponseExpectation;
+  tools?: EvalToolsExpectation;
+  tool_call?: ToolExpectation | ToolExpectation[];
+  toolCall?: ToolExpectation | ToolExpectation[];
+  tool_calls?: ToolExpectation | ToolExpectation[];
+  toolCalls?: ToolExpectation | ToolExpectation[];
+  tool_call_count?: number;
+  toolCallCount?: number;
+  tool_call_order?: string[];
+  toolCallOrder?: string[];
+  persisted_tool_call?: ToolExpectation | ToolExpectation[];
+  persistedToolCall?: ToolExpectation | ToolExpectation[];
+  persisted_tool_calls?: ToolExpectation | ToolExpectation[];
+  persistedToolCalls?: ToolExpectation | ToolExpectation[];
+};
+
+export type EvalResponseExpectation = {
   contains?: string | string[];
+  contains_all?: string | string[];
+  containsAll?: string | string[];
+  contains_any?: string | string[];
+  containsAny?: string | string[];
+  case_insensitive_contains?: string | string[];
+  caseInsensitiveContains?: string | string[];
   not_contains?: string | string[];
   notContains?: string | string[];
   regex?: string | string[];
   matches_regex?: string | string[];
   matchesRegex?: string | string[];
-  tool_call?: ToolExpectation | ToolExpectation[];
-  toolCall?: ToolExpectation | ToolExpectation[];
-  tool_calls?: ToolExpectation | ToolExpectation[];
-  toolCalls?: ToolExpectation | ToolExpectation[];
+  not_regex?: string | string[];
+  notRegex?: string | string[];
+  max_length?: number;
+  maxLength?: number;
+};
+
+export type EvalToolsExpectation =
+  | ToolExpectation
+  | ToolExpectation[]
+  | EvalToolsContainerExpectation;
+
+export type EvalToolsContainerExpectation = {
+  persisted?: ToolExpectation | ToolExpectation[];
   persisted_tool_call?: ToolExpectation | ToolExpectation[];
   persistedToolCall?: ToolExpectation | ToolExpectation[];
   persisted_tool_calls?: ToolExpectation | ToolExpectation[];
   persistedToolCalls?: ToolExpectation | ToolExpectation[];
+  called?: string | string[];
+  called_once?: string | string[];
+  calledOnce?: string | string[];
+  count?: number;
+  order?: string[];
 };
 
-type ToolExpectation =
+export type ToolExpectation =
   | string
   | {
       name?: string;
@@ -83,6 +121,10 @@ export type EvalFromConversationResult = {
   turns: number;
 };
 
+export function defineEval<const T extends EvalCase>(evalCase: T): T {
+  return evalCase;
+}
+
 export async function runEvalsFromCwd(cwd: string): Promise<EvalRunSummary> {
   const root = await findAgentCapsuleRoot(cwd);
   const evalFiles = await findEvalFiles(join(root, "evals"));
@@ -94,15 +136,29 @@ export async function runEvalsFromCwd(cwd: string): Promise<EvalRunSummary> {
   const results: EvalResult[] = [];
 
   for (const file of evalFiles) {
-    const evalCase = await loadEvalCase(file);
-    const run = await runEvalCase(root, relative(root, file), evalCase);
+    const evalFile = relative(root, file);
+    let name = evalFile;
+    let failures: string[];
+    let output: string;
+
+    try {
+      const evalCase = await loadEvalCase(file);
+      const run = await runEvalCase(root, evalFile, evalCase);
+
+      name = evalCase.name ?? evalFile;
+      failures = run.failures;
+      output = run.output;
+    } catch (error) {
+      failures = [formatEvalFailure(error)];
+      output = "";
+    }
 
     results.push({
-      name: evalCase.name ?? relative(root, file),
-      file: relative(root, file),
-      passed: run.failures.length === 0,
-      failures: run.failures,
-      output: run.output,
+      name,
+      file: evalFile,
+      passed: failures.length === 0,
+      failures,
+      output,
     });
   }
 
@@ -136,24 +192,29 @@ export async function writeEvalFromConversation(
     );
   }
 
-  const file = resolve(root, input.out ?? join("evals", `replay-${slugify(trace.title ?? trace.id)}.eval.ts`));
+  const file = resolveEvalOutputPath(root, input.out, trace.title ?? trace.id);
+  const source = `import { defineEval } from "@andreprado/agentkit";
 
-  if (!input.force && await pathExists(file)) {
-    throw new AgentKitError(
-      "validation_error",
-      `${relative(root, file)} already exists. Re-run with --force or choose --out <path>.`,
-    );
-  }
-
-  await mkdir(dirname(file), { recursive: true });
-  await writeFile(
-    file,
-    `export default {
+export default defineEval({
   name: ${JSON.stringify(input.name ?? `replay ${trace.title ?? trace.id}`)},
   turns: ${formatEvalTurns(turns)},
-};
-`,
-  );
+});
+`;
+
+  await mkdir(dirname(file), { recursive: true });
+
+  try {
+    await writeFile(file, source, { flag: input.force ? "w" : "wx" });
+  } catch (error) {
+    if (isNodeError(error) && error.code === "EEXIST") {
+      throw new AgentKitError(
+        "validation_error",
+        `${relative(root, file)} already exists. Re-run with --force or choose --out <path>.`,
+      );
+    }
+
+    throw error;
+  }
 
   return {
     root,
@@ -180,29 +241,61 @@ async function runEvalCase(
   const conversationId = `eval_${crypto.randomUUID()}`;
   const failures: string[] = [];
   let output = "";
+  const now = resolveEvalNow(file, evalCase.now);
 
   for (const [index, turn] of turns.entries()) {
-    const run = await runAgentMessageFromCwd(root, {
-      message: turn.input,
-      conversationId,
-      runtime: {
-        environment: "eval",
-        invocation: "eval",
-      },
-    });
-    const persistedToolCalls = await loadPersistedToolCalls(root, run.runId);
-    const turnFailures = evaluateExpectations(turn.expect ?? {}, run, persistedToolCalls);
+    try {
+      const run = await runAgentMessageFromCwd(root, {
+        message: turn.input,
+        conversationId,
+        now,
+        runtime: {
+          environment: "eval",
+          invocation: "eval",
+        },
+      });
+      const persistedToolCalls = await loadPersistedToolCalls(root, run.runId);
+      const turnFailures = evaluateExpectations(turn.expect ?? {}, run, persistedToolCalls);
+
+      for (const failure of turnFailures) {
+        failures.push(turns.length === 1 ? failure : `turn ${index + 1}: ${failure}`);
+      }
 
-    for (const failure of turnFailures) {
-      failures.push(turns.length === 1 ? failure : `turn ${index + 1}: ${failure}`);
+      output = run.message.content;
+    } catch (error) {
+      failures.push(turns.length === 1 ? formatEvalFailure(error) : `turn ${index + 1}: ${formatEvalFailure(error)}`);
+      break;
     }
-
-    output = run.message.content;
   }
 
   return { failures, output };
 }
 
+function resolveEvalNow(file: string, value: EvalCase["now"]): Date | undefined {
+  if (value === undefined) {
+    return undefined;
+  }
+
+  if (typeof value === "string" && !hasExplicitIsoOffset(value)) {
+    throw new AgentKitError(
+      "validation_error",
+      `${file} now must be an ISO timestamp with an explicit timezone offset, such as "2026-02-04T02:30:00.000Z" or "2026-02-04T02:30:00-05:00".`,
+    );
+  }
+
+  const date = value instanceof Date ? new Date(value.getTime()) : new Date(value);
+
+  if (Number.isNaN(date.getTime())) {
+    throw new AgentKitError("validation_error", `${file} now must be a valid ISO timestamp or Date when provided.`);
+  }
+
+  return date;
+}
+
+function hasExplicitIsoOffset(value: string): boolean {
+  return /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/.test(value.trim());
+}
+
 function normalizeEvalTurns(evalCase: EvalCase): Array<{ input: string; expect?: EvalExpect }> {
   if (evalCase.turns !== undefined) {
     if (!Array.isArray(evalCase.turns)) {
@@ -303,12 +396,89 @@ function evaluateExpectations(expect: EvalExpect, run: AgentRunResult, persisted
   const failures: string[] = [];
   const content = run.message.content;
 
-  for (const expected of list(expect.contains)) {
+  for (const responseExpectation of responseExpectations(expect)) {
+    failures.push(...evaluateResponseExpectation(responseExpectation, content));
+  }
+
+  const toolAssertions = collectToolAssertions(expect);
+
+  for (const toolExpectation of toolAssertions.persisted) {
+    const match = persistedToolCalls.find((toolCall) => matchesToolExpectation(toolCall, toolExpectation));
+
+    if (!match) {
+      failures.push(`expected persisted tool call ${formatToolExpectation(toolExpectation)}`);
+    }
+  }
+
+  for (const expectedName of toolAssertions.called) {
+    if (!persistedToolCalls.some((toolCall) => toolCall.name === expectedName)) {
+      failures.push(`expected persisted tool call named ${JSON.stringify(expectedName)}`);
+    }
+  }
+
+  for (const expectedName of toolAssertions.calledOnce) {
+    const count = persistedToolCalls.filter((toolCall) => toolCall.name === expectedName).length;
+
+    if (count !== 1) {
+      failures.push(`expected persisted tool call ${JSON.stringify(expectedName)} exactly once, got ${count}`);
+    }
+  }
+
+  for (const expectedCount of toolAssertions.counts) {
+    if (!Number.isInteger(expectedCount) || expectedCount < 0) {
+      failures.push(`expected tool call count to be a non-negative integer, got ${JSON.stringify(expectedCount)}`);
+      continue;
+    }
+
+    if (persistedToolCalls.length !== expectedCount) {
+      failures.push(`expected ${expectedCount} persisted tool call(s), got ${persistedToolCalls.length}`);
+    }
+  }
+
+  for (const expectedOrder of toolAssertions.orders) {
+    if (!namesContainSubsequence(persistedToolNames(persistedToolCalls), expectedOrder)) {
+      failures.push(
+        `expected persisted tool call order ${JSON.stringify(expectedOrder)}, got ${JSON.stringify(
+          persistedToolNames(persistedToolCalls),
+        )}`,
+      );
+    }
+  }
+
+  return failures;
+}
+
+function responseExpectations(expect: EvalExpect): EvalResponseExpectation[] {
+  return [expect, expect.response].filter((value): value is EvalResponseExpectation => value !== undefined);
+}
+
+function evaluateResponseExpectation(expect: EvalResponseExpectation, content: string): string[] {
+  const failures: string[] = [];
+
+  for (const expected of [
+    ...list(expect.contains),
+    ...list(expect.contains_all),
+    ...list(expect.containsAll),
+  ]) {
     if (!content.includes(expected)) {
       failures.push(`expected output to contain ${JSON.stringify(expected)}`);
     }
   }
 
+  const containsAny = [...list(expect.contains_any), ...list(expect.containsAny)];
+
+  if (containsAny.length > 0 && !containsAny.some((expected) => content.includes(expected))) {
+    failures.push(`expected output to contain any of ${JSON.stringify(containsAny)}`);
+  }
+
+  const lowerContent = content.toLowerCase();
+
+  for (const expected of [...list(expect.case_insensitive_contains), ...list(expect.caseInsensitiveContains)]) {
+    if (!lowerContent.includes(expected.toLowerCase())) {
+      failures.push(`expected output to contain ${JSON.stringify(expected)} case-insensitively`);
+    }
+  }
+
   for (const forbidden of [...list(expect.not_contains), ...list(expect.notContains)]) {
     if (content.includes(forbidden)) {
       failures.push(`expected output not to contain ${JSON.stringify(forbidden)}`);
@@ -321,28 +491,140 @@ function evaluateExpectations(expect: EvalExpect, run: AgentRunResult, persisted
     }
   }
 
-  const toolExpectations = [
-    ...toolList(expect.tool_call),
-    ...toolList(expect.toolCall),
-    ...toolList(expect.tool_calls),
-    ...toolList(expect.toolCalls),
-    ...toolList(expect.persisted_tool_call),
-    ...toolList(expect.persistedToolCall),
-    ...toolList(expect.persisted_tool_calls),
-    ...toolList(expect.persistedToolCalls),
-  ];
+  for (const pattern of [...list(expect.not_regex), ...list(expect.notRegex)]) {
+    if (new RegExp(pattern).test(content)) {
+      failures.push(`expected output not to match /${pattern}/`);
+    }
+  }
 
-  for (const toolExpectation of toolExpectations) {
-    const match = persistedToolCalls.find((toolCall) => matchesToolExpectation(toolCall, toolExpectation));
+  for (const expectedMaxLength of [expect.max_length, expect.maxLength]) {
+    if (expectedMaxLength === undefined) {
+      continue;
+    }
 
-    if (!match) {
-      failures.push(`expected persisted tool call ${formatToolExpectation(toolExpectation)}`);
+    if (!Number.isInteger(expectedMaxLength) || expectedMaxLength < 0) {
+      failures.push(`expected response maxLength to be a non-negative integer, got ${JSON.stringify(expectedMaxLength)}`);
+      continue;
+    }
+
+    if (content.length > expectedMaxLength) {
+      failures.push(`expected output length to be <= ${expectedMaxLength}, got ${content.length}`);
     }
   }
 
   return failures;
 }
 
+type ToolAssertions = {
+  persisted: ToolExpectation[];
+  called: string[];
+  calledOnce: string[];
+  counts: number[];
+  orders: string[][];
+};
+
+function collectToolAssertions(expect: EvalExpect): ToolAssertions {
+  const assertions: ToolAssertions = {
+    persisted: [
+      ...toolList(expect.tool_call),
+      ...toolList(expect.toolCall),
+      ...toolList(expect.tool_calls),
+      ...toolList(expect.toolCalls),
+      ...toolList(expect.persisted_tool_call),
+      ...toolList(expect.persistedToolCall),
+      ...toolList(expect.persisted_tool_calls),
+      ...toolList(expect.persistedToolCalls),
+    ],
+    called: [],
+    calledOnce: [],
+    counts: [],
+    orders: [],
+  };
+
+  if (expect.tool_call_count !== undefined) {
+    assertions.counts.push(expect.tool_call_count);
+  }
+
+  if (expect.toolCallCount !== undefined) {
+    assertions.counts.push(expect.toolCallCount);
+  }
+
+  if (expect.tool_call_order !== undefined) {
+    assertions.orders.push(expect.tool_call_order);
+  }
+
+  if (expect.toolCallOrder !== undefined) {
+    assertions.orders.push(expect.toolCallOrder);
+  }
+
+  appendNestedToolAssertions(assertions, expect.tools);
+
+  return assertions;
+}
+
+function appendNestedToolAssertions(assertions: ToolAssertions, tools: EvalToolsExpectation | undefined): void {
+  if (tools === undefined) {
+    return;
+  }
+
+  if (!isToolsContainer(tools)) {
+    assertions.persisted.push(...toolList(tools));
+    return;
+  }
+
+  assertions.persisted.push(
+    ...toolList(tools.persisted),
+    ...toolList(tools.persisted_tool_call),
+    ...toolList(tools.persistedToolCall),
+    ...toolList(tools.persisted_tool_calls),
+    ...toolList(tools.persistedToolCalls),
+  );
+  assertions.called.push(...list(tools.called));
+  assertions.calledOnce.push(...list(tools.called_once), ...list(tools.calledOnce));
+
+  if (tools.count !== undefined) {
+    assertions.counts.push(tools.count);
+  }
+
+  if (tools.order !== undefined) {
+    assertions.orders.push(tools.order);
+  }
+}
+
+function isToolsContainer(value: EvalToolsExpectation): value is EvalToolsContainerExpectation {
+  if (!isRecord(value)) {
+    return false;
+  }
+
+  if (
+    "name" in value ||
+    "input" in value ||
+    "output" in value ||
+    "rendered" in value ||
+    "status" in value ||
+    "visibility" in value
+  ) {
+    return false;
+  }
+
+  if (Object.keys(value).length === 0) {
+    return true;
+  }
+
+  return (
+    "persisted" in value ||
+    "persisted_tool_call" in value ||
+    "persistedToolCall" in value ||
+    "persisted_tool_calls" in value ||
+    "persistedToolCalls" in value ||
+    "called" in value ||
+    "called_once" in value ||
+    "calledOnce" in value ||
+    "count" in value ||
+    "order" in value
+  );
+}
+
 function list(value: string | string[] | undefined): string[] {
   if (value === undefined) {
     return [];
@@ -351,6 +633,30 @@ function list(value: string | string[] | undefined): string[] {
   return Array.isArray(value) ? value : [value];
 }
 
+function persistedToolNames(toolCalls: ToolCallSnapshot[]): string[] {
+  return toolCalls.map((toolCall) => String(toolCall.name ?? ""));
+}
+
+function namesContainSubsequence(actual: string[], expected: string[]): boolean {
+  if (expected.length === 0) {
+    return true;
+  }
+
+  let actualIndex = 0;
+
+  for (const expectedName of expected) {
+    actualIndex = actual.findIndex((actualName, index) => index >= actualIndex && actualName === expectedName);
+
+    if (actualIndex === -1) {
+      return false;
+    }
+
+    actualIndex += 1;
+  }
+
+  return true;
+}
+
 function toolList(value: ToolExpectation | ToolExpectation[] | undefined): ToolExpectation[] {
   if (value === undefined) {
     return [];
@@ -428,7 +734,9 @@ function replayTurnsFromMessages(
       ...(nextAssistant
         ? {
             expect: {
-              contains: nextAssistant.content,
+              response: {
+                contains: nextAssistant.content,
+              },
             },
           }
         : {}),
@@ -455,13 +763,40 @@ function slugify(value: string): string {
   return slug || "conversation";
 }
 
-async function pathExists(path: string): Promise<boolean> {
-  try {
-    await stat(path);
-    return true;
-  } catch {
-    return false;
+function resolveEvalOutputPath(root: string, out: string | undefined, title: string): string {
+  const outputPath = out ?? join("evals", `replay-${slugify(title)}.eval.ts`);
+
+  if (isAbsolute(outputPath)) {
+    throw new AgentKitError("validation_error", "eval --out must be a relative path inside the Agent Capsule.");
+  }
+
+  const file = resolve(root, outputPath);
+  const relativePath = relative(root, file);
+
+  if (relativePath === "" || relativePath.startsWith("..") || isAbsolute(relativePath)) {
+    throw new AgentKitError("validation_error", "eval --out must stay inside the Agent Capsule.");
+  }
+
+  if (!/\.(eval|spec)\.[cm]?[tj]s$/.test(file)) {
+    throw new AgentKitError(
+      "validation_error",
+      "eval --out must end with .eval.ts, .eval.js, .spec.ts, or another AgentKit eval/spec extension.",
+    );
   }
+
+  return file;
+}
+
+function formatEvalFailure(error: unknown): string {
+  if (isAgentKitError(error)) {
+    return `${error.code}: ${error.message}`;
+  }
+
+  if (error instanceof Error) {
+    return `runtime_error: ${error.message}`;
+  }
+
+  return `runtime_error: ${String(error)}`;
 }
 
 function jsonContains(actual: unknown, expected: unknown): boolean {

package/src/runtime/inspect.ts

@@ -6,6 +6,8 @@ import { loadAgentCapsule } from "./config";
 import { loadCapsuleEnv } from "./env";
 import { resolveKnowledgeConfig } from "./knowledge/config";
 import { KNOWLEDGE_SEARCH_TOOL_NAME } from "./knowledge/tool";
+import { resolveRuntimeTimeZone } from "./prompt-context";
+import { resolveTranscriptionConfig } from "./transcription";
 
 export type SecretState = "set" | "missing";
 
@@ -13,9 +15,26 @@ export type AgentInspectState = {
   agent: string;
   runtime: AgentRuntime;
   provider: AgentProvider;
+  timeZone: {
+    configured: string | null;
+    effective: string;
+  };
   prompt: string;
   tools: string[];
   channels: AgentInspectChannel[];
+  transcription: {
+    enabled: boolean;
+    provider: string | null;
+    model: string | null;
+    secret: string | null;
+    language: string | null;
+    prompt: string | null;
+    limits: {
+      maxDurationSeconds: number | null;
+      maxBytes: number | null;
+    };
+    rawAudioTtlSeconds: number | null;
+  };
   knowledge: {
     enabled: boolean;
     sources: string[];
@@ -75,6 +94,7 @@ export type AgentInspectChannel = {
   provider: AgentChannel["provider"];
   secrets: string[];
   buffer?: AgentChannel["buffer"];
+  audio?: AgentChannel["audio"];
 };
 
 export async function inspectAgentCapsule(
@@ -91,6 +111,7 @@ export function buildInspectState(
 ): AgentInspectState {
   const declaredSecrets = new Set(capsule.config.secrets);
   const knowledge = resolveKnowledgeConfig(capsule.config);
+  const transcription = resolveTranscriptionConfig(capsule.config.transcription);
 
   for (const tool of capsule.config.tools ?? []) {
     for (const secret of tool.secrets ?? []) {
@@ -108,6 +129,10 @@ export function buildInspectState(
     declaredSecrets.add(knowledge.embedding.secret);
   }
 
+  if (transcription.secret) {
+    declaredSecrets.add(transcription.secret);
+  }
+
   const managedSecrets = managedCloudSecrets(capsule);
   const userSecrets = Array.from(declaredSecrets).sort();
   const secrets: Record<string, SecretState> = {};
@@ -122,6 +147,13 @@ export function buildInspectState(
     agent: capsule.config.name,
     runtime: capsule.config.runtime,
     provider: capsule.config.provider,
+    timeZone: {
+      configured: capsule.config.timeZone ?? null,
+      effective: resolveRuntimeTimeZone({
+        timeZone: capsule.config.timeZone,
+        env,
+      }),
+    },
     prompt: toCapsulePath(capsule.root, capsule.instructionsPath),
     tools: (capsule.config.tools ?? []).map((tool) => tool.name),
     channels: (capsule.config.channels ?? []).map((channel) => ({
@@ -130,7 +162,21 @@ export function buildInspectState(
       provider: channel.provider,
       secrets: [...channel.secrets],
       ...(channel.buffer ? { buffer: channel.buffer } : {}),
+      ...(channel.audio ? { audio: channel.audio } : {}),
     })),
+    transcription: {
+      enabled: transcription.enabled,
+      provider: transcription.enabled ? transcription.provider : null,
+      model: transcription.enabled ? transcription.model : null,
+      secret: transcription.secret,
+      language: transcription.language,
+      prompt: transcription.prompt,
+      limits: {
+        maxDurationSeconds: transcription.limits.maxDurationSeconds ?? null,
+        maxBytes: transcription.limits.maxBytes ?? null,
+      },
+      rawAudioTtlSeconds: transcription.rawAudioTtlSeconds,
+    },
     knowledge: {
       enabled: knowledge.enabled,
       sources: knowledge.sources.map((source) => source.value),