@andreprado/agentkit 0.1.0-alpha.5 → 0.1.0-alpha.7

package/src/runtime/evals.ts

@@ -1,6 +1,6 @@
-import { readdir } from "node:fs/promises";
+import { mkdir, readdir, stat, writeFile } from "node:fs/promises";
 import type { Dirent } from "node:fs";
-import { join, relative } from "node:path";
+import { dirname, join, relative, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
 
 import type { AgentRunResult } from "./chat";
@@ -8,6 +8,7 @@ import { runAgentMessageFromCwd } from "./chat";
 import { findAgentCapsuleRoot, loadAgentCapsule } from "./config";
 import { AgentKitError } from "./errors";
 import { openCapsuleStore, type StoredToolCall } from "../storage/sqlite";
+import { getConversationTraceFromCwd } from "./traces";
 
 export type EvalRunSummary = {
   root: string;
@@ -27,9 +28,17 @@ export type EvalResult = {
 type EvalCase = {
   name?: string;
   input?: string;
+  turns?: EvalTurn[];
   expect?: EvalExpect;
 };
 
+type EvalTurn =
+  | string
+  | {
+      input: string;
+      expect?: EvalExpect;
+    };
+
 type EvalExpect = {
   contains?: string | string[];
   not_contains?: string | string[];
@@ -55,16 +64,25 @@ type ToolExpectation =
       output?: unknown;
       status?: "running" | "completed" | "failed";
       visibility?: "user" | "internal";
+      rendered?: unknown;
     };
 
 type ToolCallSnapshot = {
   name?: unknown;
   input?: unknown;
   output?: unknown;
+  rendered?: unknown;
   status?: unknown;
   visibility?: unknown;
 };
 
+export type EvalFromConversationResult = {
+  root: string;
+  file: string;
+  conversationId: string;
+  turns: number;
+};
+
 export async function runEvalsFromCwd(cwd: string): Promise<EvalRunSummary> {
   const root = await findAgentCapsuleRoot(cwd);
   const evalFiles = await findEvalFiles(join(root, "evals"));
@@ -77,28 +95,14 @@ export async function runEvalsFromCwd(cwd: string): Promise<EvalRunSummary> {
 
   for (const file of evalFiles) {
     const evalCase = await loadEvalCase(file);
-    const input = evalCase.input;
-
-    if (typeof input !== "string" || input.trim().length === 0) {
-      throw new AgentKitError("validation_error", `${relative(root, file)} must export an eval with a non-empty input string.`);
-    }
-
-    const run = await runAgentMessageFromCwd(root, {
-      message: input,
-      runtime: {
-        environment: "eval",
-        invocation: "eval",
-      },
-    });
-    const persistedToolCalls = await loadPersistedToolCalls(root, run.runId);
-    const failures = evaluateExpectations(evalCase.expect ?? {}, run, persistedToolCalls);
+    const run = await runEvalCase(root, relative(root, file), evalCase);
 
     results.push({
       name: evalCase.name ?? relative(root, file),
       file: relative(root, file),
-      passed: failures.length === 0,
-      failures,
-      output: run.message.content,
+      passed: run.failures.length === 0,
+      failures: run.failures,
+      output: run.output,
     });
   }
 
@@ -112,6 +116,133 @@ export async function runEvalsFromCwd(cwd: string): Promise<EvalRunSummary> {
   };
 }
 
+export async function writeEvalFromConversation(
+  cwd: string,
+  input: {
+    conversationId: string;
+    out?: string;
+    name?: string;
+    force?: boolean;
+  },
+): Promise<EvalFromConversationResult> {
+  const root = await findAgentCapsuleRoot(cwd);
+  const trace = await getConversationTraceFromCwd(root, input.conversationId);
+  const turns = replayTurnsFromMessages(trace.messages);
+
+  if (turns.length === 0) {
+    throw new AgentKitError(
+      "validation_error",
+      `Conversation "${input.conversationId}" does not contain user/assistant turns that can become an eval.`,
+    );
+  }
+
+  const file = resolve(root, input.out ?? join("evals", `replay-${slugify(trace.title ?? trace.id)}.eval.ts`));
+
+  if (!input.force && await pathExists(file)) {
+    throw new AgentKitError(
+      "validation_error",
+      `${relative(root, file)} already exists. Re-run with --force or choose --out <path>.`,
+    );
+  }
+
+  await mkdir(dirname(file), { recursive: true });
+  await writeFile(
+    file,
+    `export default {
+  name: ${JSON.stringify(input.name ?? `replay ${trace.title ?? trace.id}`)},
+  turns: ${formatEvalTurns(turns)},
+};
+`,
+  );
+
+  return {
+    root,
+    file,
+    conversationId: trace.id,
+    turns: turns.length,
+  };
+}
+
+async function runEvalCase(
+  root: string,
+  file: string,
+  evalCase: EvalCase,
+): Promise<{ failures: string[]; output: string }> {
+  const turns = normalizeEvalTurns(evalCase);
+
+  if (turns.length === 0) {
+    throw new AgentKitError(
+      "validation_error",
+      `${file} must export either a non-empty input string or a non-empty turns array.`,
+    );
+  }
+
+  const conversationId = `eval_${crypto.randomUUID()}`;
+  const failures: string[] = [];
+  let output = "";
+
+  for (const [index, turn] of turns.entries()) {
+    const run = await runAgentMessageFromCwd(root, {
+      message: turn.input,
+      conversationId,
+      runtime: {
+        environment: "eval",
+        invocation: "eval",
+      },
+    });
+    const persistedToolCalls = await loadPersistedToolCalls(root, run.runId);
+    const turnFailures = evaluateExpectations(turn.expect ?? {}, run, persistedToolCalls);
+
+    for (const failure of turnFailures) {
+      failures.push(turns.length === 1 ? failure : `turn ${index + 1}: ${failure}`);
+    }
+
+    output = run.message.content;
+  }
+
+  return { failures, output };
+}
+
+function normalizeEvalTurns(evalCase: EvalCase): Array<{ input: string; expect?: EvalExpect }> {
+  if (evalCase.turns !== undefined) {
+    if (!Array.isArray(evalCase.turns)) {
+      throw new AgentKitError("validation_error", "eval turns must be an array when provided.");
+    }
+
+    return evalCase.turns.map((turn, index) => normalizeEvalTurn(turn, index));
+  }
+
+  if (typeof evalCase.input === "string" && evalCase.input.trim().length > 0) {
+    return [
+      {
+        input: evalCase.input,
+        expect: evalCase.expect,
+      },
+    ];
+  }
+
+  return [];
+}
+
+function normalizeEvalTurn(turn: EvalTurn, index: number): { input: string; expect?: EvalExpect } {
+  if (typeof turn === "string") {
+    if (turn.trim().length === 0) {
+      throw new AgentKitError("validation_error", `eval turns[${index}] must be non-empty.`);
+    }
+
+    return { input: turn };
+  }
+
+  if (!isRecord(turn) || typeof turn.input !== "string" || turn.input.trim().length === 0) {
+    throw new AgentKitError("validation_error", `eval turns[${index}].input must be a non-empty string.`);
+  }
+
+  return {
+    input: turn.input,
+    ...(turn.expect ? { expect: turn.expect } : {}),
+  };
+}
+
 async function findEvalFiles(directory: string): Promise<string[]> {
   let entries: Dirent[];
 
@@ -253,6 +384,10 @@ function matchesToolExpectation(toolCall: ToolCallSnapshot, expectation: ToolExp
     return false;
   }
 
+  if ("rendered" in expectation && !jsonContains(toolCall.rendered, expectation.rendered)) {
+    return false;
+  }
+
   return true;
 }
 
@@ -269,11 +404,66 @@ function toolCallSnapshotFromStored(toolCall: StoredToolCall): ToolCallSnapshot
     name: toolCall.toolName,
     input: toolCall.input,
     output: toolCall.output,
+    rendered: toolCall.rendered,
     status: toolCall.status,
     visibility: toolCall.visibility,
   };
 }
 
+function replayTurnsFromMessages(
+  messages: Array<{ role: string; content: string }>,
+): Array<{ input: string; expect?: EvalExpect }> {
+  const turns: Array<{ input: string; expect?: EvalExpect }> = [];
+
+  for (let index = 0; index < messages.length; index += 1) {
+    const message = messages[index];
+
+    if (message.role !== "user") {
+      continue;
+    }
+
+    const nextAssistant = messages.slice(index + 1).find((candidate) => candidate.role === "assistant");
+    turns.push({
+      input: message.content,
+      ...(nextAssistant
+        ? {
+            expect: {
+              contains: nextAssistant.content,
+            },
+          }
+        : {}),
+    });
+  }
+
+  return turns;
+}
+
+function formatEvalTurns(turns: Array<{ input: string; expect?: EvalExpect }>): string {
+  return JSON.stringify(turns, null, 4)
+    .split("\n")
+    .map((line, index) => (index === 0 ? line : `  ${line}`))
+    .join("\n");
+}
+
+function slugify(value: string): string {
+  const slug = value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 48);
+
+  return slug || "conversation";
+}
+
+async function pathExists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 function jsonContains(actual: unknown, expected: unknown): boolean {
   if (Array.isArray(expected)) {
     return jsonEqual(actual, expected);

package/src/runtime/inspect.ts

@@ -4,6 +4,8 @@ import type { AgentChannel, AgentProvider, AgentRuntime, AccessMode } from "../i
 import type { LoadedAgentCapsule } from "./config";
 import { loadAgentCapsule } from "./config";
 import { loadCapsuleEnv } from "./env";
+import { resolveKnowledgeConfig } from "./knowledge/config";
+import { KNOWLEDGE_SEARCH_TOOL_NAME } from "./knowledge/tool";
 
 export type SecretState = "set" | "missing";
 
@@ -14,6 +16,20 @@ export type AgentInspectState = {
   prompt: string;
   tools: string[];
   channels: AgentInspectChannel[];
+  knowledge: {
+    enabled: boolean;
+    sources: string[];
+    embedding: {
+      provider: string;
+      model: string | null;
+      dimensions: number | null;
+    };
+    retrieval: {
+      topK: number;
+      hybrid: boolean;
+    };
+    internalTool: string | null;
+  };
   access: {
     mode: AccessMode;
   };
@@ -43,11 +59,13 @@ export type AgentInspectDatabase = {
     driver: "sqlite";
     path: string | null;
     schema: string | null;
+    migrations: string | null;
   };
   hosted: {
     driver: "none" | "turso";
     provisioning: "none" | "agentkit-managed";
     schema: string | null;
+    migrations: string | null;
   };
 };
 
@@ -56,6 +74,7 @@ export type AgentInspectChannel = {
   type: AgentChannel["type"];
   provider: AgentChannel["provider"];
   secrets: string[];
+  buffer?: AgentChannel["buffer"];
 };
 
 export async function inspectAgentCapsule(
@@ -71,6 +90,7 @@ export function buildInspectState(
   env: Record<string, string | undefined> = process.env,
 ): AgentInspectState {
   const declaredSecrets = new Set(capsule.config.secrets);
+  const knowledge = resolveKnowledgeConfig(capsule.config);
 
   for (const tool of capsule.config.tools ?? []) {
     for (const secret of tool.secrets ?? []) {
@@ -84,6 +104,10 @@ export function buildInspectState(
     }
   }
 
+  if (knowledge.embedding.secret) {
+    declaredSecrets.add(knowledge.embedding.secret);
+  }
+
   const managedSecrets = managedCloudSecrets(capsule);
   const userSecrets = Array.from(declaredSecrets).sort();
   const secrets: Record<string, SecretState> = {};
@@ -105,7 +129,19 @@ export function buildInspectState(
       type: channel.type,
       provider: channel.provider,
       secrets: [...channel.secrets],
+      ...(channel.buffer ? { buffer: channel.buffer } : {}),
     })),
+    knowledge: {
+      enabled: knowledge.enabled,
+      sources: knowledge.sources.map((source) => source.value),
+      embedding: {
+        provider: knowledge.embedding.provider,
+        model: knowledge.embedding.model,
+        dimensions: knowledge.embedding.dimensions,
+      },
+      retrieval: knowledge.retrieval,
+      internalTool: knowledge.enabled ? KNOWLEDGE_SEARCH_TOOL_NAME : null,
+    },
     access: capsule.config.access,
     storage: capsule.storagePath ? toCapsulePath(capsule.root, capsule.storagePath) : null,
     database,
@@ -137,17 +173,20 @@ function toCapsulePath(root: string, path: string): string {
 function inspectDatabase(capsule: LoadedAgentCapsule): AgentInspectDatabase {
   const database = capsule.config.storage.database;
   const schema = database?.driver === "turso" && database.schema ? database.schema : null;
+  const migrations = database?.driver === "turso" ? database.migrations ?? "migrations" : null;
 
   return {
     local: {
       driver: "sqlite",
       path: capsule.storagePath ? toCapsulePath(capsule.root, capsule.storagePath) : null,
       schema,
+      migrations,
     },
     hosted: {
       driver: database?.driver === "turso" ? "turso" : "none",
       provisioning: database?.driver === "turso" ? "agentkit-managed" : "none",
       schema,
+      migrations,
     },
   };
 }