@andreprado/agentkit 0.1.0-alpha.6 → 0.1.0-alpha.8

package/src/runtime/database.ts

@@ -1,5 +1,6 @@
-import { readFile } from "node:fs/promises";
-import { relative, resolve } from "node:path";
+import { readdir, readFile } from "node:fs/promises";
+import type { Dirent } from "node:fs";
+import { basename, relative, resolve } from "node:path";
 
 import type { DatabaseArgs, DatabaseResult, DatabaseRunner, DatabaseStatement } from "../index";
 import { openCapsuleStore, type SqliteAgentKitStore } from "../storage/sqlite";
@@ -9,6 +10,12 @@ import { AgentKitError } from "./errors";
 export type LocalDatabasePrepareResult = {
   store: SqliteAgentKitStore;
   databasePath: string;
+  migrations: {
+    configured: boolean;
+    path: string | null;
+    applied: string[];
+    skipped: string[];
+  };
   schema: {
     configured: boolean;
     applied: boolean;
@@ -20,10 +27,12 @@ export async function openPreparedCapsuleStore(capsule: LoadedAgentCapsule): Pro
   const store = await openCapsuleStore(capsule);
 
   try {
+    const migrations = await applyCapsuleDatabaseMigrations(capsule, store);
     const schema = await applyCapsuleDatabaseSchema(capsule, store);
     return {
       store,
       databasePath: store.dbPath,
+      migrations,
       schema,
     };
   } catch (error) {
@@ -32,6 +41,84 @@ export async function openPreparedCapsuleStore(capsule: LoadedAgentCapsule): Pro
   }
 }
 
+export async function applyCapsuleDatabaseMigrations(
+  capsule: LoadedAgentCapsule,
+  store: SqliteAgentKitStore,
+): Promise<LocalDatabasePrepareResult["migrations"]> {
+  const database = capsule.config.storage.database;
+
+  if (!database || database.driver !== "turso") {
+    return {
+      configured: false,
+      path: null,
+      applied: [],
+      skipped: [],
+    };
+  }
+
+  const migrationsPath = resolve(capsule.root, database.migrations ?? "migrations");
+  const migrations = await readMigrationFiles(migrationsPath);
+
+  if (migrations.length === 0) {
+    return {
+      configured: false,
+      path: migrationsPath,
+      applied: [],
+      skipped: [],
+    };
+  }
+
+  try {
+    const result = store.applyApplicationMigrations(migrations);
+    return {
+      configured: true,
+      path: migrationsPath,
+      applied: result.applied,
+      skipped: result.skipped,
+    };
+  } catch (error) {
+    if (error instanceof AgentKitError && error.code === "database_schema_error") {
+      throw new AgentKitError(
+        "database_schema_error",
+        [
+          `Could not apply database migrations from ${relative(capsule.root, migrationsPath)}.`,
+          "Migrations must be ordered .sql files and should use safe, forward-only changes.",
+          error.message,
+        ].join(" "),
+        { cause: error },
+      );
+    }
+
+    throw error;
+  }
+}
+
+async function readMigrationFiles(directory: string): Promise<Array<{ id: string; sql: string }>> {
+  let entries: Dirent[];
+
+  try {
+    entries = await readdir(directory, { withFileTypes: true });
+  } catch (error) {
+    if (isNodeError(error) && error.code === "ENOENT") {
+      return [];
+    }
+
+    throw error;
+  }
+
+  const files = entries
+    .filter((entry) => entry.isFile() && entry.name.endsWith(".sql"))
+    .map((entry) => entry.name)
+    .sort();
+
+  return Promise.all(
+    files.map(async (file) => ({
+      id: basename(file, ".sql"),
+      sql: await readFile(resolve(directory, file), "utf8"),
+    })),
+  );
+}
+
 export async function applyCapsuleDatabaseSchema(
   capsule: LoadedAgentCapsule,
   store: SqliteAgentKitStore,
@@ -163,3 +250,7 @@ function validateStatements(statements: DatabaseStatement[]): void {
     validateSql(statement.sql, statement.args ?? []);
   }
 }
+
+function isNodeError(error: unknown): error is NodeJS.ErrnoException {
+  return typeof error === "object" && error !== null && "code" in error;
+}

package/src/runtime/db-commands.ts

@@ -10,6 +10,9 @@ import { AgentKitError } from "./errors";
 export type DbMigrateResult = {
   root: string;
   databasePath: string;
+  migrationsPath: string | null;
+  migrationsApplied: string[];
+  migrationsSkipped: string[];
   schemaPath: string | null;
   schemaApplied: boolean;
 };
@@ -43,6 +46,9 @@ export async function migrateLocalDatabase(cwd = process.cwd()): Promise<DbMigra
     return {
       root: capsule.root,
       databasePath: prepared.databasePath,
+      migrationsPath: prepared.migrations.path,
+      migrationsApplied: prepared.migrations.applied,
+      migrationsSkipped: prepared.migrations.skipped,
       schemaPath: prepared.schema.path,
       schemaApplied: prepared.schema.applied,
     };
@@ -87,6 +93,9 @@ export async function seedLocalDatabase(cwd = process.cwd(), seedFile = "seed.sq
     return {
       root: capsule.root,
       databasePath: prepared.databasePath,
+      migrationsPath: prepared.migrations.path,
+      migrationsApplied: prepared.migrations.applied,
+      migrationsSkipped: prepared.migrations.skipped,
       schemaPath: prepared.schema.path,
       schemaApplied: prepared.schema.applied,
       seedPath,

package/src/runtime/dev-server.ts

@@ -147,18 +147,20 @@ function createDevHttpServer(capsule: LoadedAgentCapsule): Server {
     dedupeKeys: new Set<string>(),
     buffers: new Map<string, DevChannelBuffer>(),
   };
+  const capsuleRoot = capsule.root;
   return createHttpServer((incoming, outgoing) => {
-    void handleNodeRequest(capsule, incoming, outgoing, channelRuntime);
+    void handleNodeRequest(capsuleRoot, incoming, outgoing, channelRuntime);
   });
 }
 
 async function handleNodeRequest(
-  capsule: LoadedAgentCapsule,
+  capsuleRoot: string,
   incoming: IncomingMessage,
   outgoing: ServerResponse,
   channelRuntime: DevChannelRuntimeState,
 ): Promise<void> {
   try {
+    const capsule = await loadAgentCapsule(capsuleRoot);
     const request = nodeRequestToFetchRequest(incoming);
     const response = await handleDevServerRequest(capsule, request, channelRuntime);
     await writeFetchResponse(outgoing, response);
@@ -296,6 +298,12 @@ async function handleDevServerRequest(
       return jsonResponse(await getConversation(capsule, decodeURIComponent(conversationMatch[1])));
     }
 
+    const conversationTraceMatch = route.match(/^\/v1\/conversations\/([^/]+)\/trace$/);
+
+    if (request.method === "GET" && conversationTraceMatch) {
+      return jsonResponse(await getConversationTrace(capsule, decodeURIComponent(conversationTraceMatch[1])));
+    }
+
     return jsonError(new AgentKitError("not_found", `Route ${request.method} ${route} was not found.`), 404);
   } catch (error) {
     return jsonError(error, statusForError(error));
@@ -570,6 +578,32 @@ async function getConversation(capsule: LoadedAgentCapsule, conversationId: stri
   }
 }
 
+async function getConversationTrace(capsule: LoadedAgentCapsule, conversationId: string) {
+  if (!conversationId) {
+    throw new AgentKitError("validation_error", "Missing conversation id.");
+  }
+
+  const store = await openCapsuleStore(capsule);
+
+  try {
+    const conversation = store.getConversation(conversationId);
+
+    if (!conversation) {
+      throw new AgentKitError("conversation_not_found", `Conversation "${conversationId}" was not found.`);
+    }
+
+    return {
+      ...conversation,
+      runs: store.listRunsForConversation(conversation.id).map((run) => ({
+        ...run,
+        toolCalls: store.listToolCallsForRun(run.id),
+      })),
+    };
+  } finally {
+    store.close();
+  }
+}
+
 async function readJsonBody(request: Request): Promise<unknown> {
   const contentType = request.headers.get("content-type") ?? "";
 
@@ -1066,6 +1100,13 @@ function renderChatApp(capsule: LoadedAgentCapsule, baseUrl: string, env: Record
         messages.scrollTop = messages.scrollHeight;
       }
 
+      textarea.addEventListener("keydown", (event) => {
+        if (event.key !== "Enter" || event.shiftKey || event.isComposing) return;
+        event.preventDefault();
+        if (button.disabled) return;
+        form.requestSubmit();
+      });
+
       form.addEventListener("submit", async (event) => {
         event.preventDefault();
         const content = textarea.value.trim();

package/src/runtime/evals.ts

@@ -1,6 +1,6 @@
-import { readdir } from "node:fs/promises";
+import { mkdir, readdir, stat, writeFile } from "node:fs/promises";
 import type { Dirent } from "node:fs";
-import { join, relative } from "node:path";
+import { dirname, join, relative, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
 
 import type { AgentRunResult } from "./chat";
@@ -8,6 +8,7 @@ import { runAgentMessageFromCwd } from "./chat";
 import { findAgentCapsuleRoot, loadAgentCapsule } from "./config";
 import { AgentKitError } from "./errors";
 import { openCapsuleStore, type StoredToolCall } from "../storage/sqlite";
+import { getConversationTraceFromCwd } from "./traces";
 
 export type EvalRunSummary = {
   root: string;
@@ -27,9 +28,17 @@ export type EvalResult = {
 type EvalCase = {
   name?: string;
   input?: string;
+  turns?: EvalTurn[];
   expect?: EvalExpect;
 };
 
+type EvalTurn =
+  | string
+  | {
+      input: string;
+      expect?: EvalExpect;
+    };
+
 type EvalExpect = {
   contains?: string | string[];
   not_contains?: string | string[];
@@ -55,16 +64,25 @@ type ToolExpectation =
       output?: unknown;
       status?: "running" | "completed" | "failed";
       visibility?: "user" | "internal";
+      rendered?: unknown;
     };
 
 type ToolCallSnapshot = {
   name?: unknown;
   input?: unknown;
   output?: unknown;
+  rendered?: unknown;
   status?: unknown;
   visibility?: unknown;
 };
 
+export type EvalFromConversationResult = {
+  root: string;
+  file: string;
+  conversationId: string;
+  turns: number;
+};
+
 export async function runEvalsFromCwd(cwd: string): Promise<EvalRunSummary> {
   const root = await findAgentCapsuleRoot(cwd);
   const evalFiles = await findEvalFiles(join(root, "evals"));
@@ -77,28 +95,14 @@ export async function runEvalsFromCwd(cwd: string): Promise<EvalRunSummary> {
 
   for (const file of evalFiles) {
     const evalCase = await loadEvalCase(file);
-    const input = evalCase.input;
-
-    if (typeof input !== "string" || input.trim().length === 0) {
-      throw new AgentKitError("validation_error", `${relative(root, file)} must export an eval with a non-empty input string.`);
-    }
-
-    const run = await runAgentMessageFromCwd(root, {
-      message: input,
-      runtime: {
-        environment: "eval",
-        invocation: "eval",
-      },
-    });
-    const persistedToolCalls = await loadPersistedToolCalls(root, run.runId);
-    const failures = evaluateExpectations(evalCase.expect ?? {}, run, persistedToolCalls);
+    const run = await runEvalCase(root, relative(root, file), evalCase);
 
     results.push({
       name: evalCase.name ?? relative(root, file),
       file: relative(root, file),
-      passed: failures.length === 0,
-      failures,
-      output: run.message.content,
+      passed: run.failures.length === 0,
+      failures: run.failures,
+      output: run.output,
     });
   }
 
@@ -112,6 +116,133 @@ export async function runEvalsFromCwd(cwd: string): Promise<EvalRunSummary> {
   };
 }
 
+export async function writeEvalFromConversation(
+  cwd: string,
+  input: {
+    conversationId: string;
+    out?: string;
+    name?: string;
+    force?: boolean;
+  },
+): Promise<EvalFromConversationResult> {
+  const root = await findAgentCapsuleRoot(cwd);
+  const trace = await getConversationTraceFromCwd(root, input.conversationId);
+  const turns = replayTurnsFromMessages(trace.messages);
+
+  if (turns.length === 0) {
+    throw new AgentKitError(
+      "validation_error",
+      `Conversation "${input.conversationId}" does not contain user/assistant turns that can become an eval.`,
+    );
+  }
+
+  const file = resolve(root, input.out ?? join("evals", `replay-${slugify(trace.title ?? trace.id)}.eval.ts`));
+
+  if (!input.force && await pathExists(file)) {
+    throw new AgentKitError(
+      "validation_error",
+      `${relative(root, file)} already exists. Re-run with --force or choose --out <path>.`,
+    );
+  }
+
+  await mkdir(dirname(file), { recursive: true });
+  await writeFile(
+    file,
+    `export default {
+  name: ${JSON.stringify(input.name ?? `replay ${trace.title ?? trace.id}`)},
+  turns: ${formatEvalTurns(turns)},
+};
+`,
+  );
+
+  return {
+    root,
+    file,
+    conversationId: trace.id,
+    turns: turns.length,
+  };
+}
+
+async function runEvalCase(
+  root: string,
+  file: string,
+  evalCase: EvalCase,
+): Promise<{ failures: string[]; output: string }> {
+  const turns = normalizeEvalTurns(evalCase);
+
+  if (turns.length === 0) {
+    throw new AgentKitError(
+      "validation_error",
+      `${file} must export either a non-empty input string or a non-empty turns array.`,
+    );
+  }
+
+  const conversationId = `eval_${crypto.randomUUID()}`;
+  const failures: string[] = [];
+  let output = "";
+
+  for (const [index, turn] of turns.entries()) {
+    const run = await runAgentMessageFromCwd(root, {
+      message: turn.input,
+      conversationId,
+      runtime: {
+        environment: "eval",
+        invocation: "eval",
+      },
+    });
+    const persistedToolCalls = await loadPersistedToolCalls(root, run.runId);
+    const turnFailures = evaluateExpectations(turn.expect ?? {}, run, persistedToolCalls);
+
+    for (const failure of turnFailures) {
+      failures.push(turns.length === 1 ? failure : `turn ${index + 1}: ${failure}`);
+    }
+
+    output = run.message.content;
+  }
+
+  return { failures, output };
+}
+
+function normalizeEvalTurns(evalCase: EvalCase): Array<{ input: string; expect?: EvalExpect }> {
+  if (evalCase.turns !== undefined) {
+    if (!Array.isArray(evalCase.turns)) {
+      throw new AgentKitError("validation_error", "eval turns must be an array when provided.");
+    }
+
+    return evalCase.turns.map((turn, index) => normalizeEvalTurn(turn, index));
+  }
+
+  if (typeof evalCase.input === "string" && evalCase.input.trim().length > 0) {
+    return [
+      {
+        input: evalCase.input,
+        expect: evalCase.expect,
+      },
+    ];
+  }
+
+  return [];
+}
+
+function normalizeEvalTurn(turn: EvalTurn, index: number): { input: string; expect?: EvalExpect } {
+  if (typeof turn === "string") {
+    if (turn.trim().length === 0) {
+      throw new AgentKitError("validation_error", `eval turns[${index}] must be non-empty.`);
+    }
+
+    return { input: turn };
+  }
+
+  if (!isRecord(turn) || typeof turn.input !== "string" || turn.input.trim().length === 0) {
+    throw new AgentKitError("validation_error", `eval turns[${index}].input must be a non-empty string.`);
+  }
+
+  return {
+    input: turn.input,
+    ...(turn.expect ? { expect: turn.expect } : {}),
+  };
+}
+
 async function findEvalFiles(directory: string): Promise<string[]> {
   let entries: Dirent[];
 
@@ -253,6 +384,10 @@ function matchesToolExpectation(toolCall: ToolCallSnapshot, expectation: ToolExp
     return false;
   }
 
+  if ("rendered" in expectation && !jsonContains(toolCall.rendered, expectation.rendered)) {
+    return false;
+  }
+
   return true;
 }
 
@@ -269,11 +404,66 @@ function toolCallSnapshotFromStored(toolCall: StoredToolCall): ToolCallSnapshot
     name: toolCall.toolName,
     input: toolCall.input,
     output: toolCall.output,
+    rendered: toolCall.rendered,
     status: toolCall.status,
     visibility: toolCall.visibility,
   };
 }
 
+function replayTurnsFromMessages(
+  messages: Array<{ role: string; content: string }>,
+): Array<{ input: string; expect?: EvalExpect }> {
+  const turns: Array<{ input: string; expect?: EvalExpect }> = [];
+
+  for (let index = 0; index < messages.length; index += 1) {
+    const message = messages[index];
+
+    if (message.role !== "user") {
+      continue;
+    }
+
+    const nextAssistant = messages.slice(index + 1).find((candidate) => candidate.role === "assistant");
+    turns.push({
+      input: message.content,
+      ...(nextAssistant
+        ? {
+            expect: {
+              contains: nextAssistant.content,
+            },
+          }
+        : {}),
+    });
+  }
+
+  return turns;
+}
+
+function formatEvalTurns(turns: Array<{ input: string; expect?: EvalExpect }>): string {
+  return JSON.stringify(turns, null, 4)
+    .split("\n")
+    .map((line, index) => (index === 0 ? line : `  ${line}`))
+    .join("\n");
+}
+
+function slugify(value: string): string {
+  const slug = value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 48);
+
+  return slug || "conversation";
+}
+
+async function pathExists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 function jsonContains(actual: unknown, expected: unknown): boolean {
   if (Array.isArray(expected)) {
     return jsonEqual(actual, expected);

package/src/runtime/inspect.ts

@@ -59,11 +59,13 @@ export type AgentInspectDatabase = {
     driver: "sqlite";
     path: string | null;
     schema: string | null;
+    migrations: string | null;
   };
   hosted: {
     driver: "none" | "turso";
     provisioning: "none" | "agentkit-managed";
     schema: string | null;
+    migrations: string | null;
   };
 };
 
@@ -171,17 +173,20 @@ function toCapsulePath(root: string, path: string): string {
 function inspectDatabase(capsule: LoadedAgentCapsule): AgentInspectDatabase {
   const database = capsule.config.storage.database;
   const schema = database?.driver === "turso" && database.schema ? database.schema : null;
+  const migrations = database?.driver === "turso" ? database.migrations ?? "migrations" : null;
 
   return {
     local: {
       driver: "sqlite",
       path: capsule.storagePath ? toCapsulePath(capsule.root, capsule.storagePath) : null,
       schema,
+      migrations,
     },
     hosted: {
       driver: database?.driver === "turso" ? "turso" : "none",
       provisioning: database?.driver === "turso" ? "agentkit-managed" : "none",
       schema,
+      migrations,
     },
   };
 }