@andreprado/agentkit 0.1.0-alpha.10

package/src/runtime/docs.ts

@@ -0,0 +1,36 @@
+import { existsSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { AgentKitError } from "./errors";
+
+export type AgentKitDocsInfo = {
+  root: string;
+  llms: string;
+  llmsFull: string;
+};
+
+export function findAgentKitDocs(): AgentKitDocsInfo {
+  const candidates = [
+    resolve(dirname(fileURLToPath(import.meta.url)), "../../docs"),
+    resolve(dirname(fileURLToPath(import.meta.url)), "../../../../docs"),
+  ];
+
+  for (const root of candidates) {
+    const llms = join(root, "llms.txt");
+    const llmsFull = join(root, "llms-full.txt");
+
+    if (existsSync(llms) && existsSync(llmsFull)) {
+      return {
+        root,
+        llms,
+        llmsFull,
+      };
+    }
+  }
+
+  throw new AgentKitError(
+    "docs_not_found",
+    "Could not find packaged AgentKit docs. Reinstall @andreprado/agentkit or read the repository docs directly.",
+  );
+}

package/src/runtime/env.ts

@@ -0,0 +1,152 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { AgentKitError } from "./errors";
+
+export async function loadCapsuleEnv(
+  root: string,
+  baseEnv: Record<string, string | undefined> = process.env,
+): Promise<Record<string, string | undefined>> {
+  const parsed = await readDotEnv(join(root, ".env"));
+  return {
+    ...parsed,
+    ...baseEnv,
+  };
+}
+
+async function readDotEnv(path: string): Promise<Record<string, string>> {
+  let source: string;
+
+  try {
+    source = await readFile(path, "utf8");
+  } catch {
+    return {};
+  }
+
+  const env: Record<string, string> = {};
+
+  for (const rawLine of source.split(/\r?\n/)) {
+    const line = rawLine.trim();
+
+    if (!line || line.startsWith("#")) {
+      continue;
+    }
+
+    const equalsIndex = line.indexOf("=");
+
+    if (equalsIndex <= 0) {
+      continue;
+    }
+
+    const name = line.slice(0, equalsIndex).trim();
+    const rawValue = line.slice(equalsIndex + 1).trim();
+
+    if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) {
+      continue;
+    }
+
+    env[name] = unquote(rawValue);
+  }
+
+  return env;
+}
+
+export async function listCapsuleEnv(root: string): Promise<string[]> {
+  return Object.keys(await readDotEnv(join(root, ".env"))).sort();
+}
+
+export async function setCapsuleEnv(root: string, name: string, value: string): Promise<void> {
+  validateEnvName(name);
+
+  const path = join(root, ".env");
+  const source = await readDotEnvSource(path);
+  const line = `${name}=${quoteDotEnvValue(value)}`;
+  const lines = source.length > 0 ? source.split(/\r?\n/) : [];
+  let updated = false;
+
+  const nextLines = lines.map((existingLine) => {
+    if (isEnvAssignmentFor(existingLine, name)) {
+      updated = true;
+      return line;
+    }
+
+    return existingLine;
+  });
+
+  if (!updated) {
+    if (nextLines.length > 0 && nextLines[nextLines.length - 1] !== "") {
+      nextLines.push(line);
+    } else if (nextLines.length > 0) {
+      nextLines.splice(nextLines.length - 1, 0, line);
+    } else {
+      nextLines.push(line);
+    }
+  }
+
+  await writeFile(path, `${nextLines.join("\n").replace(/\n*$/, "")}\n`);
+}
+
+export async function unsetCapsuleEnv(root: string, name: string): Promise<boolean> {
+  validateEnvName(name);
+
+  const path = join(root, ".env");
+  const source = await readDotEnvSource(path);
+
+  if (!source) {
+    return false;
+  }
+
+  const lines = source.split(/\r?\n/);
+  const nextLines = lines.filter((line) => !isEnvAssignmentFor(line, name));
+
+  if (nextLines.length === lines.length) {
+    return false;
+  }
+
+  await writeFile(path, `${nextLines.join("\n").replace(/\n*$/, "")}\n`);
+  return true;
+}
+
+async function readDotEnvSource(path: string): Promise<string> {
+  try {
+    return await readFile(path, "utf8");
+  } catch {
+    return "";
+  }
+}
+
+function validateEnvName(name: string): void {
+  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) {
+    throw new AgentKitError(
+      "validation_error",
+      "Environment variable names must start with a letter or underscore and contain only letters, numbers, and underscores.",
+    );
+  }
+}
+
+function isEnvAssignmentFor(line: string, name: string): boolean {
+  const match = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=/);
+  return match?.[1] === name;
+}
+
+function unquote(value: string): string {
+  if (value.length >= 2 && value.startsWith('"') && value.endsWith('"')) {
+    return value
+      .slice(1, -1)
+      .replace(/\\n/g, "\n")
+      .replace(/\\r/g, "\r")
+      .replace(/\\"/g, '"')
+      .replace(/\\\\/g, "\\");
+  }
+
+  if (value.length >= 2 && value.startsWith("'") && value.endsWith("'")) {
+    return value.slice(1, -1);
+  }
+
+  const commentIndex = value.search(/\s#/);
+  return (commentIndex === -1 ? value : value.slice(0, commentIndex)).trim();
+}
+
+function quoteDotEnvValue(value: string): string {
+  return `"${value.replace(/\\/g, "\\\\").replace(/\r/g, "\\r").replace(/\n/g, "\\n").replace(/"/g, '\\"')}"`;
+}

package/src/runtime/errors.ts

@@ -0,0 +1,13 @@
+export class AgentKitError extends Error {
+  readonly code: string;
+
+  constructor(code: string, message: string, options?: ErrorOptions) {
+    super(message, options);
+    this.name = "AgentKitError";
+    this.code = code;
+  }
+}
+
+export function isAgentKitError(error: unknown): error is AgentKitError {
+  return error instanceof AgentKitError;
+}

package/src/runtime/evals.ts

@@ -0,0 +1,509 @@
+import { mkdir, readdir, stat, writeFile } from "node:fs/promises";
+import type { Dirent } from "node:fs";
+import { dirname, join, relative, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+
+import type { AgentRunResult } from "./chat";
+import { runAgentMessageFromCwd } from "./chat";
+import { findAgentCapsuleRoot, loadAgentCapsule } from "./config";
+import { AgentKitError } from "./errors";
+import { openCapsuleStore, type StoredToolCall } from "../storage/sqlite";
+import { getConversationTraceFromCwd } from "./traces";
+
+export type EvalRunSummary = {
+  root: string;
+  results: EvalResult[];
+  passed: number;
+  failed: number;
+};
+
+export type EvalResult = {
+  name: string;
+  file: string;
+  passed: boolean;
+  failures: string[];
+  output: string;
+};
+
+type EvalCase = {
+  name?: string;
+  input?: string;
+  turns?: EvalTurn[];
+  expect?: EvalExpect;
+};
+
+type EvalTurn =
+  | string
+  | {
+      input: string;
+      expect?: EvalExpect;
+    };
+
+type EvalExpect = {
+  contains?: string | string[];
+  not_contains?: string | string[];
+  notContains?: string | string[];
+  regex?: string | string[];
+  matches_regex?: string | string[];
+  matchesRegex?: string | string[];
+  tool_call?: ToolExpectation | ToolExpectation[];
+  toolCall?: ToolExpectation | ToolExpectation[];
+  tool_calls?: ToolExpectation | ToolExpectation[];
+  toolCalls?: ToolExpectation | ToolExpectation[];
+  persisted_tool_call?: ToolExpectation | ToolExpectation[];
+  persistedToolCall?: ToolExpectation | ToolExpectation[];
+  persisted_tool_calls?: ToolExpectation | ToolExpectation[];
+  persistedToolCalls?: ToolExpectation | ToolExpectation[];
+};
+
+type ToolExpectation =
+  | string
+  | {
+      name?: string;
+      input?: unknown;
+      output?: unknown;
+      status?: "running" | "completed" | "failed";
+      visibility?: "user" | "internal";
+      rendered?: unknown;
+    };
+
+type ToolCallSnapshot = {
+  name?: unknown;
+  input?: unknown;
+  output?: unknown;
+  rendered?: unknown;
+  status?: unknown;
+  visibility?: unknown;
+};
+
+export type EvalFromConversationResult = {
+  root: string;
+  file: string;
+  conversationId: string;
+  turns: number;
+};
+
+export async function runEvalsFromCwd(cwd: string): Promise<EvalRunSummary> {
+  const root = await findAgentCapsuleRoot(cwd);
+  const evalFiles = await findEvalFiles(join(root, "evals"));
+
+  if (evalFiles.length === 0) {
+    throw new AgentKitError("validation_error", "No eval files found in evals/. Add evals/<name>.eval.ts first.");
+  }
+
+  const results: EvalResult[] = [];
+
+  for (const file of evalFiles) {
+    const evalCase = await loadEvalCase(file);
+    const run = await runEvalCase(root, relative(root, file), evalCase);
+
+    results.push({
+      name: evalCase.name ?? relative(root, file),
+      file: relative(root, file),
+      passed: run.failures.length === 0,
+      failures: run.failures,
+      output: run.output,
+    });
+  }
+
+  const passed = results.filter((result) => result.passed).length;
+
+  return {
+    root,
+    results,
+    passed,
+    failed: results.length - passed,
+  };
+}
+
+export async function writeEvalFromConversation(
+  cwd: string,
+  input: {
+    conversationId: string;
+    out?: string;
+    name?: string;
+    force?: boolean;
+  },
+): Promise<EvalFromConversationResult> {
+  const root = await findAgentCapsuleRoot(cwd);
+  const trace = await getConversationTraceFromCwd(root, input.conversationId);
+  const turns = replayTurnsFromMessages(trace.messages);
+
+  if (turns.length === 0) {
+    throw new AgentKitError(
+      "validation_error",
+      `Conversation "${input.conversationId}" does not contain user/assistant turns that can become an eval.`,
+    );
+  }
+
+  const file = resolve(root, input.out ?? join("evals", `replay-${slugify(trace.title ?? trace.id)}.eval.ts`));
+
+  if (!input.force && await pathExists(file)) {
+    throw new AgentKitError(
+      "validation_error",
+      `${relative(root, file)} already exists. Re-run with --force or choose --out <path>.`,
+    );
+  }
+
+  await mkdir(dirname(file), { recursive: true });
+  await writeFile(
+    file,
+    `export default {
+  name: ${JSON.stringify(input.name ?? `replay ${trace.title ?? trace.id}`)},
+  turns: ${formatEvalTurns(turns)},
+};
+`,
+  );
+
+  return {
+    root,
+    file,
+    conversationId: trace.id,
+    turns: turns.length,
+  };
+}
+
+async function runEvalCase(
+  root: string,
+  file: string,
+  evalCase: EvalCase,
+): Promise<{ failures: string[]; output: string }> {
+  const turns = normalizeEvalTurns(evalCase);
+
+  if (turns.length === 0) {
+    throw new AgentKitError(
+      "validation_error",
+      `${file} must export either a non-empty input string or a non-empty turns array.`,
+    );
+  }
+
+  const conversationId = `eval_${crypto.randomUUID()}`;
+  const failures: string[] = [];
+  let output = "";
+
+  for (const [index, turn] of turns.entries()) {
+    const run = await runAgentMessageFromCwd(root, {
+      message: turn.input,
+      conversationId,
+      runtime: {
+        environment: "eval",
+        invocation: "eval",
+      },
+    });
+    const persistedToolCalls = await loadPersistedToolCalls(root, run.runId);
+    const turnFailures = evaluateExpectations(turn.expect ?? {}, run, persistedToolCalls);
+
+    for (const failure of turnFailures) {
+      failures.push(turns.length === 1 ? failure : `turn ${index + 1}: ${failure}`);
+    }
+
+    output = run.message.content;
+  }
+
+  return { failures, output };
+}
+
+function normalizeEvalTurns(evalCase: EvalCase): Array<{ input: string; expect?: EvalExpect }> {
+  if (evalCase.turns !== undefined) {
+    if (!Array.isArray(evalCase.turns)) {
+      throw new AgentKitError("validation_error", "eval turns must be an array when provided.");
+    }
+
+    return evalCase.turns.map((turn, index) => normalizeEvalTurn(turn, index));
+  }
+
+  if (typeof evalCase.input === "string" && evalCase.input.trim().length > 0) {
+    return [
+      {
+        input: evalCase.input,
+        expect: evalCase.expect,
+      },
+    ];
+  }
+
+  return [];
+}
+
+function normalizeEvalTurn(turn: EvalTurn, index: number): { input: string; expect?: EvalExpect } {
+  if (typeof turn === "string") {
+    if (turn.trim().length === 0) {
+      throw new AgentKitError("validation_error", `eval turns[${index}] must be non-empty.`);
+    }
+
+    return { input: turn };
+  }
+
+  if (!isRecord(turn) || typeof turn.input !== "string" || turn.input.trim().length === 0) {
+    throw new AgentKitError("validation_error", `eval turns[${index}].input must be a non-empty string.`);
+  }
+
+  return {
+    input: turn.input,
+    ...(turn.expect ? { expect: turn.expect } : {}),
+  };
+}
+
+async function findEvalFiles(directory: string): Promise<string[]> {
+  let entries: Dirent[];
+
+  try {
+    entries = await readdir(directory, { withFileTypes: true });
+  } catch (error) {
+    if (isNodeError(error) && error.code === "ENOENT") {
+      return [];
+    }
+
+    throw error;
+  }
+
+  const files = await Promise.all(
+    entries.map(async (entry) => {
+      const path = join(directory, entry.name);
+
+      if (entry.isDirectory()) {
+        return findEvalFiles(path);
+      }
+
+      if (entry.isFile() && /\.(eval|spec)\.[cm]?[tj]s$/.test(entry.name)) {
+        return [path];
+      }
+
+      return [];
+    }),
+  );
+
+  return files.flat().sort();
+}
+
+async function loadEvalCase(file: string): Promise<EvalCase> {
+  const url = pathToFileURL(file);
+  url.searchParams.set("t", String(Date.now()));
+
+  const module = (await import(url.href)) as { default?: unknown };
+
+  if (!isRecord(module.default)) {
+    throw new AgentKitError("validation_error", `${file} must default export an eval object.`);
+  }
+
+  return module.default;
+}
+
+async function loadPersistedToolCalls(root: string, runId: string): Promise<ToolCallSnapshot[]> {
+  const capsule = await loadAgentCapsule(root);
+  const store = await openCapsuleStore(capsule);
+
+  try {
+    return store.listToolCallsForRun(runId).map(toolCallSnapshotFromStored);
+  } finally {
+    store.close();
+  }
+}
+
+function evaluateExpectations(expect: EvalExpect, run: AgentRunResult, persistedToolCalls: ToolCallSnapshot[]): string[] {
+  const failures: string[] = [];
+  const content = run.message.content;
+
+  for (const expected of list(expect.contains)) {
+    if (!content.includes(expected)) {
+      failures.push(`expected output to contain ${JSON.stringify(expected)}`);
+    }
+  }
+
+  for (const forbidden of [...list(expect.not_contains), ...list(expect.notContains)]) {
+    if (content.includes(forbidden)) {
+      failures.push(`expected output not to contain ${JSON.stringify(forbidden)}`);
+    }
+  }
+
+  for (const pattern of [...list(expect.regex), ...list(expect.matches_regex), ...list(expect.matchesRegex)]) {
+    if (!new RegExp(pattern).test(content)) {
+      failures.push(`expected output to match /${pattern}/`);
+    }
+  }
+
+  const toolExpectations = [
+    ...toolList(expect.tool_call),
+    ...toolList(expect.toolCall),
+    ...toolList(expect.tool_calls),
+    ...toolList(expect.toolCalls),
+    ...toolList(expect.persisted_tool_call),
+    ...toolList(expect.persistedToolCall),
+    ...toolList(expect.persisted_tool_calls),
+    ...toolList(expect.persistedToolCalls),
+  ];
+
+  for (const toolExpectation of toolExpectations) {
+    const match = persistedToolCalls.find((toolCall) => matchesToolExpectation(toolCall, toolExpectation));
+
+    if (!match) {
+      failures.push(`expected persisted tool call ${formatToolExpectation(toolExpectation)}`);
+    }
+  }
+
+  return failures;
+}
+
+function list(value: string | string[] | undefined): string[] {
+  if (value === undefined) {
+    return [];
+  }
+
+  return Array.isArray(value) ? value : [value];
+}
+
+function toolList(value: ToolExpectation | ToolExpectation[] | undefined): ToolExpectation[] {
+  if (value === undefined) {
+    return [];
+  }
+
+  return Array.isArray(value) ? value : [value];
+}
+
+function matchesToolExpectation(toolCall: ToolCallSnapshot, expectation: ToolExpectation): boolean {
+  if (typeof expectation === "string") {
+    return toolCall.name === expectation;
+  }
+
+  if (expectation.name !== undefined && toolCall.name !== expectation.name) {
+    return false;
+  }
+
+  if ("status" in expectation && toolCall.status !== expectation.status) {
+    return false;
+  }
+
+  if ("visibility" in expectation && toolCall.visibility !== expectation.visibility) {
+    return false;
+  }
+
+  if ("input" in expectation && !jsonContains(toolCall.input, expectation.input)) {
+    return false;
+  }
+
+  if ("output" in expectation && !jsonContains(toolCall.output, expectation.output)) {
+    return false;
+  }
+
+  if ("rendered" in expectation && !jsonContains(toolCall.rendered, expectation.rendered)) {
+    return false;
+  }
+
+  return true;
+}
+
+function formatToolExpectation(expectation: ToolExpectation): string {
+  if (typeof expectation === "string") {
+    return JSON.stringify({ name: expectation });
+  }
+
+  return JSON.stringify(expectation);
+}
+
+function toolCallSnapshotFromStored(toolCall: StoredToolCall): ToolCallSnapshot {
+  return {
+    name: toolCall.toolName,
+    input: toolCall.input,
+    output: toolCall.output,
+    rendered: toolCall.rendered,
+    status: toolCall.status,
+    visibility: toolCall.visibility,
+  };
+}
+
+function replayTurnsFromMessages(
+  messages: Array<{ role: string; content: string }>,
+): Array<{ input: string; expect?: EvalExpect }> {
+  const turns: Array<{ input: string; expect?: EvalExpect }> = [];
+
+  for (let index = 0; index < messages.length; index += 1) {
+    const message = messages[index];
+
+    if (message.role !== "user") {
+      continue;
+    }
+
+    const nextAssistant = messages.slice(index + 1).find((candidate) => candidate.role === "assistant");
+    turns.push({
+      input: message.content,
+      ...(nextAssistant
+        ? {
+            expect: {
+              contains: nextAssistant.content,
+            },
+          }
+        : {}),
+    });
+  }
+
+  return turns;
+}
+
+function formatEvalTurns(turns: Array<{ input: string; expect?: EvalExpect }>): string {
+  return JSON.stringify(turns, null, 4)
+    .split("\n")
+    .map((line, index) => (index === 0 ? line : `  ${line}`))
+    .join("\n");
+}
+
+function slugify(value: string): string {
+  const slug = value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 48);
+
+  return slug || "conversation";
+}
+
+async function pathExists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function jsonContains(actual: unknown, expected: unknown): boolean {
+  if (Array.isArray(expected)) {
+    return jsonEqual(actual, expected);
+  }
+
+  if (isRecord(expected)) {
+    if (!isRecord(actual)) {
+      return false;
+    }
+
+    return Object.entries(expected).every(([key, value]) => key in actual && jsonContains(actual[key], value));
+  }
+
+  return jsonEqual(actual, expected);
+}
+
+function jsonEqual(left: unknown, right: unknown): boolean {
+  return JSON.stringify(sortJson(left)) === JSON.stringify(sortJson(right));
+}
+
+function sortJson(value: unknown): unknown {
+  if (Array.isArray(value)) {
+    return value.map(sortJson);
+  }
+
+  if (!isRecord(value)) {
+    return value;
+  }
+
+  return Object.fromEntries(
+    Object.entries(value)
+      .sort(([left], [right]) => left.localeCompare(right))
+      .map(([key, item]) => [key, sortJson(item)]),
+  );
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isNodeError(error: unknown): error is NodeJS.ErrnoException {
+  return error instanceof Error && "code" in error;
+}