@andreprado/agentkit 0.1.0-alpha.17 → 0.1.0-alpha.19

package/src/runtime/improve.ts

@@ -0,0 +1,868 @@
+import { createHash } from "node:crypto";
+import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import { dirname, isAbsolute, join, relative, resolve } from "node:path";
+
+import { listConversationsFromCwd } from "./conversations";
+import { findAgentCapsuleRoot } from "./config";
+import { AgentKitError, isAgentKitError } from "./errors";
+import { runAgentMessageFromCwd } from "./chat";
+import { getConversationTraceFromCwd } from "./traces";
+
+export type ImproveSource = "local" | "deploy";
+
+export type ImproveTraceMessage = {
+  role: "user" | "assistant" | "system" | "tool";
+  content: string;
+  created_at?: string;
+  sequence?: number;
+};
+
+export type ImproveTracePacket = {
+  id: string;
+  source: ImproveSource;
+  conversation_id: string;
+  title?: string | null;
+  updated_at?: string;
+  trace: {
+    messages: ImproveTraceMessage[];
+    runs?: unknown[];
+    raw?: unknown;
+  };
+};
+
+export type ImproveBundle = {
+  schema_version: 1;
+  created_at: string;
+  source: ImproveSource;
+  since?: string;
+  deploy_id?: string;
+  evidence?: unknown;
+  traces: ImproveTracePacket[];
+};
+
+export type ImproveCollectResult = {
+  root: string;
+  directory: string;
+  bundlePath: string;
+  reportPath: string;
+  traceCount: number;
+  source: ImproveSource;
+};
+
+export type ImproveEvalGenerationResult = {
+  root: string;
+  bundlePath: string;
+  files: string[];
+  skipped: Array<{ trace_id: string; reason: string }>;
+};
+
+export type ImproveReplayResult = {
+  root: string;
+  bundlePath: string;
+  passed: number;
+  failed: number;
+  skipped: number;
+  results: Array<{
+    trace_id: string;
+    conversation_id: string;
+    status: "passed" | "failed" | "skipped";
+    turns: number;
+    error?: string;
+  }>;
+};
+
+export type DeployImproveCollector = {
+  exportEvidence?(input: {
+    deployId: string;
+    since?: string;
+    conversationId?: string;
+  }): Promise<unknown | null>;
+  listConversations(): Promise<unknown>;
+  getTrace(conversationId: string): Promise<unknown>;
+};
+
+export async function collectLocalImproveBundle(
+  cwd: string,
+  options: {
+    since?: string;
+    conversationId?: string;
+    out?: string;
+  } = {},
+): Promise<ImproveCollectResult> {
+  const root = await findAgentCapsuleRoot(cwd);
+  const sinceDate = resolveSinceDate(options.since);
+  const conversations = options.conversationId
+    ? [{ id: options.conversationId }]
+    : (await listConversationsFromCwd(root)).filter((conversation) =>
+        sinceDate ? dateIsOnOrAfter(conversation.updatedAt, sinceDate) : true,
+      );
+  const traces: ImproveTracePacket[] = [];
+
+  for (const conversation of conversations) {
+    const trace = await getConversationTraceFromCwd(root, conversation.id);
+    traces.push(tracePacketFromPayload("local", trace.id, trace));
+  }
+
+  return writeImproveBundle(root, {
+    schema_version: 1,
+    created_at: new Date().toISOString(),
+    source: "local",
+    ...(options.since ? { since: options.since } : {}),
+    traces,
+  }, options.out);
+}
+
+export async function collectDeployImproveBundle(
+  cwd: string,
+  options: {
+    collector: DeployImproveCollector;
+    deployId: string;
+    since?: string;
+    conversationId?: string;
+    out?: string;
+  },
+): Promise<ImproveCollectResult> {
+  const root = await findAgentCapsuleRoot(cwd);
+
+  if (options.collector.exportEvidence) {
+    const exported = await options.collector.exportEvidence({
+      deployId: options.deployId,
+      ...(options.since ? { since: options.since } : {}),
+      ...(options.conversationId ? { conversationId: options.conversationId } : {}),
+    });
+
+    if (exported) {
+      const normalized = normalizeDeployEvidenceExport(exported);
+      const traces = await tracePacketsFromDeployEvidenceExport({
+        collector: options.collector,
+        deployId: options.deployId,
+        evidence: normalized,
+        source: "deploy",
+      });
+
+      return writeImproveBundle(root, {
+        schema_version: 1,
+        created_at: normalized.created_at ?? new Date().toISOString(),
+        source: "deploy",
+        deploy_id: options.deployId,
+        ...(options.since ? { since: options.since } : {}),
+        evidence: redactEvidenceValue(normalized.raw),
+        traces,
+      }, options.out);
+    }
+  }
+
+  const sinceDate = resolveSinceDate(options.since);
+  const summaries = options.conversationId
+    ? [{ id: options.conversationId }]
+    : normalizeConversationSummaries(await options.collector.listConversations()).filter((conversation) =>
+        sinceDate && conversation.updated_at ? dateIsOnOrAfter(conversation.updated_at, sinceDate) : true,
+      );
+  const traces: ImproveTracePacket[] = [];
+
+  for (const summary of summaries) {
+    const trace = await options.collector.getTrace(summary.id);
+    traces.push(tracePacketFromPayload("deploy", summary.id, trace));
+  }
+
+  return writeImproveBundle(root, {
+    schema_version: 1,
+    created_at: new Date().toISOString(),
+    source: "deploy",
+    deploy_id: options.deployId,
+    ...(options.since ? { since: options.since } : {}),
+    traces,
+  }, options.out);
+}
+
+export async function generateImproveEvals(
+  cwd: string,
+  options: {
+    bundle: string;
+    force?: boolean;
+  },
+): Promise<ImproveEvalGenerationResult> {
+  const root = await findAgentCapsuleRoot(cwd);
+  const bundlePath = await resolveBundlePath(root, options.bundle);
+  const bundle = await readImproveBundle(bundlePath);
+  const files: string[] = [];
+  const skipped: ImproveEvalGenerationResult["skipped"] = [];
+  const usedPaths = new Set<string>();
+
+  for (const trace of bundle.traces) {
+    const turns = replayTurnsFromMessages(trace.trace.messages).map(redactEvalTurn);
+
+    if (turns.length === 0) {
+      skipped.push({ trace_id: trace.id, reason: "trace has no user turns" });
+      continue;
+    }
+
+    const file = resolveUniqueEvalPath(root, trace, usedPaths);
+    usedPaths.add(file);
+    const source = renderImproveEval(trace, turns);
+
+    await mkdir(dirname(file), { recursive: true });
+    try {
+      await writeFile(file, source, { flag: options.force ? "w" : "wx" });
+    } catch (error) {
+      if (isNodeError(error) && error.code === "EEXIST") {
+        throw new AgentKitError(
+          "validation_error",
+          `${relative(root, file)} already exists. Re-run with --force after reviewing the existing eval.`,
+        );
+      }
+
+      throw error;
+    }
+
+    files.push(file);
+  }
+
+  return {
+    root,
+    bundlePath,
+    files,
+    skipped,
+  };
+}
+
+export async function replayImproveBundle(
+  cwd: string,
+  options: {
+    bundle: string;
+    against: "local";
+  },
+): Promise<ImproveReplayResult> {
+  if (options.against !== "local") {
+    throw new AgentKitError("validation_error", 'agentkit replay currently supports only --against local.');
+  }
+
+  const root = await findAgentCapsuleRoot(cwd);
+  const bundlePath = await resolveBundlePath(root, options.bundle);
+  const bundle = await readImproveBundle(bundlePath);
+  const results: ImproveReplayResult["results"] = [];
+
+  for (const trace of bundle.traces) {
+    const userTurns = trace.trace.messages.filter((message) => message.role === "user" && message.content.trim().length > 0);
+
+    if (userTurns.length === 0) {
+      results.push({
+        trace_id: trace.id,
+        conversation_id: trace.conversation_id,
+        status: "skipped",
+        turns: 0,
+        error: "trace has no user turns",
+      });
+      continue;
+    }
+
+    const replayConversationId = `replay_${slugify(trace.conversation_id)}_${crypto.randomUUID()}`;
+
+    try {
+      for (const turn of userTurns) {
+        await runAgentMessageFromCwd(root, {
+          message: turn.content,
+          conversationId: replayConversationId,
+          runtime: {
+            environment: "eval",
+            invocation: "eval",
+          },
+        });
+      }
+
+      results.push({
+        trace_id: trace.id,
+        conversation_id: trace.conversation_id,
+        status: "passed",
+        turns: userTurns.length,
+      });
+    } catch (error) {
+      results.push({
+        trace_id: trace.id,
+        conversation_id: trace.conversation_id,
+        status: "failed",
+        turns: userTurns.length,
+        error: formatError(error),
+      });
+    }
+  }
+
+  return {
+    root,
+    bundlePath,
+    passed: results.filter((result) => result.status === "passed").length,
+    failed: results.filter((result) => result.status === "failed").length,
+    skipped: results.filter((result) => result.status === "skipped").length,
+    results,
+  };
+}
+
+async function writeImproveBundle(root: string, bundle: ImproveBundle, out?: string): Promise<ImproveCollectResult> {
+  const directory = resolveImproveDirectory(root, bundle.created_at, out);
+  const bundlePath = join(directory, "bundle.json");
+  const reportPath = join(directory, "report.json");
+  const seenTraceIds = new Set<string>();
+
+  await mkdir(join(directory, "traces"), { recursive: true });
+  for (const trace of bundle.traces) {
+    if (seenTraceIds.has(trace.id)) {
+      throw new AgentKitError("validation_error", `Improve trace id ${trace.id} is duplicated and would overwrite evidence.`);
+    }
+
+    seenTraceIds.add(trace.id);
+  }
+
+  await writeFile(bundlePath, `${JSON.stringify(bundle, null, 2)}\n`);
+  await writeFile(
+    reportPath,
+    `${JSON.stringify(
+      {
+        created_at: bundle.created_at,
+        source: bundle.source,
+        deploy_id: bundle.deploy_id ?? null,
+        trace_count: bundle.traces.length,
+        conversations: bundle.traces.map((trace) => ({
+          trace_id: trace.id,
+          conversation_id: trace.conversation_id,
+          title: trace.title ?? null,
+          updated_at: trace.updated_at ?? null,
+          messages: trace.trace.messages.length,
+          runs: Array.isArray(trace.trace.runs) ? trace.trace.runs.length : 0,
+        })),
+        evidence: summarizeImproveEvidence(bundle.evidence),
+      },
+      null,
+      2,
+    )}\n`,
+  );
+
+  for (const trace of bundle.traces) {
+    await writeFile(join(directory, "traces", `${trace.id}.json`), `${JSON.stringify(trace, null, 2)}\n`);
+  }
+
+  return {
+    root,
+    directory,
+    bundlePath,
+    reportPath,
+    traceCount: bundle.traces.length,
+    source: bundle.source,
+  };
+}
+
+type NormalizedDeployEvidenceExport = {
+  created_at?: string;
+  deploy_id?: string;
+  traces: unknown[];
+  conversations: Array<{ id: string; updated_at?: string }>;
+  runs: unknown[];
+  raw: unknown;
+};
+
+function normalizeDeployEvidenceExport(payload: unknown): NormalizedDeployEvidenceExport {
+  const evidence = isRecord(payload) && isRecord(payload.evidence) ? payload.evidence : payload;
+
+  if (!isRecord(evidence)) {
+    throw new AgentKitError("deploy_improve_response_invalid", "Improve evidence export response was not an object.");
+  }
+
+  const traces = Array.isArray(evidence.traces) ? evidence.traces : [];
+  const conversations = Array.isArray(evidence.conversations)
+    ? evidence.conversations
+        .flatMap((conversation) => {
+          if (!isRecord(conversation)) {
+            return [];
+          }
+
+          const id = readString(conversation, "id") ?? readString(conversation, "conversation_id");
+
+          if (!id) {
+            return [];
+          }
+
+          const updatedAt = readString(conversation, "updated_at") ?? readString(conversation, "updatedAt");
+          return {
+            id,
+            ...(updatedAt ? { updated_at: updatedAt } : {}),
+          };
+        })
+    : [];
+
+  return {
+    created_at: readString(evidence, "created_at") ?? readString(evidence, "createdAt"),
+    deploy_id: readString(evidence, "deploy_id") ?? readString(evidence, "deployId"),
+    traces,
+    conversations,
+    runs: Array.isArray(evidence.runs) ? evidence.runs : [],
+    raw: evidence,
+  };
+}
+
+async function tracePacketsFromDeployEvidenceExport(input: {
+  collector: DeployImproveCollector;
+  deployId: string;
+  evidence: NormalizedDeployEvidenceExport;
+  source: ImproveSource;
+}): Promise<ImproveTracePacket[]> {
+  const traces: ImproveTracePacket[] = [];
+  const seenConversations = new Set<string>();
+
+  for (const rawTrace of input.evidence.traces) {
+    const trace = tracePacketFromPayload(input.source, input.deployId, rawTrace);
+    traces.push(trace);
+    seenConversations.add(trace.conversation_id);
+  }
+
+  for (const conversation of input.evidence.conversations) {
+    if (seenConversations.has(conversation.id)) {
+      continue;
+    }
+
+    try {
+      const trace = await input.collector.getTrace(conversation.id);
+      const packet = tracePacketFromPayload(input.source, conversation.id, trace);
+      traces.push(packet);
+      seenConversations.add(packet.conversation_id);
+    } catch {
+      traces.push(syntheticEvidenceTrace(input.source, conversation.id, {
+        updated_at: conversation.updated_at,
+        reason: "conversation trace was not available from the hosted runtime",
+      }));
+      seenConversations.add(conversation.id);
+    }
+  }
+
+  for (const run of input.evidence.runs) {
+    if (!isRecord(run)) {
+      continue;
+    }
+
+    const conversationId = readString(run, "conversation_id") ?? readString(run, "conversationId");
+
+    if (conversationId && seenConversations.has(conversationId)) {
+      continue;
+    }
+
+    const runId = readString(run, "id") ?? readString(run, "run_id") ?? `run_${shortHash(JSON.stringify(run))}`;
+    traces.push(syntheticEvidenceTrace(input.source, conversationId ?? runId, {
+      title: readString(run, "title") ?? runId,
+      updated_at: readString(run, "updated_at") ?? readString(run, "updatedAt"),
+      raw: run,
+    }));
+
+    if (conversationId) {
+      seenConversations.add(conversationId);
+    }
+  }
+
+  return traces;
+}
+
+function syntheticEvidenceTrace(
+  source: ImproveSource,
+  conversationId: string,
+  input: {
+    title?: string;
+    updated_at?: string;
+    reason?: string;
+    raw?: unknown;
+  },
+): ImproveTracePacket {
+  const safeConversationId = redactEvalText(conversationId);
+  const title = input.title ? redactEvalText(input.title) : "Hosted evidence";
+
+  return {
+    id: `trace_${slugify(title)}_${shortHash(`${source}:${conversationId}:${JSON.stringify(input.raw ?? input.reason ?? "")}`)}`,
+    source,
+    conversation_id: safeConversationId,
+    title,
+    updated_at: input.updated_at,
+    trace: {
+      messages: [
+        {
+          role: "system",
+          content: redactEvalText(input.reason ?? "Hosted evidence did not include replayable conversation messages."),
+          ...(input.updated_at ? { created_at: input.updated_at } : {}),
+        },
+      ],
+      raw: redactEvidenceValue(input.raw ?? input.reason ?? null),
+    },
+  };
+}
+
+function summarizeImproveEvidence(evidence: unknown): unknown {
+  if (!isRecord(evidence)) {
+    return evidence === undefined ? undefined : { available: true };
+  }
+
+  return {
+    deploy_id: readString(evidence, "deploy_id") ?? readString(evidence, "deployId") ?? null,
+    runs: Array.isArray(evidence.runs) ? evidence.runs.length : 0,
+    conversations: Array.isArray(evidence.conversations) ? evidence.conversations.length : 0,
+    deliveries: Array.isArray(evidence.deliveries) ? evidence.deliveries.length : 0,
+    deploy_events: Array.isArray(evidence.deploy_events) ? evidence.deploy_events.length : 0,
+  };
+}
+
+async function readImproveBundle(path: string): Promise<ImproveBundle> {
+  const parsed = JSON.parse(await readFile(path, "utf8")) as unknown;
+
+  if (!isRecord(parsed) || parsed.schema_version !== 1 || !Array.isArray(parsed.traces)) {
+    throw new AgentKitError("validation_error", `${path} is not an AgentKit improve bundle.`);
+  }
+
+  return parsed as ImproveBundle;
+}
+
+async function resolveBundlePath(root: string, input: string): Promise<string> {
+  if (!input) {
+    throw new AgentKitError("validation_error", "Missing improve bundle path.");
+  }
+
+  const resolved = resolveInsideCapsule(root, input, "improve bundle");
+  const info = await stat(resolved);
+
+  if (info.isDirectory()) {
+    return join(resolved, "bundle.json");
+  }
+
+  return resolved;
+}
+
+function resolveImproveDirectory(root: string, createdAt: string, out?: string): string {
+  const defaultPath = join(".agentkit", "improve", createdAt.replace(/[:.]/g, "-"));
+  return resolveInsideCapsule(root, out ?? defaultPath, "improve output");
+}
+
+function resolveUniqueEvalPath(root: string, trace: ImproveTracePacket, usedPaths: Set<string>): string {
+  const baseName = trace.id.replace(/^trace_/, "");
+  const base = join("evals", "regressions", `improve-${baseName}.eval.ts`);
+  let candidate = resolveInsideCapsule(root, base, "improve eval output");
+  let index = 2;
+
+  while (usedPaths.has(candidate)) {
+    candidate = resolveInsideCapsule(
+      root,
+      join("evals", "regressions", `improve-${baseName}-${index}.eval.ts`),
+      "improve eval output",
+    );
+    index += 1;
+  }
+
+  return candidate;
+}
+
+function resolveInsideCapsule(root: string, path: string, context: string): string {
+  if (isAbsolute(path)) {
+    throw new AgentKitError("validation_error", `${context} path must be relative to the Agent Capsule.`);
+  }
+
+  const resolved = resolve(root, path);
+  const relativePath = relative(root, resolved);
+
+  if (relativePath === "" || relativePath.startsWith("..") || isAbsolute(relativePath)) {
+    throw new AgentKitError("validation_error", `${context} path must stay inside the Agent Capsule.`);
+  }
+
+  return resolved;
+}
+
+function normalizeConversationSummaries(payload: unknown): Array<{ id: string; updated_at?: string }> {
+  const conversations = Array.isArray(payload)
+    ? payload
+    : isRecord(payload) && Array.isArray(payload.conversations)
+      ? payload.conversations
+      : null;
+
+  if (!conversations) {
+    throw new AgentKitError("deploy_conversation_response_invalid", "Conversation list response did not include conversations.");
+  }
+
+  return conversations.map((conversation) => {
+    if (!isRecord(conversation)) {
+      throw new AgentKitError("deploy_conversation_response_invalid", "Conversation list contained a non-object entry.");
+    }
+
+    const id = readString(conversation, "id") ?? readString(conversation, "conversation_id");
+
+    if (!id) {
+      throw new AgentKitError("deploy_conversation_response_invalid", "Conversation list entry did not include an id.");
+    }
+
+    return {
+      id,
+      updated_at: readString(conversation, "updated_at") ?? readString(conversation, "updatedAt"),
+    };
+  });
+}
+
+function tracePacketFromPayload(source: ImproveSource, fallbackConversationId: string, payload: unknown): ImproveTracePacket {
+  const trace = normalizeTracePayload(fallbackConversationId, payload);
+  const safeConversationId = redactEvalText(trace.conversation_id);
+  const safeTitle = trace.title ? redactEvalText(trace.title) : null;
+  const idLabel = slugify(safeTitle ?? safeConversationId);
+  const idHash = shortHash(`${source}:${trace.conversation_id}`);
+
+  return {
+    id: `trace_${idLabel}_${idHash}`,
+    source,
+    conversation_id: safeConversationId,
+    title: safeTitle,
+    updated_at: trace.updated_at,
+    trace: {
+      messages: trace.messages.map(redactTraceMessage),
+      runs: Array.isArray(trace.runs) ? trace.runs.map(redactEvidenceValue) : trace.runs,
+      raw: redactEvidenceValue(payload),
+    },
+  };
+}
+
+function normalizeTracePayload(
+  fallbackConversationId: string,
+  payload: unknown,
+): {
+  conversation_id: string;
+  title?: string | null;
+  updated_at?: string;
+  messages: ImproveTraceMessage[];
+  runs?: unknown[];
+} {
+  if (!isRecord(payload)) {
+    throw new AgentKitError("deploy_conversation_response_invalid", "Conversation trace response was not an object.");
+  }
+
+  const conversationId = readString(payload, "id") ?? readString(payload, "conversation_id") ?? fallbackConversationId;
+  const title = readString(payload, "title");
+  const updatedAt = readString(payload, "updated_at") ?? readString(payload, "updatedAt");
+  const messages = Array.isArray(payload.messages)
+    ? payload.messages.map(normalizeMessage).filter((message): message is ImproveTraceMessage => message !== null)
+    : [];
+
+  return {
+    conversation_id: conversationId,
+    title: title ?? null,
+    updated_at: updatedAt,
+    messages,
+    runs: Array.isArray(payload.runs) ? payload.runs : undefined,
+  };
+}
+
+function normalizeMessage(message: unknown): ImproveTraceMessage | null {
+  if (!isRecord(message)) {
+    return null;
+  }
+
+  const role = readString(message, "role");
+  const content = readString(message, "content");
+
+  if (!isTraceRole(role) || typeof content !== "string") {
+    return null;
+  }
+
+  return {
+    role,
+    content,
+    created_at: readString(message, "created_at") ?? readString(message, "createdAt"),
+    sequence: readNumber(message, "sequence"),
+  };
+}
+
+function redactTraceMessage(message: ImproveTraceMessage): ImproveTraceMessage {
+  return {
+    ...message,
+    content: redactEvalText(message.content),
+  };
+}
+
+type ImproveEvalTurn = {
+  input: string;
+  expect?: {
+    response:
+      | { contains: string }
+      | { notRegex: string[] };
+  };
+};
+
+function replayTurnsFromMessages(messages: ImproveTraceMessage[]): ImproveEvalTurn[] {
+  const turns: ImproveEvalTurn[] = [];
+
+  for (let index = 0; index < messages.length; index += 1) {
+    const message = messages[index];
+
+    if (message.role !== "user") {
+      continue;
+    }
+
+    const nextAssistant = messages.slice(index + 1).find((candidate) => candidate.role === "assistant");
+
+    turns.push({
+      input: message.content,
+      ...(nextAssistant ? { expect: { response: { contains: nextAssistant.content } } } : {}),
+    });
+  }
+
+  return turns;
+}
+
+function redactEvalTurn(turn: ImproveEvalTurn): ImproveEvalTurn {
+  const input = redactEvalText(turn.input);
+
+  if (!turn.expect || !("contains" in turn.expect.response)) {
+    return { input, ...(turn.expect ? { expect: turn.expect } : {}) };
+  }
+
+  const contains = redactEvalText(turn.expect.response.contains);
+  const redactionChangedTurn = input !== turn.input || contains !== turn.expect.response.contains;
+  const redactionPresent = input.includes("[redacted_") || contains.includes("[redacted_");
+
+  return {
+    input,
+    ...(redactionChangedTurn || redactionPresent
+      ? {
+          expect: {
+            response: {
+              notRegex: ["API_KEY|Bearer\\s+|(?:sk|agk|ak|dpat)[_-]"],
+            },
+          },
+        }
+      : { expect: { response: { contains } } }),
+  };
+}
+
+function redactEvalText(value: string): string {
+  return value
+    .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "[redacted_email]")
+    .replace(/\b[A-Z0-9_]*(?:API[_-]?KEY|TOKEN|SECRET|PASSWORD|PRIVATE[_-]?KEY|CLIENT[_-]?SECRET)\s*=\s*[^\s,;]+/gi, (match) => {
+      const separator = match.includes("=") ? "=" : "";
+      const name = separator ? match.slice(0, match.indexOf("=")).trim() : "SECRET";
+      return `${name}=[redacted_secret]`;
+    })
+    .replace(/\b(?:sk|agk|ak|dpat)[_-][A-Za-z0-9_-]{8,}\b/g, "[redacted_token]")
+    .replace(/\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi, "Bearer [redacted_token]")
+    .replace(/\+?\d[\d\s().-]{7,}\d/g, "[redacted_phone]");
+}
+
+function redactEvidenceValue(value: unknown): unknown {
+  if (typeof value === "string") {
+    return redactEvalText(value);
+  }
+
+  if (Array.isArray(value)) {
+    return value.map(redactEvidenceValue);
+  }
+
+  if (!isRecord(value)) {
+    return value;
+  }
+
+  const redacted: Record<string, unknown> = {};
+
+  for (const [key, nestedValue] of Object.entries(value)) {
+    redacted[key] = shouldRedactEvidenceField(key)
+      ? "[redacted_secret]"
+      : redactEvidenceValue(nestedValue);
+  }
+
+  return redacted;
+}
+
+function shouldRedactEvidenceField(key: string): boolean {
+  return /(?:token|authorization|api[_-]?key|secret|password|private[_-]?key|client[_-]?secret)$/i.test(key);
+}
+
+function renderImproveEval(
+  trace: ImproveTracePacket,
+  turns: ImproveEvalTurn[],
+): string {
+  return `import { defineEval } from "@andreprado/agentkit";
+
+export default defineEval({
+  name: ${JSON.stringify(`production replay ${trace.title ?? trace.id}`)},
+  turns: ${formatEvalTurns(turns)},
+});
+`;
+}
+
+function formatEvalTurns(turns: ImproveEvalTurn[]): string {
+  return JSON.stringify(turns, null, 4)
+    .split("\n")
+    .map((line, index) => (index === 0 ? line : `  ${line}`))
+    .join("\n");
+}
+
+function resolveSinceDate(value: string | undefined): Date | null {
+  if (!value) {
+    return null;
+  }
+
+  const trimmed = value.trim();
+  const duration = trimmed.match(/^(\d+)(m|h|d|w)$/);
+
+  if (duration) {
+    const amount = Number(duration[1]);
+    const unit = duration[2];
+    const multiplier = unit === "m" ? 60_000 : unit === "h" ? 3_600_000 : unit === "d" ? 86_400_000 : 604_800_000;
+    return new Date(Date.now() - amount * multiplier);
+  }
+
+  const parsed = new Date(trimmed);
+
+  if (Number.isNaN(parsed.getTime())) {
+    throw new AgentKitError("validation_error", 'improve --since must be an ISO timestamp or a duration like "24h", "7d", or "2w".');
+  }
+
+  return parsed;
+}
+
+function dateIsOnOrAfter(value: string, since: Date): boolean {
+  const timestamp = Date.parse(value);
+  return Number.isFinite(timestamp) && timestamp >= since.getTime();
+}
+
+function slugify(value: string): string {
+  const slug = value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 48);
+
+  return slug || "trace";
+}
+
+function shortHash(value: string): string {
+  return createHash("sha256").update(value).digest("hex").slice(0, 10);
+}
+
+function formatError(error: unknown): string {
+  if (isAgentKitError(error)) {
+    return `${error.code}: ${error.message}`;
+  }
+
+  if (error instanceof Error) {
+    return `runtime_error: ${error.message}`;
+  }
+
+  return `runtime_error: ${String(error)}`;
+}
+
+function isTraceRole(value: unknown): value is ImproveTraceMessage["role"] {
+  return value === "user" || value === "assistant" || value === "system" || value === "tool";
+}
+
+function readString(value: Record<string, unknown>, key: string): string | undefined {
+  const item = value[key];
+  return typeof item === "string" ? item : undefined;
+}
+
+function readNumber(value: Record<string, unknown>, key: string): number | undefined {
+  const item = value[key];
+  return typeof item === "number" && Number.isFinite(item) ? item : undefined;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isNodeError(error: unknown): error is NodeJS.ErrnoException {
+  return error instanceof Error && "code" in error;
+}