@andreprado/agentkit 0.1.0-alpha.17 → 0.1.0-alpha.19

package/src/cli/index.ts

@@ -21,6 +21,7 @@ import {
 import { runEvalsFromCwd, writeEvalFromConversation } from "../runtime/evals";
 import { findAgentKitDocs } from "../runtime/docs";
 import { listCapsuleEnv, setCapsuleEnv, unsetCapsuleEnv } from "../runtime/env";
+import { collectDeployImproveBundle, collectLocalImproveBundle, generateImproveEvals, replayImproveBundle } from "../runtime/improve";
 import { checkAgentSpec, initializeAgentSpec } from "../runtime/spec";
 import { checkDefaultSkillStatus, syncDefaultSkillFiles } from "../runtime/skills";
 import { initializeSyncScaffold, runCapsuleSync } from "../runtime/sync";
@@ -44,6 +45,12 @@ import {
   resolveCloudApiUrl,
   verifyCloudLogin,
   writeCloudAuth,
+  type AccountApiTokenCreateResponse,
+  type AccountApiTokenListResponse,
+  type BillingCheckoutIntentResponse,
+  type BillingCheckoutResponse,
+  type BillingClaimResponse,
+  type BillingPortalResponse,
   type CloudMeResponse,
   type DeployAccessTokenCreateResponse,
   type DeployAccessTokenListResponse,
@@ -70,6 +77,8 @@ import { handleChannelsCommand } from "./commands/channels";
 import { handleKnowledgeCommand } from "./commands/knowledge";
 import { handleTranscribeCommand } from "./commands/transcribe";
 
+const hostedDeployEntitlement = "agentkit_cloud_deploy";
+
 async function main() {
   const args = parseArgs(process.argv.slice(2));
 
@@ -336,10 +345,132 @@ async function main() {
     return;
   }
 
+  if (args.command === "improve") {
+    const [subcommand, bundlePath] = args.positional;
+
+    if (subcommand === "collect") {
+      const since = parseOptionalStringFlag(args.flags.since, "since", 'agentkit improve collect --since "24h"');
+      const conversationId = parseOptionalStringFlag(
+        args.flags["conversation-id"] ?? args.flags.conversation,
+        "conversation id",
+        "agentkit improve collect --conversation-id <id>",
+      );
+      const out = parseOptionalStringFlag(args.flags.out, "out", "agentkit improve collect --out <directory>");
+      const deployState = args.flags.deploy === true ? await requireLocalDeployState() : null;
+      const apiUrl = args.flags.deploy === true ? await resolveCloudApiUrl(args.flags.api) : null;
+      const cloudAuth = apiUrl ? await readCloudAuthForApiUrl(apiUrl) : null;
+      const result = args.flags.deploy === true
+        ? await collectDeployImproveBundle(process.cwd(), {
+            deployId: deployState?.deploy_id ?? "",
+            since,
+            conversationId,
+            out,
+            collector: {
+              ...(cloudAuth && apiUrl && deployState
+                ? {
+                    exportEvidence: (input: { deployId: string; since?: string; conversationId?: string }) =>
+                      fetchDeployImproveEvidence(apiUrl, input),
+                  }
+                : {}),
+              listConversations: () => fetchDeployConversationJson(process.cwd(), "/v1/conversations"),
+              getTrace: (id: string) =>
+                fetchDeployConversationJson(process.cwd(), `/v1/conversations/${encodeURIComponent(id)}/trace`),
+            },
+          })
+        : await collectLocalImproveBundle(process.cwd(), {
+            since,
+            conversationId,
+            out,
+          });
+
+      console.log("Improve bundle collected");
+      console.log(`Source:    ${result.source}`);
+      console.log(`Directory: ${relativePath(result.root, result.directory)}`);
+      console.log(`Bundle:    ${relativePath(result.root, result.bundlePath)}`);
+      console.log(`Report:    ${relativePath(result.root, result.reportPath)}`);
+      console.log(`Traces:    ${result.traceCount}`);
+      console.log("");
+      console.log(`Next:      agentkit improve evals ${relativePath(result.root, result.directory)}`);
+      console.log(`Replay:    agentkit replay ${relativePath(result.root, result.directory)} --against local`);
+      return;
+    }
+
+    if (subcommand === "evals") {
+      if (!bundlePath) {
+        throw new Error("Missing bundle path. Usage: agentkit improve evals <bundle-dir-or-json> [--force]");
+      }
+
+      const result = await generateImproveEvals(process.cwd(), {
+        bundle: bundlePath,
+        force: args.flags.force === true,
+      });
+
+      console.log("Improve evals generated");
+      console.log(`Bundle: ${relativePath(result.root, result.bundlePath)}`);
+      console.log(`Files:  ${result.files.length}`);
+      for (const file of result.files) {
+        console.log(`  ${relativePath(result.root, file)}`);
+      }
+      if (result.skipped.length > 0) {
+        console.log(`Skipped: ${result.skipped.length}`);
+        for (const skipped of result.skipped) {
+          console.log(`  ${skipped.trace_id}: ${skipped.reason}`);
+        }
+      }
+      return;
+    }
+
+    throw new Error("Usage: agentkit improve collect [--deploy] [--since <duration|iso>] [--conversation-id <id>] [--out <directory>] | agentkit improve evals <bundle-dir-or-json> [--force]");
+  }
+
+  if (args.command === "replay") {
+    const [bundlePath] = args.positional;
+
+    if (!bundlePath) {
+      throw new Error("Missing bundle path. Usage: agentkit replay <bundle-dir-or-json> --against local");
+    }
+
+    const against = parseOptionalStringFlag(args.flags.against, "against", "agentkit replay <bundle> --against local") ?? "local";
+
+    if (against !== "local") {
+      throw new Error("Usage: agentkit replay <bundle-dir-or-json> --against local");
+    }
+
+    const result = await replayImproveBundle(process.cwd(), {
+      bundle: bundlePath,
+      against,
+    });
+
+    for (const item of result.results) {
+      console.log(`${item.status.toUpperCase()} ${item.conversation_id} (${item.turns} turns)`);
+      if (item.error) {
+        console.log(`  - ${item.error}`);
+      }
+    }
+
+    console.log("");
+    console.log(`Replay passed: ${result.passed}`);
+    console.log(`Replay failed: ${result.failed}`);
+    console.log(`Replay skipped: ${result.skipped}`);
+
+    if (result.failed > 0) {
+      process.exitCode = 1;
+    }
+
+    return;
+  }
+
   if (args.command === "conversations") {
     const [subcommand, conversationId] = args.positional;
+    const fromDeploy = args.flags.deploy === true;
 
     if (subcommand === "list") {
+      if (fromDeploy) {
+        const payload = await fetchDeployConversationJson(process.cwd(), "/v1/conversations");
+        printJson(payload);
+        return;
+      }
+
       const conversations = await listConversationsFromCwd(process.cwd());
 
       if (conversations.length === 0) {
@@ -365,7 +496,16 @@ async function main() {
 
     if (subcommand === "show") {
       if (!conversationId) {
-        throw new Error("Missing conversation id. Usage: agentkit conversations show <conversation-id>");
+        throw new Error("Missing conversation id. Usage: agentkit conversations show <conversation-id> [--deploy]");
+      }
+
+      if (fromDeploy) {
+        const payload = await fetchDeployConversationJson(
+          process.cwd(),
+          `/v1/conversations/${encodeURIComponent(conversationId)}`,
+        );
+        printJson(payload);
+        return;
       }
 
       const conversation = await getConversationFromCwd(process.cwd(), conversationId);
@@ -384,7 +524,16 @@ async function main() {
 
     if (subcommand === "trace") {
       if (!conversationId) {
-        throw new Error("Missing conversation id. Usage: agentkit conversations trace <conversation-id>");
+        throw new Error("Missing conversation id. Usage: agentkit conversations trace <conversation-id> [--deploy]");
+      }
+
+      if (fromDeploy) {
+        const trace = await fetchDeployConversationJson(
+          process.cwd(),
+          `/v1/conversations/${encodeURIComponent(conversationId)}/trace`,
+        );
+        printJson(trace);
+        return;
       }
 
       const trace = await getConversationTraceFromCwd(process.cwd(), conversationId);
@@ -392,7 +541,7 @@ async function main() {
       return;
     }
 
-    throw new Error("Usage: agentkit conversations list | agentkit conversations show <conversation-id> | agentkit conversations trace <conversation-id>");
+    throw new Error("Usage: agentkit conversations list [--deploy] | agentkit conversations show <conversation-id> [--deploy] | agentkit conversations trace <conversation-id> [--deploy]");
   }
 
   if (args.command === "knowledge") {
@@ -492,13 +641,15 @@ async function main() {
     const email = verified.me.account?.email;
     console.log(email ? `Logged in to AgentKit Cloud as ${email}.` : "AgentKit Cloud token verified and stored.");
 
-    const hasCloudflareAlpha = (verified.capabilities.capabilities ?? []).some(
-      (capability) => capability.name === "cloudflare_deploy_alpha" && capability.status === "active",
+    const hasHostedDeploy = (verified.capabilities.capabilities ?? []).some(
+      (capability) =>
+        (capability.name === "cloudflare_deploy_alpha" || capability.name === hostedDeployEntitlement) &&
+        capability.status === "active",
     );
     console.log(
-      hasCloudflareAlpha
-        ? "Entitlement: cloudflare_deploy_alpha active."
-        : "Warning: cloudflare_deploy_alpha entitlement was not found for this account.",
+      hasHostedDeploy
+        ? "Entitlement: hosted deploy active."
+        : `Warning: cloudflare_deploy_alpha or ${hostedDeployEntitlement} entitlement was not found for this account.`,
     );
 
     if (process.env.AGENTKIT_CLOUD_API_TOKEN && process.env.AGENTKIT_CLOUD_API_TOKEN !== token) {
@@ -514,6 +665,208 @@ async function main() {
     return;
   }
 
+  if (args.command === "account") {
+    const [noun, verb, value] = args.positional;
+
+    if (noun !== "token") {
+      throw new Error("Usage: agentkit account token create <name> [--use] [--out <path>] | list | revoke <token-id> [--api <url>]");
+    }
+
+    const apiUrl = await resolveCloudApiUrl(args.flags.api);
+
+    if (verb === "create" && value) {
+      const payload = (await cloudApiRequest(apiUrl, "/v1/account/tokens", {
+        method: "POST",
+        body: JSON.stringify({ name: value }),
+      })) as AccountApiTokenCreateResponse;
+      const token = payload.token;
+
+      if (!token?.token || !token.id) {
+        throw new AgentKitError("account_token_invalid", "AgentKit Cloud returned an invalid account token payload.");
+      }
+
+      const outPath = parseOptionalStringFlag(
+        args.flags.out,
+        "out",
+        "agentkit account token create <name> [--api <url>] [--use] [--out <path>]",
+      );
+
+      if (outPath) {
+        const absolutePath = resolve(process.cwd(), outPath);
+        await writeSecretJson(absolutePath, payload);
+        console.log(`Account token written to ${absolutePath}`);
+      } else {
+        printJson(payload);
+      }
+
+      if (args.flags.use === true) {
+        await writeCloudAuth({ apiUrl, token: token.token });
+        console.log(`Saved ${token.id} as the current AgentKit Cloud login.`);
+      }
+
+      return;
+    }
+
+    if (verb === "list" && !value) {
+      const payload = (await cloudApiRequest(apiUrl, "/v1/account/tokens", {
+        method: "GET",
+      })) as AccountApiTokenListResponse;
+      printJson(payload);
+      return;
+    }
+
+    if (verb === "revoke" && value) {
+      const payload = await cloudApiRequest(apiUrl, `/v1/account/tokens/${encodeURIComponent(value)}/revoke`, {
+        method: "POST",
+      });
+      printJson(payload);
+      return;
+    }
+
+    throw new Error("Usage: agentkit account token create <name> [--use] [--out <path>] | list | revoke <token-id> [--api <url>]");
+  }
+
+  if (args.command === "billing") {
+    const [subcommand, intentId] = args.positional;
+    const apiUrl = await resolveCloudApiUrl(args.flags.api);
+
+    if (subcommand === "checkout") {
+      const slots = parsePositiveIntegerFlag(
+        args.flags.slots,
+        "slots",
+        "agentkit billing checkout --slots <count> [--email <email>] [--api <url>]",
+      );
+      const email = parseOptionalStringFlag(
+        args.flags.email,
+        "email",
+        "agentkit billing checkout --slots <count> --email <email>",
+      );
+      const payload = (await cloudApiRequest(apiUrl, "/v1/billing/checkout", {
+        method: "POST",
+        body: JSON.stringify({
+          slots,
+          ...(email ? { email } : {}),
+        }),
+      })) as BillingCheckoutResponse;
+      const checkoutUrl = payload.checkout?.url;
+      const billingIntentId = payload.billing_intent?.id;
+      const secret = payload.billing_intent?.secret;
+
+      if (!checkoutUrl || !billingIntentId || !secret) {
+        throw new AgentKitError("billing_checkout_invalid", "AgentKit Cloud returned an invalid billing checkout payload.");
+      }
+
+      if (args.flags.json === true) {
+        printJson(payload);
+      } else {
+        console.log("AgentKit Cloud checkout");
+        console.log("");
+        console.log(`URL:    ${checkoutUrl}`);
+        console.log(`Intent: ${billingIntentId}`);
+        console.log(`Secret: ${secret}`);
+        console.log("");
+        console.log(`After payment: agentkit billing claim ${billingIntentId} --secret ${secret} --api ${apiUrl}`);
+      }
+
+      if (args.flags.open === true) {
+        await openLocalUrl(checkoutUrl);
+      }
+
+      return;
+    }
+
+    if (subcommand === "status" && intentId) {
+      const secret = parseRequiredStringFlag(
+        args.flags.secret,
+        "secret",
+        "agentkit billing status <billing-intent-id> --secret <secret> [--api <url>]",
+      );
+      const payload = (await cloudApiRequest(
+        apiUrl,
+        `/v1/billing/checkout-intents/${encodeURIComponent(intentId)}?secret=${encodeURIComponent(secret)}`,
+        { method: "GET" },
+      )) as BillingCheckoutIntentResponse;
+      printJson(payload);
+      return;
+    }
+
+    if (subcommand === "claim" && intentId) {
+      const secret = parseRequiredStringFlag(
+        args.flags.secret,
+        "secret",
+        "agentkit billing claim <billing-intent-id> --secret <secret> [--token-name <name>] [--api <url>]",
+      );
+      const tokenName = parseOptionalStringFlag(
+        args.flags["token-name"],
+        "token-name",
+        "agentkit billing claim <billing-intent-id> --secret <secret> --token-name <name>",
+      );
+      const payload = (await cloudApiRequest(apiUrl, `/v1/billing/checkout-intents/${encodeURIComponent(intentId)}/claim`, {
+        method: "POST",
+        body: JSON.stringify({
+          secret,
+          ...(tokenName ? { token_name: tokenName } : {}),
+        }),
+      })) as BillingClaimResponse;
+      const token = payload.token;
+
+      if (!token?.token || !token.id) {
+        throw new AgentKitError("billing_claim_invalid", "AgentKit Cloud returned an invalid billing claim payload.");
+      }
+
+      await writeCloudAuth({ apiUrl, token: token.token });
+
+      const outPath = parseOptionalStringFlag(
+        args.flags.out,
+        "out",
+        "agentkit billing claim <billing-intent-id> --secret <secret> [--out <path>]",
+      );
+
+      if (outPath) {
+        const absolutePath = resolve(process.cwd(), outPath);
+        await writeSecretJson(absolutePath, payload);
+        console.log(`Account token written to ${absolutePath}`);
+      } else if (args.flags.json === true) {
+        printJson(payload);
+      } else {
+        console.log("AgentKit Cloud account token claimed and saved.");
+        console.log(`Token: ${token.id}`);
+        console.log(`API:   ${apiUrl}`);
+      }
+
+      return;
+    }
+
+    if (subcommand === "portal") {
+      const payload = (await cloudApiRequest(apiUrl, "/v1/billing/portal", {
+        method: "POST",
+        body: JSON.stringify({}),
+      })) as BillingPortalResponse;
+      const portalUrl = payload.portal?.url;
+
+      if (!portalUrl) {
+        throw new AgentKitError("billing_portal_invalid", "AgentKit Cloud returned an invalid billing portal payload.");
+      }
+
+      if (args.flags.json === true) {
+        printJson(payload);
+      } else {
+        console.log("AgentKit Cloud billing portal");
+        console.log(`URL: ${portalUrl}`);
+      }
+
+      if (args.flags.open === true) {
+        await openLocalUrl(portalUrl);
+      }
+
+      return;
+    }
+
+    throw new Error(
+      "Usage: agentkit billing checkout --slots <count> [--email <email>] | status <intent-id> --secret <secret> | claim <intent-id> --secret <secret> | portal [--api <url>]",
+    );
+  }
+
   if (args.command === "deploy") {
     const [deploySubcommand] = args.positional;
 
@@ -522,7 +875,7 @@ async function main() {
         parseOptionalStringFlag(args.flags.message ?? args.flags.smoke, "message", 'agentkit deploy smoke --message "hello"') ??
         "hello";
       const state = await requireLocalDeployState();
-      const token = await readDeployAccessTokenForSmoke(process.cwd(), state);
+      const token = await readDeployAccessTokenForHostedRequest(process.cwd());
       await runDeploySmoke({
         deployUrl: requireDeployUrl(state),
         message,
@@ -621,7 +974,7 @@ async function main() {
       apiUrl,
       root: result.root,
       deployId: deploy.deploy.id,
-      skip: anonymous || !auth?.token || deploy.deploy.access.mode === "public",
+      skip: anonymous || !auth?.token,
     });
 
     console.log("");
@@ -634,7 +987,7 @@ async function main() {
     if (chatAccess?.status === "ready") {
       console.log(`Chat auth: ${relativePath(process.cwd(), chatAccess.path)}`);
     } else if (chatAccess?.status === "skipped") {
-      console.log("Chat auth: public or anonymous deploy; no local chat token was created.");
+      console.log("Chat auth: anonymous deploy; no local chat token was created.");
     } else if (chatAccess?.status === "failed") {
       console.log(`Chat auth: could not create local chat token. ${formatReadinessError(chatAccess.error)}`);
       console.log(`Run:       agentkit access token create ${defaultDeployChatAccessTokenName} --out ${defaultDeployChatAccessTokenPath}`);
@@ -805,9 +1158,16 @@ async function main() {
     const apiUrl = await resolveCloudApiUrl(args.flags.api);
 
     if (subcommand === "status") {
+      const toolkit = parseOptionalStringFlag(
+        args.flags.toolkit,
+        "toolkit",
+        "agentkit integrations status --toolkit <slug>",
+      );
       const payload = await cloudApiRequest(
         apiUrl,
-        `/v1/deploys/${encodeURIComponent(state.deploy_id)}/integrations/composio/status`,
+        `/v1/deploys/${encodeURIComponent(state.deploy_id)}/integrations/composio/status${
+          toolkit ? `?toolkit=${encodeURIComponent(toolkit)}` : ""
+        }`,
         { method: "GET" },
       );
       printJson(payload);
@@ -842,6 +1202,7 @@ async function main() {
           redirect_url?: string;
           expires_at?: string;
           connected_account_id?: string;
+          auth_config_source?: string;
         };
       };
 
@@ -854,6 +1215,9 @@ async function main() {
       console.log("Composio connect link");
       console.log(`Toolkit: ${link.toolkit ?? toolkit ?? "unknown"}`);
       console.log(`User:    ${link.user_id ?? "unknown"}`);
+      if (link.auth_config_source) {
+        console.log(`Auth:    ${link.auth_config_source}`);
+      }
       console.log(`URL:     ${link.redirect_url}`);
 
       if (link.expires_at) {
@@ -1129,11 +1493,7 @@ function requireDeployUrl(state: Pick<LocalDeployState, "deploy_id" | "url">): s
   return state.url;
 }
 
-async function readDeployAccessTokenForSmoke(cwd: string, state: LocalDeployState): Promise<string | undefined> {
-  if (state.access?.mode === "public") {
-    return undefined;
-  }
-
+async function readDeployAccessTokenForHostedRequest(cwd: string): Promise<string> {
   const root = await findAgentCapsuleRoot(cwd);
   return readDeployAccessTokenForUi(join(root, defaultDeployChatAccessTokenPath));
 }
@@ -1180,6 +1540,62 @@ async function runDeploySmoke(input: { deployUrl: string; message: string; token
   }
 }
 
+async function fetchDeployConversationJson(cwd: string, path: string): Promise<unknown> {
+  const state = await requireLocalDeployState();
+  const deployUrl = requireDeployUrl(state);
+  const token = await readDeployAccessTokenForHostedRequest(cwd);
+  const headers = new Headers();
+
+  headers.set("Authorization", `Bearer ${token}`);
+
+  let response: Response;
+
+  try {
+    response = await fetch(new URL(path, deployUrl).href, {
+      method: "GET",
+      headers,
+    });
+  } catch (error) {
+    throw new AgentKitError(
+      "deploy_conversation_request_failed",
+      `Could not read hosted conversation data: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+
+  const payload = await response.json().catch(() => null);
+
+  if (!response.ok) {
+    throw new AgentKitError("deploy_conversation_request_failed", readCloudError(payload, response.status));
+  }
+
+  return payload;
+}
+
+async function fetchDeployImproveEvidence(
+  apiUrl: string,
+  input: { deployId: string; since?: string; conversationId?: string },
+): Promise<unknown | null> {
+  try {
+    return await cloudApiRequest(apiUrl, `/v1/deploys/${encodeURIComponent(input.deployId)}/improve/export`, {
+      method: "POST",
+      body: JSON.stringify({
+        status: "all",
+        ...(input.since ? { since: input.since } : {}),
+        ...(input.conversationId ? { conversation_id: input.conversationId } : {}),
+      }),
+    });
+  } catch (error) {
+    if (
+      isAgentKitError(error) &&
+      (error.code === "improve_evidence_store_not_configured" || error.code === "not_found")
+    ) {
+      return null;
+    }
+
+    throw error;
+  }
+}
+
 function printJson(value: unknown): void {
   console.log(JSON.stringify(value, null, 2));
 }
@@ -1221,6 +1637,20 @@ async function printJsonOrWrite(value: unknown, outFlag: string | boolean | unde
   console.log(`Deploy access token written to ${absolutePath}`);
 }
 
+function parsePositiveIntegerFlag(value: string | boolean | undefined, name: string, usage: string): number {
+  if (typeof value !== "string") {
+    throw new Error(`Missing ${name}. Usage: ${usage}`);
+  }
+
+  const parsed = Number(value);
+
+  if (!Number.isInteger(parsed) || parsed <= 0) {
+    throw new Error(`Invalid ${name}. Usage: ${usage}`);
+  }
+
+  return parsed;
+}
+
 function relativePath(root: string, path: string): string {
   const output = relative(root, path);
   return output.startsWith("..") ? path : output || ".";

package/src/create-project.ts

@@ -42,8 +42,7 @@ export async function createProject(options: CreateProjectOptions): Promise<Crea
   }
 
   for (const file of files) {
-    const absolutePath = join(projectPath, file.path);
-    const directory = absolutePath.slice(0, absolutePath.lastIndexOf("/"));
+    const { absolutePath, directory } = resolveProjectFilePath(projectPath, file.path);
 
     await mkdir(directory, { recursive: true });
     await writeFile(absolutePath, file.contents);
@@ -58,6 +57,18 @@ export async function createProject(options: CreateProjectOptions): Promise<Crea
   };
 }
 
+export function resolveProjectFilePath(
+  projectPath: string,
+  filePath: string,
+  pathApi: Pick<typeof import("node:path"), "dirname" | "join"> = { dirname, join },
+): { absolutePath: string; directory: string } {
+  const absolutePath = pathApi.join(projectPath, filePath);
+  return {
+    absolutePath,
+    directory: pathApi.dirname(absolutePath),
+  };
+}
+
 function resolveProjectTarget(options: CreateProjectOptions): { name: string; path: string; currentDirectory: boolean } {
   if (options.name === ".") {
     const path = resolve(options.cwd);