@andreprado/agentkit 0.1.0-alpha.15 → 0.1.0-alpha.17

package/src/runtime/config.ts

@@ -16,7 +16,11 @@ import type {
   TelegramAllowedUpdate,
   AgentKnowledgeEmbeddingProvider,
   ChannelBufferMode,
+  AgentIntegration,
+  ChannelAudioMode,
+  TranscriptionProviderName,
 } from "../index";
+import { MANAGED_COMPOSIO_TOOL_NAME } from "./integrations/composio";
 import { AgentKitError } from "./errors";
 import { isValidTimeZone } from "./prompt-context";
 
@@ -28,9 +32,14 @@ const TOOL_VISIBILITIES = new Set<ToolVisibility>(["user", "internal"]);
 const CHANNEL_TYPES = new Set<ChannelType>(["website", "telegram", "whatsapp"]);
 const CHANNEL_PROVIDERS = new Set<ChannelProvider>(["agentkit", "telegram", "zapster", "meta"]);
 const CHANNEL_BUFFER_MODES = new Set<ChannelBufferMode>(["off", "debounce"]);
+const CHANNEL_AUDIO_MODES = new Set<ChannelAudioMode>(["off", "reject", "transcribe"]);
 const TELEGRAM_ALLOWED_UPDATES = new Set<TelegramAllowedUpdate>(["message"]);
 const KNOWLEDGE_EMBEDDING_PROVIDERS = new Set<AgentKnowledgeEmbeddingProvider>(["none", "test", "openai"]);
+const TRANSCRIPTION_PROVIDERS = new Set<TranscriptionProviderName>(["test", "openai", "groq"]);
 const CHANNEL_NAME_PATTERN = /^[a-z][a-z0-9-]*$/;
+const INTEGRATION_NAME_PATTERN = /^[a-z][a-z0-9-]*$/;
+const COMPOSIO_TOOLKIT_PATTERN = /^[a-z][a-z0-9_-]*$/;
+const COMPOSIO_TOOL_SLUG_PATTERN = /^[A-Z][A-Z0-9_]*$/;
 const SECRET_NAME_PATTERN = /^[A-Z_][A-Z0-9_]*$/;
 const moduleDir = dirname(fileURLToPath(import.meta.url));
 let configImportNonce = 0;
@@ -230,7 +239,9 @@ function validateAgentConfig(value: unknown): asserts value is AgentConfig {
     }
   }
 
-  validateChannels(value.channels, value.runtime as AgentRuntime);
+  validateIntegrations(value.integrations, value.runtime as AgentRuntime, value.tools);
+  validateTranscriptionConfig(value.transcription);
+  validateChannels(value.channels, value.runtime as AgentRuntime, Boolean(value.transcription));
   validateKnowledgeConfig(value.knowledge);
 
   if (!isRecord(value.access) || !ACCESS_MODES.has(value.access.mode as AccessMode)) {
@@ -253,6 +264,169 @@ function validateAgentConfig(value: unknown): asserts value is AgentConfig {
   validateFilesConfig(value.storage.files);
 }
 
+function validateIntegrations(
+  value: unknown,
+  runtime: AgentRuntime,
+  tools: AgentConfig["tools"] = [],
+): void {
+  if (value === undefined) {
+    return;
+  }
+
+  if (!Array.isArray(value)) {
+    throw invalidConfig("integrations must be an array when provided.");
+  }
+
+  if (value.length > 0 && runtime !== "edge") {
+    throw invalidConfig('managed integrations require runtime: "edge". Use BYO defineTool wrappers for local-only integrations.');
+  }
+
+  let composioCount = 0;
+
+  for (const [index, integration] of value.entries()) {
+    if (!isRecord(integration)) {
+      throw invalidConfig(`integrations[${index}] must be an object.`);
+    }
+
+    if (integration.provider !== "composio" || integration.type !== "composio" || integration.mode !== "managed") {
+      throw invalidConfig(`integrations[${index}] must be composioManaged({...}) or { type: "composio", provider: "composio", mode: "managed", ... }.`);
+    }
+
+    composioCount += 1;
+    validateComposioManagedIntegration(integration as unknown as AgentIntegration, index);
+  }
+
+  if (composioCount > 1) {
+    throw invalidConfig("Only one managed Composio integration is supported per agent.");
+  }
+
+  const hasExplicitComposioTools = value.some(
+    (integration) => isRecord(integration) && integration.provider === "composio" && isRecord(integration.tools),
+  );
+
+  if (hasExplicitComposioTools && tools.some((tool) => isRecord(tool) && tool.name === MANAGED_COMPOSIO_TOOL_NAME)) {
+    throw invalidConfig(`tools must not declare reserved managed Composio tool name "${MANAGED_COMPOSIO_TOOL_NAME}".`);
+  }
+}
+
+function validateComposioManagedIntegration(value: AgentIntegration, index: number): void {
+  expectString(value.name, `integrations[${index}].name`);
+
+  if (!INTEGRATION_NAME_PATTERN.test(value.name)) {
+    throw invalidConfig(`integrations[${index}].name must start with a lowercase letter and contain only lowercase letters, numbers, and hyphens.`);
+  }
+
+  if (!Array.isArray(value.toolkits) || value.toolkits.length === 0) {
+    throw invalidConfig(`integrations[${index}].toolkits must include at least one Composio toolkit slug.`);
+  }
+
+  const toolkits = new Set<string>();
+
+  for (const [toolkitIndex, toolkit] of value.toolkits.entries()) {
+    expectString(toolkit, `integrations[${index}].toolkits[${toolkitIndex}]`);
+
+    if (!COMPOSIO_TOOLKIT_PATTERN.test(toolkit)) {
+      throw invalidConfig(`integrations[${index}].toolkits[${toolkitIndex}] must be a lowercase Composio toolkit slug.`);
+    }
+
+    if (toolkits.has(toolkit)) {
+      throw invalidConfig(`integrations[${index}].toolkits must not contain duplicates.`);
+    }
+
+    toolkits.add(toolkit);
+  }
+
+  if (value.tools !== undefined) {
+    if (!isRecord(value.tools)) {
+      throw invalidConfig(`integrations[${index}].tools must be an object when provided.`);
+    }
+
+    for (const [toolkit, tools] of Object.entries(value.tools)) {
+      if (!toolkits.has(toolkit)) {
+        throw invalidConfig(`integrations[${index}].tools.${toolkit} must reference a toolkit listed in integrations[${index}].toolkits.`);
+      }
+
+      if (!Array.isArray(tools) || tools.length === 0) {
+        throw invalidConfig(`integrations[${index}].tools.${toolkit} must include at least one Composio action slug.`);
+      }
+
+      const seenTools = new Set<string>();
+
+      for (const [toolIndex, tool] of tools.entries()) {
+        expectString(tool, `integrations[${index}].tools.${toolkit}[${toolIndex}]`);
+
+        if (!COMPOSIO_TOOL_SLUG_PATTERN.test(tool)) {
+          throw invalidConfig(`integrations[${index}].tools.${toolkit}[${toolIndex}] must be an uppercase Composio action slug.`);
+        }
+
+        if (seenTools.has(tool)) {
+          throw invalidConfig(`integrations[${index}].tools.${toolkit} must not contain duplicates.`);
+        }
+
+        seenTools.add(tool);
+      }
+    }
+  }
+
+  if (value.tags !== undefined) {
+    validateIntegrationTags(value.tags, `integrations[${index}].tags`);
+  }
+}
+
+function validateIntegrationTags(value: unknown, label: string): void {
+  if (!isRecord(value)) {
+    throw invalidConfig(`${label} must be an object when provided.`);
+  }
+
+  if (value.enable !== undefined) {
+    expectStringArray(value.enable, `${label}.enable`);
+  }
+
+  if (value.disable !== undefined) {
+    expectStringArray(value.disable, `${label}.disable`);
+  }
+}
+
+function validateTranscriptionConfig(value: unknown): void {
+  if (value === undefined) {
+    return;
+  }
+
+  if (!isRecord(value)) {
+    throw invalidConfig("transcription must be an object when provided.");
+  }
+
+  if (!TRANSCRIPTION_PROVIDERS.has(value.provider as TranscriptionProviderName)) {
+    throw invalidConfig(`transcription.provider must be one of: ${Array.from(TRANSCRIPTION_PROVIDERS).join(", ")}.`);
+  }
+
+  expectString(value.model, "transcription.model");
+
+  if (value.secret !== undefined) {
+    expectString(value.secret, "transcription.secret");
+    validateSecretNames([value.secret], "transcription.secret");
+  }
+
+  if (value.language !== undefined) {
+    expectString(value.language, "transcription.language");
+  }
+
+  if (value.prompt !== undefined) {
+    expectString(value.prompt, "transcription.prompt");
+  }
+
+  if (value.limits !== undefined) {
+    if (!isRecord(value.limits)) {
+      throw invalidConfig("transcription.limits must be an object when provided.");
+    }
+
+    validateOptionalPositiveInteger(value.limits.maxDurationSeconds, "transcription.limits.maxDurationSeconds");
+    validateOptionalPositiveInteger(value.limits.maxBytes, "transcription.limits.maxBytes");
+  }
+
+  validateOptionalPositiveInteger(value.rawAudioTtlSeconds, "transcription.rawAudioTtlSeconds");
+}
+
 function validateKnowledgeConfig(value: unknown): void {
   if (value === undefined) {
     return;
@@ -381,7 +555,7 @@ function validateKnowledgeRetrieval(value: unknown): void {
   }
 }
 
-function validateChannels(value: unknown, runtime: AgentRuntime): void {
+function validateChannels(value: unknown, runtime: AgentRuntime, hasTranscriptionConfig: boolean): void {
   if (value === undefined) {
     return;
   }
@@ -397,7 +571,7 @@ function validateChannels(value: unknown, runtime: AgentRuntime): void {
   const names = new Set<string>();
 
   for (const [index, channel] of value.entries()) {
-    validateChannel(channel, index);
+    validateChannel(channel, index, hasTranscriptionConfig);
 
     if (names.has(channel.name)) {
       throw invalidConfig(`channels[${index}].name must be unique within the Agent Capsule.`);
@@ -407,7 +581,7 @@ function validateChannels(value: unknown, runtime: AgentRuntime): void {
   }
 }
 
-function validateChannel(value: unknown, index: number): asserts value is AgentChannel {
+function validateChannel(value: unknown, index: number, hasTranscriptionConfig: boolean): asserts value is AgentChannel {
   if (!isRecord(value)) {
     throw invalidConfig(`channels[${index}] must be an object.`);
   }
@@ -432,6 +606,7 @@ function validateChannel(value: unknown, index: number): asserts value is AgentC
   validateSecretNames(value.secrets, `channels[${index}].secrets`);
   validateChannelLimits(value.limits, index);
   validateChannelBuffer(value.buffer, index);
+  validateChannelAudio(value.audio, index, hasTranscriptionConfig);
 
   if (value.type === "website") {
     validateWebsiteChannel(value, index);
@@ -446,6 +621,31 @@ function validateChannel(value: unknown, index: number): asserts value is AgentC
   validateWhatsappChannel(value, index);
 }
 
+function validateChannelAudio(value: unknown, index: number, hasTranscriptionConfig: boolean): void {
+  if (value === undefined) {
+    return;
+  }
+
+  if (!isRecord(value)) {
+    throw invalidConfig(`channels[${index}].audio must be an object when provided.`);
+  }
+
+  if (!CHANNEL_AUDIO_MODES.has(value.mode as ChannelAudioMode)) {
+    throw invalidConfig(`channels[${index}].audio.mode must be "off", "reject", or "transcribe".`);
+  }
+
+  if (value.mode === "transcribe" && !hasTranscriptionConfig) {
+    throw invalidConfig(`channels[${index}].audio.mode "transcribe" requires top-level transcription config.`);
+  }
+
+  if (value.mode !== "off") {
+    expectString(value.fallbackMessage, `channels[${index}].audio.fallbackMessage`);
+  }
+
+  validateOptionalPositiveInteger(value.maxDurationSeconds, `channels[${index}].audio.maxDurationSeconds`);
+  validateOptionalPositiveInteger(value.maxBytes, `channels[${index}].audio.maxBytes`);
+}
+
 function validateWebsiteChannel(value: Record<string, unknown>, index: number): void {
   if (value.provider !== "agentkit") {
     throw invalidConfig(`channels[${index}].provider must be "agentkit" when type is "website".`);
@@ -562,6 +762,16 @@ function validateSecretNames(secrets: string[], field: string): void {
   }
 }
 
+function validateOptionalPositiveInteger(value: unknown, field: string): void {
+  if (value === undefined) {
+    return;
+  }
+
+  if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) {
+    throw invalidConfig(`${field} must be a positive integer when provided.`);
+  }
+}
+
 function expectRequiredSecrets(value: unknown, requiredSecrets: string[], field: string): void {
   const declaredSecrets = new Set(Array.isArray(value) ? value : []);
   const missingSecrets = requiredSecrets.filter((secret) => !declaredSecrets.has(secret));

package/src/runtime/core/manifest.ts

@@ -2,8 +2,14 @@ import { relative, resolve } from "node:path";
 
 import type { AgentConfig, AgentTool } from "../../index";
 import type { LoadedAgentCapsule } from "../config";
+import {
+  createManagedComposioTools,
+  managedComposioRequiredSecrets,
+  resolveManagedComposioIntegrations,
+} from "../integrations/composio";
 import { resolveKnowledgeConfig } from "../knowledge/config";
 import { KNOWLEDGE_SEARCH_TOOL_NAME } from "../knowledge/tool";
+import { resolveTranscriptionConfig } from "../transcription";
 
 export type ManifestSchema = {
   source: string;
@@ -27,7 +33,28 @@ export type SharedAgentManifest = {
     permissions: string[];
     timeoutMs: number | null;
   }>;
+  integrations: Array<{
+    name: string;
+    provider: "composio";
+    mode: "managed";
+    toolkits: string[];
+    tools: string[];
+    userId: string;
+  }>;
   channels: NonNullable<AgentConfig["channels"]>;
+  transcription: {
+    enabled: boolean;
+    provider: string | null;
+    model: string | null;
+    secret: string | null;
+    language: string | null;
+    prompt: string | null;
+    limits: {
+      maxDurationSeconds: number | null;
+      maxBytes: number | null;
+    };
+    rawAudioTtlSeconds: number | null;
+  };
   knowledge: {
     enabled: boolean;
     sources: Array<{
@@ -80,6 +107,9 @@ export function buildSharedAgentManifest(
   const database = buildDatabaseManifest(capsule, options.databaseSchema ?? null);
   const files = buildFilesManifest(capsule);
   const knowledge = resolveKnowledgeConfig(capsule.config);
+  const integrationTools = createManagedComposioTools(capsule.config);
+  const tools = [...integrationTools, ...(capsule.config.tools ?? [])];
+  const transcription = resolveTranscriptionConfig(capsule.config.transcription);
 
   requiredSecrets.add("AGENTKIT_PRIVATE_TOKEN");
 
@@ -100,7 +130,15 @@ export function buildSharedAgentManifest(
     requiredSecrets.add(knowledge.embedding.secret);
   }
 
-  for (const tool of capsule.config.tools ?? []) {
+  if (transcription.secret) {
+    requiredSecrets.add(transcription.secret);
+  }
+
+  for (const secret of managedComposioRequiredSecrets(capsule.config)) {
+    requiredSecrets.add(secret);
+  }
+
+  for (const tool of tools) {
     for (const secret of tool.secrets ?? []) {
       requiredSecrets.add(secret);
     }
@@ -120,7 +158,7 @@ export function buildSharedAgentManifest(
     timeZone: capsule.config.timeZone ?? null,
     prompt: toCapsulePath(capsule, capsule.instructionsPath),
     secrets: Array.from(requiredSecrets).sort(),
-    tools: (capsule.config.tools ?? []).map((tool) => ({
+    tools: tools.map((tool) => ({
       name: tool.name,
       description: tool.description,
       inputSchema: tool.inputSchema,
@@ -129,7 +167,28 @@ export function buildSharedAgentManifest(
       permissions: [...(tool.permissions ?? [])].sort(),
       timeoutMs: tool.timeoutMs ?? null,
     })),
+    integrations: resolveManagedComposioIntegrations(capsule.config).map((integration) => ({
+      name: integration.name,
+      provider: integration.provider,
+      mode: integration.mode,
+      toolkits: integration.toolkits,
+      tools: integration.allowedTools,
+      userId: integration.userId,
+    })),
     channels: capsule.config.channels ?? [],
+    transcription: {
+      enabled: transcription.enabled,
+      provider: transcription.enabled ? transcription.provider : null,
+      model: transcription.enabled ? transcription.model : null,
+      secret: transcription.secret,
+      language: transcription.language,
+      prompt: transcription.prompt,
+      limits: {
+        maxDurationSeconds: transcription.limits.maxDurationSeconds ?? null,
+        maxBytes: transcription.limits.maxBytes ?? null,
+      },
+      rawAudioTtlSeconds: transcription.rawAudioTtlSeconds,
+    },
     knowledge: {
       enabled: knowledge.enabled,
       sources: knowledge.sources.map((source) => ({

package/src/runtime/deploy-readiness.ts

@@ -3,6 +3,10 @@ import { createHash } from "node:crypto";
 import { loadAgentCapsule } from "./config";
 import { buildSharedAgentManifest } from "./core/manifest";
 import { listCapsuleEnv } from "./env";
+import {
+  MANAGED_COMPOSIO_API_KEY_SECRET,
+  resolveManagedComposioIntegrations,
+} from "./integrations/composio";
 
 export type DeployReadinessSecretSource = {
   dotenv: boolean;
@@ -30,6 +34,14 @@ export type DeployReadinessContext = {
     embeddingProvider: string;
     databaseDriver: "none" | "turso";
   };
+  integrations: {
+    managedComposio: {
+      enabled: boolean;
+      toolkits: string[];
+      toolCount: number;
+      userId: string | null;
+    };
+  };
 };
 
 export async function loadDeployReadinessContext(
@@ -38,6 +50,7 @@ export async function loadDeployReadinessContext(
 ): Promise<DeployReadinessContext> {
   const capsule = await loadAgentCapsule(cwd);
   const manifest = buildSharedAgentManifest(capsule);
+  const managedComposio = resolveManagedComposioIntegrations(capsule.config)[0];
   const dotenvNames = new Set(await listCapsuleEnv(capsule.root));
   const requiredSecrets = manifest.secrets;
   const localSecrets: Record<string, DeployReadinessSecretSource> = {};
@@ -57,6 +70,7 @@ export async function loadDeployReadinessContext(
     isAgentKitManagedDeploySecret(name, {
       accessMode: capsule.config.access.mode,
       databaseDriver: manifest.storage.database.driver,
+      managedComposioEnabled: Boolean(managedComposio),
     }),
   );
   const managed = new Set(agentKitManagedSecrets);
@@ -82,6 +96,14 @@ export async function loadDeployReadinessContext(
       embeddingProvider: manifest.knowledge.embedding.provider,
       databaseDriver: manifest.storage.database.driver,
     },
+    integrations: {
+      managedComposio: {
+        enabled: Boolean(managedComposio),
+        toolkits: managedComposio?.toolkits ?? [],
+        toolCount: managedComposio?.allowedTools.length ?? 0,
+        userId: managedComposio?.userId ?? null,
+      },
+    },
   };
 }
 
@@ -91,7 +113,11 @@ export function stableCloudProjectId(name: string, accountId?: string | null): s
 
 function isAgentKitManagedDeploySecret(
   name: string,
-  context: { accessMode: DeployReadinessContext["accessMode"]; databaseDriver: "none" | "turso" },
+  context: {
+    accessMode: DeployReadinessContext["accessMode"];
+    databaseDriver: "none" | "turso";
+    managedComposioEnabled: boolean;
+  },
 ): boolean {
   if (context.databaseDriver === "turso" && (name === "TURSO_DATABASE_URL" || name === "TURSO_AUTH_TOKEN")) {
     return true;
@@ -101,5 +127,9 @@ function isAgentKitManagedDeploySecret(
     return true;
   }
 
+  if (context.managedComposioEnabled && name === MANAGED_COMPOSIO_API_KEY_SECRET) {
+    return true;
+  }
+
   return false;
 }

package/src/runtime/dev-server.ts

@@ -7,7 +7,7 @@ import { Readable } from "node:stream";
 
 import { openCapsuleStore, type ConversationRecord } from "../storage/sqlite";
 import type { AgentChannel, ChannelProvider, ChannelType } from "../index";
-import type { ChannelAdapter, RawWebhookEvent } from "./channels";
+import type { ChannelAdapter, NormalizedChannelMessage, RawWebhookEvent } from "./channels";
 import { telegramChannelAdapter } from "./channels/telegram";
 import { websiteChannelAdapter } from "./channels/website";
 import { metaWhatsappChannelAdapter } from "./channels/whatsapp-meta";
@@ -20,6 +20,7 @@ import { loadCapsuleEnv } from "./env";
 import { buildInspectState } from "./inspect";
 import { syncConfiguredKnowledgeSources } from "./knowledge/ingest";
 import { runToolFromCwd } from "./tool-runner";
+import { resolveTranscriptionConfig, transcribeAudio } from "./transcription";
 
 export type AgentDevServerOptions = {
   port?: number;
@@ -515,7 +516,9 @@ async function handlePortableChannel(
 
     channelRuntime.dedupeKeys.add(event.dedupeKey);
 
-    if (event.kind !== "message" || !event.message) {
+    const processedMessage = await processDevChannelMessage(capsule, adapter, channel, event.message, secrets);
+
+    if (event.kind !== "message" || !processedMessage) {
       deliveries.push({ state: "skipped", dedupeKey: event.dedupeKey, unsupportedReason: event.unsupportedReason });
       continue;
     }
@@ -531,7 +534,7 @@ async function handlePortableChannel(
       bufferDevChannelMessage(capsule, channelRuntime, {
         bufferKey: `${channel.name}:${event.externalIdentity.key}`,
         conversationId,
-        content: event.message.content,
+        content: processedMessage.content,
         receivedAt: event.receivedAt,
         buffer,
       });
@@ -545,7 +548,7 @@ async function handlePortableChannel(
     }
 
     const result = await runAgentMessage(capsule, {
-      message: event.message.content,
+      message: processedMessage.content,
       conversationId,
       signal: request.signal,
     });
@@ -560,6 +563,71 @@ async function handlePortableChannel(
   return jsonResponse({ deliveries });
 }
 
+async function processDevChannelMessage(
+  capsule: LoadedAgentCapsule,
+  adapter: ChannelAdapter,
+  channel: AgentChannel,
+  message: NormalizedChannelMessage | undefined,
+  secrets: Record<string, string>,
+): Promise<NormalizedChannelMessage | null> {
+  if (!message) {
+    return null;
+  }
+
+  if (message.contentType === "text" && message.content) {
+    return message;
+  }
+
+  if (message.contentType !== "audio" || !message.audio || channel.audio?.mode !== "transcribe") {
+    return null;
+  }
+
+  if (!adapter.downloadMedia) {
+    return null;
+  }
+
+  const env = await loadCapsuleEnv(capsule.root);
+  const transcriptionConfig = resolveTranscriptionConfig(capsule.config.transcription);
+  const secretName = transcriptionConfig.secret;
+  const download = await adapter.downloadMedia({
+    type: channel.type,
+    provider: channel.provider,
+    channelId: channel.name,
+    message,
+    secrets: {
+      ...secrets,
+      ...(secretName ? { [secretName]: env[secretName] ?? "" } : {}),
+      ...(env.AGENTKIT_TELEGRAM_API_BASE_URL ? { AGENTKIT_TELEGRAM_API_BASE_URL: env.AGENTKIT_TELEGRAM_API_BASE_URL } : {}),
+    },
+  });
+
+  if (!download.ok) {
+    return null;
+  }
+
+  const transcription = await transcribeAudio({
+    config: transcriptionConfig,
+    secrets: {
+      ...(secretName ? { [secretName]: env[secretName] ?? "" } : {}),
+    },
+    audio: download.audio,
+    filename: download.filename,
+    ...(download.mimeType ? { mimeType: download.mimeType } : {}),
+    ...(download.durationSeconds !== undefined ? { durationSeconds: download.durationSeconds } : {}),
+  });
+
+  if (!transcription.ok) {
+    return null;
+  }
+
+  return {
+    role: "user",
+    contentType: "text",
+    content: `The user sent an audio message.\n\nTranscript:\n${transcription.text}`,
+    providerMessageId: message.providerMessageId,
+  };
+}
+
 function bufferDevChannelMessage(
   capsule: LoadedAgentCapsule,
   runtime: DevChannelRuntimeState,