@andreprado/agentkit 0.1.0-alpha.15 → 0.1.0-alpha.16

package/docs/guides/add-channel.md

@@ -87,6 +87,62 @@ agentkit channels buffers clear support-whatsapp <conversation-id>
 agentkit channels buffers retry support-whatsapp <conversation-id>
 ```
 
+## Auto Transcribe Audio
+
+Use `transcription` at the agent level and `audio.mode: "transcribe"` on each channel that should accept voice notes or audio files.
+
+```ts
+export default defineAgent({
+  name: "support-agent",
+  runtime: "edge",
+  provider: { name: "openai", model: "gpt-5.4-mini" },
+  instructions: "./prompts/instructions.md",
+  transcription: {
+    provider: "groq",
+    model: "whisper-large-v3-turbo",
+    secret: "GROQ_API_KEY",
+    language: "pt",
+    limits: {
+      maxDurationSeconds: 180,
+      maxBytes: 20_000_000,
+    },
+  },
+  channels: [
+    telegramChannel({
+      name: "support-telegram",
+      audio: { mode: "transcribe" },
+    }),
+    whatsappChannel({
+      name: "support-whatsapp",
+      provider: "zapster",
+      audio: { mode: "transcribe" },
+    }),
+  ],
+  access: { mode: "public" },
+  storage: { driver: "agentkit" },
+});
+```
+
+Supported transcription providers in V1:
+
+| Provider | Default secret | Supported models |
+| --- | --- | --- |
+| `openai` | `OPENAI_API_KEY` | `gpt-4o-mini-transcribe`, `gpt-4o-transcribe`, `whisper-1` |
+| `groq` | `GROQ_API_KEY` | `whisper-large-v3-turbo`, `whisper-large-v3`, `distil-whisper-large-v3-en` |
+
+Audio transcription is paid by the capsule owner because AgentKit only passes through the configured provider secret. Hosted channel creation automatically requires the transcription secret when a channel enables `audio.mode: "transcribe"`.
+
+Processing order:
+
+1. Provider webhook is validated and deduped.
+2. Channel adapter normalizes the audio metadata.
+3. AgentKit records `audio_received` and enqueues a channel job before acknowledging the webhook.
+4. The retryable channel worker downloads the audio using the channel provider secret.
+5. The transcription adapter sends the file to the configured transcription provider.
+6. The agent receives a text message containing the transcript.
+
+V1 keeps the raw audio in memory for the request path and delivery metadata only records redacted status/error fields. `rawAudioTtlSeconds` is part of the manifest contract for future object-storage retention, but V1 does not persist raw audio by default.
+
 ## Safety Rules
 
 - Never put provider token values in `agentkit.config.ts`.
@@ -94,6 +150,7 @@ agentkit channels buffers retry support-whatsapp <conversation-id>
 - Treat channel webhook URLs as public transport endpoints. Provider validation or the AgentKit website channel token controls authenticity.
 - Keep channels separate from tools. Channels deliver user messages; tools let the agent call external systems.
 - Keep `maxMessages` and `maxChars` bounded so one burst cannot create an oversized prompt or unexpected model spend.
+- Keep `audio.limits` bounded so one voice note cannot create unexpected transcription spend.
 
 ## Verification
 
@@ -124,4 +181,10 @@ The provider webhook secret, token, or origin header does not match the managed
 `channel_limit_exceeded`:
 The channel daily message limit was reached. Website requests return `429`; Telegram and WhatsApp are acknowledged and skipped to avoid provider retry storms.
 
+`transcription_secret_missing`:
+Set the managed transcription secret declared by `agentkit inspect`, for example `OPENAI_API_KEY` or `GROQ_API_KEY`.
+
+`transcription_audio_format_unsupported`:
+The channel delivered an audio format the configured transcription provider does not accept. Telegram voice notes are OGG/Opus and work with Groq in V1; OpenAI accepts MP3, MP4, MPEG, MPGA, M4A, WAV, and WEBM in the AgentKit adapter.
+
 Buffered messages stay in `buffered` delivery state until the quiet window or max wait flushes them into one queued run.

package/docs/guides/channel-security.md

@@ -15,6 +15,7 @@ agentkit inspect
 agentkit channels status <name>
 agentkit channels deliveries show <delivery-id>
 bun test packages/agentkit/src/runtime/channels/adapters.test.ts
+bun test packages/agentkit/src/runtime/transcription.test.ts
 bun test packages/agentkit/src/runtime/deploy.test.ts
 ```
 
@@ -41,9 +42,12 @@ Zapster smoke also requires `ZAPSTER_API_KEY`, `ZAPSTER_INSTANCE_ID`, and `AGENT
 - Validate provider authenticity when the provider supports it.
 - Do not store channel plumbing in the user's Turso database.
 - Do not store raw webhook bodies in delivery records; store a SHA-256 hash.
+- Do not store raw audio in delivery records. V1 audio transcription downloads provider media into memory and stores only redacted metadata and state transitions.
 - Redact bearer tokens, bot tokens, signing secrets, provider API tokens, and phone numbers.
 - Inject only channel-declared secrets into adapter code.
+- Inject transcription secrets only when the deploy manifest declares transcription and the channel uses `audio.mode: "transcribe"`.
 - Prefer acknowledging Telegram/WhatsApp over retry storms when a channel-level limit is exceeded.
+- Keep audio duration and byte limits bounded before provider calls to avoid uncontrolled transcription spend.
 
 ## Minimal Working Example
 
@@ -54,17 +58,42 @@ telegramChannel({
 });
 ```
 
+With audio transcription:
+
+```ts
+export default defineAgent({
+  // ...
+  transcription: {
+    provider: "groq",
+    model: "whisper-large-v3-turbo",
+    secret: "GROQ_API_KEY",
+    limits: {
+      maxDurationSeconds: 180,
+      maxBytes: 20_000_000,
+    },
+  },
+  channels: [
+    telegramChannel({
+      name: "support-telegram",
+      audio: { mode: "transcribe" },
+    }),
+  ],
+});
+```
+
 The config contains secret names only. Hosted responses report:
 
 ```txt
 TELEGRAM_BOT_TOKEN: set
 TELEGRAM_WEBHOOK_SECRET: missing
+GROQ_API_KEY: set
 ```
 
 ## Verification
 
 ```sh
 bun test packages/agentkit/src/runtime/channels/adapters.test.ts
+bun test packages/agentkit/src/runtime/transcription.test.ts
 bun test packages/agentkit/src/runtime/deploy.test.ts
 npm run typecheck
 ```
@@ -79,3 +108,6 @@ Redact it to a stable partial form such as `5511******9999`.
 
 Webhook accepts invalid signatures or origin headers:
 Fix `verifyWebhook` for the adapter before enabling provider setup docs.
+
+Raw audio or transcript provider secret appears in output:
+Stop and add a regression test before changing behavior. Delivery APIs may include transcript text in normalized agent messages after successful transcription, but must never include raw bytes or provider secret values.

package/docs/guides/connect-telegram.md

@@ -27,6 +27,12 @@ TELEGRAM_BOT_TOKEN
 TELEGRAM_WEBHOOK_SECRET
 ```
 
+If Telegram audio transcription is enabled, also set the transcription provider secret declared by `agentkit inspect`, usually:
+
+```txt
+GROQ_API_KEY
+```
+
 ## Files Created Or Edited
 
 - `agentkit.config.ts`: `telegramChannel({ name: "support-telegram" })`.
@@ -66,6 +72,52 @@ telegramChannel({
 })
 ```
 
+## Auto Transcribe Telegram Audio
+
+Telegram voice notes arrive as OGG/Opus. In V1, use Groq for the most obvious Telegram voice-note path because the AgentKit Groq adapter accepts `audio/ogg`.
+
+```ts
+import { defineAgent, telegramChannel } from "@andreprado/agentkit";
+
+export default defineAgent({
+  name: "support-agent",
+  runtime: "edge",
+  provider: { name: "openai", model: "gpt-5.4-mini" },
+  instructions: "./prompts/instructions.md",
+  transcription: {
+    provider: "groq",
+    model: "whisper-large-v3-turbo",
+    secret: "GROQ_API_KEY",
+    language: "pt",
+    limits: {
+      maxDurationSeconds: 180,
+      maxBytes: 20_000_000,
+    },
+  },
+  channels: [
+    telegramChannel({
+      name: "support-telegram",
+      audio: {
+        mode: "transcribe",
+      },
+    }),
+  ],
+  access: { mode: "public" },
+  storage: { driver: "agentkit" },
+});
+```
+
+Processing order:
+
+1. AgentKit validates `X-Telegram-Bot-Api-Secret-Token`.
+2. Telegram `voice` or `audio` payloads become normalized audio messages.
+3. AgentKit records `audio_received` and enqueues a channel job before acknowledging Telegram.
+4. The retryable channel worker calls Telegram `getFile`, downloads the file with `TELEGRAM_BOT_TOKEN`, and does not log the token.
+5. AgentKit sends the audio bytes to the configured transcription provider using the user's managed secret.
+6. The agent run receives a text message with the transcript.
+
+OpenAI transcription can be used for Telegram files that arrive as MP3, MP4, MPEG, MPGA, M4A, WAV, or WEBM. Telegram voice notes are usually OGG/Opus, so they should use Groq in V1 unless the provider payload is converted before it reaches AgentKit.
+
 ## Setup Behavior
 
 `agentkit channels setup support-telegram` is read-only and prints the webhook URL.
@@ -108,3 +160,9 @@ The incoming Telegram secret token does not match `TELEGRAM_WEBHOOK_SECRET`.
 
 `channel_payload_invalid`:
 The update is malformed or is not a supported private text message.
+
+`transcription_secret_missing`:
+Set `GROQ_API_KEY` or the custom secret named in `transcription.secret` as a managed hosted secret.
+
+`transcription_audio_format_unsupported`:
+The configured transcription provider does not accept the Telegram file format. Use Groq for OGG/Opus voice notes in V1.

package/docs/guides/connect-whatsapp-zapster.md

@@ -33,6 +33,18 @@ Optional hardening secret:
 ZAPSTER_WEBHOOK_TOKEN
 ```
 
+If WhatsApp audio transcription is enabled, also set the transcription provider secret declared by `agentkit inspect`, usually:
+
+```txt
+OPENAI_API_KEY
+```
+
+or:
+
+```txt
+GROQ_API_KEY
+```
+
 ## Files Created Or Edited
 
 - `agentkit.config.ts`: `whatsappChannel({ name: "support-whatsapp", provider: "zapster" })`.
@@ -73,6 +85,53 @@ whatsappChannel({
 })
 ```
 
+## Auto Transcribe WhatsApp Audio
+
+Use `transcription` at the agent level and `audio.mode: "transcribe"` on the Zapster channel.
+
+```ts
+import { defineAgent, whatsappChannel } from "@andreprado/agentkit";
+
+export default defineAgent({
+  name: "support-agent",
+  runtime: "edge",
+  provider: { name: "openai", model: "gpt-5.4-mini" },
+  instructions: "./prompts/instructions.md",
+  transcription: {
+    provider: "openai",
+    model: "gpt-4o-mini-transcribe",
+    secret: "OPENAI_API_KEY",
+    language: "pt",
+    limits: {
+      maxDurationSeconds: 180,
+      maxBytes: 20_000_000,
+    },
+  },
+  channels: [
+    whatsappChannel({
+      name: "support-whatsapp",
+      provider: "zapster",
+      audio: {
+        mode: "transcribe",
+      },
+    }),
+  ],
+  access: { mode: "public" },
+  storage: { driver: "agentkit" },
+});
+```
+
+Processing order:
+
+1. AgentKit validates Zapster origin headers and the optional webhook token.
+2. Zapster audio payloads become normalized audio messages.
+3. AgentKit records `audio_received` and enqueues a channel job before acknowledging Zapster.
+4. The retryable channel worker downloads the media URL from a trusted Zapster HTTPS host using `ZAPSTER_API_KEY`.
+5. AgentKit sends the audio bytes to the configured transcription provider using the user's managed secret.
+6. The agent run receives a text message with the transcript.
+
+Zapster payloads must include a media download URL such as `audio.downloadUrl`, `audio.url`, `audio.mediaUrl`, or the snake_case equivalents. The URL must be HTTPS and hosted by Zapster; AgentKit rejects arbitrary webhook-provided hosts before sending `ZAPSTER_API_KEY`. If Zapster sends only a media ID without a download URL, AgentKit records `channel_audio_download_unavailable` and does not create an agent run for that audio in V1.
+
 ## Setup Behavior
 
 `agentkit channels setup support-whatsapp` prints the stable AgentKit webhook URL. Paste it into Zapster webhook settings.
@@ -125,3 +184,9 @@ Zapster is not sending the expected instance/webhook IDs, or the optional query
 
 `channel_unsupported_message_type` or skipped delivery:
 The inbound WhatsApp event was not supported text. Inspect the delivery record for provider metadata.
+
+`channel_audio_download_unavailable`:
+Zapster sent an audio event without a usable media download URL, or the URL was not an HTTPS Zapster media host. Configure Zapster to include a trusted Zapster media URL in webhook payloads, or add a Zapster media lookup adapter before enabling `audio.mode: "transcribe"`.
+
+`transcription_secret_missing`:
+Set `OPENAI_API_KEY`, `GROQ_API_KEY`, or the custom secret named in `transcription.secret` as a managed hosted secret.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andreprado/agentkit",
-  "version": "0.1.0-alpha.15",
+  "version": "0.1.0-alpha.16",
   "private": false,
   "type": "module",
   "repository": {

package/src/cli/commands/channels.ts

@@ -20,6 +20,7 @@ type CloudChannel = {
   webhook_url: string;
   required_secrets: Array<{ name: string; status: string }>;
   buffer?: AgentChannel["buffer"];
+  audio?: AgentChannel["audio"];
 };
 
 type CloudDelivery = {
@@ -386,6 +387,7 @@ async function createHostedChannel(
     secrets: localChannel?.secrets ?? defaultChannelSecretsForCli(type, provider),
     ...(localChannel?.limits ? { limits: channelLimitsForApi(localChannel.limits) } : {}),
     ...(localChannel?.buffer ? { buffer: localChannel.buffer } : {}),
+    ...(localChannel?.audio ? { audio: localChannel.audio } : {}),
   };
 
   return cloudPost<{ channel: CloudChannel }>(

package/src/index.ts

@@ -6,6 +6,8 @@ export type ChannelType = "website" | "telegram" | "whatsapp";
 export type ChannelProvider = "agentkit" | "telegram" | "zapster" | "meta";
 export type WhatsappChannelProvider = "zapster" | "meta";
 export type TelegramAllowedUpdate = "message";
+export type ChannelAudioMode = "off" | "reject" | "transcribe";
+export type TranscriptionProviderName = "test" | "openai" | "groq";
 
 export type AgentProvider = {
   name: AgentProviderName;
@@ -39,6 +41,45 @@ export type ChannelBufferConfig =
       maxChars: number;
     };
 
+export type ChannelAudioInput = {
+  mode?: ChannelAudioMode;
+  fallbackMessage?: string;
+  maxDurationSeconds?: number;
+  maxBytes?: number;
+};
+
+export type ChannelAudioConfig =
+  | {
+      mode: "off";
+    }
+  | {
+      mode: "reject";
+      fallbackMessage: string;
+      maxDurationSeconds?: number;
+      maxBytes?: number;
+    }
+  | {
+      mode: "transcribe";
+      fallbackMessage: string;
+      maxDurationSeconds?: number;
+      maxBytes?: number;
+    };
+
+export type AgentTranscriptionLimits = {
+  maxDurationSeconds?: number;
+  maxBytes?: number;
+};
+
+export type AgentTranscriptionConfig = {
+  provider: TranscriptionProviderName;
+  model: string;
+  secret?: string;
+  language?: string;
+  prompt?: string;
+  limits?: AgentTranscriptionLimits;
+  rawAudioTtlSeconds?: number;
+};
+
 export type WebsiteChannelAccess = {
   mode: "token";
 };
@@ -49,6 +90,7 @@ export type WebsiteChannelInput = {
   secrets?: string[];
   limits?: ChannelLimits;
   buffer?: ChannelBufferInput;
+  audio?: ChannelAudioInput;
 };
 
 export type WebsiteChannelConfig = {
@@ -59,6 +101,7 @@ export type WebsiteChannelConfig = {
   secrets: string[];
   limits?: ChannelLimits;
   buffer?: ChannelBufferConfig;
+  audio?: ChannelAudioConfig;
 };
 
 export type TelegramChannelInput = {
@@ -67,6 +110,7 @@ export type TelegramChannelInput = {
   allowedUpdates?: TelegramAllowedUpdate[];
   limits?: ChannelLimits;
   buffer?: ChannelBufferInput;
+  audio?: ChannelAudioInput;
 };
 
 export type TelegramChannelConfig = {
@@ -77,6 +121,7 @@ export type TelegramChannelConfig = {
   allowedUpdates: TelegramAllowedUpdate[];
   limits?: ChannelLimits;
   buffer?: ChannelBufferConfig;
+  audio?: ChannelAudioConfig;
 };
 
 export type WhatsappChannelInput = {
@@ -85,6 +130,7 @@ export type WhatsappChannelInput = {
   secrets?: string[];
   limits?: ChannelLimits;
   buffer?: ChannelBufferInput;
+  audio?: ChannelAudioInput;
 };
 
 export type WhatsappChannelConfig = {
@@ -94,6 +140,7 @@ export type WhatsappChannelConfig = {
   secrets: string[];
   limits?: ChannelLimits;
   buffer?: ChannelBufferConfig;
+  audio?: ChannelAudioConfig;
 };
 
 export type AgentChannel = WebsiteChannelConfig | TelegramChannelConfig | WhatsappChannelConfig;
@@ -270,6 +317,7 @@ export type AgentConfig = {
   secrets: string[];
   tools?: AgentTool[];
   channels?: AgentChannel[];
+  transcription?: AgentTranscriptionConfig;
   knowledge?: AgentKnowledgeConfig;
   access: {
     mode: AccessMode;
@@ -299,6 +347,7 @@ export function websiteChannel(input: WebsiteChannelInput): WebsiteChannelConfig
     secrets: normalizeChannelSecrets(input.secrets ?? ["AGENTKIT_WEBSITE_CHANNEL_TOKEN"], "website channel"),
     ...(input.limits ? { limits: input.limits } : {}),
     ...(input.buffer ? { buffer: normalizeChannelBuffer(input.buffer, "website channel") } : {}),
+    ...(input.audio ? { audio: normalizeChannelAudio(input.audio, "website channel") } : {}),
   };
 }
 
@@ -314,6 +363,7 @@ export function telegramChannel(input: TelegramChannelInput): TelegramChannelCon
     allowedUpdates: input.allowedUpdates ?? ["message"],
     ...(input.limits ? { limits: input.limits } : {}),
     ...(input.buffer ? { buffer: normalizeChannelBuffer(input.buffer, "telegram channel") } : {}),
+    ...(input.audio ? { audio: normalizeChannelAudio(input.audio, "telegram channel") } : {}),
   };
 }
 
@@ -328,6 +378,7 @@ export function whatsappChannel(input: WhatsappChannelInput): WhatsappChannelCon
     ),
     ...(input.limits ? { limits: input.limits } : {}),
     ...(input.buffer ? { buffer: normalizeChannelBuffer(input.buffer, `${input.provider} whatsapp channel`) } : {}),
+    ...(input.audio ? { audio: normalizeChannelAudio(input.audio, `${input.provider} whatsapp channel`) } : {}),
   };
 }
 
@@ -382,6 +433,33 @@ function normalizeChannelBuffer(input: ChannelBufferInput, context: string): Cha
   return buffer;
 }
 
+function normalizeChannelAudio(input: ChannelAudioInput, context: string): ChannelAudioConfig {
+  const mode = input.mode ?? "transcribe";
+
+  if (mode !== "off" && mode !== "reject" && mode !== "transcribe") {
+    throw new Error(`${context} audio.mode must be "off", "reject", or "transcribe".`);
+  }
+
+  if (mode === "off") {
+    return { mode: "off" };
+  }
+
+  if (input.maxDurationSeconds !== undefined) {
+    validatePositiveInteger(input.maxDurationSeconds, `${context} audio.maxDurationSeconds`);
+  }
+
+  if (input.maxBytes !== undefined) {
+    validatePositiveInteger(input.maxBytes, `${context} audio.maxBytes`);
+  }
+
+  return {
+    mode,
+    fallbackMessage: input.fallbackMessage ?? "I received an audio message, but I could not transcribe it. Please send the message as text.",
+    ...(input.maxDurationSeconds !== undefined ? { maxDurationSeconds: input.maxDurationSeconds } : {}),
+    ...(input.maxBytes !== undefined ? { maxBytes: input.maxBytes } : {}),
+  };
+}
+
 function validatePositiveInteger(value: number, field: string): void {
   if (!Number.isInteger(value) || value <= 0) {
     throw new Error(`${field} must be a positive integer.`);
@@ -413,10 +491,13 @@ export { AgentKitError } from "./runtime/errors";
 export type {
   ChannelAdapter,
   ChannelAdapterId,
+  ChannelAudioAttachment,
   ChannelDeliveryDirection,
   ChannelDeliveryState,
   ChannelEventKind,
   ChannelExternalIdentity,
+  ChannelMediaDownloadInput,
+  ChannelMediaDownloadResult,
   ChannelMessageContentType,
   ChannelMessageRole,
   ChannelSendInput,
@@ -429,3 +510,8 @@ export type {
   WebhookVerificationInput,
   WebhookVerificationResult,
 } from "./runtime/channels";
+export type {
+  TranscriptionAdapter,
+  TranscriptionInput,
+  TranscriptionResult,
+} from "./runtime/transcription";

package/src/runtime/channel-test-harness.ts

@@ -8,9 +8,11 @@ import type { NormalizedChannelEvent, RawWebhookEvent } from "./channels";
 export type ChannelFixtureName =
   | "website-message"
   | "telegram-message"
+  | "telegram-audio-message"
   | "telegram-duplicate-message"
   | "telegram-malformed-payload"
   | "zapster-message"
+  | "zapster-audio-message"
   | "zapster-duplicate-message"
   | "zapster-unsupported-media";