@andreprado/agentkit 0.1.0-alpha.15 → 0.1.0-alpha.16

package/src/runtime/dev-server.ts

@@ -7,7 +7,7 @@ import { Readable } from "node:stream";
 
 import { openCapsuleStore, type ConversationRecord } from "../storage/sqlite";
 import type { AgentChannel, ChannelProvider, ChannelType } from "../index";
-import type { ChannelAdapter, RawWebhookEvent } from "./channels";
+import type { ChannelAdapter, NormalizedChannelMessage, RawWebhookEvent } from "./channels";
 import { telegramChannelAdapter } from "./channels/telegram";
 import { websiteChannelAdapter } from "./channels/website";
 import { metaWhatsappChannelAdapter } from "./channels/whatsapp-meta";
@@ -20,6 +20,7 @@ import { loadCapsuleEnv } from "./env";
 import { buildInspectState } from "./inspect";
 import { syncConfiguredKnowledgeSources } from "./knowledge/ingest";
 import { runToolFromCwd } from "./tool-runner";
+import { resolveTranscriptionConfig, transcribeAudio } from "./transcription";
 
 export type AgentDevServerOptions = {
   port?: number;
@@ -515,7 +516,9 @@ async function handlePortableChannel(
 
     channelRuntime.dedupeKeys.add(event.dedupeKey);
 
-    if (event.kind !== "message" || !event.message) {
+    const processedMessage = await processDevChannelMessage(capsule, adapter, channel, event.message, secrets);
+
+    if (event.kind !== "message" || !processedMessage) {
       deliveries.push({ state: "skipped", dedupeKey: event.dedupeKey, unsupportedReason: event.unsupportedReason });
       continue;
     }
@@ -531,7 +534,7 @@ async function handlePortableChannel(
       bufferDevChannelMessage(capsule, channelRuntime, {
         bufferKey: `${channel.name}:${event.externalIdentity.key}`,
         conversationId,
-        content: event.message.content,
+        content: processedMessage.content,
         receivedAt: event.receivedAt,
         buffer,
       });
@@ -545,7 +548,7 @@ async function handlePortableChannel(
     }
 
     const result = await runAgentMessage(capsule, {
-      message: event.message.content,
+      message: processedMessage.content,
       conversationId,
       signal: request.signal,
     });
@@ -560,6 +563,71 @@ async function handlePortableChannel(
   return jsonResponse({ deliveries });
 }
 
+async function processDevChannelMessage(
+  capsule: LoadedAgentCapsule,
+  adapter: ChannelAdapter,
+  channel: AgentChannel,
+  message: NormalizedChannelMessage | undefined,
+  secrets: Record<string, string>,
+): Promise<NormalizedChannelMessage | null> {
+  if (!message) {
+    return null;
+  }
+
+  if (message.contentType === "text" && message.content) {
+    return message;
+  }
+
+  if (message.contentType !== "audio" || !message.audio || channel.audio?.mode !== "transcribe") {
+    return null;
+  }
+
+  if (!adapter.downloadMedia) {
+    return null;
+  }
+
+  const env = await loadCapsuleEnv(capsule.root);
+  const transcriptionConfig = resolveTranscriptionConfig(capsule.config.transcription);
+  const secretName = transcriptionConfig.secret;
+  const download = await adapter.downloadMedia({
+    type: channel.type,
+    provider: channel.provider,
+    channelId: channel.name,
+    message,
+    secrets: {
+      ...secrets,
+      ...(secretName ? { [secretName]: env[secretName] ?? "" } : {}),
+      ...(env.AGENTKIT_TELEGRAM_API_BASE_URL ? { AGENTKIT_TELEGRAM_API_BASE_URL: env.AGENTKIT_TELEGRAM_API_BASE_URL } : {}),
+    },
+  });
+
+  if (!download.ok) {
+    return null;
+  }
+
+  const transcription = await transcribeAudio({
+    config: transcriptionConfig,
+    secrets: {
+      ...(secretName ? { [secretName]: env[secretName] ?? "" } : {}),
+    },
+    audio: download.audio,
+    filename: download.filename,
+    ...(download.mimeType ? { mimeType: download.mimeType } : {}),
+    ...(download.durationSeconds !== undefined ? { durationSeconds: download.durationSeconds } : {}),
+  });
+
+  if (!transcription.ok) {
+    return null;
+  }
+
+  return {
+    role: "user",
+    contentType: "text",
+    content: `The user sent an audio message.\n\nTranscript:\n${transcription.text}`,
+    providerMessageId: message.providerMessageId,
+  };
+}
+
 function bufferDevChannelMessage(
   capsule: LoadedAgentCapsule,
   runtime: DevChannelRuntimeState,

package/src/runtime/inspect.ts

@@ -7,6 +7,7 @@ import { loadCapsuleEnv } from "./env";
 import { resolveKnowledgeConfig } from "./knowledge/config";
 import { KNOWLEDGE_SEARCH_TOOL_NAME } from "./knowledge/tool";
 import { resolveRuntimeTimeZone } from "./prompt-context";
+import { resolveTranscriptionConfig } from "./transcription";
 
 export type SecretState = "set" | "missing";
 
@@ -21,6 +22,19 @@ export type AgentInspectState = {
   prompt: string;
   tools: string[];
   channels: AgentInspectChannel[];
+  transcription: {
+    enabled: boolean;
+    provider: string | null;
+    model: string | null;
+    secret: string | null;
+    language: string | null;
+    prompt: string | null;
+    limits: {
+      maxDurationSeconds: number | null;
+      maxBytes: number | null;
+    };
+    rawAudioTtlSeconds: number | null;
+  };
   knowledge: {
     enabled: boolean;
     sources: string[];
@@ -80,6 +94,7 @@ export type AgentInspectChannel = {
   provider: AgentChannel["provider"];
   secrets: string[];
   buffer?: AgentChannel["buffer"];
+  audio?: AgentChannel["audio"];
 };
 
 export async function inspectAgentCapsule(
@@ -96,6 +111,7 @@ export function buildInspectState(
 ): AgentInspectState {
   const declaredSecrets = new Set(capsule.config.secrets);
   const knowledge = resolveKnowledgeConfig(capsule.config);
+  const transcription = resolveTranscriptionConfig(capsule.config.transcription);
 
   for (const tool of capsule.config.tools ?? []) {
     for (const secret of tool.secrets ?? []) {
@@ -113,6 +129,10 @@ export function buildInspectState(
     declaredSecrets.add(knowledge.embedding.secret);
   }
 
+  if (transcription.secret) {
+    declaredSecrets.add(transcription.secret);
+  }
+
   const managedSecrets = managedCloudSecrets(capsule);
   const userSecrets = Array.from(declaredSecrets).sort();
   const secrets: Record<string, SecretState> = {};
@@ -142,7 +162,21 @@ export function buildInspectState(
       provider: channel.provider,
       secrets: [...channel.secrets],
       ...(channel.buffer ? { buffer: channel.buffer } : {}),
+      ...(channel.audio ? { audio: channel.audio } : {}),
     })),
+    transcription: {
+      enabled: transcription.enabled,
+      provider: transcription.enabled ? transcription.provider : null,
+      model: transcription.enabled ? transcription.model : null,
+      secret: transcription.secret,
+      language: transcription.language,
+      prompt: transcription.prompt,
+      limits: {
+        maxDurationSeconds: transcription.limits.maxDurationSeconds ?? null,
+        maxBytes: transcription.limits.maxBytes ?? null,
+      },
+      rawAudioTtlSeconds: transcription.rawAudioTtlSeconds,
+    },
     knowledge: {
       enabled: knowledge.enabled,
       sources: knowledge.sources.map((source) => source.value),

package/src/runtime/targets/cloudflare/build.ts

@@ -37,6 +37,7 @@ export type AgentManifest = {
   secrets: SharedAgentManifest["secrets"];
   tools: SharedAgentManifest["tools"];
   channels: SharedAgentManifest["channels"];
+  transcription: SharedAgentManifest["transcription"];
   knowledge: SharedAgentManifest["knowledge"];
   access: SharedAgentManifest["access"];
   storage: {
@@ -198,6 +199,7 @@ function buildManifest(
     secrets: shared.secrets,
     tools: shared.tools,
     channels: shared.channels,
+    transcription: shared.transcription,
     knowledge: shared.knowledge,
     access: shared.access,
     storage: {

package/src/runtime/transcription.ts

@@ -0,0 +1,483 @@
+import type { AgentTranscriptionConfig, TranscriptionProviderName } from "../index";
+
+export type ResolvedTranscriptionConfig = {
+  enabled: boolean;
+  provider: TranscriptionProviderName;
+  model: string;
+  secret: string | null;
+  language: string | null;
+  prompt: string | null;
+  limits: {
+    maxDurationSeconds?: number;
+    maxBytes?: number;
+  };
+  rawAudioTtlSeconds: number | null;
+};
+
+export type TranscriptionInput = {
+  config: ResolvedTranscriptionConfig;
+  secrets: Record<string, string>;
+  audio: Uint8Array;
+  filename: string;
+  mimeType?: string;
+  durationSeconds?: number;
+  providerMetadata?: Record<string, unknown>;
+};
+
+export type TranscriptionFetch = (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
+
+export type TranscriptionResult =
+  | {
+      ok: true;
+      text: string;
+      provider: TranscriptionProviderName;
+      model: string;
+      language?: string;
+      durationSeconds?: number;
+      providerMetadata?: Record<string, unknown>;
+    }
+  | {
+      ok: false;
+      retryable: boolean;
+      code:
+        | "transcription_disabled"
+        | "transcription_secret_missing"
+        | "transcription_model_unsupported"
+        | "transcription_audio_too_large"
+        | "transcription_audio_too_long"
+        | "transcription_audio_format_unsupported"
+        | "transcription_provider_unavailable"
+        | "transcription_failed";
+      message: string;
+      provider?: TranscriptionProviderName;
+      model?: string;
+      providerMetadata?: Record<string, unknown>;
+    };
+
+export type TranscriptionAdapter = {
+  provider: TranscriptionProviderName;
+  models: string[];
+  supportedMimeTypes: string[];
+  supportedExtensions: string[];
+  maxBytes: number;
+  requiredSecret(config: ResolvedTranscriptionConfig): string | null;
+  transcribe(input: TranscriptionInput, fetcher?: TranscriptionFetch): Promise<TranscriptionResult>;
+};
+
+const DEFAULT_MAX_BYTES = 25_000_000;
+const OPENAI_MODELS = ["gpt-4o-mini-transcribe", "gpt-4o-transcribe", "whisper-1"];
+const GROQ_MODELS = ["whisper-large-v3-turbo", "whisper-large-v3", "distil-whisper-large-v3-en"];
+const OPENAI_EXTENSIONS = ["mp3", "mp4", "mpeg", "mpga", "m4a", "wav", "webm"];
+const GROQ_EXTENSIONS = ["flac", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "wav", "webm"];
+const OPENAI_MIME_TYPES = [
+  "audio/mpeg",
+  "audio/mp3",
+  "audio/mp4",
+  "audio/mpga",
+  "audio/m4a",
+  "audio/wav",
+  "audio/webm",
+  "video/mp4",
+];
+const GROQ_MIME_TYPES = [
+  ...OPENAI_MIME_TYPES,
+  "audio/flac",
+  "audio/ogg",
+  "audio/opus",
+  "application/ogg",
+];
+
+export function resolveTranscriptionConfig(config: AgentTranscriptionConfig | undefined): ResolvedTranscriptionConfig {
+  if (!config) {
+    return {
+      enabled: false,
+      provider: "test",
+      model: "fake",
+      secret: null,
+      language: null,
+      prompt: null,
+      limits: {},
+      rawAudioTtlSeconds: null,
+    };
+  }
+
+  return {
+    enabled: true,
+    provider: config.provider,
+    model: config.model,
+    secret: config.secret ?? defaultTranscriptionSecret(config.provider),
+    language: config.language ?? null,
+    prompt: config.prompt ?? null,
+    limits: {
+      ...(config.limits?.maxDurationSeconds !== undefined
+        ? { maxDurationSeconds: config.limits.maxDurationSeconds }
+        : {}),
+      ...(config.limits?.maxBytes !== undefined ? { maxBytes: config.limits.maxBytes } : {}),
+    },
+    rawAudioTtlSeconds: config.rawAudioTtlSeconds ?? 3600,
+  };
+}
+
+export async function transcribeAudio(
+  input: TranscriptionInput,
+  fetcher: TranscriptionFetch = fetch,
+): Promise<TranscriptionResult> {
+  if (!input.config.enabled) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_disabled",
+      message: "Audio transcription is not configured for this Agent Capsule.",
+    };
+  }
+
+  const adapter = transcriptionAdapterFor(input.config.provider);
+
+  if (!adapter) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_model_unsupported",
+      message: `No transcription adapter is registered for ${input.config.provider}.`,
+      provider: input.config.provider,
+      model: input.config.model,
+    };
+  }
+
+  const validation = validateTranscriptionInput(adapter, input);
+
+  if (!validation.ok) {
+    return validation;
+  }
+
+  return adapter.transcribe(input, fetcher);
+}
+
+export function transcriptionAdapterFor(provider: TranscriptionProviderName): TranscriptionAdapter | null {
+  if (provider === "test") {
+    return testTranscriptionAdapter;
+  }
+
+  if (provider === "openai") {
+    return openaiTranscriptionAdapter;
+  }
+
+  if (provider === "groq") {
+    return groqTranscriptionAdapter;
+  }
+
+  return null;
+}
+
+export function defaultTranscriptionSecret(provider: TranscriptionProviderName): string | null {
+  if (provider === "openai") {
+    return "OPENAI_API_KEY";
+  }
+
+  if (provider === "groq") {
+    return "GROQ_API_KEY";
+  }
+
+  return null;
+}
+
+function validateTranscriptionInput(adapter: TranscriptionAdapter, input: TranscriptionInput): TranscriptionResult {
+  if (!adapter.models.includes(input.config.model)) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_model_unsupported",
+      message: `${input.config.provider} transcription model ${input.config.model} is not supported by AgentKit.`,
+      provider: input.config.provider,
+      model: input.config.model,
+    };
+  }
+
+  const maxBytes = Math.min(input.config.limits.maxBytes ?? adapter.maxBytes, adapter.maxBytes);
+  if (input.audio.byteLength > maxBytes) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_audio_too_large",
+      message: `Audio file is ${input.audio.byteLength} bytes, which exceeds the configured ${maxBytes} byte limit.`,
+      provider: input.config.provider,
+      model: input.config.model,
+    };
+  }
+
+  if (
+    input.durationSeconds !== undefined &&
+    input.config.limits.maxDurationSeconds !== undefined &&
+    input.durationSeconds > input.config.limits.maxDurationSeconds
+  ) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_audio_too_long",
+      message: `Audio is ${input.durationSeconds}s, which exceeds the configured ${input.config.limits.maxDurationSeconds}s limit.`,
+      provider: input.config.provider,
+      model: input.config.model,
+    };
+  }
+
+  if (!isSupportedAudioFormat(adapter, input.filename, input.mimeType)) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_audio_format_unsupported",
+      message: `${input.config.provider} does not support audio format ${input.mimeType ?? extensionFor(input.filename) ?? "unknown"}.`,
+      provider: input.config.provider,
+      model: input.config.model,
+    };
+  }
+
+  const secret = adapter.requiredSecret(input.config);
+  if (secret && !input.secrets[secret]) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_secret_missing",
+      message: `Secret ${secret} is not set for audio transcription.`,
+      provider: input.config.provider,
+      model: input.config.model,
+    };
+  }
+
+  return {
+    ok: true,
+    text: "",
+    provider: input.config.provider,
+    model: input.config.model,
+  };
+}
+
+const testTranscriptionAdapter: TranscriptionAdapter = {
+  provider: "test",
+  models: ["fake"],
+  supportedMimeTypes: ["audio/wav", "audio/ogg", "audio/mpeg", "audio/webm", "application/octet-stream"],
+  supportedExtensions: ["wav", "ogg", "mp3", "webm", "bin"],
+  maxBytes: DEFAULT_MAX_BYTES,
+  requiredSecret() {
+    return null;
+  },
+  async transcribe(input) {
+    const text = decodeFixtureTranscript(input.audio) ?? "fake audio transcript";
+
+    return {
+      ok: true,
+      text,
+      provider: "test",
+      model: input.config.model,
+      ...(input.config.language ? { language: input.config.language } : {}),
+      ...(input.durationSeconds !== undefined ? { durationSeconds: input.durationSeconds } : {}),
+    };
+  },
+};
+
+const openaiTranscriptionAdapter: TranscriptionAdapter = {
+  provider: "openai",
+  models: OPENAI_MODELS,
+  supportedMimeTypes: OPENAI_MIME_TYPES,
+  supportedExtensions: OPENAI_EXTENSIONS,
+  maxBytes: DEFAULT_MAX_BYTES,
+  requiredSecret(config) {
+    return config.secret;
+  },
+  async transcribe(input, fetcher = fetch) {
+    return transcribeViaOpenAiCompatibleEndpoint({
+      input,
+      fetcher,
+      url: "https://api.openai.com/v1/audio/transcriptions",
+      apiKey: input.secrets[input.config.secret ?? ""],
+      provider: "openai",
+    });
+  },
+};
+
+const groqTranscriptionAdapter: TranscriptionAdapter = {
+  provider: "groq",
+  models: GROQ_MODELS,
+  supportedMimeTypes: GROQ_MIME_TYPES,
+  supportedExtensions: GROQ_EXTENSIONS,
+  maxBytes: DEFAULT_MAX_BYTES,
+  requiredSecret(config) {
+    return config.secret;
+  },
+  async transcribe(input, fetcher = fetch) {
+    return transcribeViaOpenAiCompatibleEndpoint({
+      input,
+      fetcher,
+      url: "https://api.groq.com/openai/v1/audio/transcriptions",
+      apiKey: input.secrets[input.config.secret ?? ""],
+      provider: "groq",
+    });
+  },
+};
+
+async function transcribeViaOpenAiCompatibleEndpoint(input: {
+  input: TranscriptionInput;
+  fetcher: TranscriptionFetch;
+  url: string;
+  apiKey: string | undefined;
+  provider: TranscriptionProviderName;
+}): Promise<TranscriptionResult> {
+  if (!input.apiKey) {
+    const secret = input.input.config.secret ?? defaultTranscriptionSecret(input.provider) ?? "TRANSCRIPTION_API_KEY";
+
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_secret_missing",
+      message: `Secret ${secret} is not set for audio transcription.`,
+      provider: input.provider,
+      model: input.input.config.model,
+    };
+  }
+
+  const form = new FormData();
+  form.set("model", input.input.config.model);
+  form.set(
+    "file",
+    new Blob([arrayBufferForBlob(input.input.audio)], { type: input.input.mimeType ?? "application/octet-stream" }),
+    input.input.filename,
+  );
+  form.set("response_format", "json");
+
+  if (input.input.config.language) {
+    form.set("language", input.input.config.language);
+  }
+
+  if (input.input.config.prompt) {
+    form.set("prompt", input.input.config.prompt);
+  }
+
+  let response: Response;
+
+  try {
+    response = await input.fetcher(input.url, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${input.apiKey}`,
+      },
+      body: form,
+    });
+  } catch (error) {
+    return {
+      ok: false,
+      retryable: true,
+      code: "transcription_provider_unavailable",
+      message: `Transcription provider request failed before a response: ${redactSecret(
+        error instanceof Error ? error.message : String(error),
+        input.apiKey,
+      )}`,
+      provider: input.provider,
+      model: input.input.config.model,
+    };
+  }
+
+  const payload = await response.json().catch(() => null);
+
+  if (!response.ok) {
+    const message = readProviderError(payload) ?? `Transcription provider returned HTTP ${response.status}.`;
+
+    return {
+      ok: false,
+      retryable: response.status === 429 || response.status >= 500,
+      code: response.status === 429 || response.status >= 500 ? "transcription_provider_unavailable" : "transcription_failed",
+      message: redactSecret(message, input.apiKey),
+      provider: input.provider,
+      model: input.input.config.model,
+      providerMetadata: {
+        status: response.status,
+      },
+    };
+  }
+
+  const text = isRecord(payload) && typeof payload.text === "string" ? payload.text.trim() : "";
+
+  if (!text) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_failed",
+      message: "Transcription provider returned an empty transcript.",
+      provider: input.provider,
+      model: input.input.config.model,
+      providerMetadata: {
+        status: response.status,
+      },
+    };
+  }
+
+  return {
+    ok: true,
+    text,
+    provider: input.provider,
+    model: input.input.config.model,
+    ...(input.input.config.language ? { language: input.input.config.language } : {}),
+    ...(input.input.durationSeconds !== undefined ? { durationSeconds: input.input.durationSeconds } : {}),
+    providerMetadata: {
+      status: response.status,
+    },
+  };
+}
+
+function isSupportedAudioFormat(adapter: TranscriptionAdapter, filename: string, mimeType?: string): boolean {
+  const normalizedMimeType = mimeType?.toLowerCase();
+
+  if (normalizedMimeType && adapter.supportedMimeTypes.includes(normalizedMimeType)) {
+    return true;
+  }
+
+  const extension = extensionFor(filename);
+
+  return Boolean(extension && adapter.supportedExtensions.includes(extension));
+}
+
+function extensionFor(filename: string): string | null {
+  const match = /\.([a-z0-9]+)$/i.exec(filename);
+  return match ? match[1].toLowerCase() : null;
+}
+
+function decodeFixtureTranscript(audio: Uint8Array): string | null {
+  try {
+    const text = new TextDecoder().decode(audio).trim();
+    return text.length > 0 && /^[\t\n\r -~\u00a0-\uffff]+$/.test(text) ? text : null;
+  } catch {
+    return null;
+  }
+}
+
+function arrayBufferForBlob(audio: Uint8Array): ArrayBuffer {
+  const copy = new Uint8Array(audio.byteLength);
+  copy.set(audio);
+  return copy.buffer;
+}
+
+function readProviderError(payload: unknown): string | null {
+  if (!isRecord(payload)) {
+    return null;
+  }
+
+  if (typeof payload.error === "string") {
+    return payload.error;
+  }
+
+  if (isRecord(payload.error) && typeof payload.error.message === "string") {
+    return payload.error.message;
+  }
+
+  if (typeof payload.message === "string") {
+    return payload.message;
+  }
+
+  return null;
+}
+
+function redactSecret(value: string, secret: string): string {
+  return secret ? value.replaceAll(secret, "<redacted>") : value;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}