@andreprado/agentkit 0.1.0-alpha.2 → 0.1.0-alpha.21

package/src/runtime/transcription.ts

@@ -0,0 +1,483 @@
+import type { AgentTranscriptionConfig, TranscriptionProviderName } from "../index";
+
+export type ResolvedTranscriptionConfig = {
+  enabled: boolean;
+  provider: TranscriptionProviderName;
+  model: string;
+  secret: string | null;
+  language: string | null;
+  prompt: string | null;
+  limits: {
+    maxDurationSeconds?: number;
+    maxBytes?: number;
+  };
+  rawAudioTtlSeconds: number | null;
+};
+
+export type TranscriptionInput = {
+  config: ResolvedTranscriptionConfig;
+  secrets: Record<string, string>;
+  audio: Uint8Array;
+  filename: string;
+  mimeType?: string;
+  durationSeconds?: number;
+  providerMetadata?: Record<string, unknown>;
+};
+
+export type TranscriptionFetch = (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
+
+export type TranscriptionResult =
+  | {
+      ok: true;
+      text: string;
+      provider: TranscriptionProviderName;
+      model: string;
+      language?: string;
+      durationSeconds?: number;
+      providerMetadata?: Record<string, unknown>;
+    }
+  | {
+      ok: false;
+      retryable: boolean;
+      code:
+        | "transcription_disabled"
+        | "transcription_secret_missing"
+        | "transcription_model_unsupported"
+        | "transcription_audio_too_large"
+        | "transcription_audio_too_long"
+        | "transcription_audio_format_unsupported"
+        | "transcription_provider_unavailable"
+        | "transcription_failed";
+      message: string;
+      provider?: TranscriptionProviderName;
+      model?: string;
+      providerMetadata?: Record<string, unknown>;
+    };
+
+export type TranscriptionAdapter = {
+  provider: TranscriptionProviderName;
+  models: string[];
+  supportedMimeTypes: string[];
+  supportedExtensions: string[];
+  maxBytes: number;
+  requiredSecret(config: ResolvedTranscriptionConfig): string | null;
+  transcribe(input: TranscriptionInput, fetcher?: TranscriptionFetch): Promise<TranscriptionResult>;
+};
+
+const DEFAULT_MAX_BYTES = 25_000_000;
+const OPENAI_MODELS = ["gpt-4o-mini-transcribe", "gpt-4o-transcribe", "whisper-1"];
+const GROQ_MODELS = ["whisper-large-v3-turbo", "whisper-large-v3", "distil-whisper-large-v3-en"];
+const OPENAI_EXTENSIONS = ["mp3", "mp4", "mpeg", "mpga", "m4a", "wav", "webm"];
+const GROQ_EXTENSIONS = ["flac", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "wav", "webm"];
+const OPENAI_MIME_TYPES = [
+  "audio/mpeg",
+  "audio/mp3",
+  "audio/mp4",
+  "audio/mpga",
+  "audio/m4a",
+  "audio/wav",
+  "audio/webm",
+  "video/mp4",
+];
+const GROQ_MIME_TYPES = [
+  ...OPENAI_MIME_TYPES,
+  "audio/flac",
+  "audio/ogg",
+  "audio/opus",
+  "application/ogg",
+];
+
+export function resolveTranscriptionConfig(config: AgentTranscriptionConfig | undefined): ResolvedTranscriptionConfig {
+  if (!config) {
+    return {
+      enabled: false,
+      provider: "test",
+      model: "fake",
+      secret: null,
+      language: null,
+      prompt: null,
+      limits: {},
+      rawAudioTtlSeconds: null,
+    };
+  }
+
+  return {
+    enabled: true,
+    provider: config.provider,
+    model: config.model,
+    secret: config.secret ?? defaultTranscriptionSecret(config.provider),
+    language: config.language ?? null,
+    prompt: config.prompt ?? null,
+    limits: {
+      ...(config.limits?.maxDurationSeconds !== undefined
+        ? { maxDurationSeconds: config.limits.maxDurationSeconds }
+        : {}),
+      ...(config.limits?.maxBytes !== undefined ? { maxBytes: config.limits.maxBytes } : {}),
+    },
+    rawAudioTtlSeconds: config.rawAudioTtlSeconds ?? 3600,
+  };
+}
+
+export async function transcribeAudio(
+  input: TranscriptionInput,
+  fetcher: TranscriptionFetch = fetch,
+): Promise<TranscriptionResult> {
+  if (!input.config.enabled) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_disabled",
+      message: "Audio transcription is not configured for this Agent Capsule.",
+    };
+  }
+
+  const adapter = transcriptionAdapterFor(input.config.provider);
+
+  if (!adapter) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_model_unsupported",
+      message: `No transcription adapter is registered for ${input.config.provider}.`,
+      provider: input.config.provider,
+      model: input.config.model,
+    };
+  }
+
+  const validation = validateTranscriptionInput(adapter, input);
+
+  if (!validation.ok) {
+    return validation;
+  }
+
+  return adapter.transcribe(input, fetcher);
+}
+
+export function transcriptionAdapterFor(provider: TranscriptionProviderName): TranscriptionAdapter | null {
+  if (provider === "test") {
+    return testTranscriptionAdapter;
+  }
+
+  if (provider === "openai") {
+    return openaiTranscriptionAdapter;
+  }
+
+  if (provider === "groq") {
+    return groqTranscriptionAdapter;
+  }
+
+  return null;
+}
+
+export function defaultTranscriptionSecret(provider: TranscriptionProviderName): string | null {
+  if (provider === "openai") {
+    return "OPENAI_API_KEY";
+  }
+
+  if (provider === "groq") {
+    return "GROQ_API_KEY";
+  }
+
+  return null;
+}
+
+function validateTranscriptionInput(adapter: TranscriptionAdapter, input: TranscriptionInput): TranscriptionResult {
+  if (!adapter.models.includes(input.config.model)) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_model_unsupported",
+      message: `${input.config.provider} transcription model ${input.config.model} is not supported by AgentKit.`,
+      provider: input.config.provider,
+      model: input.config.model,
+    };
+  }
+
+  const maxBytes = Math.min(input.config.limits.maxBytes ?? adapter.maxBytes, adapter.maxBytes);
+  if (input.audio.byteLength > maxBytes) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_audio_too_large",
+      message: `Audio file is ${input.audio.byteLength} bytes, which exceeds the configured ${maxBytes} byte limit.`,
+      provider: input.config.provider,
+      model: input.config.model,
+    };
+  }
+
+  if (
+    input.durationSeconds !== undefined &&
+    input.config.limits.maxDurationSeconds !== undefined &&
+    input.durationSeconds > input.config.limits.maxDurationSeconds
+  ) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_audio_too_long",
+      message: `Audio is ${input.durationSeconds}s, which exceeds the configured ${input.config.limits.maxDurationSeconds}s limit.`,
+      provider: input.config.provider,
+      model: input.config.model,
+    };
+  }
+
+  if (!isSupportedAudioFormat(adapter, input.filename, input.mimeType)) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_audio_format_unsupported",
+      message: `${input.config.provider} does not support audio format ${input.mimeType ?? extensionFor(input.filename) ?? "unknown"}.`,
+      provider: input.config.provider,
+      model: input.config.model,
+    };
+  }
+
+  const secret = adapter.requiredSecret(input.config);
+  if (secret && !input.secrets[secret]) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_secret_missing",
+      message: `Secret ${secret} is not set for audio transcription.`,
+      provider: input.config.provider,
+      model: input.config.model,
+    };
+  }
+
+  return {
+    ok: true,
+    text: "",
+    provider: input.config.provider,
+    model: input.config.model,
+  };
+}
+
+const testTranscriptionAdapter: TranscriptionAdapter = {
+  provider: "test",
+  models: ["fake"],
+  supportedMimeTypes: ["audio/wav", "audio/ogg", "audio/mpeg", "audio/webm", "application/octet-stream"],
+  supportedExtensions: ["wav", "ogg", "mp3", "webm", "bin"],
+  maxBytes: DEFAULT_MAX_BYTES,
+  requiredSecret() {
+    return null;
+  },
+  async transcribe(input) {
+    const text = decodeFixtureTranscript(input.audio) ?? "fake audio transcript";
+
+    return {
+      ok: true,
+      text,
+      provider: "test",
+      model: input.config.model,
+      ...(input.config.language ? { language: input.config.language } : {}),
+      ...(input.durationSeconds !== undefined ? { durationSeconds: input.durationSeconds } : {}),
+    };
+  },
+};
+
+const openaiTranscriptionAdapter: TranscriptionAdapter = {
+  provider: "openai",
+  models: OPENAI_MODELS,
+  supportedMimeTypes: OPENAI_MIME_TYPES,
+  supportedExtensions: OPENAI_EXTENSIONS,
+  maxBytes: DEFAULT_MAX_BYTES,
+  requiredSecret(config) {
+    return config.secret;
+  },
+  async transcribe(input, fetcher = fetch) {
+    return transcribeViaOpenAiCompatibleEndpoint({
+      input,
+      fetcher,
+      url: "https://api.openai.com/v1/audio/transcriptions",
+      apiKey: input.secrets[input.config.secret ?? ""],
+      provider: "openai",
+    });
+  },
+};
+
+const groqTranscriptionAdapter: TranscriptionAdapter = {
+  provider: "groq",
+  models: GROQ_MODELS,
+  supportedMimeTypes: GROQ_MIME_TYPES,
+  supportedExtensions: GROQ_EXTENSIONS,
+  maxBytes: DEFAULT_MAX_BYTES,
+  requiredSecret(config) {
+    return config.secret;
+  },
+  async transcribe(input, fetcher = fetch) {
+    return transcribeViaOpenAiCompatibleEndpoint({
+      input,
+      fetcher,
+      url: "https://api.groq.com/openai/v1/audio/transcriptions",
+      apiKey: input.secrets[input.config.secret ?? ""],
+      provider: "groq",
+    });
+  },
+};
+
+async function transcribeViaOpenAiCompatibleEndpoint(input: {
+  input: TranscriptionInput;
+  fetcher: TranscriptionFetch;
+  url: string;
+  apiKey: string | undefined;
+  provider: TranscriptionProviderName;
+}): Promise<TranscriptionResult> {
+  if (!input.apiKey) {
+    const secret = input.input.config.secret ?? defaultTranscriptionSecret(input.provider) ?? "TRANSCRIPTION_API_KEY";
+
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_secret_missing",
+      message: `Secret ${secret} is not set for audio transcription.`,
+      provider: input.provider,
+      model: input.input.config.model,
+    };
+  }
+
+  const form = new FormData();
+  form.set("model", input.input.config.model);
+  form.set(
+    "file",
+    new Blob([arrayBufferForBlob(input.input.audio)], { type: input.input.mimeType ?? "application/octet-stream" }),
+    input.input.filename,
+  );
+  form.set("response_format", "json");
+
+  if (input.input.config.language) {
+    form.set("language", input.input.config.language);
+  }
+
+  if (input.input.config.prompt) {
+    form.set("prompt", input.input.config.prompt);
+  }
+
+  let response: Response;
+
+  try {
+    response = await input.fetcher(input.url, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${input.apiKey}`,
+      },
+      body: form,
+    });
+  } catch (error) {
+    return {
+      ok: false,
+      retryable: true,
+      code: "transcription_provider_unavailable",
+      message: `Transcription provider request failed before a response: ${redactSecret(
+        error instanceof Error ? error.message : String(error),
+        input.apiKey,
+      )}`,
+      provider: input.provider,
+      model: input.input.config.model,
+    };
+  }
+
+  const payload = await response.json().catch(() => null);
+
+  if (!response.ok) {
+    const message = readProviderError(payload) ?? `Transcription provider returned HTTP ${response.status}.`;
+
+    return {
+      ok: false,
+      retryable: response.status === 429 || response.status >= 500,
+      code: response.status === 429 || response.status >= 500 ? "transcription_provider_unavailable" : "transcription_failed",
+      message: redactSecret(message, input.apiKey),
+      provider: input.provider,
+      model: input.input.config.model,
+      providerMetadata: {
+        status: response.status,
+      },
+    };
+  }
+
+  const text = isRecord(payload) && typeof payload.text === "string" ? payload.text.trim() : "";
+
+  if (!text) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "transcription_failed",
+      message: "Transcription provider returned an empty transcript.",
+      provider: input.provider,
+      model: input.input.config.model,
+      providerMetadata: {
+        status: response.status,
+      },
+    };
+  }
+
+  return {
+    ok: true,
+    text,
+    provider: input.provider,
+    model: input.input.config.model,
+    ...(input.input.config.language ? { language: input.input.config.language } : {}),
+    ...(input.input.durationSeconds !== undefined ? { durationSeconds: input.input.durationSeconds } : {}),
+    providerMetadata: {
+      status: response.status,
+    },
+  };
+}
+
+function isSupportedAudioFormat(adapter: TranscriptionAdapter, filename: string, mimeType?: string): boolean {
+  const normalizedMimeType = mimeType?.toLowerCase();
+
+  if (normalizedMimeType && adapter.supportedMimeTypes.includes(normalizedMimeType)) {
+    return true;
+  }
+
+  const extension = extensionFor(filename);
+
+  return Boolean(extension && adapter.supportedExtensions.includes(extension));
+}
+
+function extensionFor(filename: string): string | null {
+  const match = /\.([a-z0-9]+)$/i.exec(filename);
+  return match ? match[1].toLowerCase() : null;
+}
+
+function decodeFixtureTranscript(audio: Uint8Array): string | null {
+  try {
+    const text = new TextDecoder().decode(audio).trim();
+    return text.length > 0 && /^[\t\n\r -~\u00a0-\uffff]+$/.test(text) ? text : null;
+  } catch {
+    return null;
+  }
+}
+
+function arrayBufferForBlob(audio: Uint8Array): ArrayBuffer {
+  const copy = new Uint8Array(audio.byteLength);
+  copy.set(audio);
+  return copy.buffer;
+}
+
+function readProviderError(payload: unknown): string | null {
+  if (!isRecord(payload)) {
+    return null;
+  }
+
+  if (typeof payload.error === "string") {
+    return payload.error;
+  }
+
+  if (isRecord(payload.error) && typeof payload.error.message === "string") {
+    return payload.error.message;
+  }
+
+  if (typeof payload.message === "string") {
+    return payload.message;
+  }
+
+  return null;
+}
+
+function redactSecret(value: string, secret: string): string {
+  return secret ? value.replaceAll(secret, "<redacted>") : value;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}