@lobu/gateway 2.8.0 → 3.0.6

package/src/services/transcription-service.ts

@@ -0,0 +1,684 @@
+/**
+ * Multi-Provider Audio Service
+ *
+ * Supports speech-to-text and text-to-speech via auth profiles (installed providers):
+ * - OpenAI (chatgpt auth profile) - Whisper for STT, TTS API for speech
+ * - Google Gemini (gemini auth profile) - Audio input/output
+ * - ElevenLabs (elevenlabs auth profile) - STT and high-quality TTS
+ *
+ * STT selection: built-ins (chatgpt/openai, gemini, elevenlabs) plus optional
+ * config-driven STT providers declared in system-skills provider config.
+ * TTS selection stays built-in only (openai → gemini → elevenlabs).
+ */
+
+import type { ProviderConfigEntry } from "@lobu/core";
+import { createLogger } from "@lobu/core";
+import type { AuthProfilesManager } from "../auth/settings/auth-profiles-manager";
+
+const logger = createLogger("transcription-service");
+
+export type TranscriptionProvider = "openai" | "gemini" | "elevenlabs";
+
+interface TranscriptionConfig {
+  profileProviderId: string;
+  displayName: string;
+  provider: TranscriptionProvider;
+  apiKey: string;
+  openaiCompat?: {
+    endpointUrl: string;
+    model: string;
+  };
+}
+
+export interface TranscriptionSuccess {
+  text: string;
+  provider: TranscriptionProvider;
+}
+
+export interface TranscriptionError {
+  error: string;
+  availableProviders: TranscriptionProvider[];
+}
+
+export type TranscriptionResult = TranscriptionSuccess | TranscriptionError;
+
+export interface SynthesisSuccess {
+  audioBuffer: Buffer;
+  mimeType: string;
+  provider: TranscriptionProvider;
+}
+
+export interface SynthesisError {
+  error: string;
+  availableProviders: TranscriptionProvider[];
+}
+
+export type SynthesisResult = SynthesisSuccess | SynthesisError;
+
+// Voice options for TTS
+export interface VoiceOptions {
+  voice?: string; // Provider-specific voice ID
+  speed?: number; // Speech speed (0.5-2.0, default 1.0)
+}
+
+// Auth profile providerId → TTS provider mapping (single source of truth)
+const TTS_CAPABLE_PROVIDERS: {
+  profileProviderId: string;
+  ttsProvider: TranscriptionProvider;
+  displayName: string;
+}[] = [
+  {
+    profileProviderId: "chatgpt",
+    ttsProvider: "openai",
+    displayName: "OpenAI",
+  },
+  {
+    profileProviderId: "gemini",
+    ttsProvider: "gemini",
+    displayName: "Google Gemini",
+  },
+  {
+    profileProviderId: "elevenlabs",
+    ttsProvider: "elevenlabs",
+    displayName: "ElevenLabs",
+  },
+];
+
+function displayName(provider: TranscriptionProvider): string {
+  return (
+    TTS_CAPABLE_PROVIDERS.find((p) => p.ttsProvider === provider)
+      ?.displayName ?? provider
+  );
+}
+
+export class TranscriptionService {
+  private providerConfigSource?:
+    | (() => Promise<Record<string, ProviderConfigEntry>>)
+    | undefined;
+
+  constructor(
+    private readonly authProfilesManager: AuthProfilesManager,
+    providerConfigSource?: () => Promise<Record<string, ProviderConfigEntry>>
+  ) {
+    this.providerConfigSource = providerConfigSource;
+  }
+
+  setProviderConfigSource(
+    source: () => Promise<Record<string, ProviderConfigEntry>>
+  ): void {
+    this.providerConfigSource = source;
+  }
+
+  /**
+   * Transcribe audio buffer to text
+   */
+  async transcribe(
+    audioBuffer: Buffer,
+    agentId: string,
+    mimeType = "audio/ogg"
+  ): Promise<TranscriptionResult> {
+    const configs = await this.getTranscriptionConfigs(agentId);
+
+    if (configs.length === 0) {
+      return this.noProviderError(
+        "No transcription provider configured",
+        agentId
+      );
+    }
+
+    const attemptErrors: string[] = [];
+    for (const config of configs) {
+      logger.info("Transcribing audio", {
+        agentId,
+        provider: config.provider,
+        profileProviderId: config.profileProviderId,
+        bufferSize: audioBuffer.length,
+        mimeType,
+      });
+
+      try {
+        const text = await this.transcribeWithProvider(
+          audioBuffer,
+          config,
+          mimeType
+        );
+        logger.info("Transcription successful", {
+          agentId,
+          provider: config.provider,
+          profileProviderId: config.profileProviderId,
+          textLength: text.length,
+        });
+        return { text, provider: config.provider };
+      } catch (error) {
+        const errorMessage =
+          error instanceof Error ? error.message : String(error);
+        logger.error("Transcription failed", {
+          agentId,
+          provider: config.provider,
+          profileProviderId: config.profileProviderId,
+          error: errorMessage,
+        });
+        attemptErrors.push(`${config.displayName}: ${errorMessage}`);
+      }
+    }
+
+    return {
+      error: `Transcription failed with all configured providers: ${attemptErrors.join(" | ")}`,
+      availableProviders: [...new Set(configs.map((c) => c.provider))],
+    };
+  }
+
+  /**
+   * Get transcription config for an agent by checking installed auth profiles.
+   * First TTS-capable provider with a valid profile wins (openai → gemini → elevenlabs).
+   */
+  async getConfig(agentId: string): Promise<TranscriptionConfig | null> {
+    const configs = await this.getTranscriptionConfigs(agentId);
+    return configs[0] ?? null;
+  }
+
+  private async getSynthesisConfigs(
+    agentId: string
+  ): Promise<TranscriptionConfig[]> {
+    const configs: TranscriptionConfig[] = [];
+    for (const { profileProviderId, ttsProvider } of TTS_CAPABLE_PROVIDERS) {
+      const profile = await this.authProfilesManager.getBestProfile(
+        agentId,
+        profileProviderId
+      );
+      if (profile) {
+        configs.push({
+          profileProviderId,
+          displayName: displayName(ttsProvider),
+          provider: ttsProvider,
+          apiKey: profile.credential,
+        });
+      }
+    }
+    return configs;
+  }
+
+  private async getTranscriptionConfigs(
+    agentId: string
+  ): Promise<TranscriptionConfig[]> {
+    const configs = await this.getSynthesisConfigs(agentId);
+    const providerIds = new Set(configs.map((c) => c.profileProviderId));
+    const configDriven = await this.getConfigDrivenSttCandidates();
+
+    for (const candidate of configDriven) {
+      if (providerIds.has(candidate.profileProviderId)) continue;
+
+      const profile = await this.authProfilesManager.getBestProfile(
+        agentId,
+        candidate.profileProviderId
+      );
+      if (!profile) continue;
+
+      configs.push({
+        profileProviderId: candidate.profileProviderId,
+        displayName: candidate.displayName,
+        provider: candidate.provider,
+        apiKey: profile.credential,
+        openaiCompat: candidate.openaiCompat,
+      });
+      providerIds.add(candidate.profileProviderId);
+    }
+
+    return configs;
+  }
+
+  private async getConfigDrivenSttCandidates(): Promise<
+    Array<Omit<TranscriptionConfig, "apiKey">>
+  > {
+    if (!this.providerConfigSource) return [];
+
+    let providerConfigs: Record<string, ProviderConfigEntry>;
+    try {
+      providerConfigs = await this.providerConfigSource();
+    } catch (error) {
+      logger.warn("Failed to load provider configs for STT", {
+        error: error instanceof Error ? error.message : String(error),
+      });
+      return [];
+    }
+
+    const candidates: Array<Omit<TranscriptionConfig, "apiKey">> = [];
+    for (const [providerId, entry] of Object.entries(providerConfigs)) {
+      const stt = entry.stt;
+      const compat = stt?.sdkCompat || entry.sdkCompat;
+      const sttEnabled = stt ? stt.enabled !== false : compat === "openai";
+      if (!sttEnabled) continue;
+
+      if (compat !== "openai") {
+        logger.warn("Unsupported config-driven STT compatibility", {
+          providerId,
+          compat,
+        });
+        continue;
+      }
+
+      const endpoint = this.resolveEndpointUrl(
+        stt?.transcriptionPath,
+        stt?.baseUrl || entry.upstreamBaseUrl
+      );
+      if (!endpoint) {
+        logger.warn("Invalid STT endpoint configuration", {
+          providerId,
+          transcriptionPath: stt?.transcriptionPath,
+          baseUrl: stt?.baseUrl || entry.upstreamBaseUrl,
+        });
+        continue;
+      }
+
+      candidates.push({
+        profileProviderId: providerId,
+        displayName: entry.displayName || providerId,
+        provider: "openai",
+        openaiCompat: {
+          endpointUrl: endpoint,
+          model: stt?.model?.trim() || "whisper-1",
+        },
+      });
+    }
+    return candidates;
+  }
+
+  /**
+   * Get provider info for documentation/help messages
+   */
+  getProviderInfo(): Array<{ provider: TranscriptionProvider; name: string }> {
+    return TTS_CAPABLE_PROVIDERS.map(({ ttsProvider, displayName }) => ({
+      provider: ttsProvider,
+      name: displayName,
+    }));
+  }
+
+  // ==========================================================================
+  // Text-to-Speech (Synthesis)
+  // ==========================================================================
+
+  /**
+   * Synthesize text to audio
+   */
+  async synthesize(
+    text: string,
+    agentId: string,
+    options: VoiceOptions = {}
+  ): Promise<SynthesisResult> {
+    const config = await this.getConfig(agentId);
+
+    if (!config) {
+      return this.noProviderError("No audio provider configured", agentId);
+    }
+
+    logger.info("Synthesizing audio", {
+      agentId,
+      provider: config.provider,
+      textLength: text.length,
+      voice: options.voice,
+    });
+
+    try {
+      const result = await this.synthesizeWithProvider(text, config, options);
+      logger.info("Synthesis successful", {
+        agentId,
+        provider: config.provider,
+        audioSize: result.audioBuffer.length,
+      });
+      return { ...result, provider: config.provider };
+    } catch (error) {
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
+      logger.error("Synthesis failed", {
+        agentId,
+        provider: config.provider,
+        error: errorMessage,
+      });
+      return {
+        error: `Synthesis failed with ${displayName(config.provider)}: ${errorMessage}`,
+        availableProviders: [],
+      };
+    }
+  }
+
+  private noProviderError(message: string, agentId: string) {
+    const availableProviders = TTS_CAPABLE_PROVIDERS.map((p) => p.ttsProvider);
+    logger.info(message, { agentId, availableProviders });
+    return { error: message, availableProviders };
+  }
+
+  // ==========================================================================
+  // Provider-specific implementations - Transcription (STT)
+  // ==========================================================================
+
+  private async transcribeWithProvider(
+    buffer: Buffer,
+    config: TranscriptionConfig,
+    mimeType: string
+  ): Promise<string> {
+    switch (config.provider) {
+      case "openai":
+        return this.transcribeWithOpenAI(
+          buffer,
+          config.apiKey,
+          mimeType,
+          config.openaiCompat
+        );
+      case "gemini":
+        return this.transcribeWithGemini(buffer, config.apiKey, mimeType);
+      case "elevenlabs":
+        return this.transcribeWithElevenLabs(buffer, config.apiKey, mimeType);
+      default:
+        throw new Error(`Unknown provider: ${config.provider}`);
+    }
+  }
+
+  private async transcribeWithOpenAI(
+    buffer: Buffer,
+    apiKey: string,
+    mimeType: string,
+    options?: { endpointUrl: string; model: string }
+  ): Promise<string> {
+    const formData = new FormData();
+    const ext = this.getExtensionFromMime(mimeType);
+    formData.append(
+      "file",
+      new Blob([buffer], { type: mimeType }),
+      `audio.${ext}`
+    );
+    formData.append("model", options?.model || "whisper-1");
+
+    const resp = await fetch(
+      options?.endpointUrl || "https://api.openai.com/v1/audio/transcriptions",
+      {
+        method: "POST",
+        headers: { Authorization: `Bearer ${apiKey}` },
+        body: formData,
+      }
+    );
+
+    if (!resp.ok) {
+      const error = await resp.text();
+      throw new Error(`OpenAI API error: ${resp.status} - ${error}`);
+    }
+
+    const data = (await resp.json()) as { text: string };
+    return data.text;
+  }
+
+  private async transcribeWithGemini(
+    buffer: Buffer,
+    apiKey: string,
+    mimeType: string
+  ): Promise<string> {
+    // Gemini uses inline audio data with base64 encoding
+    const resp = await fetch(
+      `https://generativelanguage.googleapis.com/v1/models/gemini-1.5-flash:generateContent?key=${apiKey}`,
+      {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          contents: [
+            {
+              parts: [
+                {
+                  text: "Transcribe this audio exactly as spoken. Return only the transcription text, nothing else:",
+                },
+                {
+                  inline_data: {
+                    mime_type: mimeType,
+                    data: buffer.toString("base64"),
+                  },
+                },
+              ],
+            },
+          ],
+        }),
+      }
+    );
+
+    if (!resp.ok) {
+      const error = await resp.text();
+      throw new Error(`Gemini API error: ${resp.status} - ${error}`);
+    }
+
+    const data = (await resp.json()) as {
+      candidates: Array<{
+        content: { parts: Array<{ text: string }> };
+      }>;
+    };
+    return data.candidates[0]?.content?.parts[0]?.text || "";
+  }
+
+  private async transcribeWithElevenLabs(
+    buffer: Buffer,
+    apiKey: string,
+    mimeType: string
+  ): Promise<string> {
+    // ElevenLabs speech-to-text API
+    const formData = new FormData();
+    const ext = this.getExtensionFromMime(mimeType);
+    formData.append(
+      "audio",
+      new Blob([buffer], { type: mimeType }),
+      `audio.${ext}`
+    );
+
+    const resp = await fetch("https://api.elevenlabs.io/v1/speech-to-text", {
+      method: "POST",
+      headers: { "xi-api-key": apiKey },
+      body: formData,
+    });
+
+    if (!resp.ok) {
+      const error = await resp.text();
+      throw new Error(`ElevenLabs API error: ${resp.status} - ${error}`);
+    }
+
+    const data = (await resp.json()) as { text: string };
+    return data.text;
+  }
+
+  // ==========================================================================
+  // Provider-specific implementations - Synthesis (TTS)
+  // ==========================================================================
+
+  private async synthesizeWithProvider(
+    text: string,
+    config: TranscriptionConfig,
+    options: VoiceOptions
+  ): Promise<{ audioBuffer: Buffer; mimeType: string }> {
+    switch (config.provider) {
+      case "openai":
+        return this.synthesizeWithOpenAI(text, config.apiKey, options);
+      case "gemini":
+        return this.synthesizeWithGemini(text, config.apiKey);
+      case "elevenlabs":
+        return this.synthesizeWithElevenLabs(text, config.apiKey, options);
+      default:
+        throw new Error(`Unknown provider: ${config.provider}`);
+    }
+  }
+
+  private async synthesizeWithOpenAI(
+    text: string,
+    apiKey: string,
+    options: VoiceOptions
+  ): Promise<{ audioBuffer: Buffer; mimeType: string }> {
+    // OpenAI TTS API
+    // Voices: alloy, echo, fable, onyx, nova, shimmer
+    const voice = options.voice || "alloy";
+    const speed = options.speed || 1.0;
+
+    const resp = await fetch("https://api.openai.com/v1/audio/speech", {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        model: "tts-1",
+        input: text,
+        voice,
+        speed,
+        response_format: "opus", // Good for WhatsApp
+      }),
+    });
+
+    if (!resp.ok) {
+      const error = await resp.text();
+      throw new Error(`OpenAI TTS API error: ${resp.status} - ${error}`);
+    }
+
+    const arrayBuffer = await resp.arrayBuffer();
+    return {
+      audioBuffer: Buffer.from(arrayBuffer),
+      mimeType: "audio/opus",
+    };
+  }
+
+  private async synthesizeWithGemini(
+    text: string,
+    apiKey: string
+  ): Promise<{ audioBuffer: Buffer; mimeType: string }> {
+    const resp = await fetch(
+      `https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-exp:generateContent?key=${apiKey}`,
+      {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          contents: [
+            {
+              parts: [{ text: `Please speak this text aloud: "${text}"` }],
+            },
+          ],
+          generationConfig: {
+            responseModalities: ["AUDIO"],
+            speechConfig: {
+              voiceConfig: {
+                prebuiltVoiceConfig: {
+                  voiceName: "Aoede", // Default Gemini voice
+                },
+              },
+            },
+          },
+        }),
+      }
+    );
+
+    if (!resp.ok) {
+      const error = await resp.text();
+      throw new Error(`Gemini TTS API error: ${resp.status} - ${error}`);
+    }
+
+    const data = (await resp.json()) as {
+      candidates: Array<{
+        content: {
+          parts: Array<{
+            inlineData?: { mimeType: string; data: string };
+          }>;
+        };
+      }>;
+    };
+
+    const audioPart = data.candidates[0]?.content?.parts?.find((p) =>
+      p.inlineData?.mimeType?.startsWith("audio/")
+    );
+
+    if (!audioPart?.inlineData) {
+      throw new Error("Gemini did not return audio data");
+    }
+
+    return {
+      audioBuffer: Buffer.from(audioPart.inlineData.data, "base64"),
+      mimeType: audioPart.inlineData.mimeType,
+    };
+  }
+
+  private async synthesizeWithElevenLabs(
+    text: string,
+    apiKey: string,
+    options: VoiceOptions
+  ): Promise<{ audioBuffer: Buffer; mimeType: string }> {
+    // ElevenLabs TTS API
+    // Default voice: Rachel (21m00Tcm4TlvDq8ikWAM)
+    const voiceId = options.voice || "21m00Tcm4TlvDq8ikWAM";
+
+    const resp = await fetch(
+      `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`,
+      {
+        method: "POST",
+        headers: {
+          "xi-api-key": apiKey,
+          "Content-Type": "application/json",
+          Accept: "audio/mpeg",
+        },
+        body: JSON.stringify({
+          text,
+          model_id: "eleven_monolingual_v1",
+          voice_settings: {
+            stability: 0.5,
+            similarity_boost: 0.5,
+          },
+        }),
+      }
+    );
+
+    if (!resp.ok) {
+      const error = await resp.text();
+      throw new Error(`ElevenLabs TTS API error: ${resp.status} - ${error}`);
+    }
+
+    const arrayBuffer = await resp.arrayBuffer();
+    return {
+      audioBuffer: Buffer.from(arrayBuffer),
+      mimeType: "audio/mpeg",
+    };
+  }
+
+  // ==========================================================================
+  // Utility methods
+  // ==========================================================================
+
+  private getExtensionFromMime(mimeType: string): string {
+    const mimeToExt: Record<string, string> = {
+      "audio/ogg": "ogg",
+      "audio/opus": "opus",
+      "audio/mpeg": "mp3",
+      "audio/mp3": "mp3",
+      "audio/wav": "wav",
+      "audio/webm": "webm",
+      "audio/m4a": "m4a",
+      "audio/mp4": "m4a",
+    };
+    return mimeToExt[mimeType] || "ogg";
+  }
+
+  private resolveEndpointUrl(
+    transcriptionPath: string | undefined,
+    baseUrl: string | undefined
+  ): string | null {
+    const path = (
+      transcriptionPath || this.getDefaultOpenAiTranscriptionPath(baseUrl)
+    ).trim();
+    if (/^https?:\/\//i.test(path)) {
+      return path;
+    }
+
+    const base = (baseUrl || "").trim();
+    if (!base) return null;
+
+    const normalizedPath = path.startsWith("/") ? path : `/${path}`;
+    return `${base.replace(/\/+$/, "")}${normalizedPath}`;
+  }
+
+  private getDefaultOpenAiTranscriptionPath(
+    baseUrl: string | undefined
+  ): string {
+    const trimmedBase = (baseUrl || "").trim().replace(/\/+$/, "");
+    if (trimmedBase.endsWith("/v1")) {
+      return "/audio/transcriptions";
+    }
+    return "/v1/audio/transcriptions";
+  }
+}