@od-oneapp/ai-platform 0.1.0

package/src/output/multimodal/SETUP_GUIDE.md

@@ -0,0 +1,472 @@
+# Audio Module Setup Guide
+
+Quick guide to set up and use the AI SDK v6 compatible audio module.
+
+## Prerequisites
+
+- TypeScript 4.5+
+- Node.js 16+
+- API keys for desired providers (OpenAI, ElevenLabs, etc.)
+
+## Installation
+
+### 1. Import from Module
+
+```typescript
+import {
+  generateSpeech,
+  transcribe,
+  streamSpeech,
+  initializeAudioProviders,
+  type GenerateSpeechOptions,
+  type TranscribeOptions
+} from "@repo/ai-platform/output/multimodal";
+```
+
+### 2. Initialize Providers
+
+#### Option A: Using Example Providers
+
+```typescript
+import {
+  OpenAISpeechProvider,
+  OpenAITranscriptionProvider,
+  ElevenLabsSpeechProvider
+} from "@repo/ai-platform/output/multimodal/providers.example";
+
+initializeAudioProviders({
+  speechProviders: {
+    "tts-1": new OpenAISpeechProvider(process.env.OPENAI_API_KEY!),
+    eleven_: new ElevenLabsSpeechProvider(process.env.ELEVENLABS_API_KEY!)
+  },
+  transcriptionProviders: {
+    whisper: new OpenAITranscriptionProvider(process.env.OPENAI_API_KEY!)
+  }
+});
+```
+
+#### Option B: Using Official SDKs
+
+```typescript
+import { registerSpeechProvider, registerTranscriptionProvider } from "@repo/ai-platform/output/multimodal";
+import { OpenAISpeechProvider as OfficialOpenAISpeech } from "@ai-sdk/openai";
+
+// Implement adapter if needed
+class OpenAISpeechAdapter implements SpeechProvider {
+  constructor(private provider: OfficialOpenAISpeech) {}
+
+  async generateSpeech(text, options) {
+    // Adapt to interface
+  }
+}
+
+registerSpeechProvider("tts-1", new OpenAISpeechAdapter(...));
+```
+
+## Quick Start Examples
+
+### Generate Speech
+
+```typescript
+const result = await generateSpeech({
+  model: "tts-1-hd",
+  text: "Hello, how can I help you today?",
+  voice: "nova",
+  format: "mp3"
+});
+
+// Save to file
+import fs from "fs";
+fs.writeFileSync("speech.mp3", Buffer.from(result.audio));
+```
+
+### Stream Speech
+
+```typescript
+// Stream to HTTP response
+import { Readable } from "stream";
+
+const audioStream = Readable.from(
+  streamSpeech({
+    model: "tts-1",
+    text: "Your long text here...",
+    voice: "alloy"
+  })
+);
+
+response.setHeader("Content-Type", "audio/mpeg");
+audioStream.pipe(response);
+```
+
+### Transcribe Audio
+
+```typescript
+// From file
+const audioBuffer = fs.readFileSync("recording.mp3");
+
+const result = await transcribe({
+  model: "whisper-1",
+  audio: audioBuffer,
+  language: "en",
+  timestamps: true
+});
+
+console.log(result.text);
+console.log(result.segments); // [{start: 0, end: 1.5, text: "Hello"}, ...]
+```
+
+### Stream Transcription
+
+```typescript
+// From ReadableStream
+const audioStream = fs.createReadStream("recording.mp3");
+
+const result = await transcribe({
+  model: "whisper-1",
+  audio: audioStream as any, // Convert to ReadableStream<Uint8Array>
+  timestamps: true
+});
+```
+
+## Environment Variables
+
+Create `.env` or use environment setup:
+
+```bash
+# OpenAI
+OPENAI_API_KEY=sk-...
+
+# ElevenLabs
+ELEVENLABS_API_KEY=...
+
+# Other providers as needed
+DEEPGRAM_API_KEY=...
+ASSEMBLYAI_API_KEY=...
+```
+
+## Configuration Patterns
+
+### Single Provider Setup
+
+```typescript
+import { registerSpeechProvider } from "@repo/ai-platform/output/multimodal";
+import { OpenAISpeechProvider } from "@repo/ai-platform/output/multimodal/providers.example";
+
+registerSpeechProvider("tts-1", new OpenAISpeechProvider(process.env.OPENAI_API_KEY!));
+```
+
+### Multiple Providers
+
+```typescript
+initializeAudioProviders({
+  speechProviders: {
+    "tts-1": new OpenAISpeechProvider(process.env.OPENAI_API_KEY!),
+    eleven_: new ElevenLabsSpeechProvider(process.env.ELEVENLABS_API_KEY!),
+    lmnt: new LMNTSpeechProvider(process.env.LMNT_API_KEY!)
+  },
+  transcriptionProviders: {
+    whisper: new OpenAITranscriptionProvider(process.env.OPENAI_API_KEY!),
+    deepgram: new DeepgramTranscriptionProvider(process.env.DEEPGRAM_API_KEY!),
+    assemblyai: new AssemblyAITranscriptionProvider(process.env.ASSEMBLYAI_API_KEY!)
+  }
+});
+```
+
+### With Fallback Provider
+
+```typescript
+// Custom wrapper that tries multiple providers
+class FallbackSpeechProvider implements SpeechProvider {
+  constructor(private providers: SpeechProvider[]) {}
+
+  async generateSpeech(text, options) {
+    for (const provider of this.providers) {
+      try {
+        return await provider.generateSpeech(text, options);
+      } catch (error) {
+        console.warn("Provider failed, trying next...", error);
+      }
+    }
+    throw new Error("All providers failed");
+  }
+}
+```
+
+## Next.js/Express Examples
+
+### Next.js API Route
+
+```typescript
+// pages/api/speak.ts
+import { generateSpeech } from "@repo/ai-platform/output/multimodal";
+import type { NextApiRequest, NextApiResponse } from "next";
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  if (req.method !== "POST") {
+    return res.status(405).end();
+  }
+
+  const { text, voice } = req.body;
+
+  try {
+    const result = await generateSpeech({
+      model: "tts-1-hd",
+      text,
+      voice: voice || "nova"
+    });
+
+    res.setHeader("Content-Type", "audio/mpeg");
+    res.send(Buffer.from(result.audio));
+  } catch (error) {
+    res.status(500).json({ error: error instanceof Error ? error.message : "Unknown error" });
+  }
+}
+```
+
+### Express Route
+
+```typescript
+app.post("/api/transcribe", async (req, res) => {
+  const { file } = req.files;
+
+  try {
+    const result = await transcribe({
+      model: "whisper-1",
+      audio: new Uint8Array(file.data),
+      timestamps: true
+    });
+
+    res.json(result);
+  } catch (error) {
+    res.status(500).json({ error: error instanceof Error ? error.message : "Unknown error" });
+  }
+});
+```
+
+## Error Handling
+
+```typescript
+import { generateSpeech, getRegisteredProviders } from "@repo/ai-platform/output/multimodal";
+
+async function generateSpeechSafely(text: string) {
+  try {
+    // Check available providers
+    const providers = getRegisteredProviders();
+    console.log("Available providers:", providers);
+
+    const result = await generateSpeech({
+      model: "tts-1-hd",
+      text,
+      voice: "nova",
+      timeout: 30000
+    });
+
+    return result;
+  } catch (error) {
+    if (error instanceof Error) {
+      if (error.message.includes("timeout")) {
+        console.error("Speech generation timeout - try shorter text");
+      } else if (error.message.includes("No speech provider")) {
+        console.error("Provider not initialized - call initializeAudioProviders()");
+      } else {
+        console.error("Speech generation failed:", error.message);
+      }
+    }
+    throw error;
+  }
+}
+```
+
+## Debugging
+
+### Check Provider Registration
+
+```typescript
+import { getRegisteredProviders } from "@repo/ai-platform/output/multimodal";
+
+const providers = getRegisteredProviders();
+console.log("Speech providers:", providers.speechProviders); // ["tts-1", "eleven_", ...]
+console.log("Transcription providers:", providers.transcriptionProviders); // ["whisper", ...]
+```
+
+### Enable Debug Logging
+
+```typescript
+// Add logging to provider class
+class DebugOpenAISpeechProvider extends OpenAISpeechProvider {
+  async generateSpeech(text, options) {
+    console.time("OpenAI TTS");
+    try {
+      const result = await super.generateSpeech(text, options);
+      console.log("Generated audio bytes:", result.audio.length);
+      return result;
+    } finally {
+      console.timeEnd("OpenAI TTS");
+    }
+  }
+}
+```
+
+## Performance Tips
+
+### 1. Stream Long Content
+
+```typescript
+// Bad: Generate entire response at once
+const result = await generateSpeech({
+  model: "tts-1",
+  text: veryLongText // Could timeout
+});
+
+// Good: Stream with proper handling
+for await (const chunk of streamSpeech({
+  model: "tts-1",
+  text: veryLongText
+})) {
+  response.write(chunk);
+}
+```
+
+### 2. Cache Results
+
+```typescript
+const cache = new Map<string, GenerateSpeechResult>();
+
+async function getCachedSpeech(text: string) {
+  if (cache.has(text)) {
+    return cache.get(text)!;
+  }
+
+  const result = await generateSpeech({
+    model: "tts-1-hd",
+    text
+  });
+
+  cache.set(text, result);
+  return result;
+}
+```
+
+### 3. Batch Operations
+
+```typescript
+// Transcribe multiple files
+const files = [...];
+const results = await Promise.all(
+  files.map(file =>
+    transcribe({
+      model: "whisper-1",
+      audio: fs.readFileSync(file),
+    })
+  )
+);
+```
+
+### 4. Timeout Configuration
+
+```typescript
+// Adjust timeouts based on content length
+function getTimeoutForLength(textLength: number): number {
+  // Rough estimate: 100ms per 100 characters
+  return Math.max(10000, Math.ceil(textLength * 10));
+}
+
+const result = await generateSpeech({
+  model: "tts-1",
+  text,
+  timeout: getTimeoutForLength(text.length)
+});
+```
+
+## Troubleshooting
+
+### "No speech provider registered for model"
+
+**Cause**: Provider not initialized or wrong model name
+
+**Solution**:
+
+```typescript
+// Check initialization
+const providers = getRegisteredProviders();
+console.log(providers);
+
+// Initialize if missing
+initializeAudioProviders({...});
+```
+
+### "Speech generation timeout"
+
+**Cause**: Operation took too long
+
+**Solution**:
+
+```typescript
+// Increase timeout for longer text
+await generateSpeech({
+  model: "tts-1",
+  text,
+  timeout: 60000, // Increase from default 30000
+});
+
+// Or split into smaller chunks
+const chunks = text.match(/.{1,1000}/g) || [];
+for (const chunk of chunks) {
+  await generateSpeech({...});
+}
+```
+
+### "Invalid audio input"
+
+**Cause**: Unsupported audio format
+
+**Solution**:
+
+```typescript
+import { normalizeAudioInput } from "@repo/ai-platform/output/multimodal";
+
+// Use helper to normalize
+const normalizedAudio = await normalizeAudioInput(userAudio);
+```
+
+## Migration from Other Libraries
+
+### From Web Audio API
+
+```typescript
+// Before
+const response = await fetch(`https://api.openai.com/v1/audio/speech`, {
+  /* ... */
+});
+const buffer = await response.arrayBuffer();
+
+// After
+const result = await generateSpeech({
+  model: "tts-1-hd",
+  text: "Hello world"
+});
+const buffer = result.audio;
+```
+
+### From @ai-sdk/openai
+
+```typescript
+// Before
+import { experimental_generateSpeech } from "ai";
+
+// After
+import { generateSpeech } from "@repo/ai-platform/output/multimodal";
+
+// Function calls remain identical!
+const result = await generateSpeech({...});
+```
+
+## Next Steps
+
+1. **Initialize providers** in your application startup
+2. **Test with example providers** first
+3. **Implement custom providers** as needed
+4. **Handle errors gracefully**
+5. **Monitor and optimize** based on usage patterns
+
+For complete documentation, see [README.md](./README.md).