@od-oneapp/ai-platform 0.1.0

package/src/output/multimodal/audio.ts

@@ -0,0 +1,650 @@
+/**
+ * @fileoverview AI SDK v6 Compatible Audio Module
+ *
+ * Provides top-level convenience functions for speech generation and transcription
+ * that match the Vercel AI SDK v6 patterns, supporting multiple providers including
+ * OpenAI, ElevenLabs, and others.
+ *
+ * Mirrors the AI SDK's experimental_generateSpeech and experimental_transcribe APIs
+ * with unified type definitions and provider abstractions.
+ *
+ * @module @od-oneapp/ai-platform/output/multimodal/audio
+ */
+
+/**
+ * Supported speech generation models
+ */
+export type SpeechModel =
+  | 'tts-1'
+  | 'tts-1-hd'
+  | 'gpt-4o-mini-tts'
+  | 'eleven_multilingual_v2'
+  | 'eleven_turbo_v2'
+  | 'lmnt'
+  | string;
+
+/**
+ * Supported transcription models
+ */
+export type TranscriptionModel =
+  | 'whisper-1'
+  | 'eleven_multilingual_v2'
+  | 'deepgram'
+  | 'assemblyai'
+  | 'groq'
+  | string;
+
+/**
+ * Audio output format for speech generation
+ */
+export type AudioFormat = 'mp3' | 'wav' | 'opus';
+
+/**
+ * Supported voice identifiers for speech generation
+ */
+export type Voice = 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer' | string;
+
+/**
+ * Options for speech generation (text-to-speech)
+ *
+ * Matches AI SDK's generateSpeech pattern while supporting multiple providers.
+ */
+export interface GenerateSpeechOptions {
+  /**
+   * The speech model to use (e.g., "tts-1-hd", "eleven_multilingual_v2")
+   */
+  model: SpeechModel;
+
+  /**
+   * The text to convert to speech
+   */
+  text: string;
+
+  /**
+   * Voice identifier for the synthesis
+   * @default "alloy"
+   */
+  voice?: Voice;
+
+  /**
+   * Speech speed adjustment (0.25 to 4.0)
+   * @default 1.0
+   */
+  speed?: number;
+
+  /**
+   * Output audio format
+   * @default "mp3"
+   */
+  format?: AudioFormat;
+
+  /**
+   * Maximum timeout in milliseconds
+   * @default 30000
+   */
+  timeout?: number;
+
+  /**
+   * Additional provider-specific options
+   */
+  config?: Record<string, unknown>;
+}
+
+/**
+ * Result from speech generation
+ *
+ * Matches AI SDK's GenerateSpeechResult pattern.
+ */
+export interface GenerateSpeechResult {
+  /**
+   * The generated audio as Uint8Array or ArrayBuffer
+   */
+  audio: Uint8Array | ArrayBuffer;
+
+  /**
+   * The audio format of the result
+   */
+  format: AudioFormat;
+
+  /**
+   * Total duration in seconds (if available from provider)
+   */
+  duration?: number;
+
+  /**
+   * Provider metadata
+   */
+  metadata?: {
+    provider?: string;
+    modelUsed?: string;
+    voiceUsed?: string;
+  };
+}
+
+/**
+ * Options for speech-to-text transcription
+ *
+ * Matches AI SDK's transcribe pattern while supporting multiple providers.
+ */
+export interface TranscribeOptions {
+  /**
+   * The transcription model to use (e.g., "whisper-1")
+   */
+  model: TranscriptionModel;
+
+  /**
+   * The audio data to transcribe
+   * Can be Uint8Array, ArrayBuffer, ReadableStream, or File/Blob
+   */
+  audio: Uint8Array | ArrayBuffer | ReadableStream<Uint8Array> | File | Blob;
+
+  /**
+   * Language code (e.g., "en", "es", "fr")
+   * If not provided, model will auto-detect
+   */
+  language?: string;
+
+  /**
+   * Optional prompt to guide transcription
+   */
+  prompt?: string;
+
+  /**
+   * Include timestamped segments in result
+   * @default false
+   */
+  timestamps?: boolean;
+
+  /**
+   * Maximum timeout in milliseconds
+   * @default 60000
+   */
+  timeout?: number;
+
+  /**
+   * Additional provider-specific options
+   */
+  config?: Record<string, unknown>;
+}
+
+/**
+ * Timestamped segment from transcription
+ */
+export interface TranscriptionSegment {
+  /**
+   * Start time in seconds
+   */
+  start: number;
+
+  /**
+   * End time in seconds
+   */
+  end: number;
+
+  /**
+   * Transcribed text for this segment
+   */
+  text: string;
+
+  /**
+   * Confidence score (0-1) if available
+   */
+  confidence?: number;
+}
+
+/**
+ * Result from transcription (speech-to-text)
+ *
+ * Matches AI SDK's TranscribeResult pattern.
+ */
+export interface TranscribeResult {
+  /**
+   * Full transcribed text
+   */
+  text: string;
+
+  /**
+   * Timestamped segments (if requested)
+   */
+  segments?: TranscriptionSegment[];
+
+  /**
+   * Detected language code (if available)
+   */
+  language?: string;
+
+  /**
+   * Duration of audio in seconds (if available)
+   */
+  duration?: number;
+
+  /**
+   * Provider metadata
+   */
+  metadata?: {
+    provider?: string;
+    modelUsed?: string;
+    confidence?: number;
+  };
+}
+
+/**
+ * Provider interface for speech generation
+ *
+ * Implement this interface for each speech provider (OpenAI, ElevenLabs, etc.)
+ */
+export interface SpeechProvider {
+  /**
+   * Generate speech from text
+   */
+  generateSpeech(
+    text: string,
+    options: Omit<GenerateSpeechOptions, 'model' | 'text'>,
+  ): Promise<GenerateSpeechResult>;
+
+  /**
+   * Stream speech generation (optional)
+   */
+  streamSpeech?(
+    text: string,
+    options: Omit<GenerateSpeechOptions, 'model' | 'text'>,
+  ): AsyncGenerator<Uint8Array>;
+}
+
+/**
+ * Provider interface for transcription
+ *
+ * Implement this interface for each transcription provider
+ */
+export interface TranscriptionProvider {
+  /**
+   * Transcribe audio to text
+   */
+  transcribe(
+    audio: Uint8Array | ArrayBuffer | ReadableStream<Uint8Array> | File | Blob,
+    options: Omit<TranscribeOptions, 'model' | 'audio'>,
+  ): Promise<TranscribeResult>;
+}
+
+/**
+ * Provider registry for managing multiple provider implementations
+ */
+interface ProviderRegistry {
+  speechProviders: Map<string, SpeechProvider>;
+  transcriptionProviders: Map<string, TranscriptionProvider>;
+}
+
+// Global provider registry
+const providerRegistry: ProviderRegistry = {
+  speechProviders: new Map(),
+  transcriptionProviders: new Map(),
+};
+
+/**
+ * Register a speech provider
+ *
+ * @param modelPrefix The model name prefix (e.g., "tts-1", "eleven_")
+ * @param provider The provider implementation
+ */
+export function registerSpeechProvider(modelPrefix: string, provider: SpeechProvider): void {
+  providerRegistry.speechProviders.set(modelPrefix, provider);
+}
+
+/**
+ * Register a transcription provider
+ *
+ * @param modelPrefix The model name prefix (e.g., "whisper", "deepgram")
+ * @param provider The provider implementation
+ */
+export function registerTranscriptionProvider(
+  modelPrefix: string,
+  provider: TranscriptionProvider,
+): void {
+  providerRegistry.transcriptionProviders.set(modelPrefix, provider);
+}
+
+/**
+ * Get the appropriate speech provider for a model
+ *
+ * @param model The model identifier
+ * @returns The provider or null if not found
+ */
+function getSpeechProvider(model: SpeechModel): SpeechProvider | null {
+  // Try exact match first
+  const direct = providerRegistry.speechProviders.get(model);
+  if (direct) {
+    return direct;
+  }
+
+  // Try prefix match (e.g., "tts-1" matches "tts-1-hd")
+  for (const [prefix, provider] of Array.from(providerRegistry.speechProviders.entries())) {
+    if (model.startsWith(prefix)) {
+      return provider;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Get the appropriate transcription provider for a model
+ *
+ * @param model The model identifier
+ * @returns The provider or null if not found
+ */
+function getTranscriptionProvider(model: TranscriptionModel): TranscriptionProvider | null {
+  // Try exact match first
+  const direct = providerRegistry.transcriptionProviders.get(model);
+  if (direct) {
+    return direct;
+  }
+
+  // Try prefix match
+  for (const [prefix, provider] of Array.from(providerRegistry.transcriptionProviders.entries())) {
+    if (model.startsWith(prefix)) {
+      return provider;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Top-level convenience function for speech generation (text-to-speech)
+ *
+ * Matches the Vercel AI SDK v6 `experimental_generateSpeech` pattern.
+ *
+ * @param options Configuration for speech generation
+ * @returns Promise<GenerateSpeechResult> The generated audio and metadata
+ *
+ * @example
+ * ```typescript
+ * const result = await generateSpeech({
+ *   model: "tts-1-hd",
+ *   text: "Hello, how can I help you today?",
+ *   voice: "alloy",
+ *   format: "mp3"
+ * });
+ * console.log(result.audio); // Uint8Array with audio data
+ * ```
+ */
+export async function generateSpeech(
+  options: GenerateSpeechOptions,
+): Promise<GenerateSpeechResult> {
+  const { model, timeout = 30000, ...restOptions } = options;
+
+  // Validate input
+  if (!model) {
+    throw new Error('Model is required for speech generation');
+  }
+
+  if (!options.text) {
+    throw new Error('Text is required for speech generation');
+  }
+
+  // Get provider
+  const provider = getSpeechProvider(model);
+  if (!provider) {
+    throw new Error(
+      `No speech provider registered for model: ${model}. ` +
+        `Available providers: ${Array.from(providerRegistry.speechProviders.keys()).join(', ')}`,
+    );
+  }
+
+  // Create timeout promise
+  const timeoutPromise = new Promise<never>((resolve, reject) => {
+    void resolve;
+    setTimeout(
+      () => reject(new Error(`Speech generation timeout after ${timeout}ms for model: ${model}`)),
+      timeout,
+    );
+  });
+
+  // Execute with timeout
+  return Promise.race([provider.generateSpeech(options.text, restOptions), timeoutPromise]);
+}
+
+/**
+ * Stream speech generation (text-to-speech)
+ *
+ * Provides streaming audio output for progressive playback.
+ * Supports providers that implement streaming.
+ *
+ * @param options Configuration for speech generation
+ * @returns AsyncGenerator that yields audio chunks as Uint8Array
+ *
+ * @example
+ * ```typescript
+ * for await (const chunk of streamSpeech({
+ *   model: "tts-1",
+ *   text: "Hello world",
+ *   voice: "nova"
+ * })) {
+ *   audioBuffer.write(chunk);
+ * }
+ * ```
+ */
+export async function* streamSpeech(options: GenerateSpeechOptions): AsyncGenerator<Uint8Array> {
+  const { model, timeout = 30000, ...restOptions } = options;
+
+  // Validate input
+  if (!model) {
+    throw new Error('Model is required for speech generation');
+  }
+
+  if (!options.text) {
+    throw new Error('Text is required for speech generation');
+  }
+
+  // Get provider
+  const provider = getSpeechProvider(model);
+  if (!provider) {
+    throw new Error(
+      `No speech provider registered for model: ${model}. ` +
+        `Available providers: ${Array.from(providerRegistry.speechProviders.keys()).join(', ')}`,
+    );
+  }
+
+  // Check if provider supports streaming
+  if (!provider.streamSpeech) {
+    throw new Error(
+      `Provider for model ${model} does not support streaming. ` +
+        `Use generateSpeech() for non-streaming audio generation.`,
+    );
+  }
+
+  // Create timeout mechanism for generator
+  const timeoutController = new AbortController();
+  const timeoutId = setTimeout(() => {
+    timeoutController.abort();
+  }, timeout);
+
+  try {
+    for await (const chunk of provider.streamSpeech(options.text, restOptions)) {
+      if (timeoutController.signal.aborted) {
+        throw new Error(`Speech generation timeout after ${timeout}ms for model: ${model}`);
+      }
+      yield chunk;
+    }
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
+
+/**
+ * Top-level convenience function for transcription (speech-to-text)
+ *
+ * Matches the Vercel AI SDK v6 `experimental_transcribe` pattern.
+ *
+ * @param options Configuration for transcription
+ * @returns Promise<TranscribeResult> Transcribed text and metadata
+ *
+ * @example
+ * ```typescript
+ * const result = await transcribe({
+ *   model: "whisper-1",
+ *   audio: audioBuffer,
+ *   language: "en",
+ *   timestamps: true
+ * });
+ * console.log(result.text); // Full transcribed text
+ * console.log(result.segments); // Timestamped segments
+ * ```
+ */
+export async function transcribe(options: TranscribeOptions): Promise<TranscribeResult> {
+  const { model, timeout = 60000, ...restOptions } = options;
+
+  // Validate input
+  if (!model) {
+    throw new Error('Model is required for transcription');
+  }
+
+  if (!options.audio) {
+    throw new Error('Audio data is required for transcription');
+  }
+
+  // Get provider
+  const provider = getTranscriptionProvider(model);
+  if (!provider) {
+    throw new Error(
+      `No transcription provider registered for model: ${model}. ` +
+        `Available providers: ${Array.from(providerRegistry.transcriptionProviders.keys()).join(', ')}`,
+    );
+  }
+
+  // Create timeout promise
+  const timeoutPromise = new Promise<never>((resolve, reject) => {
+    void resolve;
+    setTimeout(
+      () => reject(new Error(`Transcription timeout after ${timeout}ms for model: ${model}`)),
+      timeout,
+    );
+  });
+
+  // Execute with timeout
+  return Promise.race([provider.transcribe(options.audio, restOptions), timeoutPromise]);
+}
+
+/**
+ * Convert audio to Uint8Array if necessary
+ *
+ * Helper function for normalizing audio input across different formats.
+ *
+ * @param audio Input audio in various formats
+ * @returns Uint8Array representation of audio
+ */
+export async function normalizeAudioInput(
+  audio: Uint8Array | ArrayBuffer | ReadableStream<Uint8Array> | File | Blob,
+): Promise<Uint8Array> {
+  // Already Uint8Array
+  if (audio instanceof Uint8Array) {
+    return audio;
+  }
+
+  // ArrayBuffer
+  if (audio instanceof ArrayBuffer) {
+    return new Uint8Array(audio);
+  }
+
+  // File or Blob
+  if (audio instanceof File || audio instanceof Blob) {
+    const buffer = await audio.arrayBuffer();
+    return new Uint8Array(buffer);
+  }
+
+  // ReadableStream
+  if (audio instanceof ReadableStream) {
+    const chunks: Uint8Array[] = [];
+    const reader = audio.getReader();
+
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        chunks.push(value);
+      }
+    } finally {
+      reader.releaseLock();
+    }
+
+    // Combine chunks
+    const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
+    const result = new Uint8Array(totalLength);
+    let offset = 0;
+
+    for (const chunk of chunks) {
+      result.set(chunk, offset);
+      offset += chunk.length;
+    }
+
+    return result;
+  }
+
+  throw new Error(
+    'Invalid audio input: must be Uint8Array, ArrayBuffer, ReadableStream, File, or Blob',
+  );
+}
+
+/**
+ * Configuration interface for initializing providers
+ */
+export interface AudioProviderConfig {
+  speechProviders?: {
+    [modelPrefix: string]: SpeechProvider;
+  };
+  transcriptionProviders?: {
+    [modelPrefix: string]: TranscriptionProvider;
+  };
+}
+
+/**
+ * Initialize audio providers
+ *
+ * Call this function with your provider implementations to set up the module.
+ *
+ * @param config Provider configuration
+ *
+ * @example
+ * ```typescript
+ * initializeAudioProviders({
+ *   speechProviders: {
+ *     "tts-1": openaiSpeechProvider,
+ *     "eleven_": elevenLabsSpeechProvider
+ *   },
+ *   transcriptionProviders: {
+ *     "whisper": openaiTranscriptionProvider,
+ *     "deepgram": deepgramProvider
+ *   }
+ * });
+ * ```
+ */
+export function initializeAudioProviders(config: AudioProviderConfig): void {
+  if (config.speechProviders) {
+    for (const [key, provider] of Object.entries(config.speechProviders)) {
+      registerSpeechProvider(key, provider);
+    }
+  }
+
+  if (config.transcriptionProviders) {
+    for (const [key, provider] of Object.entries(config.transcriptionProviders)) {
+      registerTranscriptionProvider(key, provider);
+    }
+  }
+}
+
+/**
+ * Get registered providers (for debugging/monitoring)
+ *
+ * @returns Current provider registry state
+ */
+export function getRegisteredProviders(): {
+  speechProviders: string[];
+  transcriptionProviders: string[];
+} {
+  return {
+    speechProviders: Array.from(providerRegistry.speechProviders.keys()),
+    transcriptionProviders: Array.from(providerRegistry.transcriptionProviders.keys()),
+  };
+}
+
+// Types are already exported above in their definitions

package/src/output/multimodal/image.ts

@@ -0,0 +1,22 @@
+/**
+ * @fileoverview Multimodal Image Utilities
+ *
+ * Provides wrappers for image generation.
+ *
+ * @module @od-oneapp/ai-platform/multimodal/image
+ */
+
+import { generateImage } from '../../shared';
+
+// Re-export generateImage, potentially adding middleware/compliance wrappers later.
+export { generateImage };
+
+/**
+ * Standard inputs for image generation to normalize across models.
+ */
+export interface ImageGenerationOptions {
+  prompt: string;
+  aspectRatio?: '1:1' | '16:9' | '4:3';
+  size?: string;
+  n?: number;
+}

package/src/output/multimodal/index.ts

@@ -0,0 +1,32 @@
+/**
+ * @fileoverview Multimodal output module exports
+ *
+ * Barrel export for audio and speech/transcription functionality.
+ *
+ * @module @od-oneapp/ai-platform/output/multimodal
+ */
+
+export {
+  // Functions
+  generateSpeech,
+  streamSpeech,
+  transcribe,
+  normalizeAudioInput,
+  initializeAudioProviders,
+  getRegisteredProviders,
+  registerSpeechProvider,
+  registerTranscriptionProvider,
+  // Types
+  type GenerateSpeechOptions,
+  type GenerateSpeechResult,
+  type TranscribeOptions,
+  type TranscribeResult,
+  type TranscriptionSegment,
+  type SpeechProvider,
+  type TranscriptionProvider,
+  type AudioProviderConfig,
+  type SpeechModel,
+  type TranscriptionModel,
+  type AudioFormat,
+  type Voice,
+} from './audio';