@juspay/neurolink 9.61.1 → 9.62.0

package/dist/voice/providers/OpenAISTT.js

@@ -0,0 +1,285 @@
+/**
+ * OpenAI Whisper Speech-to-Text Handler
+ *
+ * Implementation of STT using OpenAI's Whisper model.
+ *
+ * @module voice/providers/OpenAISTT
+ */
+import { logger } from "../../utils/logger.js";
+import { STTError } from "../errors.js";
+/**
+ * OpenAI Whisper Speech-to-Text Handler
+ *
+ * Supports transcription and translation using OpenAI's Whisper model.
+ *
+ * @see https://platform.openai.com/docs/api-reference/audio
+ */
+export class OpenAISTT {
+    apiKey;
+    baseUrl = "https://api.openai.com/v1";
+    /**
+     * Maximum audio duration in seconds (25 minutes)
+     */
+    maxAudioDuration = 25 * 60;
+    /**
+     * Whisper does not support streaming
+     */
+    supportsStreaming = false;
+    constructor(apiKey) {
+        const resolvedKey = (apiKey ?? process.env.OPENAI_API_KEY ?? "").trim();
+        this.apiKey = resolvedKey.length > 0 ? resolvedKey : null;
+    }
+    isConfigured() {
+        return this.apiKey !== null;
+    }
+    getSupportedFormats() {
+        // OpenAI Whisper transcription API accepts: flac, m4a, mp3, mp4, mpeg,
+        // mpga, oga, ogg, opus, wav, webm. Keep this in sync with TTSAudioFormat
+        // — formats not listed in TTSAudioFormat are filtered out by the type.
+        return [
+            "mp3",
+            "wav",
+            "ogg",
+            "opus",
+            "m4a",
+            "flac",
+            "webm",
+            "mp4",
+            "mpeg",
+            "mpga",
+        ];
+    }
+    async getSupportedLanguages() {
+        // Whisper supports 100+ languages
+        // Return the most common ones
+        return [
+            {
+                code: "en",
+                name: "English",
+                supportsDiarization: false,
+                supportsPunctuation: true,
+            },
+            {
+                code: "es",
+                name: "Spanish",
+                supportsDiarization: false,
+                supportsPunctuation: true,
+            },
+            {
+                code: "fr",
+                name: "French",
+                supportsDiarization: false,
+                supportsPunctuation: true,
+            },
+            {
+                code: "de",
+                name: "German",
+                supportsDiarization: false,
+                supportsPunctuation: true,
+            },
+            {
+                code: "it",
+                name: "Italian",
+                supportsDiarization: false,
+                supportsPunctuation: true,
+            },
+            {
+                code: "pt",
+                name: "Portuguese",
+                supportsDiarization: false,
+                supportsPunctuation: true,
+            },
+            {
+                code: "ru",
+                name: "Russian",
+                supportsDiarization: false,
+                supportsPunctuation: true,
+            },
+            {
+                code: "ja",
+                name: "Japanese",
+                supportsDiarization: false,
+                supportsPunctuation: true,
+            },
+            {
+                code: "ko",
+                name: "Korean",
+                supportsDiarization: false,
+                supportsPunctuation: true,
+            },
+            {
+                code: "zh",
+                name: "Chinese",
+                supportsDiarization: false,
+                supportsPunctuation: true,
+            },
+            {
+                code: "ar",
+                name: "Arabic",
+                supportsDiarization: false,
+                supportsPunctuation: true,
+            },
+            {
+                code: "hi",
+                name: "Hindi",
+                supportsDiarization: false,
+                supportsPunctuation: true,
+            },
+        ];
+    }
+    async transcribe(audio, options = {}) {
+        if (!this.apiKey) {
+            throw STTError.providerNotConfigured("whisper");
+        }
+        const audioBuffer = Buffer.isBuffer(audio) ? audio : Buffer.from(audio);
+        if (audioBuffer.length === 0) {
+            throw STTError.audioEmpty("whisper");
+        }
+        const whisperOptions = options;
+        const startTime = Date.now();
+        try {
+            // Prepare form data
+            const formData = new FormData();
+            // Add audio file - convert Buffer to Uint8Array for compatibility
+            const audioBlob = new Blob([new Uint8Array(audioBuffer)], {
+                type: this.getMimeType(options.format ?? "wav"),
+            });
+            formData.append("file", audioBlob, `audio.${options.format ?? "wav"}`);
+            // Add model
+            formData.append("model", whisperOptions.model ?? "whisper-1");
+            // Add optional parameters
+            if (options.language) {
+                formData.append("language", options.language);
+            }
+            if (whisperOptions.prompt) {
+                formData.append("prompt", whisperOptions.prompt);
+            }
+            if (whisperOptions.temperature !== undefined) {
+                formData.append("temperature", whisperOptions.temperature.toString());
+            }
+            // Request verbose_json for detailed response
+            const responseFormat = whisperOptions.responseFormat ?? "verbose_json";
+            formData.append("response_format", responseFormat);
+            // Add timestamp granularities for word-level timestamps
+            if (options.wordTimestamps && responseFormat === "verbose_json") {
+                formData.append("timestamp_granularities[]", "word");
+                formData.append("timestamp_granularities[]", "segment");
+            }
+            // Choose endpoint based on translation option
+            const endpoint = whisperOptions.translate
+                ? `${this.baseUrl}/audio/translations`
+                : `${this.baseUrl}/audio/transcriptions`;
+            const controller = new AbortController();
+            const timeoutId = setTimeout(() => controller.abort(), 30000);
+            let response;
+            try {
+                response = await fetch(endpoint, {
+                    method: "POST",
+                    headers: {
+                        Authorization: `Bearer ${this.apiKey}`,
+                    },
+                    body: formData,
+                    signal: controller.signal,
+                });
+            }
+            catch (fetchErr) {
+                if (fetchErr instanceof Error && fetchErr.name === "AbortError") {
+                    throw STTError.transcriptionFailed("OpenAI STT request timed out after 30 seconds", "whisper", fetchErr);
+                }
+                throw fetchErr;
+            }
+            finally {
+                clearTimeout(timeoutId);
+            }
+            if (!response.ok) {
+                const errorData = await response
+                    .json()
+                    .catch(() => Object.create(null));
+                const errorMessage = errorData.error?.message ||
+                    `HTTP ${response.status}`;
+                throw STTError.transcriptionFailed(errorMessage, "whisper");
+            }
+            const latency = Date.now() - startTime;
+            // Parse response based on format
+            if (responseFormat === "text") {
+                const text = await response.text();
+                return {
+                    text,
+                    confidence: 0.95, // Whisper doesn't return confidence
+                    metadata: {
+                        latency,
+                        provider: "whisper",
+                        model: whisperOptions.model ?? "whisper-1",
+                    },
+                };
+            }
+            const data = (await response.json());
+            // Build result
+            const result = {
+                text: data.text,
+                confidence: 0.95, // Whisper doesn't return per-result confidence
+                language: data.language,
+                duration: data.duration,
+                metadata: {
+                    latency,
+                    provider: "whisper",
+                    model: whisperOptions.model ?? "whisper-1",
+                    task: data.task,
+                },
+            };
+            // Add word timings if available
+            if (data.words && data.words.length > 0) {
+                result.words = data.words.map((word) => ({
+                    word: word.word,
+                    startTime: word.start,
+                    endTime: word.end,
+                }));
+            }
+            // Add segments
+            if (data.segments && data.segments.length > 0) {
+                result.segments = data.segments.map((segment, index) => ({
+                    index,
+                    text: segment.text,
+                    isFinal: true,
+                    confidence: Math.exp(segment.avg_logprob), // Convert log prob to confidence
+                    startTime: segment.start,
+                    endTime: segment.end,
+                }));
+            }
+            logger.info(`[WhisperSTTHandler] Transcribed ${data.duration?.toFixed(1) ?? "?"}s audio in ${latency}ms`);
+            return result;
+        }
+        catch (err) {
+            if (err instanceof STTError) {
+                throw err;
+            }
+            const errorMessage = err instanceof Error ? err.message : String(err || "Unknown error");
+            logger.error(`[WhisperSTTHandler] Transcription failed: ${errorMessage}`);
+            throw STTError.transcriptionFailed(errorMessage, "whisper", err instanceof Error ? err : undefined);
+        }
+    }
+    /**
+     * Get MIME type for audio format. Whisper auto-detects from headers, but
+     * sending a correct MIME helps providers / proxies that sniff Content-Type.
+     * Must stay aligned with `getSupportedFormats()`.
+     */
+    getMimeType(format) {
+        const mimeTypes = {
+            mp3: "audio/mpeg",
+            wav: "audio/wav",
+            ogg: "audio/ogg",
+            opus: "audio/opus",
+            m4a: "audio/mp4",
+            flac: "audio/flac",
+            webm: "audio/webm",
+            mp4: "audio/mp4",
+            mpeg: "audio/mpeg",
+            mpga: "audio/mpeg",
+        };
+        return mimeTypes[format] ?? "audio/wav";
+    }
+}
+// Export as named exports for compatibility
+export { OpenAISTT as WhisperSTT };
+export { OpenAISTT as WhisperSTTHandler };
+export { OpenAISTT as OpenAISTTHandler };

package/dist/voice/providers/OpenAITTS.d.ts

@@ -0,0 +1,49 @@
+/**
+ * OpenAI Text-to-Speech Handler
+ *
+ * Implementation of TTS using OpenAI's TTS API.
+ *
+ * @module voice/providers/OpenAITTS
+ */
+import type { TTSHandler, TTSOptions, TTSResult, TTSVoice } from "../../types/index.js";
+/**
+ * OpenAI Text-to-Speech Handler
+ *
+ * Supports high-quality neural TTS with multiple voices.
+ *
+ * @see https://platform.openai.com/docs/api-reference/audio/createSpeech
+ */
+export declare class OpenAITTS implements TTSHandler {
+    private readonly apiKey;
+    private readonly baseUrl;
+    /**
+     * Maximum text length (4096 characters)
+     */
+    readonly maxTextLength = 4096;
+    /**
+     * Available voices
+     */
+    private static readonly VOICES;
+    constructor(apiKey?: string);
+    isConfigured(): boolean;
+    getVoices(languageCode?: string): Promise<TTSVoice[]>;
+    synthesize(text: string, options?: TTSOptions): Promise<TTSResult>;
+    /**
+     * Map TTSAudioFormat to OpenAI response_format.
+     * OpenAI TTS supports: mp3, wav, opus (ogg maps to opus).
+     * Unsupported formats are coerced to mp3 with a warning.
+     */
+    private mapFormat;
+    /**
+     * Get sample rate for format
+     */
+    private getSampleRate;
+    /**
+     * Map the OpenAI `response_format` string back to the canonical
+     * `TTSAudioFormat` so `TTSResult.format` reflects what the API actually
+     * returned (mapFormat() coerces unsupported requests to "mp3"). Note:
+     * OpenAI returns Ogg-Opus for both "ogg" and "opus" requests — both
+     * surface as "opus" since the bytes are an .ogg/Opus container.
+     */
+    private effectiveFormat;
+}

package/dist/voice/providers/OpenAITTS.js

@@ -0,0 +1,270 @@
+/**
+ * OpenAI Text-to-Speech Handler
+ *
+ * Implementation of TTS using OpenAI's TTS API.
+ *
+ * @module voice/providers/OpenAITTS
+ */
+import { ErrorCategory, ErrorSeverity } from "../../constants/enums.js";
+import { logger } from "../../utils/logger.js";
+import { TTS_ERROR_CODES, TTSError } from "../../utils/ttsProcessor.js";
+/**
+ * OpenAI Text-to-Speech Handler
+ *
+ * Supports high-quality neural TTS with multiple voices.
+ *
+ * @see https://platform.openai.com/docs/api-reference/audio/createSpeech
+ */
+export class OpenAITTS {
+    apiKey;
+    baseUrl = "https://api.openai.com/v1";
+    /**
+     * Maximum text length (4096 characters)
+     */
+    maxTextLength = 4096;
+    /**
+     * Available voices
+     */
+    static VOICES = [
+        {
+            id: "alloy",
+            name: "Alloy",
+            languageCode: "en",
+            languageCodes: ["en"],
+            gender: "neutral",
+            type: "neural",
+        },
+        {
+            id: "echo",
+            name: "Echo",
+            languageCode: "en",
+            languageCodes: ["en"],
+            gender: "male",
+            type: "neural",
+        },
+        {
+            id: "fable",
+            name: "Fable",
+            languageCode: "en",
+            languageCodes: ["en"],
+            gender: "neutral",
+            type: "neural",
+        },
+        {
+            id: "onyx",
+            name: "Onyx",
+            languageCode: "en",
+            languageCodes: ["en"],
+            gender: "male",
+            type: "neural",
+        },
+        {
+            id: "nova",
+            name: "Nova",
+            languageCode: "en",
+            languageCodes: ["en"],
+            gender: "female",
+            type: "neural",
+        },
+        {
+            id: "shimmer",
+            name: "Shimmer",
+            languageCode: "en",
+            languageCodes: ["en"],
+            gender: "female",
+            type: "neural",
+        },
+    ];
+    constructor(apiKey) {
+        const resolvedKey = (apiKey ?? process.env.OPENAI_API_KEY ?? "").trim();
+        this.apiKey = resolvedKey.length > 0 ? resolvedKey : null;
+    }
+    isConfigured() {
+        return this.apiKey !== null;
+    }
+    async getVoices(languageCode) {
+        // OpenAI voices are pre-defined, filter by language if provided
+        if (languageCode && !languageCode.startsWith("en")) {
+            // OpenAI TTS works with multiple languages but voices are English-named
+            return OpenAITTS.VOICES;
+        }
+        return OpenAITTS.VOICES;
+    }
+    async synthesize(text, options = {}) {
+        if (!this.apiKey) {
+            throw new TTSError({
+                code: TTS_ERROR_CODES.PROVIDER_NOT_CONFIGURED,
+                message: "OpenAI TTS API key not configured",
+                category: ErrorCategory.CONFIGURATION,
+                severity: ErrorSeverity.HIGH,
+                retriable: false,
+            });
+        }
+        const startTime = Date.now();
+        const openaiOptions = options;
+        try {
+            // Determine model based on quality
+            const model = openaiOptions.model ??
+                (options.quality === "hd" ? "tts-1-hd" : "tts-1");
+            // Determine voice
+            const voice = options.voice ?? "alloy";
+            // Determine format
+            const responseFormat = this.mapFormat(options.format ?? "mp3");
+            // Build request
+            const requestBody = {
+                model,
+                input: text,
+                voice,
+                response_format: responseFormat,
+                speed: options.speed ?? 1.0,
+            };
+            const controller = new AbortController();
+            const timeoutId = setTimeout(() => controller.abort(), 30000);
+            let response;
+            try {
+                response = await fetch(`${this.baseUrl}/audio/speech`, {
+                    method: "POST",
+                    headers: {
+                        Authorization: `Bearer ${this.apiKey}`,
+                        "Content-Type": "application/json",
+                    },
+                    body: JSON.stringify(requestBody),
+                    signal: controller.signal,
+                });
+            }
+            catch (fetchErr) {
+                if (fetchErr instanceof Error && fetchErr.name === "AbortError") {
+                    throw new TTSError({
+                        code: TTS_ERROR_CODES.SYNTHESIS_FAILED,
+                        message: "OpenAI TTS request timed out after 30 seconds",
+                        category: ErrorCategory.NETWORK,
+                        severity: ErrorSeverity.HIGH,
+                        retriable: true,
+                        originalError: fetchErr,
+                    });
+                }
+                throw fetchErr;
+            }
+            finally {
+                clearTimeout(timeoutId);
+            }
+            if (!response.ok) {
+                const errorData = await response
+                    .json()
+                    .catch(() => Object.create(null));
+                const errorMessage = errorData.error?.message ||
+                    `HTTP ${response.status}`;
+                // Preserve HTTP status so the outer catch doesn't mark a permanent
+                // 4xx (auth, bad input) as retriable and trigger pointless retry loops.
+                const retriable = response.status === 408 ||
+                    response.status === 429 ||
+                    response.status >= 500;
+                throw new TTSError({
+                    code: TTS_ERROR_CODES.SYNTHESIS_FAILED,
+                    message: errorMessage,
+                    category: retriable ? ErrorCategory.NETWORK : ErrorCategory.EXECUTION,
+                    severity: ErrorSeverity.HIGH,
+                    retriable,
+                    context: { status: response.status, model, responseFormat },
+                });
+            }
+            const latency = Date.now() - startTime;
+            // Get audio buffer
+            const arrayBuffer = await response.arrayBuffer();
+            const audioBuffer = Buffer.from(arrayBuffer);
+            // Use the *effective* output format (post-mapFormat fallback), not the
+            // requested format — otherwise mp3-coerced "m4a" requests would mislabel
+            // the buffer and break consumer file-extension routing.
+            const effectiveFormat = this.effectiveFormat(responseFormat);
+            const result = {
+                buffer: audioBuffer,
+                format: effectiveFormat,
+                size: audioBuffer.length,
+                voice,
+                sampleRate: this.getSampleRate(effectiveFormat),
+                metadata: {
+                    latency,
+                    provider: "openai-tts",
+                    model,
+                    requestedFormat: options.format,
+                    responseFormat,
+                },
+            };
+            logger.info(`[OpenAITTSHandler] Synthesized ${audioBuffer.length} bytes in ${latency}ms`);
+            return result;
+        }
+        catch (err) {
+            if (err instanceof TTSError) {
+                throw err;
+            }
+            const errorMessage = err instanceof Error ? err.message : String(err || "Unknown error");
+            logger.error(`[OpenAITTSHandler] Synthesis failed: ${errorMessage}`);
+            throw new TTSError({
+                code: TTS_ERROR_CODES.SYNTHESIS_FAILED,
+                message: `Synthesis failed: ${errorMessage}`,
+                category: ErrorCategory.EXECUTION,
+                severity: ErrorSeverity.HIGH,
+                retriable: true,
+                context: { textLength: text.length },
+                originalError: err instanceof Error ? err : undefined,
+            });
+        }
+    }
+    /**
+     * Map TTSAudioFormat to OpenAI response_format.
+     * OpenAI TTS supports: mp3, wav, opus (ogg maps to opus).
+     * Unsupported formats are coerced to mp3 with a warning.
+     */
+    mapFormat(format) {
+        const formats = {
+            mp3: "mp3",
+            wav: "wav",
+            ogg: "opus", // OpenAI uses opus for ogg
+            opus: "opus",
+            // OpenAI's "pcm" is raw 16-bit signed LE @ 24kHz (no header) — maps to
+            // canonical pcm16 in TTSResult.format. See effectiveFormat() below.
+            pcm16: "pcm",
+        };
+        const mapped = formats[format];
+        if (mapped === undefined) {
+            logger.warn(`[OpenAITTSHandler] Unsupported format "${format}" — falling back to "mp3". Supported formats: mp3, wav, ogg, opus, pcm16.`);
+            return "mp3";
+        }
+        return mapped;
+    }
+    /**
+     * Get sample rate for format
+     */
+    getSampleRate(format) {
+        switch (format) {
+            case "opus":
+            case "ogg":
+                return 48000;
+            default:
+                return 24000;
+        }
+    }
+    /**
+     * Map the OpenAI `response_format` string back to the canonical
+     * `TTSAudioFormat` so `TTSResult.format` reflects what the API actually
+     * returned (mapFormat() coerces unsupported requests to "mp3"). Note:
+     * OpenAI returns Ogg-Opus for both "ogg" and "opus" requests — both
+     * surface as "opus" since the bytes are an .ogg/Opus container.
+     */
+    effectiveFormat(responseFormat) {
+        switch (responseFormat) {
+            case "mp3":
+                return "mp3";
+            case "wav":
+                return "wav";
+            case "opus":
+                return "opus";
+            // Raw PCM (16-bit signed LE @ 24kHz, no header) — keep semantics in
+            // TTSResult.format so consumers don't write raw bytes to a .wav file.
+            case "pcm":
+                return "pcm16";
+            default:
+                return "mp3";
+        }
+    }
+}