@juspay/neurolink 9.61.1 → 9.62.0

package/dist/lib/voice/RealtimeVoiceAPI.js

@@ -0,0 +1,439 @@
+/**
+ * Realtime Voice API Infrastructure
+ *
+ * Base handler and processor for realtime voice communication.
+ * Supports bidirectional audio streaming with providers like OpenAI and Gemini.
+ *
+ * @module voice/RealtimeVoiceAPI
+ */
+import { logger } from "../utils/logger.js";
+import { RealtimeError } from "./errors.js";
+import { DEFAULT_REALTIME_CONFIG, REALTIME_ERROR_CODES, } from "../types/index.js";
+import { ErrorCategory, ErrorSeverity } from "../constants/enums.js";
+/**
+ * Realtime Processor class for orchestrating realtime voice operations
+ *
+ * Provides a unified interface for realtime voice across multiple providers.
+ *
+ * @example
+ * ```typescript
+ * // Register a handler (typically done in providerRegistry.ts on startup)
+ * RealtimeProcessor.registerHandler('openai-realtime', openaiHandler);
+ *
+ * // Connect to a session — the first arg is the registered handler key,
+ * // and `config.provider` must match the same key.
+ * const session = await RealtimeProcessor.connect('openai-realtime', {
+ *   provider: 'openai-realtime',
+ *   voice: 'alloy',
+ *   systemPrompt: 'You are a helpful assistant.'
+ * });
+ *
+ * // Send audio
+ * await RealtimeProcessor.sendAudio('openai-realtime', audioBuffer);
+ *
+ * // Disconnect
+ * await RealtimeProcessor.disconnect('openai-realtime');
+ * ```
+ */
+export class RealtimeProcessor {
+    /**
+     * Handler registry mapping provider names to Realtime handlers
+     */
+    static handlers = new Map();
+    /**
+     * Active sessions by provider
+     */
+    static sessions = new Map();
+    /**
+     * Register a Realtime handler for a specific provider
+     *
+     * @param providerName - Provider identifier (e.g., 'openai', 'gemini')
+     * @param handler - Realtime handler implementation
+     */
+    static registerHandler(providerName, handler) {
+        if (!providerName) {
+            throw new Error("Provider name is required");
+        }
+        if (!handler) {
+            throw new Error("Handler is required");
+        }
+        const normalizedName = providerName.toLowerCase();
+        if (this.handlers.has(normalizedName)) {
+            logger.warn(`[RealtimeProcessor] Overwriting existing handler for provider: ${normalizedName}`);
+        }
+        this.handlers.set(normalizedName, handler);
+        logger.debug(`[RealtimeProcessor] Registered Realtime handler for provider: ${normalizedName}`);
+    }
+    /**
+     * Get a registered Realtime handler by provider name
+     */
+    static getHandler(providerName) {
+        const normalizedName = providerName.toLowerCase();
+        return this.handlers.get(normalizedName);
+    }
+    /**
+     * Check if a provider is supported
+     */
+    static supports(providerName) {
+        if (!providerName) {
+            return false;
+        }
+        const normalizedName = providerName.toLowerCase();
+        return this.handlers.has(normalizedName);
+    }
+    /**
+     * Get list of all registered providers
+     */
+    static getProviders() {
+        return Array.from(this.handlers.keys());
+    }
+    /**
+     * Connect to a realtime session
+     *
+     * @param provider - Provider identifier
+     * @param config - Session configuration
+     * @param handlers - Event handlers
+     * @returns Session information
+     */
+    static async connect(provider, config, handlers) {
+        const handler = this.getHandler(provider);
+        if (!handler) {
+            throw RealtimeError.providerNotSupported(provider, Array.from(this.handlers.keys()));
+        }
+        if (!handler.isConfigured()) {
+            throw RealtimeError.providerNotConfigured(provider);
+        }
+        // Check for existing session
+        if (handler.isConnected()) {
+            throw RealtimeError.sessionAlreadyActive(provider);
+        }
+        // Merge with defaults
+        const mergedConfig = {
+            ...DEFAULT_REALTIME_CONFIG,
+            ...config,
+        };
+        // Register event handlers if provided
+        if (handlers) {
+            handler.on(handlers);
+        }
+        try {
+            logger.debug(`[RealtimeProcessor] Connecting to provider: ${provider}`);
+            const session = await handler.connect(mergedConfig);
+            this.sessions.set(provider.toLowerCase(), session);
+            logger.info(`[RealtimeProcessor] Connected to ${provider} session: ${session.id}`);
+            return session;
+        }
+        catch (err) {
+            if (handlers) {
+                handler.off();
+            }
+            if (err instanceof RealtimeError) {
+                throw err;
+            }
+            const errorMessage = err instanceof Error ? err.message : String(err || "Unknown error");
+            throw RealtimeError.connectionFailed(errorMessage, provider, err instanceof Error ? err : undefined);
+        }
+    }
+    /**
+     * Disconnect from a realtime session
+     *
+     * @param provider - Provider identifier
+     */
+    static async disconnect(provider) {
+        const handler = this.getHandler(provider);
+        if (!handler) {
+            throw RealtimeError.providerNotSupported(provider, Array.from(this.handlers.keys()));
+        }
+        if (!handler.isConnected()) {
+            logger.warn(`[RealtimeProcessor] No active session for provider: ${provider}`);
+            return;
+        }
+        try {
+            await handler.disconnect();
+            this.sessions.delete(provider.toLowerCase());
+            handler.off();
+            logger.info(`[RealtimeProcessor] Disconnected from ${provider}`);
+        }
+        catch (err) {
+            if (err instanceof RealtimeError) {
+                throw err;
+            }
+            const errorMessage = err instanceof Error ? err.message : String(err || "Unknown error");
+            throw RealtimeError.protocolError(`Disconnect failed: ${errorMessage}`, provider, err instanceof Error ? err : undefined);
+        }
+    }
+    /**
+     * Send audio to a realtime session
+     *
+     * @param provider - Provider identifier
+     * @param audio - Audio data
+     */
+    static async sendAudio(provider, audio) {
+        const handler = this.getHandler(provider);
+        if (!handler) {
+            throw RealtimeError.providerNotSupported(provider, Array.from(this.handlers.keys()));
+        }
+        if (!handler.isConnected()) {
+            throw RealtimeError.sessionNotActive(provider);
+        }
+        try {
+            await handler.sendAudio(audio);
+        }
+        catch (err) {
+            if (err instanceof RealtimeError) {
+                throw err;
+            }
+            const errorMessage = err instanceof Error ? err.message : String(err || "Unknown error");
+            throw RealtimeError.audioStreamError(errorMessage, provider);
+        }
+    }
+    /**
+     * Send text to a realtime session
+     *
+     * @param provider - Provider identifier
+     * @param text - Text to send
+     */
+    static async sendText(provider, text) {
+        const handler = this.getHandler(provider);
+        if (!handler) {
+            throw RealtimeError.providerNotSupported(provider, Array.from(this.handlers.keys()));
+        }
+        if (!handler.isConnected()) {
+            throw RealtimeError.sessionNotActive(provider);
+        }
+        if (!handler.sendText) {
+            throw new RealtimeError({
+                code: REALTIME_ERROR_CODES.PROTOCOL_ERROR,
+                message: `Provider "${provider}" does not support text input`,
+                category: ErrorCategory.VALIDATION,
+                severity: ErrorSeverity.MEDIUM,
+                context: { provider },
+            });
+        }
+        // Normalize provider exceptions into RealtimeError so callers see a
+        // consistent error taxonomy across sendAudio/sendText/triggerResponse/
+        // cancelResponse — previously raw provider errors leaked from the
+        // text/control paths while sendAudio wrapped them (CodeRabbit review).
+        try {
+            await handler.sendText(text);
+        }
+        catch (err) {
+            if (err instanceof RealtimeError) {
+                throw err;
+            }
+            throw new RealtimeError({
+                code: REALTIME_ERROR_CODES.PROTOCOL_ERROR,
+                message: `sendText failed: ${err instanceof Error ? err.message : String(err)}`,
+                category: ErrorCategory.NETWORK,
+                severity: ErrorSeverity.MEDIUM,
+                retriable: true,
+                context: { provider },
+                originalError: err instanceof Error ? err : undefined,
+            });
+        }
+    }
+    /**
+     * Trigger a response from the model (manual turn detection)
+     *
+     * @param provider - Provider identifier
+     */
+    static async triggerResponse(provider) {
+        const handler = this.getHandler(provider);
+        if (!handler) {
+            throw RealtimeError.providerNotSupported(provider, Array.from(this.handlers.keys()));
+        }
+        if (!handler.isConnected()) {
+            throw RealtimeError.sessionNotActive(provider);
+        }
+        if (handler.triggerResponse) {
+            try {
+                await handler.triggerResponse();
+            }
+            catch (err) {
+                if (err instanceof RealtimeError) {
+                    throw err;
+                }
+                throw new RealtimeError({
+                    code: REALTIME_ERROR_CODES.PROTOCOL_ERROR,
+                    message: `triggerResponse failed: ${err instanceof Error ? err.message : String(err)}`,
+                    category: ErrorCategory.NETWORK,
+                    severity: ErrorSeverity.MEDIUM,
+                    retriable: true,
+                    context: { provider },
+                    originalError: err instanceof Error ? err : undefined,
+                });
+            }
+        }
+    }
+    /**
+     * Cancel the current response
+     *
+     * @param provider - Provider identifier
+     */
+    static async cancelResponse(provider) {
+        const handler = this.getHandler(provider);
+        if (!handler) {
+            throw RealtimeError.providerNotSupported(provider, Array.from(this.handlers.keys()));
+        }
+        if (!handler.isConnected()) {
+            return; // Nothing to cancel
+        }
+        if (handler.cancelResponse) {
+            try {
+                await handler.cancelResponse();
+            }
+            catch (err) {
+                if (err instanceof RealtimeError) {
+                    throw err;
+                }
+                throw new RealtimeError({
+                    code: REALTIME_ERROR_CODES.PROTOCOL_ERROR,
+                    message: `cancelResponse failed: ${err instanceof Error ? err.message : String(err)}`,
+                    category: ErrorCategory.NETWORK,
+                    severity: ErrorSeverity.MEDIUM,
+                    retriable: true,
+                    context: { provider },
+                    originalError: err instanceof Error ? err : undefined,
+                });
+            }
+        }
+    }
+    /**
+     * Get current session for a provider
+     *
+     * @param provider - Provider identifier
+     * @returns Session or null
+     */
+    static getSession(provider) {
+        const handler = this.getHandler(provider);
+        return handler?.getSession() ?? null;
+    }
+    /**
+     * Check if a provider has an active session
+     *
+     * @param provider - Provider identifier
+     */
+    static isConnected(provider) {
+        const handler = this.getHandler(provider);
+        return handler?.isConnected() ?? false;
+    }
+    /**
+     * Get supported formats for a provider
+     *
+     * @param provider - Provider identifier
+     */
+    static getSupportedFormats(provider) {
+        const handler = this.getHandler(provider);
+        return handler?.getSupportedFormats() ?? [];
+    }
+    /**
+     * Clear all handlers and sessions (for testing)
+     */
+    static clearHandlers() {
+        // Disconnect all active sessions
+        for (const [provider] of this.sessions) {
+            const handler = this.handlers.get(provider);
+            if (handler?.isConnected()) {
+                handler.disconnect().catch(() => {
+                    // Ignore errors during cleanup
+                });
+            }
+        }
+        this.handlers.clear();
+        this.sessions.clear();
+        logger.debug("[RealtimeProcessor] Cleared all handlers and sessions");
+    }
+}
+/**
+ * Base Realtime Handler with common functionality
+ *
+ * Providers can extend this class for common behavior.
+ */
+export class BaseRealtimeHandler {
+    session = null;
+    eventHandlers = null;
+    state = "disconnected";
+    isConnected() {
+        return this.state === "connected";
+    }
+    getSession() {
+        return this.session;
+    }
+    on(handlers) {
+        this.eventHandlers = handlers;
+    }
+    off() {
+        this.eventHandlers = null;
+    }
+    /**
+     * Emit state change event
+     */
+    emitStateChange(newState) {
+        this.state = newState;
+        if (this.session) {
+            this.session.state = newState;
+            this.session.lastActivityAt = new Date();
+        }
+        this.eventHandlers?.onStateChange?.(newState);
+    }
+    /**
+     * Emit audio event
+     */
+    emitAudio(chunk) {
+        this.eventHandlers?.onAudio?.(chunk);
+    }
+    /**
+     * Emit transcript event
+     */
+    emitTranscript(text, isFinal) {
+        this.eventHandlers?.onTranscript?.(text, isFinal);
+    }
+    /**
+     * Emit text event
+     */
+    emitText(text, isFinal) {
+        this.eventHandlers?.onText?.(text, isFinal);
+    }
+    /**
+     * Emit function call event
+     */
+    async emitFunctionCall(name, args) {
+        if (this.eventHandlers?.onFunctionCall) {
+            return this.eventHandlers.onFunctionCall(name, args);
+        }
+        return undefined;
+    }
+    /**
+     * Emit error event
+     */
+    emitError(error) {
+        this.eventHandlers?.onError?.(error);
+    }
+    /**
+     * Emit turn start event
+     */
+    emitTurnStart() {
+        this.eventHandlers?.onTurnStart?.();
+    }
+    /**
+     * Emit turn end event
+     */
+    emitTurnEnd() {
+        this.eventHandlers?.onTurnEnd?.();
+    }
+    /**
+     * Create a session object
+     */
+    createSession(id, config) {
+        return {
+            id,
+            state: "connected",
+            provider: this.name,
+            model: config.model,
+            createdAt: new Date(),
+            lastActivityAt: new Date(),
+            config,
+        };
+    }
+}
+//# sourceMappingURL=RealtimeVoiceAPI.js.map

package/dist/lib/voice/audio-utils.d.ts

@@ -0,0 +1,135 @@
+/**
+ * Audio Utilities for Voice Module
+ *
+ * Provides audio format conversion, duration calculation, and buffer utilities.
+ *
+ * @module voice/audio-utils
+ */
+import type { TTSAudioFormat } from "../types/index.js";
+/**
+ * Detect audio format from buffer
+ *
+ * @param buffer - Audio data buffer
+ * @returns Detected audio format or null
+ */
+export declare function detectAudioFormat(buffer: Buffer): TTSAudioFormat | null;
+/**
+ * Get MIME type for audio format
+ *
+ * @param format - Audio format
+ * @returns MIME type string
+ */
+export declare function getMimeType(format: TTSAudioFormat): string;
+/**
+ * Get file extension for audio format
+ *
+ * @param format - Audio format
+ * @returns File extension with dot
+ */
+export declare function getFileExtension(format: TTSAudioFormat): string;
+/**
+ * Calculate audio duration from buffer
+ *
+ * @param buffer - Audio data buffer
+ * @param format - Audio format (optional, will be detected if not provided)
+ * @param sampleRate - Sample rate in Hz (optional, will be extracted if possible)
+ * @returns Duration in seconds, or undefined if cannot be calculated
+ */
+export declare function calculateDuration(buffer: Buffer, format?: TTSAudioFormat, sampleRate?: number): number | undefined;
+/**
+ * Convert audio format (basic conversion)
+ *
+ * Note: For full format conversion, external tools like ffmpeg would be needed.
+ * This provides basic PCM resampling only.
+ *
+ * @param buffer - Input audio buffer
+ * @param fromFormat - Source format
+ * @param toFormat - Target format
+ * @param options - Conversion options
+ * @returns Converted audio buffer
+ */
+export declare function convertAudioFormat(buffer: Buffer, fromFormat: TTSAudioFormat, toFormat: TTSAudioFormat, _options?: Record<string, unknown>): Promise<Buffer>;
+/**
+ * Create PCM audio buffer from raw samples
+ *
+ * @param samples - Array of sample values (-1 to 1)
+ * @param sampleRate - Sample rate in Hz
+ * @param bitDepth - Bit depth (8, 16, 24, or 32)
+ * @returns PCM audio buffer
+ */
+export declare function createPcmBuffer(samples: number[], _sampleRate?: number, bitDepth?: 8 | 16 | 24 | 32): Buffer;
+/**
+ * Extract PCM samples from buffer
+ *
+ * @param buffer - PCM audio buffer
+ * @param bitDepth - Bit depth (8, 16, 24, or 32)
+ * @returns Array of sample values (-1 to 1)
+ */
+export declare function extractPcmSamples(buffer: Buffer, bitDepth?: 8 | 16 | 24 | 32): number[];
+/**
+ * Resample PCM audio
+ *
+ * @param samples - Input samples
+ * @param fromSampleRate - Source sample rate
+ * @param toSampleRate - Target sample rate
+ * @returns Resampled samples
+ */
+export declare function resamplePcm(samples: number[], fromSampleRate: number, toSampleRate: number): number[];
+/**
+ * Normalize audio levels
+ *
+ * @param samples - Input samples
+ * @param targetPeak - Target peak level (0 to 1)
+ * @returns Normalized samples
+ */
+export declare function normalizeAudio(samples: number[], targetPeak?: number): number[];
+/**
+ * Create a WAV header
+ *
+ * @param dataSize - Size of audio data in bytes
+ * @param sampleRate - Sample rate in Hz
+ * @param channels - Number of channels
+ * @param bitDepth - Bit depth
+ * @returns WAV header buffer
+ */
+export declare function createWavHeader(dataSize: number, sampleRate?: number, channels?: number, bitDepth?: number): Buffer;
+/**
+ * Create a complete WAV file from PCM data
+ *
+ * @param pcmData - PCM audio data
+ * @param sampleRate - Sample rate in Hz
+ * @param channels - Number of channels
+ * @param bitDepth - Bit depth
+ * @returns Complete WAV file buffer
+ */
+export declare function createWavFile(pcmData: Buffer, sampleRate?: number, channels?: number, bitDepth?: number): Buffer;
+/**
+ * Split audio buffer into chunks
+ *
+ * @param buffer - Audio buffer to split
+ * @param chunkDurationMs - Duration of each chunk in milliseconds
+ * @param sampleRate - Sample rate in Hz
+ * @param bytesPerSample - Bytes per sample (channels * bitDepth / 8)
+ * @returns Array of audio chunks
+ */
+export declare function splitIntoChunks(buffer: Buffer, chunkDurationMs: number, sampleRate?: number, bytesPerSample?: number): Buffer[];
+/**
+ * Audio format signatures for detection
+ */
+export declare const AUDIO_SIGNATURES: {
+    readonly wav: Buffer<ArrayBuffer>;
+    readonly mp3: {
+        readonly id3: Buffer<ArrayBuffer>;
+        readonly frameSync: Buffer<ArrayBuffer>;
+    };
+    readonly ogg: Buffer<ArrayBuffer>;
+};
+/**
+ * MIME types for audio formats
+ */
+export declare const MIME_TYPES: {
+    readonly wav: "audio/wav";
+    readonly mp3: "audio/mpeg";
+    readonly ogg: "audio/ogg";
+    readonly opus: "audio/opus";
+};