@juspay/neurolink 9.61.1 → 9.62.0

package/dist/lib/adapters/tts/googleTTSHandler.js

@@ -107,7 +107,7 @@ export class GoogleTTSHandler {
                 const languageCodes = voice.languageCodes;
                 const primaryLanguageCode = languageCodes[0];
                 const voiceType = this.detectVoiceType(voiceName);
-                // Map Google's ssmlGender → internal Gender
+                // Map Google's ssmlGender → internal TTSGender
                 const gender = voice.ssmlGender === "MALE"
                     ? "male"
                     : voice.ssmlGender === "FEMALE"

package/dist/lib/core/baseProvider.js

@@ -249,6 +249,11 @@ export class BaseProvider {
                 excludeTools: options.excludeTools,
                 skipToolPromptInjection: options.skipToolPromptInjection,
                 timeout: options.timeout,
+                stt: options.stt,
+                // Forward TTS options too — without this, the fake-streaming fallback
+                // path silently drops `tts` and the resulting StreamResult never
+                // produces a `tts_audio` chunk even when synthesis was requested.
+                tts: options.tts,
             };
             logger.debug(`Calling generate for fake streaming`, {
                 provider: this.providerName,
@@ -299,6 +304,23 @@ export class BaseProvider {
                             imageOutput: result.imageOutput,
                         };
                     }
+                    // Yield synthesized audio so callers using stream() with tts.enabled
+                    // still receive a tts_audio chunk on the fake-streaming fallback
+                    // path (matches the discriminator used by the real streaming path).
+                    if (result?.audio) {
+                        yield {
+                            type: "tts_audio",
+                            audio: {
+                                data: result.audio.buffer,
+                                format: result.audio.format,
+                                index: 0,
+                                isFinal: true,
+                                cumulativeSize: result.audio.size,
+                                voice: result.audio.voice,
+                                sampleRate: result.audio.sampleRate,
+                            },
+                        };
+                    }
                 })(),
                 usage: result?.usage,
                 provider: result?.provider,
@@ -587,7 +609,7 @@ export class BaseProvider {
             if (!options.tts) {
                 return this.enhanceResult(baseResult, options, startTime);
             }
-            baseResult.audio = await TTSProcessor.synthesize(textToSynthesize, options.provider ?? this.providerName, options.tts);
+            baseResult.audio = await TTSProcessor.synthesize(textToSynthesize, options.tts.provider ?? options.provider ?? this.providerName, options.tts);
         }
         catch (ttsError) {
             logger.error(`TTS synthesis failed in Mode 1 (direct input synthesis):`, ttsError);
@@ -691,20 +713,21 @@ export class BaseProvider {
         const { toolsUsed, toolExecutions } = this.extractToolInformation(generateResult);
         let enhancedResult = this.formatEnhancedResult(generateResult, tools, toolsUsed, toolExecutions, options);
         enhancedResult = await this.synthesizeAIResponseIfNeeded(enhancedResult, options);
-        return this.enhanceResult(enhancedResult, options, startTime);
+        const finalResult = await this.enhanceResult(enhancedResult, options, startTime);
+        return finalResult;
     }
     async synthesizeAIResponseIfNeeded(enhancedResult, options) {
         if (!options.tts?.enabled || !options.tts?.useAiResponse) {
             return enhancedResult;
         }
         const aiResponse = enhancedResult.content;
-        const provider = options.provider ?? this.providerName;
-        if (!aiResponse || !provider) {
+        const ttsProvider = options.tts?.provider ?? options.provider ?? this.providerName;
+        if (!aiResponse || !ttsProvider) {
             logger.warn(`TTS synthesis skipped despite being enabled`, {
                 provider: this.providerName,
                 hasAiResponse: !!aiResponse,
                 aiResponseLength: aiResponse?.length ?? 0,
-                hasProvider: !!provider,
+                hasProvider: !!ttsProvider,
                 ttsConfig: {
                     enabled: options.tts?.enabled,
                     useAiResponse: options.tts?.useAiResponse,
@@ -716,7 +739,7 @@ export class BaseProvider {
             return enhancedResult;
         }
         try {
-            const ttsResult = await TTSProcessor.synthesize(aiResponse, provider, options.tts);
+            const ttsResult = await TTSProcessor.synthesize(aiResponse, ttsProvider, options.tts);
             return {
                 ...enhancedResult,
                 audio: ttsResult,

package/dist/lib/factories/providerRegistry.d.ts

@@ -7,6 +7,20 @@ export declare class ProviderRegistry {
     private static registered;
     private static registrationPromise;
     private static options;
+    /**
+     * NEW4: per-handler registration outcomes for the realtime voice
+     * providers. `"ok"` = registered; any other string = the error message.
+     * Empty until the first `registerAllProviders()` call.
+     */
+    static realtimeRegistration: Record<string, "ok" | string>;
+    /**
+     * Returns a snapshot of voice provider registration outcomes so callers
+     * can detect at runtime which voice handlers are usable. Useful in
+     * health-check endpoints and CI startup probes.
+     */
+    static getRegistrationReport(): {
+        realtime: Record<string, "ok" | string>;
+    };
     /**
      * Register all providers with the factory
      */

package/dist/lib/factories/providerRegistry.js

@@ -11,6 +11,20 @@ export class ProviderRegistry {
     static options = {
         enableManualMCP: false, // Default to disabled for safety
     };
+    /**
+     * NEW4: per-handler registration outcomes for the realtime voice
+     * providers. `"ok"` = registered; any other string = the error message.
+     * Empty until the first `registerAllProviders()` call.
+     */
+    static realtimeRegistration = {};
+    /**
+     * Returns a snapshot of voice provider registration outcomes so callers
+     * can detect at runtime which voice handlers are usable. Useful in
+     * health-check endpoints and CI startup probes.
+     */
+    static getRegistrationReport() {
+        return { realtime: { ...this.realtimeRegistration } };
+    }
     /**
      * Register all providers with the factory
      */
@@ -152,8 +166,7 @@ export class ProviderRegistry {
                 const { LlamaCppProvider } = await import("../providers/llamaCpp.js");
                 return new LlamaCppProvider(modelName, sdk, undefined, llamaCppCreds);
             }, process.env.LLAMACPP_MODEL || undefined, ["llamacpp", "llama.cpp", "llama-cpp"]);
-            logger.debug("All providers registered successfully");
-            this.registered = true;
+            logger.debug("All AI providers registered successfully");
             // ===== TTS HANDLER REGISTRATION =====
             try {
                 // Create handler instance and register explicitly
@@ -172,6 +185,128 @@ export class ProviderRegistry {
                 });
                 // Don't throw - TTS is optional functionality
             }
+            // New TTS providers
+            try {
+                const { TTSProcessor } = await import("../utils/ttsProcessor.js");
+                const { OpenAITTS } = await import("../voice/providers/OpenAITTS.js");
+                TTSProcessor.registerHandler("openai-tts", new OpenAITTS());
+            }
+            catch (err) {
+                logger.debug(`[ProviderRegistry] openai-tts registration skipped: ${err instanceof Error ? err.message : String(err)}`);
+            }
+            try {
+                const { TTSProcessor } = await import("../utils/ttsProcessor.js");
+                const { ElevenLabsTTS } = await import("../voice/providers/ElevenLabsTTS.js");
+                const elevenLabsHandler = new ElevenLabsTTS();
+                TTSProcessor.registerHandler("elevenlabs", elevenLabsHandler);
+                TTSProcessor.registerHandler("elevenlabs-tts", elevenLabsHandler);
+            }
+            catch (err) {
+                logger.debug(`[ProviderRegistry] elevenlabs registration skipped: ${err instanceof Error ? err.message : String(err)}`);
+            }
+            try {
+                const { TTSProcessor } = await import("../utils/ttsProcessor.js");
+                const { AzureTTS } = await import("../voice/providers/AzureTTS.js");
+                TTSProcessor.registerHandler("azure-tts", new AzureTTS());
+            }
+            catch (err) {
+                logger.debug(`[ProviderRegistry] azure-tts registration skipped: ${err instanceof Error ? err.message : String(err)}`);
+            }
+            // ===== STT HANDLER REGISTRATION =====
+            try {
+                const { STTProcessor } = await import("../utils/sttProcessor.js");
+                try {
+                    const { OpenAISTT } = await import("../voice/providers/OpenAISTT.js");
+                    const openAISTT = new OpenAISTT();
+                    STTProcessor.registerHandler("whisper", openAISTT);
+                    STTProcessor.registerHandler("openai-stt", openAISTT);
+                }
+                catch (err) {
+                    logger.debug(`[ProviderRegistry] whisper/openai-stt registration skipped: ${err instanceof Error ? err.message : String(err)}`);
+                }
+                try {
+                    const { DeepgramSTT } = await import("../voice/providers/DeepgramSTT.js");
+                    STTProcessor.registerHandler("deepgram", new DeepgramSTT());
+                }
+                catch (err) {
+                    logger.debug(`[ProviderRegistry] deepgram registration skipped: ${err instanceof Error ? err.message : String(err)}`);
+                }
+                try {
+                    const { GoogleSTT } = await import("../voice/providers/GoogleSTT.js");
+                    STTProcessor.registerHandler("google-stt", new GoogleSTT());
+                }
+                catch (err) {
+                    logger.debug(`[ProviderRegistry] google-stt registration skipped: ${err instanceof Error ? err.message : String(err)}`);
+                }
+                try {
+                    const { AzureSTT } = await import("../voice/providers/AzureSTT.js");
+                    STTProcessor.registerHandler("azure-stt", new AzureSTT());
+                }
+                catch (err) {
+                    logger.debug(`[ProviderRegistry] azure-stt registration skipped: ${err instanceof Error ? err.message : String(err)}`);
+                }
+                logger.debug("STT handlers registered successfully", {
+                    providers: ["whisper", "deepgram", "google-stt", "azure-stt"],
+                });
+            }
+            catch (sttError) {
+                logger.warn("Failed to register STT handlers - STT functionality will be unavailable", {
+                    error: sttError instanceof Error ? sttError.message : String(sttError),
+                });
+            }
+            // ===== REALTIME HANDLER REGISTRATION =====
+            try {
+                const { RealtimeProcessor } = await import("../voice/RealtimeVoiceAPI.js");
+                // M9 + NEW4: track per-handler registration outcomes so the final
+                // log accurately reflects which voice providers succeeded vs which
+                // were skipped — instead of unconditionally claiming "registered
+                // successfully" or hiding failures at debug level.
+                const realtimeOutcomes = {};
+                try {
+                    const { OpenAIRealtime } = await import("../voice/providers/OpenAIRealtime.js");
+                    RealtimeProcessor.registerHandler("openai-realtime", new OpenAIRealtime());
+                    realtimeOutcomes["openai-realtime"] = "ok";
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    realtimeOutcomes["openai-realtime"] = msg;
+                    // M9: promote per-handler failures to error level so users can
+                    // see which shipped voice provider failed to register at startup.
+                    logger.error(`[ProviderRegistry] openai-realtime registration failed: ${msg}`);
+                }
+                try {
+                    const { GeminiLive } = await import("../voice/providers/GeminiLive.js");
+                    RealtimeProcessor.registerHandler("gemini-live", new GeminiLive());
+                    realtimeOutcomes["gemini-live"] = "ok";
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    realtimeOutcomes["gemini-live"] = msg;
+                    logger.error(`[ProviderRegistry] gemini-live registration failed: ${msg}`);
+                }
+                // NEW4: report the actual per-handler outcomes instead of an
+                // unconditional success log. Stored on the registry so callers can
+                // introspect via getRegistrationReport().
+                ProviderRegistry.realtimeRegistration = realtimeOutcomes;
+                const skipped = Object.entries(realtimeOutcomes).filter(([, v]) => v !== "ok");
+                if (skipped.length === 0) {
+                    logger.info("[ProviderRegistry] Realtime handlers registered: openai-realtime, gemini-live");
+                }
+                else {
+                    logger.warn(`[ProviderRegistry] Realtime handlers partial: ${skipped.length} skipped`, { outcomes: realtimeOutcomes });
+                }
+            }
+            catch (realtimeError) {
+                logger.warn("Failed to register Realtime handlers - Realtime functionality will be unavailable", {
+                    error: realtimeError instanceof Error
+                        ? realtimeError.message
+                        : String(realtimeError),
+                });
+            }
+            // Mark registered ONLY after all blocks (AI + voice) attempted, so a
+            // subsequent registerAllProviders() call does not short-circuit when an
+            // optional handler block silently failed.
+            this.registered = true;
         }
         catch (error) {
             logger.error("Failed to register providers:", error);
@@ -191,6 +326,10 @@ export class ProviderRegistry {
         ProviderFactory.clearRegistrations();
         this.registered = false;
         this.registrationPromise = null;
+        // Reset realtime registration too — otherwise getRegistrationReport()
+        // can surface stale data from a previous run if the realtime block
+        // failed before reaching `realtimeRegistration = realtimeOutcomes`.
+        ProviderRegistry.realtimeRegistration = {};
     }
     /**
      * Set registry options (should be called before initialization)

package/dist/lib/mcp/toolRegistry.js

@@ -301,12 +301,18 @@ export class MCPToolRegistry extends MCPRegistry {
                 span.setAttribute("tool.arguments_size", argsStr.length);
                 // HITL Safety Check: Request confirmation if required
                 let finalArgs = args;
-                if (this.hitlManager && this.hitlManager.isEnabled()) {
+                const HITLState = context?.hitlState;
+                if (!HITLState?.triggered &&
+                    this.hitlManager &&
+                    this.hitlManager.isEnabled()) {
                     const requiresConfirmation = this.hitlManager.requiresConfirmation(toolName, args);
                     if (requiresConfirmation) {
                         registryLogger.info(`Tool '${toolName}' requires HITL confirmation`);
                         span.addEvent("tool.hitl_requested");
                         try {
+                            if (HITLState) {
+                                HITLState.triggered = true;
+                            }
                             const confirmationResult = await this.hitlManager.requestConfirmation(toolName, args, {
                                 serverId: tool.serverId,
                                 sessionId: execContext.sessionId,

package/dist/lib/neurolink.d.ts

@@ -764,6 +764,25 @@ export declare class NeuroLink {
     private validateStreamRequestOptions;
     private maybeHandleWorkflowStreamRequest;
     private runStandardStreamRequest;
+    /**
+     * TTS Mode 2 synthesis helper for the stream() pipeline.
+     *
+     * m5 — extracted from runStandardStreamRequest so the surrounding generator
+     * stays under the max-lines-per-function lint budget. Behaviour preserved
+     * exactly:
+     * - When Mode 2 is enabled (`tts.enabled && tts.useAiResponse`) AND the
+     *   model produced non-empty content: synthesises one final audio buffer
+     *   and returns it as an `audioChunk` for the caller to `yield`. Resolves
+     *   `ttsResolver` with the `TTSResult`.
+     * - When Mode 2 is enabled but synthesis fails: logs a warning and resolves
+     *   `ttsResolver` with `undefined`.
+     * - When Mode 2 is requested but skipped (empty content / wrong mode):
+     *   resolves `ttsResolver` with `undefined` early so callers awaiting
+     *   `result.audio` unblock before the surrounding `finally` cleanup
+     *   completes (Issue 7 latency micro-opt — the finally block also resolves
+     *   defensively, so this is a redundant early signal, not a coverage fix).
+     */
+    private synthesizeStreamModeTwo;
     /**
      * Prepare stream options: initialize memory, MCP, retrieval, orchestration,
      * Ollama tool auto-disable, factory processing, and tool detection.