@bytexbyte/nxtlinq-ai-agent-ui-react-development 0.1.7 → 0.1.9

package/dist/context/ChatBotContext.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ChatBotContext.d.ts","sourceRoot":"","sources":["../../src/context/ChatBotContext.tsx"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAwB/B,OAAO,EAEL,kBAAkB,EAClB,YAAY,EAEb,MAAM,uBAAuB,CAAC;AAM/B,eAAO,MAAM,UAAU,0BAMtB,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,KAAK,CAAC,EAAE,CAAC,YAAY,CA4mGlD,CAAC"}
+{"version":3,"file":"ChatBotContext.d.ts","sourceRoot":"","sources":["../../src/context/ChatBotContext.tsx"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAyB/B,OAAO,EAEL,kBAAkB,EAClB,YAAY,EAEb,MAAM,uBAAuB,CAAC;AAM/B,eAAO,MAAM,UAAU,0BAMtB,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,KAAK,CAAC,EAAE,CAAC,YAAY,CA+qGlD,CAAC"}

package/dist/context/ChatBotContext.js

@@ -4,7 +4,7 @@ import stringify from 'fast-json-stable-stringify';
 import * as React from 'react';
 import { flushSync } from 'react-dom';
 import { v4 as uuidv4 } from 'uuid';
-import { createNxtlinqApi, setApiHosts, synthesizeSpeechToBuffer, useLocalStorage, useSessionStorage, useSpeechToTextFromMic, useVoiceMode, metakeepClient, getEthers, sleep, walletTextUtils, } from '@bytexbyte/nxtlinq-ai-agent-web-development';
+import { createNxtlinqApi, setApiHosts, synthesizeSpeechToBuffer, streamSpeechToAudioContext, useLocalStorage, useSessionStorage, useSpeechToTextFromMic, useVoiceMode, metakeepClient, getEthers, sleep, walletTextUtils, } from '@bytexbyte/nxtlinq-ai-agent-web-development';
 const MIC_ENABLED_SESSION_KEY = 'chatbot-mic-enabled';
 const ChatBotContext = React.createContext(undefined);
 export const useChatBot = () => {
@@ -88,6 +88,10 @@ piiDisplayMode = 'redacted', }) => {
     const audioCtxRef = React.useRef(null);
     const audioSourceRef = React.useRef(null);
     const audioElementRef = React.useRef(null);
+    // Streaming TTS (OpenAI PCM16 path) — tracks abort controller and all
+    // scheduled AudioBufferSourceNodes so stopTextToSpeech can cancel them.
+    const ttsAbortControllerRef = React.useRef(null);
+    const streamingSourcesRef = React.useRef([]);
     const speechingRef = React.useRef(false);
     const [isTtsProcessing, setIsTtsProcessing] = React.useState(false);
     const [requiresGesture, setRequiresGesture] = React.useState(false);
@@ -197,6 +201,58 @@ piiDisplayMode = 'redacted', }) => {
             setIsTtsProcessing(false);
             return;
         }
+        // OpenAI provider: stream PCM16 directly via fetch streaming so playback
+        // starts before the full audio is downloaded.
+        const provider = clientTtsVoiceRef.current?.provider ?? 'azure';
+        if (provider === 'openai') {
+            ttsAbortControllerRef.current?.abort();
+            ttsAbortControllerRef.current = null;
+            // Stop any still-playing nodes from the previous streaming call before
+            // clearing the ref — otherwise old sources keep playing and overlap.
+            for (const src of streamingSourcesRef.current) {
+                try {
+                    src.stop();
+                    src.disconnect();
+                }
+                catch { /* already ended */ }
+            }
+            streamingSourcesRef.current = [];
+            const abortController = new AbortController();
+            ttsAbortControllerRef.current = abortController;
+            // Always ensure a 24 kHz AudioContext for PCM16 streaming. Azure TTS
+            // uses the default OS rate (48 kHz), so reusing that context would
+            // pitch-shift the audio and cause the first chunk to sound distorted.
+            if (!audioCtxRef.current || audioCtxRef.current.state === 'closed' || audioCtxRef.current.sampleRate !== 24000) {
+                audioCtxRef.current = new AudioContext({ sampleRate: 24000 });
+            }
+            await streamSpeechToAudioContext({
+                text,
+                apiKey: apiKeyRef.current,
+                apiSecret: apiSecretRef.current,
+                audioCtx: audioCtxRef.current,
+                signal: abortController.signal,
+                onSourceScheduled: (source) => {
+                    streamingSourcesRef.current.push(source);
+                },
+                onFirstChunk: () => {
+                    if (abortController.signal.aborted)
+                        return;
+                    speechingRef.current = true;
+                    if (messageIndex !== undefined)
+                        setSpeechingIndex(messageIndex);
+                    setIsTtsProcessing(false);
+                    setRequiresGesture(false);
+                },
+                onEnded: () => {
+                    if (abortController.signal.aborted)
+                        return;
+                    speechingRef.current = false;
+                    setSpeechingIndex(undefined);
+                    streamingSourcesRef.current = [];
+                },
+            });
+            return;
+        }
         const synth = await synthesizeSpeechToBuffer({
             text,
             apiKey: apiKeyRef.current,
@@ -2130,6 +2186,19 @@ piiDisplayMode = 'redacted', }) => {
     };
     // Stop text-to-speech (does not clear queue)
     const stopTextToSpeech = React.useCallback(() => {
+        // Cancel any in-flight streaming TTS fetch and stop all scheduled nodes.
+        if (ttsAbortControllerRef.current) {
+            ttsAbortControllerRef.current.abort();
+            ttsAbortControllerRef.current = null;
+        }
+        for (const source of streamingSourcesRef.current) {
+            try {
+                source.stop();
+                source.disconnect();
+            }
+            catch { /* already stopped */ }
+        }
+        streamingSourcesRef.current = [];
         if (speechingRef.current) {
             // Stop AudioContext source if exists
             if (audioSourceRef.current) {
@@ -2176,6 +2245,9 @@ piiDisplayMode = 'redacted', }) => {
         getMessages: () => messagesRef.current,
         setMessages,
         onError,
+        onToolCall: onToolUse
+            ? (event) => { void onToolUse({ name: event.name, input: event.args }); }
+            : undefined,
         stopRecording,
         stopTextToSpeech: stopTextToSpeechAndReset,
         voiceTransport: 'ws-realtime',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bytexbyte/nxtlinq-ai-agent-ui-react-development",
-  "version": "0.1.7",
+  "version": "0.1.9",
   "description": "Official React Web UI for nxtlinq AI Agent — drop-in chat widget",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -38,8 +38,8 @@
     "react-dom": ">=18.0.0"
   },
   "dependencies": {
-    "@bytexbyte/nxtlinq-ai-agent-core-development": "0.4.1",
-    "@bytexbyte/nxtlinq-ai-agent-web-development": "0.1.7",
+    "@bytexbyte/nxtlinq-ai-agent-core-development": "0.4.4",
+    "@bytexbyte/nxtlinq-ai-agent-web-development": "0.1.9",
     "@emotion/react": "^11.14.0",
     "@emotion/styled": "^11.14.1",
     "@mui/icons-material": "^7.2.0",

package/src/context/ChatBotContext.tsx

@@ -7,6 +7,7 @@ import {
   createNxtlinqApi,
   setApiHosts,
   synthesizeSpeechToBuffer,
+  streamSpeechToAudioContext,
   useLocalStorage,
   useSessionStorage,
   useSpeechToTextFromMic,
@@ -143,6 +144,10 @@ export const ChatBotProvider: React.FC<ChatBotProps> = ({
   const audioCtxRef = React.useRef<AudioContext | null>(null);
   const audioSourceRef = React.useRef<AudioBufferSourceNode | null>(null);
   const audioElementRef = React.useRef<HTMLAudioElement | null>(null);
+  // Streaming TTS (OpenAI PCM16 path) — tracks abort controller and all
+  // scheduled AudioBufferSourceNodes so stopTextToSpeech can cancel them.
+  const ttsAbortControllerRef = React.useRef<AbortController | null>(null);
+  const streamingSourcesRef = React.useRef<AudioBufferSourceNode[]>([]);
   const speechingRef = React.useRef(false);
   const [isTtsProcessing, setIsTtsProcessing] = React.useState(false);
   const [requiresGesture, setRequiresGesture] = React.useState(false);
@@ -268,6 +273,56 @@ export const ChatBotProvider: React.FC<ChatBotProps> = ({
       setIsTtsProcessing(false);
       return;
     }
+
+    // OpenAI provider: stream PCM16 directly via fetch streaming so playback
+    // starts before the full audio is downloaded.
+    const provider = clientTtsVoiceRef.current?.provider ?? 'azure';
+    if (provider === 'openai') {
+      ttsAbortControllerRef.current?.abort();
+      ttsAbortControllerRef.current = null;
+      // Stop any still-playing nodes from the previous streaming call before
+      // clearing the ref — otherwise old sources keep playing and overlap.
+      for (const src of streamingSourcesRef.current) {
+        try { src.stop(); src.disconnect(); } catch { /* already ended */ }
+      }
+      streamingSourcesRef.current = [];
+
+      const abortController = new AbortController();
+      ttsAbortControllerRef.current = abortController;
+
+      // Always ensure a 24 kHz AudioContext for PCM16 streaming. Azure TTS
+      // uses the default OS rate (48 kHz), so reusing that context would
+      // pitch-shift the audio and cause the first chunk to sound distorted.
+      if (!audioCtxRef.current || audioCtxRef.current.state === 'closed' || audioCtxRef.current.sampleRate !== 24000) {
+        audioCtxRef.current = new AudioContext({ sampleRate: 24000 });
+      }
+
+      await streamSpeechToAudioContext({
+        text,
+        apiKey: apiKeyRef.current,
+        apiSecret: apiSecretRef.current,
+        audioCtx: audioCtxRef.current,
+        signal: abortController.signal,
+        onSourceScheduled: (source: AudioBufferSourceNode) => {
+          streamingSourcesRef.current.push(source);
+        },
+        onFirstChunk: () => {
+          if (abortController.signal.aborted) return;
+          speechingRef.current = true;
+          if (messageIndex !== undefined) setSpeechingIndex(messageIndex);
+          setIsTtsProcessing(false);
+          setRequiresGesture(false);
+        },
+        onEnded: () => {
+          if (abortController.signal.aborted) return;
+          speechingRef.current = false;
+          setSpeechingIndex(undefined);
+          streamingSourcesRef.current = [];
+        },
+      });
+      return;
+    }
+
     const synth = await synthesizeSpeechToBuffer({
       text,
       apiKey: apiKeyRef.current,
@@ -2373,6 +2428,16 @@ export const ChatBotProvider: React.FC<ChatBotProps> = ({
 
   // Stop text-to-speech (does not clear queue)
   const stopTextToSpeech = React.useCallback(() => {
+    // Cancel any in-flight streaming TTS fetch and stop all scheduled nodes.
+    if (ttsAbortControllerRef.current) {
+      ttsAbortControllerRef.current.abort();
+      ttsAbortControllerRef.current = null;
+    }
+    for (const source of streamingSourcesRef.current) {
+      try { source.stop(); source.disconnect(); } catch { /* already stopped */ }
+    }
+    streamingSourcesRef.current = [];
+
     if (speechingRef.current) {
       // Stop AudioContext source if exists
       if (audioSourceRef.current) {
@@ -2432,6 +2497,9 @@ export const ChatBotProvider: React.FC<ChatBotProps> = ({
     getMessages: () => messagesRef.current,
     setMessages,
     onError,
+    onToolCall: onToolUse
+      ? (event) => { void onToolUse({ name: event.name, input: event.args }); }
+      : undefined,
     stopRecording,
     stopTextToSpeech: stopTextToSpeechAndReset,
     voiceTransport: 'ws-realtime',