@bytexbyte/nxtlinq-ai-agent-web-development 0.1.7 → 0.1.9

package/dist/hooks/useNxtlinqAgent.d.ts

@@ -12,6 +12,8 @@ export type UseNxtlinqAgentResult = NxtlinqAgentSnapshot & {
     setMessages: (messages: Message[]) => void;
     postTextTts: (text: string) => Promise<PostTextTtsResult>;
     buildTextTtsPlaybackUri: (result: PostTextTtsResult) => string;
+    /** Stream PCM16 TTS using browser fetch streaming. Web-only. */
+    streamTextTts: (text: string, onChunk: (pcm16: Int16Array) => void, signal?: AbortSignal) => Promise<void>;
 };
 /** Subscribe to {@link NxtlinqAgent} state for custom Web chat UIs. */
 export declare function useNxtlinqAgent(): UseNxtlinqAgentResult;

package/dist/hooks/useNxtlinqAgent.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"useNxtlinqAgent.d.ts","sourceRoot":"","sources":["../../src/hooks/useNxtlinqAgent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,oBAAoB,EACpB,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,8CAA8C,CAAC;AAEtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAExE,MAAM,MAAM,qBAAqB,GAAG,oBAAoB,GAAG;IACzD,KAAK,EAAE,UAAU,CAAC,OAAO,sBAAsB,CAAC,CAAC;IACjD,WAAW,EAAE,CACX,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,KACvC,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,WAAW,EAAE,CAAC,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5D,oBAAoB,EAAE,CAAC,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACrE,WAAW,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IAC3C,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAC1D,uBAAuB,EAAE,CAAC,MAAM,EAAE,iBAAiB,KAAK,MAAM,CAAC;CAChE,CAAC;AAEF,uEAAuE;AACvE,wBAAgB,eAAe,IAAI,qBAAqB,CAiDvD"}
+{"version":3,"file":"useNxtlinqAgent.d.ts","sourceRoot":"","sources":["../../src/hooks/useNxtlinqAgent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,oBAAoB,EACpB,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,8CAA8C,CAAC;AAEtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAExE,MAAM,MAAM,qBAAqB,GAAG,oBAAoB,GAAG;IACzD,KAAK,EAAE,UAAU,CAAC,OAAO,sBAAsB,CAAC,CAAC;IACjD,WAAW,EAAE,CACX,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,KACvC,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,WAAW,EAAE,CAAC,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5D,oBAAoB,EAAE,CAAC,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACrE,WAAW,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IAC3C,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAC1D,uBAAuB,EAAE,CAAC,MAAM,EAAE,iBAAiB,KAAK,MAAM,CAAC;IAC/D,gEAAgE;IAChE,aAAa,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,EAAE,MAAM,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5G,CAAC;AAEF,uEAAuE;AACvE,wBAAgB,eAAe,IAAI,qBAAqB,CAwDvD"}

package/dist/hooks/useNxtlinqAgent.js

@@ -10,6 +10,7 @@ export function useNxtlinqAgent() {
     const setMessages = useCallback((messages) => agent.setMessages(messages), [agent]);
     const postTextTts = useCallback((text) => agent.postTextTts(text), [agent]);
     const buildTextTtsPlaybackUri = useCallback((result) => agent.buildTextTtsPlaybackUri(result), [agent]);
+    const streamTextTts = useCallback((text, onChunk, signal) => agent.streamTextTts(text, onChunk, signal), [agent]);
     return {
         agent,
         ...snapshot,
@@ -19,5 +20,6 @@ export function useNxtlinqAgent() {
         setMessages,
         postTextTts,
         buildTextTtsPlaybackUri,
+        streamTextTts,
     };
 }

package/dist/index.d.ts

@@ -7,6 +7,6 @@ export { WEB_AGENT_DEFAULTS, applyWebAgentDefaults } from './webAgentDefaults';
 export { fileToAttachment } from './utils/fileToAttachment';
 export type { Message, Attachment, AgentResponse, SendMessageOptions, NxtlinqAgentSnapshot, ToolUse, StartVoiceSessionOptions, VoiceSession, VoiceStatus, VoiceTranscriptEvent, VoiceDoneEvent, VoiceUserInputOptions, VoiceGreetingOptions, } from '@bytexbyte/nxtlinq-ai-agent-core-development';
 export { NxtlinqAgent, setApiHosts, createBrowserWebRTCPort, createBrowserStoragePort, createDefaultHttpPort, VoiceNotSupportedError, mapServerHistoryToMessages, appendServerHistoryIntoMessages, STORAGE_KEYS, } from '@bytexbyte/nxtlinq-ai-agent-core-development';
-export { createNxtlinqApi, getAiAgentApiHost, startVoiceSession, createVoiceApi, useDraggable, useLocalStorage, useSessionStorage, useResizable, synthesizeSpeechToBuffer, useSpeechToTextFromMic, useVoiceMode, metakeepClient, getEthers, sleep, walletTextUtils, connectWallet, disconnectWallet, validateToken, createAITMetadata, generateAITId, prepareAITCreation, createNotification, getNotificationIcon, containsUrls, convertUrlsToLinks, convertUrlsToHtml, } from './legacy';
+export { createNxtlinqApi, getAiAgentApiHost, startVoiceSession, createVoiceApi, useDraggable, useLocalStorage, useSessionStorage, useResizable, synthesizeSpeechToBuffer, streamSpeechToAudioContext, useSpeechToTextFromMic, useVoiceMode, metakeepClient, getEthers, sleep, walletTextUtils, connectWallet, disconnectWallet, validateToken, createAITMetadata, generateAITId, prepareAITCreation, createNotification, getNotificationIcon, containsUrls, convertUrlsToLinks, convertUrlsToHtml, } from './legacy';
 export type { ResizeCorner } from './legacy';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,qBAAqB,EACrB,KAAK,4BAA4B,GAClC,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,KAAK,yBAAyB,EAC9B,KAAK,WAAW,GACjB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAE,eAAe,EAAE,KAAK,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AACtF,OAAO,EACL,eAAe,EACf,KAAK,qBAAqB,EAC1B,KAAK,sBAAsB,GAC5B,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,sBAAsB,EACtB,KAAK,6BAA6B,GACnC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAE5D,YAAY,EACV,OAAO,EACP,UAAU,EACV,aAAa,EACb,kBAAkB,EAClB,oBAAoB,EACpB,OAAO,EACP,wBAAwB,EACxB,YAAY,EACZ,WAAW,EACX,oBAAoB,EACpB,cAAc,EACd,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,8CAA8C,CAAC;AAEtD,OAAO,EACL,YAAY,EACZ,WAAW,EACX,uBAAuB,EACvB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,0BAA0B,EAC1B,+BAA+B,EAC/B,YAAY,GACb,MAAM,8CAA8C,CAAC;AAEtD,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,iBAAiB,EACjB,cAAc,EACd,YAAY,EACZ,eAAe,EACf,iBAAiB,EACjB,YAAY,EACZ,wBAAwB,EACxB,sBAAsB,EACtB,YAAY,EACZ,cAAc,EACd,SAAS,EACT,KAAK,EACL,eAAe,EACf,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,iBAAiB,EACjB,aAAa,EACb,kBAAkB,EAClB,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,UAAU,CAAC;AAElB,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,qBAAqB,EACrB,KAAK,4BAA4B,GAClC,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,KAAK,yBAAyB,EAC9B,KAAK,WAAW,GACjB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAE,eAAe,EAAE,KAAK,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AACtF,OAAO,EACL,eAAe,EACf,KAAK,qBAAqB,EAC1B,KAAK,sBAAsB,GAC5B,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,sBAAsB,EACtB,KAAK,6BAA6B,GACnC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAE5D,YAAY,EACV,OAAO,EACP,UAAU,EACV,aAAa,EACb,kBAAkB,EAClB,oBAAoB,EACpB,OAAO,EACP,wBAAwB,EACxB,YAAY,EACZ,WAAW,EACX,oBAAoB,EACpB,cAAc,EACd,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,8CAA8C,CAAC;AAEtD,OAAO,EACL,YAAY,EACZ,WAAW,EACX,uBAAuB,EACvB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,0BAA0B,EAC1B,+BAA+B,EAC/B,YAAY,GACb,MAAM,8CAA8C,CAAC;AAEtD,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,iBAAiB,EACjB,cAAc,EACd,YAAY,EACZ,eAAe,EACf,iBAAiB,EACjB,YAAY,EACZ,wBAAwB,EACxB,0BAA0B,EAC1B,sBAAsB,EACtB,YAAY,EACZ,cAAc,EACd,SAAS,EACT,KAAK,EACL,eAAe,EACf,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,iBAAiB,EACjB,aAAa,EACb,kBAAkB,EAClB,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,UAAU,CAAC;AAElB,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}

package/dist/index.js

@@ -6,4 +6,4 @@ export { createWebPlatformPorts, } from './ports/createWebPlatformPorts';
 export { WEB_AGENT_DEFAULTS, applyWebAgentDefaults } from './webAgentDefaults';
 export { fileToAttachment } from './utils/fileToAttachment';
 export { NxtlinqAgent, setApiHosts, createBrowserWebRTCPort, createBrowserStoragePort, createDefaultHttpPort, VoiceNotSupportedError, mapServerHistoryToMessages, appendServerHistoryIntoMessages, STORAGE_KEYS, } from '@bytexbyte/nxtlinq-ai-agent-core-development';
-export { createNxtlinqApi, getAiAgentApiHost, startVoiceSession, createVoiceApi, useDraggable, useLocalStorage, useSessionStorage, useResizable, synthesizeSpeechToBuffer, useSpeechToTextFromMic, useVoiceMode, metakeepClient, getEthers, sleep, walletTextUtils, connectWallet, disconnectWallet, validateToken, createAITMetadata, generateAITId, prepareAITCreation, createNotification, getNotificationIcon, containsUrls, convertUrlsToLinks, convertUrlsToHtml, } from './legacy';
+export { createNxtlinqApi, getAiAgentApiHost, startVoiceSession, createVoiceApi, useDraggable, useLocalStorage, useSessionStorage, useResizable, synthesizeSpeechToBuffer, streamSpeechToAudioContext, useSpeechToTextFromMic, useVoiceMode, metakeepClient, getEthers, sleep, walletTextUtils, connectWallet, disconnectWallet, validateToken, createAITMetadata, generateAITId, prepareAITCreation, createNotification, getNotificationIcon, containsUrls, convertUrlsToLinks, convertUrlsToHtml, } from './legacy';

package/dist/legacy/core/lib/textToSpeech.d.ts

@@ -11,4 +11,14 @@ export declare function synthesizeSpeechToBuffer(params: {
     buffer: ArrayBuffer;
     mimeType: string;
 } | undefined>;
+export declare function streamSpeechToAudioContext(params: {
+    text: string;
+    apiKey: string;
+    apiSecret: string;
+    audioCtx: AudioContext;
+    onSourceScheduled?: (source: AudioBufferSourceNode) => void;
+    onFirstChunk?: () => void;
+    onEnded?: () => void;
+    signal?: AbortSignal;
+}): Promise<void>;
 //# sourceMappingURL=textToSpeech.d.ts.map

package/dist/legacy/core/lib/textToSpeech.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"textToSpeech.d.ts","sourceRoot":"","sources":["../../../../src/legacy/core/lib/textToSpeech.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,8CAA8C,CAAC;AA8E3F;;GAEG;AACH,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,sBAAsB,CAAC;CACnC,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,WAAW,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAAC,CAsCjE"}
+{"version":3,"file":"textToSpeech.d.ts","sourceRoot":"","sources":["../../../../src/legacy/core/lib/textToSpeech.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,8CAA8C,CAAC;AA8E3F;;GAEG;AACH,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,sBAAsB,CAAC;CACnC,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,WAAW,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAAC,CAsCjE;AAmBD,wBAAsB,0BAA0B,CAAC,MAAM,EAAE;IACvD,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,YAAY,CAAC;IACvB,iBAAiB,CAAC,EAAE,CAAC,MAAM,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC5D,YAAY,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;IACrB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,IAAI,CAAC,CA+DhB"}

package/dist/legacy/core/lib/textToSpeech.js

@@ -1,4 +1,5 @@
 import { SpeechConfig, SpeechSynthesizer } from 'microsoft-cognitiveservices-speech-sdk';
+import { streamTextTts } from '@bytexbyte/nxtlinq-ai-agent-core-development';
 import { getAiAgentApiHost } from '../../api/nxtlinq-api';
 const DEFAULT_AZURE_VOICE = 'en-US-JennyNeural';
 async function fetchAzureTokenFromAgent(agentHost, apiKey, apiSecret) {
@@ -80,3 +81,79 @@ export async function synthesizeSpeechToBuffer(params) {
         return undefined;
     return { buffer, mimeType: 'audio/wav' };
 }
+/**
+ * Stream OpenAI TTS audio progressively into a Web AudioContext.
+ *
+ * Receives raw PCM16 (24 kHz mono) chunks from the server and schedules
+ * them back-to-back on the AudioContext so playback starts as soon as the
+ * first chunk arrives, before the full audio is downloaded.
+ *
+ * @param onSourceScheduled  Called for every AudioBufferSourceNode that is
+ *   scheduled — collect these to stop playback early (e.g. user clicks stop).
+ * @param onFirstChunk  Called when the first audio chunk is ready to play.
+ * @param onEnded       Called when all scheduled buffers have finished playing.
+ */
+// Number of chunks to pre-buffer before starting playback. Each chunk is
+// ~8192 bytes = ~170ms of 24kHz PCM16 audio. Buffering 3 chunks gives ~510ms
+// of headroom to absorb network jitter and prevent gaps at the start.
+const PREBUFFER_CHUNKS = 3;
+export async function streamSpeechToAudioContext(params) {
+    const { text, apiKey, apiSecret, audioCtx, onSourceScheduled, onFirstChunk, onEnded, signal } = params;
+    if (audioCtx.state === 'suspended')
+        await audioCtx.resume();
+    let nextPlayTime = 0;
+    let pendingCount = 0;
+    let streamDone = false;
+    // Pre-buffer: collect initial chunks before scheduling any playback so the
+    // AudioContext has enough backlog to play without gaps from network jitter.
+    const prebuffer = [];
+    let prebuffering = true;
+    const checkEnded = () => {
+        if (streamDone && pendingCount === 0)
+            onEnded?.();
+    };
+    const scheduleBuffer = (audioBuffer) => {
+        const source = audioCtx.createBufferSource();
+        source.buffer = audioBuffer;
+        source.connect(audioCtx.destination);
+        pendingCount++;
+        source.start(nextPlayTime);
+        nextPlayTime += audioBuffer.duration;
+        onSourceScheduled?.(source);
+        source.onended = () => {
+            pendingCount--;
+            checkEnded();
+        };
+    };
+    const flushPrebuffer = () => {
+        nextPlayTime = audioCtx.currentTime + 0.05;
+        onFirstChunk?.();
+        for (const buf of prebuffer)
+            scheduleBuffer(buf);
+        prebuffer.length = 0;
+        prebuffering = false;
+    };
+    await streamTextTts({ apiKey, apiSecret, text }, (pcm16) => {
+        if (signal?.aborted)
+            return;
+        const float32 = new Float32Array(pcm16.length);
+        for (let i = 0; i < pcm16.length; i++)
+            float32[i] = pcm16[i] / 32768;
+        const audioBuffer = audioCtx.createBuffer(1, float32.length, 24000);
+        audioBuffer.copyToChannel(float32, 0);
+        if (prebuffering) {
+            prebuffer.push(audioBuffer);
+            if (prebuffer.length >= PREBUFFER_CHUNKS)
+                flushPrebuffer();
+        }
+        else {
+            scheduleBuffer(audioBuffer);
+        }
+    }, signal);
+    // Stream finished — flush any remaining prebuffered chunks (short text that
+    // never reached PREBUFFER_CHUNKS) and mark done.
+    if (prebuffering && prebuffer.length > 0)
+        flushPrebuffer();
+    streamDone = true;
+    checkEnded();
+}

package/dist/legacy/core/lib/useVoiceMode.d.ts

@@ -1,6 +1,6 @@
 import * as React from 'react';
 import type { AITApi, Message, VoiceTransport } from '@bytexbyte/nxtlinq-ai-agent-core-development';
-import { type VoiceStatus } from '../../api/voice';
+import { type VoiceStatus, type VoiceToolCallEvent, type VoiceToolResultEvent } from '../../api/voice';
 export type UseVoiceModeOptions = {
     apiKey: string;
     apiSecret: string;
@@ -14,11 +14,13 @@ export type UseVoiceModeOptions = {
     getMessages: () => Message[];
     setMessages: React.Dispatch<React.SetStateAction<Message[]>>;
     onError?: (error: Error) => void;
+    onToolCall?: (event: VoiceToolCallEvent) => void;
+    onToolResult?: (event: VoiceToolResultEvent) => void;
     stopRecording: () => void;
     stopTextToSpeech: () => void;
     voiceTransport?: VoiceTransport;
 };
-export declare function useVoiceMode({ apiKey, apiSecret, pseudoId, externalId, aitId, walletAddress, aitToken, metadata, nxtlinqApi, getMessages, setMessages, onError, stopRecording, stopTextToSpeech, voiceTransport, }: UseVoiceModeOptions): {
+export declare function useVoiceMode({ apiKey, apiSecret, pseudoId, externalId, aitId, walletAddress, aitToken, metadata, nxtlinqApi, getMessages, setMessages, onError, onToolCall, onToolResult, stopRecording, stopTextToSpeech, voiceTransport, }: UseVoiceModeOptions): {
     isVoiceMode: boolean;
     voiceStatus: VoiceStatus;
     isVoiceConnecting: boolean;

package/dist/legacy/core/lib/useVoiceMode.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"useVoiceMode.d.ts","sourceRoot":"","sources":["../../../../src/legacy/core/lib/useVoiceMode.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAE/B,OAAO,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;AACpG,OAAO,EAKL,KAAK,WAAW,EAEjB,MAAM,iBAAiB,CAAC;AAgBzB,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,OAAO,EAAE,CAAC;IAC7B,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IACjC,aAAa,EAAE,MAAM,IAAI,CAAC;IAC1B,gBAAgB,EAAE,MAAM,IAAI,CAAC;IAC7B,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC,CAAC;AAEF,wBAAgB,YAAY,CAAC,EAC3B,MAAM,EACN,SAAS,EACT,QAAQ,EACR,UAAU,EACV,KAAK,EACL,aAAa,EACb,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,WAAW,EACX,WAAW,EACX,OAAO,EACP,aAAa,EACb,gBAAgB,EAChB,cAA8B,GAC/B,EAAE,mBAAmB;;;;;;;;;;EAoXrB"}
+{"version":3,"file":"useVoiceMode.d.ts","sourceRoot":"","sources":["../../../../src/legacy/core/lib/useVoiceMode.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAE/B,OAAO,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;AACpG,OAAO,EAKL,KAAK,WAAW,EAChB,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EAE1B,MAAM,iBAAiB,CAAC;AAgBzB,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,OAAO,EAAE,CAAC;IAC7B,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,kBAAkB,KAAK,IAAI,CAAC;IACjD,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,IAAI,CAAC;IACrD,aAAa,EAAE,MAAM,IAAI,CAAC;IAC1B,gBAAgB,EAAE,MAAM,IAAI,CAAC;IAC7B,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC,CAAC;AAEF,wBAAgB,YAAY,CAAC,EAC3B,MAAM,EACN,SAAS,EACT,QAAQ,EACR,UAAU,EACV,KAAK,EACL,aAAa,EACb,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,WAAW,EACX,WAAW,EACX,OAAO,EACP,UAAU,EACV,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAA8B,GAC/B,EAAE,mBAAmB;;;;;;;;;;EA0XrB"}

package/dist/legacy/core/lib/useVoiceMode.js

@@ -12,7 +12,7 @@ const USER_TRANSCRIPT_BLOCKED = new Set([
 const ASSISTANT_MIC_HOLD_STATUSES = new Set([
     'transcribing', 'thinking', 'generating', 'speaking',
 ]);
-export function useVoiceMode({ apiKey, apiSecret, pseudoId, externalId, aitId, walletAddress, aitToken, metadata, nxtlinqApi, getMessages, setMessages, onError, stopRecording, stopTextToSpeech, voiceTransport = 'ws-realtime', }) {
+export function useVoiceMode({ apiKey, apiSecret, pseudoId, externalId, aitId, walletAddress, aitToken, metadata, nxtlinqApi, getMessages, setMessages, onError, onToolCall, onToolResult, stopRecording, stopTextToSpeech, voiceTransport = 'ws-realtime', }) {
     const [isVoiceMode, setIsVoiceMode] = React.useState(false);
     const [voiceStatus, setVoiceStatus] = React.useState('idle');
     const [isVoiceConnecting, setIsVoiceConnecting] = React.useState(false);
@@ -170,6 +170,8 @@ export function useVoiceMode({ apiKey, apiSecret, pseudoId, externalId, aitId, w
         onStatus: (_status) => { },
         onTranscript: (_event) => { },
         onDone: (_event) => { },
+        onToolCall: (_event) => { },
+        onToolResult: (_event) => { },
     });
     const stopVoiceSession = React.useCallback(async (reason = 'unknown') => {
         const session = sessionRef.current;
@@ -289,6 +291,8 @@ export function useVoiceMode({ apiKey, apiSecret, pseudoId, externalId, aitId, w
                 onStatus: (status) => voiceHandlersRef.current.onStatus(status),
                 onTranscript: (event) => voiceHandlersRef.current.onTranscript(event),
                 onDone: (event) => voiceHandlersRef.current.onDone(event),
+                onToolCall: (event) => voiceHandlersRef.current.onToolCall(event),
+                onToolResult: (event) => voiceHandlersRef.current.onToolResult(event),
                 onError: (err) => {
                     if (isConnectCancelled())
                         return;
@@ -358,8 +362,10 @@ export function useVoiceMode({ apiKey, apiSecret, pseudoId, externalId, aitId, w
                 handleTranscriptUi(event);
             },
             onDone: handleDone,
+            onToolCall: (event) => onToolCall?.(event),
+            onToolResult: (event) => onToolResult?.(event),
         };
-    }, [handleStatus, handleTranscriptTurn, handleTranscriptUi, handleDone]);
+    }, [handleStatus, handleTranscriptTurn, handleTranscriptUi, handleDone, onToolCall, onToolResult]);
     React.useEffect(() => () => { void stopVoiceSessionRef.current('unmount'); }, []);
     // 追踪当前的 isVoiceMode 状态，用于 visibilitychange 事件
     const isVoiceModeRef = React.useRef(isVoiceMode);

package/dist/legacy/index.d.ts

@@ -6,7 +6,7 @@ export { default as useLocalStorage } from './core/lib/useLocalStorage';
 export { default as useSessionStorage } from './core/lib/useSessionStorage';
 export { useResizable } from './core/lib/useResizable';
 export type { ResizeCorner } from './core/lib/useResizable';
-export { synthesizeSpeechToBuffer } from './core/lib/textToSpeech';
+export { synthesizeSpeechToBuffer, streamSpeechToAudioContext } from './core/lib/textToSpeech';
 export { useSpeechToTextFromMic } from './core/lib/useSpeechToTextFromMic';
 export { useVoiceMode } from './core/lib/useVoiceMode';
 export { default as metakeepClient } from './core/metakeepClient';

package/dist/legacy/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/legacy/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACrF,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAChE,mBAAmB,aAAa,CAAC;AAEjC,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,OAAO,IAAI,eAAe,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EAAE,OAAO,IAAI,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAC5E,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,YAAY,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,sBAAsB,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAEvD,OAAO,EAAE,OAAO,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,eAAe,MAAM,8BAA8B,CAAC;AAChE,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,aAAa,GACd,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,kBAAkB,GACnB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,uBAAuB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/legacy/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACrF,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAChE,mBAAmB,aAAa,CAAC;AAEjC,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,OAAO,IAAI,eAAe,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EAAE,OAAO,IAAI,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAC5E,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,YAAY,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AAC/F,OAAO,EAAE,sBAAsB,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAEvD,OAAO,EAAE,OAAO,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,eAAe,MAAM,8BAA8B,CAAC;AAChE,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,aAAa,GACd,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,kBAAkB,GACnB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,uBAAuB,CAAC"}

package/dist/legacy/index.js

@@ -4,7 +4,7 @@ export { useDraggable } from './core/lib/useDraggable';
 export { default as useLocalStorage } from './core/lib/useLocalStorage';
 export { default as useSessionStorage } from './core/lib/useSessionStorage';
 export { useResizable } from './core/lib/useResizable';
-export { synthesizeSpeechToBuffer } from './core/lib/textToSpeech';
+export { synthesizeSpeechToBuffer, streamSpeechToAudioContext } from './core/lib/textToSpeech';
 export { useSpeechToTextFromMic } from './core/lib/useSpeechToTextFromMic';
 export { useVoiceMode } from './core/lib/useVoiceMode';
 export { default as metakeepClient } from './core/metakeepClient';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bytexbyte/nxtlinq-ai-agent-web-development",
-  "version": "0.1.7",
+  "version": "0.1.9",
   "description": "React Web headless SDK for nxtlinq AI Agent — hooks and browser ports",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -37,7 +37,7 @@
     "react-dom": ">=18.0.0"
   },
   "dependencies": {
-    "@bytexbyte/nxtlinq-ai-agent-core-development": "0.4.1",
+    "@bytexbyte/nxtlinq-ai-agent-core-development": "0.4.4",
     "ethers": "^6.16.0",
     "fast-json-stable-stringify": "^2.1.0",
     "metakeep": "^2.2.8",

package/src/hooks/useNxtlinqAgent.ts

@@ -18,6 +18,8 @@ export type UseNxtlinqAgentResult = NxtlinqAgentSnapshot & {
   setMessages: (messages: Message[]) => void;
   postTextTts: (text: string) => Promise<PostTextTtsResult>;
   buildTextTtsPlaybackUri: (result: PostTextTtsResult) => string;
+  /** Stream PCM16 TTS using browser fetch streaming. Web-only. */
+  streamTextTts: (text: string, onChunk: (pcm16: Int16Array) => void, signal?: AbortSignal) => Promise<void>;
 };
 
 /** Subscribe to {@link NxtlinqAgent} state for custom Web chat UIs. */
@@ -60,6 +62,12 @@ export function useNxtlinqAgent(): UseNxtlinqAgentResult {
     [agent],
   );
 
+  const streamTextTts = useCallback(
+    (text: string, onChunk: (pcm16: Int16Array) => void, signal?: AbortSignal) =>
+      agent.streamTextTts(text, onChunk, signal),
+    [agent],
+  );
+
   return {
     agent,
     ...snapshot,
@@ -69,5 +77,6 @@ export function useNxtlinqAgent(): UseNxtlinqAgentResult {
     setMessages,
     postTextTts,
     buildTextTtsPlaybackUri,
+    streamTextTts,
   };
 }

package/src/index.ts

@@ -62,6 +62,7 @@ export {
   useSessionStorage,
   useResizable,
   synthesizeSpeechToBuffer,
+  streamSpeechToAudioContext,
   useSpeechToTextFromMic,
   useVoiceMode,
   metakeepClient,

package/src/legacy/core/lib/textToSpeech.ts

@@ -1,4 +1,5 @@
 import { SpeechConfig, SpeechSynthesisResult, SpeechSynthesizer } from 'microsoft-cognitiveservices-speech-sdk';
+import { streamTextTts } from '@bytexbyte/nxtlinq-ai-agent-core-development';
 import { getAiAgentApiHost } from '../../api/nxtlinq-api';
 import type { ClientTtsVoiceSettings } from '@bytexbyte/nxtlinq-ai-agent-core-development';
 
@@ -125,3 +126,94 @@ export async function synthesizeSpeechToBuffer(params: {
   if (!buffer) return undefined;
   return { buffer, mimeType: 'audio/wav' };
 }
+
+/**
+ * Stream OpenAI TTS audio progressively into a Web AudioContext.
+ *
+ * Receives raw PCM16 (24 kHz mono) chunks from the server and schedules
+ * them back-to-back on the AudioContext so playback starts as soon as the
+ * first chunk arrives, before the full audio is downloaded.
+ *
+ * @param onSourceScheduled  Called for every AudioBufferSourceNode that is
+ *   scheduled — collect these to stop playback early (e.g. user clicks stop).
+ * @param onFirstChunk  Called when the first audio chunk is ready to play.
+ * @param onEnded       Called when all scheduled buffers have finished playing.
+ */
+// Number of chunks to pre-buffer before starting playback. Each chunk is
+// ~8192 bytes = ~170ms of 24kHz PCM16 audio. Buffering 3 chunks gives ~510ms
+// of headroom to absorb network jitter and prevent gaps at the start.
+const PREBUFFER_CHUNKS = 3;
+
+export async function streamSpeechToAudioContext(params: {
+  text: string;
+  apiKey: string;
+  apiSecret: string;
+  audioCtx: AudioContext;
+  onSourceScheduled?: (source: AudioBufferSourceNode) => void;
+  onFirstChunk?: () => void;
+  onEnded?: () => void;
+  signal?: AbortSignal;
+}): Promise<void> {
+  const { text, apiKey, apiSecret, audioCtx, onSourceScheduled, onFirstChunk, onEnded, signal } =
+    params;
+
+  if (audioCtx.state === 'suspended') await audioCtx.resume();
+
+  let nextPlayTime = 0;
+  let pendingCount = 0;
+  let streamDone = false;
+  // Pre-buffer: collect initial chunks before scheduling any playback so the
+  // AudioContext has enough backlog to play without gaps from network jitter.
+  const prebuffer: AudioBuffer[] = [];
+  let prebuffering = true;
+
+  const checkEnded = () => {
+    if (streamDone && pendingCount === 0) onEnded?.();
+  };
+
+  const scheduleBuffer = (audioBuffer: AudioBuffer) => {
+    const source = audioCtx.createBufferSource();
+    source.buffer = audioBuffer;
+    source.connect(audioCtx.destination);
+    pendingCount++;
+    source.start(nextPlayTime);
+    nextPlayTime += audioBuffer.duration;
+    onSourceScheduled?.(source);
+    source.onended = () => {
+      pendingCount--;
+      checkEnded();
+    };
+  };
+
+  const flushPrebuffer = () => {
+    nextPlayTime = audioCtx.currentTime + 0.05;
+    onFirstChunk?.();
+    for (const buf of prebuffer) scheduleBuffer(buf);
+    prebuffer.length = 0;
+    prebuffering = false;
+  };
+
+  await streamTextTts({ apiKey, apiSecret, text }, (pcm16: Int16Array) => {
+    if (signal?.aborted) return;
+
+    const float32 = new Float32Array(pcm16.length);
+    for (let i = 0; i < pcm16.length; i++) float32[i] = pcm16[i] / 32768;
+
+    const audioBuffer = audioCtx.createBuffer(1, float32.length, 24000);
+    audioBuffer.copyToChannel(float32, 0);
+
+    if (prebuffering) {
+      prebuffer.push(audioBuffer);
+      if (prebuffer.length >= PREBUFFER_CHUNKS) flushPrebuffer();
+    } else {
+      scheduleBuffer(audioBuffer);
+    }
+  }, signal);
+
+  // Stream finished — flush any remaining prebuffered chunks (short text that
+  // never reached PREBUFFER_CHUNKS) and mark done.
+  if (prebuffering && prebuffer.length > 0) flushPrebuffer();
+
+  streamDone = true;
+  checkEnded();
+}

package/src/legacy/core/lib/useVoiceMode.ts

@@ -7,6 +7,8 @@ import {
   type VoiceDoneEvent,
   type VoiceSession,
   type VoiceStatus,
+  type VoiceToolCallEvent,
+  type VoiceToolResultEvent,
   type VoiceTranscriptEvent,
 } from '../../api/voice';
 import { appendServerHistoryIntoMessages } from './messageHistory';
@@ -37,6 +39,8 @@ export type UseVoiceModeOptions = {
   getMessages: () => Message[];
   setMessages: React.Dispatch<React.SetStateAction<Message[]>>;
   onError?: (error: Error) => void;
+  onToolCall?: (event: VoiceToolCallEvent) => void;
+  onToolResult?: (event: VoiceToolResultEvent) => void;
   stopRecording: () => void;
   stopTextToSpeech: () => void;
   voiceTransport?: VoiceTransport;
@@ -55,6 +59,8 @@ export function useVoiceMode({
   getMessages,
   setMessages,
   onError,
+  onToolCall,
+  onToolResult,
   stopRecording,
   stopTextToSpeech,
   voiceTransport = 'ws-realtime',
@@ -222,6 +228,8 @@ export function useVoiceMode({
     onStatus: (_status: VoiceStatus) => {},
     onTranscript: (_event: VoiceTranscriptEvent) => {},
     onDone: (_event: VoiceDoneEvent) => {},
+    onToolCall: (_event: VoiceToolCallEvent) => {},
+    onToolResult: (_event: VoiceToolResultEvent) => {},
   });
 
   const stopVoiceSession = React.useCallback(async (reason = 'unknown') => {
@@ -333,6 +341,8 @@ export function useVoiceMode({
         onStatus: (status) => voiceHandlersRef.current.onStatus(status),
         onTranscript: (event) => voiceHandlersRef.current.onTranscript(event),
         onDone: (event) => voiceHandlersRef.current.onDone(event),
+        onToolCall: (event) => voiceHandlersRef.current.onToolCall(event),
+        onToolResult: (event) => voiceHandlersRef.current.onToolResult(event),
         onError: (err) => {
           if (isConnectCancelled()) return;
           onError?.(err);
@@ -396,8 +406,10 @@ export function useVoiceMode({
         handleTranscriptUi(event);
       },
       onDone: handleDone,
+      onToolCall: (event) => onToolCall?.(event),
+      onToolResult: (event) => onToolResult?.(event),
     };
-  }, [handleStatus, handleTranscriptTurn, handleTranscriptUi, handleDone]);
+  }, [handleStatus, handleTranscriptTurn, handleTranscriptUi, handleDone, onToolCall, onToolResult]);
 
   React.useEffect(() => () => { void stopVoiceSessionRef.current('unmount'); }, []);
 

package/src/legacy/index.ts

@@ -7,7 +7,7 @@ export { default as useLocalStorage } from './core/lib/useLocalStorage';
 export { default as useSessionStorage } from './core/lib/useSessionStorage';
 export { useResizable } from './core/lib/useResizable';
 export type { ResizeCorner } from './core/lib/useResizable';
-export { synthesizeSpeechToBuffer } from './core/lib/textToSpeech';
+export { synthesizeSpeechToBuffer, streamSpeechToAudioContext } from './core/lib/textToSpeech';
 export { useSpeechToTextFromMic } from './core/lib/useSpeechToTextFromMic';
 export { useVoiceMode } from './core/lib/useVoiceMode';