modular-voice-agent-sdk 1.0.0

package/dist/client/audio-recorder.js

@@ -0,0 +1,128 @@
+/**
+ * Audio Recorder
+ *
+ * Handles microphone capture with AudioWorklet for real-time PCM streaming.
+ */
+const PCM_WORKLET_CODE = `
+class PCMProcessor extends AudioWorkletProcessor {
+  process(inputs) {
+    const input = inputs[0][0];
+    if (input) {
+      // Convert to Int16 for transmission efficiency
+      const int16 = new Int16Array(input.length);
+      for (let i = 0; i < input.length; i++) {
+        int16[i] = Math.max(-32768, Math.min(32767, input[i] * 32768));
+      }
+      this.port.postMessage(int16);
+    }
+    return true;
+  }
+}
+registerProcessor('pcm-processor', PCMProcessor);
+`;
+export class AudioRecorder {
+    sampleRate;
+    audioContext = null;
+    mediaStream = null;
+    mediaStreamSource = null;
+    workletNode = null;
+    workletRegistered = false;
+    isRecording = false;
+    onChunkCallback = null;
+    constructor(config = {}) {
+        this.sampleRate = config.sampleRate ?? 16000;
+    }
+    /**
+     * Check if currently recording
+     */
+    get recording() {
+        return this.isRecording;
+    }
+    /**
+     * Set callback for audio chunks (called during streaming recording)
+     */
+    onChunk(callback) {
+        this.onChunkCallback = callback;
+    }
+    /**
+     * Start recording from microphone
+     */
+    async start() {
+        if (this.isRecording)
+            return;
+        this.isRecording = true;
+        // Create fresh AudioContext for each session
+        if (this.audioContext) {
+            await this.audioContext.close();
+            this.workletRegistered = false;
+        }
+        this.audioContext = new AudioContext({ sampleRate: this.sampleRate });
+        // Register worklet if needed
+        if (!this.workletRegistered) {
+            const blob = new Blob([PCM_WORKLET_CODE], { type: 'application/javascript' });
+            await this.audioContext.audioWorklet.addModule(URL.createObjectURL(blob));
+            this.workletRegistered = true;
+        }
+        // Get microphone stream
+        this.mediaStream = await navigator.mediaDevices.getUserMedia({
+            audio: {
+                sampleRate: this.sampleRate,
+                channelCount: 1,
+                echoCancellation: true,
+                noiseSuppression: true,
+            },
+        });
+        // Create audio graph
+        this.mediaStreamSource = this.audioContext.createMediaStreamSource(this.mediaStream);
+        this.workletNode = new AudioWorkletNode(this.audioContext, 'pcm-processor');
+        // Handle audio chunks
+        this.workletNode.port.onmessage = (e) => {
+            if (this.onChunkCallback) {
+                // Convert Int16 back to Float32
+                const float32 = new Float32Array(e.data.length);
+                for (let i = 0; i < e.data.length; i++) {
+                    float32[i] = e.data[i] / 32768.0;
+                }
+                this.onChunkCallback(float32);
+            }
+        };
+        // Connect the graph
+        this.mediaStreamSource.connect(this.workletNode);
+    }
+    /**
+     * Stop recording and clean up resources
+     */
+    async stop() {
+        if (!this.isRecording)
+            return;
+        this.isRecording = false;
+        // Disconnect nodes
+        if (this.mediaStreamSource) {
+            this.mediaStreamSource.disconnect();
+            this.mediaStreamSource = null;
+        }
+        if (this.workletNode) {
+            this.workletNode.disconnect();
+            this.workletNode = null;
+        }
+        // Stop media tracks
+        if (this.mediaStream) {
+            this.mediaStream.getTracks().forEach((track) => track.stop());
+            this.mediaStream = null;
+        }
+        // Close audio context
+        if (this.audioContext) {
+            await this.audioContext.close();
+            this.audioContext = null;
+            this.workletRegistered = false;
+        }
+    }
+    /**
+     * Clean up all resources
+     */
+    async dispose() {
+        await this.stop();
+        this.onChunkCallback = null;
+    }
+}
+//# sourceMappingURL=audio-recorder.js.map

package/dist/client/audio-recorder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"audio-recorder.js","sourceRoot":"","sources":["../../src/client/audio-recorder.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AASH,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;CAgBxB,CAAC;AAEF,MAAM,OAAO,aAAa;IAChB,UAAU,CAAS;IACnB,YAAY,GAAwB,IAAI,CAAC;IACzC,WAAW,GAAuB,IAAI,CAAC;IACvC,iBAAiB,GAAsC,IAAI,CAAC;IAC5D,WAAW,GAA4B,IAAI,CAAC;IAC5C,iBAAiB,GAAG,KAAK,CAAC;IAC1B,WAAW,GAAG,KAAK,CAAC;IACpB,eAAe,GAA8B,IAAI,CAAC;IAE1D,YAAY,SAA8B,EAAE;QAC1C,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,QAA4B;QAClC,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,6CAA6C;QAC7C,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;QACjC,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAEtE,6BAA6B;QAC7B,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,gBAAgB,CAAC,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;YAC9E,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;YAC1E,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAChC,CAAC;QAED,wBAAwB;QACxB,IAAI,CAAC,WAAW,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,YAAY,CAAC;YAC3D,KAAK,EAAE;gBACL,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,YAAY,EAAE,CAAC;gBACf,gBAAgB,EAAE,IAAI;gBACtB,gBAAgB,EAAE,IAAI;aACvB;SACF,CAAC,CAAC;QAEH,qBAAqB;QACrB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,YAAY,CAAC,uBAAuB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACrF,IAAI,CAAC,WAAW,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;QAE5E,sBAAsB;QACtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAA2B,EAAE,EAAE;YAChE,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBACzB,gCAAgC;gBAChC,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACvC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;gBACnC,CAAC;gBACD,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAChC,CAAC;QACH,CAAC,CAAC;QAEF,oBAAoB;QACpB,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,OAAO;QAC9B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QAEzB,mBAAmB;QACnB,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,CAAC;YACpC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAChC,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;YAC9B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;QAED,oBAAoB;QACpB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YAC9D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;QAED,sBAAsB;QACtB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;YACzB,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;QACjC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;IAC9B,CAAC;CACF"}

package/dist/client/index.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Modular Voice Agent SDK - Browser Client
+ *
+ * High-level SDK for building voice assistant browser clients.
+ *
+ * @example
+ * ```typescript
+ * import { VoiceClient } from 'modular-voice-agent-sdk/client';
+ *
+ * const client = new VoiceClient({ serverUrl: 'ws://localhost:8080' });
+ *
+ * client.on('transcript', (text) => console.log('User:', text));
+ * client.on('responseChunk', (chunk) => console.log('Assistant:', chunk));
+ * client.on('status', (status) => console.log('Status:', status));
+ *
+ * client.connect();
+ *
+ * // Push-to-talk
+ * button.onmousedown = () => client.startRecording();
+ * button.onmouseup = () => client.stopRecording();
+ * ```
+ */
+export { VoiceClient, createVoiceClient } from './voice-client';
+export type { VoiceClientConfig, VoiceClientEvents, VoiceClientStatus, BrowserSupport } from './voice-client';
+export { WebSpeechSTT } from './web-speech-stt';
+export type { WebSpeechSTTConfig, WebSpeechSTTResult } from './web-speech-stt';
+export { WebSpeechTTS } from './web-speech-tts';
+export type { WebSpeechTTSConfig } from './web-speech-tts';
+export { AudioRecorder } from './audio-recorder';
+export type { AudioRecorderConfig, AudioChunkCallback } from './audio-recorder';
+export { AudioPlayer } from './audio-player';
+export type { AudioPlayerConfig } from './audio-player';
+export { float32ToBase64, base64ToFloat32, type ClientMessage, type ServerMessage, type AudioMessage, type EndAudioMessage, type ClearHistoryMessage, type TextMessage, type TranscriptMessage, type ResponseChunkMessage, type AudioResponseMessage, type CompleteMessage, type ErrorMessage, } from './protocol';
+//# sourceMappingURL=index.d.ts.map

package/dist/client/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAGH,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAChE,YAAY,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAG9G,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,YAAY,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE/E,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAG3D,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,YAAY,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEhF,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAGxD,OAAO,EACL,eAAe,EACf,eAAe,EACf,KAAK,aAAa,EAClB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAChB,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,KAAK,eAAe,EACpB,KAAK,YAAY,GAClB,MAAM,YAAY,CAAC"}

package/dist/client/index.js

@@ -0,0 +1,33 @@
+/**
+ * Modular Voice Agent SDK - Browser Client
+ *
+ * High-level SDK for building voice assistant browser clients.
+ *
+ * @example
+ * ```typescript
+ * import { VoiceClient } from 'modular-voice-agent-sdk/client';
+ *
+ * const client = new VoiceClient({ serverUrl: 'ws://localhost:8080' });
+ *
+ * client.on('transcript', (text) => console.log('User:', text));
+ * client.on('responseChunk', (chunk) => console.log('Assistant:', chunk));
+ * client.on('status', (status) => console.log('Status:', status));
+ *
+ * client.connect();
+ *
+ * // Push-to-talk
+ * button.onmousedown = () => client.startRecording();
+ * button.onmouseup = () => client.stopRecording();
+ * ```
+ */
+// Main SDK
+export { VoiceClient, createVoiceClient } from './voice-client';
+// Browser speech APIs (for local STT/TTS)
+export { WebSpeechSTT } from './web-speech-stt';
+export { WebSpeechTTS } from './web-speech-tts';
+// Lower-level utilities (for custom implementations)
+export { AudioRecorder } from './audio-recorder';
+export { AudioPlayer } from './audio-player';
+// Protocol types and utilities
+export { float32ToBase64, base64ToFloat32, } from './protocol';
+//# sourceMappingURL=index.js.map

package/dist/client/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,WAAW;AACX,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAGhE,0CAA0C;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAGhD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAGhD,qDAAqD;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAG7C,+BAA+B;AAC/B,OAAO,EACL,eAAe,EACf,eAAe,GAYhB,MAAM,YAAY,CAAC"}

package/dist/client/protocol.d.ts

@@ -0,0 +1,80 @@
+/**
+ * WebSocket Protocol Types for Voice Pipeline
+ *
+ * Shared types for client-server communication.
+ */
+export type AudioMessage = {
+    type: 'audio';
+    data: string;
+    sampleRate: number;
+};
+export type EndAudioMessage = {
+    type: 'end_audio';
+};
+export type ClearHistoryMessage = {
+    type: 'clear_history';
+};
+/**
+ * Text message - sent when client does STT locally
+ * Server skips STT and goes straight to LLM
+ */
+export type TextMessage = {
+    type: 'text';
+    text: string;
+};
+/**
+ * Capabilities message - sent on connect to tell server what client handles
+ */
+export type CapabilitiesMessage = {
+    type: 'capabilities';
+    hasSTT: boolean;
+    hasTTS: boolean;
+};
+export type ClientMessage = AudioMessage | EndAudioMessage | ClearHistoryMessage | TextMessage | CapabilitiesMessage;
+export type TranscriptMessage = {
+    type: 'transcript';
+    text: string;
+};
+export type ResponseChunkMessage = {
+    type: 'response_chunk';
+    text: string;
+};
+export type AudioResponseMessage = {
+    type: 'audio';
+    data: string;
+    sampleRate: number;
+};
+/**
+ * Sent when a tool call is being executed
+ */
+export type ToolCallMessage = {
+    type: 'tool_call';
+    toolCallId: string;
+    name: string;
+    arguments: Record<string, unknown>;
+};
+/**
+ * Sent when a tool call completes
+ */
+export type ToolResultMessage = {
+    type: 'tool_result';
+    toolCallId: string;
+    result: unknown;
+};
+export type CompleteMessage = {
+    type: 'complete';
+};
+export type ErrorMessage = {
+    type: 'error';
+    message: string;
+};
+export type ServerMessage = TranscriptMessage | ResponseChunkMessage | AudioResponseMessage | ToolCallMessage | ToolResultMessage | CompleteMessage | ErrorMessage;
+/**
+ * Encode Float32Array to base64 string
+ */
+export declare function float32ToBase64(audio: Float32Array): string;
+/**
+ * Decode base64 string to Float32Array
+ */
+export declare function base64ToFloat32(data: string): Float32Array;
+//# sourceMappingURL=protocol.d.ts.map

package/dist/client/protocol.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../src/client/protocol.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,IAAI,EAAE,eAAe,CAAC;CACvB,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,YAAY,GAAG,eAAe,GAAG,mBAAmB,GAAG,WAAW,GAAG,mBAAmB,CAAC;AAIrH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,YAAY,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,gBAAgB,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,WAAW,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,aAAa,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,UAAU,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,aAAa,GACrB,iBAAiB,GACjB,oBAAoB,GACpB,oBAAoB,GACpB,eAAe,GACf,iBAAiB,GACjB,eAAe,GACf,YAAY,CAAC;AAIjB;;GAEG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAO3D;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CAO1D"}

package/dist/client/protocol.js

@@ -0,0 +1,29 @@
+/**
+ * WebSocket Protocol Types for Voice Pipeline
+ *
+ * Shared types for client-server communication.
+ */
+// ============ Encoding Utilities ============
+/**
+ * Encode Float32Array to base64 string
+ */
+export function float32ToBase64(audio) {
+    const bytes = new Uint8Array(audio.buffer, audio.byteOffset, audio.byteLength);
+    let binary = '';
+    for (let i = 0; i < bytes.length; i++) {
+        binary += String.fromCharCode(bytes[i]);
+    }
+    return btoa(binary);
+}
+/**
+ * Decode base64 string to Float32Array
+ */
+export function base64ToFloat32(data) {
+    const binary = atob(data);
+    const bytes = new Uint8Array(binary.length);
+    for (let i = 0; i < binary.length; i++) {
+        bytes[i] = binary.charCodeAt(i);
+    }
+    return new Float32Array(bytes.buffer);
+}
+//# sourceMappingURL=protocol.js.map

package/dist/client/protocol.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"protocol.js","sourceRoot":"","sources":["../../src/client/protocol.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AA6FH,+CAA+C;AAE/C;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAAmB;IACjD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IAC/E,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AACxC,CAAC"}

package/dist/client/voice-client.d.ts

@@ -0,0 +1,249 @@
+/**
+ * Voice Client
+ *
+ * Unified browser SDK for voice assistants.
+ * Handles three modes:
+ * 1. Fully local - all components run in browser, no server needed
+ * 2. Fully remote - all processing on server via WebSocket
+ * 3. Hybrid - mix of local and server components
+ *
+ * Component logic:
+ * - Component provided → runs locally
+ * - Component is null + serverUrl → server handles it
+ * - All components local → no WebSocket needed
+ */
+import type { STTPipeline, LLMPipeline, TTSPipeline } from '../types';
+import { WebSpeechSTT } from './web-speech-stt';
+import { WebSpeechTTS } from './web-speech-tts';
+export interface BrowserSupport {
+    /** Web Speech API speech recognition (Chrome, Edge, Safari only) */
+    webSpeechSTT: boolean;
+    /** Web Speech API speech synthesis (most browsers) */
+    webSpeechTTS: boolean;
+    /** WebGPU for ML acceleration (Chrome, Edge) */
+    webGPU: boolean;
+    /** MediaDevices API for microphone access */
+    mediaDevices: boolean;
+    /** WebSocket support */
+    webSocket: boolean;
+    /** AudioContext for audio processing */
+    audioContext: boolean;
+}
+export type VoiceClientStatus = 'disconnected' | 'connecting' | 'initializing' | 'ready' | 'listening' | 'processing' | 'speaking';
+export interface VoiceClientConfig {
+    /**
+     * STT backend - runs locally in browser
+     * - Provide an STTPipeline (e.g., TransformersSTT) or WebSpeechSTT for local STT
+     * - Set to null to use server-side STT (requires serverUrl)
+     * @default null (server handles)
+     */
+    stt?: STTPipeline | WebSpeechSTT | null;
+    /**
+     * LLM backend - runs locally in browser
+     * - Provide an LLMPipeline (e.g., TransformersLLM) for local LLM
+     * - Set to null to use server-side LLM (requires serverUrl)
+     * @default null (server handles)
+     */
+    llm?: LLMPipeline | null;
+    /**
+     * TTS backend - runs locally in browser
+     * - Provide a TTSPipeline (e.g., TransformersTTS) or WebSpeechTTS for local TTS
+     * - Set to null to use server-side TTS (requires serverUrl)
+     * @default null (server handles)
+     */
+    tts?: TTSPipeline | WebSpeechTTS | null;
+    /**
+     * System prompt for the LLM (required if llm is provided)
+     */
+    systemPrompt?: string;
+    /**
+     * WebSocket server URL (required if any component is null)
+     */
+    serverUrl?: string;
+    /**
+     * Audio sample rate for recording (default: 16000)
+     */
+    sampleRate?: number;
+    /**
+     * Auto-reconnect on disconnect - only applies when using server (default: true)
+     */
+    autoReconnect?: boolean;
+    /**
+     * Reconnect delay in ms (default: 2000)
+     */
+    reconnectDelay?: number;
+}
+export interface ToolCallInfo {
+    id: string;
+    name: string;
+    arguments: Record<string, unknown>;
+}
+export interface VoiceClientEvents {
+    /** Connection/initialization status changed */
+    status: (status: VoiceClientStatus) => void;
+    /** User transcript received (from STT) */
+    transcript: (text: string) => void;
+    /** Assistant response chunk (streaming) */
+    responseChunk: (text: string) => void;
+    /** Full response complete */
+    responseComplete: (fullText: string) => void;
+    /** Tool call started (function calling) */
+    toolCall: (toolCall: ToolCallInfo) => void;
+    /** Tool call result received */
+    toolResult: (toolCallId: string, result: unknown) => void;
+    /** Error occurred */
+    error: (error: Error) => void;
+    /** Model loading progress (for local models) */
+    progress: (info: {
+        status: string;
+        file?: string;
+        progress?: number;
+    }) => void;
+}
+type EventName = keyof VoiceClientEvents;
+export declare class VoiceClient {
+    /**
+     * Check browser support for voice features.
+     * Call this before creating a VoiceClient to determine what's available.
+     *
+     * @example
+     * const support = VoiceClient.getBrowserSupport();
+     * if (!support.webSpeechSTT) {
+     *   showMessage("Voice input requires Chrome, Edge, or Safari");
+     * }
+     */
+    static getBrowserSupport(): BrowserSupport;
+    /**
+     * Get a human-readable description of what's not supported.
+     * Returns null if everything needed for basic operation is supported.
+     */
+    static getUnsupportedFeatures(): string[];
+    private config;
+    private mode;
+    private needsServer;
+    private localSTT;
+    private localLLM;
+    private localTTS;
+    private localPipeline;
+    private ws;
+    private recorder;
+    private player;
+    private status;
+    private listeners;
+    private currentResponse;
+    private reconnectTimer;
+    private pendingTTSText;
+    private ttsQueue;
+    private isSpeaking;
+    private conversationId;
+    private history;
+    private audioContext;
+    private mediaRecorder;
+    private audioChunks;
+    private mediaRecording;
+    private generateConversationId;
+    constructor(config: VoiceClientConfig);
+    private setupComponents;
+    private setupWebSpeechSTT;
+    /**
+     * Initialize and connect (if using server)
+     */
+    connect(): Promise<void>;
+    private initializeLocalComponents;
+    private connectWebSocket;
+    /**
+     * Disconnect and clean up
+     */
+    disconnect(): void;
+    /**
+     * Start recording/listening
+     */
+    startRecording(): Promise<void>;
+    /**
+     * Stop recording/listening and process
+     */
+    stopRecording(): Promise<void>;
+    private startMediaRecorder;
+    private stopMediaRecorder;
+    private processLocalAudio;
+    private processTextLocally;
+    private runLocalPipeline;
+    private runLocalLLM;
+    /**
+     * Clear conversation history
+     */
+    clearHistory(): void;
+    /**
+     * Get current status
+     */
+    getStatus(): VoiceClientStatus;
+    /**
+     * Check if ready for interaction
+     */
+    isReady(): boolean;
+    /**
+     * Check if currently recording
+     */
+    isRecording(): boolean;
+    /**
+     * Get current mode
+     */
+    getMode(): 'local' | 'remote' | 'hybrid';
+    /**
+     * Check which components are local
+     */
+    getLocalComponents(): {
+        stt: boolean;
+        llm: boolean;
+        tts: boolean;
+    };
+    /**
+     * Subscribe to events
+     */
+    on<K extends EventName>(event: K, callback: VoiceClientEvents[K]): void;
+    /**
+     * Unsubscribe from events
+     */
+    off<K extends EventName>(event: K, callback: VoiceClientEvents[K]): void;
+    /**
+     * Clean up all resources
+     */
+    dispose(): Promise<void>;
+    private send;
+    private handleServerMessage;
+    private handleLocalTTSChunk;
+    private flushLocalTTS;
+    private processLocalTTSQueue;
+    private setStatus;
+    private emit;
+    private scheduleReconnect;
+    private validateBrowserSupport;
+}
+/**
+ * Create a VoiceClient instance
+ * @example
+ * // Fully local
+ * const client = createVoiceClient({
+ *   stt: new TransformersSTT({ model: '...' }),
+ *   llm: new TransformersLLM({ model: '...' }),
+ *   tts: new WebSpeechTTS(),
+ *   systemPrompt: 'You are a helpful assistant.',
+ * });
+ *
+ * @example
+ * // Fully remote
+ * const client = createVoiceClient({
+ *   serverUrl: 'ws://localhost:3000',
+ * });
+ *
+ * @example
+ * // Hybrid: local STT/TTS, server LLM
+ * const client = createVoiceClient({
+ *   stt: new WebSpeechSTT(),
+ *   tts: new WebSpeechTTS(),
+ *   serverUrl: 'ws://localhost:3000',
+ * });
+ */
+export declare function createVoiceClient(config: VoiceClientConfig): VoiceClient;
+export {};
+//# sourceMappingURL=voice-client.d.ts.map

package/dist/client/voice-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"voice-client.d.ts","sourceRoot":"","sources":["../../src/client/voice-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAmC,MAAM,UAAU,CAAC;AAIvG,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAUhD,MAAM,WAAW,cAAc;IAC7B,oEAAoE;IACpE,YAAY,EAAE,OAAO,CAAC;IACtB,sDAAsD;IACtD,YAAY,EAAE,OAAO,CAAC;IACtB,gDAAgD;IAChD,MAAM,EAAE,OAAO,CAAC;IAChB,6CAA6C;IAC7C,YAAY,EAAE,OAAO,CAAC;IACtB,wBAAwB;IACxB,SAAS,EAAE,OAAO,CAAC;IACnB,wCAAwC;IACxC,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,MAAM,iBAAiB,GACzB,cAAc,GACd,YAAY,GACZ,cAAc,GACd,OAAO,GACP,WAAW,GACX,YAAY,GACZ,UAAU,CAAC;AAEf,MAAM,WAAW,iBAAiB;IAChC;;;;;OAKG;IACH,GAAG,CAAC,EAAE,WAAW,GAAG,YAAY,GAAG,IAAI,CAAC;IAExC;;;;;OAKG;IACH,GAAG,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IAEzB;;;;;OAKG;IACH,GAAG,CAAC,EAAE,WAAW,GAAG,YAAY,GAAG,IAAI,CAAC;IAExC;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IAExB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,iBAAiB;IAChC,+CAA+C;IAC/C,MAAM,EAAE,CAAC,MAAM,EAAE,iBAAiB,KAAK,IAAI,CAAC;IAC5C,0CAA0C;IAC1C,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,2CAA2C;IAC3C,aAAa,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACtC,6BAA6B;IAC7B,gBAAgB,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IAC7C,2CAA2C;IAC3C,QAAQ,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;IAC3C,gCAAgC;IAChC,UAAU,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,CAAC;IAC1D,qBAAqB;IACrB,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IAC9B,gDAAgD;IAChD,QAAQ,EAAE,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CAChF;AAED,KAAK,SAAS,GAAG,MAAM,iBAAiB,CAAC;AAezC,qBAAa,WAAW;IAGtB;;;;;;;;;OASG;IACH,MAAM,CAAC,iBAAiB,IAAI,cAAc;IAmB1C;;;OAGG;IACH,MAAM,CAAC,sBAAsB,IAAI,MAAM,EAAE;IAsBzC,OAAO,CAAC,MAAM,CAMZ;IAGF,OAAO,CAAC,IAAI,CAAgC;IAC5C,OAAO,CAAC,WAAW,CAAU;IAG7B,OAAO,CAAC,QAAQ,CAA2C;IAC3D,OAAO,CAAC,QAAQ,CAA4B;IAC5C,OAAO,CAAC,QAAQ,CAA2C;IAC3D,OAAO,CAAC,aAAa,CAA8B;IAGnD,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,QAAQ,CAA8B;IAC9C,OAAO,CAAC,MAAM,CAA4B;IAG1C,OAAO,CAAC,MAAM,CAAqC;IACnD,OAAO,CAAC,SAAS,CAAuD;IACxE,OAAO,CAAC,eAAe,CAAM;IAC7B,OAAO,CAAC,cAAc,CAA8C;IACpE,OAAO,CAAC,cAAc,CAAM;IAC5B,OAAO,CAAC,QAAQ,CAAgB;IAChC,OAAO,CAAC,UAAU,CAAS;IAG3B,OAAO,CAAC,cAAc,CAAiC;IACvD,OAAO,CAAC,OAAO,CAAkF;IAGjG,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,cAAc,CAAS;IAE/B,OAAO,CAAC,sBAAsB;gBAIlB,MAAM,EAAE,iBAAiB;IAgDrC,OAAO,CAAC,eAAe;IAiDvB,OAAO,CAAC,iBAAiB;IAiCzB;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;YAehB,yBAAyB;YAsCzB,gBAAgB;IAsC9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAiBlB;;OAEG;IACG,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBrC;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;YAiBtB,kBAAkB;YAclB,iBAAiB;YAMjB,iBAAiB;YAuCjB,kBAAkB;YAYlB,gBAAgB;YAkDhB,WAAW;IAgDzB;;OAEG;IACH,YAAY,IAAI,IAAI;IAepB;;OAEG;IACH,SAAS,IAAI,iBAAiB;IAI9B;;OAEG;IACH,OAAO,IAAI,OAAO;IAOlB;;OAEG;IACH,WAAW,IAAI,OAAO;IAQtB;;OAEG;IACH,OAAO,IAAI,OAAO,GAAG,QAAQ,GAAG,QAAQ;IAIxC;;OAEG;IACH,kBAAkB,IAAI;QAAE,GAAG,EAAE,OAAO,CAAC;QAAC,GAAG,EAAE,OAAO,CAAC;QAAC,GAAG,EAAE,OAAO,CAAA;KAAE;IAQlE;;OAEG;IACH,EAAE,CAAC,CAAC,SAAS,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,IAAI;IAOvE;;OAEG;IACH,GAAG,CAAC,CAAC,SAAS,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,IAAI;IAIxE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAY9B,OAAO,CAAC,IAAI;IAMZ,OAAO,CAAC,mBAAmB;IA+D3B,OAAO,CAAC,mBAAmB;IAmB3B,OAAO,CAAC,aAAa;YASP,oBAAoB;IA2BlC,OAAO,CAAC,SAAS;IAOjB,OAAO,CAAC,IAAI;IAaZ,OAAO,CAAC,iBAAiB;IAQzB,OAAO,CAAC,sBAAsB;CAwD/B;AAID;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,iBAAiB,GAAG,WAAW,CAExE"}