@llmrtc/llmrtc-web-client 1.0.0

package/dist/index.js

@@ -0,0 +1,708 @@
+import EventEmitter from 'eventemitter3';
+import { z } from 'zod';
+import { NativePeer } from './native-peer.js';
+import { ConnectionStateMachine, ConnectionState } from './connection-state.js';
+import { PROTOCOL_VERSION } from '@llmrtc/llmrtc-core';
+// Re-export for convenience
+export { ConnectionState } from './connection-state.js';
+export { NativePeer } from './native-peer.js';
+export { PROTOCOL_VERSION } from '@llmrtc/llmrtc-core';
+const MessageSchema = z.object({ type: z.string() }).passthrough();
+const HEARTBEAT_INTERVAL_MS = 15000;
+const HEARTBEAT_TIMEOUT_MS = 10000;
+const MAX_MISSED_HEARTBEATS = 2;
+/**
+ * Default ICE servers - Metered STUN server
+ * Used when no custom ICE servers configured and server doesn't provide any
+ */
+const DEFAULT_ICE_SERVERS = [
+    { urls: 'stun:stun.metered.ca:80' }
+];
+export class LLMRTCWebClient extends EventEmitter {
+    constructor(config) {
+        super();
+        Object.defineProperty(this, "config", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: config
+        });
+        Object.defineProperty(this, "ws", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "peer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "stateMachine", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "sessionId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "heartbeatInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "heartbeatTimeout", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "missedHeartbeats", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "reconnectTimeout", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** ICE servers received from server in ready message */
+        Object.defineProperty(this, "serverIceServers", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        // Media state
+        Object.defineProperty(this, "audioTrack", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "audioStream", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "audioSender", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "videoCapture", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "screenCapture", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // Default reconnection to enabled
+        const reconnectionConfig = {
+            enabled: true,
+            ...config.reconnection
+        };
+        this.stateMachine = new ConnectionStateMachine(reconnectionConfig);
+        this.stateMachine.on('stateChange', ({ to }) => {
+            this.emit('stateChange', to);
+        });
+    }
+    /**
+     * Get the current connection state.
+     */
+    get state() {
+        return this.stateMachine.state;
+    }
+    /**
+     * Get the session ID assigned by the server.
+     */
+    get currentSessionId() {
+        return this.sessionId;
+    }
+    /**
+     * Start the connection to the server.
+     */
+    async start() {
+        if (this.stateMachine.state !== ConnectionState.DISCONNECTED) {
+            throw new Error('Client already started or connecting');
+        }
+        this.stateMachine.transition(ConnectionState.CONNECTING);
+        try {
+            await this.connect();
+            this.stateMachine.transition(ConnectionState.CONNECTED);
+        }
+        catch (error) {
+            this.handleConnectionError(error);
+            throw error;
+        }
+    }
+    async connect() {
+        // 1. Establish WebSocket
+        await this.connectWebSocket();
+        // 2. Wait for ready message with session ID and ICE servers
+        await this.waitForReady();
+        // 3. Resolve ICE servers
+        // Priority: client config > server-provided > default STUN
+        const iceServers = this.config.iceServers?.length
+            ? this.config.iceServers
+            : this.serverIceServers?.length
+                ? this.serverIceServers
+                : DEFAULT_ICE_SERVERS;
+        console.log('[web-client] Using', iceServers.length, 'ICE servers');
+        // 4. Create peer connection
+        this.peer = new NativePeer({ iceServers, trickle: false }, true);
+        this.setupPeerEventHandlers();
+        // 5. Create offer (triggers signal event which sends to server)
+        await this.peer.createOffer();
+        // 6. Wait for peer connection to be established
+        await this.waitForPeerConnection();
+        // 7. Start heartbeat
+        this.startHeartbeat();
+    }
+    connectWebSocket() {
+        return new Promise((resolve, reject) => {
+            console.log('[web-client] Connecting to', this.config.signallingUrl);
+            this.ws = new WebSocket(this.config.signallingUrl);
+            const onOpen = () => {
+                console.log('[web-client] WebSocket connected');
+                cleanup();
+                resolve();
+            };
+            const onError = (e) => {
+                console.error('[web-client] WebSocket error:', e);
+                cleanup();
+                reject(new Error('WebSocket connection failed'));
+            };
+            const cleanup = () => {
+                this.ws?.removeEventListener('open', onOpen);
+                this.ws?.removeEventListener('error', onError);
+            };
+            this.ws.addEventListener('open', onOpen);
+            this.ws.addEventListener('error', onError);
+            this.ws.onmessage = (ev) => this.handleSignalingMessage(ev.data);
+            this.ws.onclose = () => this.handleWebSocketClose();
+        });
+    }
+    waitForReady() {
+        return new Promise((resolve, reject) => {
+            const timeout = setTimeout(() => {
+                reject(new Error('Timeout waiting for ready message'));
+            }, 10000);
+            const handler = (ev) => {
+                try {
+                    const msg = JSON.parse(ev.data);
+                    if (msg.type === 'ready') {
+                        clearTimeout(timeout);
+                        this.sessionId = msg.id;
+                        // Store server-provided ICE servers
+                        if (msg.iceServers?.length) {
+                            this.serverIceServers = msg.iceServers;
+                            console.log('[web-client] Received', msg.iceServers.length, 'ICE servers from server');
+                        }
+                        // Check protocol version
+                        const serverVersion = msg.protocolVersion ?? 0;
+                        if (serverVersion !== PROTOCOL_VERSION) {
+                            console.warn(`[web-client] Protocol version mismatch: client=${PROTOCOL_VERSION}, server=${serverVersion}`);
+                        }
+                        console.log('[web-client] Session ID:', this.sessionId, 'Protocol version:', serverVersion);
+                        this.ws?.removeEventListener('message', handler);
+                        resolve();
+                    }
+                }
+                catch {
+                    // Ignore parse errors
+                }
+            };
+            this.ws?.addEventListener('message', handler);
+        });
+    }
+    setupPeerEventHandlers() {
+        if (!this.peer)
+            return;
+        this.peer.on('signal', (signal) => {
+            console.log('[web-client] Sending offer signal');
+            this.ws?.send(JSON.stringify({ type: 'offer', signal }));
+        });
+        this.peer.on('data', (data) => {
+            const str = typeof data === 'string' ? data : new TextDecoder().decode(data);
+            this.handlePayload(str);
+        });
+        this.peer.on('connect', () => {
+            console.log('[web-client] Peer connected');
+        });
+        this.peer.on('close', () => {
+            console.log('[web-client] Peer closed');
+            if (this.stateMachine.state === ConnectionState.CONNECTED) {
+                this.scheduleReconnect();
+            }
+        });
+        this.peer.on('error', (err) => {
+            console.error('[web-client] Peer error:', err.message);
+            this.emit('error', {
+                code: 'WEBRTC_ERROR',
+                message: err.message,
+                recoverable: true
+            });
+        });
+        this.peer.on('track', (track, stream) => {
+            if (track.kind === 'audio') {
+                console.log('[web-client] Received TTS audio track from server');
+                this.emit('ttsTrack', stream);
+            }
+        });
+        this.peer.on('connectionStateChange', (state) => {
+            console.log('[web-client] Connection state changed:', state);
+            if (state === 'failed' || state === 'disconnected') {
+                if (this.stateMachine.state === ConnectionState.CONNECTED) {
+                    this.scheduleReconnect();
+                }
+            }
+        });
+    }
+    waitForPeerConnection() {
+        return new Promise((resolve, reject) => {
+            if (this.peer?.connected) {
+                resolve();
+                return;
+            }
+            const timeout = setTimeout(() => {
+                cleanup();
+                reject(new Error('Timeout waiting for peer connection'));
+            }, 30000);
+            const onConnect = () => {
+                clearTimeout(timeout);
+                cleanup();
+                resolve();
+            };
+            const onError = (err) => {
+                clearTimeout(timeout);
+                cleanup();
+                reject(err);
+            };
+            const cleanup = () => {
+                this.peer?.off('connect', onConnect);
+                this.peer?.off('error', onError);
+            };
+            this.peer?.on('connect', onConnect);
+            this.peer?.on('error', onError);
+        });
+    }
+    handleSignalingMessage(raw) {
+        try {
+            const parsed = MessageSchema.safeParse(JSON.parse(raw));
+            if (!parsed.success)
+                return;
+            const msg = parsed.data;
+            switch (msg.type) {
+                case 'signal':
+                    if (this.peer && !this.peer.destroyed) {
+                        console.log('[web-client] Received answer signal');
+                        this.peer.signal(msg.signal);
+                    }
+                    break;
+                case 'pong':
+                    // Reset missed heartbeats on pong
+                    this.missedHeartbeats = 0;
+                    if (this.heartbeatTimeout) {
+                        clearTimeout(this.heartbeatTimeout);
+                        this.heartbeatTimeout = undefined;
+                    }
+                    break;
+                case 'reconnect-ack':
+                    console.log('[web-client] Reconnect acknowledged:', msg.historyRecovered ? 'history recovered' : 'new session', 'sessionId:', msg.sessionId);
+                    // Update session ID to the one confirmed by the server
+                    // This ensures sessionId is preserved across reconnections when history is recovered
+                    if (msg.sessionId) {
+                        this.sessionId = msg.sessionId;
+                    }
+                    break;
+                default:
+                    // Only process payload messages from WebSocket if DataChannel is NOT connected
+                    // This prevents duplicate message handling since backend sends to both channels
+                    if (!this.peer?.connected) {
+                        this.handlePayload(raw);
+                    }
+                // When DataChannel is connected, ignore payload messages from WebSocket
+                // They will be handled by the DataChannel's 'data' event
+            }
+        }
+        catch (err) {
+            console.error('[web-client] Error handling signal:', err);
+        }
+    }
+    handlePayload(raw) {
+        try {
+            const msg = JSON.parse(raw);
+            switch (msg.type) {
+                case 'transcript':
+                    this.emit('transcript', msg.text);
+                    break;
+                case 'llm-chunk':
+                    if (msg.content)
+                        this.emit('llmChunk', msg.content);
+                    break;
+                case 'llm':
+                    this.emit('llm', msg.text);
+                    break;
+                case 'tts':
+                    if (msg.data)
+                        this.emit('tts', base64ToArrayBuffer(msg.data), msg.format ?? 'mp3');
+                    break;
+                case 'tts-start':
+                    this.emit('ttsStart');
+                    break;
+                case 'tts-complete':
+                    this.emit('ttsComplete');
+                    break;
+                case 'tts-cancelled':
+                    this.emit('ttsCancelled');
+                    break;
+                case 'speech-start':
+                    this.emit('speechStart');
+                    break;
+                case 'speech-end':
+                    this.sendAttachments();
+                    this.emit('speechEnd');
+                    break;
+                case 'tool-call-start':
+                    this.emit('toolCallStart', {
+                        name: msg.name,
+                        callId: msg.callId,
+                        arguments: msg.arguments ?? {}
+                    });
+                    break;
+                case 'tool-call-end':
+                    this.emit('toolCallEnd', {
+                        callId: msg.callId,
+                        result: msg.result,
+                        error: msg.error,
+                        durationMs: msg.durationMs ?? 0
+                    });
+                    break;
+                case 'stage-change':
+                    this.emit('stageChange', {
+                        from: msg.from,
+                        to: msg.to,
+                        reason: msg.reason ?? ''
+                    });
+                    break;
+                case 'error':
+                    this.emit('error', {
+                        code: msg.code ?? 'SERVER_ERROR',
+                        message: msg.message ?? 'Unknown error',
+                        recoverable: false
+                    });
+                    break;
+            }
+        }
+        catch (err) {
+            console.error('[web-client] Error handling payload:', err);
+        }
+    }
+    handleWebSocketClose() {
+        console.log('[web-client] WebSocket closed');
+        if (this.stateMachine.state === ConnectionState.CONNECTED) {
+            this.scheduleReconnect();
+        }
+    }
+    handleConnectionError(error) {
+        console.error('[web-client] Connection error:', error.message);
+        if (this.stateMachine.reconnectionEnabled) {
+            this.scheduleReconnect();
+        }
+        else {
+            this.stateMachine.transition(ConnectionState.FAILED);
+            this.emit('error', {
+                code: 'CONNECTION_ERROR',
+                message: error.message,
+                recoverable: false
+            });
+        }
+    }
+    scheduleReconnect() {
+        if (!this.stateMachine.reconnectionEnabled) {
+            this.stateMachine.transition(ConnectionState.FAILED);
+            return;
+        }
+        // Clear any existing reconnect timeout
+        if (this.reconnectTimeout) {
+            clearTimeout(this.reconnectTimeout);
+        }
+        this.stateMachine.transition(ConnectionState.RECONNECTING);
+        const delay = this.stateMachine.getNextRetryDelay();
+        if (delay === null) {
+            // Max retries exceeded
+            this.stateMachine.transition(ConnectionState.FAILED);
+            this.emit('error', {
+                code: 'RECONNECTION_FAILED',
+                message: 'Maximum reconnection attempts exceeded',
+                recoverable: false
+            });
+            return;
+        }
+        console.log(`[web-client] Reconnecting in ${delay}ms (attempt ${this.stateMachine.retryCount}/${this.stateMachine.maxRetries})`);
+        this.emit('reconnecting', this.stateMachine.retryCount, this.stateMachine.maxRetries);
+        this.reconnectTimeout = setTimeout(() => this.attemptReconnect(), delay);
+    }
+    async attemptReconnect() {
+        // Save old session ID BEFORE cleanup/connect (connect() will set a new sessionId)
+        const oldSessionId = this.sessionId;
+        // Clean up existing connections but keep session ID
+        this.cleanup(false);
+        this.stateMachine.transition(ConnectionState.CONNECTING);
+        try {
+            await this.connect();
+            // Send reconnect message with OLD session ID to recover the previous session
+            // Note: connect() sets this.sessionId to a new value from the server's ready message,
+            // so we must use oldSessionId here to request session recovery
+            if (oldSessionId) {
+                console.log('[web-client] Sending reconnect request for session:', oldSessionId);
+                this.ws?.send(JSON.stringify({
+                    type: 'reconnect',
+                    sessionId: oldSessionId
+                }));
+            }
+            // Re-add audio track if we had one
+            if (this.audioStream && this.audioTrack) {
+                console.log('[web-client] Re-adding audio track after reconnect');
+                this.audioSender = this.peer?.addTrack(this.audioTrack, this.audioStream);
+            }
+            this.stateMachine.transition(ConnectionState.CONNECTED);
+        }
+        catch (error) {
+            console.error('[web-client] Reconnect attempt failed:', error);
+            this.scheduleReconnect();
+        }
+    }
+    startHeartbeat() {
+        this.stopHeartbeat();
+        this.heartbeatInterval = setInterval(() => {
+            if (this.ws?.readyState === WebSocket.OPEN) {
+                this.ws.send(JSON.stringify({ type: 'ping', timestamp: Date.now() }));
+                this.heartbeatTimeout = setTimeout(() => {
+                    this.missedHeartbeats++;
+                    console.warn(`[web-client] Missed heartbeat (${this.missedHeartbeats}/${MAX_MISSED_HEARTBEATS})`);
+                    if (this.missedHeartbeats >= MAX_MISSED_HEARTBEATS) {
+                        console.warn('[web-client] Too many missed heartbeats, reconnecting');
+                        this.scheduleReconnect();
+                    }
+                }, HEARTBEAT_TIMEOUT_MS);
+            }
+        }, HEARTBEAT_INTERVAL_MS);
+    }
+    stopHeartbeat() {
+        if (this.heartbeatInterval) {
+            clearInterval(this.heartbeatInterval);
+            this.heartbeatInterval = undefined;
+        }
+        if (this.heartbeatTimeout) {
+            clearTimeout(this.heartbeatTimeout);
+            this.heartbeatTimeout = undefined;
+        }
+        this.missedHeartbeats = 0;
+    }
+    /**
+     * Share audio with the server.
+     * Speech detection is handled server-side using Silero VAD.
+     */
+    async shareAudio(stream) {
+        if (!this.peer?.connected) {
+            throw new Error('Peer not connected');
+        }
+        this.audioStream = stream;
+        this.audioTrack = stream.getAudioTracks()[0];
+        if (!this.audioTrack) {
+            throw new Error('No audio track in stream');
+        }
+        console.log('[web-client] Adding audio track to peer connection');
+        this.audioSender = this.peer.addTrack(this.audioTrack, stream);
+        // Wait for signaling to stabilize
+        await this.waitForStableSignaling();
+        console.log('[web-client] Audio track added - VAD handled server-side');
+        return {
+            stop: async () => {
+                console.log('[web-client] Stopping audio sharing');
+                if (this.audioSender && this.peer) {
+                    this.peer.removeTrack(this.audioSender);
+                    this.audioSender = undefined;
+                }
+                this.audioTrack?.stop();
+                stream.getTracks().forEach((t) => t.stop());
+                this.audioTrack = undefined;
+                this.audioStream = undefined;
+            }
+        };
+    }
+    waitForStableSignaling() {
+        return new Promise((resolve) => {
+            const checkState = () => {
+                if (this.peer?.signalingState === 'stable') {
+                    resolve();
+                }
+                else {
+                    setTimeout(checkState, 100);
+                }
+            };
+            setTimeout(checkState, 100);
+        });
+    }
+    /**
+     * Send vision attachments to include with next speech segment.
+     */
+    sendAttachments() {
+        const attachments = this.gatherAttachments();
+        if (attachments.length > 0 && this.peer?.connected) {
+            this.peer.send(JSON.stringify({ type: 'attachments', attachments }));
+        }
+    }
+    gatherAttachments() {
+        const attachments = [];
+        const cam = this.videoCapture?.getLastFrame();
+        const screen = this.screenCapture?.getLastFrame();
+        if (cam)
+            attachments.push({ data: cam, mimeType: 'image/jpeg', alt: 'camera frame' });
+        if (screen)
+            attachments.push({ data: screen, mimeType: 'image/jpeg', alt: 'screen frame' });
+        return attachments;
+    }
+    shareVideo(stream, intervalMs = 1000) {
+        this.videoCapture?.stop();
+        const ctrl = startFrameCapture(stream, intervalMs);
+        this.videoCapture = ctrl;
+        return ctrl;
+    }
+    shareScreen(stream, intervalMs = 1200) {
+        this.screenCapture?.stop();
+        const ctrl = startFrameCapture(stream, intervalMs);
+        this.screenCapture = ctrl;
+        return ctrl;
+    }
+    /**
+     * Close the connection.
+     */
+    close() {
+        this.stateMachine.transition(ConnectionState.CLOSED);
+        this.cleanup(true);
+    }
+    cleanup(fullCleanup) {
+        this.stopHeartbeat();
+        if (this.reconnectTimeout) {
+            clearTimeout(this.reconnectTimeout);
+            this.reconnectTimeout = undefined;
+        }
+        this.videoCapture?.stop();
+        this.screenCapture?.stop();
+        this.peer?.destroy();
+        this.peer = null;
+        if (this.ws) {
+            this.ws.onclose = null; // Prevent triggering reconnect
+            this.ws.close();
+            this.ws = null;
+        }
+        if (fullCleanup) {
+            this.sessionId = null;
+            this.serverIceServers = null;
+            this.audioTrack = undefined;
+            this.audioStream = undefined;
+            this.audioSender = undefined;
+            this.stateMachine.reset();
+        }
+    }
+}
+// Helper functions
+function base64ToArrayBuffer(base64) {
+    const binary = atob(base64);
+    const bytes = new Uint8Array(binary.length);
+    for (let i = 0; i < binary.length; i++) {
+        bytes[i] = binary.charCodeAt(i);
+    }
+    return bytes.buffer;
+}
+export async function recordOnce(durationMs = 4000) {
+    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    const recorder = new MediaRecorder(stream, { mimeType: 'audio/webm' });
+    const chunks = [];
+    return new Promise((resolve) => {
+        recorder.ondataavailable = (e) => {
+            if (e.data.size > 0)
+                chunks.push(e.data);
+        };
+        recorder.onstop = async () => {
+            const blob = new Blob(chunks, { type: 'audio/webm' });
+            const buf = await blob.arrayBuffer();
+            stream.getTracks().forEach((t) => t.stop());
+            resolve(buf);
+        };
+        recorder.start();
+        setTimeout(() => recorder.stop(), durationMs);
+    });
+}
+export async function captureScreenFrame() {
+    const screen = await navigator.mediaDevices.getDisplayMedia({ video: true });
+    const track = screen.getVideoTracks()[0];
+    const image = await grabFrame(track);
+    track.stop();
+    return image;
+}
+async function grabFrame(track) {
+    const capture = new window.ImageCapture(track);
+    const bitmap = await capture.grabFrame();
+    const canvas = document.createElement('canvas');
+    canvas.width = bitmap.width;
+    canvas.height = bitmap.height;
+    const ctx = canvas.getContext('2d');
+    if (!ctx)
+        throw new Error('canvas context missing');
+    ctx.drawImage(bitmap, 0, 0);
+    const dataUrl = canvas.toDataURL('image/jpeg', 0.6);
+    return dataUrl;
+}
+function startFrameCapture(stream, intervalMs) {
+    const track = stream.getVideoTracks()[0];
+    let stopped = false;
+    const canvas = document.createElement('canvas');
+    const ctx = canvas.getContext('2d');
+    let lastFrame = null;
+    const tick = async () => {
+        if (stopped)
+            return;
+        try {
+            const capture = new window.ImageCapture(track);
+            const bitmap = await capture.grabFrame();
+            canvas.width = bitmap.width;
+            canvas.height = bitmap.height;
+            ctx?.drawImage(bitmap, 0, 0);
+            lastFrame = canvas.toDataURL('image/jpeg', 0.6);
+        }
+        catch (err) {
+            // ignore frame errors
+        }
+        timer = window.setTimeout(tick, intervalMs);
+    };
+    let timer = window.setTimeout(tick, intervalMs);
+    return {
+        stop: () => {
+            stopped = true;
+            window.clearTimeout(timer);
+            stream.getTracks().forEach((t) => t.stop());
+        },
+        getLastFrame: () => lastFrame
+    };
+}
+//# sourceMappingURL=index.js.map