@theoremjs/react 0.0.0-stage → 0.3.0

package/client/live-client.js

@@ -0,0 +1,638 @@
+/**
+ * Pure client-side SDK for THEOREM Gemini 3.1 Flash Live sessions over WebSocket relay.
+ *
+ * Handles:
+ * - Bidirectional WebSocket connection to `/api/live/relay`.
+ * - 16-bit 16kHz PCM microphone audio recording & streaming.
+ * - Gapless 24kHz PCM / WAV model voice playback scheduling.
+ * - Barge-in interruption cancellation (instant audio queue flush).
+ * - Quiet-mic gate while model audio plays (reduces false barge-in from speaker bleed).
+ * - UI tool execution and response routing.
+ *
+ * @module
+ */
+import { describeError, TheoremError, } from '@theoremjs/agents';
+import { base64ToBytes, bytesToBase64 } from '@theoremjs/agents/kernel';
+import { float32Rms, float32RmsToLevel, timeDomainBytesToLevel } from "./audio-level.js";
+import { isPermissionDeniedError } from "./live-errors.js";
+import { applyLiveToolTurnEvent, liveTranscriptFromEvidence, shouldForwardMicFrame, } from "./live/live-mic-forward.js";
+import { parseLiveServerEnvelope, } from "./live-messages.js";
+import { MIC_CAPTURE_PROCESSOR, micCaptureWorkletUrl } from "./mic-capture.js";
+import { downsampleAndConvertToInt16, pcm16BytesToFloat32 } from "./pcm-downsample.js";
+import { hostError } from "./transport.js";
+/** One message to the relay, in the shape a relay checks with `parseLiveClientMessage`. */
+function clientMessage(message) {
+    return JSON.stringify(message);
+}
+function emptyInboundTurnAccum() {
+    return {
+        toolCalls: [],
+        mediaChunks: [],
+        cancelledToolIds: new Set(),
+    };
+}
+/**
+ * While the model is playing audio, mic frames below this RMS are not forwarded.
+ * Speaker bleed / keyboard / room noise sit under this; intentional barge-in speech
+ * sits above it. Gemini VAD only offers LOW|HIGH start sensitivity — this is the
+ * fine-grained "less choppy barge-in" knob.
+ */
+const BARGE_IN_RMS_WHILE_SPEAKING = 0.05;
+/** A caught value as an `Error`; one that is not keeps its text as `internal` detail. */
+function asError(err) {
+    return err instanceof Error ? err : new TheoremError('internal', describeError(err), { cause: err });
+}
+function safeDisconnect(node) {
+    if (!node || typeof node.disconnect !== 'function')
+        return;
+    try {
+        node.disconnect();
+    }
+    catch {
+        /* ignore */
+    }
+}
+function stopMediaTracks(stream) {
+    if (!stream || typeof stream.getTracks !== 'function')
+        return;
+    for (const track of stream.getTracks()) {
+        track.stop();
+    }
+}
+function checkMicFrameForward(args) {
+    const rms = float32Rms(args.inputFloat32);
+    const modelPlaying = args.hasPlaybackNodes || args.status === 'speaking';
+    return shouldForwardMicFrame({
+        isMuted: args.isMuted,
+        socketOpen: args.wsOpen,
+        modelPlaying,
+        rms,
+        bargeInRmsWhileSpeaking: BARGE_IN_RMS_WHILE_SPEAKING,
+    });
+}
+function resolveWorkingStatusTransition(currentStatus, serverWorking) {
+    if (currentStatus === 'listening' && serverWorking)
+        return 'working';
+    if (currentStatus === 'working' && !serverWorking)
+        return 'listening';
+    return null;
+}
+export class LiveSessionClient {
+    ws = null;
+    audioContext = null;
+    micStream = null;
+    micSource = null;
+    micWorklet = null;
+    micWorkletModuleLoaded = false;
+    playbackNodes = [];
+    playbackBus = null;
+    playbackAnalyser = null;
+    playbackMeterFrame = 0;
+    playbackMeterBuffer = null;
+    nextPlaybackTime = 0;
+    status = 'disconnected';
+    /** Server reported background work (reasoning / async tools) still in flight. */
+    serverWorking = false;
+    micActivating = false;
+    connectTimeout = null;
+    isMuted = false;
+    options;
+    /** Serialize control-plane handling (tools / status) without waiting on audio decode. */
+    inboundChain = Promise.resolve();
+    /** Ordered model-audio playback queue (separate so tools are not stuck behind decode). */
+    audioChain = Promise.resolve();
+    /**
+     * Tool calls, one at a time in the order the model made them. Apart from
+     * `inboundChain`: a call waits on the relay's reply, which arrives inbound.
+     */
+    toolChain = Promise.resolve();
+    /** Bumped on barge-in / cancel so stale audioChain work is skipped. */
+    audioEpoch = 0;
+    pendingExecuteResults = new Map();
+    constructor(options) {
+        this.options = options;
+    }
+    setStatus(newStatus) {
+        this.status = newStatus;
+        if (newStatus === 'listening' || newStatus === 'disconnected' || newStatus === 'error') {
+            this.clearConnectTimeout();
+        }
+        this.options.onStatusChange?.(newStatus);
+    }
+    setConnectPhase(phase) {
+        this.options.onConnectPhase?.(phase);
+    }
+    clearConnectTimeout() {
+        if (this.connectTimeout) {
+            clearTimeout(this.connectTimeout);
+            this.connectTimeout = null;
+        }
+    }
+    detachWebSocket() {
+        for (const [callId, pending] of this.pendingExecuteResults) {
+            // lexicon-exempt: internal diagnostic; the user reads error.network
+            pending.reject(new TheoremError('network', `live session closed before call ${callId} settled`));
+        }
+        this.pendingExecuteResults.clear();
+        if (!this.ws)
+            return;
+        this.ws.onopen = null;
+        this.ws.onmessage = null;
+        this.ws.onclose = null;
+        this.ws.onerror = null;
+        try {
+            this.ws.close();
+        }
+        catch {
+            /* ignore */
+        }
+        this.ws = null;
+    }
+    teardownConnection() {
+        this.clearConnectTimeout();
+        this.micActivating = false;
+        this.setConnectPhase(null);
+        this.detachWebSocket();
+        this.cleanupAudio();
+    }
+    /** End the session on `error`: a failed connect or a lost socket. */
+    failSession(error) {
+        if (this.status === 'error' || this.status === 'disconnected')
+            return;
+        this.teardownConnection();
+        this.options.onError?.(error);
+        this.setStatus('error');
+    }
+    createSocket(url) {
+        return this.options.createSocket?.() ?? new WebSocket(url);
+    }
+    // fallow-ignore-next-line unused-class-member -- called from useLiveRunnerControls via client refs
+    async connect() {
+        this.teardownConnection();
+        this.setStatus('connecting');
+        this.setConnectPhase('socket');
+        try {
+            const AudioContextClass = globalThis.AudioContext;
+            this.audioContext = new AudioContextClass();
+            if (this.audioContext.state === 'suspended') {
+                await this.audioContext.resume();
+            }
+            const protocol = globalThis.location.protocol === 'https:' ? 'wss:' : 'ws:';
+            const profileParam = this.options.profile
+                ? `?profile=${encodeURIComponent(this.options.profile)}`
+                : '';
+            const defaultUrl = `${protocol}//${globalThis.location.host}/api/live/relay${profileParam}`;
+            const url = this.options.relayUrl || defaultUrl;
+            this.ws = this.createSocket(url);
+            this.ws.binaryType = 'arraybuffer';
+            const { openMessage } = this.options;
+            if (openMessage) {
+                this.ws.onopen = () => {
+                    this.ws?.send(JSON.stringify(openMessage));
+                };
+            }
+            this.connectTimeout = setTimeout(() => {
+                if (this.status === 'connecting') {
+                    // lexicon-exempt: internal diagnostic; the user reads error.timeout
+                    this.failSession(new TheoremError('timeout', 'live connect timed out'));
+                }
+            }, 20_000);
+            this.ws.onmessage = (event) => {
+                this.enqueueServerMessage(event.data);
+            };
+            this.ws.onclose = () => {
+                if (this.status === 'connecting') {
+                    // lexicon-exempt: internal diagnostic; the user reads error.network
+                    this.failSession(new TheoremError('network', 'live socket closed before ready'));
+                    return;
+                }
+                this.teardownConnection();
+                this.setStatus('disconnected');
+            };
+            this.ws.onerror = () => {
+                // lexicon-exempt: internal diagnostic; the user reads error.network
+                this.failSession(new TheoremError('network', 'live socket error'));
+            };
+        }
+        catch (err) {
+            this.failSession(new TheoremError('internal', describeError(err), { cause: err }));
+        }
+    }
+    async ensureAudioContext() {
+        if (!this.audioContext || this.audioContext.state === 'closed') {
+            const AudioContextClass = globalThis.AudioContext;
+            this.audioContext = new AudioContextClass();
+            if (this.audioContext.state === 'suspended') {
+                await this.audioContext.resume();
+            }
+        }
+    }
+    async activateMicrophone() {
+        if (this.micActivating || this.status !== 'connecting')
+            return;
+        this.micActivating = true;
+        this.setConnectPhase('microphone');
+        try {
+            await this.ensureAudioContext();
+            this.micStream = await navigator.mediaDevices.getUserMedia({
+                audio: {
+                    channelCount: 1,
+                    echoCancellation: true,
+                    noiseSuppression: true,
+                    autoGainControl: true,
+                },
+            });
+            await this.setupMicrophonePipeline();
+            this.setConnectPhase(null);
+            this.setStatus('listening');
+        }
+        catch (err) {
+            const denied = isPermissionDeniedError(err);
+            this.failSession(new TheoremError(denied ? 'auth' : 'internal', describeError(err), {
+                cause: err,
+                copy: { key: denied ? 'voice.permission' : 'voice.unavailable' },
+            }));
+        }
+        finally {
+            this.micActivating = false;
+        }
+    }
+    sendMicFrame(inputFloat32, sampleRate) {
+        const pcm16 = downsampleAndConvertToInt16(inputFloat32, sampleRate, 16000);
+        const base64 = bytesToBase64(new Uint8Array(pcm16.buffer));
+        const socket = this.ws;
+        if (socket?.readyState === WebSocket.OPEN) {
+            socket.send(clientMessage({ type: 'audio', data: base64 }));
+        }
+    }
+    reportMicVolume(inputFloat32) {
+        if (this.isMuted)
+            return;
+        this.options.onVolumeLevel?.(float32RmsToLevel(inputFloat32), true);
+    }
+    forwardMicBuffer(inputFloat32) {
+        this.reportMicVolume(inputFloat32);
+        const forward = checkMicFrameForward({
+            isMuted: this.isMuted,
+            wsOpen: this.ws?.readyState === WebSocket.OPEN,
+            hasPlaybackNodes: this.playbackNodes.length > 0,
+            status: this.status,
+            inputFloat32,
+        });
+        if (forward) {
+            const sampleRate = this.audioContext ? this.audioContext.sampleRate : 48000;
+            this.sendMicFrame(inputFloat32, sampleRate);
+        }
+    }
+    async setupMicrophonePipeline() {
+        if (!this.micStream || !this.audioContext)
+            return;
+        this.micSource = this.audioContext.createMediaStreamSource(this.micStream);
+        const silent = this.audioContext.createGain();
+        silent.gain.value = 0;
+        if (!this.micWorkletModuleLoaded) {
+            await this.audioContext.audioWorklet.addModule(micCaptureWorkletUrl());
+            this.micWorkletModuleLoaded = true;
+        }
+        this.micWorklet = new AudioWorkletNode(this.audioContext, MIC_CAPTURE_PROCESSOR);
+        this.micWorklet.port.onmessage = (event) => {
+            const inputFloat32 = new Float32Array(event.data);
+            this.forwardMicBuffer(inputFloat32);
+        };
+        this.micSource.connect(this.micWorklet);
+        this.micWorklet.connect(silent);
+        silent.connect(this.audioContext.destination);
+    }
+    ensurePlaybackBus() {
+        if (!this.audioContext || this.playbackBus)
+            return;
+        this.playbackBus = this.audioContext.createGain();
+        this.playbackAnalyser = this.audioContext.createAnalyser();
+        this.playbackAnalyser.fftSize = 512;
+        this.playbackAnalyser.smoothingTimeConstant = 0.35;
+        this.playbackBus.connect(this.playbackAnalyser);
+        this.playbackAnalyser.connect(this.audioContext.destination);
+        this.playbackMeterBuffer = new Uint8Array(this.playbackAnalyser.fftSize);
+    }
+    stopPlaybackMeter() {
+        if (this.playbackMeterFrame) {
+            cancelAnimationFrame(this.playbackMeterFrame);
+            this.playbackMeterFrame = 0;
+        }
+        this.options.onVolumeLevel?.(0, false);
+    }
+    startPlaybackMeter() {
+        if (this.playbackMeterFrame)
+            return;
+        const tick = () => {
+            if (this.playbackNodes.length === 0 || !this.playbackAnalyser || !this.playbackMeterBuffer) {
+                this.stopPlaybackMeter();
+                return;
+            }
+            this.playbackAnalyser.getByteTimeDomainData(this.playbackMeterBuffer);
+            this.options.onVolumeLevel?.(timeDomainBytesToLevel(this.playbackMeterBuffer), false);
+            this.playbackMeterFrame = requestAnimationFrame(tick);
+        };
+        this.playbackMeterFrame = requestAnimationFrame(tick);
+    }
+    async processServerEnvelope(payload) {
+        if (await this.tryHandleControlEnvelope(payload))
+            return;
+        if (payload.type !== 'events')
+            return;
+        const accum = this.collectInboundTurn(payload.events);
+        const runnableTools = accum.toolCalls.filter((call) => !accum.cancelledToolIds.has(call.id));
+        for (const call of runnableTools) {
+            this.toolChain = this.toolChain
+                .then(() => this.options.onToolCall(call.name, call.arguments, { callId: call.id,
+            }))
+                .catch((err) => {
+                this.options.onError?.(asError(err));
+            });
+        }
+        this.scheduleMediaChunks(accum.mediaChunks);
+    }
+    enqueueServerMessage(data) {
+        this.inboundChain = this.inboundChain
+            .then(async () => {
+            if (typeof data !== 'string')
+                return;
+            const payload = parseLiveServerEnvelope(data);
+            // An envelope the client can't use reaches onTurnEvent like any event; the call goes on.
+            if (payload.type === 'unsupported' || payload.type === 'malformed') {
+                this.options.onTurnEvent?.(payload);
+            }
+            else
+                await this.processServerEnvelope(payload);
+        })
+            .catch((err) => {
+            this.options.onError?.(asError(err));
+        });
+    }
+    async handleReadyEnvelope(payload) {
+        this.options.onSessionReady?.({
+            sessionId: payload.sessionId,
+            profile: payload.profile,
+        });
+        if (this.options.voiceIngress === false) {
+            this.setConnectPhase(null);
+            this.setStatus('listening');
+        }
+        else {
+            await this.activateMicrophone();
+        }
+    }
+    handleExecuteToolResultEnvelope(payload) {
+        const pending = this.pendingExecuteResults.get(payload.callId);
+        if (!pending)
+            return;
+        this.pendingExecuteResults.delete(payload.callId);
+        if (payload.status === 'refused') {
+            pending.reject(hostError(payload.body, 'request'));
+        }
+        else if (payload.status === 'gated') {
+            pending.resolve({ status: 'gated', gate: payload.gate });
+        }
+        else
+            pending.resolve({ status: 'settled' });
+    }
+    async tryHandleControlEnvelope(payload) {
+        if (payload.type === 'ready') {
+            await this.handleReadyEnvelope(payload);
+            return true;
+        }
+        if (payload.type === 'executeToolResult') {
+            this.handleExecuteToolResultEnvelope(payload);
+            return true;
+        }
+        if (payload.type === 'trace') {
+            this.options.onTrace?.(payload.record);
+            return true;
+        }
+        if (payload.type === 'error') {
+            // The relay's kind and wording when it sent them, else `unavailable`.
+            this.options.onError?.(hostError(payload, 'unavailable'));
+            this.setStatus('error');
+            return true;
+        }
+        return false;
+    }
+    collectInboundTurn(events) {
+        const accum = emptyInboundTurnAccum();
+        for (const event of events) {
+            this.processTurnEvent(event, accum);
+        }
+        return accum;
+    }
+    processTurnEvent(event, accum) {
+        this.options.onTurnEvent?.(event);
+        switch (event.type) {
+            case 'evidence':
+                this.handleEvidenceTurnEvent(event);
+                break;
+            case 'session':
+                this.handleSessionTurnEvent(event);
+                break;
+            case 'media':
+                this.collectMediaTurnEvent(event, accum);
+                break;
+            case 'tool':
+                applyLiveToolTurnEvent(event.tool, accum);
+                break;
+            case 'done':
+                this.handleDoneTurnEvent(event);
+                break;
+        }
+    }
+    handleEvidenceTurnEvent(event) {
+        const transcript = liveTranscriptFromEvidence(event);
+        if (transcript) {
+            this.options.onTranscript?.(transcript.text, transcript.isUser, { interim: transcript.interim,
+            });
+        }
+    }
+    handleSessionTurnEvent(event) {
+        if (this.notifySessionLifecycle(event.session))
+            return;
+        this.serverWorking = event.session.kind === 'working';
+        const nextStatus = resolveWorkingStatusTransition(this.status, this.serverWorking);
+        if (nextStatus)
+            this.setStatus(nextStatus);
+    }
+    /** Hand the session's closing and end to the host; `true` when it was one of them. */
+    notifySessionLifecycle(session) {
+        if (session.kind === 'closing_soon') {
+            this.options.onSessionClosing?.(session.timeLeftMs);
+            return true;
+        }
+        if (session.kind !== 'ended')
+            return false;
+        this.options.onSessionEnded?.(session);
+        return true;
+    }
+    /** Status to settle into once model speech stops. */
+    restingStatus() {
+        return this.serverWorking ? 'working' : 'listening';
+    }
+    collectMediaTurnEvent(event, accum) {
+        if (!event.media.data)
+            return;
+        this.setStatus('speaking');
+        accum.mediaChunks.push({
+            data: event.media.data,
+            mimeType: event.media.mimeType,
+        });
+    }
+    handleDoneTurnEvent(event) {
+        if (event.interrupted) {
+            this.cancelPlayback();
+        }
+        if (event.stop.kind !== 'generation_complete') {
+            this.serverWorking = false;
+            this.setStatus('listening');
+        }
+    }
+    scheduleMediaChunks(mediaChunks) {
+        if (mediaChunks.length === 0)
+            return;
+        const epoch = this.audioEpoch;
+        for (const chunk of mediaChunks) {
+            this.audioChain = this.audioChain
+                .then(async () => {
+                if (epoch !== this.audioEpoch)
+                    return;
+                await this.enqueueAudioChunk(chunk.data, chunk.mimeType);
+            })
+                .catch((err) => {
+                this.options.onError?.(asError(err));
+            });
+        }
+    }
+    /** The relay runs `LiveSession.executeTool`; see `ExecuteToolOnRelay`. */
+    executeToolOnRelay(args) {
+        if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+            throw new TheoremError('request', 'live session is not connected'); // lexicon-exempt: internal diagnostic
+        }
+        const resultPromise = new Promise((resolve, reject) => {
+            this.pendingExecuteResults.set(args.callId, { resolve, reject });
+        });
+        this.ws.send(clientMessage({ type: 'executeTool', ...args }));
+        return resultPromise;
+    }
+    async enqueueAudioChunk(base64Data, mimeType) {
+        if (!this.audioContext)
+            return;
+        try {
+            this.ensurePlaybackBus();
+            if (!this.playbackBus)
+                return;
+            const bytes = base64ToBytes(base64Data);
+            let audioBuffer;
+            if (mimeType?.includes('wav')) {
+                const wavBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
+                audioBuffer = await this.audioContext.decodeAudioData(wavBuffer);
+            }
+            else {
+                // Raw PCM 24kHz 1-channel 16-bit LE
+                const float32 = pcm16BytesToFloat32(bytes);
+                audioBuffer = this.audioContext.createBuffer(1, float32.length, 24000);
+                audioBuffer.getChannelData(0).set(float32);
+            }
+            const source = this.audioContext.createBufferSource();
+            source.buffer = audioBuffer;
+            source.connect(this.playbackBus);
+            const now = this.audioContext.currentTime;
+            const startTime = Math.max(now, this.nextPlaybackTime);
+            source.start(startTime);
+            this.nextPlaybackTime = startTime + audioBuffer.duration;
+            this.playbackNodes.push(source);
+            this.startPlaybackMeter();
+            source.onended = () => {
+                const idx = this.playbackNodes.indexOf(source);
+                if (idx !== -1)
+                    this.playbackNodes.splice(idx, 1);
+                if (this.playbackNodes.length === 0) {
+                    this.stopPlaybackMeter();
+                    if (this.status === 'speaking') {
+                        this.setStatus(this.restingStatus());
+                    }
+                }
+            };
+        }
+        catch (err) {
+            this.options.onError?.(asError(err));
+        }
+    }
+    cancelPlayback() {
+        this.audioEpoch += 1;
+        this.audioChain = Promise.resolve();
+        for (const node of this.playbackNodes) {
+            try {
+                node.stop();
+                node.disconnect();
+            }
+            catch {
+                /* ignore */
+            }
+        }
+        this.playbackNodes = [];
+        this.stopPlaybackMeter();
+        if (this.audioContext) {
+            this.nextPlaybackTime = this.audioContext.currentTime;
+        }
+    }
+    // fallow-ignore-next-line unused-class-member -- called from useLiveRunnerControls via client refs
+    toggleMute() {
+        this.isMuted = !this.isMuted;
+        return this.isMuted;
+    }
+    // fallow-ignore-next-line unused-class-member -- called from useLiveRunnerControls via client refs
+    sendText(text) {
+        if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            this.ws.send(clientMessage({ type: 'text', text }));
+        }
+    }
+    // fallow-ignore-next-line unused-class-member -- called by hosts that tell the model where the visitor is
+    sendContext(text) {
+        if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            this.ws.send(clientMessage({ type: 'context', text }));
+        }
+    }
+    // fallow-ignore-next-line unused-class-member -- called from useLiveRunnerControls via client refs
+    sendVideo(data, mimeType = 'image/jpeg') {
+        if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            this.ws.send(clientMessage({ type: 'video', data, mimeType }));
+        }
+    }
+    // fallow-ignore-next-line unused-class-member -- called from useLiveSessionClient teardown
+    disconnect() {
+        this.teardownConnection();
+        this.setStatus('disconnected');
+    }
+    cleanupAudio() {
+        this.cancelPlayback();
+        this.stopPlaybackMeter();
+        safeDisconnect(this.playbackBus);
+        this.playbackBus = null;
+        this.playbackAnalyser = null;
+        this.playbackMeterBuffer = null;
+        if (this.micWorklet) {
+            this.micWorklet.port.onmessage = null;
+            safeDisconnect(this.micWorklet);
+            this.micWorklet = null;
+        }
+        safeDisconnect(this.micSource);
+        this.micSource = null;
+        stopMediaTracks(this.micStream);
+        this.micStream = null;
+        if (this.audioContext && this.audioContext.state !== 'closed') {
+            try {
+                void this.audioContext.close();
+            }
+            catch {
+                /* ignore */
+            }
+            this.audioContext = null;
+        }
+    }
+}

package/client/live-errors.d.ts

@@ -0,0 +1,2 @@
+/** Whether a DOM/media error indicates the user denied microphone access. */
+export declare function isPermissionDeniedError(err: unknown): boolean;

package/client/live-errors.js

@@ -0,0 +1,9 @@
+/** Whether a DOM/media error indicates the user denied microphone access. */
+export function isPermissionDeniedError(err) {
+    if (!(err instanceof Error))
+        return false;
+    const name = 'name' in err && typeof err.name === 'string' ? err.name : '';
+    return (name === 'NotAllowedError' ||
+        name === 'PermissionDeniedError' ||
+        /permission|not allowed|denied/i.test(err.message));
+}