@theoremjs/react 0.0.0-stage → 0.3.1

package/client/live-client.js

@@ -0,0 +1,829 @@
+// why: This is browser code; a Deno program that reaches it through `@theoremjs/react/client` checks it with the DOM.
+/// <reference lib="dom" />
+/**
+ * Pure client-side SDK for THEOREM Gemini 3.1 Flash Live sessions over WebSocket relay.
+ *
+ * Handles:
+ * - Bidirectional WebSocket connection to `/api/live/relay`.
+ * - 16-bit 16kHz PCM microphone audio recording & streaming.
+ * - Gapless 24kHz PCM / WAV model voice playback scheduling.
+ * - Barge-in interruption cancellation (instant audio queue flush).
+ * - Quiet-mic gate while model audio plays (reduces false barge-in from speaker bleed).
+ * - UI tool execution and response routing.
+ *
+ * @module
+ */
+import { describeError, labelText, TheoremError, } from '@theoremjs/agents';
+import { base64ToBytes, bytesToBase64 } from '@theoremjs/agents/kernel';
+import { float32Rms, float32RmsToLevel, timeDomainBytesToLevel } from "./audio-level.js";
+import { applyLiveToolTurnEvent, liveTranscriptFromEvidence, shouldForwardMicFrame, } from "./live/live-mic-forward.js";
+import { isPermissionDeniedError } from "./live-errors.js";
+import { liveClientMessageSchema, parseLiveServerEnvelope, } from "./live-messages.js";
+import { MIC_CAPTURE_PROCESSOR, micCaptureWorkletUrl } from "./mic-capture.js";
+import { downsampleAndConvertToInt16, pcm16BytesToFloat32 } from "./pcm-downsample.js";
+import { hostError } from "./transport.js";
+import { issuedOperationId } from "./turn-client.js";
+/** One message to the relay, in the shape a relay checks with `parseLiveClientMessage`. */
+function clientMessage(message) {
+    return JSON.stringify(message);
+}
+function emptyInboundTurnAccum() {
+    return {
+        toolCalls: [],
+        mediaChunks: [],
+        cancelledToolIds: new Set(),
+    };
+}
+/**
+ * While the model is playing audio, mic frames below this RMS are not forwarded.
+ * Speaker bleed / keyboard / room noise sit under this; intentional barge-in speech
+ * sits above it. Gemini VAD only offers LOW|HIGH start sensitivity — this is the
+ * fine-grained "less choppy barge-in" knob.
+ */
+const BARGE_IN_RMS_WHILE_SPEAKING = 0.05;
+/** How long the client waits before each try at taking a dropped call up again. After the last, the call fails. */
+const LIVE_RECONNECT_DELAYS_MS = [500, 1000, 2000, 4000, 8000];
+/** A caught value as an `Error`; one that is not keeps its text as `internal` detail. */
+function asError(err) {
+    return err instanceof Error
+        ? err
+        : new TheoremError('internal', describeError(err), { cause: err });
+}
+function safeDisconnect(node) {
+    if (!node || typeof node.disconnect !== 'function')
+        return;
+    try {
+        node.disconnect();
+    }
+    catch {
+        // why: disconnect() throws for a node that is not connected; it is disconnected either way.
+    }
+}
+function stopMediaTracks(stream) {
+    if (!stream || typeof stream.getTracks !== 'function')
+        return;
+    for (const track of stream.getTracks()) {
+        track.stop();
+    }
+}
+function checkMicFrameForward(args) {
+    const rms = float32Rms(args.inputFloat32);
+    const modelPlaying = args.hasPlaybackNodes || args.status === 'speaking';
+    return shouldForwardMicFrame({
+        isMuted: args.isMuted,
+        socketOpen: args.wsOpen,
+        modelPlaying,
+        rms,
+        bargeInRmsWhileSpeaking: BARGE_IN_RMS_WHILE_SPEAKING,
+    });
+}
+function resolveWorkingStatusTransition(currentStatus, serverWorking) {
+    if (currentStatus === 'listening' && serverWorking)
+        return 'working';
+    if (currentStatus === 'working' && !serverWorking)
+        return 'listening';
+    return null;
+}
+export class LiveSessionClient {
+    ws = null;
+    audioContext = null;
+    micStream = null;
+    micSource = null;
+    micWorklet = null;
+    micWorkletModuleLoaded = false;
+    playbackNodes = [];
+    playbackBus = null;
+    playbackAnalyser = null;
+    playbackMeterFrame = 0;
+    playbackMeterBuffer = null;
+    nextPlaybackTime = 0;
+    status = 'disconnected';
+    /** Server reported background work (reasoning / async tools) still in flight. */
+    serverWorking = false;
+    micActivating = false;
+    connectTimeout = null;
+    isMuted = false;
+    options;
+    /** Serialize control-plane handling (tools / status) without waiting on audio decode. */
+    inboundChain = Promise.resolve();
+    /** Ordered model-audio playback queue (separate so tools are not stuck behind decode). */
+    audioChain = Promise.resolve();
+    /**
+     * Tool calls, one at a time in the order the model made them. Apart from
+     * `inboundChain`: a call waits on the relay's reply, which arrives inbound.
+     */
+    toolChain = Promise.resolve();
+    /** Bumped on barge-in / cancel so stale audioChain work is skipped. */
+    audioEpoch = 0;
+    pendingExecuteResults = new Map();
+    /** The page's context package, and the JSON of the one the relay last got. */
+    context;
+    sentContext;
+    /** The provider's latest handle for taking this call up again after a drop. */
+    history = [];
+    providerState;
+    /** The session said it ended, or failed: a close after that is the end, not a drop. */
+    sessionOver = false;
+    /** When the call dropped, for the time away a resume reports. */
+    droppedAt;
+    /** When the call that could not reconnect dropped: the next call takes it up again. */
+    lostAt;
+    reconnectTries = 0;
+    reconnectTimer = null;
+    constructor(options) {
+        this.options = options;
+        this.context = options.context;
+    }
+    setStatus(newStatus) {
+        this.status = newStatus;
+        if (newStatus === 'listening' || newStatus === 'disconnected' || newStatus === 'error') {
+            this.clearConnectTimeout();
+        }
+        this.options.onStatusChange?.(newStatus);
+    }
+    /** Still opening: the first connect, or a try at taking a dropped call up again. */
+    get opening() {
+        return this.status === 'connecting' || this.status === 'reconnecting';
+    }
+    setConnectPhase(phase) {
+        this.options.onConnectPhase?.(phase);
+    }
+    clearConnectTimeout() {
+        if (this.connectTimeout) {
+            clearTimeout(this.connectTimeout);
+            this.connectTimeout = null;
+        }
+    }
+    detachWebSocket() {
+        for (const [callId, pending] of this.pendingExecuteResults) {
+            // labels-exempt: internal diagnostic; the user reads error.network
+            pending.reject(new TheoremError('network', `live session closed before call ${callId} settled`));
+        }
+        this.pendingExecuteResults.clear();
+        if (!this.ws)
+            return;
+        this.ws.onopen = null;
+        this.ws.onmessage = null;
+        this.ws.onclose = null;
+        this.ws.onerror = null;
+        try {
+            this.ws.close();
+        }
+        catch {
+            // why: closing a socket that already closed throws; the connection is over either way.
+        }
+        this.ws = null;
+    }
+    clearReconnect() {
+        if (this.reconnectTimer)
+            clearTimeout(this.reconnectTimer);
+        this.reconnectTimer = null;
+        this.reconnectTries = 0;
+        this.droppedAt = undefined;
+    }
+    teardownConnection() {
+        this.clearConnectTimeout();
+        this.micActivating = false;
+        this.setConnectPhase(null);
+        this.detachWebSocket();
+        this.cleanupAudio();
+    }
+    /** End the session on `error`: a failed connect or a lost socket. */
+    failSession(error) {
+        if (this.status === 'error' || this.status === 'disconnected')
+            return;
+        this.clearReconnect();
+        this.teardownConnection();
+        this.options.onError?.(error);
+        this.setStatus('error');
+    }
+    createSocket(url) {
+        return this.options.createSocket?.() ?? new WebSocket(url);
+    }
+    /**
+     * Start a call. After one that dropped and could not reconnect, it takes that call up again,
+     * so the agent neither greets twice nor forgets; after a hang-up it is a new call.
+     */
+    connect() {
+        const lostAt = this.lostAt;
+        this.clearReconnect();
+        this.lostAt = undefined;
+        if (lostAt === undefined) {
+            this.providerState = undefined;
+            this.history = [];
+        }
+        else
+            this.droppedAt = lostAt;
+        return this.open('connecting');
+    }
+    /** A lost call that cannot be taken up again, its handle refused or expired, starts as a new one. */
+    startOver() {
+        if (this.status !== 'connecting' || !this.providerState)
+            return false;
+        this.providerState = undefined;
+        this.droppedAt = undefined;
+        void this.open('connecting');
+        return true;
+    }
+    /** The first message: the call's values, and the handle and time away when it takes a dropped call up again. */
+    openMessage() {
+        const { slots, openMessage: host } = this.options;
+        return {
+            type: 'open',
+            ...(this.history.length ? { history: structuredClone(this.history) } : {}),
+            ...(slots ? { slots } : {}),
+            ...(this.context === undefined ? {} : { context: this.context }),
+            ...(this.providerState
+                ? {
+                    resume: {
+                        providerState: this.providerState,
+                        awayMs: this.droppedAt === undefined ? 0 : Date.now() - this.droppedAt,
+                    },
+                }
+                : {}),
+            ...(host ? { host } : {}),
+        };
+    }
+    /** The socket closed while the call was open: take it up again if the provider gave a handle, else end. */
+    handleDrop() {
+        if (!this.providerState || this.sessionOver) {
+            this.teardownConnection();
+            this.setStatus('disconnected');
+            return;
+        }
+        this.droppedAt ??= Date.now();
+        this.teardownConnection();
+        this.setStatus('reconnecting');
+        this.scheduleReconnect();
+    }
+    scheduleReconnect() {
+        const delay = LIVE_RECONNECT_DELAYS_MS[this.reconnectTries];
+        if (delay === undefined) {
+            const droppedAt = this.droppedAt;
+            // labels-exempt: internal diagnostic; the user reads error.network
+            this.failSession(new TheoremError('network', 'live call dropped and could not reconnect'));
+            this.lostAt = droppedAt;
+            return;
+        }
+        this.reconnectTries += 1;
+        this.reconnectTimer = setTimeout(() => {
+            this.reconnectTimer = null;
+            void this.open('reconnecting');
+        }, delay);
+    }
+    /** A try at opening failed: a reconnect waits and tries again, a first connect fails the call. */
+    failOpen(error) {
+        if (this.status !== 'reconnecting') {
+            if (!this.startOver())
+                this.failSession(error);
+            return;
+        }
+        this.teardownConnection();
+        this.scheduleReconnect();
+    }
+    async open(as) {
+        this.teardownConnection();
+        this.sessionOver = false;
+        this.setStatus(as);
+        if (as === 'connecting')
+            this.setConnectPhase('socket');
+        try {
+            const AudioContextClass = globalThis.AudioContext;
+            this.audioContext = new AudioContextClass();
+            if (this.audioContext.state === 'suspended') {
+                await this.audioContext.resume();
+            }
+            const protocol = globalThis.location.protocol === 'https:' ? 'wss:' : 'ws:';
+            const profileParam = this.options.profile
+                ? `?profile=${encodeURIComponent(this.options.profile)}`
+                : '';
+            const defaultUrl = `${protocol}//${globalThis.location.host}/api/live/relay${profileParam}`;
+            const url = this.options.relayUrl || defaultUrl;
+            this.ws = this.createSocket(url);
+            this.ws.binaryType = 'arraybuffer';
+            this.ws.onopen = () => {
+                const open = this.openMessage();
+                this.sentContext = JSON.stringify(open.context);
+                this.ws?.send(JSON.stringify(open));
+            };
+            this.connectTimeout = setTimeout(() => {
+                if (this.opening) {
+                    // labels-exempt: internal diagnostic; the user reads error.timeout
+                    this.failOpen(new TheoremError('timeout', 'live connect timed out'));
+                }
+            }, 20_000);
+            this.ws.onmessage = (event) => {
+                this.enqueueServerMessage(event.data);
+            };
+            this.ws.onclose = () => {
+                if (this.opening) {
+                    // labels-exempt: internal diagnostic; the user reads error.network
+                    this.failOpen(new TheoremError('network', 'live socket closed before ready'));
+                    return;
+                }
+                this.handleDrop();
+            };
+            this.ws.onerror = () => {
+                if (!this.opening && this.providerState && !this.sessionOver) {
+                    this.handleDrop();
+                    return;
+                }
+                // labels-exempt: internal diagnostic; the user reads error.network
+                this.failOpen(new TheoremError('network', 'live socket error'));
+            };
+        }
+        catch (err) {
+            this.failOpen(new TheoremError('internal', describeError(err), { cause: err }));
+        }
+    }
+    async ensureAudioContext() {
+        if (!this.audioContext || this.audioContext.state === 'closed') {
+            const AudioContextClass = globalThis.AudioContext;
+            this.audioContext = new AudioContextClass();
+            if (this.audioContext.state === 'suspended') {
+                await this.audioContext.resume();
+            }
+        }
+    }
+    async activateMicrophone() {
+        if (this.micActivating || !this.opening)
+            return;
+        this.micActivating = true;
+        if (this.status === 'connecting')
+            this.setConnectPhase('microphone');
+        try {
+            await this.ensureAudioContext();
+            this.micStream = await navigator.mediaDevices.getUserMedia({
+                audio: {
+                    channelCount: 1,
+                    echoCancellation: true,
+                    noiseSuppression: true,
+                    autoGainControl: true,
+                },
+            });
+            await this.setupMicrophonePipeline();
+            this.becomeReady();
+        }
+        catch (err) {
+            const denied = isPermissionDeniedError(err);
+            this.failSession(new TheoremError(denied ? 'auth' : 'internal', describeError(err), {
+                cause: err,
+                copy: { key: denied ? 'voice.permission' : 'voice.unavailable' },
+            }));
+        }
+        finally {
+            this.micActivating = false;
+        }
+    }
+    sendMicFrame(inputFloat32, sampleRate) {
+        const pcm16 = downsampleAndConvertToInt16(inputFloat32, sampleRate, 16000);
+        const base64 = bytesToBase64(new Uint8Array(pcm16.buffer));
+        const socket = this.ws;
+        if (socket?.readyState === WebSocket.OPEN) {
+            socket.send(clientMessage({ type: 'audio', data: base64 }));
+        }
+    }
+    reportMicVolume(inputFloat32) {
+        if (this.isMuted)
+            return;
+        this.options.onVolumeLevel?.(float32RmsToLevel(inputFloat32), true);
+    }
+    forwardMicBuffer(inputFloat32) {
+        this.reportMicVolume(inputFloat32);
+        const forward = checkMicFrameForward({
+            isMuted: this.isMuted,
+            wsOpen: this.ws?.readyState === WebSocket.OPEN,
+            hasPlaybackNodes: this.playbackNodes.length > 0,
+            status: this.status,
+            inputFloat32,
+        });
+        if (forward) {
+            const sampleRate = this.audioContext ? this.audioContext.sampleRate : 48000;
+            this.sendMicFrame(inputFloat32, sampleRate);
+        }
+    }
+    async setupMicrophonePipeline() {
+        if (!this.micStream || !this.audioContext)
+            return;
+        this.micSource = this.audioContext.createMediaStreamSource(this.micStream);
+        const silent = this.audioContext.createGain();
+        silent.gain.value = 0;
+        if (!this.micWorkletModuleLoaded) {
+            await this.audioContext.audioWorklet.addModule(micCaptureWorkletUrl());
+            this.micWorkletModuleLoaded = true;
+        }
+        this.micWorklet = new AudioWorkletNode(this.audioContext, MIC_CAPTURE_PROCESSOR);
+        this.micWorklet.port.onmessage = (event) => {
+            const inputFloat32 = new Float32Array(event.data);
+            this.forwardMicBuffer(inputFloat32);
+        };
+        this.micSource.connect(this.micWorklet);
+        this.micWorklet.connect(silent);
+        silent.connect(this.audioContext.destination);
+    }
+    ensurePlaybackBus() {
+        if (!this.audioContext || this.playbackBus)
+            return;
+        this.playbackBus = this.audioContext.createGain();
+        this.playbackAnalyser = this.audioContext.createAnalyser();
+        this.playbackAnalyser.fftSize = 512;
+        this.playbackAnalyser.smoothingTimeConstant = 0.35;
+        this.playbackBus.connect(this.playbackAnalyser);
+        this.playbackAnalyser.connect(this.audioContext.destination);
+        this.playbackMeterBuffer = new Uint8Array(this.playbackAnalyser.fftSize);
+    }
+    stopPlaybackMeter() {
+        if (this.playbackMeterFrame) {
+            cancelAnimationFrame(this.playbackMeterFrame);
+            this.playbackMeterFrame = 0;
+        }
+        this.options.onVolumeLevel?.(0, false);
+    }
+    startPlaybackMeter() {
+        if (this.playbackMeterFrame)
+            return;
+        const tick = () => {
+            if (this.playbackNodes.length === 0 || !this.playbackAnalyser || !this.playbackMeterBuffer) {
+                this.stopPlaybackMeter();
+                return;
+            }
+            this.playbackAnalyser.getByteTimeDomainData(this.playbackMeterBuffer);
+            this.options.onVolumeLevel?.(timeDomainBytesToLevel(this.playbackMeterBuffer), false);
+            this.playbackMeterFrame = requestAnimationFrame(tick);
+        };
+        this.playbackMeterFrame = requestAnimationFrame(tick);
+    }
+    async processServerEnvelope(payload) {
+        if (await this.tryHandleControlEnvelope(payload))
+            return;
+        if (payload.type !== 'events')
+            return;
+        const accum = this.collectInboundTurn(payload.events);
+        const runnableTools = accum.toolCalls.filter((call) => !accum.cancelledToolIds.has(call.id));
+        for (const call of runnableTools) {
+            this.toolChain = this.toolChain
+                .then(() => this.options.onToolCall(call.name, call.arguments, {
+                callId: call.id,
+                operationId: issuedOperationId(call),
+            }))
+                .catch((err) => {
+                this.options.onError?.(asError(err));
+            });
+        }
+        this.scheduleMediaChunks(accum.mediaChunks);
+    }
+    enqueueServerMessage(data) {
+        this.inboundChain = this.inboundChain
+            .then(async () => {
+            if (typeof data !== 'string')
+                return;
+            const payload = parseLiveServerEnvelope(data);
+            // why: An envelope the client can't use reaches onTurnEvent like any event; the call goes on.
+            if (payload.type === 'unsupported' || payload.type === 'malformed') {
+                this.options.onTurnEvent?.(payload);
+            }
+            else
+                await this.processServerEnvelope(payload);
+        })
+            .catch((err) => {
+            this.options.onError?.(asError(err));
+        });
+    }
+    async handleReadyEnvelope(payload) {
+        this.options.onSessionReady?.({
+            sessionId: payload.sessionId,
+            profile: payload.profile,
+        });
+        if (this.options.voiceIngress === false) {
+            this.becomeReady();
+        }
+        else {
+            await this.activateMicrophone();
+        }
+    }
+    /** The call is open, first or again: the page's context catches up with what changed meanwhile. */
+    becomeReady() {
+        this.clearReconnect();
+        this.setConnectPhase(null);
+        this.setStatus('listening');
+        this.flushContext();
+    }
+    handleExecuteToolResultEnvelope(payload) {
+        const pending = this.pendingExecuteResults.get(payload.operationId);
+        if (!pending || pending.callId !== payload.callId)
+            return;
+        this.pendingExecuteResults.delete(payload.operationId);
+        if (payload.status === 'refused') {
+            pending.reject(hostError(payload.body, 'request'));
+        }
+        else if (payload.status === 'dispatched') {
+            pending.resolve({
+                status: 'dispatched',
+                grantId: payload.grantId,
+                arguments: payload.arguments,
+            });
+        }
+        else if (payload.status === 'gated') {
+            pending.resolve({ status: 'gated', gate: payload.gate });
+        }
+        else
+            pending.resolve({ status: 'settled' });
+    }
+    async tryHandleControlEnvelope(payload) {
+        if (payload.type === 'ready') {
+            await this.handleReadyEnvelope(payload);
+            return true;
+        }
+        if (payload.type === 'executeToolResult') {
+            this.handleExecuteToolResultEnvelope(payload);
+            return true;
+        }
+        if (payload.type === 'trace') {
+            this.options.onTrace?.(payload.record);
+            return true;
+        }
+        if (payload.type === 'error') {
+            if (this.startOver())
+                return true;
+            this.sessionOver = true;
+            this.options.onError?.(hostError(payload, 'unavailable'));
+            this.setStatus('error');
+            return true;
+        }
+        return false;
+    }
+    collectInboundTurn(events) {
+        const accum = emptyInboundTurnAccum();
+        for (const event of events) {
+            this.processTurnEvent(event, accum);
+        }
+        return accum;
+    }
+    processTurnEvent(event, accum) {
+        if (event.type === 'provider_checkpoint' && event.history)
+            this.history = structuredClone(event.history);
+        if ((event.type === 'provider_checkpoint' || event.type === 'done') && event.providerState)
+            this.providerState = event.providerState;
+        if (event.type === 'provider_warning')
+            this.providerState = undefined;
+        this.options.onTurnEvent?.(event);
+        switch (event.type) {
+            case 'evidence':
+                this.handleEvidenceTurnEvent(event);
+                break;
+            case 'session':
+                this.handleSessionTurnEvent(event);
+                break;
+            case 'media':
+                this.collectMediaTurnEvent(event, accum);
+                break;
+            case 'tool':
+                applyLiveToolTurnEvent(event.tool, accum);
+                break;
+            case 'done':
+                this.handleDoneTurnEvent(event);
+                break;
+        }
+    }
+    handleEvidenceTurnEvent(event) {
+        const transcript = liveTranscriptFromEvidence(event);
+        if (transcript) {
+            this.options.onTranscript?.(transcript.text, transcript.isUser, {
+                interim: transcript.interim,
+            });
+        }
+    }
+    handleSessionTurnEvent(event) {
+        if (this.notifySessionLifecycle(event.session))
+            return;
+        this.serverWorking = event.session.kind === 'working';
+        const nextStatus = resolveWorkingStatusTransition(this.status, this.serverWorking);
+        if (nextStatus)
+            this.setStatus(nextStatus);
+    }
+    /** Hand the session's closing and end to the host; `true` when it was one of them. */
+    notifySessionLifecycle(session) {
+        if (session.kind === 'closing_soon') {
+            this.options.onSessionClosing?.(session.timeLeftMs);
+            return true;
+        }
+        if (session.kind !== 'ended')
+            return false;
+        // why: The provider ends a call at its time limit. With a handle the call is taken up
+        // again, so the person is not told it ended.
+        if (this.providerState) {
+            this.handleDrop();
+            return true;
+        }
+        this.sessionOver = true;
+        this.options.onSessionEnded?.(session);
+        return true;
+    }
+    /** Status to settle into once model speech stops. */
+    restingStatus() {
+        return this.serverWorking ? 'working' : 'listening';
+    }
+    collectMediaTurnEvent(event, accum) {
+        if (!event.media.data)
+            return;
+        this.setStatus('speaking');
+        accum.mediaChunks.push({
+            data: event.media.data,
+            mimeType: event.media.mimeType,
+        });
+    }
+    handleDoneTurnEvent(event) {
+        if (event.interrupted) {
+            this.cancelPlayback();
+        }
+        if (event.stop.kind !== 'generation_complete') {
+            this.serverWorking = false;
+            this.setStatus('listening');
+        }
+    }
+    scheduleMediaChunks(mediaChunks) {
+        if (mediaChunks.length === 0)
+            return;
+        const epoch = this.audioEpoch;
+        for (const chunk of mediaChunks) {
+            this.audioChain = this.audioChain
+                .then(async () => {
+                if (epoch !== this.audioEpoch)
+                    return;
+                await this.enqueueAudioChunk(chunk.data, chunk.mimeType);
+            })
+                .catch((err) => {
+                this.options.onError?.(asError(err));
+            });
+        }
+    }
+    /** The relay runs `LiveSession.executeTool`; see `ExecuteToolOnRelay`. */
+    executeToolOnRelay(args) {
+        const socket = this.ws;
+        if (!socket || socket.readyState !== WebSocket.OPEN) {
+            throw new TheoremError('request', 'live session is not connected'); // labels-exempt: internal diagnostic
+        }
+        const checked = liveClientMessageSchema.safeParse({ ...args, type: 'executeTool' });
+        if (!checked.success || checked.data.type !== 'executeTool') {
+            throw new TheoremError('request', labelText('error.request'));
+        }
+        const request = checked.data;
+        if (this.pendingExecuteResults.has(request.operationId)) {
+            throw new TheoremError('request', labelText('session.gate_expired'));
+        }
+        return new Promise((resolve, reject) => {
+            this.pendingExecuteResults.set(request.operationId, {
+                callId: request.callId,
+                resolve,
+                reject,
+            });
+            try {
+                socket.send(clientMessage(request));
+            }
+            catch (error) {
+                this.pendingExecuteResults.delete(request.operationId);
+                reject(asError(error));
+            }
+        });
+    }
+    async enqueueAudioChunk(base64Data, mimeType) {
+        if (!this.audioContext)
+            return;
+        try {
+            this.ensurePlaybackBus();
+            if (!this.playbackBus)
+                return;
+            const bytes = base64ToBytes(base64Data);
+            let audioBuffer;
+            if (mimeType?.includes('wav')) {
+                const wavBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
+                audioBuffer = await this.audioContext.decodeAudioData(wavBuffer);
+            }
+            else {
+                // why: Raw PCM 24kHz 1-channel 16-bit LE
+                const float32 = pcm16BytesToFloat32(bytes);
+                audioBuffer = this.audioContext.createBuffer(1, float32.length, 24000);
+                audioBuffer.getChannelData(0).set(float32);
+            }
+            const source = this.audioContext.createBufferSource();
+            source.buffer = audioBuffer;
+            source.connect(this.playbackBus);
+            const now = this.audioContext.currentTime;
+            const startTime = Math.max(now, this.nextPlaybackTime);
+            source.start(startTime);
+            this.nextPlaybackTime = startTime + audioBuffer.duration;
+            this.playbackNodes.push(source);
+            this.startPlaybackMeter();
+            source.onended = () => {
+                const idx = this.playbackNodes.indexOf(source);
+                if (idx !== -1)
+                    this.playbackNodes.splice(idx, 1);
+                if (this.playbackNodes.length === 0) {
+                    this.stopPlaybackMeter();
+                    if (this.status === 'speaking') {
+                        this.setStatus(this.restingStatus());
+                    }
+                }
+            };
+        }
+        catch (err) {
+            this.options.onError?.(asError(err));
+        }
+    }
+    cancelPlayback() {
+        this.audioEpoch += 1;
+        this.audioChain = Promise.resolve();
+        for (const node of this.playbackNodes) {
+            try {
+                node.stop();
+                node.disconnect();
+            }
+            catch {
+                // why: a node that already stopped throws on stop() and disconnect(); it leaves the list either way.
+            }
+        }
+        this.playbackNodes = [];
+        this.stopPlaybackMeter();
+        if (this.audioContext) {
+            this.nextPlaybackTime = this.audioContext.currentTime;
+        }
+    }
+    toggleMute() {
+        this.isMuted = !this.isMuted;
+        return this.isMuted;
+    }
+    sendText(text) {
+        if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            this.ws.send(clientMessage({ type: 'text', text }));
+        }
+    }
+    /**
+     * The call's slot values. An open call is taken up again with them when the provider gave a
+     * handle; with none, the next call has them.
+     */
+    setSlots(slots) {
+        if (JSON.stringify(slots) === JSON.stringify(this.options.slots))
+            return;
+        this.options = { ...this.options, slots };
+        if (this.providerState && !this.opening && this.ws?.readyState === WebSocket.OPEN) {
+            this.handleDrop();
+        }
+    }
+    /**
+     * Replace the page's context package. In a call it reaches the agent as
+     * background, with no reply; before one, it goes with the call's opening.
+     */
+    setContext(context) {
+        this.context = context;
+        this.flushContext();
+    }
+    flushContext() {
+        if (this.opening || this.ws?.readyState !== WebSocket.OPEN)
+            return;
+        const next = JSON.stringify(this.context);
+        if (next === this.sentContext)
+            return;
+        this.sentContext = next;
+        this.ws.send(clientMessage({ type: 'context', context: this.context }));
+    }
+    sendVideo(data, mimeType = 'image/jpeg') {
+        if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            this.ws.send(clientMessage({ type: 'video', data, mimeType }));
+        }
+    }
+    disconnect() {
+        this.clearReconnect();
+        this.lostAt = undefined;
+        this.providerState = undefined;
+        this.teardownConnection();
+        this.setStatus('disconnected');
+    }
+    cleanupAudio() {
+        this.cancelPlayback();
+        this.stopPlaybackMeter();
+        safeDisconnect(this.playbackBus);
+        this.playbackBus = null;
+        this.playbackAnalyser = null;
+        this.playbackMeterBuffer = null;
+        if (this.micWorklet) {
+            this.micWorklet.port.onmessage = null;
+            safeDisconnect(this.micWorklet);
+            this.micWorklet = null;
+        }
+        safeDisconnect(this.micSource);
+        this.micSource = null;
+        stopMediaTracks(this.micStream);
+        this.micStream = null;
+        if (this.audioContext && this.audioContext.state !== 'closed') {
+            try {
+                void this.audioContext.close();
+            }
+            catch {
+                // why: close() throws on a context that already closed.
+            }
+            this.audioContext = null;
+            // why: A worklet module belongs to its context; the next context loads it again.
+            this.micWorkletModuleLoaded = false;
+        }
+    }
+}