@layercode/js-sdk 2.8.2 → 2.8.3

package/dist/layercode-js-sdk.min.js

@@ -5318,13 +5318,15 @@ registerProcessor('audio_processor', AudioProcessor);
 	   * @returns {Promise<true>}
 	   */
 	  async requestPermission() {
+	    console.log('ensureUserMediaAccess');
 	    try {
-	      console.log('ensureUserMediaAccess');
-	      await navigator.mediaDevices.getUserMedia({
+	      const stream = await navigator.mediaDevices.getUserMedia({
 	        audio: true,
 	      });
+	      // Stop the tracks immediately after getting permission
+	      stream.getTracks().forEach(track => track.stop());
 	    } catch (fallbackError) {
-	      window.alert('You must grant microphone access to use this feature.');
+	      console.error('getUserMedia failed:', fallbackError.name, fallbackError.message);
 	      throw fallbackError;
 	    }
 	    return true;
@@ -5972,6 +5974,7 @@ registerProcessor('audio_processor', AudioProcessor);
 	        this.recorderStarted = false;
 	        this.readySent = false;
 	        this.currentTurnId = null;
+	        this.sentReplayFinishedForDisabledOutput = false;
 	        this.audioBuffer = [];
 	        this.vadConfig = null;
 	        this.activeDeviceId = null;
@@ -6131,11 +6134,14 @@ registerProcessor('audio_processor', AudioProcessor);
 	        this.options.onAgentSpeakingChange(shouldReportSpeaking);
 	    }
 	    _setUserSpeaking(isSpeaking) {
-	        const shouldReportSpeaking = this._shouldCaptureUserAudio() && isSpeaking;
+	        const shouldCapture = this._shouldCaptureUserAudio();
+	        const shouldReportSpeaking = shouldCapture && isSpeaking;
+	        console.log('_setUserSpeaking called:', isSpeaking, 'shouldCapture:', shouldCapture, 'shouldReportSpeaking:', shouldReportSpeaking, 'current userIsSpeaking:', this.userIsSpeaking);
 	        if (this.userIsSpeaking === shouldReportSpeaking) {
 	            return;
 	        }
 	        this.userIsSpeaking = shouldReportSpeaking;
+	        console.log('_setUserSpeaking: updated userIsSpeaking to:', this.userIsSpeaking);
 	        this.options.onUserIsSpeakingChange(shouldReportSpeaking);
 	    }
 	    /**
@@ -6185,6 +6191,7 @@ registerProcessor('audio_processor', AudioProcessor);
 	     * @param {MessageEvent} event - The WebSocket message event
 	     */
 	    async _handleWebSocketMessage(event) {
+	        var _a, _b;
 	        try {
 	            const message = JSON.parse(event.data);
 	            if (message.type !== 'response.audio') {
@@ -6197,6 +6204,20 @@ registerProcessor('audio_processor', AudioProcessor);
 	                        // Start tracking new agent turn
 	                        console.debug('Agent turn started, will track new turn ID from audio/text');
 	                        this._setUserSpeaking(false);
+	                        // Reset the flag for the new assistant turn
+	                        this.sentReplayFinishedForDisabledOutput = false;
+	                        // When assistant's turn starts but we're not playing audio,
+	                        // we need to tell the server we're "done" with playback so it can
+	                        // transition the turn back to user. Use a small delay to let any
+	                        // response.audio/response.end messages arrive first.
+	                        if (!this.audioOutput) {
+	                            setTimeout(() => {
+	                                if (!this.audioOutput && !this.sentReplayFinishedForDisabledOutput) {
+	                                    this.sentReplayFinishedForDisabledOutput = true;
+	                                    this._clientResponseAudioReplayFinished();
+	                                }
+	                            }, 1000);
+	                        }
 	                    }
 	                    else if (message.role === 'user' && !this.pushToTalkEnabled) {
 	                        // Interrupt any playing agent audio if this is a turn triggered by the server (and not push to talk, which will have already called interrupt)
@@ -6216,7 +6237,25 @@ registerProcessor('audio_processor', AudioProcessor);
 	                    });
 	                    break;
 	                }
+	                case 'response.end': {
+	                    // When audioOutput is disabled, notify server that "playback" is complete
+	                    if (!this.audioOutput && !this.sentReplayFinishedForDisabledOutput) {
+	                        this.sentReplayFinishedForDisabledOutput = true;
+	                        this._clientResponseAudioReplayFinished();
+	                    }
+	                    (_b = (_a = this.options).onMessage) === null || _b === void 0 ? void 0 : _b.call(_a, message);
+	                    break;
+	                }
 	                case 'response.audio': {
+	                    // Skip audio playback if audioOutput is disabled
+	                    if (!this.audioOutput) {
+	                        // Send replay_finished so server knows we're "done" with playback (only once per turn)
+	                        if (!this.sentReplayFinishedForDisabledOutput) {
+	                            this.sentReplayFinishedForDisabledOutput = true;
+	                            this._clientResponseAudioReplayFinished();
+	                        }
+	                        break;
+	                    }
 	                    await this._waitForAudioOutputReady();
 	                    const audioBuffer = base64ToArrayBuffer(message.content);
 	                    const hasAudioSamples = audioBuffer.byteLength > 0;
@@ -6351,6 +6390,9 @@ registerProcessor('audio_processor', AudioProcessor);
 	    }
 	    _sendReadyIfNeeded() {
 	        var _a;
+	        // Send client.ready when either:
+	        // 1. Recorder is started (audio mode active)
+	        // 2. audioInput is false (text-only mode, but server should still be ready)
 	        const audioReady = this.recorderStarted || !this.audioInput;
 	        if (audioReady && ((_a = this.ws) === null || _a === void 0 ? void 0 : _a.readyState) === WebSocket.OPEN && !this.readySent) {
 	            this._wsSend({ type: 'client.ready' });
@@ -6416,12 +6458,16 @@ registerProcessor('audio_processor', AudioProcessor);
 	    }
 	    async audioInputConnect() {
 	        // Turn mic ON
+	        console.log('audioInputConnect: requesting permission');
 	        await this.wavRecorder.requestPermission();
+	        console.log('audioInputConnect: setting up device change listener');
 	        await this._setupDeviceChangeListener();
 	        // If the recorder hasn't spun up yet, proactively select a device.
 	        if (!this.recorderStarted && this.deviceChangeListener) {
+	            console.log('audioInputConnect: initializing recorder with default device');
 	            await this._initializeRecorderWithDefaultDevice();
 	        }
+	        console.log('audioInputConnect: done, recorderStarted =', this.recorderStarted);
 	    }
 	    async audioInputDisconnect() {
 	        try {
@@ -6453,11 +6499,25 @@ registerProcessor('audio_processor', AudioProcessor);
 	        }
 	    }
 	    async setAudioOutput(state) {
+	        console.log('setAudioOutput called with state:', state, 'current:', this.audioOutput);
 	        if (this.audioOutput !== state) {
 	            this.audioOutput = state;
 	            this._emitAudioOutput();
 	            if (state) {
-	                this.wavPlayer.unmute();
+	                // Initialize audio output if not already connected
+	                // This happens when audioOutput was initially false and is now being enabled
+	                if (!this.wavPlayer.context) {
+	                    console.log('setAudioOutput: initializing audio output (no context yet)');
+	                    // Store the promise so _waitForAudioOutputReady() can await it
+	                    // This prevents response.audio from running before AudioContext is ready
+	                    const setupPromise = this.setupAudioOutput();
+	                    this.audioOutputReady = setupPromise;
+	                    await setupPromise;
+	                }
+	                else {
+	                    console.log('setAudioOutput: unmuting existing player');
+	                    this.wavPlayer.unmute();
+	                }
 	                // Sync agentSpeaking state with actual playback state when enabling audio output
 	                this._syncAgentSpeakingState();
 	            }
@@ -6614,6 +6674,11 @@ registerProcessor('audio_processor', AudioProcessor);
 	        return authorizeSessionResponseBody;
 	    }
 	    async setupAudioOutput() {
+	        // Only initialize audio player if audioOutput is enabled
+	        // This prevents AudioContext creation before user gesture when audio is disabled
+	        if (!this.audioOutput) {
+	            return;
+	        }
 	        // Initialize audio player
 	        // wavRecorder will be started from the onDeviceSwitched callback,
 	        // which is called when the device is first initialized and also when the device is switched
@@ -6624,12 +6689,7 @@ registerProcessor('audio_processor', AudioProcessor);
 	        if (!this.options.enableAmplitudeMonitoring) {
 	            this.agentAudioAmplitude = 0;
 	        }
-	        if (this.audioOutput) {
-	            this.wavPlayer.unmute();
-	        }
-	        else {
-	            this.wavPlayer.mute();
-	        }
+	        this.wavPlayer.unmute();
 	    }
 	    async connectToAudioInput() {
 	        if (!this.audioInput) {
@@ -6678,6 +6738,7 @@ registerProcessor('audio_processor', AudioProcessor);
 	     */
 	    async setInputDevice(deviceId) {
 	        var _a, _b, _c;
+	        console.log('setInputDevice called with:', deviceId, 'audioInput:', this.audioInput);
 	        const normalizedDeviceId = !deviceId || deviceId === 'default' ? null : deviceId;
 	        this.useSystemDefaultDevice = normalizedDeviceId === null;
 	        this.deviceId = normalizedDeviceId;
@@ -6686,6 +6747,7 @@ registerProcessor('audio_processor', AudioProcessor);
 	            return;
 	        }
 	        try {
+	            console.log('setInputDevice: calling _queueRecorderRestart');
 	            // Restart recording with the new device
 	            await this._queueRecorderRestart();
 	            // Reinitialize VAD with the new audio stream if VAD is enabled
@@ -6769,12 +6831,15 @@ registerProcessor('audio_processor', AudioProcessor);
 	        return run;
 	    }
 	    async _initializeRecorderWithDefaultDevice() {
+	        console.log('_initializeRecorderWithDefaultDevice called, deviceChangeListener:', !!this.deviceChangeListener);
 	        if (!this.deviceChangeListener) {
 	            return;
 	        }
 	        try {
 	            const devices = await this.wavRecorder.listDevices();
+	            console.log('_initializeRecorderWithDefaultDevice: got devices:', devices.length);
 	            if (devices.length) {
+	                console.log('_initializeRecorderWithDefaultDevice: calling deviceChangeListener');
 	                await this.deviceChangeListener(devices);
 	                return;
 	            }
@@ -6784,6 +6849,7 @@ registerProcessor('audio_processor', AudioProcessor);
 	            console.warn('Unable to prime audio devices from listDevices()', error);
 	        }
 	        try {
+	            console.log('_initializeRecorderWithDefaultDevice: calling setInputDevice default');
 	            await this.setInputDevice('default');
 	        }
 	        catch (error) {
@@ -6832,6 +6898,7 @@ registerProcessor('audio_processor', AudioProcessor);
 	            });
 	            this.deviceChangeListener = async (devices) => {
 	                var _a;
+	                console.log('deviceChangeListener called, devices:', devices.length, 'recorderStarted:', this.recorderStarted);
 	                try {
 	                    // Notify user that devices have changed
 	                    this.options.onDevicesChanged(devices);
@@ -6840,6 +6907,7 @@ registerProcessor('audio_processor', AudioProcessor);
 	                    const previousDefaultDeviceKey = this.lastKnownSystemDefaultDeviceKey;
 	                    const currentDefaultDeviceKey = this._getDeviceComparisonKey(defaultDevice);
 	                    let shouldSwitch = !this.recorderStarted;
+	                    console.log('deviceChangeListener: shouldSwitch initial:', shouldSwitch);
 	                    if (!shouldSwitch) {
 	                        if (usingDefaultDevice) {
 	                            if (!defaultDevice) {
@@ -6859,6 +6927,7 @@ registerProcessor('audio_processor', AudioProcessor);
 	                        }
 	                    }
 	                    this.lastKnownSystemDefaultDeviceKey = currentDefaultDeviceKey;
+	                    console.log('deviceChangeListener: final shouldSwitch:', shouldSwitch);
 	                    if (shouldSwitch) {
 	                        console.debug('Selecting audio input device after change');
 	                        let targetDeviceId = null;