osborn 0.9.34 → 0.9.36

package/dist/index.js

@@ -1,7 +1,7 @@
 // Load environment variables FIRST before any other imports
 import 'dotenv/config';
 import { voice, initializeLogger } from '@livekit/agents';
-import { Room, RoomEvent } from '@livekit/rtc-node';
+import { Room, RoomEvent, RemoteParticipant } from '@livekit/rtc-node';
 import { AccessToken } from 'livekit-server-sdk';
 // Initialize logger before anything else
 initializeLogger({ pretty: true, level: 'info' });
@@ -2492,6 +2492,51 @@ async function main() {
         console.log('✅ Connected to room:', roomName);
         localParticipant = room.localParticipant;
     });
+    // EARLIEST possible "user is speaking" signal in our setup. Driven by LiveKit's
+    // server-side audio-level VAD on the participant's WebRTC track — fires ~50-100ms
+    // after mic onset, independent of Deepgram STT or any local VAD (we don't run one).
+    //
+    // Flow: user starts talking → ActiveSpeakersChanged includes a RemoteParticipant →
+    // if agent is currently speaking → interrupt the SpeechHandle to flush TTS playback.
+    // The existing handleSpeechDone callback (around line 1320) captures the spoken-text
+    // + JSONL context into lastInterruption; PipelineDirectLLM consumes it on the next
+    // chat() call to enrich the user's message with [INTERRUPTED] context — so the
+    // post-interrupt note flow is preserved even though we're cutting TTS earlier.
+    //
+    // Filter is `instanceof RemoteParticipant`. The agent IS the LocalParticipant in this
+    // room, and when its TTS plays it appears in the active-speakers list too. An earlier
+    // attempt that compared `s.identity !== room.localParticipant?.identity` failed because
+    // localParticipant.identity could be undefined at event-fire time, letting the agent's
+    // own speech trigger a self-interrupt. The type check is bulletproof.
+    //
+    // Realtime mode skipped — the SDK handles interruption internally there, and manual
+    // interrupt for Gemini realtime crashes its state machine (code 1008, memory v0.4.5).
+    let lastActiveSpeakerInterruptAt = 0;
+    room.on(RoomEvent.ActiveSpeakersChanged, (speakers) => {
+        if (!Array.isArray(speakers) || speakers.length === 0)
+            return;
+        const remoteSpeakers = speakers.filter((s) => s instanceof RemoteParticipant);
+        if (remoteSpeakers.length === 0)
+            return;
+        if (currentVoiceMode === 'realtime')
+            return;
+        if (agentState !== 'speaking')
+            return;
+        const now = Date.now();
+        const debounced = now - lastActiveSpeakerInterruptAt < 1000;
+        lastActiveSpeakerInterruptAt = now;
+        try {
+            if (!debounced) {
+                const ids = remoteSpeakers.map((s) => s.identity).join(',');
+                console.log(`🎤 ActiveSpeakersChanged: remote speakers [${ids}] + agent speaking → interrupting TTS`);
+            }
+            currentSession?.interrupt();
+        }
+        catch (err) {
+            if (!debounced)
+                console.warn('⚠️ active-speaker interrupt failed:', err instanceof Error ? err.message : err);
+        }
+    });
     room.on(RoomEvent.Disconnected, () => {
         console.log('👋 Disconnected from room');
         // Clean up active research and voice queue
@@ -2784,10 +2829,26 @@ async function main() {
                     setTimeout(() => processVoiceQueue(), 500); // 500ms to let model settle
                 }
             });
-            // User state tracking — prevents queue from colliding with server-side VAD
+            // User state tracking — prevents queue from colliding with server-side VAD.
+            // Also a secondary interrupt trigger: when Deepgram STT classifies speech onset
+            // it propagates here via agent_activity.onStartOfSpeech → _updateUserState('speaking').
+            // Fires later than ActiveSpeakersChanged (Deepgram has ~100-300ms classification
+            // latency vs LiveKit's ~50-100ms audio-level) but acts as a redundant fallback in
+            // case the room-level event drops. interrupt() is idempotent on an already-
+            // interrupted SpeechHandle so calling both paths is harmless.
             sess.on('user_state_changed', (ev) => {
+                const prev = userState;
                 userState = ev.newState;
-                console.log(`👤 User state: ${ev.newState}`);
+                console.log(`👤 User state: ${prev} → ${ev.newState} (agent: ${agentState})`);
+                if (ev.newState === 'speaking' && agentState === 'speaking' && sessionVoiceMode !== 'realtime') {
+                    try {
+                        console.log('🎤 user_state_changed=speaking + agent speaking → interrupting TTS (fallback)');
+                        currentSession?.interrupt();
+                    }
+                    catch (err) {
+                        console.warn('⚠️ user-state interrupt failed:', err instanceof Error ? err.message : err);
+                    }
+                }
                 // When user stops speaking, retry voice queue — items may be waiting
                 if (ev.newState === 'listening' && voiceQueue.length > 0) {
                     setTimeout(() => processVoiceQueue(), 500);
@@ -3081,27 +3142,10 @@ async function main() {
                 clearInterval(readyInterval);
                 console.log('✅ agent_ready retries complete');
             }, 20000);
-            // Stop agent_ready retries on user speech, and interrupt agent TTS at VAD onset.
-            // Previously the interrupt only fired when STT committed a full transcript (chat()
-            // call), which let the agent talk over the user for the full utterance. Firing it
-            // here cuts TTS the moment VAD detects speech.
-            // Realtime providers (OpenAI/Gemini) handle interruption server-side via their own
-            // VAD — calling interrupt() manually for Gemini specifically crashes its state
-            // machine (code 1008, hangs in 'speaking'), so skip those.
+            // Stop agent_ready retries on user speech
             session.on('input_speech_started', () => {
                 readySent = true;
                 clearInterval(readyInterval);
-                if (agentState !== 'speaking')
-                    return;
-                if (sessionVoiceMode === 'realtime')
-                    return;
-                try {
-                    console.log('🎤 VAD onset → interrupting agent TTS');
-                    currentSession?.interrupt();
-                }
-                catch (err) {
-                    console.warn('⚠️ VAD-onset interrupt failed:', err instanceof Error ? err.message : err);
-                }
             });
             // Greet user via TTS (delayed if resume prompt will be shown)
             // For realtime mode: use generateReply() since there's no standalone TTS

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.34",
+  "version": "0.9.36",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {