osborn 0.9.35 → 0.9.37

package/dist/index.js

@@ -1,7 +1,7 @@
 // Load environment variables FIRST before any other imports
 import 'dotenv/config';
 import { voice, initializeLogger } from '@livekit/agents';
-import { Room, RoomEvent } from '@livekit/rtc-node';
+import { Room, RoomEvent, RemoteParticipant } from '@livekit/rtc-node';
 import { AccessToken } from 'livekit-server-sdk';
 // Initialize logger before anything else
 initializeLogger({ pretty: true, level: 'info' });
@@ -1294,9 +1294,34 @@ async function main() {
                 console.warn('⚠️ Failed to read JSONL for interruption context:', err);
             }
         }
-        // Store — consumed when user's next message arrives via chat()
-        lastInterruption = { spokenText: fullText, recentMessages, timestamp: Date.now() };
-        console.log(`📋 Interruption context stored (text: ${fullText.length} chars, JSONL: ${recentMessages.length} chars)`);
+        // Store — consumed when user's next message arrives via chat().
+        // Preserve any already-buffered suppressedText (the user may have started speaking
+        // BEFORE the previous TTS completed, and we may have already suppressed in-flight
+        // tts_say events that arrived during that overlap).
+        const carriedSuppressed = lastInterruption?.suppressedText ?? '';
+        lastInterruption = { spokenText: fullText, recentMessages, suppressedText: carriedSuppressed, timestamp: Date.now() };
+        console.log(`📋 Interruption context stored (text: ${fullText.length} chars, JSONL: ${recentMessages.length} chars, suppressed carried: ${carriedSuppressed.length} chars)`);
+    }
+    /**
+     * Append text the agent tried to say while the user was speaking, but which we
+     * suppressed at the tts_say gate to avoid talking over them. Folded into
+     * lastInterruption so it travels to Claude in the next chat() call.
+     * If no interruption context exists yet (e.g. user just started speaking with no
+     * prior TTS interrupt), creates a fresh entry.
+     */
+    function appendSuppressedText(text) {
+        const t = text.trim();
+        if (!t)
+            return;
+        if (lastInterruption) {
+            const sep = lastInterruption.suppressedText ? '\n' : '';
+            lastInterruption.suppressedText = lastInterruption.suppressedText + sep + t;
+            lastInterruption.timestamp = Date.now();
+        }
+        else {
+            lastInterruption = { spokenText: '', recentMessages: '', suppressedText: t, timestamp: Date.now() };
+        }
+        console.log(`🤐 Suppressed text buffered (+${t.length} chars, total ${lastInterruption.suppressedText.length}): "${t.substring(0, 80)}${t.length > 80 ? '...' : ''}"`);
     }
     /**
      * Callback for PipelineDirectLLM — returns pending interruption context and clears it.
@@ -1311,7 +1336,11 @@ async function main() {
             lastInterruption = null;
             return null;
         }
-        const ctx = { spokenText: lastInterruption.spokenText, recentMessages: lastInterruption.recentMessages };
+        const ctx = {
+            spokenText: lastInterruption.spokenText,
+            recentMessages: lastInterruption.recentMessages,
+            suppressedText: lastInterruption.suppressedText,
+        };
         lastInterruption = null;
         return ctx;
     }
@@ -1821,6 +1850,15 @@ async function main() {
                 console.log(`🔇 tts_say fired but text is empty — skipping`);
                 return;
             }
+            // Suppress while the user is mid-utterance. Without this, agent text generated
+            // in parallel by the Claude SDK plays right over the user — same problem as
+            // pre-interrupt overlap, but at the *output* side. The suppressed text gets
+            // folded into lastInterruption so the next chat() to Claude carries it as
+            // "you wrote this but the user did not hear it — re-articulate if relevant."
+            if (userState === 'speaking') {
+                appendSuppressedText(data.text);
+                return;
+            }
             const sayId = Date.now(); // simple ID to correlate start/end logs
             console.log(`🗣️ [${sayId}] session.say START (${data.text.length} chars): "${data.text}"`);
             // Forward spoken text + audio to meeting output page when bot is in a meeting.
@@ -2495,46 +2533,46 @@ async function main() {
     // EARLIEST possible "user is speaking" signal in our setup. Driven by LiveKit's
     // server-side audio-level VAD on the participant's WebRTC track — fires ~50-100ms
     // after mic onset, independent of Deepgram STT or any local VAD (we don't run one).
-    // Same signal the LiveKit room uses to identify active speakers, so it's tuned for
-    // real speech and ignores low-level noise.
     //
-    // Flow: user starts talking → ActiveSpeakersChanged includes the remote participant →
-    // if agent is currently speaking → interrupt the SpeechHandle to flush TTS.
-    // The existing handleSpeechDone callback captures the spoken-text + JSONL context
-    // (lastInterruption) and PipelineDirectLLM consumes it on the next chat() call to
-    // enrich the user's message with [INTERRUPTED] context — so the post-interrupt
-    // note flow is preserved even though we're interrupting earlier.
+    // Flow: user starts talking → ActiveSpeakersChanged includes a RemoteParticipant →
+    // if agent is currently speaking → interrupt the SpeechHandle to flush TTS playback.
+    // The existing handleSpeechDone callback (around line 1320) captures the spoken-text
+    // + JSONL context into lastInterruption; PipelineDirectLLM consumes it on the next
+    // chat() call to enrich the user's message with [INTERRUPTED] context — so the
+    // post-interrupt note flow is preserved even though we're cutting TTS earlier.
+    //
+    // Filter is `instanceof RemoteParticipant`. The agent IS the LocalParticipant in this
+    // room, and when its TTS plays it appears in the active-speakers list too. An earlier
+    // attempt that compared `s.identity !== room.localParticipant?.identity` failed because
+    // localParticipant.identity could be undefined at event-fire time, letting the agent's
+    // own speech trigger a self-interrupt. The type check is bulletproof.
     //
     // Realtime mode skipped — the SDK handles interruption internally there, and manual
     // interrupt for Gemini realtime crashes its state machine (code 1008, memory v0.4.5).
     let lastActiveSpeakerInterruptAt = 0;
     room.on(RoomEvent.ActiveSpeakersChanged, (speakers) => {
+        if (!Array.isArray(speakers) || speakers.length === 0)
+            return;
+        const remoteSpeakers = speakers.filter((s) => s instanceof RemoteParticipant);
+        if (remoteSpeakers.length === 0)
+            return;
         if (currentVoiceMode === 'realtime')
             return;
         if (agentState !== 'speaking')
             return;
-        const localIdentity = room.localParticipant?.identity;
-        const remoteSpeaking = Array.isArray(speakers) && speakers.some((s) => s && s.identity && s.identity !== localIdentity);
-        if (!remoteSpeaking)
-            return;
-        // Debounce: avoid log spam when audio level oscillates above/below threshold.
-        // interrupt() itself is idempotent on an already-interrupted SpeechHandle, but
-        // we suppress duplicate logs within 1s.
         const now = Date.now();
-        if (now - lastActiveSpeakerInterruptAt < 1000) {
-            try {
-                currentSession?.interrupt();
-            }
-            catch { }
-            return;
-        }
+        const debounced = now - lastActiveSpeakerInterruptAt < 1000;
         lastActiveSpeakerInterruptAt = now;
         try {
-            console.log('🎤 ActiveSpeakersChanged: remote speaker + agent speaking → interrupting TTS');
+            if (!debounced) {
+                const ids = remoteSpeakers.map((s) => s.identity).join(',');
+                console.log(`🎤 ActiveSpeakersChanged: remote speakers [${ids}] + agent speaking → interrupting TTS`);
+            }
             currentSession?.interrupt();
         }
         catch (err) {
-            console.warn('⚠️ active-speaker interrupt failed:', err instanceof Error ? err.message : err);
+            if (!debounced)
+                console.warn('⚠️ active-speaker interrupt failed:', err instanceof Error ? err.message : err);
         }
     });
     room.on(RoomEvent.Disconnected, () => {
@@ -2830,28 +2868,23 @@ async function main() {
                 }
             });
             // User state tracking — prevents queue from colliding with server-side VAD.
-            // ALSO: interrupt the agent's TTS the moment Deepgram STT says the user is speaking.
-            // Why here: in STT pipeline mode without a local VAD, the SDK's own auto-interrupt
-            // (interruptByAudioActivity, agent_activity.js:651) is dead because it only fires
-            // from onVADInferenceDone. The STT path (Deepgram START_OF_SPEECH) reaches us via
-            // agent_activity.onStartOfSpeech → _updateUserState('speaking') → this event. That
-            // is the earliest "user is speaking, not noise" signal we get without bringing back
-            // a local VAD. interrupt() drains the currentSpeech + speech queue, killing TTS
-            // playback in-flight. handleSpeechDone still captures the spoken-text + JSONL
-            // context, consumed by PipelineDirectLLM on the next chat() call.
-            // Realtime mode skipped — the SDK handles interruption internally there, and manual
-            // interrupt for Gemini realtime crashes its state machine (code 1008, memory v0.4.5).
+            // Also a secondary interrupt trigger: when Deepgram STT classifies speech onset
+            // it propagates here via agent_activity.onStartOfSpeech → _updateUserState('speaking').
+            // Fires later than ActiveSpeakersChanged (Deepgram has ~100-300ms classification
+            // latency vs LiveKit's ~50-100ms audio-level) but acts as a redundant fallback in
+            // case the room-level event drops. interrupt() is idempotent on an already-
+            // interrupted SpeechHandle so calling both paths is harmless.
             sess.on('user_state_changed', (ev) => {
                 const prev = userState;
                 userState = ev.newState;
                 console.log(`👤 User state: ${prev} → ${ev.newState} (agent: ${agentState})`);
                 if (ev.newState === 'speaking' && agentState === 'speaking' && sessionVoiceMode !== 'realtime') {
                     try {
-                        console.log('🎤 User started speaking while agent was speaking → interrupting TTS');
+                        console.log('🎤 user_state_changed=speaking + agent speaking → interrupting TTS (fallback)');
                         currentSession?.interrupt();
                     }
                     catch (err) {
-                        console.warn('⚠️ user-onset interrupt failed:', err instanceof Error ? err.message : err);
+                        console.warn('⚠️ user-state interrupt failed:', err instanceof Error ? err.message : err);
                     }
                 }
                 // When user stops speaking, retry voice queue — items may be waiting
@@ -3147,10 +3180,7 @@ async function main() {
                 clearInterval(readyInterval);
                 console.log('✅ agent_ready retries complete');
             }, 20000);
-            // Stop agent_ready retries on user speech.
-            // NB: input_speech_started is realtime-only — the SDK never emits it in STT pipeline
-            // mode. The earliest onset signal in pipeline mode is user_state_changed → 'speaking',
-            // wired further down. Don't add interrupt logic here.
+            // Stop agent_ready retries on user speech
             session.on('input_speech_started', () => {
                 readySent = true;
                 clearInterval(readyInterval);

package/dist/pipeline-direct-llm.d.ts

@@ -14,6 +14,13 @@ import { EventEmitter } from 'events';
 export interface InterruptionContext {
     spokenText: string;
     recentMessages: string;
+    /**
+     * Text the agent generated while the user was still speaking, which we
+     * suppressed at session.say() to avoid talking over the user. The agent
+     * receives this so it knows what it tried to say but the user did not hear,
+     * and can re-articulate the relevant bits in its next response.
+     */
+    suppressedText: string;
 }
 export interface PipelineDirectOptions extends ClaudeLLMOptions {
     onFastBrainResult?: (result: FastBrainPanelResult) => void;

package/dist/pipeline-direct-llm.js

@@ -83,31 +83,89 @@ export class PipelineDirectLLM extends llm.LLM {
             }
         }
         console.log(`📥 [pipeline] chat() call #${callN} (${userText.length} chars): "${userText}"`);
-        // Check for pending interruption context — enrich user message if interrupted
+        // Always check the pending playback context — it can carry two independent
+        // signals: (a) an actual interruption (spokenText + recentMessages) when the
+        // user cut Osborn off mid-TTS, OR (b) suppressed text generated by the SDK
+        // while the user was speaking, regardless of whether they were actually
+        // interrupting active TTS. We need to forward BOTH cases so the agent knows
+        // what it produced that the user didn't hear, and so the buffer is cleared
+        // every turn even when there was no interrupt.
         const interruptCtx = this.#opts.getAndConsumeInterruptionContext?.();
         if (interruptCtx && userText.trim()) {
-            console.log(`🔇 [pipeline] Enriching user message with interruption context`);
-            // Interrupt Claude's current work before sending enriched message
-            this.#claudeLLM.interruptQuery().catch(() => { });
-            // Replace user message in chatCtx with context-enriched version
-            const enrichedMessage = [
-                `[INTERRUPTED] The user interrupted your response mid-speech.`,
-                ``,
-                `What the user heard before cutoff:`,
-                `"${interruptCtx.spokenText}"`,
-                ``,
-                `Your recent messages (full untruncated — you wrote these):`,
-                interruptCtx.recentMessages || '(no recent messages found)',
-                ``,
-                `User's message: "${userText}"`,
-                ``,
-                `RESPOND with speech first, then act:`,
-                `- ALWAYS reply with at least one spoken sentence before doing any tool calls`,
-                `- If it's a quick side question, answer it then continue where you left off`,
-                `- If they want to change direction, acknowledge and follow their lead`,
-                `- Clarify when asked to or the question requires going over what you just said`,
-                `- If relevant details were cut off — whether they answer the current question or an earlier one — weave them back in naturally so the user stays in context without having to ask again.`,
-            ].join('\n');
+            const hasInterrupt = !!interruptCtx.spokenText;
+            const hasSuppressed = !!interruptCtx.suppressedText;
+            const suppressedBlock = hasSuppressed
+                ? [
+                    ``,
+                    `Text you generated while the user was speaking — NOT played (we suppressed it so we wouldn't talk over them):`,
+                    `"${interruptCtx.suppressedText}"`,
+                    `If any of that is still relevant to the user's current message, re-articulate the key points naturally. If it's no longer relevant, drop it.`,
+                ].join('\n')
+                : '';
+            let enrichedMessage;
+            if (hasInterrupt) {
+                // Actual mid-TTS interruption — keep the full [INTERRUPTED] template
+                console.log(`🔇 [pipeline] Enriching: interrupt (spoken=${interruptCtx.spokenText.length} chars, suppressed=${interruptCtx.suppressedText.length} chars)`);
+                this.#claudeLLM.interruptQuery().catch(() => { });
+                enrichedMessage = [
+                    `[INTERRUPTED] The user interrupted your response mid-speech.`,
+                    ``,
+                    `What the user heard before cutoff:`,
+                    `"${interruptCtx.spokenText}"`,
+                    ``,
+                    `WHAT THE USER DID NOT HEAR (you wrote this but it was cut off):`,
+                    `Anything in "Your recent messages" below that appears AFTER the quoted heard text is content the user did not hear. The user has no memory of it.`,
+                    ``,
+                    `Your recent messages (full untruncated — you wrote these):`,
+                    interruptCtx.recentMessages || '(no recent messages found)',
+                    suppressedBlock,
+                    ``,
+                    `User's message: "${userText}"`,
+                    ``,
+                    `CONTEXT PRESERVATION (READ THIS):`,
+                    `The user has NO memory of unheard content. If any of it bears on their current message — answers a question they just asked, sets up a follow-up they're now asking about, or shows a knowledge gap and fills in a detail they're reacting to — you MUST surface it. Briefly is fine if their message is off-topic or explores a direction they haven't asked about yet. But never assume they remember what they never heard.`,
+                    ``,
+                    `RESPOND with speech first, then act:`,
+                    `- ALWAYS reply with at least one spoken sentence before doing any tool calls`,
+                    `- If it's a quick side question, answer it then continue where you left off`,
+                    `- If they want to change direction, acknowledge and follow their lead`,
+                    `- Clarify when asked to or the question requires going over what you just said`,
+                    `- If relevant details were cut off — whether they answer the current question or an earlier one — weave them back in naturally so the user stays in context without having to ask again.`,
+                ].join('\n');
+            }
+            else if (hasSuppressed) {
+                // No real interrupt — user was speaking while we had text queued. They
+                // weren't cutting Osborn off, just talking over a gap. Don't claim an
+                // interrupt happened; symmetric structure to [INTERRUPTED] so Claude
+                // treats both signals consistently.
+                console.log(`🤐 [pipeline] Enriching: suppressed-only (${interruptCtx.suppressedText.length} chars, no interrupt)`);
+                enrichedMessage = [
+                    `[CONTEXT] You generated speech while the user was already talking. None of it played.`,
+                    ``,
+                    `What the user is saying now:`,
+                    `"${userText}"`,
+                    ``,
+                    `Text you produced that the user did NOT hear:`,
+                    `"${interruptCtx.suppressedText}"`,
+                    ``,
+                    `CONTEXT PRESERVATION (READ THIS):`,
+                    `The user has NO memory of the unheard text. If any of it bears on their current message — answers a question they just asked, sets up a follow-up they're now asking about, or shows a knowledge gap and fills in a detail they're reacting to — you MUST surface it. Briefly is fine if their message is off-topic or explores a direction they haven't asked about yet. But never assume they remember what they never heard.`,
+                    ``,
+                    `RESPOND with speech first, then act:`,
+                    `- ALWAYS reply with at least one spoken sentence before doing any tool calls`,
+                    `- Three likely cases — figure out which applies:`,
+                    `  (a) the user didn't realize you were responding → forward the key points of the unheard text`,
+                    `  (b) the user changed direction → drop the unheard text, follow their lead`,
+                    `  (c) the user's message builds on the unheard text → integrate it as if they'd heard it`,
+                    `- Keep it tight — this is a voice response.`,
+                ].join('\n');
+            }
+            else {
+                // Context object existed but both fields empty — defensive no-op,
+                // shouldn't happen because appendSuppressedText only creates entries
+                // when text is non-empty.
+                enrichedMessage = userText;
+            }
             // Modify the last user message in chatCtx
             for (let i = chatCtx.items.length - 1; i >= 0; i--) {
                 const item = chatCtx.items[i];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.35",
+  "version": "0.9.37",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {