osborn 0.9.116 → 0.9.118

package/dist/index.js

@@ -276,6 +276,12 @@ async function synthMp3(text) {
 }
 // THE speak path for meetings (2026-08-01): Recall native output_audio FIRST
 // (direct, loud, no canvas capture chain), canvas Web-Audio say as FALLBACK.
+// THE meeting speak path (2026-08-01, live-verified A/B): VOICE via Recall
+// output_audio (reliable, doesn't depend on the headless canvas AudioContext),
+// VISUAL caption pushed to the canvas WITHOUT audio (kind:'caption') so the
+// bot's camera shows what it's saying — no double-audio. output_audio +
+// canvas camera coexist (confirmed: user heard output_audio while the canvas
+// was showing). Falls back to canvas 'say' (audio) only if output_audio fails.
 async function speakIntoMeeting(text) {
     const recall = getRecallClient();
     const botId = recall?.getActiveBotIds?.()[0];
@@ -283,11 +289,12 @@ async function speakIntoMeeting(text) {
         const mp3 = await synthMp3(text);
         if (mp3 && await recall.outputAudio(botId, mp3)) {
             console.log(`📢 spoke via Recall output_audio (${mp3.length}b): "${text.slice(0, 60)}"`);
+            pushCanvas({ kind: 'caption', text }); // visual only, no audio
             markMeetingSpeaking(text);
             return;
         }
     }
-    console.log(`📽️ falling back to canvas say: "${text.slice(0, 50)}"`);
+    console.log(`📽️ falling back to canvas say (audio): "${text.slice(0, 50)}"`);
     pushCanvas({ kind: 'say', text });
     markMeetingSpeaking(text);
 }
@@ -715,8 +722,8 @@ function startApiServer(workingDir, port) {
             req.on('end', () => {
                 try {
                     const evt = JSON.parse(body || '{}');
-                    if (evt.kind !== 'say' && evt.kind !== 'show' && evt.kind !== 'stop')
-                        throw new Error("kind must be 'say', 'show', or 'stop'");
+                    if (evt.kind !== 'say' && evt.kind !== 'caption' && evt.kind !== 'show' && evt.kind !== 'stop')
+                        throw new Error("kind must be 'say', 'caption', 'show', or 'stop'");
                     if (evt.kind === 'say') {
                         // Native-first speak path (Recall output_audio → canvas fallback).
                         void speakIntoMeeting(evt.text);
@@ -2644,44 +2651,46 @@ async function main() {
         // Wire up TTS say — bypass LiveKit's BufferedTokenStream, speak directly via session.say()
         // Each text block from Claude gets spoken immediately as it arrives, no internal buffering
         directLLM.events.on('tts_say', (data) => {
-            // Guard: session must be alive — TTS errors can kill the session while background query runs
-            if (!currentSession) {
-                console.warn(`⚠️ tts_say fired but currentSession is null — text dropped (${data.text?.length || 0} chars): "${data.text || ''}"`);
-                return;
-            }
             if (!data.text?.trim()) {
                 console.log(`🔇 tts_say fired but text is empty — skipping`);
                 return;
             }
-            // Suppress browser TTS for meeting-chunk responses — the agent speaks INTO
-            // the meeting via /canvas, not the laptop speakers (which the Meet mic
-            // re-captures in the same room → feedback). Set by PipelineDirectLLM.chat()
-            // when the turn is a [MEETING —] chunk. Normal user turns are unaffected.
+            // MEETING PATH (fix 2026-08-01) — runs BEFORE the currentSession guard.
+            // A meeting reply routes to the meeting via speakIntoMeeting → Recall
+            // output_audio, which needs NO LiveKit voice session. Keeping this ahead
+            // of the guard is what lets the bot keep replying AFTER the browser voice
+            // session dropped (currentLLM is deliberately kept alive for exactly this;
+            // see handleParticipantDisconnected). Before the fix a participant blip
+            // nulled currentSession and EVERY meeting reply was "text dropped" — the
+            // deaf-bot bug proven live 2026-08-01 via synthetic transcript inject.
+            // Set by PipelineDirectLLM.chat() when the turn is a [MEETING —] chunk.
             if (directLLM.suppressMeetingTTS) {
-                // MEETING MODE CONDITION (user directive 2026-08-01): says route to
-                // the NATIVE Recall sink by default — no room/Chrome relay (that path
-                // was moderately-to-extremely choppy). The LiveKit parity relay stays
-                // available behind OSBORN_MEETING_AUDIO=parity for future testing.
-                if (process.env.OSBORN_MEETING_AUDIO === 'parity' && meetingCanvasInRoom && activeMeetingBotId && Date.now() < meetingAddressedUntil) {
+                // Parity relay (behind OSBORN_MEETING_AUDIO=parity) speaks via
+                // session.say, so it still needs a live session — it falls through to
+                // the normal browser-TTS path below only when currentSession exists.
+                const parity = process.env.OSBORN_MEETING_AUDIO === 'parity' && meetingCanvasInRoom && activeMeetingBotId && Date.now() < meetingAddressedUntil && !!currentSession;
+                if (parity) {
                     console.log(`🔊🎼 meeting reply via NATIVE session.say (canvas relays): "${data.text.slice(0, 50)}"`);
                     markMeetingSpeaking(data.text);
-                    // no return — normal say proceeds below
-                }
-                else 
-                // ADDRESSED turn → REDIRECT the normal streaming reply into the meeting
-                // (reuse of the regular speak path; no Bash roundtrip). Silent-observer
-                // turns stay suppressed as before.
-                if (activeMeetingBotId && Date.now() < meetingAddressedUntil) {
-                    // ONE say per reply (2026-08-01 quality directive): chunking created
-                    // seams no scheduler fully hid. A single utterance = a single clean
-                    // audio file = zero seams — quality over ~1s of latency. (Structural
-                    // fix queued: canvas as LiveKit subscriber playing the agent's real
-                    // TTS track — the true "same browser experience" reuse.)
+                    // fall through to normal browser TTS (currentSession present)
+                }
+                else if (activeMeetingBotId && Date.now() < meetingAddressedUntil) {
+                    // NATIVE sink: one clean output_audio push into the meeting, NO
+                    // LiveKit session required — this is the path that survives a
+                    // voice-session drop.
                     console.log(`🔊➡️📢 tts_say → meeting (native audio) t=${new Date().toISOString()}: "${data.text.slice(0, 60)}"`);
                     void speakIntoMeeting(data.text);
                     return;
                 }
-                console.log(`🔇 tts_say suppressed (meeting turn — response goes to /canvas, not browser): "${data.text.slice(0, 60)}"`);
+                else {
+                    console.log(`🔇 tts_say suppressed (meeting turn — response goes to /canvas, not browser): "${data.text.slice(0, 60)}"`);
+                    return;
+                }
+            }
+            // NORMAL BROWSER TTS (and the parity fall-through): needs a live session.
+            // TTS errors can kill the session while a background query still runs.
+            if (!currentSession) {
+                console.warn(`⚠️ tts_say fired but currentSession is null — text dropped (${data.text?.length || 0} chars): "${data.text || ''}"`);
                 return;
             }
             // Suppress while the user is mid-utterance. Without this, agent text generated
@@ -4142,6 +4151,17 @@ async function main() {
             // Close handler with auto-recovery for crashes (both realtime and direct modes)
             sess.on('close', async (ev) => {
                 console.log('🚪 Session closed:', ev.reason);
+                // ORPHAN-BOT GUARD (fix 2026-08-01): an abrupt AgentSession close
+                // (e.g. a driver tab killed → reason 'user_initiated', with NO clean
+                // ParticipantDisconnected) must not leave the Recall bot stuck in-call
+                // draining credits. Arm the meeting leave-grace so endMeeting() fires
+                // (bot leaves + LLM released) once the room is genuinely empty. A rejoin
+                // cancels it; error/disconnected closes still auto-recover below and the
+                // grace is harmless there (recovery + rejoin cancels it).
+                if (activeMeetingBotId) {
+                    console.log('📽️ Meeting active on session close — arming leave-grace (bot leaves if no user returns)');
+                    armMeetingLeaveGrace();
+                }
                 // TTS abort from user interruption — SDK already killed the session internally,
                 // so we MUST recover (can't just reset state — STT pipeline is dead).
                 // Log it distinctly so we know it's an interrupt recovery, not a real crash.
@@ -4526,24 +4546,34 @@ async function main() {
             })();
         }
         currentAgent = null;
-        // Kill the Claude SDK subprocess BEFORE dropping the reference, otherwise the
-        // persistent session keeps running tools and pushing TTS into a dead session.
-        killCurrentLLM('participant_disconnected');
-        currentLLM = null;
-        clearFastBrainSession();
-        clearPipelineFastBrainSession();
-        // Auto-leave any active meeting bot when user disconnects from the room.
-        // DELIBERATE COUPLING: the bot's lifecycle follows the voice session — a
-        // decoupled always-on meeting bot means untracked background agents (no
-        // status surface for them yet). Revisit only WITH a tracking UI.
-        // endMeeting() centralizes the teardown (flush, poller, max-timer, Recall
-        // leave, frontend notify).
-        armMeetingLeaveGrace();
-        // 0.9.83: a real session just ended → use the FAST leave (~20s), not the
-        // 3-min alone grace. Runs on the agent, so it fires even on an abrupt tab
-        // close. Cancelled if a user rejoins within the grace (handled where the
-        // participant-join cancels timers).
-        armFastLeaveTimer();
+        if (activeMeetingBotId) {
+            // MEETING SURVIVES A VOICE-SESSION DROP (fix 2026-08-01). The bot's brain
+            // is the persistent Claude LLM; killing it on every participant blip left
+            // the Recall bot permanently DEAF mid-call ("Addressed" fired but never
+            // "Flushed" — currentLLM was null; proven via synthetic transcript inject).
+            // Keep currentLLM alive so meeting transcripts still flush + reply
+            // (speakIntoMeeting is Recall-direct, needs no LiveKit session). The
+            // leave-grace below owns teardown: if the user is truly gone at 75s,
+            // endMeeting() releases the LLM (its !userPresent branch). A rejoin cancels
+            // the grace. DO NOT arm the 20s fast-leave here — it would destroy the room
+            // before the meeting grace and orphan the bot; the meeting grace is the
+            // single teardown while a meeting is live.
+            console.log('📽️ Meeting active — keeping currentLLM alive across the voice-session drop (leave-grace owns teardown)');
+            armMeetingLeaveGrace();
+        }
+        else {
+            // Kill the Claude SDK subprocess BEFORE dropping the reference, otherwise the
+            // persistent session keeps running tools and pushing TTS into a dead session.
+            killCurrentLLM('participant_disconnected');
+            currentLLM = null;
+            clearFastBrainSession();
+            clearPipelineFastBrainSession();
+            // Auto-leave path for a NON-meeting session. 0.9.83: a real session just
+            // ended → use the FAST leave (~20s), not the 3-min alone grace. Fires even
+            // on an abrupt tab close; cancelled if a user rejoins within the grace.
+            armMeetingLeaveGrace(); // no-op when no meeting, kept for symmetry
+            armFastLeaveTimer();
+        }
         console.log('⏳ Waiting for new user...\n');
     };
     const handleDataReceived = async (payload, participant, kind, topic) => {
@@ -5210,16 +5240,18 @@ async function main() {
                             // the bot casts the meeting canvas as its camera+mic (below), so
                             // it can show visuals and speak into the meeting on demand.
                             await sendToFrontend({ type: 'meeting_joining', message: 'Osborn is joining your meeting...' });
-                            // AUDIO-FIRST DEFAULT (2026-08-01, proven live): the webpage-camera
-                            // output_media routes the bot's AUDIO through the canvas page's
-                            // Web Audio — which sits SUSPENDED in Recall's headless Chrome (no
-                            // user gesture) → the bot was inaudible all night. Default now = NO
-                            // webpage camera, so the bot's voice is the direct Recall
-                            // output_audio track (speakIntoMeeting → recall.outputAudio, ~1.8s
-                            // synth, verified audible + clear). The browser CAST is opt-in via
-                            // explicit data.castUrl / OSBORN_MEETING_CAST_URL until we split
-                            // video(webpage) from audio(output_audio) to reclaim it cleanly.
-                            const castUrl = data.castUrl || process.env.OSBORN_MEETING_CAST_URL || undefined;
+                            // CANVAS + AUDIO (2026-08-01, live-verified A/B): the canvas
+                            // webpage camera shows VISUALS (captions, screenshots, transcript
+                            // animation) AND Recall output_audio delivers the VOICE — the two
+                            // coexist (confirmed: output_audio audible while the canvas camera
+                            // was showing). speakIntoMeeting routes voice→output_audio +
+                            // caption→canvas (no audio), so no double-audio. The canvas page's
+                            // OWN Web Audio is NOT used for voice (it's suspended headless).
+                            const frontendUrl = (process.env.OSBORN_FRONTEND_URL || 'https://www.voice-native.com').replace(/\/$/, '');
+                            const canvasUrl = /^https:\/\//.test(webhookBase)
+                                ? `${frontendUrl}/meeting-canvas?agent=${encodeURIComponent(webhookBase)}`
+                                : undefined;
+                            const castUrl = data.castUrl || process.env.OSBORN_MEETING_CAST_URL || canvasUrl;
                             const botId = await recallJoin.joinMeeting(meetingUrl, webhookBase, { castUrl });
                             const sessionId = currentLLM?.sessionId || currentResumeSessionId || 'default';
                             recallJoin.registerBot(botId, sessionId);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.116",
+  "version": "0.9.118",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {