osborn 0.9.40 → 0.9.42

package/dist/index.js

@@ -1,7 +1,7 @@
 // Load environment variables FIRST before any other imports
 import 'dotenv/config';
 import { voice, initializeLogger } from '@livekit/agents';
-import { Room, RoomEvent } from '@livekit/rtc-node';
+import { Room, RoomEvent, AudioSource, AudioFrame, LocalAudioTrack, TrackPublishOptions, TrackSource, } from '@livekit/rtc-node';
 import { AccessToken } from 'livekit-server-sdk';
 // Initialize logger before anything else
 initializeLogger({ pretty: true, level: 'info' });
@@ -149,6 +149,12 @@ process.on('uncaughtException', (error) => {
 let currentRoomCode = null;
 // Meeting output WebSocket — module-level so both startApiServer and main() can access it
 let meetingOutputWs = null;
+// Module-level AgentSession reference so /meeting-audio-in WS handler can switch
+// the RoomIO-linked participant when meeting audio starts/stops (B2 design).
+let activeAgentSession = null;
+// Identity of the local user participant the session was originally listening to
+// — captured at the moment we switch to the meeting publisher, restored on cleanup.
+let preMeetingUserIdentity = null;
 function sendToMeetingOutput(msg) {
     if (meetingOutputWs && meetingOutputWs.readyState === WebSocket.OPEN) {
         try {
@@ -960,20 +966,270 @@ function startApiServer(workingDir, port) {
     cleanStaleUploadDirs();
     setInterval(cleanStaleUploadDirs, 10 * 60 * 1000);
     // ============================================================
-    // Meeting Output WebSocket — /meeting-audio
+    // Meeting Output WebSocket — /meeting-audio (LEGACY)
     // ============================================================
-    // Recall's headless browser opens meeting-output.html which connects here.
-    // We push: JSON { type: 'speak', text } for display, binary PCM for audio (future).
+    // Recall's headless browser used to open meeting-output.html which connects
+    // here. With the new /meeting-bot Next.js page (Phase 2 + LiveKit), Recall
+    // points at frontend/meeting-bot instead — this handler exists only for
+    // backwards-compat with old machine images still serving the legacy path.
     const meetingOutputWss = new WebSocketServer({ noServer: true });
     meetingOutputWss.on('connection', (ws) => {
-        console.log('📺 Meeting output browser connected');
+        console.log('📺 Meeting output browser connected (legacy /meeting-audio)');
         meetingOutputWs = ws;
         ws.on('close', () => {
-            console.log('📺 Meeting output browser disconnected');
+            console.log('📺 Meeting output browser disconnected (legacy)');
             if (meetingOutputWs === ws)
                 meetingOutputWs = null;
         });
     });
+    // ============================================================
+    // Recall.ai meeting-audio-in WebSocket — /meeting-audio-in
+    // ============================================================
+    // Recall.ai's per-participant real-time audio protocol. Bot is configured
+    // (in recall-client.ts joinMeeting) with audio_separate_raw + a realtime
+    // endpoint pointing at this URL. Recall sends JSON events containing
+    // base64-encoded PCM (S16LE, 16kHz, mono) for every meeting participant
+    // (bot's own audio NOT included by default — no feedback loop possible).
+    //
+    // Flow: Recall → /meeting-audio-in → open a SECOND LiveKit connection from
+    //       this agent process as a publisher participant → publish PCM as an
+    //       audio track in the same LiveKit room → the existing AgentSession's
+    //       STT subscribes to it as a remote track → routes to currentLLM.chat()
+    //       via the same pipeline as voice-native user mic.
+    //
+    // The advantage of this design vs a parallel STT pipeline: meeting audio
+    // becomes "just another participant" in the LiveKit room — same end-of-turn
+    // detection, same interrupt handling, same conversation context, no parallel
+    // chat() paths to maintain.
+    //
+    // Wait until activeAgentSession._roomIO exists AND the publisher participant
+    // is visible to the agent's room. Both can race against join_meeting:
+    //   - Agent session may still be starting up when Recall connects.
+    //   - LiveKit takes a moment to propagate the publisher's join to the agent
+    //     side after publishTrack() returns on our side.
+    // Bounded poll (200ms cadence) avoids both timing gaps.
+    async function waitForRoomIOAndParticipant(publisherIdentity, timeoutMs) {
+        const deadline = Date.now() + timeoutMs;
+        let roomIO = null;
+        let participantVisible = false;
+        while (Date.now() < deadline) {
+            roomIO = activeAgentSession?._roomIO;
+            if (roomIO && typeof roomIO.setParticipant === 'function') {
+                const agentRoom = roomIO.rtcRoom;
+                const remotes = agentRoom?.remoteParticipants;
+                if (remotes && typeof remotes.values === 'function') {
+                    for (const p of remotes.values()) {
+                        if (p?.identity === publisherIdentity) {
+                            participantVisible = true;
+                            break;
+                        }
+                    }
+                }
+                if (participantVisible)
+                    return { roomIO, participantVisible };
+            }
+            await new Promise(r => setTimeout(r, 200));
+        }
+        // Timed out — return whatever we have. Caller decides whether to proceed.
+        return { roomIO, participantVisible };
+    }
+    const meetingAudioInWss = new WebSocketServer({ noServer: true });
+    meetingAudioInWss.on('connection', async (recallWs) => {
+        console.log('🎙️ Recall audio-in WebSocket connected — setting up LiveKit publisher');
+        const livekitUrl = process.env.LIVEKIT_URL;
+        const apiKey = process.env.LIVEKIT_API_KEY;
+        const apiSecret = process.env.LIVEKIT_API_SECRET;
+        if (!livekitUrl || !apiKey || !apiSecret) {
+            console.warn('⚠️ LIVEKIT_URL / LIVEKIT_API_KEY / LIVEKIT_API_SECRET not set — meeting audio publisher disabled');
+            recallWs.close();
+            return;
+        }
+        if (!currentRoomCode) {
+            console.warn('⚠️ No active LiveKit room (currentRoomCode null) — meeting audio publisher cannot attach');
+            recallWs.close();
+            return;
+        }
+        const roomName = `osborn-${currentRoomCode}`;
+        // Mint a publisher token via livekit-server-sdk (already imported for
+        // /api/token style flows). Long TTL — meetings can run for hours.
+        const identity = `meeting-audio-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+        const at = new AccessToken(apiKey, apiSecret, {
+            identity,
+            ttl: 14400, // 4 hours
+            metadata: JSON.stringify({ role: 'meeting-audio-publisher' }),
+        });
+        at.addGrant({ roomJoin: true, room: roomName, canPublish: true, canSubscribe: false });
+        const token = await at.toJwt();
+        let room = null;
+        let source = null;
+        let track = null;
+        const cleanup = async () => {
+            // Restore AgentSession STT input to the original user participant before
+            // tearing down the publisher track. If we don't switch back, the session
+            // will be stuck waiting on a participant that's about to disappear.
+            try {
+                const roomIO = activeAgentSession?._roomIO;
+                if (roomIO && typeof roomIO.setParticipant === 'function') {
+                    if (preMeetingUserIdentity) {
+                        roomIO.setParticipant(preMeetingUserIdentity);
+                        console.log(`🔁 Restored AgentSession STT input to user: ${preMeetingUserIdentity}`);
+                    }
+                    else {
+                        roomIO.unsetParticipant();
+                        console.log('🔁 Cleared AgentSession STT input (no original user to restore)');
+                    }
+                }
+            }
+            catch (err) {
+                console.warn('⚠️ Failed to restore RoomIO participant on cleanup:', err.message);
+            }
+            preMeetingUserIdentity = null;
+            try {
+                if (track)
+                    await track.close(true);
+            }
+            catch { }
+            try {
+                if (source)
+                    await source.close();
+            }
+            catch { }
+            try {
+                if (room)
+                    await room.disconnect();
+            }
+            catch { }
+            room = null;
+            source = null;
+            track = null;
+        };
+        try {
+            room = new Room();
+            await room.connect(livekitUrl, token);
+            if (!room.localParticipant)
+                throw new Error('LiveKit connected but localParticipant missing');
+            // Recall sends S16LE PCM at 16kHz mono. AudioSource matches the format.
+            source = new AudioSource(16000, 1);
+            track = LocalAudioTrack.createAudioTrack('meeting-audio', source);
+            await room.localParticipant.publishTrack(track, new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }));
+            console.log(`🎙️ Meeting audio publisher connected to ${roomName} as ${identity}`);
+            // B2 — switch the existing AgentSession's RoomIO input from the local user
+            // to this meeting-audio publisher. While the meeting is active, the user
+            // talks via the meeting (Recall captures it and sends PCM here), and the
+            // agent treats this publisher as the "speaking" participant for STT/EOT.
+            // Original user identity is stashed so cleanup() can restore it.
+            //
+            // 15s timeout accommodates: session-start race (agent still booting when
+            // user clicks "join meeting"), LiveKit participant-join propagation
+            // (~hundreds of ms), and Fly cold-path latency on first request.
+            try {
+                const { roomIO, participantVisible } = await waitForRoomIOAndParticipant(identity, 15000);
+                if (!roomIO) {
+                    console.warn('⚠️ Timed out waiting for AgentSession._roomIO (15s) — meeting audio published but STT not switched. Meeting audio will be ignored until a session starts.');
+                }
+                else if (!participantVisible) {
+                    // RoomIO exists but our publisher hasn't propagated to the agent's
+                    // room view yet. setParticipant stores the identity and links on
+                    // participant-connected event, so this is still safe to call —
+                    // RoomIO will pick up the link when the event arrives.
+                    preMeetingUserIdentity = roomIO.linkedParticipant?.identity ?? null;
+                    roomIO.setParticipant(identity);
+                    console.log(`🔁 Switched AgentSession STT input (publisher not yet visible — will link on connect): ${preMeetingUserIdentity ?? '(none)'} → ${identity}`);
+                }
+                else {
+                    preMeetingUserIdentity = roomIO.linkedParticipant?.identity ?? null;
+                    roomIO.setParticipant(identity);
+                    console.log(`🔁 Switched AgentSession STT input: ${preMeetingUserIdentity ?? '(none)'} → ${identity}`);
+                }
+            }
+            catch (err) {
+                console.warn('⚠️ Failed to switch RoomIO participant:', err.message);
+            }
+        }
+        catch (err) {
+            console.error('❌ Failed to set up LiveKit publisher for meeting audio:', err instanceof Error ? err.message : err);
+            try {
+                recallWs.close();
+            }
+            catch { }
+            await cleanup();
+            return;
+        }
+        // Recall → us: JSON events with base64-encoded PCM. Decode, wrap as
+        // AudioFrame, and capture into the source. AgentSession in the main room
+        // will subscribe to this published track and STT it via the normal pipeline.
+        // Payload shape from
+        // docs.recall.ai/docs/how-to-get-separate-audio-per-participant-realtime:
+        //   { event: 'audio_separate_raw.data', data: { data: { buffer: '<base64>', ... }, participant: {...} } }
+        //
+        // Diagnostic counters so we can tell from prod logs whether (a) Recall is
+        // streaming any frames at all, (b) they're decoding correctly, and (c)
+        // captureFrame is succeeding. Logged every 100 frames (~5s at 50fps).
+        let totalMessages = 0;
+        let audioFrames = 0;
+        let bytesIn = 0;
+        let lastSpeakerSeen;
+        const startTs = Date.now();
+        recallWs.on('message', async (raw) => {
+            totalMessages++;
+            if (!source)
+                return;
+            try {
+                const msg = JSON.parse(raw.toString());
+                if (msg.event !== 'audio_separate_raw.data') {
+                    // First-time event-type diagnostic — log unknown event types once so
+                    // we know if Recall's payload shape changed
+                    if (totalMessages <= 3) {
+                        console.log(`[meeting-audio-in] non-audio event: ${msg.event}`);
+                    }
+                    return;
+                }
+                const b64 = msg.data?.data?.buffer;
+                if (!b64) {
+                    if (audioFrames === 0) {
+                        console.warn(`[meeting-audio-in] first audio event had no buffer field. payload keys=${Object.keys(msg.data?.data ?? {}).join(',')}`);
+                    }
+                    return;
+                }
+                const pcmBuf = Buffer.from(b64, 'base64');
+                bytesIn += pcmBuf.byteLength;
+                const speakerName = msg.data?.data?.participant?.name || msg.data?.participant?.name;
+                if (speakerName && speakerName !== lastSpeakerSeen) {
+                    console.log(`[meeting-audio-in] now hearing: ${speakerName}`);
+                    lastSpeakerSeen = speakerName;
+                }
+                // AudioFrame expects Int16Array. The PCM buffer is S16LE — view it
+                // directly without copy. Length / 2 = samples (each sample 2 bytes).
+                const samplesPerChannel = pcmBuf.byteLength / 2;
+                const int16 = new Int16Array(pcmBuf.buffer, pcmBuf.byteOffset, samplesPerChannel);
+                const frame = new AudioFrame(int16, 16000, 1, samplesPerChannel);
+                await source.captureFrame(frame);
+                audioFrames++;
+                if (audioFrames === 1) {
+                    console.log(`[meeting-audio-in] FIRST audio frame captured (${pcmBuf.byteLength} bytes, ${samplesPerChannel} samples)`);
+                }
+                if (audioFrames % 100 === 0) {
+                    const elapsed = ((Date.now() - startTs) / 1000).toFixed(1);
+                    console.log(`[meeting-audio-in] heartbeat: ${audioFrames} frames, ${(bytesIn / 1024).toFixed(1)} KB in ${elapsed}s (last speaker: ${lastSpeakerSeen ?? 'unknown'})`);
+                }
+            }
+            catch (err) {
+                // Don't log every frame parse failure — could be noisy if Recall sends
+                // non-audio_separate_raw events on the same channel.
+                if (err.message?.includes('JSON'))
+                    return;
+                console.warn('⚠️ meeting audio capture error:', err instanceof Error ? err.message : err);
+            }
+        });
+        recallWs.on('close', async () => {
+            const elapsed = ((Date.now() - startTs) / 1000).toFixed(1);
+            console.log(`🎙️ Recall audio-in WebSocket closed — tearing down LiveKit publisher. Total: ${audioFrames} audio frames / ${totalMessages} messages / ${(bytesIn / 1024).toFixed(1)} KB over ${elapsed}s`);
+            await cleanup();
+        });
+        recallWs.on('error', (err) => {
+            console.warn('⚠️ Recall WS error:', err instanceof Error ? err.message : err);
+        });
+    });
     server.on('upgrade', (req, socket, head) => {
         const url = new URL(req.url || '/', `http://localhost:${port}`);
         if (url.pathname === '/meeting-audio') {
@@ -981,6 +1237,11 @@ function startApiServer(workingDir, port) {
                 meetingOutputWss.emit('connection', ws, req);
             });
         }
+        else if (url.pathname === '/meeting-audio-in') {
+            meetingAudioInWss.handleUpgrade(req, socket, head, (ws) => {
+                meetingAudioInWss.emit('connection', ws, req);
+            });
+        }
         else {
             socket.destroy();
         }
@@ -2630,6 +2891,7 @@ async function main() {
         }
         lastCompletedResearch = null;
         currentSession = null;
+        activeAgentSession = null;
         currentAgent = null;
         // Same disconnect-leak fix as the other two cleanup sites — kill the Claude SDK
         // subprocess BEFORE dropping the reference. See killCurrentLLM() for full context.
@@ -2675,6 +2937,7 @@ async function main() {
             }
             catch { }
             currentSession = null;
+            activeAgentSession = null;
             currentAgent = null;
             // Same disconnect-leak fix — kill the previous user's Claude subprocess
             // before binding currentLLM to the new user's session below.
@@ -2829,6 +3092,7 @@ async function main() {
             agent = result.agent;
         }
         currentSession = session;
+        activeAgentSession = session;
         currentAgent = agent; // Store for updateChatCtx() context injection
         // ============================================================
         // Session event wiring — extracted into function for auto-recovery
@@ -2988,6 +3252,7 @@ async function main() {
                     }
                     catch { }
                     currentSession = null;
+                    activeAgentSession = null;
                     currentAgent = null;
                     // Clear stale state from crashed session
                     voiceQueue.length = 0;
@@ -3049,6 +3314,7 @@ async function main() {
                         const newSession = result.session;
                         const newAgent = result.agent;
                         currentSession = newSession;
+                        activeAgentSession = newSession;
                         currentAgent = newAgent;
                         // Re-wire event listeners on the new session
                         wireSessionEvents(newSession, newAgent);
@@ -3105,6 +3371,7 @@ async function main() {
                     }
                     catch { }
                     currentSession = null;
+                    activeAgentSession = null;
                     currentAgent = null;
                     // Clear voice queue — stale injections from the crashed session
                     voiceQueue.length = 0;
@@ -3128,6 +3395,7 @@ async function main() {
                         const newSession = result.session;
                         const newAgent = result.agent;
                         currentSession = newSession;
+                        activeAgentSession = newSession;
                         currentAgent = newAgent;
                         // Re-wire event listeners on the new session
                         wireSessionEvents(newSession, newAgent);
@@ -3322,6 +3590,7 @@ async function main() {
         if (currentSession) {
             const sessionToClose = currentSession;
             currentSession = null;
+            activeAgentSession = null;
             // Track async close so new connections can wait for byte stream handler to be released
             pendingSessionClose = (async () => {
                 try {

package/dist/recall-client.js

@@ -35,6 +35,23 @@ export class RecallClient extends EventEmitter {
         //   - `url` and `events` are flat on the endpoint object (NOT nested under `config`)
         //   - `transcription_options` does NOT exist — use `transcript.provider`
         //   - Both transcript.provider AND realtime_endpoints must be set, or no events delivered
+        //
+        // ARCHITECTURE (post-2026-05-22 redesign):
+        //   Input (meeting → osborn): Recall's documented WebSocket audio protocol.
+        //     `audio_separate_raw` config + websocket realtime endpoint streams
+        //     per-participant PCM (S16LE 16kHz mono, base64 in JSON) to the agent's
+        //     /meeting-audio-in WS handler. Bot's own audio is excluded by default
+        //     → zero possibility of feedback loop, no echo cancellation needed.
+        //   Output (osborn → meeting): webpage output_media (LiveKit-on-page). Bot
+        //     page subscribes to osborn's LiveKit audio track and plays it via
+        //     track.attach(); Recall captures the page's audio output and injects
+        //     into the meeting.
+        //   Webhook transcripts (transcript.data): retained as a SECONDARY signal —
+        //     the agent index.ts handler for this event currently logs but does NOT
+        //     forward to the LLM (intentionally disabled). The Deepgram WS path
+        //     above is the LLM input.
+        const httpBase = webhookBaseUrl.replace(/\/$/, '');
+        const wsBase = httpBase.replace(/^https?:\/\//, m => m === 'https://' ? 'wss://' : 'ws://');
         const res = await fetch(`${RECALL_BASE_URL}/bot`, {
             method: 'POST',
             headers: {
@@ -49,25 +66,39 @@ export class RecallClient extends EventEmitter {
                         provider: {
                             // recallai_streaming is built-in — no external API key needed,
                             // low-latency, works across all meeting platforms.
+                            // Kept for the secondary webhook signal (display / future use);
+                            // LLM input now comes from the Deepgram WS pipe below.
                             recallai_streaming: {
                                 mode: 'prioritize_low_latency',
                                 language_code: 'en',
                             },
                         },
                     },
-                    realtime_endpoints: [{
+                    // Per-participant raw PCM audio stream. Bot's own audio is excluded
+                    // (we don't set include_bot_in_recording.audio:true).
+                    audio_separate_raw: {},
+                    realtime_endpoints: [
+                        {
+                            // Transcript webhook (secondary signal; LLM forwarding disabled).
                             type: 'webhook',
-                            url: `${webhookBaseUrl}/webhook/recall`,
+                            url: `${httpBase}/webhook/recall`,
                             events: ['transcript.data'],
-                        }],
+                        },
+                        {
+                            // Per-participant PCM audio → agent's Deepgram STT pipe.
+                            type: 'websocket',
+                            url: `${wsBase}/meeting-audio-in`,
+                            events: ['audio_separate_raw.data'],
+                        },
+                    ],
                 },
                 output_media: {
                     camera: {
                         // `kind` (not `type`) — confirmed from prior debugging.
-                        // The page Recall renders is responsible for joining the same LiveKit
-                        // room as the osborn agent: meeting audio captured via getUserMedia is
-                        // published into the room; osborn's TTS audio (already in the room) is
-                        // played by the page and captured by Recall as the bot's mic output.
+                        // The page Recall renders connects to LiveKit and plays osborn's
+                        // TTS audio via track.attach(); Recall captures the page audio.
+                        // The page does NOT call getUserMedia anymore — input now comes
+                        // from the audio_separate_raw WebSocket above.
                         kind: 'webpage',
                         config: {
                             url: outputPageUrl,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.40",
+  "version": "0.9.42",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {