osborn 0.9.39 → 0.9.41

package/dist/index.js

@@ -1,7 +1,7 @@
 // Load environment variables FIRST before any other imports
 import 'dotenv/config';
 import { voice, initializeLogger } from '@livekit/agents';
-import { Room, RoomEvent } from '@livekit/rtc-node';
+import { Room, RoomEvent, AudioSource, AudioFrame, LocalAudioTrack, TrackPublishOptions, TrackSource, } from '@livekit/rtc-node';
 import { AccessToken } from 'livekit-server-sdk';
 // Initialize logger before anything else
 initializeLogger({ pretty: true, level: 'info' });
@@ -149,6 +149,12 @@ process.on('uncaughtException', (error) => {
 let currentRoomCode = null;
 // Meeting output WebSocket — module-level so both startApiServer and main() can access it
 let meetingOutputWs = null;
+// Module-level AgentSession reference so /meeting-audio-in WS handler can switch
+// the RoomIO-linked participant when meeting audio starts/stops (B2 design).
+let activeAgentSession = null;
+// Identity of the local user participant the session was originally listening to
+// — captured at the moment we switch to the meeting publisher, restored on cleanup.
+let preMeetingUserIdentity = null;
 function sendToMeetingOutput(msg) {
     if (meetingOutputWs && meetingOutputWs.readyState === WebSocket.OPEN) {
         try {
@@ -960,20 +966,235 @@ function startApiServer(workingDir, port) {
     cleanStaleUploadDirs();
     setInterval(cleanStaleUploadDirs, 10 * 60 * 1000);
     // ============================================================
-    // Meeting Output WebSocket — /meeting-audio
+    // Meeting Output WebSocket — /meeting-audio (LEGACY)
     // ============================================================
-    // Recall's headless browser opens meeting-output.html which connects here.
-    // We push: JSON { type: 'speak', text } for display, binary PCM for audio (future).
+    // Recall's headless browser used to open meeting-output.html which connects
+    // here. With the new /meeting-bot Next.js page (Phase 2 + LiveKit), Recall
+    // points at frontend/meeting-bot instead — this handler exists only for
+    // backwards-compat with old machine images still serving the legacy path.
     const meetingOutputWss = new WebSocketServer({ noServer: true });
     meetingOutputWss.on('connection', (ws) => {
-        console.log('📺 Meeting output browser connected');
+        console.log('📺 Meeting output browser connected (legacy /meeting-audio)');
         meetingOutputWs = ws;
         ws.on('close', () => {
-            console.log('📺 Meeting output browser disconnected');
+            console.log('📺 Meeting output browser disconnected (legacy)');
             if (meetingOutputWs === ws)
                 meetingOutputWs = null;
         });
     });
+    // ============================================================
+    // Recall.ai meeting-audio-in WebSocket — /meeting-audio-in
+    // ============================================================
+    // Recall.ai's per-participant real-time audio protocol. Bot is configured
+    // (in recall-client.ts joinMeeting) with audio_separate_raw + a realtime
+    // endpoint pointing at this URL. Recall sends JSON events containing
+    // base64-encoded PCM (S16LE, 16kHz, mono) for every meeting participant
+    // (bot's own audio NOT included by default — no feedback loop possible).
+    //
+    // Flow: Recall → /meeting-audio-in → open a SECOND LiveKit connection from
+    //       this agent process as a publisher participant → publish PCM as an
+    //       audio track in the same LiveKit room → the existing AgentSession's
+    //       STT subscribes to it as a remote track → routes to currentLLM.chat()
+    //       via the same pipeline as voice-native user mic.
+    //
+    // The advantage of this design vs a parallel STT pipeline: meeting audio
+    // becomes "just another participant" in the LiveKit room — same end-of-turn
+    // detection, same interrupt handling, same conversation context, no parallel
+    // chat() paths to maintain.
+    //
+    // Wait until activeAgentSession._roomIO exists AND the publisher participant
+    // is visible to the agent's room. Both can race against join_meeting:
+    //   - Agent session may still be starting up when Recall connects.
+    //   - LiveKit takes a moment to propagate the publisher's join to the agent
+    //     side after publishTrack() returns on our side.
+    // Bounded poll (200ms cadence) avoids both timing gaps.
+    async function waitForRoomIOAndParticipant(publisherIdentity, timeoutMs) {
+        const deadline = Date.now() + timeoutMs;
+        let roomIO = null;
+        let participantVisible = false;
+        while (Date.now() < deadline) {
+            roomIO = activeAgentSession?._roomIO;
+            if (roomIO && typeof roomIO.setParticipant === 'function') {
+                const agentRoom = roomIO.rtcRoom;
+                const remotes = agentRoom?.remoteParticipants;
+                if (remotes && typeof remotes.values === 'function') {
+                    for (const p of remotes.values()) {
+                        if (p?.identity === publisherIdentity) {
+                            participantVisible = true;
+                            break;
+                        }
+                    }
+                }
+                if (participantVisible)
+                    return { roomIO, participantVisible };
+            }
+            await new Promise(r => setTimeout(r, 200));
+        }
+        // Timed out — return whatever we have. Caller decides whether to proceed.
+        return { roomIO, participantVisible };
+    }
+    const meetingAudioInWss = new WebSocketServer({ noServer: true });
+    meetingAudioInWss.on('connection', async (recallWs) => {
+        console.log('🎙️ Recall audio-in WebSocket connected — setting up LiveKit publisher');
+        const livekitUrl = process.env.LIVEKIT_URL;
+        const apiKey = process.env.LIVEKIT_API_KEY;
+        const apiSecret = process.env.LIVEKIT_API_SECRET;
+        if (!livekitUrl || !apiKey || !apiSecret) {
+            console.warn('⚠️ LIVEKIT_URL / LIVEKIT_API_KEY / LIVEKIT_API_SECRET not set — meeting audio publisher disabled');
+            recallWs.close();
+            return;
+        }
+        if (!currentRoomCode) {
+            console.warn('⚠️ No active LiveKit room (currentRoomCode null) — meeting audio publisher cannot attach');
+            recallWs.close();
+            return;
+        }
+        const roomName = `osborn-${currentRoomCode}`;
+        // Mint a publisher token via livekit-server-sdk (already imported for
+        // /api/token style flows). Long TTL — meetings can run for hours.
+        const identity = `meeting-audio-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+        const at = new AccessToken(apiKey, apiSecret, {
+            identity,
+            ttl: 14400, // 4 hours
+            metadata: JSON.stringify({ role: 'meeting-audio-publisher' }),
+        });
+        at.addGrant({ roomJoin: true, room: roomName, canPublish: true, canSubscribe: false });
+        const token = await at.toJwt();
+        let room = null;
+        let source = null;
+        let track = null;
+        const cleanup = async () => {
+            // Restore AgentSession STT input to the original user participant before
+            // tearing down the publisher track. If we don't switch back, the session
+            // will be stuck waiting on a participant that's about to disappear.
+            try {
+                const roomIO = activeAgentSession?._roomIO;
+                if (roomIO && typeof roomIO.setParticipant === 'function') {
+                    if (preMeetingUserIdentity) {
+                        roomIO.setParticipant(preMeetingUserIdentity);
+                        console.log(`🔁 Restored AgentSession STT input to user: ${preMeetingUserIdentity}`);
+                    }
+                    else {
+                        roomIO.unsetParticipant();
+                        console.log('🔁 Cleared AgentSession STT input (no original user to restore)');
+                    }
+                }
+            }
+            catch (err) {
+                console.warn('⚠️ Failed to restore RoomIO participant on cleanup:', err.message);
+            }
+            preMeetingUserIdentity = null;
+            try {
+                if (track)
+                    await track.close(true);
+            }
+            catch { }
+            try {
+                if (source)
+                    await source.close();
+            }
+            catch { }
+            try {
+                if (room)
+                    await room.disconnect();
+            }
+            catch { }
+            room = null;
+            source = null;
+            track = null;
+        };
+        try {
+            room = new Room();
+            await room.connect(livekitUrl, token);
+            if (!room.localParticipant)
+                throw new Error('LiveKit connected but localParticipant missing');
+            // Recall sends S16LE PCM at 16kHz mono. AudioSource matches the format.
+            source = new AudioSource(16000, 1);
+            track = LocalAudioTrack.createAudioTrack('meeting-audio', source);
+            await room.localParticipant.publishTrack(track, new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }));
+            console.log(`🎙️ Meeting audio publisher connected to ${roomName} as ${identity}`);
+            // B2 — switch the existing AgentSession's RoomIO input from the local user
+            // to this meeting-audio publisher. While the meeting is active, the user
+            // talks via the meeting (Recall captures it and sends PCM here), and the
+            // agent treats this publisher as the "speaking" participant for STT/EOT.
+            // Original user identity is stashed so cleanup() can restore it.
+            //
+            // 15s timeout accommodates: session-start race (agent still booting when
+            // user clicks "join meeting"), LiveKit participant-join propagation
+            // (~hundreds of ms), and Fly cold-path latency on first request.
+            try {
+                const { roomIO, participantVisible } = await waitForRoomIOAndParticipant(identity, 15000);
+                if (!roomIO) {
+                    console.warn('⚠️ Timed out waiting for AgentSession._roomIO (15s) — meeting audio published but STT not switched. Meeting audio will be ignored until a session starts.');
+                }
+                else if (!participantVisible) {
+                    // RoomIO exists but our publisher hasn't propagated to the agent's
+                    // room view yet. setParticipant stores the identity and links on
+                    // participant-connected event, so this is still safe to call —
+                    // RoomIO will pick up the link when the event arrives.
+                    preMeetingUserIdentity = roomIO.linkedParticipant?.identity ?? null;
+                    roomIO.setParticipant(identity);
+                    console.log(`🔁 Switched AgentSession STT input (publisher not yet visible — will link on connect): ${preMeetingUserIdentity ?? '(none)'} → ${identity}`);
+                }
+                else {
+                    preMeetingUserIdentity = roomIO.linkedParticipant?.identity ?? null;
+                    roomIO.setParticipant(identity);
+                    console.log(`🔁 Switched AgentSession STT input: ${preMeetingUserIdentity ?? '(none)'} → ${identity}`);
+                }
+            }
+            catch (err) {
+                console.warn('⚠️ Failed to switch RoomIO participant:', err.message);
+            }
+        }
+        catch (err) {
+            console.error('❌ Failed to set up LiveKit publisher for meeting audio:', err instanceof Error ? err.message : err);
+            try {
+                recallWs.close();
+            }
+            catch { }
+            await cleanup();
+            return;
+        }
+        // Recall → us: JSON events with base64-encoded PCM. Decode, wrap as
+        // AudioFrame, and capture into the source. AgentSession in the main room
+        // will subscribe to this published track and STT it via the normal pipeline.
+        // Payload shape from
+        // docs.recall.ai/docs/how-to-get-separate-audio-per-participant-realtime:
+        //   { event: 'audio_separate_raw.data', data: { data: { buffer: '<base64>', ... }, participant: {...} } }
+        recallWs.on('message', async (raw) => {
+            if (!source)
+                return;
+            try {
+                const msg = JSON.parse(raw.toString());
+                if (msg.event !== 'audio_separate_raw.data')
+                    return;
+                const b64 = msg.data?.data?.buffer;
+                if (!b64)
+                    return;
+                const pcmBuf = Buffer.from(b64, 'base64');
+                // AudioFrame expects Int16Array. The PCM buffer is S16LE — view it
+                // directly without copy. Length / 2 = samples (each sample 2 bytes).
+                const samplesPerChannel = pcmBuf.byteLength / 2;
+                const int16 = new Int16Array(pcmBuf.buffer, pcmBuf.byteOffset, samplesPerChannel);
+                const frame = new AudioFrame(int16, 16000, 1, samplesPerChannel);
+                await source.captureFrame(frame);
+            }
+            catch (err) {
+                // Don't log every frame parse failure — could be noisy if Recall sends
+                // non-audio_separate_raw events on the same channel.
+                if (err.message?.includes('JSON'))
+                    return;
+                console.warn('⚠️ meeting audio capture error:', err instanceof Error ? err.message : err);
+            }
+        });
+        recallWs.on('close', async () => {
+            console.log('🎙️ Recall audio-in WebSocket closed — tearing down LiveKit publisher');
+            await cleanup();
+        });
+        recallWs.on('error', (err) => {
+            console.warn('⚠️ Recall WS error:', err instanceof Error ? err.message : err);
+        });
+    });
     server.on('upgrade', (req, socket, head) => {
         const url = new URL(req.url || '/', `http://localhost:${port}`);
         if (url.pathname === '/meeting-audio') {
@@ -981,6 +1202,11 @@ function startApiServer(workingDir, port) {
                 meetingOutputWss.emit('connection', ws, req);
             });
         }
+        else if (url.pathname === '/meeting-audio-in') {
+            meetingAudioInWss.handleUpgrade(req, socket, head, (ws) => {
+                meetingAudioInWss.emit('connection', ws, req);
+            });
+        }
         else {
             socket.destroy();
         }
@@ -1224,28 +1450,46 @@ async function main() {
     let lastCompletedResearch = null;
     // No manual queuing — the Claude SDK handles sequential queries internally
     // ============================================================
-    // Recall.ai — Meeting Transcript Routing
+    // Recall.ai — Meeting Transcript Listener
     // ============================================================
+    // NOTE: LLM-forwarding via Recall webhook STT was DISABLED in the Phase 2
+    // LiveKit-based meeting-bot migration. Reason: Recall sends transcripts as
+    // sentence-level fragments (e.g. "transcript.data" events fire ~once per
+    // sentence). The old code below called currentLLM.chat() PER FRAGMENT, which
+    // meant the agent fired ~10 chat() calls during a single user utterance —
+    // each one prompting a separate response. The agent ended up speaking over
+    // itself answering partial fragments.
+    //
+    // Phase 2 routes meeting audio through LiveKit instead (see
+    // frontend/src/app/meeting-bot/page.tsx). The agent's existing Deepgram Flux
+    // STT processes that audio via end-of-turn detection — ONE chat() call per
+    // actual completed utterance, no fragment storms.
+    //
+    // We keep the listener registered so we have a hook for future work (e.g.
+    // forwarding the live transcript to the frontend chat panel as a read-only
+    // "what was said in the meeting" display, separate from the LLM input path).
     const recall = getRecallClient();
     if (recall) {
-        console.log('🎥 Recall.ai client initialized (RECALL_API_KEY present)');
+        console.log('🎥 Recall.ai client initialized (webhook STT receiver — LLM forwarding disabled, see meeting-bot Phase 2)');
         recall.on('transcript', ({ botId, speaker, text }) => {
             console.log(`📝 Meeting transcript [${speaker}]: ${text}`);
-            // Route meeting transcripts to Claude as user text with speaker attribution
-            if (currentLLM && currentSession) {
-                const meetingText = `[Meeting — ${speaker}]: ${text}`;
-                // Use the same pipeline as user_text data channel messages
-                try {
-                    if (currentVoiceMode === 'pipeline' || currentVoiceMode === 'direct') {
-                        const chatCtx = new llm.ChatContext();
-                        chatCtx.addMessage({ role: 'user', content: meetingText });
-                        currentLLM.chat({ chatCtx });
-                    }
-                }
-                catch (err) {
-                    console.error('❌ Failed to route meeting transcript:', err);
-                }
-            }
+            // INTENTIONALLY DISABLED — see comment above. Audio path is now LiveKit
+            // → meeting-bot page publishes meeting audio → agent STT processes it.
+            // The line below is preserved as a reference for future re-enablement
+            // (e.g. as a display-only feature, NOT as LLM input).
+            //
+            // if (currentLLM && currentSession) {
+            //   const meetingText = `[Meeting — ${speaker}]: ${text}`
+            //   try {
+            //     if (currentVoiceMode === 'pipeline' || currentVoiceMode === 'direct') {
+            //       const chatCtx = new llm.ChatContext()
+            //       chatCtx.addMessage({ role: 'user', content: meetingText })
+            //       ;(currentLLM as any).chat({ chatCtx })
+            //     }
+            //   } catch (err) {
+            //     console.error('❌ Failed to route meeting transcript:', err)
+            //   }
+            // }
         });
     }
     // ============================================================
@@ -1656,9 +1900,34 @@ async function main() {
             skipTTSQueue: true,
             onCompactionEvent: (event) => {
                 try {
-                    // Forward every field — frontend renders stage + detail + skill list during compaction.
-                    // Spread covers compaction_started/progress/complete (different fields per type).
+                    // Forward the raw event so the dedicated banner UI can render it (if/when fixed).
                     sendToFrontend({ ...event });
+                    // ALSO emit as a claude_output chat bubble — reuses the existing message path
+                    // that's already working end-to-end. PreCompact → in-progress bubble.
+                    // PostCompact → completion bubble with the skills summary. The dedicated
+                    // banner has been unreliable in production (data path works on backend, banner
+                    // never appears on iPad/iPhone where dev tools aren't accessible). Chat bubbles
+                    // are visible without dev tools.
+                    if (event.type === 'compaction_started') {
+                        const triggerLabel = event.trigger ? ` (${event.trigger})` : '';
+                        sendToFrontend({
+                            type: 'claude_output',
+                            text: `🧠 _Crystallizing session memory…_${triggerLabel}`,
+                            agentRole: 'direct',
+                        });
+                    }
+                    else if (event.type === 'compaction_complete') {
+                        const ev = event;
+                        const n = ev.skillsWritten ?? 0;
+                        const names = Array.isArray(ev.skillNames) && ev.skillNames.length > 0
+                            ? ` — ${ev.skillNames.join(', ')}`
+                            : '';
+                        sendToFrontend({
+                            type: 'claude_output',
+                            text: `🧠 Memory crystallized — ${n} skill${n === 1 ? '' : 's'} updated${names}.`,
+                            agentRole: 'direct',
+                        });
+                    }
                 }
                 catch { /* non-fatal */ }
             },
@@ -1862,14 +2131,17 @@ async function main() {
             const sayId = Date.now(); // simple ID to correlate start/end logs
             console.log(`🗣️ [${sayId}] session.say START (${data.text.length} chars): "${data.text}"`);
             // Forward spoken text + audio to meeting output page when bot is in a meeting.
-            // Text appears immediately; audio uses the same configured TTS (directConfig.tts)
-            // so voice/provider stays consistent — no separate hardcoded provider.
+            // Uses DIRECT_MODE_TTS (same OpenAI fable voice as the live session) — was
+            // previously using directConfig.tts which falls back to DEFAULT_CONFIG.direct.tts
+            // (Deepgram aura-2-asteria-en) when no user config exists, producing a different
+            // voice in the meeting than what the user hears in voice-native. Both paths now
+            // share the single source of truth.
             // PCM frames are WAV-encoded and pushed as binary WebSocket frames.
             // Recall captures the browser page's audio output and injects it into the meeting.
             if (activeMeetingBotId) {
                 sendToMeetingOutput({ type: 'speak', text: data.text });
                 if (meetingOutputWs) {
-                    synthesizeForMeeting(data.text, directConfig.tts).catch((err) => console.warn('⚠️ Meeting TTS error:', err));
+                    synthesizeForMeeting(data.text, DIRECT_MODE_TTS).catch((err) => console.warn('⚠️ Meeting TTS error:', err));
                 }
             }
             try {
@@ -2011,9 +2283,34 @@ async function main() {
             resumeSessionId,
             onCompactionEvent: (event) => {
                 try {
-                    // Forward every field — frontend renders stage + detail + skill list during compaction.
-                    // Spread covers compaction_started/progress/complete (different fields per type).
+                    // Forward the raw event so the dedicated banner UI can render it (if/when fixed).
                     sendToFrontend({ ...event });
+                    // ALSO emit as a claude_output chat bubble — reuses the existing message path
+                    // that's already working end-to-end. PreCompact → in-progress bubble.
+                    // PostCompact → completion bubble with the skills summary. The dedicated
+                    // banner has been unreliable in production (data path works on backend, banner
+                    // never appears on iPad/iPhone where dev tools aren't accessible). Chat bubbles
+                    // are visible without dev tools.
+                    if (event.type === 'compaction_started') {
+                        const triggerLabel = event.trigger ? ` (${event.trigger})` : '';
+                        sendToFrontend({
+                            type: 'claude_output',
+                            text: `🧠 _Crystallizing session memory…_${triggerLabel}`,
+                            agentRole: 'direct',
+                        });
+                    }
+                    else if (event.type === 'compaction_complete') {
+                        const ev = event;
+                        const n = ev.skillsWritten ?? 0;
+                        const names = Array.isArray(ev.skillNames) && ev.skillNames.length > 0
+                            ? ` — ${ev.skillNames.join(', ')}`
+                            : '';
+                        sendToFrontend({
+                            type: 'claude_output',
+                            text: `🧠 Memory crystallized — ${n} skill${n === 1 ? '' : 's'} updated${names}.`,
+                            agentRole: 'direct',
+                        });
+                    }
                 }
                 catch { /* non-fatal */ }
             },
@@ -2559,6 +2856,7 @@ async function main() {
         }
         lastCompletedResearch = null;
         currentSession = null;
+        activeAgentSession = null;
         currentAgent = null;
         // Same disconnect-leak fix as the other two cleanup sites — kill the Claude SDK
         // subprocess BEFORE dropping the reference. See killCurrentLLM() for full context.
@@ -2604,6 +2902,7 @@ async function main() {
             }
             catch { }
             currentSession = null;
+            activeAgentSession = null;
             currentAgent = null;
             // Same disconnect-leak fix — kill the previous user's Claude subprocess
             // before binding currentLLM to the new user's session below.
@@ -2758,6 +3057,7 @@ async function main() {
             agent = result.agent;
         }
         currentSession = session;
+        activeAgentSession = session;
         currentAgent = agent; // Store for updateChatCtx() context injection
         // ============================================================
         // Session event wiring — extracted into function for auto-recovery
@@ -2917,6 +3217,7 @@ async function main() {
                     }
                     catch { }
                     currentSession = null;
+                    activeAgentSession = null;
                     currentAgent = null;
                     // Clear stale state from crashed session
                     voiceQueue.length = 0;
@@ -2978,6 +3279,7 @@ async function main() {
                         const newSession = result.session;
                         const newAgent = result.agent;
                         currentSession = newSession;
+                        activeAgentSession = newSession;
                         currentAgent = newAgent;
                         // Re-wire event listeners on the new session
                         wireSessionEvents(newSession, newAgent);
@@ -3034,6 +3336,7 @@ async function main() {
                     }
                     catch { }
                     currentSession = null;
+                    activeAgentSession = null;
                     currentAgent = null;
                     // Clear voice queue — stale injections from the crashed session
                     voiceQueue.length = 0;
@@ -3057,6 +3360,7 @@ async function main() {
                         const newSession = result.session;
                         const newAgent = result.agent;
                         currentSession = newSession;
+                        activeAgentSession = newSession;
                         currentAgent = newAgent;
                         // Re-wire event listeners on the new session
                         wireSessionEvents(newSession, newAgent);
@@ -3251,6 +3555,7 @@ async function main() {
         if (currentSession) {
             const sessionToClose = currentSession;
             currentSession = null;
+            activeAgentSession = null;
             // Track async close so new connections can wait for byte stream handler to be released
             pendingSessionClose = (async () => {
                 try {
@@ -3855,8 +4160,57 @@ async function main() {
                                 (process.env.FLY_APP_NAME
                                     ? `https://${process.env.FLY_APP_NAME}.fly.dev`
                                     : `http://localhost:${apiPort}`);
+                            // Try to mint a LiveKit bot token + construct the frontend-hosted
+                            // meeting-bot page URL. The bot page joins the same LiveKit room
+                            // as this agent so meeting audio flows through LiveKit directly
+                            // (no agent-side WebSocket+WAV pipe). Falls back to the legacy
+                            // /meeting-output webpage if no frontend URL is resolvable, so
+                            // the old code path keeps working during the migration window.
+                            //
+                            // Frontend URL resolution (in priority order):
+                            //   1. data.frontendBase — the public URL the user's browser is on,
+                            //      passed through the join_meeting data channel message. Works
+                            //      automatically for localhost dev + production without any
+                            //      env var.
+                            //   2. OSBORN_FRONTEND_URL — existing convention from sprites.ts
+                            //      (frontend/src/lib/sprites.ts:241) that injects the public
+                            //      frontend URL into sandbox env vars. Defense in depth.
+                            //
+                            // Auth: the endpoint uses LiveKit room-presence as the auth check
+                            // — no shared secret needed. The agent must already be in the
+                            // requested room (which it is by this point) for the mint to
+                            // succeed.
+                            let outputPageUrl;
+                            const frontendUrl = data.frontendBase
+                                || process.env.OSBORN_FRONTEND_URL;
+                            if (frontendUrl) {
+                                try {
+                                    const botLkId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+                                    const tokenRes = await fetch(`${frontendUrl}/api/meeting-bot-token`, {
+                                        method: 'POST',
+                                        headers: { 'Content-Type': 'application/json' },
+                                        body: JSON.stringify({ botId: botLkId, roomName }),
+                                    });
+                                    if (tokenRes.ok) {
+                                        const { token, url } = await tokenRes.json();
+                                        const params = new URLSearchParams({ token, url, room: roomName, botId: botLkId });
+                                        outputPageUrl = `${frontendUrl}/meeting-bot?${params.toString()}`;
+                                        console.log(`🎫 Meeting-bot token minted for room=${roomName} bot=${botLkId}`);
+                                    }
+                                    else {
+                                        const errText = await tokenRes.text().catch(() => '');
+                                        console.warn(`⚠️ meeting-bot-token mint failed (HTTP ${tokenRes.status}: ${errText.substring(0, 120)}) — falling back to legacy /meeting-output path`);
+                                    }
+                                }
+                                catch (mintErr) {
+                                    console.warn(`⚠️ meeting-bot-token mint threw — falling back: ${mintErr.message}`);
+                                }
+                            }
+                            else {
+                                console.log('ℹ️ No frontend URL (data.frontendBase + OSBORN_FRONTEND_URL both empty) — using legacy /meeting-output path');
+                            }
                             await sendToFrontend({ type: 'meeting_joining', message: 'Osborn is joining your meeting...' });
-                            const botId = await recallJoin.joinMeeting(meetingUrl, webhookBase);
+                            const botId = await recallJoin.joinMeeting(meetingUrl, webhookBase, { outputPageUrl });
                             const sessionId = currentLLM?.sessionId || currentResumeSessionId || 'default';
                             recallJoin.registerBot(botId, sessionId);
                             activeMeetingBotId = botId;

package/dist/recall-client.d.ts

@@ -36,7 +36,23 @@ export interface TranscriptPayload {
 export declare class RecallClient extends EventEmitter {
     #private;
     constructor(apiKey: string);
-    joinMeeting(meetingUrl: string, webhookBaseUrl: string, botName?: string): Promise<string>;
+    /**
+     * Join a meeting via Recall.ai.
+     *
+     * @param meetingUrl       Zoom / Google Meet / Teams URL the bot should dial in to
+     * @param webhookBaseUrl   Base URL for the agent's HTTP endpoints (transcript webhook)
+     * @param opts.outputPageUrl  Full URL for the bot's camera/audio page. If provided,
+     *                            replaces the default `${webhookBaseUrl}/meeting-output`.
+     *                            Used to point at the frontend-hosted /meeting-bot page
+     *                            with token + room embedded as query params, so the page
+     *                            connects to LiveKit and audio flows through the same
+     *                            room as the osborn agent (no separate WebSocket+WAV pipe).
+     * @param opts.botName     Display name of the bot in the meeting
+     */
+    joinMeeting(meetingUrl: string, webhookBaseUrl: string, opts?: {
+        outputPageUrl?: string;
+        botName?: string;
+    }): Promise<string>;
     leaveMeeting(botId: string): Promise<void>;
     getBotStatus(botId: string): Promise<string>;
     handleWebhook(payload: TranscriptPayload): void;

package/dist/recall-client.js

@@ -8,7 +8,22 @@ export class RecallClient extends EventEmitter {
         super();
         this.#apiKey = apiKey;
     }
-    async joinMeeting(meetingUrl, webhookBaseUrl, botName = 'Osborn') {
+    /**
+     * Join a meeting via Recall.ai.
+     *
+     * @param meetingUrl       Zoom / Google Meet / Teams URL the bot should dial in to
+     * @param webhookBaseUrl   Base URL for the agent's HTTP endpoints (transcript webhook)
+     * @param opts.outputPageUrl  Full URL for the bot's camera/audio page. If provided,
+     *                            replaces the default `${webhookBaseUrl}/meeting-output`.
+     *                            Used to point at the frontend-hosted /meeting-bot page
+     *                            with token + room embedded as query params, so the page
+     *                            connects to LiveKit and audio flows through the same
+     *                            room as the osborn agent (no separate WebSocket+WAV pipe).
+     * @param opts.botName     Display name of the bot in the meeting
+     */
+    async joinMeeting(meetingUrl, webhookBaseUrl, opts) {
+        const botName = opts?.botName ?? 'Osborn';
+        const outputPageUrl = opts?.outputPageUrl ?? `${webhookBaseUrl}/meeting-output`;
         // Authoritative structure per https://docs.recall.ai/reference/bot_create
         // and https://docs.recall.ai/docs/real-time-transcription:
         //
@@ -20,6 +35,23 @@ export class RecallClient extends EventEmitter {
         //   - `url` and `events` are flat on the endpoint object (NOT nested under `config`)
         //   - `transcription_options` does NOT exist — use `transcript.provider`
         //   - Both transcript.provider AND realtime_endpoints must be set, or no events delivered
+        //
+        // ARCHITECTURE (post-2026-05-22 redesign):
+        //   Input (meeting → osborn): Recall's documented WebSocket audio protocol.
+        //     `audio_separate_raw` config + websocket realtime endpoint streams
+        //     per-participant PCM (S16LE 16kHz mono, base64 in JSON) to the agent's
+        //     /meeting-audio-in WS handler. Bot's own audio is excluded by default
+        //     → zero possibility of feedback loop, no echo cancellation needed.
+        //   Output (osborn → meeting): webpage output_media (LiveKit-on-page). Bot
+        //     page subscribes to osborn's LiveKit audio track and plays it via
+        //     track.attach(); Recall captures the page's audio output and injects
+        //     into the meeting.
+        //   Webhook transcripts (transcript.data): retained as a SECONDARY signal —
+        //     the agent index.ts handler for this event currently logs but does NOT
+        //     forward to the LLM (intentionally disabled). The Deepgram WS path
+        //     above is the LLM input.
+        const httpBase = webhookBaseUrl.replace(/\/$/, '');
+        const wsBase = httpBase.replace(/^https?:\/\//, m => m === 'https://' ? 'wss://' : 'ws://');
         const res = await fetch(`${RECALL_BASE_URL}/bot`, {
             method: 'POST',
             headers: {
@@ -34,25 +66,42 @@ export class RecallClient extends EventEmitter {
                         provider: {
                             // recallai_streaming is built-in — no external API key needed,
                             // low-latency, works across all meeting platforms.
+                            // Kept for the secondary webhook signal (display / future use);
+                            // LLM input now comes from the Deepgram WS pipe below.
                             recallai_streaming: {
                                 mode: 'prioritize_low_latency',
                                 language_code: 'en',
                             },
                         },
                     },
-                    realtime_endpoints: [{
+                    // Per-participant raw PCM audio stream. Bot's own audio is excluded
+                    // (we don't set include_bot_in_recording.audio:true).
+                    audio_separate_raw: {},
+                    realtime_endpoints: [
+                        {
+                            // Transcript webhook (secondary signal; LLM forwarding disabled).
                             type: 'webhook',
-                            url: `${webhookBaseUrl}/webhook/recall`,
+                            url: `${httpBase}/webhook/recall`,
                             events: ['transcript.data'],
-                        }],
+                        },
+                        {
+                            // Per-participant PCM audio → agent's Deepgram STT pipe.
+                            type: 'websocket',
+                            url: `${wsBase}/meeting-audio-in`,
+                            events: ['audio_separate_raw.data'],
+                        },
+                    ],
                 },
                 output_media: {
                     camera: {
                         // `kind` (not `type`) — confirmed from prior debugging.
-                        // Output webpage plays TTS audio so meeting participants can hear the agent.
+                        // The page Recall renders connects to LiveKit and plays osborn's
+                        // TTS audio via track.attach(); Recall captures the page audio.
+                        // The page does NOT call getUserMedia anymore — input now comes
+                        // from the audio_separate_raw WebSocket above.
                         kind: 'webpage',
                         config: {
-                            url: `${webhookBaseUrl}/meeting-output`,
+                            url: outputPageUrl,
                         },
                     },
                 },
@@ -63,7 +112,7 @@ export class RecallClient extends EventEmitter {
             throw new Error(`Recall.ai join failed: ${res.status} ${err}`);
         }
         const bot = (await res.json());
-        console.log(`🤖 Recall.ai bot joined meeting: ${bot.id}`);
+        console.log(`🤖 Recall.ai bot joined meeting: ${bot.id} (output page: ${outputPageUrl})`);
         return bot.id;
     }
     async leaveMeeting(botId) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.39",
+  "version": "0.9.41",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {