osborn 0.9.112 → 0.9.114

package/dist/index.js

@@ -192,6 +192,11 @@ const livekitState = {
 // connect-with-retry loop) so the module-level HTTP server can drive them.
 let intentionalLeave = false;
 let connectRoomHook = null;
+// Mints a LISTEN-ONLY LiveKit token for the meeting canvas so it can join the
+// agent's room and play the agent's REAL TTS audio track — bit-identical to
+// the browser voice experience (2026-08-01 quality architecture; replaces the
+// synth-file chain when connected). Set from main() where the room name lives.
+let canvasTokenHook = null;
 let leaveRoomHook = null;
 // Hook for the bug-reporter skill. The /report-bug HTTP endpoint validates the
 // payload + generates the reportId in the module-level handler, then delegates
@@ -230,6 +235,62 @@ let meetingAgentSpeakingText = '';
 // Prepended to the next flush: what the bot was cut off saying + who interrupted
 // (same pattern as voice-native interruptions).
 let meetingInterruptContext = '';
+// Synthesize speech as MP3 (Deepgram fast path, OpenAI fallback) — for
+// Recall native output_audio, which requires mp3.
+async function synthMp3(text) {
+    const t0 = Date.now();
+    const dgKey = process.env.DEEPGRAM_API_KEY;
+    if (dgKey) {
+        try {
+            const dg = await fetch('https://api.deepgram.com/v1/speak?model=aura-2-thalia-en&encoding=mp3&bit_rate=48000', {
+                method: 'POST',
+                headers: { 'Authorization': `Token ${dgKey}`, 'Content-Type': 'application/json' },
+                body: JSON.stringify({ text: text.slice(0, 4000) }),
+                signal: AbortSignal.timeout(12000),
+            });
+            if (dg.ok) {
+                const buf = Buffer.from(await dg.arrayBuffer());
+                console.log(`🗣️ synthMp3 deepgram ${buf.length}b in ${Date.now() - t0}ms`);
+                return buf;
+            }
+        }
+        catch (e) {
+            console.warn(`⚠️ synthMp3 deepgram: ${e.message}`);
+        }
+    }
+    const oa = process.env.OPENAI_API_KEY;
+    if (oa) {
+        try {
+            const r = await fetch('https://api.openai.com/v1/audio/speech', {
+                method: 'POST',
+                headers: { 'Authorization': `Bearer ${oa}`, 'Content-Type': 'application/json' },
+                body: JSON.stringify({ model: 'gpt-4o-mini-tts', voice: 'alloy', input: text.slice(0, 4000), response_format: 'mp3' }),
+                signal: AbortSignal.timeout(15000),
+            });
+            if (r.ok)
+                return Buffer.from(await r.arrayBuffer());
+        }
+        catch { /* fall through */ }
+    }
+    return null;
+}
+// THE speak path for meetings (2026-08-01): Recall native output_audio FIRST
+// (direct, loud, no canvas capture chain), canvas Web-Audio say as FALLBACK.
+async function speakIntoMeeting(text) {
+    const recall = getRecallClient();
+    const botId = recall?.getActiveBotIds?.()[0];
+    if (recall && botId) {
+        const mp3 = await synthMp3(text);
+        if (mp3 && await recall.outputAudio(botId, mp3)) {
+            console.log(`📢 spoke via Recall output_audio (${mp3.length}b): "${text.slice(0, 60)}"`);
+            markMeetingSpeaking(text);
+            return;
+        }
+    }
+    console.log(`📽️ falling back to canvas say: "${text.slice(0, 50)}"`);
+    pushCanvas({ kind: 'say', text });
+    markMeetingSpeaking(text);
+}
 function markMeetingSpeaking(text) {
     meetingAgentSpeaking = true;
     meetingAgentSpeakingText = text;
@@ -286,6 +347,18 @@ function startApiServer(workingDir, port) {
             }
             return;
         }
+        if (req.method === 'GET' && url.pathname === '/canvas-token') {
+            // Listen-only LiveKit credentials for the meeting canvas (see hook doc).
+            const out = canvasTokenHook ? await canvasTokenHook() : null;
+            if (!out) {
+                res.writeHead(503, { 'Content-Type': 'application/json' });
+                res.end(JSON.stringify({ error: 'no active room' }));
+                return;
+            }
+            res.writeHead(200, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify(out));
+            return;
+        }
         if (req.method === 'GET' && url.pathname === '/skills') {
             // Installed skills — same list the chat's get_skills data-channel message
             // returns, exposed over HTTP so the DASHBOARD (no LiveKit connection) can
@@ -644,10 +717,13 @@ function startApiServer(workingDir, port) {
                     const evt = JSON.parse(body || '{}');
                     if (evt.kind !== 'say' && evt.kind !== 'show' && evt.kind !== 'stop')
                         throw new Error("kind must be 'say', 'show', or 'stop'");
-                    pushCanvas(evt);
-                    // Track that the bot is now speaking into the meeting → enables interruption.
-                    if (evt.kind === 'say')
-                        markMeetingSpeaking(evt.text);
+                    if (evt.kind === 'say') {
+                        // Native-first speak path (Recall output_audio → canvas fallback).
+                        void speakIntoMeeting(evt.text);
+                    }
+                    else {
+                        pushCanvas(evt);
+                    }
                     res.writeHead(200, { 'Content-Type': 'application/json' });
                     res.end(JSON.stringify({ ok: true, clients: canvasClients.size }));
                 }
@@ -1656,6 +1732,10 @@ async function main() {
     // REDIRECTED into the meeting canvas instead of suppressed — the regular
     // voice pipeline reused with the meeting as the sink (latency fix).
     let meetingAddressedUntil = 0;
+    // True while the meeting-canvas page is connected to the LiveKit room as a
+    // listener (browser-parity audio path). When set, meeting replies use the
+    // NORMAL session.say pipeline — no suppression, no redirect, no synth chain.
+    let meetingCanvasInRoom = false;
     // Distinct human speakers seen in the current meeting. In a 1:1 (one human
     // + the bot) EVERY utterance is addressed to the bot — no name needed
     // (2026-08-01: "can you open carfax..." got observer-suppressed because
@@ -1870,6 +1950,16 @@ async function main() {
                 // mis-transcriptions; mild false-positive risk is acceptable in a
                 // room that invited the bot. AND: in a 1:1 meeting (one human + the
                 // bot) every utterance is addressed — no name needed.
+                // Spoken mode switch: "interactive mode"/"go interactive" opens the
+                // conversation latch; "observer mode"/"go silent" closes it.
+                if (/\b(interactive mode|go interactive|start responding|talk to (me|us))\b/i.test(text)) {
+                    meetingAddressedUntil = Date.now() + 6 * 60 * 60 * 1000;
+                    void speakIntoMeeting('Interactive mode on — I will respond out loud.');
+                }
+                else if (/\b(observer mode|silent mode|go silent|stop responding)\b/i.test(text)) {
+                    meetingAddressedUntil = 0;
+                    void speakIntoMeeting('Going silent — just taking notes. Say interactive mode to bring me back.');
+                }
                 const oneOnOne = meetingSpeakers.size <= 1;
                 if (oneOnOne || /\b(osborne?|oz\s?born|os\s?born|was born|is born|ozborn|osbourne?|austin\b.{0,8}(hear|there|can you))/i.test(text)) {
                     console.log(`📓 Addressed (${oneOnOne ? '1:1 meeting' : 'by name'}) — immediate flush for a response`);
@@ -2563,6 +2653,16 @@ async function main() {
             // re-captures in the same room → feedback). Set by PipelineDirectLLM.chat()
             // when the turn is a [MEETING —] chunk. Normal user turns are unaffected.
             if (directLLM.suppressMeetingTTS) {
+                // BROWSER-PARITY PATH: canvas is in the LiveKit room → the normal
+                // session.say audio reaches the meeting through it. Fall through to
+                // the regular pipeline for addressed turns (identical audio to the
+                // browser experience); observer turns still stay silent.
+                if (meetingCanvasInRoom && activeMeetingBotId && Date.now() < meetingAddressedUntil) {
+                    console.log(`🔊🎼 meeting reply via NATIVE session.say (canvas relays): "${data.text.slice(0, 50)}"`);
+                    markMeetingSpeaking(data.text);
+                    // no return — normal say proceeds below
+                }
+                else 
                 // ADDRESSED turn → REDIRECT the normal streaming reply into the meeting
                 // (reuse of the regular speak path; no Bash roundtrip). Silent-observer
                 // turns stay suppressed as before.
@@ -2572,9 +2672,8 @@ async function main() {
                     // audio file = zero seams — quality over ~1s of latency. (Structural
                     // fix queued: canvas as LiveKit subscriber playing the agent's real
                     // TTS track — the true "same browser experience" reuse.)
-                    console.log(`🔊➡️📽️ tts_say → canvas (single utterance) t=${new Date().toISOString()}: "${data.text.slice(0, 60)}"`);
-                    pushCanvas({ kind: 'say', text: data.text });
-                    markMeetingSpeaking(data.text);
+                    console.log(`🔊➡️📢 tts_say → meeting (native audio) t=${new Date().toISOString()}: "${data.text.slice(0, 60)}"`);
+                    void speakIntoMeeting(data.text);
                     return;
                 }
                 console.log(`🔇 tts_say suppressed (meeting turn — response goes to /canvas, not browser): "${data.text.slice(0, 60)}"`);
@@ -3510,6 +3609,14 @@ async function main() {
     // connect callback, not via events).
     participantConnectedHandler = async (participant) => {
         console.log(`\n👤 User joined: ${participant.identity}`);
+        // The meeting canvas joining as a listener = browser-parity audio path is
+        // LIVE: the agent's normal session.say plays through the canvas into the
+        // meeting, so tts_say suppression/redirect must stand down.
+        if (participant.identity === 'meeting-canvas') {
+            meetingCanvasInRoom = true;
+            console.log('📽️🔊 meeting-canvas joined the LiveKit room — native session.say relays to the meeting');
+            return; // not a user; skip session setup for it
+        }
         // A user (re)arriving cancels the meeting leave-grace — the bot stays.
         cancelMeetingLeaveGrace();
         // A user is present — cancel any pending agent-side "alone" leave.
@@ -5413,6 +5520,17 @@ async function main() {
     // { roomName } (which connectRoomHook resolves to). During rollout, the legacy
     // GET /room-code endpoint keeps returning the last-created room NAME so old
     // frontends still function.
+    canvasTokenHook = async () => {
+        if (!activeRoomName)
+            return null;
+        const t = new AccessToken(apiKey, apiSecret, {
+            identity: 'meeting-canvas',
+            name: 'Meeting Canvas',
+            metadata: JSON.stringify({ type: 'canvas' }),
+        });
+        t.addGrant({ roomJoin: true, room: activeRoomName, canPublish: false, canSubscribe: true, canPublishData: false });
+        return { token: await t.toJwt(), url: process.env.LIVEKIT_URL || '', room: activeRoomName };
+    };
     connectRoomHook = async () => {
         intentionalLeave = false;
         cancelIdleExitTimer();

package/dist/recall-client.d.ts

@@ -103,6 +103,13 @@ export declare class RecallClient extends EventEmitter {
      */
     getTranscript(botId: string): Promise<TranscriptTurn[]>;
     leaveMeeting(botId: string): Promise<void>;
+    /**
+     * Speak DIRECTLY through the bot via Recall's native output_audio (mp3).
+     * No canvas page, no Web Audio, no capture chain — Recall plays the file as
+     * the bot's voice. Returns true on 2xx. (2026-08-01: canvas-audio quality
+     * was "barely bearable"; this is the direct loud path.)
+     */
+    outputAudio(botId: string, mp3: Buffer): Promise<boolean>;
     getBotStatus(botId: string): Promise<string>;
     handleWebhook(payload: TranscriptPayload): void;
     registerBot(botId: string, sessionId: string): void;

package/dist/recall-client.js

@@ -146,6 +146,25 @@ export class RecallClient extends EventEmitter {
         this.#activeBots.delete(botId);
         console.log(`👋 Recall.ai bot left meeting: ${botId}`);
     }
+    /**
+     * Speak DIRECTLY through the bot via Recall's native output_audio (mp3).
+     * No canvas page, no Web Audio, no capture chain — Recall plays the file as
+     * the bot's voice. Returns true on 2xx. (2026-08-01: canvas-audio quality
+     * was "barely bearable"; this is the direct loud path.)
+     */
+    async outputAudio(botId, mp3) {
+        const res = await fetch(`${RECALL_BASE_URL}/bot/${botId}/output_audio/`, {
+            method: 'POST',
+            headers: { 'Authorization': `Token ${this.#apiKey}`, 'Content-Type': 'application/json' },
+            body: JSON.stringify({ kind: 'mp3', b64_data: mp3.toString('base64') }),
+        });
+        if (!res.ok) {
+            const e = await res.text().catch(() => '');
+            console.warn(`⚠️ Recall output_audio ${res.status}: ${e.slice(0, 160)}`);
+            return false;
+        }
+        return true;
+    }
     async getBotStatus(botId) {
         const res = await fetch(`${RECALL_BASE_URL}/bot/${botId}`, {
             headers: { 'Authorization': `Token ${this.#apiKey}` },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.112",
+  "version": "0.9.114",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {