osborn 0.9.111 → 0.9.113

package/dist/index.js

@@ -230,6 +230,62 @@ let meetingAgentSpeakingText = '';
 // Prepended to the next flush: what the bot was cut off saying + who interrupted
 // (same pattern as voice-native interruptions).
 let meetingInterruptContext = '';
+// Synthesize speech as MP3 (Deepgram fast path, OpenAI fallback) — for
+// Recall native output_audio, which requires mp3.
+async function synthMp3(text) {
+    const t0 = Date.now();
+    const dgKey = process.env.DEEPGRAM_API_KEY;
+    if (dgKey) {
+        try {
+            const dg = await fetch('https://api.deepgram.com/v1/speak?model=aura-2-thalia-en&encoding=mp3&bit_rate=48000', {
+                method: 'POST',
+                headers: { 'Authorization': `Token ${dgKey}`, 'Content-Type': 'application/json' },
+                body: JSON.stringify({ text: text.slice(0, 4000) }),
+                signal: AbortSignal.timeout(12000),
+            });
+            if (dg.ok) {
+                const buf = Buffer.from(await dg.arrayBuffer());
+                console.log(`🗣️ synthMp3 deepgram ${buf.length}b in ${Date.now() - t0}ms`);
+                return buf;
+            }
+        }
+        catch (e) {
+            console.warn(`⚠️ synthMp3 deepgram: ${e.message}`);
+        }
+    }
+    const oa = process.env.OPENAI_API_KEY;
+    if (oa) {
+        try {
+            const r = await fetch('https://api.openai.com/v1/audio/speech', {
+                method: 'POST',
+                headers: { 'Authorization': `Bearer ${oa}`, 'Content-Type': 'application/json' },
+                body: JSON.stringify({ model: 'gpt-4o-mini-tts', voice: 'alloy', input: text.slice(0, 4000), response_format: 'mp3' }),
+                signal: AbortSignal.timeout(15000),
+            });
+            if (r.ok)
+                return Buffer.from(await r.arrayBuffer());
+        }
+        catch { /* fall through */ }
+    }
+    return null;
+}
+// THE speak path for meetings (2026-08-01): Recall native output_audio FIRST
+// (direct, loud, no canvas capture chain), canvas Web-Audio say as FALLBACK.
+async function speakIntoMeeting(text) {
+    const recall = getRecallClient();
+    const botId = recall?.getActiveBotIds?.()[0];
+    if (recall && botId) {
+        const mp3 = await synthMp3(text);
+        if (mp3 && await recall.outputAudio(botId, mp3)) {
+            console.log(`📢 spoke via Recall output_audio (${mp3.length}b): "${text.slice(0, 60)}"`);
+            markMeetingSpeaking(text);
+            return;
+        }
+    }
+    console.log(`📽️ falling back to canvas say: "${text.slice(0, 50)}"`);
+    pushCanvas({ kind: 'say', text });
+    markMeetingSpeaking(text);
+}
 function markMeetingSpeaking(text) {
     meetingAgentSpeaking = true;
     meetingAgentSpeakingText = text;
@@ -644,10 +700,13 @@ function startApiServer(workingDir, port) {
                     const evt = JSON.parse(body || '{}');
                     if (evt.kind !== 'say' && evt.kind !== 'show' && evt.kind !== 'stop')
                         throw new Error("kind must be 'say', 'show', or 'stop'");
-                    pushCanvas(evt);
-                    // Track that the bot is now speaking into the meeting → enables interruption.
-                    if (evt.kind === 'say')
-                        markMeetingSpeaking(evt.text);
+                    if (evt.kind === 'say') {
+                        // Native-first speak path (Recall output_audio → canvas fallback).
+                        void speakIntoMeeting(evt.text);
+                    }
+                    else {
+                        pushCanvas(evt);
+                    }
                     res.writeHead(200, { 'Content-Type': 'application/json' });
                     res.end(JSON.stringify({ ok: true, clients: canvasClients.size }));
                 }
@@ -1656,6 +1715,11 @@ async function main() {
     // REDIRECTED into the meeting canvas instead of suppressed — the regular
     // voice pipeline reused with the meeting as the sink (latency fix).
     let meetingAddressedUntil = 0;
+    // Distinct human speakers seen in the current meeting. In a 1:1 (one human
+    // + the bot) EVERY utterance is addressed to the bot — no name needed
+    // (2026-08-01: "can you open carfax..." got observer-suppressed because
+    // the fresh session's latch was never opened by the name).
+    const meetingSpeakers = new Set();
     let activeMeetingPoller = null; // Transcript poller bound to that bot
     // LIVE meeting transcript → LLM (buffered webhook finals). See recall.on('transcript').
     const meetingTranscriptBuffer = [];
@@ -1747,6 +1811,7 @@ async function main() {
         }
         activeMeetingBotId = null;
         meetingAddressedUntil = 0;
+        meetingSpeakers.clear();
         // leaveBot=false when Recall itself reported the meeting over (bot already gone).
         if (opts.leaveBot !== false) {
             const recall = getRecallClient();
@@ -1857,13 +1922,26 @@ async function main() {
                 // the agent replies out loud into the meeting. This is the chat-mode path:
                 // named/asked → prompt response. Un-addressed chunks stay in the silent
                 // 20s batch for note-taking. That's the two-mode seam, mechanically.
+                meetingSpeakers.add(speaker);
                 // Fuzzy name match: meeting STT routinely mangles "Osborn" — observed
                 // live 2026-08-01: "Osborn can you hear me" → "i was born can you hear
                 // me" (name trigger missed, bot stayed silent). Accept common
                 // mis-transcriptions; mild false-positive risk is acceptable in a
-                // room that invited the bot.
-                if (/\b(osborne?|oz\s?born|os\s?born|was born|is born|ozborn|osbourne?|austin\b.{0,8}(hear|there|can you))/i.test(text)) {
-                    console.log('📓 Addressed by name (fuzzy) — immediate flush for a response');
+                // room that invited the bot. AND: in a 1:1 meeting (one human + the
+                // bot) every utterance is addressed — no name needed.
+                // Spoken mode switch: "interactive mode"/"go interactive" opens the
+                // conversation latch; "observer mode"/"go silent" closes it.
+                if (/\b(interactive mode|go interactive|start responding|talk to (me|us))\b/i.test(text)) {
+                    meetingAddressedUntil = Date.now() + 6 * 60 * 60 * 1000;
+                    void speakIntoMeeting('Interactive mode on — I will respond out loud.');
+                }
+                else if (/\b(observer mode|silent mode|go silent|stop responding)\b/i.test(text)) {
+                    meetingAddressedUntil = 0;
+                    void speakIntoMeeting('Going silent — just taking notes. Say interactive mode to bring me back.');
+                }
+                const oneOnOne = meetingSpeakers.size <= 1;
+                if (oneOnOne || /\b(osborne?|oz\s?born|os\s?born|was born|is born|ozborn|osbourne?|austin\b.{0,8}(hear|there|can you))/i.test(text)) {
+                    console.log(`📓 Addressed (${oneOnOne ? '1:1 meeting' : 'by name'}) — immediate flush for a response`);
                     flushMeetingBuffer(botId, true);
                 }
             }
@@ -2563,9 +2641,8 @@ async function main() {
                     // audio file = zero seams — quality over ~1s of latency. (Structural
                     // fix queued: canvas as LiveKit subscriber playing the agent's real
                     // TTS track — the true "same browser experience" reuse.)
-                    console.log(`🔊➡️📽️ tts_say → canvas (single utterance) t=${new Date().toISOString()}: "${data.text.slice(0, 60)}"`);
-                    pushCanvas({ kind: 'say', text: data.text });
-                    markMeetingSpeaking(data.text);
+                    console.log(`🔊➡️📢 tts_say → meeting (native audio) t=${new Date().toISOString()}: "${data.text.slice(0, 60)}"`);
+                    void speakIntoMeeting(data.text);
                     return;
                 }
                 console.log(`🔇 tts_say suppressed (meeting turn — response goes to /canvas, not browser): "${data.text.slice(0, 60)}"`);

package/dist/recall-client.d.ts

@@ -103,6 +103,13 @@ export declare class RecallClient extends EventEmitter {
      */
     getTranscript(botId: string): Promise<TranscriptTurn[]>;
     leaveMeeting(botId: string): Promise<void>;
+    /**
+     * Speak DIRECTLY through the bot via Recall's native output_audio (mp3).
+     * No canvas page, no Web Audio, no capture chain — Recall plays the file as
+     * the bot's voice. Returns true on 2xx. (2026-08-01: canvas-audio quality
+     * was "barely bearable"; this is the direct loud path.)
+     */
+    outputAudio(botId: string, mp3: Buffer): Promise<boolean>;
     getBotStatus(botId: string): Promise<string>;
     handleWebhook(payload: TranscriptPayload): void;
     registerBot(botId: string, sessionId: string): void;

package/dist/recall-client.js

@@ -146,6 +146,25 @@ export class RecallClient extends EventEmitter {
         this.#activeBots.delete(botId);
         console.log(`👋 Recall.ai bot left meeting: ${botId}`);
     }
+    /**
+     * Speak DIRECTLY through the bot via Recall's native output_audio (mp3).
+     * No canvas page, no Web Audio, no capture chain — Recall plays the file as
+     * the bot's voice. Returns true on 2xx. (2026-08-01: canvas-audio quality
+     * was "barely bearable"; this is the direct loud path.)
+     */
+    async outputAudio(botId, mp3) {
+        const res = await fetch(`${RECALL_BASE_URL}/bot/${botId}/output_audio/`, {
+            method: 'POST',
+            headers: { 'Authorization': `Token ${this.#apiKey}`, 'Content-Type': 'application/json' },
+            body: JSON.stringify({ kind: 'mp3', b64_data: mp3.toString('base64') }),
+        });
+        if (!res.ok) {
+            const e = await res.text().catch(() => '');
+            console.warn(`⚠️ Recall output_audio ${res.status}: ${e.slice(0, 160)}`);
+            return false;
+        }
+        return true;
+    }
     async getBotStatus(botId) {
         const res = await fetch(`${RECALL_BASE_URL}/bot/${botId}`, {
             headers: { 'Authorization': `Token ${this.#apiKey}` },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.111",
+  "version": "0.9.113",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {