osborn 0.9.91 → 0.9.93

package/dist/index.js

@@ -215,7 +215,28 @@ function pushCanvas(evt) {
             canvasClients.delete(res);
         }
     }
-    console.log(`🖼️ canvas ${evt.kind}: ${evt.kind === 'say' ? evt.text.slice(0, 60) : evt.mode} → ${canvasClients.size} client(s)`);
+    const desc = evt.kind === 'say' ? evt.text.slice(0, 60) : evt.kind === 'show' ? evt.mode : evt.kind;
+    console.log(`🖼️ canvas ${evt.kind}: ${desc} → ${canvasClients.size} client(s)`);
+}
+// ── Meeting interruption state (module scope — shared by the /canvas HTTP
+// handler and the recall speech handler in main()) ──────────────────────────
+const meetingBotName = 'Osborn'; // the bot's name in the meeting; ignore its own speech_on
+// True while the bot's TTS is (probably) still playing into the meeting. Set when
+// a /canvas say is pushed; cleared after an estimated duration OR on interruption.
+// A HUMAN's speech_on while this is true → interrupt.
+let meetingAgentSpeaking = false;
+let meetingSpeakClearTimer = null;
+let meetingAgentSpeakingText = '';
+// Prepended to the next flush: what the bot was cut off saying + who interrupted
+// (same pattern as voice-native interruptions).
+let meetingInterruptContext = '';
+function markMeetingSpeaking(text) {
+    meetingAgentSpeaking = true;
+    meetingAgentSpeakingText = text;
+    if (meetingSpeakClearTimer)
+        clearTimeout(meetingSpeakClearTimer);
+    const ms = Math.min(30_000, 3_000 + (text.split(/\s+/).length / 2.5) * 1000); // ~2.5 wps + ~3s Recall lag
+    meetingSpeakClearTimer = setTimeout(() => { meetingAgentSpeaking = false; meetingAgentSpeakingText = ''; }, ms);
 }
 function startApiServer(workingDir, port) {
     const server = createServer(async (req, res) => {
@@ -558,9 +579,12 @@ function startApiServer(workingDir, port) {
             req.on('end', () => {
                 try {
                     const evt = JSON.parse(body || '{}');
-                    if (evt.kind !== 'say' && evt.kind !== 'show')
-                        throw new Error("kind must be 'say' or 'show'");
+                    if (evt.kind !== 'say' && evt.kind !== 'show' && evt.kind !== 'stop')
+                        throw new Error("kind must be 'say', 'show', or 'stop'");
                     pushCanvas(evt);
+                    // Track that the bot is now speaking into the meeting → enables interruption.
+                    if (evt.kind === 'say')
+                        markMeetingSpeaking(evt.text);
                     res.writeHead(200, { 'Content-Type': 'application/json' });
                     res.end(JSON.stringify({ ok: true, clients: canvasClients.size }));
                 }
@@ -1559,24 +1583,33 @@ async function main() {
     // LIVE meeting transcript → LLM (buffered webhook finals). See recall.on('transcript').
     const meetingTranscriptBuffer = [];
     let meetingFlushTimer = null;
+    // Flush the buffered meeting turns to the LLM. `addressed=true` (the agent was
+    // called by name / asked directly) tags the message so the agent RESPONDS by
+    // speaking into the meeting; otherwise it's a silent-observer note-taking batch.
+    const flushMeetingBuffer = (botId, addressed = false) => {
+        if (!meetingTranscriptBuffer.length || !currentLLM)
+            return;
+        const turns = meetingTranscriptBuffer.splice(0); // drain
+        const header = addressed
+            ? `[MEETING — ${botId}] — YOU WERE ADDRESSED. Respond OUT LOUD into the meeting now: POST http://localhost:${apiPort}/canvas {"kind":"say","text":"..."} with a short, direct reply. Then note it.`
+            : `[MEETING — ${botId}]:`;
+        // Prepend + consume any interruption context (bot was cut off mid-sentence).
+        const interrupt = meetingInterruptContext ? `${meetingInterruptContext}\n` : '';
+        meetingInterruptContext = '';
+        try {
+            const ctx = new llm.ChatContext();
+            ctx.addMessage({ role: 'user', content: `${interrupt}${header}\n${turns.join('\n')}` });
+            currentLLM.chat({ chatCtx: ctx });
+            console.log(`📓 Flushed ${turns.length} meeting turn(s) to LLM${addressed ? ' (ADDRESSED — respond)' : ''}`);
+        }
+        catch (err) {
+            console.warn(`⚠️ Meeting flush failed: ${err.message}`);
+        }
+    };
     const startMeetingFlush = (botId) => {
         stopMeetingFlush();
         console.log(`📓 Meeting transcript flush timer started (bot ${botId}, 20s)`);
-        meetingFlushTimer = setInterval(() => {
-            if (!meetingTranscriptBuffer.length || !currentLLM)
-                return;
-            const turns = meetingTranscriptBuffer.splice(0); // drain
-            const tagged = `[MEETING — ${botId}]:\n${turns.join('\n')}`;
-            try {
-                const ctx = new llm.ChatContext();
-                ctx.addMessage({ role: 'user', content: tagged });
-                currentLLM.chat({ chatCtx: ctx });
-                console.log(`📓 Flushed ${turns.length} meeting turn(s) to LLM`);
-            }
-            catch (err) {
-                console.warn(`⚠️ Meeting flush failed: ${err.message}`);
-            }
-        }, 20_000);
+        meetingFlushTimer = setInterval(() => flushMeetingBuffer(botId, false), 20_000);
     };
     const stopMeetingFlush = () => {
         if (meetingFlushTimer) {
@@ -1633,8 +1666,40 @@ async function main() {
         // is empty until the meeting ENDS, so mid-call the LLM saw nothing.
         recall.on('transcript', ({ botId, speaker, text, partial }) => {
             console.log(`📝 Meeting transcript [${speaker}]${partial ? ' (partial)' : ''}: ${text}`);
-            if (!partial && text.trim())
+            if (!partial && text.trim()) {
                 meetingTranscriptBuffer.push(`${speaker}: ${text.trim()}`);
+                // Addressed by name → flush NOW (don't wait the 20s batch) and tag it so
+                // the agent replies out loud into the meeting. This is the chat-mode path:
+                // named/asked → prompt response. Un-addressed chunks stay in the silent
+                // 20s batch for note-taking. That's the two-mode seam, mechanically.
+                if (/\bosborn\b/i.test(text)) {
+                    console.log('📓 Addressed by name — immediate flush for a response');
+                    flushMeetingBuffer(botId, true);
+                }
+            }
+        });
+        // Participant speech VAD → interruption + silence-boundary chunking.
+        recall.on('speech', ({ botId, participant, active }) => {
+            const isBot = participant === meetingBotName || participant === 'Osborn' || participant === 'Unknown';
+            if (active) {
+                // A HUMAN started talking. If the bot is mid-speech → INTERRUPT: stop the
+                // bot's audio and record what it was cut off saying so the next flush can
+                // tell it what it missed (same pattern as voice-native interruptions).
+                if (meetingAgentSpeaking && !isBot) {
+                    console.log(`✋ Interruption — ${participant} spoke while bot was talking. Stopping bot audio.`);
+                    pushCanvas({ kind: 'stop' });
+                    meetingInterruptContext = `[MEETING — interrupted] You were speaking ("${meetingAgentSpeakingText.slice(0, 140)}") when ${participant} started talking and cut you off. They likely didn't hear the rest. When you respond, briefly acknowledge and adapt — don't just repeat.`;
+                    meetingAgentSpeaking = false;
+                    if (meetingSpeakClearTimer)
+                        clearTimeout(meetingSpeakClearTimer);
+                }
+            }
+            else {
+                // speech_off — a natural silence boundary. Flush the buffered turns now
+                // (chunk on conversational pauses, not an arbitrary clock), unless empty.
+                if (!isBot && meetingTranscriptBuffer.length)
+                    flushMeetingBuffer(botId, false);
+            }
         });
     }
     // ============================================================
@@ -2287,6 +2352,14 @@ async function main() {
                 console.log(`🔇 tts_say fired but text is empty — skipping`);
                 return;
             }
+            // Suppress browser TTS for meeting-chunk responses — the agent speaks INTO
+            // the meeting via /canvas, not the laptop speakers (which the Meet mic
+            // re-captures in the same room → feedback). Set by PipelineDirectLLM.chat()
+            // when the turn is a [MEETING —] chunk. Normal user turns are unaffected.
+            if (directLLM.suppressMeetingTTS) {
+                console.log(`🔇 tts_say suppressed (meeting turn — response goes to /canvas, not browser): "${data.text.slice(0, 60)}"`);
+                return;
+            }
             // Suppress while the user is mid-utterance. Without this, agent text generated
             // in parallel by the Claude SDK plays right over the user — same problem as
             // pre-interrupt overlap, but at the *output* side. The suppressed text gets

package/dist/pipeline-direct-llm.d.ts

@@ -42,6 +42,7 @@ export interface FastBrainPanelResult {
 }
 export declare class PipelineDirectLLM extends llm.LLM {
     #private;
+    suppressMeetingTTS: boolean;
     constructor(opts: PipelineDirectOptions);
     /** Stop the index watcher (call on disconnect/session switch) */
     stopIndexWatcher(): void;

package/dist/pipeline-direct-llm.js

@@ -18,6 +18,10 @@ export class PipelineDirectLLM extends llm.LLM {
     #turnAbort = null;
     #indexWatcher = null;
     #indexBuilding = false;
+    // True while the in-flight turn is a [MEETING —] chunk — the tts_say gate in
+    // index.ts reads this to skip browser session.say() (meeting audio goes via
+    // /canvas, not the laptop speakers → no same-room feedback). Set per chat().
+    suppressMeetingTTS = false;
     constructor(opts) {
         super();
         this.#claudeLLM = new ClaudeLLM(opts);
@@ -175,10 +179,20 @@ export class PipelineDirectLLM extends llm.LLM {
                 }
             }
         }
+        // Meeting chunks ([MEETING —]) are the silent-observer / addressed-response
+        // path. They must NOT drive the browser voice session: (a) no session.say()
+        // TTS — the Meet mic re-captures browser audio in the same room → feedback;
+        // the agent speaks INTO the meeting via /canvas instead; (b) no fast brain —
+        // it's chat-panel noise ("Is there something you'd like to know?") on meeting
+        // speech. Set BEFORE the response streams so the tts_say gate (index.ts) sees
+        // it. Cleared implicitly by the next real user turn (which isn't a [MEETING]).
+        const isMeetingChunk = userText.startsWith('[MEETING');
+        this.suppressMeetingTTS = isMeetingChunk;
         // Fire Claude
         const claudeStream = this.#claudeLLM.chat({ chatCtx, toolCtx, connOptions, abortController });
-        // Fire pipeline fast brain in background — no await, no blocking
-        if (userText.trim()) {
+        // Fire pipeline fast brain in background — no await, no blocking. Skip for
+        // meeting chunks (silent observer / meeting-response path, not a user turn).
+        if (userText.trim() && !isMeetingChunk) {
             this.#firePipelineFastBrain(userText);
         }
         return claudeStream;

package/dist/recall-client.js

@@ -45,8 +45,12 @@ export class RecallClient extends EventEmitter {
         // transcript turn to /webhook/recall as it's spoken — the receiver
         // (handleWebhook) already exists. Polling stays as the after-the-fact
         // backstop. Only wire the webhook when we have a public URL to receive on.
+        // Subscribe to transcript (content) AND participant speech VAD events. The
+        // speech_on/speech_off events fire from raw audio (faster than transcription)
+        // and tell us WHO is speaking WHEN — used for interruption (human speaks while
+        // the bot is talking → stop) and for chunking on natural silence boundaries.
         const realtime = /^https:\/\//.test(webhookBaseUrl)
-            ? [{ type: 'webhook', url: `${webhookBaseUrl}/webhook/recall`, events: ['transcript.data', 'transcript.partial_data'] }]
+            ? [{ type: 'webhook', url: `${webhookBaseUrl}/webhook/recall`, events: ['transcript.data', 'transcript.partial_data', 'participant_events.speech_on', 'participant_events.speech_off'] }]
             : [];
         if (!realtime.length)
             console.log('⚠️ Recall realtime webhook skipped (no public https URL) — polling only');
@@ -154,6 +158,21 @@ export class RecallClient extends EventEmitter {
     // grows meaningfully or finalizes.
     #lastEmitted = new Map();
     handleWebhook(payload) {
+        // Participant speech VAD (speech_on/speech_off) — fires from raw audio, ahead
+        // of transcription. Emit a 'speech' event so index.ts can (a) interrupt the
+        // bot when a HUMAN starts talking over it, and (b) chunk the transcript on
+        // natural silence boundaries. The bot's own output_media audio may also fire
+        // speech_on for its own participant — index.ts filters that by name.
+        if (payload.event === 'participant_events.speech_on' || payload.event === 'participant_events.speech_off') {
+            const p = payload.data;
+            this.emit('speech', {
+                botId: p?.bot?.id ?? 'unknown',
+                participant: p?.data?.participant?.name ?? 'Unknown',
+                isHost: !!p?.data?.participant?.is_host,
+                active: payload.event === 'participant_events.speech_on',
+            });
+            return;
+        }
         // Accept BOTH finals (transcript.data) AND partials (transcript.partial_data).
         // recallai_streaming in prioritize_low_latency mode emits partials DURING
         // the call and finals lag — ignoring partials (the old behavior) meant no

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.91",
+  "version": "0.9.93",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {