osborn 0.9.90 → 0.9.93

package/.claude/skills/meetings/SKILL.md

@@ -39,6 +39,35 @@ Task(
 
 Give the writer everything it needs in the prompt: the session-workspace path, the bot ID, the `us-west-2.recall.ai` endpoint rule, and the `meeting-todos.md` structure. The writer runs the curl/jq/Write steps itself. For research, delegate to the `researcher` sub-agent the same way.
 
+## How to SPEAK INTO the meeting (out of silent mode)
+
+You can talk directly into the Google Meet / Zoom — your words play as the bot's
+voice. The bot casts a "meeting canvas" webpage as its camera+mic; POSTing to
+your own HTTP API pushes speech (and visuals) to it, and Recall pipes the
+canvas audio into the call. This is ONE Bash call, within budget.
+
+**Speak into the meeting:**
+```bash
+curl -sS -X POST http://localhost:8741/canvas -H 'Content-Type: application/json' \
+  -d '{"kind":"say","text":"YOUR WORDS HERE"}'
+```
+(Port is `OSBORN_API_PORT`, default 8741.)
+
+**Show a visual on the bot's camera** (notes, a link, a title):
+```bash
+curl -sS -X POST http://localhost:8741/canvas -H 'Content-Type: application/json' \
+  -d '{"kind":"show","mode":"notes","title":"...","items":["...","..."]}'
+```
+`mode` = `idle` | `notes` (title+items) | `link` (url) | `web` (iframe url) | `text` (title+text).
+
+**When to speak into the meeting:** By DEFAULT stay silent (observer) for
+`[MEETING — *]:` chunks — take notes, don't interrupt. Speak into the meeting
+ONLY when: (a) the voice-native user explicitly tells you to say something to the
+meeting / "tell them X" / "answer that", or (b) you're directly addressed by name
+in the meeting and the user has enabled active mode. When you do speak, keep it
+short and let the room continue. This is the toggle between silent-observer and
+active-participant.
+
 ## How to behave (auto-tagged chunks)
 
 For every `[MEETING — *]:` message:

package/Dockerfile.sandbox

@@ -164,7 +164,14 @@ PKG_SKILLS_DIR="/usr/local/lib/node_modules/osborn/.claude/skills"
 SEED_VERSION_FILE="${HOME_SKILLS_DIR}/.seed-version"
 mkdir -p "$HOME_SKILLS_DIR"
 CURRENT_SEED_VERSION=$(cat "$SEED_VERSION_FILE" 2>/dev/null | tr -d '[:space:]' || echo "")
-IMAGE_SEED_VERSION="${OSBORN_IMAGE_VERSION:-latest}"
+# Seed version = the INSTALLED package's own version (reliable on every boot),
+# NOT the OSBORN_IMAGE_VERSION env — that env isn't threaded through
+# `fly machine update`, so it went stale and left the marker permanently pinned
+# at 0.9.48, meaning the refresh below never fired and new-version skills never
+# reached the load path. Reading package.json makes CURRENT != IMAGE trip on a
+# real version bump. Falls back to the env / "latest" if the package is unreadable.
+IMAGE_SEED_VERSION=$(grep -m1 '"version"' /usr/local/lib/node_modules/osborn/package.json 2>/dev/null | sed 's/.*"version"[^"]*"\([^"]*\)".*/\1/')
+[ -z "$IMAGE_SEED_VERSION" ] && IMAGE_SEED_VERSION="${OSBORN_IMAGE_VERSION:-latest}"
 if [ -d "$PKG_SKILLS_DIR" ]; then
   REFRESHED=0
   SEEDED=0

package/dist/index.js

@@ -215,7 +215,28 @@ function pushCanvas(evt) {
             canvasClients.delete(res);
         }
     }
-    console.log(`🖼️ canvas ${evt.kind}: ${evt.kind === 'say' ? evt.text.slice(0, 60) : evt.mode} → ${canvasClients.size} client(s)`);
+    const desc = evt.kind === 'say' ? evt.text.slice(0, 60) : evt.kind === 'show' ? evt.mode : evt.kind;
+    console.log(`🖼️ canvas ${evt.kind}: ${desc} → ${canvasClients.size} client(s)`);
+}
+// ── Meeting interruption state (module scope — shared by the /canvas HTTP
+// handler and the recall speech handler in main()) ──────────────────────────
+const meetingBotName = 'Osborn'; // the bot's name in the meeting; ignore its own speech_on
+// True while the bot's TTS is (probably) still playing into the meeting. Set when
+// a /canvas say is pushed; cleared after an estimated duration OR on interruption.
+// A HUMAN's speech_on while this is true → interrupt.
+let meetingAgentSpeaking = false;
+let meetingSpeakClearTimer = null;
+let meetingAgentSpeakingText = '';
+// Prepended to the next flush: what the bot was cut off saying + who interrupted
+// (same pattern as voice-native interruptions).
+let meetingInterruptContext = '';
+function markMeetingSpeaking(text) {
+    meetingAgentSpeaking = true;
+    meetingAgentSpeakingText = text;
+    if (meetingSpeakClearTimer)
+        clearTimeout(meetingSpeakClearTimer);
+    const ms = Math.min(30_000, 3_000 + (text.split(/\s+/).length / 2.5) * 1000); // ~2.5 wps + ~3s Recall lag
+    meetingSpeakClearTimer = setTimeout(() => { meetingAgentSpeaking = false; meetingAgentSpeakingText = ''; }, ms);
 }
 function startApiServer(workingDir, port) {
     const server = createServer(async (req, res) => {
@@ -558,9 +579,12 @@ function startApiServer(workingDir, port) {
             req.on('end', () => {
                 try {
                     const evt = JSON.parse(body || '{}');
-                    if (evt.kind !== 'say' && evt.kind !== 'show')
-                        throw new Error("kind must be 'say' or 'show'");
+                    if (evt.kind !== 'say' && evt.kind !== 'show' && evt.kind !== 'stop')
+                        throw new Error("kind must be 'say', 'show', or 'stop'");
                     pushCanvas(evt);
+                    // Track that the bot is now speaking into the meeting → enables interruption.
+                    if (evt.kind === 'say')
+                        markMeetingSpeaking(evt.text);
                     res.writeHead(200, { 'Content-Type': 'application/json' });
                     res.end(JSON.stringify({ ok: true, clients: canvasClients.size }));
                 }
@@ -1556,6 +1580,45 @@ async function main() {
     let currentUserId = '';
     let activeMeetingBotId = null; // Recall.ai bot ID if in a meeting
     let activeMeetingPoller = null; // Transcript poller bound to that bot
+    // LIVE meeting transcript → LLM (buffered webhook finals). See recall.on('transcript').
+    const meetingTranscriptBuffer = [];
+    let meetingFlushTimer = null;
+    // Flush the buffered meeting turns to the LLM. `addressed=true` (the agent was
+    // called by name / asked directly) tags the message so the agent RESPONDS by
+    // speaking into the meeting; otherwise it's a silent-observer note-taking batch.
+    const flushMeetingBuffer = (botId, addressed = false) => {
+        if (!meetingTranscriptBuffer.length || !currentLLM)
+            return;
+        const turns = meetingTranscriptBuffer.splice(0); // drain
+        const header = addressed
+            ? `[MEETING — ${botId}] — YOU WERE ADDRESSED. Respond OUT LOUD into the meeting now: POST http://localhost:${apiPort}/canvas {"kind":"say","text":"..."} with a short, direct reply. Then note it.`
+            : `[MEETING — ${botId}]:`;
+        // Prepend + consume any interruption context (bot was cut off mid-sentence).
+        const interrupt = meetingInterruptContext ? `${meetingInterruptContext}\n` : '';
+        meetingInterruptContext = '';
+        try {
+            const ctx = new llm.ChatContext();
+            ctx.addMessage({ role: 'user', content: `${interrupt}${header}\n${turns.join('\n')}` });
+            currentLLM.chat({ chatCtx: ctx });
+            console.log(`📓 Flushed ${turns.length} meeting turn(s) to LLM${addressed ? ' (ADDRESSED — respond)' : ''}`);
+        }
+        catch (err) {
+            console.warn(`⚠️ Meeting flush failed: ${err.message}`);
+        }
+    };
+    const startMeetingFlush = (botId) => {
+        stopMeetingFlush();
+        console.log(`📓 Meeting transcript flush timer started (bot ${botId}, 20s)`);
+        meetingFlushTimer = setInterval(() => flushMeetingBuffer(botId, false), 20_000);
+    };
+    const stopMeetingFlush = () => {
+        if (meetingFlushTimer) {
+            clearInterval(meetingFlushTimer);
+            meetingFlushTimer = null;
+            console.log('📓 Meeting flush timer stopped');
+        }
+        meetingTranscriptBuffer.length = 0;
+    };
     // Track the active resume session ID across scopes (ParticipantConnected + DataReceived)
     // Updated by resume_session, session_selected, continue_session, switch_session handlers
     let currentResumeSessionId;
@@ -1593,26 +1656,50 @@ async function main() {
     // "what was said in the meeting" display, separate from the LLM input path).
     const recall = getRecallClient();
     if (recall) {
-        console.log('🎥 Recall.ai client initialized (webhook STT receiver — LLM forwarding disabled, see meeting-bot Phase 2)');
-        recall.on('transcript', ({ botId, speaker, text }) => {
-            console.log(`📝 Meeting transcript [${speaker}]: ${text}`);
-            // INTENTIONALLY DISABLED — see comment above. Audio path is now LiveKit
-            // → meeting-bot page publishes meeting audio → agent STT processes it.
-            // The line below is preserved as a reference for future re-enablement
-            // (e.g. as a display-only feature, NOT as LLM input).
-            //
-            // if (currentLLM && currentSession) {
-            //   const meetingText = `[Meeting — ${speaker}]: ${text}`
-            //   try {
-            //     if (currentVoiceMode === 'pipeline' || currentVoiceMode === 'direct') {
-            //       const chatCtx = new llm.ChatContext()
-            //       chatCtx.addMessage({ role: 'user', content: meetingText })
-            //       ;(currentLLM as any).chat({ chatCtx })
-            //     }
-            //   } catch (err) {
-            //     console.error('❌ Failed to route meeting transcript:', err)
-            //   }
-            // }
+        console.log('🎥 Recall.ai client initialized — LIVE webhook finals buffered → LLM (batch poller download_url is empty mid-call, so the webhook is the only live transcript source)');
+        // LIVE meeting transcript → LLM. The realtime webhook (handleWebhook) emits
+        // every partial + final. We buffer FINALS (partials are too noisy) and a
+        // flush timer (started on join_meeting) batches them to currentLLM.chat()
+        // as [MEETING — botId]: every ~20s — so the agent actually sees the meeting
+        // and can take notes / delegate to the writer. Previously this was DISABLED
+        // and the poller (batch endpoint) was the only LLM path — but that endpoint
+        // is empty until the meeting ENDS, so mid-call the LLM saw nothing.
+        recall.on('transcript', ({ botId, speaker, text, partial }) => {
+            console.log(`📝 Meeting transcript [${speaker}]${partial ? ' (partial)' : ''}: ${text}`);
+            if (!partial && text.trim()) {
+                meetingTranscriptBuffer.push(`${speaker}: ${text.trim()}`);
+                // Addressed by name → flush NOW (don't wait the 20s batch) and tag it so
+                // the agent replies out loud into the meeting. This is the chat-mode path:
+                // named/asked → prompt response. Un-addressed chunks stay in the silent
+                // 20s batch for note-taking. That's the two-mode seam, mechanically.
+                if (/\bosborn\b/i.test(text)) {
+                    console.log('📓 Addressed by name — immediate flush for a response');
+                    flushMeetingBuffer(botId, true);
+                }
+            }
+        });
+        // Participant speech VAD → interruption + silence-boundary chunking.
+        recall.on('speech', ({ botId, participant, active }) => {
+            const isBot = participant === meetingBotName || participant === 'Osborn' || participant === 'Unknown';
+            if (active) {
+                // A HUMAN started talking. If the bot is mid-speech → INTERRUPT: stop the
+                // bot's audio and record what it was cut off saying so the next flush can
+                // tell it what it missed (same pattern as voice-native interruptions).
+                if (meetingAgentSpeaking && !isBot) {
+                    console.log(`✋ Interruption — ${participant} spoke while bot was talking. Stopping bot audio.`);
+                    pushCanvas({ kind: 'stop' });
+                    meetingInterruptContext = `[MEETING — interrupted] You were speaking ("${meetingAgentSpeakingText.slice(0, 140)}") when ${participant} started talking and cut you off. They likely didn't hear the rest. When you respond, briefly acknowledge and adapt — don't just repeat.`;
+                    meetingAgentSpeaking = false;
+                    if (meetingSpeakClearTimer)
+                        clearTimeout(meetingSpeakClearTimer);
+                }
+            }
+            else {
+                // speech_off — a natural silence boundary. Flush the buffered turns now
+                // (chunk on conversational pauses, not an arbitrary clock), unless empty.
+                if (!isBot && meetingTranscriptBuffer.length)
+                    flushMeetingBuffer(botId, false);
+            }
         });
     }
     // ============================================================
@@ -2265,6 +2352,14 @@ async function main() {
                 console.log(`🔇 tts_say fired but text is empty — skipping`);
                 return;
             }
+            // Suppress browser TTS for meeting-chunk responses — the agent speaks INTO
+            // the meeting via /canvas, not the laptop speakers (which the Meet mic
+            // re-captures in the same room → feedback). Set by PipelineDirectLLM.chat()
+            // when the turn is a [MEETING —] chunk. Normal user turns are unaffected.
+            if (directLLM.suppressMeetingTTS) {
+                console.log(`🔇 tts_say suppressed (meeting turn — response goes to /canvas, not browser): "${data.text.slice(0, 60)}"`);
+                return;
+            }
             // Suppress while the user is mid-utterance. Without this, agent text generated
             // in parallel by the Claude SDK plays right over the user — same problem as
             // pre-interrupt overlap, but at the *output* side. The suppressed text gets
@@ -4101,6 +4196,7 @@ async function main() {
         clearFastBrainSession();
         clearPipelineFastBrainSession();
         // Auto-leave any active meeting bot when user disconnects from the room
+        stopMeetingFlush();
         if (activeMeetingPoller) {
             activeMeetingPoller.stop();
             activeMeetingPoller = null;
@@ -4786,6 +4882,9 @@ async function main() {
                                 },
                             });
                             activeMeetingPoller.start();
+                            // LIVE path: buffer webhook finals + flush to the LLM every 20s.
+                            // (The poller above only lands data after the meeting ENDS.)
+                            startMeetingFlush(botId);
                         }
                         catch (err) {
                             console.error('❌ Recall.ai join error:', err);
@@ -4799,8 +4898,9 @@ async function main() {
                 const recallLeave = getRecallClient();
                 if (recallLeave && botId) {
                     try {
-                        // Stop the transcript poller FIRST so no more transcript chunks get
-                        // forwarded to the LLM during the leave.
+                        // Stop the transcript poller + live flush FIRST so no more chunks
+                        // get forwarded to the LLM during the leave.
+                        stopMeetingFlush();
                         if (activeMeetingPoller) {
                             activeMeetingPoller.stop();
                             activeMeetingPoller = null;

package/dist/pipeline-direct-llm.d.ts

@@ -42,6 +42,7 @@ export interface FastBrainPanelResult {
 }
 export declare class PipelineDirectLLM extends llm.LLM {
     #private;
+    suppressMeetingTTS: boolean;
     constructor(opts: PipelineDirectOptions);
     /** Stop the index watcher (call on disconnect/session switch) */
     stopIndexWatcher(): void;

package/dist/pipeline-direct-llm.js

@@ -18,6 +18,10 @@ export class PipelineDirectLLM extends llm.LLM {
     #turnAbort = null;
     #indexWatcher = null;
     #indexBuilding = false;
+    // True while the in-flight turn is a [MEETING —] chunk — the tts_say gate in
+    // index.ts reads this to skip browser session.say() (meeting audio goes via
+    // /canvas, not the laptop speakers → no same-room feedback). Set per chat().
+    suppressMeetingTTS = false;
     constructor(opts) {
         super();
         this.#claudeLLM = new ClaudeLLM(opts);
@@ -175,10 +179,20 @@ export class PipelineDirectLLM extends llm.LLM {
                 }
             }
         }
+        // Meeting chunks ([MEETING —]) are the silent-observer / addressed-response
+        // path. They must NOT drive the browser voice session: (a) no session.say()
+        // TTS — the Meet mic re-captures browser audio in the same room → feedback;
+        // the agent speaks INTO the meeting via /canvas instead; (b) no fast brain —
+        // it's chat-panel noise ("Is there something you'd like to know?") on meeting
+        // speech. Set BEFORE the response streams so the tts_say gate (index.ts) sees
+        // it. Cleared implicitly by the next real user turn (which isn't a [MEETING]).
+        const isMeetingChunk = userText.startsWith('[MEETING');
+        this.suppressMeetingTTS = isMeetingChunk;
         // Fire Claude
         const claudeStream = this.#claudeLLM.chat({ chatCtx, toolCtx, connOptions, abortController });
-        // Fire pipeline fast brain in background — no await, no blocking
-        if (userText.trim()) {
+        // Fire pipeline fast brain in background — no await, no blocking. Skip for
+        // meeting chunks (silent observer / meeting-response path, not a user turn).
+        if (userText.trim() && !isMeetingChunk) {
             this.#firePipelineFastBrain(userText);
         }
         return claudeStream;

package/dist/recall-client.js

@@ -45,8 +45,12 @@ export class RecallClient extends EventEmitter {
         // transcript turn to /webhook/recall as it's spoken — the receiver
         // (handleWebhook) already exists. Polling stays as the after-the-fact
         // backstop. Only wire the webhook when we have a public URL to receive on.
+        // Subscribe to transcript (content) AND participant speech VAD events. The
+        // speech_on/speech_off events fire from raw audio (faster than transcription)
+        // and tell us WHO is speaking WHEN — used for interruption (human speaks while
+        // the bot is talking → stop) and for chunking on natural silence boundaries.
         const realtime = /^https:\/\//.test(webhookBaseUrl)
-            ? [{ type: 'webhook', url: `${webhookBaseUrl}/webhook/recall`, events: ['transcript.data', 'transcript.partial_data'] }]
+            ? [{ type: 'webhook', url: `${webhookBaseUrl}/webhook/recall`, events: ['transcript.data', 'transcript.partial_data', 'participant_events.speech_on', 'participant_events.speech_off'] }]
             : [];
         if (!realtime.length)
             console.log('⚠️ Recall realtime webhook skipped (no public https URL) — polling only');
@@ -154,6 +158,21 @@ export class RecallClient extends EventEmitter {
     // grows meaningfully or finalizes.
     #lastEmitted = new Map();
     handleWebhook(payload) {
+        // Participant speech VAD (speech_on/speech_off) — fires from raw audio, ahead
+        // of transcription. Emit a 'speech' event so index.ts can (a) interrupt the
+        // bot when a HUMAN starts talking over it, and (b) chunk the transcript on
+        // natural silence boundaries. The bot's own output_media audio may also fire
+        // speech_on for its own participant — index.ts filters that by name.
+        if (payload.event === 'participant_events.speech_on' || payload.event === 'participant_events.speech_off') {
+            const p = payload.data;
+            this.emit('speech', {
+                botId: p?.bot?.id ?? 'unknown',
+                participant: p?.data?.participant?.name ?? 'Unknown',
+                isHost: !!p?.data?.participant?.is_host,
+                active: payload.event === 'participant_events.speech_on',
+            });
+            return;
+        }
         // Accept BOTH finals (transcript.data) AND partials (transcript.partial_data).
         // recallai_streaming in prioritize_low_latency mode emits partials DURING
         // the call and finals lag — ignoring partials (the old behavior) meant no

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.90",
+  "version": "0.9.93",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {