osborn 0.9.90 → 0.9.91

package/.claude/skills/meetings/SKILL.md

@@ -39,6 +39,35 @@ Task(
 
 Give the writer everything it needs in the prompt: the session-workspace path, the bot ID, the `us-west-2.recall.ai` endpoint rule, and the `meeting-todos.md` structure. The writer runs the curl/jq/Write steps itself. For research, delegate to the `researcher` sub-agent the same way.
 
+## How to SPEAK INTO the meeting (out of silent mode)
+
+You can talk directly into the Google Meet / Zoom — your words play as the bot's
+voice. The bot casts a "meeting canvas" webpage as its camera+mic; POSTing to
+your own HTTP API pushes speech (and visuals) to it, and Recall pipes the
+canvas audio into the call. This is ONE Bash call, within budget.
+
+**Speak into the meeting:**
+```bash
+curl -sS -X POST http://localhost:8741/canvas -H 'Content-Type: application/json' \
+  -d '{"kind":"say","text":"YOUR WORDS HERE"}'
+```
+(Port is `OSBORN_API_PORT`, default 8741.)
+
+**Show a visual on the bot's camera** (notes, a link, a title):
+```bash
+curl -sS -X POST http://localhost:8741/canvas -H 'Content-Type: application/json' \
+  -d '{"kind":"show","mode":"notes","title":"...","items":["...","..."]}'
+```
+`mode` = `idle` | `notes` (title+items) | `link` (url) | `web` (iframe url) | `text` (title+text).
+
+**When to speak into the meeting:** By DEFAULT stay silent (observer) for
+`[MEETING — *]:` chunks — take notes, don't interrupt. Speak into the meeting
+ONLY when: (a) the voice-native user explicitly tells you to say something to the
+meeting / "tell them X" / "answer that", or (b) you're directly addressed by name
+in the meeting and the user has enabled active mode. When you do speak, keep it
+short and let the room continue. This is the toggle between silent-observer and
+active-participant.
+
 ## How to behave (auto-tagged chunks)
 
 For every `[MEETING — *]:` message:

package/Dockerfile.sandbox

@@ -164,7 +164,14 @@ PKG_SKILLS_DIR="/usr/local/lib/node_modules/osborn/.claude/skills"
 SEED_VERSION_FILE="${HOME_SKILLS_DIR}/.seed-version"
 mkdir -p "$HOME_SKILLS_DIR"
 CURRENT_SEED_VERSION=$(cat "$SEED_VERSION_FILE" 2>/dev/null | tr -d '[:space:]' || echo "")
-IMAGE_SEED_VERSION="${OSBORN_IMAGE_VERSION:-latest}"
+# Seed version = the INSTALLED package's own version (reliable on every boot),
+# NOT the OSBORN_IMAGE_VERSION env — that env isn't threaded through
+# `fly machine update`, so it went stale and left the marker permanently pinned
+# at 0.9.48, meaning the refresh below never fired and new-version skills never
+# reached the load path. Reading package.json makes CURRENT != IMAGE trip on a
+# real version bump. Falls back to the env / "latest" if the package is unreadable.
+IMAGE_SEED_VERSION=$(grep -m1 '"version"' /usr/local/lib/node_modules/osborn/package.json 2>/dev/null | sed 's/.*"version"[^"]*"\([^"]*\)".*/\1/')
+[ -z "$IMAGE_SEED_VERSION" ] && IMAGE_SEED_VERSION="${OSBORN_IMAGE_VERSION:-latest}"
 if [ -d "$PKG_SKILLS_DIR" ]; then
   REFRESHED=0
   SEEDED=0

package/dist/index.js

@@ -1556,6 +1556,36 @@ async function main() {
     let currentUserId = '';
     let activeMeetingBotId = null; // Recall.ai bot ID if in a meeting
     let activeMeetingPoller = null; // Transcript poller bound to that bot
+    // LIVE meeting transcript → LLM (buffered webhook finals). See recall.on('transcript').
+    const meetingTranscriptBuffer = [];
+    let meetingFlushTimer = null;
+    const startMeetingFlush = (botId) => {
+        stopMeetingFlush();
+        console.log(`📓 Meeting transcript flush timer started (bot ${botId}, 20s)`);
+        meetingFlushTimer = setInterval(() => {
+            if (!meetingTranscriptBuffer.length || !currentLLM)
+                return;
+            const turns = meetingTranscriptBuffer.splice(0); // drain
+            const tagged = `[MEETING — ${botId}]:\n${turns.join('\n')}`;
+            try {
+                const ctx = new llm.ChatContext();
+                ctx.addMessage({ role: 'user', content: tagged });
+                currentLLM.chat({ chatCtx: ctx });
+                console.log(`📓 Flushed ${turns.length} meeting turn(s) to LLM`);
+            }
+            catch (err) {
+                console.warn(`⚠️ Meeting flush failed: ${err.message}`);
+            }
+        }, 20_000);
+    };
+    const stopMeetingFlush = () => {
+        if (meetingFlushTimer) {
+            clearInterval(meetingFlushTimer);
+            meetingFlushTimer = null;
+            console.log('📓 Meeting flush timer stopped');
+        }
+        meetingTranscriptBuffer.length = 0;
+    };
     // Track the active resume session ID across scopes (ParticipantConnected + DataReceived)
     // Updated by resume_session, session_selected, continue_session, switch_session handlers
     let currentResumeSessionId;
@@ -1593,26 +1623,18 @@ async function main() {
     // "what was said in the meeting" display, separate from the LLM input path).
     const recall = getRecallClient();
     if (recall) {
-        console.log('🎥 Recall.ai client initialized (webhook STT receiver — LLM forwarding disabled, see meeting-bot Phase 2)');
-        recall.on('transcript', ({ botId, speaker, text }) => {
-            console.log(`📝 Meeting transcript [${speaker}]: ${text}`);
-            // INTENTIONALLY DISABLED — see comment above. Audio path is now LiveKit
-            // → meeting-bot page publishes meeting audio → agent STT processes it.
-            // The line below is preserved as a reference for future re-enablement
-            // (e.g. as a display-only feature, NOT as LLM input).
-            //
-            // if (currentLLM && currentSession) {
-            //   const meetingText = `[Meeting — ${speaker}]: ${text}`
-            //   try {
-            //     if (currentVoiceMode === 'pipeline' || currentVoiceMode === 'direct') {
-            //       const chatCtx = new llm.ChatContext()
-            //       chatCtx.addMessage({ role: 'user', content: meetingText })
-            //       ;(currentLLM as any).chat({ chatCtx })
-            //     }
-            //   } catch (err) {
-            //     console.error('❌ Failed to route meeting transcript:', err)
-            //   }
-            // }
+        console.log('🎥 Recall.ai client initialized — LIVE webhook finals buffered → LLM (batch poller download_url is empty mid-call, so the webhook is the only live transcript source)');
+        // LIVE meeting transcript → LLM. The realtime webhook (handleWebhook) emits
+        // every partial + final. We buffer FINALS (partials are too noisy) and a
+        // flush timer (started on join_meeting) batches them to currentLLM.chat()
+        // as [MEETING — botId]: every ~20s — so the agent actually sees the meeting
+        // and can take notes / delegate to the writer. Previously this was DISABLED
+        // and the poller (batch endpoint) was the only LLM path — but that endpoint
+        // is empty until the meeting ENDS, so mid-call the LLM saw nothing.
+        recall.on('transcript', ({ botId, speaker, text, partial }) => {
+            console.log(`📝 Meeting transcript [${speaker}]${partial ? ' (partial)' : ''}: ${text}`);
+            if (!partial && text.trim())
+                meetingTranscriptBuffer.push(`${speaker}: ${text.trim()}`);
         });
     }
     // ============================================================
@@ -4101,6 +4123,7 @@ async function main() {
         clearFastBrainSession();
         clearPipelineFastBrainSession();
         // Auto-leave any active meeting bot when user disconnects from the room
+        stopMeetingFlush();
         if (activeMeetingPoller) {
             activeMeetingPoller.stop();
             activeMeetingPoller = null;
@@ -4786,6 +4809,9 @@ async function main() {
                                 },
                             });
                             activeMeetingPoller.start();
+                            // LIVE path: buffer webhook finals + flush to the LLM every 20s.
+                            // (The poller above only lands data after the meeting ENDS.)
+                            startMeetingFlush(botId);
                         }
                         catch (err) {
                             console.error('❌ Recall.ai join error:', err);
@@ -4799,8 +4825,9 @@ async function main() {
                 const recallLeave = getRecallClient();
                 if (recallLeave && botId) {
                     try {
-                        // Stop the transcript poller FIRST so no more transcript chunks get
-                        // forwarded to the LLM during the leave.
+                        // Stop the transcript poller + live flush FIRST so no more chunks
+                        // get forwarded to the LLM during the leave.
+                        stopMeetingFlush();
                         if (activeMeetingPoller) {
                             activeMeetingPoller.stop();
                             activeMeetingPoller = null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.90",
+  "version": "0.9.91",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {