osborn 0.9.85 → 0.9.87

package/dist/index.js

@@ -201,6 +201,22 @@ let leaveRoomHook = null;
 // as the existing fetch-log/save-log flow so we don't ship Supabase credentials
 // to the Fly machine.
 let bugReportHook = null;
+const canvasClients = new Set();
+let latestCanvasShow = null; // last 'show' so a reconnecting canvas resyncs its visual
+function pushCanvas(evt) {
+    if (evt.kind === 'show')
+        latestCanvasShow = evt;
+    const line = `data: ${JSON.stringify(evt)}\n\n`;
+    for (const res of canvasClients) {
+        try {
+            res.write(line);
+        }
+        catch {
+            canvasClients.delete(res);
+        }
+    }
+    console.log(`🖼️ canvas ${evt.kind}: ${evt.kind === 'say' ? evt.text.slice(0, 60) : evt.mode} → ${canvasClients.size} client(s)`);
+}
 function startApiServer(workingDir, port) {
     const server = createServer(async (req, res) => {
         // CORS headers for cloud frontend
@@ -303,6 +319,12 @@ function startApiServer(workingDir, port) {
             req.on('end', () => {
                 try {
                     const payload = JSON.parse(body);
+                    // 0.9.86: log every receipt so we can SEE what Recall streams
+                    // (event type + word count) — the handler was silent, making it
+                    // impossible to tell whether webhooks arrive or are just filtered.
+                    const evt = payload?.event ?? 'unknown';
+                    const wc = (payload?.data?.data?.words ?? []).length;
+                    console.log(`📨 Recall webhook: event=${evt} words=${wc}`);
                     const recall = getRecallClient();
                     if (recall)
                         recall.handleWebhook(payload);
@@ -470,6 +492,50 @@ function startApiServer(workingDir, port) {
             console.log('[events] SSE client connected');
             return;
         }
+        // ── Meeting canvas: SSE stream the Recall webpage subscribes to ──────────
+        // The canvas page (frontend /meeting-canvas) opens this as an EventSource.
+        // Recall renders that page as the bot's camera+mic, so whatever we push here
+        // becomes what the meeting sees + hears. On connect we resync the current
+        // visual so a page reload doesn't blank the bot's camera.
+        if (req.method === 'GET' && url.pathname === '/canvas-stream') {
+            res.writeHead(200, {
+                'Content-Type': 'text/event-stream',
+                'Cache-Control': 'no-cache',
+                'Connection': 'keep-alive',
+                'X-Accel-Buffering': 'no',
+            });
+            res.write(`: canvas connected ${new Date().toISOString()}\n\n`);
+            res.write(`data: ${JSON.stringify(latestCanvasShow ?? { kind: 'show', mode: 'idle' })}\n\n`);
+            canvasClients.add(res);
+            const hb = setInterval(() => { try {
+                res.write(`: ping ${Date.now()}\n\n`);
+            }
+            catch { } }, 10_000);
+            req.on('close', () => { clearInterval(hb); canvasClients.delete(res); console.log('[canvas] client disconnected'); });
+            console.log(`[canvas] client connected (${canvasClients.size} total)`);
+            return;
+        }
+        // ── Meeting canvas: control endpoint (director / agent tool) ─────────────
+        // POST /canvas  { kind:'say', text } | { kind:'show', mode, title?, items?, url?, text? }
+        if (req.method === 'POST' && url.pathname === '/canvas') {
+            let body = '';
+            req.on('data', (c) => { body += c; });
+            req.on('end', () => {
+                try {
+                    const evt = JSON.parse(body || '{}');
+                    if (evt.kind !== 'say' && evt.kind !== 'show')
+                        throw new Error("kind must be 'say' or 'show'");
+                    pushCanvas(evt);
+                    res.writeHead(200, { 'Content-Type': 'application/json' });
+                    res.end(JSON.stringify({ ok: true, clients: canvasClients.size }));
+                }
+                catch (e) {
+                    res.writeHead(400, { 'Content-Type': 'application/json' });
+                    res.end(JSON.stringify({ error: e.message }));
+                }
+            });
+            return;
+        }
         // GET /sessions/export — stream a gzipped tar of ~/.claude/projects/ AND
         // ~/.claude/skills/ to the client. Both directories ship in one archive so
         // a sync covers conversations (projects/) and learned skills together —
@@ -4619,15 +4685,22 @@ async function main() {
                                 (process.env.FLY_APP_NAME
                                     ? `https://${process.env.FLY_APP_NAME}.fly.dev`
                                     : `http://localhost:${apiPort}`);
-                            // Polling architecture (post-2026-05-22): the bot joins by name
-                            // only — no output_media webpage, no LiveKit republish, no audio
-                            // pipeline at all. Recall captures the meeting audio internally
-                            // and we pull the transcript via its REST API every ~30s.
+                            // Transcript: Recall captures meeting audio internally; we get it
+                            // live via the realtime webhook (+ REST poll backstop). Output:
+                            // the bot casts the meeting canvas as its camera+mic (below), so
+                            // it can show visuals and speak into the meeting on demand.
                             await sendToFrontend({ type: 'meeting_joining', message: 'Osborn is joining your meeting...' });
-                            // 0.9.85: optional castUrl — a webpage to display as the bot's
-                            // camera (live feed / seeded site). Passed from the frontend or
-                            // env OSBORN_MEETING_CAST_URL for testing.
-                            const castUrl = data.castUrl || process.env.OSBORN_MEETING_CAST_URL || undefined;
+                            // Cast target = the bot's camera+mic webpage (Recall output_media).
+                            // DEFAULT is the meeting canvas (frontend /meeting-canvas), pointed
+                            // at THIS agent's public URL so it can subscribe to /canvas-stream
+                            // and become the bot's face (visuals) + voice (TTS it speaks). An
+                            // explicit castUrl from the frontend or OSBORN_MEETING_CAST_URL
+                            // overrides it (e.g. to cast a live feed or a seeded site instead).
+                            const frontendUrl = (process.env.OSBORN_FRONTEND_URL || 'https://www.voice-native.com').replace(/\/$/, '');
+                            const canvasUrl = /^https:\/\//.test(webhookBase)
+                                ? `${frontendUrl}/meeting-canvas?agent=${encodeURIComponent(webhookBase)}`
+                                : undefined;
+                            const castUrl = data.castUrl || process.env.OSBORN_MEETING_CAST_URL || canvasUrl;
                             const botId = await recallJoin.joinMeeting(meetingUrl, webhookBase, { castUrl });
                             const sessionId = currentLLM?.sessionId || currentResumeSessionId || 'default';
                             recallJoin.registerBot(botId, sessionId);

package/dist/recall-client.js

@@ -149,9 +149,19 @@ export class RecallClient extends EventEmitter {
         const bot = (await res.json());
         return bot.status_changes?.at(-1)?.code ?? 'unknown';
     }
+    // 0.9.86: track the last text we emitted per speaker so partials don't
+    // spam the LLM with every incremental word — only emit when the utterance
+    // grows meaningfully or finalizes.
+    #lastEmitted = new Map();
     handleWebhook(payload) {
-        // Only process final transcripts (transcript.data), skip partials
-        if (payload.event !== 'transcript.data')
+        // Accept BOTH finals (transcript.data) AND partials (transcript.partial_data).
+        // recallai_streaming in prioritize_low_latency mode emits partials DURING
+        // the call and finals lag — ignoring partials (the old behavior) meant no
+        // live transcript at all (confirmed 2026-07-29: 0 webhook-processed turns
+        // mid-call). We emit partials for liveness, deduped against the last text.
+        const isFinal = payload.event === 'transcript.data';
+        const isPartial = payload.event === 'transcript.partial_data';
+        if (!isFinal && !isPartial)
             return;
         const words = payload.data?.data?.words ?? [];
         const text = words.map(w => w.text).join(' ').trim();
@@ -159,7 +169,15 @@ export class RecallClient extends EventEmitter {
             return;
         const speaker = payload.data?.data?.participant?.name ?? 'Unknown';
         const botId = payload.data?.bot?.id ?? 'unknown';
-        this.emit('transcript', { botId, speaker, text });
+        const key = `${botId}:${speaker}`;
+        // Dedup: skip a partial that just repeats/prefixes what we already sent.
+        const prev = this.#lastEmitted.get(key) ?? '';
+        if (!isFinal && (text === prev || (prev && text.startsWith(prev) && text.length - prev.length < 8)))
+            return;
+        this.#lastEmitted.set(key, text);
+        if (isFinal)
+            this.#lastEmitted.delete(key); // reset for the next utterance
+        this.emit('transcript', { botId, speaker, text, partial: !isFinal });
     }
     registerBot(botId, sessionId) {
         this.#activeBots.set(botId, sessionId);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.85",
+  "version": "0.9.87",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {