osborn 0.9.44 → 0.9.46

package/Dockerfile.sandbox

@@ -54,9 +54,37 @@ EXPOSE 8741
 
 # Entrypoint: credential persistence + onboarding suppression + start
 COPY <<'ENTRYPOINT' /entrypoint.sh
-#!/bin/sh
+#!/bin/bash
 set -e
 
+# Persistent log capture for post-disconnect upload to Supabase Storage.
+# Fly Machines has NO REST endpoint for fetching machine logs (the previous
+# implementation hit /v1/apps/{app}/machines/{id}/logs which returns 404 → that
+# 404 error string was getting uploaded as "the log" for every session).
+# Volume-backed /workspace/osborn.log survives reboots and is readable via
+# the documented /exec endpoint (`tail -n 500 /workspace/osborn.log`).
+#
+# We use process substitution to tee output to BOTH the log file AND the
+# original stdout (so `flyctl logs` keeps working for ad-hoc debugging).
+# Requires bash, hence the #!/bin/bash shebang.
+#
+# Size cap: if log grows past 100 MB, keep only the last 50 MB. Prevents
+# disk-fill from long-running retry loops (we saw 17h × ~1 line/min = ~1000
+# lines today, but anything connecting to LiveKit produces orders of
+# magnitude more output).
+LOGFILE=/workspace/osborn.log
+mkdir -p /workspace
+if [ -f "$LOGFILE" ] && [ "$(stat -c%s "$LOGFILE" 2>/dev/null || echo 0)" -gt 104857600 ]; then
+  echo "[sandbox] Rotating /workspace/osborn.log (>100MB, keeping last 50MB)"
+  tail -c 52428800 "$LOGFILE" > "$LOGFILE.tmp" && mv "$LOGFILE.tmp" "$LOGFILE"
+fi
+echo "[sandbox] === boot at $(date -Iseconds) ===" >> "$LOGFILE"
+# Redirect all subsequent stdout+stderr from this script (and the eventual
+# `exec osborn`) to both the original fd (Fly stdout collector) AND the
+# append-only log file. tee runs as a backgrounded subshell that survives
+# the final exec replacement.
+exec > >(tee -a "$LOGFILE") 2>&1
+
 # Claude credential persistence (volume at /workspace)
 mkdir -p /workspace/.claude
 rm -rf /root/.claude

package/dist/claude-llm.js

@@ -660,6 +660,28 @@ export class ClaudeLLM extends llm.LLM {
                         }
                     }
                 }
+                // Compaction signals observed on the SDK iterator (parallel to hook path).
+                // The SDK emits TWO message subtypes during compaction independent of
+                // hook registration:
+                //   - type:'system', subtype:'compact_boundary'  (with compact_metadata)
+                //   - type:'system', subtype:'status', status:'compacting' | null
+                // We DON'T route these through onCompactionEvent (to avoid duplicate
+                // chat bubbles — hooks already do that), but we LOG them. If the hook
+                // path ever silently fails, these logs will be the only signal that
+                // compaction actually happened — making the failure obvious in fly logs.
+                if (msg.type === 'system' && msg.subtype === 'compact_boundary') {
+                    const meta = msg.compact_metadata || {};
+                    console.log(`[COMPACT-SDK-ITER] compact_boundary observed: trigger=${meta.trigger ?? '?'} pre_tokens=${meta.pre_tokens ?? '?'} preserved=${meta.preserved_segment ? 'yes' : 'no'}`);
+                    // Fire onCompactionEvent as a FALLBACK if hooks didn't fire — we
+                    // detect this by checking whether we've seen 'compaction_started'
+                    // recently. For now, log only; can wire as fallback if hooks fail.
+                }
+                if (msg.type === 'system' && msg.subtype === 'status') {
+                    const status = msg.status;
+                    if (status === 'compacting' || status === null) {
+                        console.log(`[COMPACT-SDK-ITER] status change: ${status === 'compacting' ? 'ENTERED compacting' : 'EXITED compacting'} session=${msg.session_id?.substring(0, 8) ?? '?'}`);
+                    }
+                }
                 // Checkpoint capture
                 if (msg.type === 'user' && msg.uuid) {
                     callbacks.onCheckpoint(msg.uuid);

package/dist/index.js

@@ -147,6 +147,24 @@ process.on('uncaughtException', (error) => {
 // ============================================================
 // Module-level room code so the HTTP server can expose it via GET /room-code
 let currentRoomCode = null;
+// Module-level LiveKit connection state. Shared between main() (which runs the
+// connect-with-retry loop) and the /health handler in startApiServer (which
+// reports it to the frontend so the user sees a meaningful error instead of a
+// dashboard redirect when LiveKit is unreachable / out of quota / etc).
+//
+// We deliberately do NOT 503 /health on connect failure — Fly's machine
+// health-check uses /health, and returning non-2xx triggers a restart loop
+// which (a) burns the same failing LiveKit calls every 30s and (b) gets the
+// machine killed after 3 failed restarts. By staying 200 OK and surfacing the
+// status as a field, we keep the container alive long enough for LiveKit to
+// recover (auto-retry) or for the user to read the error and upgrade quota.
+const livekitState = {
+    status: 'connecting',
+    error: null,
+    errorCode: null,
+    lastAttemptAt: null,
+    attemptCount: 0,
+};
 function startApiServer(workingDir, port) {
     const server = createServer(async (req, res) => {
         // CORS headers for cloud frontend
@@ -221,7 +239,22 @@ function startApiServer(workingDir, port) {
             }
             catch { /* version optional */ }
             res.writeHead(200, { 'Content-Type': 'application/json' });
-            res.end(JSON.stringify({ status: 'ok', workingDir, version }));
+            res.end(JSON.stringify({
+                status: 'ok',
+                workingDir,
+                version,
+                // LiveKit subsystem status — frontend can use this to surface a real
+                // error instead of treating the sandbox as totally broken. The HTTP
+                // status code stays 200 so Fly health-check stays green and the
+                // container isn't restart-looped while LiveKit is unreachable.
+                livekit: {
+                    status: livekitState.status,
+                    error: livekitState.error,
+                    errorCode: livekitState.errorCode,
+                    attemptCount: livekitState.attemptCount,
+                    lastAttemptAt: livekitState.lastAttemptAt,
+                },
+            }));
             return;
         }
         // POST /webhook/recall — Recall.ai real-time transcript webhooks
@@ -1545,31 +1578,52 @@ async function main() {
     // previously skipped this entirely, so compaction events fired into the void
     // in pipeline mode.
     const buildOnCompactionEvent = () => (event) => {
+        // CRITICAL diagnostic — every compaction event MUST appear in the agent
+        // log first. If you don't see [COMPACT-AGENT-RX] for an event type, the
+        // ClaudeLLM hook isn't calling this callback (most likely culprits:
+        // PreCompact/PostCompact hook never registered, or the callback wasn't
+        // passed through createPipelineDirectLLM's opts). If you see RX but no
+        // CHAT-EMIT, the type didn't match the chat-emit branch. If you see both
+        // but the frontend log never shows [COMPACT-FRONTEND], the data channel
+        // dropped the message (room not connected, payload too big, etc.).
+        console.log(`[COMPACT-AGENT-RX] type=${event.type} keys=[${Object.keys(event).filter(k => k !== 'type').join(',')}]`);
         try {
             // Raw event → banner state machine (compaction_started/progress/complete handlers in VoiceRoom.tsx).
             sendToFrontend({ ...event });
+            console.log(`[COMPACT-AGENT-RAW-SENT] type=${event.type}`);
             // Inline chat bubble — reuses the existing claude_output path that's already working.
             if (event.type === 'compaction_started') {
                 const triggerLabel = event.trigger ? ` (${event.trigger})` : '';
+                const text = `🧠 _Crystallizing session memory…_${triggerLabel}`;
                 sendToFrontend({
                     type: 'claude_output',
-                    text: `🧠 _Crystallizing session memory…_${triggerLabel}`,
+                    text,
                     agentRole: 'direct',
                 });
+                console.log(`[COMPACT-AGENT-CHAT-EMIT] started → "${text.substring(0, 60)}"`);
             }
             else if (event.type === 'compaction_complete') {
                 const n = event.skillsWritten ?? 0;
                 const names = Array.isArray(event.skillNames) && event.skillNames.length > 0
                     ? ` — ${event.skillNames.join(', ')}`
                     : '';
+                const text = `🧠 Memory crystallized — ${n} skill${n === 1 ? '' : 's'} updated${names}.`;
                 sendToFrontend({
                     type: 'claude_output',
-                    text: `🧠 Memory crystallized — ${n} skill${n === 1 ? '' : 's'} updated${names}.`,
+                    text,
                     agentRole: 'direct',
                 });
+                console.log(`[COMPACT-AGENT-CHAT-EMIT] complete → "${text.substring(0, 80)}"`);
             }
+            else {
+                // progress events don't get a chat bubble (too noisy) — they only feed the banner.
+                // Log at debug level so we can confirm they fired.
+                console.log(`[COMPACT-AGENT-CHAT-SKIP] type=${event.type} (progress events feed the banner only, no inline bubble)`);
+            }
+        }
+        catch (err) {
+            console.error(`[COMPACT-AGENT-ERROR] ${err.message}`);
         }
-        catch { /* non-fatal */ }
     };
     // Create DIRECT session (STT + Claude Agent SDK + TTS)
     async function createDirectSession(resumeSessionId, llmOverride) {
@@ -3931,22 +3985,87 @@ async function main() {
     // ============================================================
     // Connect to Room
     // ============================================================
-    try {
-        await room.connect(livekitUrl, jwt, {
-            autoSubscribe: true,
-            dynacast: true,
-        });
-        localParticipant = room.localParticipant;
-        console.log('✅ Connected to room:', roomName);
-        console.log('\n⏳ Waiting for user to connect...');
-        console.log(`   Room: ${roomCode}\n`);
-        // Keep process alive
-        await new Promise(() => { });
-    }
-    catch (err) {
-        console.error('❌ Failed to connect:', err);
-        process.exit(1);
-    }
+    // Connect to LiveKit with retry-on-failure.
+    //
+    // Earlier behavior: a single attempt followed by `process.exit(1)` on error.
+    // Combined with Fly's restart policy this produced a tight restart loop that
+    // (a) hit the same 429 / auth error every ~30s, (b) burned the LiveKit
+    // quota's retry budget, and (c) hit Fly's max-restart-count (3) and killed
+    // the machine — at which point the frontend's /api/sandbox probe saw the
+    // sandbox as failed and bounced the user back to the dashboard with no
+    // useful error.
+    //
+    // New behavior: bounded-backoff retry, infinite attempts. The API server
+    // stays up the whole time serving /health (which surfaces the LiveKit error
+    // as a field, NOT as an HTTP failure — see the /health handler comment).
+    // When the underlying issue is resolved (quota reset, key fixed, LiveKit
+    // service back), the next retry succeeds and the agent picks up where it
+    // left off without anyone needing to manually restart.
+    //
+    // Backoff: 5s → 10s → 20s → 40s → 60s (capped). Resets to 5s after each
+    // successful connect (so a single transient hiccup doesn't disable fast
+    // recovery on the next disconnect).
+    const connectWithRetry = async () => {
+        const backoffSchedule = [5_000, 10_000, 20_000, 40_000, 60_000];
+        let backoffIdx = 0;
+        while (true) {
+            livekitState.status = livekitState.attemptCount === 0 ? 'connecting' : 'retrying';
+            livekitState.lastAttemptAt = Date.now();
+            livekitState.attemptCount += 1;
+            try {
+                await room.connect(livekitUrl, jwt, {
+                    autoSubscribe: true,
+                    dynacast: true,
+                });
+                localParticipant = room.localParticipant;
+                livekitState.status = 'connected';
+                livekitState.error = null;
+                livekitState.errorCode = null;
+                backoffIdx = 0;
+                console.log('✅ Connected to room:', roomName);
+                console.log('\n⏳ Waiting for user to connect...');
+                console.log(`   Room: ${roomCode}\n`);
+                return;
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                // Categorize the error so the frontend can show specific guidance.
+                // Substring matching on the message because LiveKit's rtc-node SDK
+                // wraps the underlying HTTP status in a generic ConnectError.
+                let errorCode;
+                if (/429|connection minutes limit/i.test(msg))
+                    errorCode = 'quota_exceeded';
+                else if (/401|403|unauthorized|invalid/i.test(msg))
+                    errorCode = 'auth';
+                else if (/ENOTFOUND|ECONNREFUSED|ETIMEDOUT|network/i.test(msg))
+                    errorCode = 'network';
+                else
+                    errorCode = 'unknown';
+                livekitState.status = 'failed';
+                livekitState.error = msg;
+                livekitState.errorCode = errorCode;
+                const waitMs = backoffSchedule[Math.min(backoffIdx, backoffSchedule.length - 1)];
+                backoffIdx += 1;
+                console.error(`❌ LiveKit connect failed (${errorCode}, attempt ${livekitState.attemptCount}): ${msg.substring(0, 200)}`);
+                console.error(`   Retrying in ${waitMs / 1000}s — process staying alive; /health remains 200 with livekit.status='failed'`);
+                await new Promise(r => setTimeout(r, waitMs));
+                // loop — try again
+            }
+        }
+    };
+    // Fire and forget; the retry loop keeps the process alive on its own (so
+    // we don't need the explicit `new Promise(() => {})` keepalive anymore).
+    // Errors that escape the retry loop should never happen, but if they do,
+    // log them rather than crash.
+    connectWithRetry().catch(err => {
+        console.error('❌ Unrecoverable error in LiveKit retry loop (should not happen):', err);
+        livekitState.status = 'failed';
+        livekitState.error = err instanceof Error ? err.message : String(err);
+        livekitState.errorCode = 'unrecoverable';
+    });
+    // Keep main() alive forever — without this the await chain ends and Node
+    // exits 0, which Fly treats as a clean shutdown.
+    await new Promise(() => { });
 }
 // Run
 main().catch(console.error);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.44",
+  "version": "0.9.46",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {