groove-dev 0.27.212 → 0.27.213

package/daemon-bridge.js

@@ -0,0 +1,87 @@
+// FSL-1.1-Apache-2.0 — see LICENSE
+import { existsSync, readdirSync } from 'fs';
+import { createRequire } from 'module';
+import { dirname, join } from 'path';
+import { pathToFileURL } from 'url';
+
+const port = 31415;
+const projectDir = process.argv[2] || process.cwd();
+
+function preflightCheck(daemonPath) {
+  if (!existsSync(daemonPath)) {
+    throw new Error(
+      `Daemon entry point not found at ${daemonPath}. ` +
+      'The app may not have been packaged correctly — try reinstalling Groove.'
+    );
+  }
+
+  const daemonDir = dirname(daemonPath);
+  const require = createRequire(daemonPath);
+  const critical = ['express', 'ws'];
+  const missing = critical.filter(dep => {
+    try { require.resolve(dep); return false; } catch { return true; }
+  });
+
+  if (missing.length) {
+    const diag = [`Missing deps: ${missing.join(', ')}`, `GROOVE_DAEMON_PATH: ${daemonPath}`];
+    let walkDir = daemonDir;
+    for (let i = 0; i < 5 && walkDir !== dirname(walkDir); i++) {
+      try {
+        const entries = readdirSync(walkDir);
+        diag.push(`${walkDir}/: [${entries.join(', ')}]`);
+      } catch { diag.push(`${walkDir}/: (unreadable)`); }
+      walkDir = dirname(walkDir);
+    }
+    throw new Error(
+      `Daemon is missing dependencies. ` +
+      `Diagnostics:\n${diag.join('\n')}\n` +
+      'The app bundle may be incomplete — try reinstalling Groove.'
+    );
+  }
+}
+
+async function main() {
+  let Daemon;
+  const daemonPath = process.env.GROOVE_DAEMON_PATH;
+
+  if (daemonPath) {
+    preflightCheck(daemonPath);
+    const mod = await import(pathToFileURL(daemonPath).href);
+    Daemon = mod.Daemon;
+  } else {
+    const mod = await import('@groove-dev/daemon');
+    Daemon = mod.Daemon;
+  }
+
+  const daemon = new Daemon({ port, projectDir });
+  await daemon.start();
+
+  process.send({ type: 'ready', port: daemon.port });
+
+  process.on('message', (msg) => {
+    if (msg.type === 'auth-token') {
+      (async () => {
+        try { await daemon.setAuthToken(msg.token); } catch (err) {
+          process.stderr.write(`[daemon-bridge] setAuthToken failed: ${err.message}\n`);
+        }
+      })();
+    }
+  });
+
+  process.on('SIGTERM', async () => {
+    await daemon.stop();
+    process.exit(0);
+  });
+
+  process.on('SIGINT', async () => {
+    await daemon.stop();
+    process.exit(0);
+  });
+}
+
+main().catch((err) => {
+  if (process.send) {
+    process.send({ type: 'error', message: err.message });
+  }
+  process.exit(1);
+});

package/node_modules/@groove-dev/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@groove-dev/cli",
-  "version": "0.27.212",
+  "version": "0.27.213",
   "description": "GROOVE CLI — manage AI coding agents from your terminal",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",

package/node_modules/@groove-dev/daemon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@groove-dev/daemon",
-  "version": "0.27.212",
+  "version": "0.27.213",
   "description": "GROOVE daemon — agent orchestration engine",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",

package/node_modules/@groove-dev/daemon/src/tunnel-manager.js

@@ -20,6 +20,12 @@ const MAX_PORT_ATTEMPTS = 10;
 const HEALTH_INTERVAL = 30000;
 const HEALTH_TIMEOUT = 5000;
 const MAX_FAIL_COUNT = 3;
+// Long-timeout probe used to CONFIRM death before killing a tunnel — a busy
+// remote daemon can sit on /api/health well past the 5s routine probe.
+const CONFIRM_TIMEOUT = 15000;
+// At most one automatic rebuild per tunnel per window; beyond that it stays
+// disconnected rather than thrashing against a host that keeps dying.
+const REBUILD_COOLDOWN_MS = 10 * 60 * 1000;
 
 const INJECTION_CHARS = /[;|&`$(){}[\]<>!#\n\r\\]/;
 
@@ -295,6 +301,8 @@ export class TunnelManager {
         return { localPort: existing.localPort, pid: existing.pid, name: config.name };
       }
       console.log(`[Groove:Tunnel] ${config.name}: existing tunnel is not responding — rebuilding`);
+      // Reuse the dead tunnel's port so any GUI window pointed at it heals.
+      opts = { ...opts, preferredPort: opts.preferredPort || existing.localPort };
       await this.disconnect(id);
     }
 
@@ -328,7 +336,14 @@ export class TunnelManager {
     // Establish SSH tunnel
     this.daemon.broadcast({ type: 'tunnel.status', data: { id, step: 'connecting' } });
 
-    const localPort = await this._findAvailablePort();
+    // A rebuild wants its old port back: the remote GUI window is pointed at it
+    // and will self-heal over WebSocket retry only if the port stays the same.
+    let localPort;
+    if (opts.preferredPort && !(await this._isPortInUse(opts.preferredPort))) {
+      localPort = opts.preferredPort;
+    } else {
+      localPort = await this._findAvailablePort();
+    }
     const target = `${config.user}@${config.host}`;
     const keyArgs = config.sshKeyPath ? ['-i', config.sshKeyPath] : [];
 
@@ -445,7 +460,7 @@ export class TunnelManager {
   // Does the tunnel actually serve a request, as opposed to merely holding an
   // open listening socket? A wedged forward passes a TCP connect test but never
   // answers, so only an HTTP round-trip proves it.
-  async _tunnelResponds(localPort, timeoutMs = HEALTH_TIMEOUT) {
+  async _tunnelResponds(localPort, timeoutMs = this.healthTimeout ?? HEALTH_TIMEOUT) {
     try {
       const res = await fetch(`http://localhost:${localPort}/api/health`, {
         signal: AbortSignal.timeout(timeoutMs),
@@ -915,22 +930,26 @@ export class TunnelManager {
 
   async _healthCheckPass() {
     // Timers don't fire while the machine is asleep, so an interval that should
-    // have run every HEALTH_INTERVAL arriving far later means we just woke up.
-    // Every tunnel is suspect at that point: verify them now rather than waiting
-    // for MAX_FAIL_COUNT normal cycles to notice.
+    // have run every HEALTH_INTERVAL arriving far later means we PROBABLY just
+    // woke up — but not certainly: this daemon also blocks its event loop for
+    // long stretches (execFileSync ssh calls in test/upgrade paths), which
+    // produces the same gap on a machine that never slept. So a gap only makes
+    // tunnels *suspect* — it fast-tracks them to the confirmation ladder below.
+    // It must never lower the bar for killing one (that misdiagnosis dropped a
+    // healthy DGX tunnel twice in ten minutes).
     const now = Date.now();
     const gap = now - (this._lastHealthCheck || now);
     this._lastHealthCheck = now;
-    const wokeFromSleep = gap > HEALTH_INTERVAL * 3;
-    if (wokeFromSleep && this.active.size > 0) {
-      console.log(`[Groove:Tunnel] Detected a ${Math.round(gap / 1000)}s gap (system sleep) — verifying tunnels`);
+    const suspectAll = gap > HEALTH_INTERVAL * 3;
+    if (suspectAll && this.active.size > 0) {
+      console.log(`[Groove:Tunnel] ${Math.round(gap / 1000)}s timer gap (sleep or blocked loop) — verifying tunnels`);
     }
 
     for (const [id, conn] of this.active) {
       try {
         const start = Date.now();
         const res = await fetch(`http://localhost:${conn.localPort}/api/health`, {
-          signal: AbortSignal.timeout(HEALTH_TIMEOUT),
+          signal: AbortSignal.timeout(this.healthTimeout ?? HEALTH_TIMEOUT),
         });
         if (res.ok) {
           conn.latencyMs = Date.now() - start;
@@ -942,22 +961,29 @@ export class TunnelManager {
         }
       } catch {
         conn.failCount = (conn.failCount || 0) + 1;
-        // After a sleep gap, one failure is enough — the tunnel was almost
-        // certainly cut with the network.
-        const limit = wokeFromSleep ? 1 : MAX_FAIL_COUNT;
-        if (conn.failCount >= limit) {
-          conn.healthy = false;
-          this.daemon.broadcast({ type: 'tunnel.unhealthy', data: { id } });
-
-          // Reap it. Previously a dead tunnel was only FLAGGED, so it stayed in
-          // `active` indefinitely and every later connect() handed back its dead
-          // port — the reason reconnecting never helped and only a full restart
-          // did. Removing it means the next connect() builds a real tunnel.
-          if (conn.failCount >= limit + 1) {
-            console.log(`[Groove:Tunnel] Reaping dead tunnel ${id} after ${conn.failCount} failed checks`);
-            this.daemon.broadcast({ type: 'tunnel.status', data: { id, step: 'reaping' } });
-            await this.disconnect(id);
-            continue;
+        // A failed 5s probe is WEAK evidence: it can't distinguish a dead
+        // tunnel from a remote daemon that's briefly busy or our own blocked
+        // event loop. Never kill on it. Once failures accumulate (or a timer
+        // gap makes everything suspect), run the confirmation ladder, which
+        // can — a healthy verdict there resets the count.
+        if (conn.failCount >= MAX_FAIL_COUNT || suspectAll) {
+          const verdict = await this._confirmDead(conn);
+          if (verdict === 'alive') {
+            conn.failCount = 0;
+            conn.healthy = true;
+            conn._wedgedStreak = 0;
+          } else {
+            conn.healthy = false;
+            this.daemon.broadcast({ type: 'tunnel.unhealthy', data: { id } });
+            // 'wedged' (port accepts, HTTP silent even at long timeout) is the
+            // one verdict with a false-positive path — a remote event loop
+            // blocked 15s+ — so demand it twice in a row. proc-dead/port-dead
+            // are unambiguous: the ssh client is gone or nothing is listening.
+            conn._wedgedStreak = verdict === 'wedged' ? (conn._wedgedStreak || 0) + 1 : 0;
+            if (verdict !== 'wedged' || conn._wedgedStreak >= 2) {
+              await this._reapAndRebuild(id, conn, verdict);
+              continue;
+            }
           }
         }
       }
@@ -968,6 +994,51 @@ export class TunnelManager {
     }
   }
 
+  // Escalating evidence that a tunnel is actually dead, not merely slow:
+  //   'alive'     — answered a long-timeout HTTP probe; leave it alone
+  //   'proc-dead' — the ssh client process is gone
+  //   'port-dead' — nothing is listening on the local port
+  //   'wedged'    — port accepts TCP but HTTP never answers (dead forward)
+  async _confirmDead(conn) {
+    if (await this._tunnelResponds(conn.localPort, this.confirmTimeout ?? CONFIRM_TIMEOUT)) return 'alive';
+    if (conn.pid) {
+      try { process.kill(conn.pid, 0); } catch { return 'proc-dead'; }
+    }
+    if (!(await this._isPortInUse(conn.localPort))) return 'port-dead';
+    return 'wedged';
+  }
+
+  // Tear down a confirmed-dead tunnel and immediately rebuild it on the SAME
+  // local port. The remote GUI window points at that port and its WebSocket
+  // retries every 2s, so a same-port rebuild heals an open window without the
+  // user noticing. Only if the rebuild fails does this surface as a disconnect.
+  // Rate-limited so a genuinely dead host degrades to disconnected instead of
+  // thrashing reconnect attempts forever.
+  async _reapAndRebuild(id, conn, reason) {
+    const { localPort } = conn;
+    console.log(`[Groove:Tunnel] Tunnel ${id} confirmed dead (${reason}) — rebuilding`);
+    this.daemon.audit.log('tunnel.reap', { id, reason, failCount: conn.failCount });
+    await this.disconnect(id);
+
+    const lastRebuild = this._rebuildAt?.get(id) || 0;
+    if (Date.now() - lastRebuild < REBUILD_COOLDOWN_MS) {
+      console.log(`[Groove:Tunnel] ${id} already auto-rebuilt recently — leaving disconnected`);
+      return;
+    }
+    this._rebuildAt = this._rebuildAt || new Map();
+    this._rebuildAt.set(id, Date.now());
+
+    try {
+      this.daemon.broadcast({ type: 'tunnel.status', data: { id, step: 'reconnecting' } });
+      await this.connect(id, { preferredPort: localPort });
+      console.log(`[Groove:Tunnel] ${id} rebuilt on port ${localPort}`);
+    } catch (err) {
+      console.warn(`[Groove:Tunnel] Auto-rebuild of ${id} failed: ${err.message}`);
+      // disconnect() above already broadcast tunnel.disconnected — the GUI is
+      // consistent; the user can reconnect manually when the host is back.
+    }
+  }
+
   // Signal 0 only tests for existence — no signal is delivered.
   async _waitForExit(pid, timeoutMs) {
     const deadline = Date.now() + timeoutMs;

package/node_modules/@groove-dev/daemon/test/tunnel-manager.test.js

@@ -66,6 +66,10 @@ describe('TunnelManager — wake-from-sleep recovery', () => {
     grooveDir = mkdtempSync(resolve(tmpdir(), 'groove-tunnel-'));
     daemon = makeDaemon(grooveDir);
     mgr = new TunnelManager(daemon);
+    // Shrink probe timeouts (production: 5s routine / 15s confirm) so the
+    // wedged-listener tests don't burn real minutes waiting them out.
+    mgr.healthTimeout = 400;
+    mgr.confirmTimeout = 1200;
   });
 
   afterEach(() => {
@@ -84,7 +88,7 @@ describe('TunnelManager — wake-from-sleep recovery', () => {
   it('_tunnelResponds rejects a tunnel that accepts but never answers', async () => {
     const wedged = await startWedgedListener();
     try {
-      assert.equal(await mgr._tunnelResponds(wedged.port, 1500), false);
+      assert.equal(await mgr._tunnelResponds(wedged.port, 800), false);
     } finally { wedged.close(); }
   });
 
@@ -132,19 +136,25 @@ describe('TunnelManager — wake-from-sleep recovery', () => {
     } finally { live.close(); }
   });
 
-  it('the health pass reaps a dead tunnel so the next connect starts clean', async () => {
+  it('the health pass reaps a proc-dead tunnel and tries to rebuild it', async () => {
     const wedged = await startWedgedListener();
     try {
       mgr.saved.set('s19', { id: 's19', name: 'S19 Agency', host: 'example.invalid', user: 'ops', port: 22 });
       mgr.active.set('s19', {
-        pid: 999999, localPort: wedged.port, healthy: true,
+        pid: 999999, // no such process → verdict 'proc-dead', reap on first confirm
+        localPort: wedged.port, healthy: true,
         failCount: 99, // already past the failure limit
         startedAt: new Date().toISOString(),
       });
 
+      const rebuilds = [];
+      mgr.connect = async (id, opts) => { rebuilds.push({ id, opts }); throw new Error('host unreachable'); };
+
       await mgr._healthCheckAll();
 
       assert.equal(mgr.active.has('s19'), false, 'dead tunnel removed from active');
+      assert.equal(rebuilds.length, 1, 'an automatic rebuild was attempted');
+      assert.equal(rebuilds[0].opts.preferredPort, wedged.port, 'rebuild asks for the same port');
       assert.ok(
         daemon.broadcasts.some((b) => b.type === 'tunnel.unhealthy'),
         'the GUI is told the tunnel went unhealthy',
@@ -152,24 +162,84 @@ describe('TunnelManager — wake-from-sleep recovery', () => {
     } finally { wedged.close(); }
   });
 
-  it('treats a long timer gap as a sleep and drops the failure tolerance', async () => {
+  it('a slow-but-alive tunnel is never reaped — the long confirm probe clears it', async () => {
+    // Answers /api/health slower than the routine probe allows but inside the
+    // confirmation window. This is a busy DGX, not a dead tunnel.
+    const sockets = [];
+    const slow = createServer((sock) => {
+      sockets.push(sock);
+      sock.on('data', () => setTimeout(() => {
+        const body = '{"ok":true}';
+        try {
+          sock.end('HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n'
+            + `Content-Length: ${body.length}\r\nConnection: close\r\n\r\n${body}`);
+        } catch { /* closed */ }
+      }, 700));
+    });
+    await new Promise((r) => slow.listen(0, '127.0.0.1', r));
+    const port = slow.address().port;
+    try {
+      mgr.saved.set('dgx', { id: 'dgx', name: 'Axom Spark', host: 'edgexpert.local', user: 'rok', port: 22 });
+      mgr.active.set('dgx', {
+        pid: process.pid, // definitely alive
+        localPort: port, healthy: true, failCount: 99,
+        startedAt: new Date().toISOString(),
+      });
+
+      mgr.connect = async () => { throw new Error('rebuild must not be attempted'); };
+      await mgr._healthCheckAll();
+
+      const conn = mgr.active.get('dgx');
+      assert.ok(conn, 'the tunnel survived');
+      assert.equal(conn.healthy, true, 'confirmed alive by the long probe');
+      assert.equal(conn.failCount, 0, 'failure count reset');
+    } finally {
+      for (const s of sockets) s.destroy();
+      slow.close();
+    }
+  });
+
+  it('a wedged tunnel needs two consecutive confirmations before it is reaped', { timeout: 30000 }, async () => {
     const wedged = await startWedgedListener();
     try {
       mgr.saved.set('s19', { id: 's19', name: 'S19 Agency', host: 'example.invalid', user: 'ops', port: 22 });
       mgr.active.set('s19', {
-        pid: 999999, localPort: wedged.port, healthy: true, failCount: 0,
+        pid: process.pid, // ssh "alive" → verdict is 'wedged', not 'proc-dead'
+        localPort: wedged.port, healthy: true, failCount: 99,
         startedAt: new Date().toISOString(),
       });
+      mgr.connect = async () => { throw new Error('unreachable'); };
 
-      // Last check was 10 minutes ago — the machine was asleep.
-      mgr._lastHealthCheck = Date.now() - 10 * 60 * 1000;
       await mgr._healthCheckAll();
+      assert.equal(mgr.active.has('s19'), true, 'first wedged verdict only flags it');
+      assert.equal(mgr.active.get('s19').healthy, false);
 
-      const conn = mgr.active.get('s19');
-      assert.equal(conn?.healthy, false, 'one failure after a sleep gap is enough');
+      mgr.active.get('s19').failCount = 99; // still failing next tick
+      await mgr._healthCheckAll();
+      assert.equal(mgr.active.has('s19'), false, 'second wedged verdict reaps it');
     } finally { wedged.close(); }
   });
 
+  it('a timer gap fast-tracks confirmation but cannot kill a live tunnel', async () => {
+    const live = await startHealthyListener();
+    try {
+      mgr.saved.set('dgx', { id: 'dgx', name: 'Axom Spark', host: 'edgexpert.local', user: 'rok', port: 22 });
+      mgr.active.set('dgx', {
+        pid: process.pid, localPort: live.port, healthy: true, failCount: 0,
+        startedAt: new Date().toISOString(),
+      });
+
+      // 10-minute gap: sleep — or just a blocked event loop on a busy daemon.
+      mgr._lastHealthCheck = Date.now() - 10 * 60 * 1000;
+      mgr.connect = async () => { throw new Error('rebuild must not be attempted'); };
+      await mgr._healthCheckAll();
+
+      const conn = mgr.active.get('dgx');
+      assert.ok(conn, 'live tunnel survived the suspicious gap');
+      assert.equal(conn.healthy, true);
+    } finally { live.close(); }
+  });
+
   it('_waitForPortFree reports a released port', async () => {
     const live = await startHealthyListener();
     assert.equal(await mgr._waitForPortFree(live.port, 600), false, 'still held while listening');

package/node_modules/@groove-dev/gui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@groove-dev/gui",
-  "version": "0.27.212",
+  "version": "0.27.213",
   "description": "GROOVE GUI — visual agent control plane",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "groove-dev",
-  "version": "0.27.212",
+  "version": "0.27.213",
   "description": "Open-source agent orchestration layer — the AI company OS. Local model agent engine (GGUF/Ollama/llama-server), HuggingFace model browser, MCP integrations (Slack, Gmail, Stripe, 15+), agent scheduling (cron), business roles (CMO, CFO, EA). GUI dashboard, multi-agent coordination, zero cold-start, infinite sessions. Works with Claude Code, Codex, Gemini CLI, Ollama, any local model.",
   "license": "FSL-1.1-Apache-2.0",
   "author": "Groove Dev <hello@groovedev.ai> (https://groovedev.ai)",

package/packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@groove-dev/cli",
-  "version": "0.27.212",
+  "version": "0.27.213",
   "description": "GROOVE CLI — manage AI coding agents from your terminal",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",

package/packages/daemon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@groove-dev/daemon",
-  "version": "0.27.212",
+  "version": "0.27.213",
   "description": "GROOVE daemon — agent orchestration engine",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",

package/packages/daemon/src/tunnel-manager.js

@@ -20,6 +20,12 @@ const MAX_PORT_ATTEMPTS = 10;
 const HEALTH_INTERVAL = 30000;
 const HEALTH_TIMEOUT = 5000;
 const MAX_FAIL_COUNT = 3;
+// Long-timeout probe used to CONFIRM death before killing a tunnel — a busy
+// remote daemon can sit on /api/health well past the 5s routine probe.
+const CONFIRM_TIMEOUT = 15000;
+// At most one automatic rebuild per tunnel per window; beyond that it stays
+// disconnected rather than thrashing against a host that keeps dying.
+const REBUILD_COOLDOWN_MS = 10 * 60 * 1000;
 
 const INJECTION_CHARS = /[;|&`$(){}[\]<>!#\n\r\\]/;
 
@@ -295,6 +301,8 @@ export class TunnelManager {
         return { localPort: existing.localPort, pid: existing.pid, name: config.name };
       }
       console.log(`[Groove:Tunnel] ${config.name}: existing tunnel is not responding — rebuilding`);
+      // Reuse the dead tunnel's port so any GUI window pointed at it heals.
+      opts = { ...opts, preferredPort: opts.preferredPort || existing.localPort };
       await this.disconnect(id);
     }
 
@@ -328,7 +336,14 @@ export class TunnelManager {
     // Establish SSH tunnel
     this.daemon.broadcast({ type: 'tunnel.status', data: { id, step: 'connecting' } });
 
-    const localPort = await this._findAvailablePort();
+    // A rebuild wants its old port back: the remote GUI window is pointed at it
+    // and will self-heal over WebSocket retry only if the port stays the same.
+    let localPort;
+    if (opts.preferredPort && !(await this._isPortInUse(opts.preferredPort))) {
+      localPort = opts.preferredPort;
+    } else {
+      localPort = await this._findAvailablePort();
+    }
     const target = `${config.user}@${config.host}`;
     const keyArgs = config.sshKeyPath ? ['-i', config.sshKeyPath] : [];
 
@@ -445,7 +460,7 @@ export class TunnelManager {
   // Does the tunnel actually serve a request, as opposed to merely holding an
   // open listening socket? A wedged forward passes a TCP connect test but never
   // answers, so only an HTTP round-trip proves it.
-  async _tunnelResponds(localPort, timeoutMs = HEALTH_TIMEOUT) {
+  async _tunnelResponds(localPort, timeoutMs = this.healthTimeout ?? HEALTH_TIMEOUT) {
     try {
       const res = await fetch(`http://localhost:${localPort}/api/health`, {
         signal: AbortSignal.timeout(timeoutMs),
@@ -915,22 +930,26 @@ export class TunnelManager {
 
   async _healthCheckPass() {
     // Timers don't fire while the machine is asleep, so an interval that should
-    // have run every HEALTH_INTERVAL arriving far later means we just woke up.
-    // Every tunnel is suspect at that point: verify them now rather than waiting
-    // for MAX_FAIL_COUNT normal cycles to notice.
+    // have run every HEALTH_INTERVAL arriving far later means we PROBABLY just
+    // woke up — but not certainly: this daemon also blocks its event loop for
+    // long stretches (execFileSync ssh calls in test/upgrade paths), which
+    // produces the same gap on a machine that never slept. So a gap only makes
+    // tunnels *suspect* — it fast-tracks them to the confirmation ladder below.
+    // It must never lower the bar for killing one (that misdiagnosis dropped a
+    // healthy DGX tunnel twice in ten minutes).
     const now = Date.now();
     const gap = now - (this._lastHealthCheck || now);
     this._lastHealthCheck = now;
-    const wokeFromSleep = gap > HEALTH_INTERVAL * 3;
-    if (wokeFromSleep && this.active.size > 0) {
-      console.log(`[Groove:Tunnel] Detected a ${Math.round(gap / 1000)}s gap (system sleep) — verifying tunnels`);
+    const suspectAll = gap > HEALTH_INTERVAL * 3;
+    if (suspectAll && this.active.size > 0) {
+      console.log(`[Groove:Tunnel] ${Math.round(gap / 1000)}s timer gap (sleep or blocked loop) — verifying tunnels`);
     }
 
     for (const [id, conn] of this.active) {
       try {
         const start = Date.now();
         const res = await fetch(`http://localhost:${conn.localPort}/api/health`, {
-          signal: AbortSignal.timeout(HEALTH_TIMEOUT),
+          signal: AbortSignal.timeout(this.healthTimeout ?? HEALTH_TIMEOUT),
         });
         if (res.ok) {
           conn.latencyMs = Date.now() - start;
@@ -942,22 +961,29 @@ export class TunnelManager {
         }
       } catch {
         conn.failCount = (conn.failCount || 0) + 1;
-        // After a sleep gap, one failure is enough — the tunnel was almost
-        // certainly cut with the network.
-        const limit = wokeFromSleep ? 1 : MAX_FAIL_COUNT;
-        if (conn.failCount >= limit) {
-          conn.healthy = false;
-          this.daemon.broadcast({ type: 'tunnel.unhealthy', data: { id } });
-
-          // Reap it. Previously a dead tunnel was only FLAGGED, so it stayed in
-          // `active` indefinitely and every later connect() handed back its dead
-          // port — the reason reconnecting never helped and only a full restart
-          // did. Removing it means the next connect() builds a real tunnel.
-          if (conn.failCount >= limit + 1) {
-            console.log(`[Groove:Tunnel] Reaping dead tunnel ${id} after ${conn.failCount} failed checks`);
-            this.daemon.broadcast({ type: 'tunnel.status', data: { id, step: 'reaping' } });
-            await this.disconnect(id);
-            continue;
+        // A failed 5s probe is WEAK evidence: it can't distinguish a dead
+        // tunnel from a remote daemon that's briefly busy or our own blocked
+        // event loop. Never kill on it. Once failures accumulate (or a timer
+        // gap makes everything suspect), run the confirmation ladder, which
+        // can — a healthy verdict there resets the count.
+        if (conn.failCount >= MAX_FAIL_COUNT || suspectAll) {
+          const verdict = await this._confirmDead(conn);
+          if (verdict === 'alive') {
+            conn.failCount = 0;
+            conn.healthy = true;
+            conn._wedgedStreak = 0;
+          } else {
+            conn.healthy = false;
+            this.daemon.broadcast({ type: 'tunnel.unhealthy', data: { id } });
+            // 'wedged' (port accepts, HTTP silent even at long timeout) is the
+            // one verdict with a false-positive path — a remote event loop
+            // blocked 15s+ — so demand it twice in a row. proc-dead/port-dead
+            // are unambiguous: the ssh client is gone or nothing is listening.
+            conn._wedgedStreak = verdict === 'wedged' ? (conn._wedgedStreak || 0) + 1 : 0;
+            if (verdict !== 'wedged' || conn._wedgedStreak >= 2) {
+              await this._reapAndRebuild(id, conn, verdict);
+              continue;
+            }
           }
         }
       }
@@ -968,6 +994,51 @@ export class TunnelManager {
     }
   }
 
+  // Escalating evidence that a tunnel is actually dead, not merely slow:
+  //   'alive'     — answered a long-timeout HTTP probe; leave it alone
+  //   'proc-dead' — the ssh client process is gone
+  //   'port-dead' — nothing is listening on the local port
+  //   'wedged'    — port accepts TCP but HTTP never answers (dead forward)
+  async _confirmDead(conn) {
+    if (await this._tunnelResponds(conn.localPort, this.confirmTimeout ?? CONFIRM_TIMEOUT)) return 'alive';
+    if (conn.pid) {
+      try { process.kill(conn.pid, 0); } catch { return 'proc-dead'; }
+    }
+    if (!(await this._isPortInUse(conn.localPort))) return 'port-dead';
+    return 'wedged';
+  }
+
+  // Tear down a confirmed-dead tunnel and immediately rebuild it on the SAME
+  // local port. The remote GUI window points at that port and its WebSocket
+  // retries every 2s, so a same-port rebuild heals an open window without the
+  // user noticing. Only if the rebuild fails does this surface as a disconnect.
+  // Rate-limited so a genuinely dead host degrades to disconnected instead of
+  // thrashing reconnect attempts forever.
+  async _reapAndRebuild(id, conn, reason) {
+    const { localPort } = conn;
+    console.log(`[Groove:Tunnel] Tunnel ${id} confirmed dead (${reason}) — rebuilding`);
+    this.daemon.audit.log('tunnel.reap', { id, reason, failCount: conn.failCount });
+    await this.disconnect(id);
+
+    const lastRebuild = this._rebuildAt?.get(id) || 0;
+    if (Date.now() - lastRebuild < REBUILD_COOLDOWN_MS) {
+      console.log(`[Groove:Tunnel] ${id} already auto-rebuilt recently — leaving disconnected`);
+      return;
+    }
+    this._rebuildAt = this._rebuildAt || new Map();
+    this._rebuildAt.set(id, Date.now());
+
+    try {
+      this.daemon.broadcast({ type: 'tunnel.status', data: { id, step: 'reconnecting' } });
+      await this.connect(id, { preferredPort: localPort });
+      console.log(`[Groove:Tunnel] ${id} rebuilt on port ${localPort}`);
+    } catch (err) {
+      console.warn(`[Groove:Tunnel] Auto-rebuild of ${id} failed: ${err.message}`);
+      // disconnect() above already broadcast tunnel.disconnected — the GUI is
+      // consistent; the user can reconnect manually when the host is back.
+    }
+  }
+
   // Signal 0 only tests for existence — no signal is delivered.
   async _waitForExit(pid, timeoutMs) {
     const deadline = Date.now() + timeoutMs;

package/packages/gui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@groove-dev/gui",
-  "version": "0.27.212",
+  "version": "0.27.213",
   "description": "GROOVE GUI — visual agent control plane",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",