groove-dev 0.27.213 → 0.27.215

package/packages/daemon/src/tunnel-manager.js

@@ -4,7 +4,8 @@
 import { execFileSync, spawn } from 'child_process';
 import { existsSync, writeFileSync, readFileSync, statSync } from 'fs';
 import { resolve } from 'path';
-import { createConnection } from 'net';
+import { createConnection, isIP } from 'net';
+import { lookup } from 'dns/promises';
 import crypto from 'crypto';
 
 function getLocalVersion() {
@@ -15,6 +16,13 @@ function getLocalVersion() {
 }
 
 const REMOTE_PORT = 31415;
+
+// Every remote `groove start` must run from the directory of the daemon's
+// EXISTING world (grooveDir is derived from cwd). The remote daemon records
+// that directory in ~/.groove/last-run-dir on each boot; starting anywhere
+// else boots a fresh empty .groove — which reads as "all my teams are gone".
+// Falls back to $HOME (matching old behavior) when no anchor exists yet.
+const ANCHOR_CD = `cd "$(cat "$HOME/.groove/last-run-dir" 2>/dev/null || echo "$HOME")" 2>/dev/null || cd "$HOME"; `;
 const DEFAULT_LOCAL_PORT = 31416;
 const MAX_PORT_ATTEMPTS = 10;
 const HEALTH_INTERVAL = 30000;
@@ -26,6 +34,10 @@ const CONFIRM_TIMEOUT = 15000;
 // At most one automatic rebuild per tunnel per window; beyond that it stays
 // disconnected rather than thrashing against a host that keeps dying.
 const REBUILD_COOLDOWN_MS = 10 * 60 * 1000;
+// Restarting a crashed REMOTE daemon is cheaper and safer than a rebuild, so
+// its cooldown is shorter — but still bounded: a daemon that dies right after
+// every start has a real problem more starts won't fix.
+const REMOTE_START_COOLDOWN_MS = 2 * 60 * 1000;
 
 const INJECTION_CHARS = /[;|&`$(){}[\]<>!#\n\r\\]/;
 
@@ -56,10 +68,57 @@ function isPermissionError(output) {
 
 const PERMISSION_HINT = 'npm global install requires write access. Either install Node via nvm (recommended) or configure passwordless sudo for npm on the remote server.';
 
+// A hostname can resolve to several addresses on different interfaces — a
+// dual-homed LAN box (wired + Wi-Fi) advertises all of them over mDNS, and ssh
+// just takes the resolver's first pick. Landing on a weak Wi-Fi address gives
+// a tunnel that dies of keepalive timeout minutes later, and every reconnect
+// re-rolls the dice. Probe all candidates with a TCP handshake to the ssh port
+// and take the fastest responder — on a LAN that reliably picks wired over
+// Wi-Fi. Falls back to the original hostname if resolution or every probe
+// fails, so behavior is unchanged for the cases that already worked.
+export async function resolveBestAddress(host, port = 22, probeTimeoutMs = 2500) {
+  if (isIP(host)) return host; // literal IP — nothing to choose
+  let addrs;
+  try {
+    addrs = await lookup(host, { all: true, verbatim: true });
+  } catch { return host; }
+  if (!Array.isArray(addrs) || addrs.length <= 1) return host;
+
+  const handshake = (address) => new Promise((res) => {
+    const started = Date.now();
+    let sock;
+    try { sock = createConnection({ host: address, port }); } catch { return res(null); }
+    sock.setTimeout(probeTimeoutMs);
+    sock.on('connect', () => { sock.destroy(); res(Date.now() - started); });
+    sock.on('error', () => res(null));
+    sock.on('timeout', () => { sock.destroy(); res(null); });
+  });
+
+  // Median of three handshakes per address: one lucky round-trip can make a
+  // weak link look fine, but a flaky link rarely wins three in a row — a
+  // single retransmit (or drop, scored as the timeout) sinks its median.
+  const probe = async (address) => {
+    const times = [];
+    for (let i = 0; i < 3; i++) times.push(await handshake(address));
+    const scored = times.map((t) => (t === null ? probeTimeoutMs : t)).sort((a, b) => a - b);
+    if (times.every((t) => t === null)) return null; // never connected at all
+    return { address, ms: scored[1] };
+  };
+
+  const results = (await Promise.all(addrs.map((a) => probe(a.address)))).filter(Boolean);
+  if (results.length === 0) return host;
+  results.sort((a, b) => a.ms - b.ms);
+  return results[0].address;
+}
+
 export class TunnelManager {
   constructor(daemon) {
     this.daemon = daemon;
     this.remotesPath = resolve(daemon.grooveDir, 'remotes.json');
+    // Live tunnel state, persisted separately from the configs so a daemon
+    // restart can re-adopt still-running ssh processes instead of forgetting
+    // them (the configs file is user data; this is runtime state).
+    this.activePath = resolve(daemon.grooveDir, 'tunnels-active.json');
     this.saved = new Map();
     this.active = new Map();
     this._healthInterval = null;
@@ -88,8 +147,9 @@ export class TunnelManager {
   }
 
   async init() {
+    await this._readopt();
     for (const [id, config] of this.saved) {
-      if (config.autoConnect) {
+      if (config.autoConnect && !this.active.has(id)) {
         try {
           await this.connect(id);
         } catch (err) {
@@ -99,6 +159,71 @@ export class TunnelManager {
     }
   }
 
+  // Re-adopt tunnels whose detached ssh processes survived a daemon restart.
+  // Without this, every daemon/app restart orphaned the ssh (or shutdown killed
+  // it) and the new daemon started amnesiac — remote windows died mid-session
+  // and the user had to reconnect everything by hand.
+  async _readopt() {
+    let entries = [];
+    try {
+      if (existsSync(this.activePath)) entries = JSON.parse(readFileSync(this.activePath, 'utf8'));
+    } catch { /* corrupt — treat as none */ }
+    if (!Array.isArray(entries) || entries.length === 0) return;
+
+    let adopted = 0;
+    for (const e of entries) {
+      if (!e?.id || !e.pid || !e.localPort || !this.saved.has(e.id)) continue;
+      // Only re-adopt what is provably OUR ssh still doing THIS job: the pid
+      // must be alive, be an ssh process forwarding this port, and the port
+      // must serve HTTP.
+      let alive = false;
+      try { process.kill(e.pid, 0); alive = true; } catch { /* gone */ }
+      if (alive) alive = this._looksLikeOurSsh(e.pid, e.localPort);
+      if (alive && await this._tunnelResponds(e.localPort)) {
+        this.active.set(e.id, {
+          pid: e.pid,
+          localPort: e.localPort,
+          startedAt: e.startedAt || new Date().toISOString(),
+          lastPing: Date.now(),
+          latencyMs: null,
+          healthy: true,
+          failCount: 0,
+        });
+        adopted++;
+        const name = this.saved.get(e.id)?.name || e.id;
+        console.log(`[Groove:Tunnel] Re-adopted live tunnel to ${name} on port ${e.localPort}`);
+        this.daemon.broadcast({ type: 'tunnel.connected', data: { id: e.id, name, localPort: e.localPort, host: this.saved.get(e.id)?.host, url: `http://localhost:${e.localPort}?instance=${encodeURIComponent(name)}` } });
+      } else if (alive) {
+        // ssh survives but doesn't serve — a corpse from before the restart.
+        try { process.kill(e.pid, 'SIGTERM'); } catch { /* gone */ }
+      }
+    }
+    if (adopted > 0 && !this._healthInterval) {
+      this._healthInterval = setInterval(() => this._healthCheckAll(), HEALTH_INTERVAL);
+    }
+    this._saveActive();
+  }
+
+  // Identity check for re-adoption: is this pid an ssh forwarding this port?
+  // Guards against pid recycling handing us an unrelated process.
+  _looksLikeOurSsh(pid, localPort) {
+    try {
+      const cmd = execFileSync('ps', ['-p', String(pid), '-o', 'command='], {
+        encoding: 'utf8', timeout: 3000,
+      }).trim();
+      return cmd.includes('ssh') && cmd.includes(String(localPort));
+    } catch { return false; }
+  }
+
+  _saveActive() {
+    try {
+      const entries = [...this.active.entries()].map(([id, c]) => ({
+        id, pid: c.pid, localPort: c.localPort, startedAt: c.startedAt,
+      }));
+      writeFileSync(this.activePath, JSON.stringify(entries, null, 2), { mode: 0o600 });
+    } catch { /* best effort */ }
+  }
+
   getSaved() {
     return Array.from(this.saved.values()).map(s => ({
       ...this._sanitize(s),
@@ -228,7 +353,7 @@ export class TunnelManager {
     const config = this.saved.get(id);
     if (!config) throw new Error(`Remote ${id} not found`);
 
-    const target = `${config.user}@${config.host}`;
+    const target = `${config.user}@${await resolveBestAddress(config.host, config.port || 22)}`;
     const keyArgs = config.sshKeyPath ? ['-i', config.sshKeyPath] : [];
 
     try {
@@ -344,18 +469,28 @@ export class TunnelManager {
     } else {
       localPort = await this._findAvailablePort();
     }
-    const target = `${config.user}@${config.host}`;
+    // Multi-homed hosts (mDNS names especially): pick the address that actually
+    // answers fastest instead of letting the resolver gamble on an interface.
+    const connectHost = await resolveBestAddress(config.host, config.port || 22);
+    if (connectHost !== config.host) {
+      console.log(`[Groove:Tunnel] ${config.name}: ${config.host} → ${connectHost} (fastest responding address)`);
+    }
+    const target = `${config.user}@${connectHost}`;
     const keyArgs = config.sshKeyPath ? ['-i', config.sshKeyPath] : [];
+    // Keep the host key pinned to the NAME when we connect by address, so every
+    // address of the same box shares one known_hosts entry.
+    const aliasArgs = connectHost !== config.host ? ['-o', `HostKeyAlias=${config.host}`] : [];
 
     const sshArgs = [
       '-N',
       '-L', `127.0.0.1:${localPort}:localhost:${REMOTE_PORT}`,
       '-p', String(config.port || 22),
-      '-o', 'ServerAliveInterval=30',
-      '-o', 'ServerAliveCountMax=3',
+      '-o', 'ServerAliveInterval=15',
+      '-o', 'ServerAliveCountMax=4',
       '-o', 'ExitOnForwardFailure=yes',
       '-o', 'StrictHostKeyChecking=accept-new',
       '-o', 'GSSAPIAuthentication=no',
+      ...aliasArgs,
       ...keyArgs,
       target,
     ];
@@ -406,6 +541,7 @@ export class TunnelManager {
       healthy: true,
       failCount: 0,
     });
+    this._saveActive();
 
     // Verify daemon is reachable through tunnel, start if needed
     let remoteAlive = false;
@@ -496,6 +632,7 @@ export class TunnelManager {
     if (localPort) await this._waitForPortFree(localPort, 3000);
 
     this.active.delete(id);
+    this._saveActive();
 
     const config = this.saved.get(id);
     this.daemon.audit.log('tunnel.disconnect', { id, name: config?.name });
@@ -559,7 +696,7 @@ export class TunnelManager {
       }
 
       // Restart remote daemon — fire and forget the SSH, verify through the tunnel
-      const cdPrefix = config.projectDir ? `cd "${config.projectDir}" && ` : '';
+      const cdPrefix = config.projectDir ? `cd "${config.projectDir}" && ` : ANCHOR_CD;
       try {
         execFileSync('ssh', [...sshBase, sshCmd(`kill $(lsof -t -i:${REMOTE_PORT}) 2>/dev/null || true; sleep 1; ${cdPrefix}GROOVE_BIN=$(which groove) && nohup "$GROOVE_BIN" start > /tmp/groove-daemon.log 2>&1 < /dev/null & disown`)], {
           encoding: 'utf8', timeout: 15000, stdio: ['pipe', 'pipe', 'pipe'],
@@ -666,7 +803,7 @@ export class TunnelManager {
     const config = this.saved.get(id);
     if (!config) throw new Error(`Remote ${id} not found`);
 
-    const target = `${config.user}@${config.host}`;
+    const target = `${config.user}@${await resolveBestAddress(config.host, config.port || 22)}`;
     const keyArgs = config.sshKeyPath ? ['-i', config.sshKeyPath] : [];
 
     // Build the remote bash command:
@@ -676,7 +813,7 @@ export class TunnelManager {
     //   4. explicitly POST /api/project-dir so the daemon's projectDir matches
     //      config.projectDir even if the backgrounded cwd didn't stick (this
     //      also updates the editor root used for /api/browse, /api/files/*)
-    const cdPrefix = config.projectDir ? `cd "${config.projectDir}" && ` : '';
+    const cdPrefix = config.projectDir ? `cd "${config.projectDir}" && ` : ANCHOR_CD;
     const setProjectDir = config.projectDir
       ? `curl -sf -X POST -H 'Content-Type: application/json' --data '{"path":"${config.projectDir}"}' http://localhost:${REMOTE_PORT}/api/project-dir > /dev/null 2>&1 || true; `
       : '';
@@ -798,7 +935,7 @@ export class TunnelManager {
     try {
       const result = execFileSync('ssh', [
         ...sshBase,
-        remoteCmd(`GROOVE_BIN=$(which groove) && nohup "$GROOVE_BIN" start > /tmp/groove-daemon.log 2>&1 < /dev/null & disown; sleep 5; curl -sf http://localhost:${REMOTE_PORT}/api/health > /dev/null && echo __DAEMON_OK__ || (echo __DAEMON_FAIL__; tail -20 /tmp/groove-daemon.log 2>/dev/null)`),
+        remoteCmd(`${ANCHOR_CD}GROOVE_BIN=$(which groove) && nohup "$GROOVE_BIN" start > /tmp/groove-daemon.log 2>&1 < /dev/null & disown; sleep 5; curl -sf http://localhost:${REMOTE_PORT}/api/health > /dev/null && echo __DAEMON_OK__ || (echo __DAEMON_FAIL__; tail -20 /tmp/groove-daemon.log 2>/dev/null)`),
       ], {
         encoding: 'utf8',
         timeout: 45000,
@@ -868,7 +1005,7 @@ export class TunnelManager {
     }).trim();
     const installedVer = verOutput.replace(/[^0-9.]/g, '') || verOutput.trim();
 
-    const restartCmd = `kill $(lsof -t -i:${REMOTE_PORT}) 2>/dev/null || true; sleep 2; GROOVE_BIN=$(which groove) && nohup "$GROOVE_BIN" start > /tmp/groove-daemon.log 2>&1 < /dev/null & disown; sleep 4; curl -sf http://localhost:${REMOTE_PORT}/api/status`;
+    const restartCmd = `kill $(lsof -t -i:${REMOTE_PORT}) 2>/dev/null || true; sleep 2; ${ANCHOR_CD}GROOVE_BIN=$(which groove) && nohup "$GROOVE_BIN" start > /tmp/groove-daemon.log 2>&1 < /dev/null & disown; sleep 4; curl -sf http://localhost:${REMOTE_PORT}/api/status`;
     const restartResult = execFileSync('ssh', [...sshBase, sshCmd(restartCmd)], {
       encoding: 'utf8',
       timeout: 60000,
@@ -972,6 +1109,12 @@ export class TunnelManager {
             conn.failCount = 0;
             conn.healthy = true;
             conn._wedgedStreak = 0;
+          } else if (verdict === 'remote-down') {
+            // The tunnel is carrying traffic correctly — the far daemon is what
+            // died (typically mid-upgrade). Rebuild would be useless; start it.
+            conn.healthy = false;
+            this.daemon.broadcast({ type: 'tunnel.unhealthy', data: { id } });
+            await this._startRemoteDaemon(id, conn);
           } else {
             conn.healthy = false;
             this.daemon.broadcast({ type: 'tunnel.unhealthy', data: { id } });
@@ -995,19 +1138,71 @@ export class TunnelManager {
   }
 
   // Escalating evidence that a tunnel is actually dead, not merely slow:
-  //   'alive'     — answered a long-timeout HTTP probe; leave it alone
-  //   'proc-dead' — the ssh client process is gone
-  //   'port-dead' — nothing is listening on the local port
-  //   'wedged'    — port accepts TCP but HTTP never answers (dead forward)
+  //   'alive'       — answered a long-timeout HTTP probe; leave it alone
+  //   'proc-dead'   — the ssh client process is gone
+  //   'port-dead'   — nothing is listening on the local port
+  //   'remote-down' — the tunnel forwards fine but the REMOTE end refuses:
+  //                   the probe fails fast with a connection error, not a
+  //                   timeout. Killing the tunnel won't fix that — the remote
+  //                   daemon needs starting (e.g. it died during an upgrade).
+  //   'wedged'      — port accepts TCP but HTTP hangs to timeout (dead forward)
   async _confirmDead(conn) {
-    if (await this._tunnelResponds(conn.localPort, this.confirmTimeout ?? CONFIRM_TIMEOUT)) return 'alive';
+    const started = Date.now();
+    let probeErr = null;
+    try {
+      const res = await fetch(`http://localhost:${conn.localPort}/api/health`, {
+        signal: AbortSignal.timeout(this.confirmTimeout ?? CONFIRM_TIMEOUT),
+      });
+      if (res.ok) return 'alive';
+    } catch (err) { probeErr = err; }
+
     if (conn.pid) {
       try { process.kill(conn.pid, 0); } catch { return 'proc-dead'; }
     }
     if (!(await this._isPortInUse(conn.localPort))) return 'port-dead';
+
+    // ssh is alive and its port listens. A hang (timeout) means the forward is
+    // dead; a FAST connection-level error means ssh relayed the remote side's
+    // refusal — the tunnel works, the far daemon doesn't.
+    const failedFast = Date.now() - started < 2000;
+    const timedOut = probeErr && (probeErr.name === 'TimeoutError' || probeErr.name === 'AbortError');
+    if (failedFast && !timedOut) return 'remote-down';
     return 'wedged';
   }
 
+  // The tunnel is healthy; the daemon on the far side is what's down. Start it
+  // over ssh rather than pointlessly rebuilding the tunnel. Rate-limited: if
+  // the remote daemon won't stay up, repeated starts won't save it.
+  async _startRemoteDaemon(id, conn) {
+    this._remoteStartAt = this._remoteStartAt || new Map();
+    const last = this._remoteStartAt.get(id) || 0;
+    if (Date.now() - last < REMOTE_START_COOLDOWN_MS) return;
+    this._remoteStartAt.set(id, Date.now());
+
+    console.log(`[Groove:Tunnel] ${id}: tunnel is fine but the remote daemon is down — starting it`);
+    this.daemon.audit.log('tunnel.remote-daemon-start', { id });
+    this.daemon.broadcast({ type: 'tunnel.status', data: { id, step: 'starting' } });
+    try {
+      await this.autoStart(id);
+      // Confirm it came up; a success resets the failure counters immediately
+      // instead of waiting out another health cycle.
+      for (let i = 0; i < 10; i++) {
+        await new Promise((r) => setTimeout(r, 2000));
+        if (await this._tunnelResponds(conn.localPort)) {
+          conn.failCount = 0;
+          conn.healthy = true;
+          conn._wedgedStreak = 0;
+          console.log(`[Groove:Tunnel] ${id}: remote daemon is back`);
+          this.daemon.broadcast({ type: 'tunnel.health', data: { id, latencyMs: conn.latencyMs, healthy: true } });
+          return;
+        }
+      }
+      console.warn(`[Groove:Tunnel] ${id}: remote daemon did not come back after start`);
+    } catch (err) {
+      console.warn(`[Groove:Tunnel] ${id}: could not start remote daemon: ${err.message}`);
+    }
+  }
+
   // Tear down a confirmed-dead tunnel and immediately rebuild it on the SAME
   // local port. The remote GUI window points at that port and its WebSocket
   // retries every 2s, so a same-port rebuild heals an open window without the
@@ -1075,25 +1270,17 @@ export class TunnelManager {
     throw new Error(`No available local port found (tried ${DEFAULT_LOCAL_PORT}-${DEFAULT_LOCAL_PORT + MAX_PORT_ATTEMPTS - 1})`);
   }
 
+  // Deliberately does NOT kill the ssh processes. They are spawned detached and
+  // are the user's live sessions: killing them on every daemon restart (app
+  // upgrade, promote, crash) is what nuked remote windows mid-session. State is
+  // persisted; the next daemon re-adopts whatever is still alive and serving.
+  // Explicit disconnect()/delete() remain the paths that actually kill a tunnel.
   shutdown() {
     if (this._healthInterval) {
       clearInterval(this._healthInterval);
       this._healthInterval = null;
     }
-    for (const [id] of this.active) {
-      try {
-        const conn = this.active.get(id);
-        if (conn?.pid) {
-          const cmd = execFileSync('ps', ['-p', String(conn.pid), '-o', 'command='], {
-            encoding: 'utf8',
-            timeout: 3000,
-          }).trim();
-          if (cmd.includes('ssh')) {
-            process.kill(conn.pid, 'SIGTERM');
-          }
-        }
-      } catch { /* ignore */ }
-    }
+    this._saveActive();
     this.active.clear();
   }
 }