@venturewild/workspace 0.2.0 → 0.2.2

package/server/src/tunnel-watchdog.mjs

@@ -0,0 +1,153 @@
+// TunnelWatchdog — RC2 self-heal for "slug-linked but the public URL is dead".
+//
+// The bug from the first external install: after a restart the daemon was
+// "running" but never re-linked to the relay, so `<slug>.venturewild.llc` was 502
+// while `localhost:5173` was perfectly healthy. Nothing noticed the half: the
+// install thought it was online. (`docs/remote-support-and-self-healing-design.md`
+// RC2 — a sibling of the half-open-after-sleep fix already shipped in the daemon.)
+//
+// This watchdog closes that gap from the workspace server (no Rust change): it
+// periodically asks our OWN public URL for /api/health. That request travels the
+// full chain — out to Cloudflare, through the relay, down the daemon's tunnel,
+// back to this server — so a 200 proves the whole path works. When the public
+// side fails repeatedly WHILE the local server is healthy (so the fault is the
+// LINK, not the server), it relinks the daemon — the same remedy as the operator
+// `relink-account` action, applied automatically.
+//
+// Conservative by design: it acts only on a sustained failure (threshold), never
+// relinks more often than `minRelinkIntervalMs`, and treats a down LOCAL server
+// as "not my job" (the WorkspaceSupervisor owns that). Every touch-point is an
+// injected seam so the suite never hits the network.
+//
+// SAFETY against thrash: a relink only helps when THIS daemon's link to the relay
+// is the broken part. When the relay itself is globally down (or otherwise can't
+// accept the link), relinking can't help — so retrying every interval just churns
+// the daemon. Relinks that DON'T restore the tunnel are counted, and after
+// `maxIneffectiveRelinks` the watchdog escalates to a long `longCooldownMs` quiet
+// period; a relink that works clears the counter (the next probe is healthy), so
+// the genuine RC2 case still self-heals fast.
+//
+// (Product model: one machine = one install = one daemon = one public slug;
+// multi-folder work is VS-Code-style within that single install. So co-tenant
+// daemon contention is NOT a supported state — this guard is for the relay-down /
+// transient-unreachable case, which is the real one. A two-install-per-machine
+// setup, as seen while dogfooding on 2026-06-07, is a test artifact only.)
+
+const DEFAULT_INTERVAL_MS = 60_000;
+const DEFAULT_PROBE_TIMEOUT_MS = 8_000;
+
+export class TunnelWatchdog {
+  /**
+   * @param {object} opts
+   * @param {string} opts.publicBaseUrl     e.g. https://<slug>.venturewild.llc
+   * @param {Function} opts.relink          async () => relink the daemon (stop+ensureRunning)
+   * @param {Function} [opts.localHealthy]  async () => boolean; default assumes healthy
+   * @param {Function} [opts.fetchImpl]
+   * @param {Function} [opts.nowImpl]
+   * @param {Function} [opts.log]
+   */
+  constructor({
+    publicBaseUrl,
+    relink,
+    localHealthy = async () => true,
+    fetchImpl = (...a) => globalThis.fetch(...a),
+    nowImpl = () => Date.now(),
+    log = () => {},
+    intervalMs = DEFAULT_INTERVAL_MS,
+    failureThreshold = 3,
+    minRelinkIntervalMs = 120_000,
+    maxIneffectiveRelinks = 3,
+    longCooldownMs = 1_800_000, // 30 min quiet period once relinks stop helping
+    probeTimeoutMs = DEFAULT_PROBE_TIMEOUT_MS,
+  } = {}) {
+    this.publicBaseUrl = String(publicBaseUrl || '').replace(/\/+$/, '');
+    this.relink = relink;
+    this.localHealthy = localHealthy;
+    this.fetchImpl = fetchImpl;
+    this.nowImpl = nowImpl;
+    this.log = log;
+    this.intervalMs = intervalMs;
+    this.failureThreshold = failureThreshold;
+    this.minRelinkIntervalMs = minRelinkIntervalMs;
+    this.maxIneffectiveRelinks = maxIneffectiveRelinks;
+    this.longCooldownMs = longCooldownMs;
+    this.probeTimeoutMs = probeTimeoutMs;
+    this.failures = 0;
+    this.lastRelink = 0;
+    this.consecutiveRelinks = 0; // relinks since the last healthy probe (thrash guard)
+    this.timer = null;
+  }
+
+  /** Probe the public /api/health. Returns true iff it answers (any HTTP status). */
+  async probePublic() {
+    if (!this.publicBaseUrl) return false;
+    const ctrl = new AbortController();
+    const t = setTimeout(() => ctrl.abort(), this.probeTimeoutMs);
+    try {
+      const res = await this.fetchImpl(`${this.publicBaseUrl}/api/health`, {
+        signal: ctrl.signal,
+        // never cached by the edge — we want the live tunnel, not a CDN copy
+        headers: { 'cache-control': 'no-cache' },
+      });
+      // A 5xx from the tunnel layer (502/504) means the link is down even though
+      // we got an HTTP response from the edge; treat only <500 as "reachable".
+      return res.status < 500;
+    } catch {
+      return false; // network error / timeout / abort
+    } finally {
+      clearTimeout(t);
+    }
+  }
+
+  /** One watchdog step. Returns its decision (exposed for tests). */
+  async tick() {
+    // Local server down → the WorkspaceSupervisor's problem, not ours. A failing
+    // public probe in that state says nothing about the LINK, so don't relink.
+    if (!(await this.localHealthy())) {
+      this.failures = 0;
+      return 'local-down';
+    }
+    if (await this.probePublic()) {
+      this.failures = 0;
+      this.consecutiveRelinks = 0; // the link is up → any prior relink worked
+      return 'healthy';
+    }
+    this.failures += 1;
+    if (this.failures < this.failureThreshold) return 'degraded';
+    const now = this.nowImpl();
+    // Relinks that aren't restoring the tunnel (shared daemon / relay down) earn an
+    // escalating quiet period so we never thrash a co-tenant's daemon (real finding).
+    const cooldown =
+      this.consecutiveRelinks >= this.maxIneffectiveRelinks
+        ? this.longCooldownMs
+        : this.minRelinkIntervalMs;
+    if (now - this.lastRelink < cooldown) return 'cooldown';
+    this.lastRelink = now;
+    this.failures = 0;
+    this.consecutiveRelinks += 1;
+    this.log(
+      `public tunnel unreachable while local is healthy — relinking daemon` +
+        ` (attempt ${this.consecutiveRelinks} since last healthy)`,
+    );
+    try {
+      await this.relink();
+      return 'relinked';
+    } catch (e) {
+      this.log(`relink failed: ${e?.message || e}`);
+      return 'relink-error';
+    }
+  }
+
+  start() {
+    if (this.timer) return this;
+    this.timer = setInterval(() => {
+      this.tick().catch((e) => this.log(`tick error: ${e?.message || e}`));
+    }, this.intervalMs);
+    if (this.timer.unref) this.timer.unref(); // never keep the process alive
+    return this;
+  }
+
+  stop() {
+    if (this.timer) { clearInterval(this.timer); this.timer = null; }
+  }
+}