@evomap/evolver 1.87.4 → 1.88.0

package/src/proxy/lifecycle/manager.js

@@ -23,12 +23,30 @@ const HEARTBEAT_BACKOFF_CAP_MS = 15 * 60_000;
 const HELLO_TIMEOUT = 15_000;
 const HEARTBEAT_TIMEOUT = 10_000;
 const MAX_REAUTH_ATTEMPTS = 2;
-// First failure = 30 min, subsequent consecutive failures double up to ~4h.
-// Without escalation a daemon stuck on a bad secret gets re-poked every 30
-// minutes by inbound auth errors and fills the log forever.
-const REAUTH_BACKOFF_BASE_MS = 30 * 60_000;
+// First failure = 2 min, subsequent consecutive failures double up to ~4h.
+// Aligned with a2aProtocol.js Round-9 reduction (was 30 min, caused
+// "idle-death" for proxy-mode users: one benign 401 silenced the node for 30
+// min, triggering stagnation kills and manual restart loops).
+const REAUTH_BACKOFF_BASE_MS = 2 * 60_000;
 const REAUTH_BACKOFF_MAX_MS = 4 * 60 * 60_000;
 
+// Wall-clock drift detector tunables. Mirrors DRIFT_CHECK_MS /
+// DRIFT_SLEEP_THRESHOLD_MS / DRIFT_LONG_SLEEP_THRESHOLD_MS in
+// src/gep/a2aProtocol.js. setTimeout / setInterval fire on libuv's
+// monotonic clock, which freezes while the host is suspended -- so a
+// laptop closed for hours and reopened would not trigger any heartbeat
+// tick until the next scheduled time, which under exponential backoff
+// can sit at HEARTBEAT_BACKOFF_CAP_MS (15 min). Sampling Date.now()
+// (wall clock) every DRIFT_CHECK_MS lets us detect the jump and
+// immediately poke the heartbeat so recovery does not have to wait for
+// the next natural tick. Long-sleep gap also clears reauth backoff:
+// hub-side state we cached is almost certainly stale after a 30min+
+// suspend, so force a clean retry path on wake instead of carrying the
+// pre-sleep penalty through. R10 (#544).
+const DRIFT_CHECK_MS = 30 * 1000;
+const DRIFT_SLEEP_THRESHOLD_MS = 90 * 1000;
+const DRIFT_LONG_SLEEP_THRESHOLD_MS = 30 * 60_000;
+
 let _cachedFingerprint = null;
 function _getEnvFingerprint() {
   if (_cachedFingerprint) return _cachedFingerprint;
@@ -101,6 +119,8 @@ class LifecycleManager {
     this._helloRateLimitUntil = 0;
     this._reauthBackoffUntil = 0;
     this._consecutiveReauthFailures = 0;
+    this._driftInterval = null;
+    this._lastDriftCheckAt = 0;
   }
 
   get nodeId() {
@@ -510,6 +530,55 @@ class LifecycleManager {
     // to schedule its own next timer (a fresher path already owns it).
     this._heartbeatGen = (this._heartbeatGen || 0) + 1;
     this._heartbeatTick(this._heartbeatGen);
+    this._startDriftDetector();
+  }
+
+  // Sample wall-clock every DRIFT_CHECK_MS so macOS sleep / hypervisor
+  // pause / debugger break is detected and the heartbeat loop is poked
+  // back into action without waiting for the (possibly 15-min) backoff
+  // timer to fire on libuv's monotonic clock. R10 (#544).
+  _startDriftDetector() {
+    if (this._driftInterval) return;
+    this._lastDriftCheckAt = Date.now();
+    this._driftInterval = setInterval(() => {
+      // Wrap the whole body in try/catch -- this is a setInterval
+      // callback; any throw escaping it kills the detector itself,
+      // which is the bug we're protecting against.
+      try {
+        if (!this._running) return;
+        const now = Date.now();
+        const gap = now - this._lastDriftCheckAt;
+        this._lastDriftCheckAt = now;
+        if (gap > DRIFT_SLEEP_THRESHOLD_MS) {
+          try {
+            this.logger.warn(
+              `[lifecycle] wall-clock jump detected (+${Math.round(gap / 1000)}s); ` +
+                'likely sleep/wake or process suspension, poking heartbeat'
+            );
+          } catch (_) { /* logger broken; detector must still poke */ }
+          // Long-sleep recovery: the hub-side cached state we carried
+          // through the suspend is almost certainly stale. Clear reauth
+          // backoff so the next tick can try a clean recovery path
+          // instead of sitting out a pre-sleep penalty for up to 4h.
+          if (gap > DRIFT_LONG_SLEEP_THRESHOLD_MS) {
+            this._consecutiveReauthFailures = 0;
+            this._reauthBackoffUntil = 0;
+            try {
+              this.logger.warn(
+                `[lifecycle] long sleep (+${Math.round(gap / 60_000)}min) cleared reauth backoff`
+              );
+            } catch (_) { /* logger broken; non-fatal */ }
+          }
+          this.pokeHeartbeatLoop();
+        }
+      } catch (err) {
+        try { this.logger.error(`[lifecycle] drift detector threw: ${err && err.message}`); }
+        catch (_) { /* never let the detector escape */ }
+      }
+    }, DRIFT_CHECK_MS);
+    // Don't keep the event loop alive on behalf of the detector alone --
+    // matches the unref() used on _heartbeatTimer.
+    if (this._driftInterval.unref) this._driftInterval.unref();
   }
 
   async _heartbeatTick(myGen) {
@@ -574,6 +643,10 @@ class LifecycleManager {
       clearTimeout(this._heartbeatTimer);
       this._heartbeatTimer = null;
     }
+    if (this._driftInterval) {
+      clearInterval(this._driftInterval);
+      this._driftInterval = null;
+    }
   }
 
   _shouldUpgrade(minVersion) {

package/src/proxy/mailbox/store.js

@@ -68,6 +68,27 @@ function safeParse(payload) {
   try { return JSON.parse(payload); } catch { return payload; }
 }
 
+// Round-9: the round-8 cross-process append lock (§21.7) was REMOVED.
+// Its premise -- that fs.appendFileSync to a regular file can interleave
+// bytes mid-line unless each write stays under PIPE_BUF (512 B darwin,
+// 4096 B linux) -- conflated two different POSIX guarantees. PIPE_BUF
+// atomicity is defined for PIPES/FIFOs, not regular files. A single
+// O_APPEND write() to a regular file is positioned atomically at EOF and
+// is not interleaved with other appenders on the local filesystems evolver
+// uses (~/.evomap); this was verified empirically on darwin/APFS --
+// concurrent 4 KB..1 MB appends from 6 writers produced zero torn lines.
+// So the lock guarded a non-problem. Worse, its 5 s deadline with a
+// busy-wait (Atomics.wait, then a spin-loop fallback) ran on the single
+// JS thread, so under any real contention it BLOCKED the event loop --
+// starving the very heartbeat/SSE/HTTP it shared the process with, i.e.
+// it could itself produce the "process alive but inert" symptom it claimed
+// to prevent. fs.appendFileSync writes the whole buffer with O_APPEND, so
+// a single record lands as one atomic append.
+//
+// Windows note: PIPE_BUF is a POSIX concept; it does not exist on Windows.
+// Windows NTFS provides the same atomicity guarantee for O_APPEND writes to
+// regular files that POSIX local filesystems do, so the removal above is
+// equally valid on Windows. No platform-specific code is needed here.
 function appendLine(filePath, obj) {
   fs.appendFileSync(filePath, JSON.stringify(obj) + '\n', 'utf8');
 }
@@ -135,8 +156,27 @@ class MailboxStore {
   _persistState() {
     const dir = path.dirname(this._stateFile);
     if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
-    const tmp = `${this._stateFile}.tmp`;
+    // Round-7 (§20.5): per-PID tmp path. Two evolver processes (daemon +
+    // ad-hoc CLI / proxy + loop) writing to the same `${stateFile}.tmp`
+    // would otherwise interleave: process B's writeFileSync truncates
+    // A's tmp mid-write, then B's rename completes with B's truncated
+    // payload as the final state.json. `state.json` holds the cached
+    // node_secret after a hub rotation -- a torn write here is the
+    // load-bearing trigger for the "401-loop -> reauth backoff -> dead
+    // for 30 min..4 h" symptom this branch targets. Matches the
+    // precedent set by _persistNodeSecret in src/gep/a2aProtocol.js.
+    const tmp = `${this._stateFile}.${process.pid}.tmp`;
     fs.writeFileSync(tmp, JSON.stringify(this._state, null, 2) + '\n', 'utf8');
+    // Windows: fs.renameSync throws EPERM when the destination file already
+    // exists, unlike POSIX where rename(2) atomically replaces the target.
+    // Remove the destination first so the rename succeeds on all platforms.
+    // The window between unlink and rename is intentionally tiny; a crash in
+    // that window leaves the tmp file behind (recovered on next _persistState).
+    if (process.platform === 'win32') {
+      try { fs.unlinkSync(this._stateFile); } catch (e) {
+        if (e.code !== 'ENOENT') throw e;
+      }
+    }
     fs.renameSync(tmp, this._stateFile);
   }
 
@@ -384,7 +424,10 @@ class MailboxStore {
   // --- Compaction (reduces JSONL file size by rewriting only current state) ---
 
   compact() {
-    const tmpFile = this._messagesFile + '.tmp';
+    // Round-7 (§20.5): same per-PID tmp rationale as _persistState.
+    // Two concurrent compact() calls (daemon + ad-hoc CLI) racing on
+    // the same `${messagesFile}.tmp` lose the loser's compacted log.
+    const tmpFile = `${this._messagesFile}.${process.pid}.tmp`;
     const entries = [];
     for (const [, msg] of this._messages) {
       entries.push(msg);
@@ -396,6 +439,13 @@ class MailboxStore {
       fs.writeSync(fd, JSON.stringify(msg) + '\n');
     }
     fs.closeSync(fd);
+    // Windows: renameSync throws EPERM when the destination already exists.
+    // Remove it first so the swap succeeds on all platforms.
+    if (process.platform === 'win32') {
+      try { fs.unlinkSync(this._messagesFile); } catch (e) {
+        if (e.code !== 'ENOENT') throw e;
+      }
+    }
     fs.renameSync(tmpFile, this._messagesFile);
     this._rebuildIndex();
   }

package/src/proxy/server/settings.js

@@ -37,9 +37,13 @@ function writeSettings(data) {
   }
   const current = readSettings();
   const merged = { ...current, ...data };
+  // NOTE(windows): mode 0o600 is silently ignored on Windows. The settings
+  // file (which may contain proxy credentials) will NOT be owner-read-only.
+  // Only Windows user-profile directory ACLs provide isolation. The chmodSync
+  // call below is also a no-op on Windows but is retained for Unix correctness.
   fs.writeFileSync(file, JSON.stringify(merged, null, 2), { encoding: 'utf8', mode: 0o600 });
   // mode: 0o600 only applies on creation; explicitly chmod to tighten pre-existing files
-  try { fs.chmodSync(file, 0o600); } catch {}
+  try { fs.chmodSync(file, 0o600); } catch { /* best-effort; no-op on Windows */ }
   return merged;
 }
 
@@ -59,9 +63,19 @@ function isStaleProxy() {
   const pid = settings.proxy?.pid;
   if (!pid) return false;
   try {
+    // process.kill(pid, 0) probes whether the process exists without sending a
+    // signal. On POSIX it throws ESRCH when the PID is gone. On Windows the
+    // Node.js runtime maps this to the same behavior (ESRCH via uv_kill), so
+    // the cross-platform semantics are consistent. If the current process does
+    // not have permission to query the target PID, EPERM is thrown -- that
+    // means the PID exists and is owned by another user, so we treat it as
+    // live (not stale) rather than crashing.
     process.kill(pid, 0);
     return false;
-  } catch {
+  } catch (err) {
+    // ESRCH: process does not exist -> stale.
+    // EPERM: process exists but is not ours -> not stale (leave settings alone).
+    if (err.code === 'EPERM') return false;
     return true;
   }
 }

package/src/proxy/sync/inbound.js

@@ -83,12 +83,25 @@ class InboundSync {
 
     try {
       const senderId = this.store.getState('node_id');
-      await hubFetch(endpoint, {
+      // Round-8 (§21.5): drain the response body so the undici long-poll
+      // dispatcher pool is not leaked one socket per ack. ackDelivered
+      // is called every inbound poll cycle (default 1-10s); the
+      // pre-round-8 code captured no reference to res and never called
+      // .json()/.text()/body.cancel(), so each ack pinned a socket
+      // until GC. After a few minutes of activity the strict-pool was
+      // exhausted and proxy-mode heartbeats hung on next acquire --
+      // matches the "alive once then dead" user symptom in proxy mode.
+      const res = await hubFetch(endpoint, {
         method: 'POST',
         headers: this.getHeaders(),
         body: JSON.stringify({ sender_id: senderId, message_ids: delivered.map(m => m.id) }),
         signal: AbortSignal.timeout(10_000),
       });
+      try {
+        if (res && res.body && typeof res.body.cancel === 'function') {
+          await res.body.cancel().catch(() => {});
+        }
+      } catch (_) { /* never escape the drain helper */ }
       return { acked: delivered.length };
     } catch (err) {
       this.logger.error(`[inbound] ack failed: ${err.message}`);