ai-or-die 0.1.66 → 0.1.67

package/bin/supervisor.js

@@ -6,11 +6,38 @@ const { spawn } = require('child_process');
 const path = require('path');
 const { RESTART_EXIT_CODE } = require('../src/restart-manager');
 
-const RESTART_DELAY_MS = 1000;
-const CRASH_RESTART_DELAY_MS = 3000;
-const CIRCUIT_BREAKER_WINDOW_MS = 30000;
-const CIRCUIT_BREAKER_MAX_CRASHES = 3;
-const SHUTDOWN_TIMEOUT_MS = 10000;
+// ---------------------------------------------------------------------------
+// Tunables — all overridable via env vars so the regression test can shrink
+// the windows from hours/minutes to ms. See docs/audits/proc-supervisor-breaker.md
+// for the rationale behind every default.
+// ---------------------------------------------------------------------------
+
+const RESTART_DELAY_MS         = parseInt(process.env.RESTART_DELAY_MS, 10)         || 1000;     // clean RESTART_EXIT_CODE respawn
+const CRASH_RESTART_DELAY_MS   = parseInt(process.env.CRASH_RESTART_DELAY_MS, 10)   || 3000;     // normal crash respawn
+const SHUTDOWN_TIMEOUT_MS      = parseInt(process.env.SHUTDOWN_TIMEOUT_MS, 10)      || 10000;    // SIGINT/SIGTERM hard-kill fallback
+
+// Tier 1 — tight crash loop. 3 crashes in 30 s historically tripped a hard
+// process.exit(1). The fix replaces that with an extended restart delay
+// (and a loud log) so the daemon ALWAYS comes back; permanent halt strands
+// the user's single browser session with no way to recover short of SSH.
+const CIRCUIT_BREAKER_WINDOW_MS    = parseInt(process.env.CIRCUIT_BREAKER_WINDOW_MS, 10)    || 30000;   // 30 s
+const CIRCUIT_BREAKER_MAX_CRASHES  = parseInt(process.env.CIRCUIT_BREAKER_MAX_CRASHES, 10)  || 3;
+const TIER1_RESTART_DELAY_MS       = parseInt(process.env.TIER1_RESTART_DELAY_MS, 10)       || 60000;   // 1 min
+
+// Tier 2 — sustained slow churn. The old breaker missed this entirely:
+// a server that crashed once every 31 s respawned forever at the normal
+// cadence, masking the underlying bug. Tier 2 catches 5 crashes in 1 h
+// and slows respawn to 5 min, dropping log volume by ~100×.
+const SUSTAINED_CRASH_WINDOW_MS    = parseInt(process.env.SUSTAINED_CRASH_WINDOW_MS, 10)    || 3600000; // 1 h
+const SUSTAINED_CRASH_MAX          = parseInt(process.env.SUSTAINED_CRASH_MAX, 10)          || 5;
+const TIER2_RESTART_DELAY_MS       = parseInt(process.env.TIER2_RESTART_DELAY_MS, 10)       || 300000;  // 5 min
+
+// Hard cap on the crashTimestamps array so a pathological 100/sec crash loop
+// over an hour can't grow it to 360 k entries. 1024 is comfortably more than
+// any realistic backoff cadence would produce in 1 h (even at tier-2's
+// minimum 5-min cadence that's 12 entries/h; at tier-1's 60-s cadence it's
+// 60/h). Trimming oldest-first preserves the most-recent-N invariant.
+const CRASH_TIMESTAMPS_CAP         = parseInt(process.env.CRASH_TIMESTAMPS_CAP, 10)         || 1024;
 
 const serverScript = process.env.SUPERVISOR_CHILD_SCRIPT
   || path.join(__dirname, 'ai-or-die.js');
@@ -21,6 +48,64 @@ let shuttingDown = false;
 let crashTimestamps = [];
 let pendingRestartTimer = null;
 
+// Queued IPC message delivered to the NEXT spawned child once its IPC channel
+// is open. Used to forward tier-2 escalation downstream so the in-process
+// server can surface it to the browser ("supervisor is throttling restarts").
+let pendingWarning = null;
+
+// Test-only: when SUPERVISOR_ESCALATION_OBSERVER=1, the supervisor emits a
+// {type:'supervisor_escalation', tier, count, ...} IPC message to ITS parent
+// after each classification, so a regression test can deterministically watch
+// tier transitions without parsing log strings. Production runs leave it null.
+let escalationObserver = process.env.SUPERVISOR_ESCALATION_OBSERVER === '1'
+  ? (info) => { try { if (process.send) process.send({ type: 'supervisor_escalation', ...info }); } catch (_) {} }
+  : null;
+
+function classifyCrash(now) {
+  // Trim to the longer of the two windows so the array stays bounded.
+  const cutoff = now - Math.max(CIRCUIT_BREAKER_WINDOW_MS, SUSTAINED_CRASH_WINDOW_MS);
+  crashTimestamps = crashTimestamps.filter((t) => t >= cutoff);
+
+  // Defence-in-depth cap (SUP-REL review). The time-window trim already
+  // bounds the array to "crashes in the last hour"; this guards against
+  // an extreme pathological case (e.g. CRASH_RESTART_DELAY_MS overridden
+  // to 0 in a test, or a future env-var injection raising the window).
+  // Keep the most-recent entries — classification only ever cares about
+  // the head of the array.
+  if (crashTimestamps.length > CRASH_TIMESTAMPS_CAP) {
+    crashTimestamps = crashTimestamps.slice(-CRASH_TIMESTAMPS_CAP);
+  }
+
+  const inSustained = crashTimestamps.filter((t) => now - t < SUSTAINED_CRASH_WINDOW_MS).length;
+  const inTight     = crashTimestamps.filter((t) => now - t < CIRCUIT_BREAKER_WINDOW_MS).length;
+
+  // Higher tier wins.
+  if (inSustained >= SUSTAINED_CRASH_MAX) {
+    return { tier: 2, count: inSustained, windowMs: SUSTAINED_CRASH_WINDOW_MS, delayMs: TIER2_RESTART_DELAY_MS };
+  }
+  if (inTight >= CIRCUIT_BREAKER_MAX_CRASHES) {
+    return { tier: 1, count: inTight, windowMs: CIRCUIT_BREAKER_WINDOW_MS, delayMs: TIER1_RESTART_DELAY_MS };
+  }
+  return { tier: 0, count: inTight, windowMs: CIRCUIT_BREAKER_WINDOW_MS, delayMs: CRASH_RESTART_DELAY_MS };
+}
+
+function logEscalation(decision) {
+  if (decision.tier === 2) {
+    console.error(
+      `\n[supervisor] ⚠ TIER 2 ESCALATION: ${decision.count} crashes within ` +
+      `${Math.round(decision.windowMs / 60000)}m. Throttling restart to ` +
+      `${Math.round(decision.delayMs / 60000)}m. Underlying defect is likely real — ` +
+      `inspect server logs.\n`
+    );
+  } else if (decision.tier === 1) {
+    console.error(
+      `\n[supervisor] ⚠ TIER 1 ESCALATION: ${decision.count} crashes within ` +
+      `${Math.round(decision.windowMs / 1000)}s. Throttling restart to ` +
+      `${Math.round(decision.delayMs / 1000)}s.\n`
+    );
+  }
+}
+
 function startServer() {
   pendingRestartTimer = null;
   const nodeArgs = ['--expose-gc', serverScript, ...forwardedArgs];
@@ -30,6 +115,32 @@ function startServer() {
     env: { ...process.env, SUPERVISED: '1' }
   });
 
+  // Flush a queued supervisor_warning into the new child's IPC channel.
+  // SUP-REL review: the immediately-after-spawn `child.connected` is false
+  // (the IPC handshake hasn't completed yet), so this block used to silently
+  // drop the warning. Defer via the 'spawn' event, which Node fires AFTER
+  // the child has been successfully spawned and the IPC channel is wired.
+  // Future CLIENT-04 server-side wiring will then receive it deterministically.
+  if (pendingWarning) {
+    const warning = pendingWarning;
+    pendingWarning = null;
+    const flush = () => {
+      try {
+        if (child && child.connected) child.send(warning);
+      } catch (_) { /* IPC may have closed between spawn and send — best-effort */ }
+    };
+    // Node ≥ 16: 'spawn' event fires once when spawn succeeds. If the child
+    // already crashed before 'spawn' fires, we never send; that's correct
+    // behaviour — the next-next child will get its own warning if the crash
+    // sequence re-escalates.
+    if (typeof child.once === 'function') {
+      child.once('spawn', flush);
+    } else {
+      // Defensive: pre-Node-16 fallback (unsupported but harmless).
+      process.nextTick(flush);
+    }
+  }
+
   child.on('exit', (code, signal) => {
     child = null;
 
@@ -52,21 +163,34 @@ function startServer() {
       return;
     }
 
-    // Unexpected exit — check circuit breaker
+    // Unexpected exit — classify against both windows.
     const now = Date.now();
     crashTimestamps.push(now);
-    // Remove timestamps outside the window
-    crashTimestamps = crashTimestamps.filter(t => now - t < CIRCUIT_BREAKER_WINDOW_MS);
-
-    if (crashTimestamps.length >= CIRCUIT_BREAKER_MAX_CRASHES) {
-      console.error(`[supervisor] Circuit breaker: ${CIRCUIT_BREAKER_MAX_CRASHES} crashes within ${CIRCUIT_BREAKER_WINDOW_MS / 1000}s. Stopping.`);
-      process.exit(1);
-      return;
+    const decision = classifyCrash(now);
+
+    if (decision.tier > 0) {
+      logEscalation(decision);
+      // Queue a downstream warning so the next-spawned server can surface
+      // it to the browser UI. (Receiver-side wiring is a future task — for
+      // now this is a no-op on the child side but adds zero risk.)
+      pendingWarning = {
+        type: 'supervisor_warning',
+        tier: decision.tier,
+        crashes: decision.count,
+        windowMs: decision.windowMs,
+        nextDelayMs: decision.delayMs,
+      };
     }
 
+    if (escalationObserver) escalationObserver(decision);
+
     const exitInfo = signal ? `signal ${signal}` : `code ${code}`;
-    console.warn(`[supervisor] Server exited unexpectedly (${exitInfo}), restarting in ${CRASH_RESTART_DELAY_MS}ms... (crash ${crashTimestamps.length}/${CIRCUIT_BREAKER_MAX_CRASHES})`);
-    pendingRestartTimer = setTimeout(startServer, CRASH_RESTART_DELAY_MS);
+    console.warn(
+      `[supervisor] Server exited unexpectedly (${exitInfo}), restarting in ` +
+      `${decision.delayMs}ms... (crash ${decision.count} in ` +
+      `${Math.round(decision.windowMs / 1000)}s window, tier ${decision.tier})`
+    );
+    pendingRestartTimer = setTimeout(startServer, decision.delayMs);
   });
 
   child.on('error', (err) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-or-die",
-  "version": "0.1.66",
+  "version": "0.1.67",
   "description": "Universal AI coding terminal — Claude, Copilot, Gemini & more in your browser",
   "main": "src/server.js",
   "bin": {
@@ -15,6 +15,11 @@
     "test": "mocha --exit test/*.test.js",
     "test:integration": "mocha --exit --timeout 60000 test/supervisor-integration.test.js test/integration/*.test.js",
     "test:browser": "npx playwright test --config e2e/playwright.config.js",
+    "soak": "node test/longevity/harness/cli.js",
+    "test:longevity-smoke": "mocha --exit --timeout 120000 test/longevity/smoke.test.js",
+    "test:longevity:server": "mocha --exit --timeout 120000 --recursive --extension .test.js test/longevity/event-loop test/longevity/disk test/longevity/process test/longevity/browser-sampler.test.js test/longevity/cli.test.js test/longevity/gate-evaluator-directional.test.js test/longevity/gate-evaluator-vacuous.test.js test/longevity/resume.test.js test/longevity/smoke.test.js",
+    "test:longevity:browser": "playwright test --config test/longevity/playwright.config.js",
+    "test:longevity": "npm run test:longevity:server && npm run test:longevity:browser",
     "build:bundle": "node scripts/build-sea.js bundle",
     "build:sea": "node scripts/build-sea.js",
     "release:pr": "bash scripts/release-pr.sh"

package/src/public/app.js

@@ -5781,3 +5781,118 @@ document.addEventListener('DOMContentLoaded', () => {
     window.app = app;
     app.startHeartbeat();
 });
+
+// CLIENT-03 (stability-hardening-2026): browser-side diagnostics surface.
+// Mirrors the server `/api/diagnostics` shape so SUP-SOAK's browser soak
+// can sample uniformly. Installed at module level so it is callable from
+// the moment app.js finishes loading — independent of when `window.app`
+// is constructed or whether a session has been joined. All sub-collectors
+// are wrapped in try/catch so the function never throws; safe to call
+// pre-session. Returns a Promise (the optional
+// `performance.measureUserAgentSpecificMemory()` call is async).
+// Idempotent: re-loading app.js (e.g. HMR) overwrites the previous
+// install — last loader wins.
+// Spec: docs/specs/client-longevity.md
+window.__diagnostics = async function __diagnostics() {
+    const ts = Date.now();
+    const snap = {
+        ts: ts,
+        dom: { total_nodes: 0 },
+        buffers: { plan_detector_bytes: 0, xterm_scrollback_lines: 0 },
+        ws: { state: null, url: null },
+        sse: { connected: false, streams: 0 },
+        memory: null
+    };
+
+    // dom.total_nodes
+    try {
+        snap.dom.total_nodes = document.querySelectorAll('*').length;
+    } catch (_) { /* leave 0 */ }
+
+    // dom.listeners_tracked — only emit if a tracker exists. No tracker
+    // ships today; SUP-SOAK must tolerate absence per spec v1.
+    try {
+        if (typeof window.__listenerCount === 'number') {
+            snap.dom.listeners_tracked = window.__listenerCount;
+        }
+    } catch (_) { /* leave omitted */ }
+
+    // buffers.plan_detector_bytes — prefer the CLIENT-01 `bufferBytes`
+    // field; fall back to inline sum if running against an older detector.
+    try {
+        const pd = window.app && window.app.planDetector;
+        if (pd) {
+            if (typeof pd.bufferBytes === 'number') {
+                snap.buffers.plan_detector_bytes = pd.bufferBytes;
+            } else if (Array.isArray(pd.outputBuffer)) {
+                let sum = 0;
+                for (let i = 0; i < pd.outputBuffer.length; i++) {
+                    const e = pd.outputBuffer[i];
+                    if (e && typeof e.data === 'string') sum += e.data.length;
+                }
+                snap.buffers.plan_detector_bytes = sum;
+            }
+        }
+    } catch (_) { /* leave 0 */ }
+
+    // buffers.xterm_scrollback_lines — xterm.js buffer line count.
+    try {
+        const term = window.app && window.app.terminal;
+        if (term && term.buffer && term.buffer.active
+                && typeof term.buffer.active.length === 'number') {
+            snap.buffers.xterm_scrollback_lines = term.buffer.active.length;
+        }
+    } catch (_) { /* leave 0 */ }
+
+    // ws.state / ws.url
+    try {
+        const sock = window.app && window.app.socket;
+        if (sock) {
+            if (typeof sock.readyState === 'number') snap.ws.state = sock.readyState;
+            if (typeof sock.url === 'string') snap.ws.url = sock.url;
+        }
+    } catch (_) { /* leave null */ }
+
+    // sse — best-effort walk of known holders. No global EventSource count
+    // is exposed by browsers, so this is a lower bound.
+    try {
+        let streams = 0;
+        const candidates = [
+            window.app && window.app._fileBrowserPanel
+                && window.app._fileBrowserPanel._fileWatcher
+                && window.app._fileBrowserPanel._fileWatcher._eventSource,
+            window.app && window.app._fileSearchPanel
+                && window.app._fileSearchPanel._eventSource,
+            window.app && window.app._fileWatcher
+                && window.app._fileWatcher._eventSource,
+        ];
+        for (let i = 0; i < candidates.length; i++) {
+            const es = candidates[i];
+            // EventSource.OPEN === 1; CONNECTING === 0; CLOSED === 2.
+            if (es && typeof es.readyState === 'number' && es.readyState !== 2) {
+                streams++;
+            }
+        }
+        snap.sse.streams = streams;
+        snap.sse.connected = streams > 0;
+    } catch (_) { /* leave defaults */ }
+
+    // memory — cross-origin-isolated Chrome only; fall back to
+    // navigator.deviceMemory; else null.
+    try {
+        if (typeof performance !== 'undefined'
+                && typeof performance.measureUserAgentSpecificMemory === 'function') {
+            try {
+                snap.memory = await performance.measureUserAgentSpecificMemory();
+            } catch (_) {
+                if (typeof navigator !== 'undefined' && typeof navigator.deviceMemory === 'number') {
+                    snap.memory = { deviceMemoryGB: navigator.deviceMemory };
+                }
+            }
+        } else if (typeof navigator !== 'undefined' && typeof navigator.deviceMemory === 'number') {
+            snap.memory = { deviceMemoryGB: navigator.deviceMemory };
+        }
+    } catch (_) { /* leave null */ }
+
+    return snap;
+};

package/src/public/plan-detector.js

@@ -2,12 +2,18 @@ class PlanDetector {
   constructor() {
     this.isMonitoring = false;
     this.outputBuffer = [];
+    // Byte-count cap (data.length is a faithful proxy for V8 string heap
+    // cost — see docs/audits/client-plan-detector.md). 8 MB hard cap; FIFO
+    // eviction when exceeded. Replaces the older 10 000-item cap, which
+    // permitted ~80 MB of retained string memory per tab under sustained
+    // heavy PTY output.
+    this.maxBufferBytes = 8 * 1024 * 1024;
+    this.bufferBytes = 0;
     this.planModeActive = false;
     this.currentPlan = null;
     this.currentTool = null;
     this.planStartMarker = '## Implementation Plan:';
     this.planEndMarker = 'User has approved your plan';
-    this.maxBufferSize = 10000;
     this.onPlanDetected = null;
     this.onPlanModeChange = null;
     this.onStepProgress = null;
@@ -64,11 +70,18 @@ class PlanDetector {
       timestamp: Date.now(),
       data: data
     });
-
-    // Keep buffer size manageable
-    if (this.outputBuffer.length > this.maxBufferSize) {
-      this.outputBuffer = this.outputBuffer.slice(-this.maxBufferSize / 2);
+    this.bufferBytes += data.length;
+
+    // FIFO eviction: pop oldest until under the byte cap. O(k) per call
+    // where k is the number of entries to evict — usually 1 unless a
+    // single huge chunk pushes us multiple entries over.
+    while (this.bufferBytes > this.maxBufferBytes && this.outputBuffer.length > 0) {
+      const evicted = this.outputBuffer.shift();
+      this.bufferBytes -= evicted.data.length;
     }
+    // Defensive: clamp accounting drift caused by arithmetic on unusual
+    // string types (should never happen, but cheap insurance).
+    if (this.bufferBytes < 0) this.bufferBytes = 0;
 
     // Stage 1: Quick trigger scan on the new chunk only (O(k) where k = chunk size).
     // Prepend overlap from the previous chunk to catch triggers spanning boundaries.
@@ -405,6 +418,7 @@ class PlanDetector {
   startMonitoring() {
     this.isMonitoring = true;
     this.outputBuffer = [];
+    this.bufferBytes = 0;
     this.planModeActive = false;
     this.currentPlan = null;
     this._lastChunkTail = '';
@@ -413,6 +427,7 @@ class PlanDetector {
   stopMonitoring() {
     this.isMonitoring = false;
     this.outputBuffer = [];
+    this.bufferBytes = 0;
     this.planModeActive = false;
     this.currentPlan = null;
     this._lastChunkTail = '';
@@ -420,6 +435,7 @@ class PlanDetector {
 
   clearBuffer() {
     this.outputBuffer = [];
+    this.bufferBytes = 0;
     this.currentPlan = null;
     this._lastChunkTail = '';
   }