instar 1.3.107 → 1.3.108

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/dist/commands/server.d.ts.map +1 -1
  2. package/dist/commands/server.js +6 -0
  3. package/dist/commands/server.js.map +1 -1
  4. package/dist/core/SessionManager.d.ts +23 -0
  5. package/dist/core/SessionManager.d.ts.map +1 -1
  6. package/dist/core/SessionManager.js +53 -0
  7. package/dist/core/SessionManager.js.map +1 -1
  8. package/dist/core/StuckInputSentinel.d.ts.map +1 -1
  9. package/dist/core/StuckInputSentinel.js +7 -6
  10. package/dist/core/StuckInputSentinel.js.map +1 -1
  11. package/dist/core/claudeActivityIndicators.d.ts +37 -0
  12. package/dist/core/claudeActivityIndicators.d.ts.map +1 -0
  13. package/dist/core/claudeActivityIndicators.js +45 -0
  14. package/dist/core/claudeActivityIndicators.js.map +1 -0
  15. package/dist/core/cpuStarvation.d.ts +30 -0
  16. package/dist/core/cpuStarvation.d.ts.map +1 -0
  17. package/dist/core/cpuStarvation.js +44 -0
  18. package/dist/core/cpuStarvation.js.map +1 -0
  19. package/dist/lifeline/ServerSupervisor.d.ts +25 -0
  20. package/dist/lifeline/ServerSupervisor.d.ts.map +1 -1
  21. package/dist/lifeline/ServerSupervisor.js +71 -46
  22. package/dist/lifeline/ServerSupervisor.js.map +1 -1
  23. package/dist/monitoring/CompactionSentinel.d.ts +39 -1
  24. package/dist/monitoring/CompactionSentinel.d.ts.map +1 -1
  25. package/dist/monitoring/CompactionSentinel.js +45 -1
  26. package/dist/monitoring/CompactionSentinel.js.map +1 -1
  27. package/package.json +1 -1
  28. package/src/data/builtin-manifest.json +3 -3
  29. package/upgrades/1.3.108.md +94 -0
  30. package/upgrades/side-effects/compaction-busy-session-defer.md +94 -0
  31. package/upgrades/side-effects/supervisor-cpu-starvation-restart-guard.md +83 -0
@@ -0,0 +1,37 @@
1
+ /**
2
+ * claudeActivityIndicators — the canonical "Claude Code is mid-turn right now"
3
+ * footer hints.
4
+ *
5
+ * Claude Code renders these strings in the status footer ONLY while a turn is
6
+ * in flight (model generating, tool running, extended-think computing). They
7
+ * disappear the instant the turn lands and the session returns to its idle
8
+ * prompt. That structural property is exactly what makes them a reliable
9
+ * "actively working" tell: a session sitting idle at the prompt — or a session
10
+ * that has WEDGED and fast-fails every turn — never shows them.
11
+ *
12
+ * Why this is its own module: more than one recovery surface needs to answer
13
+ * "is this pane actively working?" before taking a disruptive action —
14
+ * • StuckInputSentinel / SessionManager.verifyInjection refuse to fire a
15
+ * recovery Enter into a working pane (the input is correctly queued and
16
+ * will submit when the turn ends).
17
+ * • CompactionSentinel refuses to RE-INJECT a recovery prompt into a working
18
+ * pane — re-injecting buries the user's real message under stacked
19
+ * recovery bootstraps (the false "session is restarting" loop).
20
+ * Centralizing the list keeps those surfaces from drifting apart.
21
+ *
22
+ * IMPORTANT — footer hints ONLY. We deliberately do NOT include the Braille
23
+ * spinner glyphs (⠋⠙⠹…) here: a frozen last frame of a dead pane can still
24
+ * contain a spinner glyph, which would make a genuinely-stuck session read as
25
+ * "working" and starve it of recovery. The footer hints are present only while
26
+ * a turn is actually in flight, so they cannot double-fire against a dead pane.
27
+ * (See frameworkActivitySignals.ts for the spinner-inclusive, framework-portable
28
+ * signal used by the triage nurse, where the trade-off is different.)
29
+ */
30
+ export declare const CLAUDE_WORKING_INDICATORS: readonly string[];
31
+ /**
32
+ * Pure check: does this captured tmux pane show Claude Code actively working?
33
+ * Footer-hint based — see CLAUDE_WORKING_INDICATORS for why that's the precise
34
+ * signal. Returns false for empty/missing panes.
35
+ */
36
+ export declare function paneShowsClaudeWorking(pane: string | null | undefined): boolean;
37
+ //# sourceMappingURL=claudeActivityIndicators.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"claudeActivityIndicators.d.ts","sourceRoot":"","sources":["../../src/core/claudeActivityIndicators.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,eAAO,MAAM,yBAAyB,EAAE,SAAS,MAAM,EAItD,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CAG/E"}
@@ -0,0 +1,45 @@
1
+ /**
2
+ * claudeActivityIndicators — the canonical "Claude Code is mid-turn right now"
3
+ * footer hints.
4
+ *
5
+ * Claude Code renders these strings in the status footer ONLY while a turn is
6
+ * in flight (model generating, tool running, extended-think computing). They
7
+ * disappear the instant the turn lands and the session returns to its idle
8
+ * prompt. That structural property is exactly what makes them a reliable
9
+ * "actively working" tell: a session sitting idle at the prompt — or a session
10
+ * that has WEDGED and fast-fails every turn — never shows them.
11
+ *
12
+ * Why this is its own module: more than one recovery surface needs to answer
13
+ * "is this pane actively working?" before taking a disruptive action —
14
+ * • StuckInputSentinel / SessionManager.verifyInjection refuse to fire a
15
+ * recovery Enter into a working pane (the input is correctly queued and
16
+ * will submit when the turn ends).
17
+ * • CompactionSentinel refuses to RE-INJECT a recovery prompt into a working
18
+ * pane — re-injecting buries the user's real message under stacked
19
+ * recovery bootstraps (the false "session is restarting" loop).
20
+ * Centralizing the list keeps those surfaces from drifting apart.
21
+ *
22
+ * IMPORTANT — footer hints ONLY. We deliberately do NOT include the Braille
23
+ * spinner glyphs (⠋⠙⠹…) here: a frozen last frame of a dead pane can still
24
+ * contain a spinner glyph, which would make a genuinely-stuck session read as
25
+ * "working" and starve it of recovery. The footer hints are present only while
26
+ * a turn is actually in flight, so they cannot double-fire against a dead pane.
27
+ * (See frameworkActivitySignals.ts for the spinner-inclusive, framework-portable
28
+ * signal used by the triage nurse, where the trade-off is different.)
29
+ */
30
+ export const CLAUDE_WORKING_INDICATORS = [
31
+ 'esc to interrupt', // Claude Code's "task in progress" hint
32
+ 'ctrl+t to hide tasks', // Multi-task display
33
+ 'tokens · esc', // Token-counting + interrupt hint
34
+ ];
35
+ /**
36
+ * Pure check: does this captured tmux pane show Claude Code actively working?
37
+ * Footer-hint based — see CLAUDE_WORKING_INDICATORS for why that's the precise
38
+ * signal. Returns false for empty/missing panes.
39
+ */
40
+ export function paneShowsClaudeWorking(pane) {
41
+ if (!pane)
42
+ return false;
43
+ return CLAUDE_WORKING_INDICATORS.some(ind => pane.includes(ind));
44
+ }
45
+ //# sourceMappingURL=claudeActivityIndicators.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"claudeActivityIndicators.js","sourceRoot":"","sources":["../../src/core/claudeActivityIndicators.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAsB;IAC1D,kBAAkB,EAAM,wCAAwC;IAChE,sBAAsB,EAAE,qBAAqB;IAC7C,cAAc,EAAU,kCAAkC;CAC3D,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAA+B;IACpE,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,OAAO,yBAAyB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AACnE,CAAC"}
@@ -0,0 +1,30 @@
1
+ /**
2
+ * cpuStarvation — a tiny shared helper for "is this machine so oversubscribed
3
+ * that a live process can't get scheduled?"
4
+ *
5
+ * The signal is `loadavg[0] / cpuCount`. Above ~1.5 the run-queue is longer
6
+ * than the core count, so a perfectly-alive event loop can stall for many
7
+ * seconds simply because it isn't getting CPU time. Two watchers need to tell
8
+ * this apart from a genuine fault, or they take a destructive action that makes
9
+ * things worse:
10
+ * - SleepWakeDetector: a short heartbeat drift under CPU starvation is NOT a
11
+ * real sleep/wake (it suppresses the wake — see SleepWakeDetector).
12
+ * - ServerSupervisor: a server that can't answer /health under CPU starvation
13
+ * is NOT dead — restarting it doesn't cure the starvation (the fresh server
14
+ * is starved too), it just drops the in-flight message and loops.
15
+ *
16
+ * SleepWakeDetector keeps its own injectable providers for unit testing; this
17
+ * module is the canonical default ratio + computation that other callers
18
+ * (ServerSupervisor) share so the fleet uses one definition of "CPU-starved."
19
+ */
20
+ /** Above this loadavg[0]/cpuCount ratio the machine is treated as CPU-starved. */
21
+ export declare const DEFAULT_MAX_LOAD_RATIO = 1.5;
22
+ /**
23
+ * 1-minute load average divided by CPU count. Returns 0 on any error (so a
24
+ * failure to read system load never trips a starvation branch). Providers are
25
+ * injectable for tests.
26
+ */
27
+ export declare function cpuLoadRatio(loadAvg1?: number, cpuCount?: number): number;
28
+ /** True when the machine is CPU-starved (load ratio strictly above maxRatio). */
29
+ export declare function isCpuStarved(maxRatio?: number, loadAvg1?: number, cpuCount?: number): boolean;
30
+ //# sourceMappingURL=cpuStarvation.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cpuStarvation.d.ts","sourceRoot":"","sources":["../../src/core/cpuStarvation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH,kFAAkF;AAClF,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C;;;;GAIG;AACH,wBAAgB,YAAY,CAC1B,QAAQ,CAAC,EAAE,MAAM,EACjB,QAAQ,CAAC,EAAE,MAAM,GAChB,MAAM,CASR;AAED,iFAAiF;AACjF,wBAAgB,YAAY,CAC1B,QAAQ,GAAE,MAA+B,EACzC,QAAQ,CAAC,EAAE,MAAM,EACjB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAET"}
@@ -0,0 +1,44 @@
1
+ /**
2
+ * cpuStarvation — a tiny shared helper for "is this machine so oversubscribed
3
+ * that a live process can't get scheduled?"
4
+ *
5
+ * The signal is `loadavg[0] / cpuCount`. Above ~1.5 the run-queue is longer
6
+ * than the core count, so a perfectly-alive event loop can stall for many
7
+ * seconds simply because it isn't getting CPU time. Two watchers need to tell
8
+ * this apart from a genuine fault, or they take a destructive action that makes
9
+ * things worse:
10
+ * - SleepWakeDetector: a short heartbeat drift under CPU starvation is NOT a
11
+ * real sleep/wake (it suppresses the wake — see SleepWakeDetector).
12
+ * - ServerSupervisor: a server that can't answer /health under CPU starvation
13
+ * is NOT dead — restarting it doesn't cure the starvation (the fresh server
14
+ * is starved too), it just drops the in-flight message and loops.
15
+ *
16
+ * SleepWakeDetector keeps its own injectable providers for unit testing; this
17
+ * module is the canonical default ratio + computation that other callers
18
+ * (ServerSupervisor) share so the fleet uses one definition of "CPU-starved."
19
+ */
20
+ import os from 'node:os';
21
+ /** Above this loadavg[0]/cpuCount ratio the machine is treated as CPU-starved. */
22
+ export const DEFAULT_MAX_LOAD_RATIO = 1.5;
23
+ /**
24
+ * 1-minute load average divided by CPU count. Returns 0 on any error (so a
25
+ * failure to read system load never trips a starvation branch). Providers are
26
+ * injectable for tests.
27
+ */
28
+ export function cpuLoadRatio(loadAvg1, cpuCount) {
29
+ try {
30
+ const load = loadAvg1 ?? os.loadavg()[0];
31
+ const cpus = cpuCount ?? os.cpus().length;
32
+ if (!Number.isFinite(load) || !Number.isFinite(cpus) || cpus < 1)
33
+ return 0;
34
+ return load / cpus;
35
+ }
36
+ catch {
37
+ return 0;
38
+ }
39
+ }
40
+ /** True when the machine is CPU-starved (load ratio strictly above maxRatio). */
41
+ export function isCpuStarved(maxRatio = DEFAULT_MAX_LOAD_RATIO, loadAvg1, cpuCount) {
42
+ return cpuLoadRatio(loadAvg1, cpuCount) > maxRatio;
43
+ }
44
+ //# sourceMappingURL=cpuStarvation.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cpuStarvation.js","sourceRoot":"","sources":["../../src/core/cpuStarvation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,kFAAkF;AAClF,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAC1B,QAAiB,EACjB,QAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,QAAQ,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;QAC1C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC;YAAE,OAAO,CAAC,CAAC;QAC3E,OAAO,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,YAAY,CAC1B,WAAmB,sBAAsB,EACzC,QAAiB,EACjB,QAAiB;IAEjB,OAAO,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,QAAQ,CAAC;AACrD,CAAC"}
@@ -127,6 +127,9 @@ export declare class ServerSupervisor extends EventEmitter {
127
127
  private consecutiveBindFailures;
128
128
  private readonly bindFailureEscalationThreshold;
129
129
  private readonly processAliveThreshold;
130
+ private readonly maxLoadRatio;
131
+ private readonly starvationRestartThreshold;
132
+ private readonly loadRatioProvider;
130
133
  private stateDir;
131
134
  private maintenanceWaitStartedAt;
132
135
  private maintenanceWaitMs;
@@ -165,6 +168,8 @@ export declare class ServerSupervisor extends EventEmitter {
165
168
  maintenanceWaitMinutes?: number;
166
169
  /** How long to wait after spawning before starting health checks. Default: 180 seconds (3 minutes). */
167
170
  startupGraceSeconds?: number;
171
+ /** Injectable CPU-load-ratio source (loadavg[0]/cpuCount) for tests. Default: cpuLoadRatio. */
172
+ loadRatioProvider?: () => number;
168
173
  });
169
174
  /**
170
175
  * Start the server and begin monitoring.
@@ -343,6 +348,26 @@ export declare class ServerSupervisor extends EventEmitter {
343
348
  getPendingRemediatorRequestCount(): number;
344
349
  /** Test helper — simulate a healthy tick driving completion notifications. */
345
350
  triggerHealthyTickForTests(): void;
351
+ /**
352
+ * Decide what to do about a server that has failed `>= unhealthyThreshold`
353
+ * consecutive health checks. Shared by the health-check loop's two failure
354
+ * paths (unhealthy /health response, and a thrown check).
355
+ *
356
+ * - Process dead → restart immediately.
357
+ * - Process alive but unresponsive, below the (load-lenient) threshold →
358
+ * keep waiting.
359
+ * - Process alive, threshold reached, but the box is CPU-starved → DEFER the
360
+ * restart (restarting a starved server only drops the in-flight message
361
+ * and loops) until starvation eases or the hard cap forces it.
362
+ * - Process alive, threshold reached, not starved → restart.
363
+ */
364
+ private evaluateUnhealthyServer;
365
+ /**
366
+ * True when we should hold off restarting an alive-but-unresponsive server
367
+ * because the machine is CPU-starved — but only up to the hard cap. Past the
368
+ * cap we restart regardless (the server may be genuinely hung, not starved).
369
+ */
370
+ private deferRestartForCpuStarvation;
346
371
  private handleUnhealthy;
347
372
  /**
348
373
  * Capture crash output from multiple sources:
@@ -1 +1 @@
1
- {"version":3,"file":"ServerSupervisor.d.ts","sourceRoot":"","sources":["../../src/lifeline/ServerSupervisor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAIH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAiB3C,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,EAAE,CAAC;IACb,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,gBAAgB,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACpC,aAAa,EAAE,CAAC,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;IAChE,qBAAqB,EAAE,CAAC,OAAO,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAClF,mBAAmB,EAAE,CAAC,IAAI,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACxE;wFACoF;IACpF,aAAa,EAAE,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;CACxC;AAYD;;;;;;GAMG;AACH,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,SAAS,GAAG,SAAS,GAAG,OAAO,CAAC;IAC7C,gEAAgE;IAChE,WAAW,EAAE,MAAM,CAAC;IACpB,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB,6DAA6D;IAC7D,gBAAgB,EAAE,MAAM,CAAC;IACzB,iFAAiF;IACjF,IAAI,EAAE,MAAM,CAAC;CACd;AAED,kEAAkE;AAClE,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,OAAO,CAAC;IAClB;oFACgF;IAChF,MAAM,EAAE,MAAM,CAAC;IACf,+DAA+D;IAC/D,0BAA0B,EAAE,MAAM,CAAC;IACnC,4DAA4D;IAC5D,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAUD;;;;;;GAMG;AACH,MAAM,WAAW,qCAAqC;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,WAAW,CAAC;IACzB,4EAA4E;IAC5E,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qEAAqE;IACrE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,mCAAmC;IAClD,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;CAC/D;AAED,MAAM,WAAW,mCAAmC;IAClD,OAAO,EAAE,SAAS,GAAG,SAAS,CAAC;IAC/B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAyDD,wEAAwE;AACxE,MAAM,WAAW,oBAAoB;IACnC,8DAA8D;IAC9D,iBAAiB,EAAE,CAAC,GAAG,EAAE;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACxD,0EAA0E;IAC1E,oBAAoB,EAAE,MAAM,MAAM,CAAC;CACpC;AAED,qBAAa,gBAAiB,SAAQ,YAAY;IAChD,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,QAAQ,CAAgB;IAChC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,mBAAmB,CAA+C;IAC1E,OAAO,CAAC,iBAAiB,CAAK;IAC9B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAc;IAC7C,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,kBAAkB,CAAK;IAC/B,OAAO,CAAC,gBAAgB,CAAQ;IAChC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,cAAc,CAAW;IACjC,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,eAAe,CAAc;IACrC,OAAO,CAAC,qBAAqB,CAAK;IAClC,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAK;IAMxC,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,QAAQ,CAAC,8BAA8B,CAAK;IACpD,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAK;IAC3C,OAAO,CAAC,QAAQ,CAAgB;IAGhC,OAAO,CAAC,wBAAwB,CAAK;IACrC,OAAO,CAAC,iBAAiB,CAAc;IACvC,OAAO,CAAC,oBAAoB,CAAuB;IAGnD,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAM;IAC9C,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAe;IACtD,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,wBAAwB,CAAK;IACrC,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAe;IAC7D,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAK;IAC9C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAmB;IACvD,OAAO,CAAC,kBAAkB,CAAK;IAC/B,OAAO,CAAC,eAAe,CAAM;IAC7B,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,iBAAiB,CAAkC;IAC3D,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;IAG3C,0EAA0E;IAC1E,MAAM,CAAC,QAAQ,CAAC,0BAA0B,KAAK;IAC/C,yEAAyE;IACzE,MAAM,CAAC,QAAQ,CAAC,0BAA0B,SAAc;IACxD,OAAO,CAAC,oBAAoB,CAAqC;IACjE,qFAAqF;IACrF,OAAO,CAAC,yBAAyB,CAAmF;IACpH,oFAAoF;IACpF,OAAO,CAAC,iBAAiB,CAA+D;gBAE5E,OAAO,EAAE;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,yGAAyG;QACzG,sBAAsB,CAAC,EAAE,MAAM,CAAC;QAChC,uGAAuG;QACvG,mBAAmB,CAAC,EAAE,MAAM,CAAC;KAC9B;IAiBD;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC;IAyD/B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAyB3B;;OAEG;IACH,IAAI,OAAO,IAAI,OAAO,CAErB;IAED;;OAEG;IACH,SAAS,IAAI;QACX,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,OAAO,CAAC;QACjB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,EAAE,OAAO,CAAC;QACrB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,aAAa,EAAE,OAAO,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;QACtB,eAAe,EAAE,MAAM,CAAC;QACxB,wBAAwB,EAAE,MAAM,CAAC;QACjC,wBAAwB,EAAE,MAAM,CAAC;QACjC,iBAAiB,EAAE,OAAO,CAAC;QAC3B,wBAAwB,EAAE,MAAM,CAAC;QACjC,gBAAgB,EAAE,OAAO,CAAC;QAC1B,yBAAyB,EAAE,MAAM,CAAC;KACnC;IA2BD;;;OAGG;IACH,mBAAmB,IAAI,IAAI;IAa3B;;;OAGG;IACH,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAI5C;;;;;OAKG;IACG,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAsC9D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACG,oBAAoB,CACxB,GAAG,EAAE,qCAAqC,EAC1C,QAAQ,CAAC,EAAE,mCAAmC,GAC7C,OAAO,CAAC,mCAAmC,CAAC;IAmG/C;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IA8QzB;;OAEG;IACH,OAAO,CAAC,WAAW;IAoBnB;;OAEG;IACH,OAAO,CAAC,YAAY;IAgBpB,OAAO,CAAC,WAAW;IAmFnB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,iBAAiB;IAkKzB,OAAO,CAAC,gBAAgB;YAWV,WAAW;IAoBzB;;;;;;;;;;OAUG;YACW,sBAAsB;IAYpC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;IA2E3B;;;OAGG;IACH,OAAO,CAAC,wBAAwB;IAiDhC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAoC3B;;;;;OAKG;IACH,kBAAkB,CAAC,UAAU,EAAE,oBAAoB,GAAG,IAAI;IAK1D,8DAA8D;IAC9D,2BAA2B,IAAI,MAAM;IAIrC,2DAA2D;IAC3D,oBAAoB,IAAI,MAAM;IAI9B,oEAAoE;IACpE,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAK3C;;;;;;;;;;;;;;OAcG;IACG,sBAAsB,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwG9F;;;;;;OAMG;IACH,OAAO,CAAC,wCAAwC;IAkBhD;;;;;;;OAOG;IACH,OAAO,CAAC,4BAA4B;IAiBpC,kEAAkE;IAClE,gCAAgC,IAAI,MAAM;IAI1C,8EAA8E;IAC9E,0BAA0B,IAAI,IAAI;IAMlC,OAAO,CAAC,eAAe;IAwKvB;;;;OAIG;IAOH,OAAO,CAAC,kBAAkB;IAgC1B;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAgC7B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAgB9B,OAAO,CAAC,sBAAsB;IAa9B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,uBAAuB;IAwC/B;;OAEG;IACH,OAAO,CAAC,sBAAsB;CAS/B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,sEAAsE;IACtE,UAAU,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,uBAAuB,CAAC,eAAe,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAqDpF;AAID;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,6BAA6B,CAAC,OAAO,EAAE,uBAAuB,GAAG,MAAM,CAuCtF"}
1
+ {"version":3,"file":"ServerSupervisor.d.ts","sourceRoot":"","sources":["../../src/lifeline/ServerSupervisor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAIH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAkB3C,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,EAAE,CAAC;IACb,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,gBAAgB,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACpC,aAAa,EAAE,CAAC,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;IAChE,qBAAqB,EAAE,CAAC,OAAO,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAClF,mBAAmB,EAAE,CAAC,IAAI,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACxE;wFACoF;IACpF,aAAa,EAAE,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;CACxC;AAYD;;;;;;GAMG;AACH,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,SAAS,GAAG,SAAS,GAAG,OAAO,CAAC;IAC7C,gEAAgE;IAChE,WAAW,EAAE,MAAM,CAAC;IACpB,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB,6DAA6D;IAC7D,gBAAgB,EAAE,MAAM,CAAC;IACzB,iFAAiF;IACjF,IAAI,EAAE,MAAM,CAAC;CACd;AAED,kEAAkE;AAClE,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,OAAO,CAAC;IAClB;oFACgF;IAChF,MAAM,EAAE,MAAM,CAAC;IACf,+DAA+D;IAC/D,0BAA0B,EAAE,MAAM,CAAC;IACnC,4DAA4D;IAC5D,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAUD;;;;;;GAMG;AACH,MAAM,WAAW,qCAAqC;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,WAAW,CAAC;IACzB,4EAA4E;IAC5E,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qEAAqE;IACrE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,mCAAmC;IAClD,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;CAC/D;AAED,MAAM,WAAW,mCAAmC;IAClD,OAAO,EAAE,SAAS,GAAG,SAAS,CAAC;IAC/B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAyDD,wEAAwE;AACxE,MAAM,WAAW,oBAAoB;IACnC,8DAA8D;IAC9D,iBAAiB,EAAE,CAAC,GAAG,EAAE;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACxD,0EAA0E;IAC1E,oBAAoB,EAAE,MAAM,MAAM,CAAC;CACpC;AAED,qBAAa,gBAAiB,SAAQ,YAAY;IAChD,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,QAAQ,CAAgB;IAChC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,mBAAmB,CAA+C;IAC1E,OAAO,CAAC,iBAAiB,CAAK;IAC9B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAc;IAC7C,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,kBAAkB,CAAK;IAC/B,OAAO,CAAC,gBAAgB,CAAQ;IAChC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,cAAc,CAAW;IACjC,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,eAAe,CAAc;IACrC,OAAO,CAAC,qBAAqB,CAAK;IAClC,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAK;IAMxC,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,QAAQ,CAAC,8BAA8B,CAAK;IACpD,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAK;IAQ3C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA0B;IACvD,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAAM;IACjD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAe;IACjD,OAAO,CAAC,QAAQ,CAAgB;IAGhC,OAAO,CAAC,wBAAwB,CAAK;IACrC,OAAO,CAAC,iBAAiB,CAAc;IACvC,OAAO,CAAC,oBAAoB,CAAuB;IAGnD,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAM;IAC9C,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAe;IACtD,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,wBAAwB,CAAK;IACrC,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAe;IAC7D,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAK;IAC9C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAmB;IACvD,OAAO,CAAC,kBAAkB,CAAK;IAC/B,OAAO,CAAC,eAAe,CAAM;IAC7B,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,iBAAiB,CAAkC;IAC3D,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;IAG3C,0EAA0E;IAC1E,MAAM,CAAC,QAAQ,CAAC,0BAA0B,KAAK;IAC/C,yEAAyE;IACzE,MAAM,CAAC,QAAQ,CAAC,0BAA0B,SAAc;IACxD,OAAO,CAAC,oBAAoB,CAAqC;IACjE,qFAAqF;IACrF,OAAO,CAAC,yBAAyB,CAAmF;IACpH,oFAAoF;IACpF,OAAO,CAAC,iBAAiB,CAA+D;gBAE5E,OAAO,EAAE;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,yGAAyG;QACzG,sBAAsB,CAAC,EAAE,MAAM,CAAC;QAChC,uGAAuG;QACvG,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,+FAA+F;QAC/F,iBAAiB,CAAC,EAAE,MAAM,MAAM,CAAC;KAClC;IAkBD;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC;IAyD/B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAyB3B;;OAEG;IACH,IAAI,OAAO,IAAI,OAAO,CAErB;IAED;;OAEG;IACH,SAAS,IAAI;QACX,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,OAAO,CAAC;QACjB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,EAAE,OAAO,CAAC;QACrB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,aAAa,EAAE,OAAO,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;QACtB,eAAe,EAAE,MAAM,CAAC;QACxB,wBAAwB,EAAE,MAAM,CAAC;QACjC,wBAAwB,EAAE,MAAM,CAAC;QACjC,iBAAiB,EAAE,OAAO,CAAC;QAC3B,wBAAwB,EAAE,MAAM,CAAC;QACjC,gBAAgB,EAAE,OAAO,CAAC;QAC1B,yBAAyB,EAAE,MAAM,CAAC;KACnC;IA2BD;;;OAGG;IACH,mBAAmB,IAAI,IAAI;IAa3B;;;OAGG;IACH,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAI5C;;;;;OAKG;IACG,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAsC9D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACG,oBAAoB,CACxB,GAAG,EAAE,qCAAqC,EAC1C,QAAQ,CAAC,EAAE,mCAAmC,GAC7C,OAAO,CAAC,mCAAmC,CAAC;IAmG/C;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IA8QzB;;OAEG;IACH,OAAO,CAAC,WAAW;IAoBnB;;OAEG;IACH,OAAO,CAAC,YAAY;IAgBpB,OAAO,CAAC,WAAW;IAmFnB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,iBAAiB;IA0HzB,OAAO,CAAC,gBAAgB;YAWV,WAAW;IAoBzB;;;;;;;;;;OAUG;YACW,sBAAsB;IAYpC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;IA2E3B;;;OAGG;IACH,OAAO,CAAC,wBAAwB;IAiDhC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAoC3B;;;;;OAKG;IACH,kBAAkB,CAAC,UAAU,EAAE,oBAAoB,GAAG,IAAI;IAK1D,8DAA8D;IAC9D,2BAA2B,IAAI,MAAM;IAIrC,2DAA2D;IAC3D,oBAAoB,IAAI,MAAM;IAI9B,oEAAoE;IACpE,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAK3C;;;;;;;;;;;;;;OAcG;IACG,sBAAsB,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwG9F;;;;;;OAMG;IACH,OAAO,CAAC,wCAAwC;IAkBhD;;;;;;;OAOG;IACH,OAAO,CAAC,4BAA4B;IAiBpC,kEAAkE;IAClE,gCAAgC,IAAI,MAAM;IAI1C,8EAA8E;IAC9E,0BAA0B,IAAI,IAAI;IAMlC;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,uBAAuB;IAoC/B;;;;OAIG;IACH,OAAO,CAAC,4BAA4B;IAKpC,OAAO,CAAC,eAAe;IAwKvB;;;;OAIG;IAOH,OAAO,CAAC,kBAAkB;IAgC1B;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAgC7B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAgB9B,OAAO,CAAC,sBAAsB;IAa9B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,uBAAuB;IAwC/B;;OAEG;IACH,OAAO,CAAC,sBAAsB;CAS/B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,sEAAsE;IACtE,UAAU,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,uBAAuB,CAAC,eAAe,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAqDpF;AAID;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,6BAA6B,CAAC,OAAO,EAAE,uBAAuB,GAAG,MAAM,CAuCtF"}
@@ -23,6 +23,7 @@ import { fileURLToPath } from 'node:url';
23
23
  const __dirname = path.dirname(fileURLToPath(import.meta.url));
24
24
  import { detectTmuxPath } from '../core/Config.js';
25
25
  import { SleepWakeDetector } from '../core/SleepWakeDetector.js';
26
+ import { cpuLoadRatio, DEFAULT_MAX_LOAD_RATIO } from '../core/cpuStarvation.js';
26
27
  import { SafeFsExecutor } from '../core/SafeFsExecutor.js';
27
28
  import { SafeGitExecutor } from '../core/SafeGitExecutor.js';
28
29
  /** Execute a shell command safely, returning stdout. */
@@ -110,6 +111,16 @@ export class ServerSupervisor extends EventEmitter {
110
111
  consecutiveBindFailures = 0;
111
112
  bindFailureEscalationThreshold = 2;
112
113
  processAliveThreshold = 6; // When process is alive but unresponsive (e.g., high CPU load), require 6 failures (~60s) before restarting
114
+ // CPU-starvation restart guard: when the box is so oversubscribed (load >>
115
+ // cores) that the live server can't answer /health, restarting it does NOT
116
+ // cure the starvation — the fresh server is starved too, it just drops the
117
+ // in-flight message and loops (the 2026-05-29 restart-loop incident). So while
118
+ // CPU-starved we DEFER the restart up to a hard cap, then force it (in case
119
+ // the server is genuinely hung rather than merely starved). Same load-ratio
120
+ // signal SleepWakeDetector uses (see core/cpuStarvation).
121
+ maxLoadRatio = DEFAULT_MAX_LOAD_RATIO; // loadavg[0]/cpuCount above this = starved
122
+ starvationRestartThreshold = 30; // ~5min at 10s checks — force a restart even if starved if unresponsiveness persists this long
123
+ loadRatioProvider; // injectable for tests; default cpuLoadRatio
113
124
  stateDir;
114
125
  // Planned restart / maintenance wait — suppress alerts during expected downtime
115
126
  maintenanceWaitStartedAt = 0;
@@ -150,6 +161,7 @@ export class ServerSupervisor extends EventEmitter {
150
161
  this.stateDir = options.stateDir ?? null;
151
162
  this.tmuxPath = detectTmuxPath();
152
163
  this.serverSessionName = `${this.projectName}-server`;
164
+ this.loadRatioProvider = options.loadRatioProvider ?? (() => cpuLoadRatio());
153
165
  if (options.maintenanceWaitMinutes !== undefined) {
154
166
  this.maintenanceWaitMs = options.maintenanceWaitMinutes * 60_000;
155
167
  }
@@ -993,58 +1005,14 @@ export class ServerSupervisor extends EventEmitter {
993
1005
  else {
994
1006
  this.consecutiveFailures++;
995
1007
  if (this.consecutiveFailures >= this.unhealthyThreshold) {
996
- const processAlive = this.isServerSessionAlive();
997
- if (processAlive) {
998
- // Server process exists but isn't responding to health checks.
999
- // Under high CPU load (or during wake transitions), this is normal —
1000
- // the event loop is stalled, not the process dead. Use a much higher
1001
- // threshold to avoid killing a server that would recover on its own.
1002
- const effectiveThreshold = Date.now() < this.wakeTransitionUntil
1003
- ? this.unhealthyThreshold // During wake transition: already lenient via counter reset
1004
- : this.processAliveThreshold;
1005
- if (this.consecutiveFailures < effectiveThreshold) {
1006
- if (this.consecutiveFailures === this.unhealthyThreshold) {
1007
- console.log(`[Supervisor] Health check failed but server process is alive — waiting for ${effectiveThreshold} consecutive failures before restart (${this.consecutiveFailures}/${effectiveThreshold})`);
1008
- }
1009
- }
1010
- else {
1011
- console.log(`[Supervisor] Server process alive but unresponsive for ${this.consecutiveFailures} checks (~${this.consecutiveFailures * 10}s) — restarting`);
1012
- this.handleUnhealthy();
1013
- }
1014
- if (Date.now() < this.wakeTransitionUntil) {
1015
- this.consecutiveFailures = 0; // Reset during wake transition as before
1016
- }
1017
- }
1018
- else {
1019
- this.handleUnhealthy();
1020
- }
1008
+ this.evaluateUnhealthyServer();
1021
1009
  }
1022
1010
  }
1023
1011
  }
1024
1012
  catch {
1025
1013
  this.consecutiveFailures++;
1026
1014
  if (this.consecutiveFailures >= this.unhealthyThreshold) {
1027
- const processAlive = this.isServerSessionAlive();
1028
- if (processAlive) {
1029
- const effectiveThreshold = Date.now() < this.wakeTransitionUntil
1030
- ? this.unhealthyThreshold
1031
- : this.processAliveThreshold;
1032
- if (this.consecutiveFailures < effectiveThreshold) {
1033
- if (this.consecutiveFailures === this.unhealthyThreshold) {
1034
- console.log(`[Supervisor] Health check failed but server process is alive — waiting for ${effectiveThreshold} consecutive failures before restart (${this.consecutiveFailures}/${effectiveThreshold})`);
1035
- }
1036
- }
1037
- else {
1038
- console.log(`[Supervisor] Server process alive but unresponsive for ${this.consecutiveFailures} checks (~${this.consecutiveFailures * 10}s) — restarting`);
1039
- this.handleUnhealthy();
1040
- }
1041
- if (Date.now() < this.wakeTransitionUntil) {
1042
- this.consecutiveFailures = 0;
1043
- }
1044
- }
1045
- else {
1046
- this.handleUnhealthy();
1047
- }
1015
+ this.evaluateUnhealthyServer();
1048
1016
  }
1049
1017
  }
1050
1018
  // Check for restart requests from the server (e.g., auto-updater)
@@ -1462,6 +1430,63 @@ export class ServerSupervisor extends EventEmitter {
1462
1430
  this.notifyPendingRemediatorRequestsOnHealthy();
1463
1431
  }
1464
1432
  // ── Unhealthy handling ──────────────────────────────────────────
1433
+ /**
1434
+ * Decide what to do about a server that has failed `>= unhealthyThreshold`
1435
+ * consecutive health checks. Shared by the health-check loop's two failure
1436
+ * paths (unhealthy /health response, and a thrown check).
1437
+ *
1438
+ * - Process dead → restart immediately.
1439
+ * - Process alive but unresponsive, below the (load-lenient) threshold →
1440
+ * keep waiting.
1441
+ * - Process alive, threshold reached, but the box is CPU-starved → DEFER the
1442
+ * restart (restarting a starved server only drops the in-flight message
1443
+ * and loops) until starvation eases or the hard cap forces it.
1444
+ * - Process alive, threshold reached, not starved → restart.
1445
+ */
1446
+ evaluateUnhealthyServer() {
1447
+ if (!this.isServerSessionAlive()) {
1448
+ // Process is genuinely gone — restart immediately.
1449
+ this.handleUnhealthy();
1450
+ return;
1451
+ }
1452
+ // Server process exists but isn't responding to health checks. Under high
1453
+ // CPU load (or during wake transitions), this is normal — the event loop is
1454
+ // stalled, not the process dead. Use a much higher threshold to avoid
1455
+ // killing a server that would recover on its own.
1456
+ const inWakeTransition = Date.now() < this.wakeTransitionUntil;
1457
+ const effectiveThreshold = inWakeTransition
1458
+ ? this.unhealthyThreshold // During wake transition: already lenient via counter reset
1459
+ : this.processAliveThreshold;
1460
+ if (this.consecutiveFailures < effectiveThreshold) {
1461
+ if (this.consecutiveFailures === this.unhealthyThreshold) {
1462
+ console.log(`[Supervisor] Health check failed but server process is alive — waiting for ${effectiveThreshold} consecutive failures before restart (${this.consecutiveFailures}/${effectiveThreshold})`);
1463
+ }
1464
+ }
1465
+ else if (this.deferRestartForCpuStarvation()) {
1466
+ // Box is CPU-starved — bouncing the server won't help, it only drops the
1467
+ // in-flight message. Defer; the next healthy tick resets the counter.
1468
+ if (this.consecutiveFailures === effectiveThreshold) {
1469
+ console.log(`[Supervisor] Server alive but unresponsive (${this.consecutiveFailures} checks) AND the box is CPU-starved (load ratio ${this.loadRatioProvider().toFixed(2)} > ${this.maxLoadRatio}) — DEFERRING restart; restarting a starved server only drops in-flight messages. Will force-restart at ${this.starvationRestartThreshold} checks (~${this.starvationRestartThreshold * 10}s) if it persists.`);
1470
+ }
1471
+ }
1472
+ else {
1473
+ console.log(`[Supervisor] Server process alive but unresponsive for ${this.consecutiveFailures} checks (~${this.consecutiveFailures * 10}s) — restarting`);
1474
+ this.handleUnhealthy();
1475
+ }
1476
+ if (inWakeTransition) {
1477
+ this.consecutiveFailures = 0; // Reset during wake transition as before
1478
+ }
1479
+ }
1480
+ /**
1481
+ * True when we should hold off restarting an alive-but-unresponsive server
1482
+ * because the machine is CPU-starved — but only up to the hard cap. Past the
1483
+ * cap we restart regardless (the server may be genuinely hung, not starved).
1484
+ */
1485
+ deferRestartForCpuStarvation() {
1486
+ if (this.consecutiveFailures >= this.starvationRestartThreshold)
1487
+ return false; // hard cap — restart even if starved
1488
+ return this.loadRatioProvider() > this.maxLoadRatio;
1489
+ }
1465
1490
  handleUnhealthy() {
1466
1491
  // Circuit breaker — periodic retry instead of permanent death
1467
1492
  if (this.circuitBroken) {