instar 0.7.49 → 0.7.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/setup-wizard/skill.md +9 -40
- package/.vercel/README.txt +11 -0
- package/.vercel/project.json +1 -0
- package/dist/commands/server.js +141 -1
- package/dist/core/AutoUpdater.js +12 -5
- package/dist/core/CaffeinateManager.d.ts +50 -0
- package/dist/core/CaffeinateManager.js +180 -0
- package/dist/core/SessionManager.d.ts +6 -0
- package/dist/core/SessionManager.js +31 -27
- package/dist/core/UpdateChecker.d.ts +2 -1
- package/dist/core/UpdateChecker.js +37 -10
- package/dist/core/types.d.ts +8 -0
- package/dist/lifeline/ServerSupervisor.d.ts +2 -0
- package/dist/lifeline/ServerSupervisor.js +8 -1
- package/dist/lifeline/TelegramLifeline.d.ts +5 -0
- package/dist/lifeline/TelegramLifeline.js +86 -0
- package/dist/monitoring/HealthChecker.d.ts +4 -1
- package/dist/monitoring/HealthChecker.js +36 -1
- package/dist/monitoring/MemoryPressureMonitor.d.ts +83 -0
- package/dist/monitoring/MemoryPressureMonitor.js +242 -0
- package/dist/monitoring/SessionWatchdog.d.ts +83 -0
- package/dist/monitoring/SessionWatchdog.js +326 -0
- package/dist/scaffold/templates.js +31 -8
- package/dist/server/AgentServer.d.ts +2 -0
- package/dist/server/AgentServer.js +1 -0
- package/dist/server/middleware.js +5 -0
- package/dist/server/routes.d.ts +2 -0
- package/dist/server/routes.js +82 -3
- package/package.json +1 -1
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MemoryPressureMonitor - Detect and respond to system memory pressure.
|
|
3
|
+
*
|
|
4
|
+
* Platform-aware: uses macOS `vm_stat` or Linux `/proc/meminfo`.
|
|
5
|
+
* EventEmitter pattern consistent with Instar conventions.
|
|
6
|
+
*
|
|
7
|
+
* Thresholds:
|
|
8
|
+
* - normal (< 60%): all operations allowed
|
|
9
|
+
* - warning (60-75%): log trend, notify
|
|
10
|
+
* - elevated (75-90%): restrict session spawning
|
|
11
|
+
* - critical (90%+): block all spawns, alert
|
|
12
|
+
*
|
|
13
|
+
* Includes trend tracking via ring buffer + linear regression.
|
|
14
|
+
*/
|
|
15
|
+
import { EventEmitter } from 'node:events';
|
|
16
|
+
import { execSync } from 'node:child_process';
|
|
17
|
+
import * as fs from 'node:fs';
|
|
18
|
+
const DEFAULT_THRESHOLDS = {
|
|
19
|
+
warning: 60,
|
|
20
|
+
elevated: 75,
|
|
21
|
+
critical: 90,
|
|
22
|
+
};
|
|
23
|
+
const RING_BUFFER_SIZE = 20;
|
|
24
|
+
const TREND_WINDOW = 6;
|
|
25
|
+
const PAGE_SIZE_BYTES = 16384; // macOS Apple Silicon
|
|
26
|
+
// Adaptive intervals
|
|
27
|
+
const INTERVALS = {
|
|
28
|
+
normal: 30_000,
|
|
29
|
+
warning: 15_000,
|
|
30
|
+
elevated: 10_000,
|
|
31
|
+
critical: 5_000,
|
|
32
|
+
};
|
|
33
|
+
export class MemoryPressureMonitor extends EventEmitter {
|
|
34
|
+
timeout = null;
|
|
35
|
+
currentState = 'normal';
|
|
36
|
+
stateChangedAt = new Date().toISOString();
|
|
37
|
+
lastChecked = new Date().toISOString();
|
|
38
|
+
lastPressurePercent = 0;
|
|
39
|
+
lastFreeGB = 0;
|
|
40
|
+
lastTotalGB = 0;
|
|
41
|
+
ringBuffer = [];
|
|
42
|
+
currentTrend = 'stable';
|
|
43
|
+
currentRatePerMin = 0;
|
|
44
|
+
thresholds;
|
|
45
|
+
baseIntervalMs;
|
|
46
|
+
constructor(config = {}) {
|
|
47
|
+
super();
|
|
48
|
+
this.thresholds = {
|
|
49
|
+
...DEFAULT_THRESHOLDS,
|
|
50
|
+
...config.thresholds,
|
|
51
|
+
};
|
|
52
|
+
this.baseIntervalMs = config.checkIntervalMs ?? 30_000;
|
|
53
|
+
}
|
|
54
|
+
start() {
|
|
55
|
+
if (this.timeout)
|
|
56
|
+
return;
|
|
57
|
+
this.check();
|
|
58
|
+
this.scheduleNext();
|
|
59
|
+
console.log(`[MemoryPressureMonitor] Started (platform: ${process.platform}, thresholds: ${JSON.stringify(this.thresholds)})`);
|
|
60
|
+
}
|
|
61
|
+
stop() {
|
|
62
|
+
if (this.timeout) {
|
|
63
|
+
clearTimeout(this.timeout);
|
|
64
|
+
this.timeout = null;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
getState() {
|
|
68
|
+
return {
|
|
69
|
+
pressurePercent: this.lastPressurePercent,
|
|
70
|
+
freeGB: this.lastFreeGB,
|
|
71
|
+
totalGB: this.lastTotalGB,
|
|
72
|
+
state: this.currentState,
|
|
73
|
+
trend: this.currentTrend,
|
|
74
|
+
ratePerMin: this.currentRatePerMin,
|
|
75
|
+
lastChecked: this.lastChecked,
|
|
76
|
+
stateChangedAt: this.stateChangedAt,
|
|
77
|
+
platform: process.platform,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Can a new session be spawned?
|
|
82
|
+
*/
|
|
83
|
+
canSpawnSession() {
|
|
84
|
+
switch (this.currentState) {
|
|
85
|
+
case 'normal':
|
|
86
|
+
case 'warning':
|
|
87
|
+
return { allowed: true };
|
|
88
|
+
case 'elevated':
|
|
89
|
+
return {
|
|
90
|
+
allowed: false,
|
|
91
|
+
reason: `Memory pressure elevated (${this.lastPressurePercent.toFixed(1)}%) — session spawn blocked`,
|
|
92
|
+
};
|
|
93
|
+
case 'critical':
|
|
94
|
+
return {
|
|
95
|
+
allowed: false,
|
|
96
|
+
reason: `Memory pressure critical (${this.lastPressurePercent.toFixed(1)}%) — all spawns blocked`,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
scheduleNext() {
|
|
101
|
+
const intervalMs = INTERVALS[this.currentState] || this.baseIntervalMs;
|
|
102
|
+
this.timeout = setTimeout(() => {
|
|
103
|
+
this.check();
|
|
104
|
+
this.scheduleNext();
|
|
105
|
+
}, intervalMs);
|
|
106
|
+
this.timeout.unref(); // Don't prevent process exit
|
|
107
|
+
}
|
|
108
|
+
check() {
|
|
109
|
+
try {
|
|
110
|
+
const { pressurePercent, freeGB, totalGB } = this.readSystemMemory();
|
|
111
|
+
this.lastPressurePercent = pressurePercent;
|
|
112
|
+
this.lastFreeGB = freeGB;
|
|
113
|
+
this.lastTotalGB = totalGB;
|
|
114
|
+
this.lastChecked = new Date().toISOString();
|
|
115
|
+
// Ring buffer
|
|
116
|
+
this.ringBuffer.push({ timestamp: Date.now(), pressurePercent });
|
|
117
|
+
if (this.ringBuffer.length > RING_BUFFER_SIZE) {
|
|
118
|
+
this.ringBuffer.shift();
|
|
119
|
+
}
|
|
120
|
+
// Trend
|
|
121
|
+
const { trend, ratePerMin } = this.detectTrend();
|
|
122
|
+
this.currentTrend = trend;
|
|
123
|
+
this.currentRatePerMin = ratePerMin;
|
|
124
|
+
const newState = this.classifyState(pressurePercent);
|
|
125
|
+
if (newState !== this.currentState) {
|
|
126
|
+
const from = this.currentState;
|
|
127
|
+
this.currentState = newState;
|
|
128
|
+
this.stateChangedAt = new Date().toISOString();
|
|
129
|
+
console.log(`[MemoryPressureMonitor] ${from} -> ${newState} (${pressurePercent.toFixed(1)}%, ${freeGB.toFixed(1)}GB free, trend: ${trend})`);
|
|
130
|
+
this.emit('stateChange', { from, to: newState, state: this.getState() });
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
console.error('[MemoryPressureMonitor] Check failed:', error);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
classifyState(pressurePercent) {
|
|
138
|
+
if (pressurePercent >= this.thresholds.critical)
|
|
139
|
+
return 'critical';
|
|
140
|
+
if (pressurePercent >= this.thresholds.elevated)
|
|
141
|
+
return 'elevated';
|
|
142
|
+
if (pressurePercent >= this.thresholds.warning)
|
|
143
|
+
return 'warning';
|
|
144
|
+
return 'normal';
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Read system memory — platform-aware.
|
|
148
|
+
*/
|
|
149
|
+
readSystemMemory() {
|
|
150
|
+
if (process.platform === 'darwin') {
|
|
151
|
+
return this.parseVmStat();
|
|
152
|
+
}
|
|
153
|
+
else if (process.platform === 'linux') {
|
|
154
|
+
return this.parseProcMeminfo();
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
// Fallback: use Node's process.memoryUsage (very rough)
|
|
158
|
+
const mem = process.memoryUsage();
|
|
159
|
+
const totalGB = require('os').totalmem() / (1024 ** 3);
|
|
160
|
+
const usedGB = mem.rss / (1024 ** 3);
|
|
161
|
+
return {
|
|
162
|
+
pressurePercent: (usedGB / totalGB) * 100,
|
|
163
|
+
freeGB: totalGB - usedGB,
|
|
164
|
+
totalGB,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* macOS: parse vm_stat
|
|
170
|
+
*/
|
|
171
|
+
parseVmStat() {
|
|
172
|
+
const output = execSync('vm_stat', { encoding: 'utf-8', timeout: 5000 });
|
|
173
|
+
const pageSizeMatch = output.match(/page size of (\d+) bytes/);
|
|
174
|
+
const pageSize = pageSizeMatch ? parseInt(pageSizeMatch[1], 10) : PAGE_SIZE_BYTES;
|
|
175
|
+
const parsePages = (label) => {
|
|
176
|
+
const match = output.match(new RegExp(`${label}:\\s+(\\d+)`));
|
|
177
|
+
return match ? parseInt(match[1], 10) : 0;
|
|
178
|
+
};
|
|
179
|
+
const freePages = parsePages('Pages free');
|
|
180
|
+
const activePages = parsePages('Pages active');
|
|
181
|
+
const inactivePages = parsePages('Pages inactive');
|
|
182
|
+
const wiredPages = parsePages('Pages wired down');
|
|
183
|
+
const compressorPages = parsePages('Pages occupied by compressor');
|
|
184
|
+
const purgeablePages = parsePages('Pages purgeable');
|
|
185
|
+
const totalPages = freePages + activePages + inactivePages + wiredPages + compressorPages;
|
|
186
|
+
const totalBytes = totalPages * pageSize;
|
|
187
|
+
const totalGB = totalBytes / (1024 ** 3);
|
|
188
|
+
const availablePages = freePages + inactivePages + purgeablePages;
|
|
189
|
+
const availableBytes = availablePages * pageSize;
|
|
190
|
+
const freeGB = availableBytes / (1024 ** 3);
|
|
191
|
+
const usedPages = totalPages - availablePages;
|
|
192
|
+
const pressurePercent = totalPages > 0 ? (usedPages / totalPages) * 100 : 0;
|
|
193
|
+
return { pressurePercent, freeGB, totalGB };
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Linux: parse /proc/meminfo
|
|
197
|
+
*/
|
|
198
|
+
parseProcMeminfo() {
|
|
199
|
+
const content = fs.readFileSync('/proc/meminfo', 'utf-8');
|
|
200
|
+
const parseKB = (key) => {
|
|
201
|
+
const match = content.match(new RegExp(`${key}:\\s+(\\d+)`));
|
|
202
|
+
return match ? parseInt(match[1], 10) : 0;
|
|
203
|
+
};
|
|
204
|
+
const totalKB = parseKB('MemTotal');
|
|
205
|
+
const availableKB = parseKB('MemAvailable') || (parseKB('MemFree') + parseKB('Buffers') + parseKB('Cached'));
|
|
206
|
+
const totalGB = totalKB / (1024 * 1024);
|
|
207
|
+
const freeGB = availableKB / (1024 * 1024);
|
|
208
|
+
const pressurePercent = totalKB > 0 ? ((totalKB - availableKB) / totalKB) * 100 : 0;
|
|
209
|
+
return { pressurePercent, freeGB, totalGB };
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Linear regression over recent readings.
|
|
213
|
+
*/
|
|
214
|
+
detectTrend() {
|
|
215
|
+
if (this.ringBuffer.length < 3) {
|
|
216
|
+
return { trend: 'stable', ratePerMin: 0 };
|
|
217
|
+
}
|
|
218
|
+
const readings = this.ringBuffer.slice(-TREND_WINDOW);
|
|
219
|
+
const n = readings.length;
|
|
220
|
+
const firstTs = readings[0].timestamp;
|
|
221
|
+
const xs = readings.map(r => (r.timestamp - firstTs) / 1000);
|
|
222
|
+
const ys = readings.map(r => r.pressurePercent);
|
|
223
|
+
const sumX = xs.reduce((a, b) => a + b, 0);
|
|
224
|
+
const sumY = ys.reduce((a, b) => a + b, 0);
|
|
225
|
+
const sumXY = xs.reduce((a, x, i) => a + x * ys[i], 0);
|
|
226
|
+
const sumX2 = xs.reduce((a, x) => a + x * x, 0);
|
|
227
|
+
const denom = n * sumX2 - sumX * sumX;
|
|
228
|
+
if (denom === 0)
|
|
229
|
+
return { trend: 'stable', ratePerMin: 0 };
|
|
230
|
+
const slope = (n * sumXY - sumX * sumY) / denom;
|
|
231
|
+
const ratePerMin = slope * 60;
|
|
232
|
+
let trend;
|
|
233
|
+
if (ratePerMin > 0.5)
|
|
234
|
+
trend = 'rising';
|
|
235
|
+
else if (ratePerMin < -0.5)
|
|
236
|
+
trend = 'falling';
|
|
237
|
+
else
|
|
238
|
+
trend = 'stable';
|
|
239
|
+
return { trend, ratePerMin };
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
//# sourceMappingURL=MemoryPressureMonitor.js.map
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SessionWatchdog — Auto-remediation for stuck Claude sessions (Instar port).
|
|
3
|
+
*
|
|
4
|
+
* Detects when a Claude session has a long-running bash command and escalates
|
|
5
|
+
* from gentle (Ctrl+C) to forceful (SIGKILL + session kill). Adapted from
|
|
6
|
+
* Dawn Server's SessionWatchdog for Instar's self-contained architecture.
|
|
7
|
+
*
|
|
8
|
+
* Escalation pipeline:
|
|
9
|
+
* Level 0: Monitoring (default)
|
|
10
|
+
* Level 1: Ctrl+C via tmux send-keys
|
|
11
|
+
* Level 2: SIGTERM the stuck child PID
|
|
12
|
+
* Level 3: SIGKILL the stuck child PID
|
|
13
|
+
* Level 4: Kill tmux session
|
|
14
|
+
*/
|
|
15
|
+
import { EventEmitter } from 'node:events';
|
|
16
|
+
import type { SessionManager } from '../core/SessionManager.js';
|
|
17
|
+
import type { StateManager } from '../core/StateManager.js';
|
|
18
|
+
import type { InstarConfig } from '../core/types.js';
|
|
19
|
+
export declare enum EscalationLevel {
|
|
20
|
+
Monitoring = 0,
|
|
21
|
+
CtrlC = 1,
|
|
22
|
+
SigTerm = 2,
|
|
23
|
+
SigKill = 3,
|
|
24
|
+
KillSession = 4
|
|
25
|
+
}
|
|
26
|
+
interface EscalationState {
|
|
27
|
+
level: EscalationLevel;
|
|
28
|
+
levelEnteredAt: number;
|
|
29
|
+
stuckChildPid: number;
|
|
30
|
+
stuckCommand: string;
|
|
31
|
+
retryCount: number;
|
|
32
|
+
}
|
|
33
|
+
export interface InterventionEvent {
|
|
34
|
+
sessionName: string;
|
|
35
|
+
level: EscalationLevel;
|
|
36
|
+
action: string;
|
|
37
|
+
stuckCommand: string;
|
|
38
|
+
stuckPid: number;
|
|
39
|
+
timestamp: number;
|
|
40
|
+
}
|
|
41
|
+
export interface WatchdogEvents {
|
|
42
|
+
intervention: [event: InterventionEvent];
|
|
43
|
+
recovery: [sessionName: string, fromLevel: EscalationLevel];
|
|
44
|
+
}
|
|
45
|
+
export declare class SessionWatchdog extends EventEmitter {
|
|
46
|
+
private config;
|
|
47
|
+
private sessionManager;
|
|
48
|
+
private state;
|
|
49
|
+
private interval;
|
|
50
|
+
private escalationState;
|
|
51
|
+
private interventionHistory;
|
|
52
|
+
private enabled;
|
|
53
|
+
private running;
|
|
54
|
+
private stuckThresholdMs;
|
|
55
|
+
private pollIntervalMs;
|
|
56
|
+
constructor(config: InstarConfig, sessionManager: SessionManager, state: StateManager);
|
|
57
|
+
start(): void;
|
|
58
|
+
stop(): void;
|
|
59
|
+
setEnabled(enabled: boolean): void;
|
|
60
|
+
isEnabled(): boolean;
|
|
61
|
+
isManaging(sessionName: string): boolean;
|
|
62
|
+
getStatus(): {
|
|
63
|
+
enabled: boolean;
|
|
64
|
+
sessions: Array<{
|
|
65
|
+
name: string;
|
|
66
|
+
escalation: EscalationState | null;
|
|
67
|
+
}>;
|
|
68
|
+
interventionHistory: InterventionEvent[];
|
|
69
|
+
};
|
|
70
|
+
private poll;
|
|
71
|
+
private checkSession;
|
|
72
|
+
private handleEscalation;
|
|
73
|
+
private getClaudePid;
|
|
74
|
+
private getChildProcesses;
|
|
75
|
+
private isExcluded;
|
|
76
|
+
private parseElapsed;
|
|
77
|
+
private sendSignal;
|
|
78
|
+
private isProcessAlive;
|
|
79
|
+
private killTmuxSession;
|
|
80
|
+
private recordIntervention;
|
|
81
|
+
}
|
|
82
|
+
export {};
|
|
83
|
+
//# sourceMappingURL=SessionWatchdog.d.ts.map
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SessionWatchdog — Auto-remediation for stuck Claude sessions (Instar port).
|
|
3
|
+
*
|
|
4
|
+
* Detects when a Claude session has a long-running bash command and escalates
|
|
5
|
+
* from gentle (Ctrl+C) to forceful (SIGKILL + session kill). Adapted from
|
|
6
|
+
* Dawn Server's SessionWatchdog for Instar's self-contained architecture.
|
|
7
|
+
*
|
|
8
|
+
* Escalation pipeline:
|
|
9
|
+
* Level 0: Monitoring (default)
|
|
10
|
+
* Level 1: Ctrl+C via tmux send-keys
|
|
11
|
+
* Level 2: SIGTERM the stuck child PID
|
|
12
|
+
* Level 3: SIGKILL the stuck child PID
|
|
13
|
+
* Level 4: Kill tmux session
|
|
14
|
+
*/
|
|
15
|
+
import { execSync } from 'node:child_process';
|
|
16
|
+
import { EventEmitter } from 'node:events';
|
|
17
|
+
export var EscalationLevel;
|
|
18
|
+
(function (EscalationLevel) {
|
|
19
|
+
EscalationLevel[EscalationLevel["Monitoring"] = 0] = "Monitoring";
|
|
20
|
+
EscalationLevel[EscalationLevel["CtrlC"] = 1] = "CtrlC";
|
|
21
|
+
EscalationLevel[EscalationLevel["SigTerm"] = 2] = "SigTerm";
|
|
22
|
+
EscalationLevel[EscalationLevel["SigKill"] = 3] = "SigKill";
|
|
23
|
+
EscalationLevel[EscalationLevel["KillSession"] = 4] = "KillSession";
|
|
24
|
+
})(EscalationLevel || (EscalationLevel = {}));
|
|
25
|
+
// Processes that are long-running by design
|
|
26
|
+
const EXCLUDED_PATTERNS = [
|
|
27
|
+
'playwright-mcp', 'playwright-persistent', '@playwright/mcp',
|
|
28
|
+
'chrome-native-host', 'claude-in-chrome-mcp', 'payments-mcp',
|
|
29
|
+
'mcp-remote', '/mcp/', '.mcp/', 'caffeinate', 'exa-mcp-server',
|
|
30
|
+
];
|
|
31
|
+
const EXCLUDED_PREFIXES = [
|
|
32
|
+
'/bin/zsh -c -l source',
|
|
33
|
+
'/bin/bash -c -l source',
|
|
34
|
+
];
|
|
35
|
+
// Escalation delays (ms to wait before advancing to next level)
|
|
36
|
+
const ESCALATION_DELAYS = {
|
|
37
|
+
[EscalationLevel.Monitoring]: 0,
|
|
38
|
+
[EscalationLevel.CtrlC]: 0,
|
|
39
|
+
[EscalationLevel.SigTerm]: 15_000,
|
|
40
|
+
[EscalationLevel.SigKill]: 10_000,
|
|
41
|
+
[EscalationLevel.KillSession]: 5_000,
|
|
42
|
+
};
|
|
43
|
+
const DEFAULT_STUCK_THRESHOLD_MS = 180_000; // 3 minutes
|
|
44
|
+
const DEFAULT_POLL_INTERVAL_MS = 30_000;
|
|
45
|
+
const MAX_RETRIES = 2;
|
|
46
|
+
export class SessionWatchdog extends EventEmitter {
|
|
47
|
+
config;
|
|
48
|
+
sessionManager;
|
|
49
|
+
state;
|
|
50
|
+
interval = null;
|
|
51
|
+
escalationState = new Map();
|
|
52
|
+
interventionHistory = [];
|
|
53
|
+
enabled = true;
|
|
54
|
+
running = false;
|
|
55
|
+
stuckThresholdMs;
|
|
56
|
+
pollIntervalMs;
|
|
57
|
+
constructor(config, sessionManager, state) {
|
|
58
|
+
super();
|
|
59
|
+
this.config = config;
|
|
60
|
+
this.sessionManager = sessionManager;
|
|
61
|
+
this.state = state;
|
|
62
|
+
const wdConfig = config.monitoring.watchdog;
|
|
63
|
+
this.stuckThresholdMs = (wdConfig?.stuckCommandSec ?? 180) * 1000;
|
|
64
|
+
this.pollIntervalMs = wdConfig?.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
65
|
+
}
|
|
66
|
+
start() {
|
|
67
|
+
if (this.interval)
|
|
68
|
+
return;
|
|
69
|
+
console.log(`[Watchdog] Starting (poll: ${this.pollIntervalMs / 1000}s, threshold: ${this.stuckThresholdMs / 1000}s)`);
|
|
70
|
+
this.interval = setInterval(() => this.poll(), this.pollIntervalMs);
|
|
71
|
+
setTimeout(() => this.poll(), 5000);
|
|
72
|
+
}
|
|
73
|
+
stop() {
|
|
74
|
+
if (this.interval) {
|
|
75
|
+
clearInterval(this.interval);
|
|
76
|
+
this.interval = null;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
setEnabled(enabled) {
|
|
80
|
+
this.enabled = enabled;
|
|
81
|
+
if (!enabled) {
|
|
82
|
+
this.escalationState.clear();
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
isEnabled() {
|
|
86
|
+
return this.enabled;
|
|
87
|
+
}
|
|
88
|
+
isManaging(sessionName) {
|
|
89
|
+
const s = this.escalationState.get(sessionName);
|
|
90
|
+
return s !== undefined && s.level > EscalationLevel.Monitoring;
|
|
91
|
+
}
|
|
92
|
+
getStatus() {
|
|
93
|
+
const runningSessions = this.sessionManager.listRunningSessions();
|
|
94
|
+
const sessions = runningSessions.map(s => ({
|
|
95
|
+
name: s.tmuxSession,
|
|
96
|
+
escalation: this.escalationState.get(s.tmuxSession) ?? null,
|
|
97
|
+
}));
|
|
98
|
+
return {
|
|
99
|
+
enabled: this.enabled,
|
|
100
|
+
sessions,
|
|
101
|
+
interventionHistory: this.interventionHistory.slice(-20),
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
// --- Core polling ---
|
|
105
|
+
async poll() {
|
|
106
|
+
if (!this.enabled || this.running)
|
|
107
|
+
return;
|
|
108
|
+
this.running = true;
|
|
109
|
+
try {
|
|
110
|
+
const sessions = this.sessionManager.listRunningSessions();
|
|
111
|
+
for (const session of sessions) {
|
|
112
|
+
try {
|
|
113
|
+
this.checkSession(session.tmuxSession);
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
console.error(`[Watchdog] Error checking "${session.tmuxSession}":`, err);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
finally {
|
|
121
|
+
this.running = false;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
checkSession(tmuxSession) {
|
|
125
|
+
const existing = this.escalationState.get(tmuxSession);
|
|
126
|
+
if (existing && existing.level > EscalationLevel.Monitoring) {
|
|
127
|
+
this.handleEscalation(tmuxSession, existing);
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
// Find Claude PID in the tmux session
|
|
131
|
+
const claudePid = this.getClaudePid(tmuxSession);
|
|
132
|
+
if (!claudePid)
|
|
133
|
+
return;
|
|
134
|
+
const children = this.getChildProcesses(claudePid);
|
|
135
|
+
const stuckChild = children.find(c => !this.isExcluded(c.command) && c.elapsedMs > this.stuckThresholdMs);
|
|
136
|
+
if (stuckChild) {
|
|
137
|
+
const state = {
|
|
138
|
+
level: EscalationLevel.CtrlC,
|
|
139
|
+
levelEnteredAt: Date.now(),
|
|
140
|
+
stuckChildPid: stuckChild.pid,
|
|
141
|
+
stuckCommand: stuckChild.command,
|
|
142
|
+
retryCount: existing?.retryCount ?? 0,
|
|
143
|
+
};
|
|
144
|
+
this.escalationState.set(tmuxSession, state);
|
|
145
|
+
console.log(`[Watchdog] "${tmuxSession}": stuck command (${Math.round(stuckChild.elapsedMs / 1000)}s): ` +
|
|
146
|
+
`${stuckChild.command.slice(0, 80)} — sending Ctrl+C`);
|
|
147
|
+
this.sessionManager.sendKey(tmuxSession, 'C-c');
|
|
148
|
+
this.recordIntervention(tmuxSession, EscalationLevel.CtrlC, 'Sent Ctrl+C', stuckChild);
|
|
149
|
+
}
|
|
150
|
+
else if (existing) {
|
|
151
|
+
this.escalationState.delete(tmuxSession);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
handleEscalation(tmuxSession, state) {
|
|
155
|
+
const now = Date.now();
|
|
156
|
+
if (!this.isProcessAlive(state.stuckChildPid)) {
|
|
157
|
+
console.log(`[Watchdog] "${tmuxSession}": stuck process ${state.stuckChildPid} died — recovered`);
|
|
158
|
+
this.emit('recovery', tmuxSession, state.level);
|
|
159
|
+
this.escalationState.delete(tmuxSession);
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
const timeInLevel = now - state.levelEnteredAt;
|
|
163
|
+
const nextLevel = state.level + 1;
|
|
164
|
+
if (nextLevel > EscalationLevel.KillSession) {
|
|
165
|
+
if (state.retryCount >= MAX_RETRIES) {
|
|
166
|
+
console.log(`[Watchdog] "${tmuxSession}": max retries reached — giving up`);
|
|
167
|
+
this.escalationState.delete(tmuxSession);
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
state.level = EscalationLevel.CtrlC;
|
|
171
|
+
state.levelEnteredAt = now;
|
|
172
|
+
state.retryCount++;
|
|
173
|
+
this.sessionManager.sendKey(tmuxSession, 'C-c');
|
|
174
|
+
this.recordIntervention(tmuxSession, EscalationLevel.CtrlC, `Retry ${state.retryCount}: Sent Ctrl+C`, {
|
|
175
|
+
pid: state.stuckChildPid, command: state.stuckCommand, elapsedMs: 0,
|
|
176
|
+
});
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
const delayForNext = ESCALATION_DELAYS[nextLevel] ?? 15_000;
|
|
180
|
+
if (timeInLevel < delayForNext)
|
|
181
|
+
return;
|
|
182
|
+
state.level = nextLevel;
|
|
183
|
+
state.levelEnteredAt = now;
|
|
184
|
+
const child = { pid: state.stuckChildPid, command: state.stuckCommand, elapsedMs: 0 };
|
|
185
|
+
switch (state.level) {
|
|
186
|
+
case EscalationLevel.SigTerm:
|
|
187
|
+
console.log(`[Watchdog] "${tmuxSession}": sending SIGTERM to ${state.stuckChildPid}`);
|
|
188
|
+
this.sendSignal(state.stuckChildPid, 'SIGTERM');
|
|
189
|
+
this.recordIntervention(tmuxSession, EscalationLevel.SigTerm, `SIGTERM ${state.stuckChildPid}`, child);
|
|
190
|
+
break;
|
|
191
|
+
case EscalationLevel.SigKill:
|
|
192
|
+
console.log(`[Watchdog] "${tmuxSession}": sending SIGKILL to ${state.stuckChildPid}`);
|
|
193
|
+
this.sendSignal(state.stuckChildPid, 'SIGKILL');
|
|
194
|
+
this.recordIntervention(tmuxSession, EscalationLevel.SigKill, `SIGKILL ${state.stuckChildPid}`, child);
|
|
195
|
+
break;
|
|
196
|
+
case EscalationLevel.KillSession:
|
|
197
|
+
console.log(`[Watchdog] "${tmuxSession}": killing tmux session`);
|
|
198
|
+
this.killTmuxSession(tmuxSession);
|
|
199
|
+
this.recordIntervention(tmuxSession, EscalationLevel.KillSession, 'Killed tmux session', child);
|
|
200
|
+
this.escalationState.delete(tmuxSession);
|
|
201
|
+
break;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
// --- Process utilities (self-contained, no shared module) ---
|
|
205
|
+
getClaudePid(tmuxSession) {
|
|
206
|
+
try {
|
|
207
|
+
// Get pane PID
|
|
208
|
+
const panePidStr = execSync(`${this.config.sessions.tmuxPath} list-panes -t "=${tmuxSession}" -F "#{pane_pid}" 2>/dev/null`, { encoding: 'utf-8', timeout: 5000 }).trim().split('\n')[0];
|
|
209
|
+
if (!panePidStr)
|
|
210
|
+
return null;
|
|
211
|
+
const panePid = parseInt(panePidStr, 10);
|
|
212
|
+
if (isNaN(panePid))
|
|
213
|
+
return null;
|
|
214
|
+
// Find claude child
|
|
215
|
+
const claudePidStr = execSync(`pgrep -P ${panePid} -f claude 2>/dev/null | head -1`, { encoding: 'utf-8', timeout: 5000 }).trim();
|
|
216
|
+
if (!claudePidStr)
|
|
217
|
+
return null;
|
|
218
|
+
const pid = parseInt(claudePidStr, 10);
|
|
219
|
+
return isNaN(pid) ? null : pid;
|
|
220
|
+
}
|
|
221
|
+
catch {
|
|
222
|
+
return null;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
getChildProcesses(pid) {
|
|
226
|
+
try {
|
|
227
|
+
const childPidsStr = execSync(`pgrep -P ${pid} 2>/dev/null`, { encoding: 'utf-8', timeout: 5000 }).trim();
|
|
228
|
+
if (!childPidsStr)
|
|
229
|
+
return [];
|
|
230
|
+
const childPids = childPidsStr.split('\n').filter(Boolean).join(',');
|
|
231
|
+
if (!childPids)
|
|
232
|
+
return [];
|
|
233
|
+
const output = execSync(`ps -o pid=,etime=,command= -p ${childPids} 2>/dev/null`, { encoding: 'utf-8', timeout: 5000 }).trim();
|
|
234
|
+
if (!output)
|
|
235
|
+
return [];
|
|
236
|
+
const results = [];
|
|
237
|
+
for (const line of output.split('\n')) {
|
|
238
|
+
const match = line.trim().match(/^(\d+)\s+([\d:.-]+)\s+(.+)$/);
|
|
239
|
+
if (!match)
|
|
240
|
+
continue;
|
|
241
|
+
const childPid = parseInt(match[1], 10);
|
|
242
|
+
if (isNaN(childPid))
|
|
243
|
+
continue;
|
|
244
|
+
results.push({
|
|
245
|
+
pid: childPid,
|
|
246
|
+
command: match[3],
|
|
247
|
+
elapsedMs: this.parseElapsed(match[2]),
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
return results;
|
|
251
|
+
}
|
|
252
|
+
catch {
|
|
253
|
+
return [];
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
isExcluded(command) {
|
|
257
|
+
for (const pattern of EXCLUDED_PATTERNS) {
|
|
258
|
+
if (command.includes(pattern))
|
|
259
|
+
return true;
|
|
260
|
+
}
|
|
261
|
+
for (const prefix of EXCLUDED_PREFIXES) {
|
|
262
|
+
if (command.startsWith(prefix))
|
|
263
|
+
return true;
|
|
264
|
+
}
|
|
265
|
+
return false;
|
|
266
|
+
}
|
|
267
|
+
parseElapsed(elapsed) {
|
|
268
|
+
let days = 0;
|
|
269
|
+
let timePart = elapsed;
|
|
270
|
+
if (elapsed.includes('-')) {
|
|
271
|
+
const [d, t] = elapsed.split('-');
|
|
272
|
+
days = parseInt(d, 10);
|
|
273
|
+
timePart = t;
|
|
274
|
+
}
|
|
275
|
+
const parts = timePart.split(':').map(Number);
|
|
276
|
+
let seconds = 0;
|
|
277
|
+
if (parts.length === 3)
|
|
278
|
+
seconds = parts[0] * 3600 + parts[1] * 60 + parts[2];
|
|
279
|
+
else if (parts.length === 2)
|
|
280
|
+
seconds = parts[0] * 60 + parts[1];
|
|
281
|
+
else
|
|
282
|
+
seconds = parts[0];
|
|
283
|
+
return (days * 86400 + seconds) * 1000;
|
|
284
|
+
}
|
|
285
|
+
sendSignal(pid, signal) {
|
|
286
|
+
try {
|
|
287
|
+
process.kill(pid, signal);
|
|
288
|
+
}
|
|
289
|
+
catch (err) {
|
|
290
|
+
if (err.code !== 'ESRCH') {
|
|
291
|
+
console.error(`[Watchdog] Failed to send ${signal} to ${pid}:`, err);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
isProcessAlive(pid) {
|
|
296
|
+
try {
|
|
297
|
+
process.kill(pid, 0);
|
|
298
|
+
return true;
|
|
299
|
+
}
|
|
300
|
+
catch {
|
|
301
|
+
return false;
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
killTmuxSession(tmuxSession) {
|
|
305
|
+
try {
|
|
306
|
+
execSync(`${this.config.sessions.tmuxPath} kill-session -t "=${tmuxSession}" 2>/dev/null`, { timeout: 5000, stdio: 'ignore' });
|
|
307
|
+
}
|
|
308
|
+
catch { }
|
|
309
|
+
}
|
|
310
|
+
recordIntervention(sessionName, level, action, child) {
|
|
311
|
+
const event = {
|
|
312
|
+
sessionName,
|
|
313
|
+
level,
|
|
314
|
+
action,
|
|
315
|
+
stuckCommand: child.command.slice(0, 200),
|
|
316
|
+
stuckPid: child.pid,
|
|
317
|
+
timestamp: Date.now(),
|
|
318
|
+
};
|
|
319
|
+
this.interventionHistory.push(event);
|
|
320
|
+
if (this.interventionHistory.length > 50) {
|
|
321
|
+
this.interventionHistory = this.interventionHistory.slice(-50);
|
|
322
|
+
}
|
|
323
|
+
this.emit('intervention', event);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
//# sourceMappingURL=SessionWatchdog.js.map
|