@songsid/agend 2.0.12 → 2.1.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/antigravity.js +3 -1
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/codex.js +3 -1
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/kiro.js +3 -2
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +12 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/cli.js +43 -6
- package/dist/cli.js.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/context-guardian.d.ts +2 -0
- package/dist/context-guardian.js +10 -2
- package/dist/context-guardian.js.map +1 -1
- package/dist/daemon-entry.js +2 -1
- package/dist/daemon-entry.js.map +1 -1
- package/dist/daemon.d.ts +78 -3
- package/dist/daemon.js +496 -27
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-context.d.ts +7 -1
- package/dist/fleet-manager.d.ts +23 -2
- package/dist/fleet-manager.js +207 -65
- package/dist/fleet-manager.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +6 -1
- package/dist/instance-lifecycle.js +45 -3
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/logger.js +6 -3
- package/dist/logger.js.map +1 -1
- package/dist/outbound-handlers.d.ts +5 -0
- package/dist/outbound-handlers.js +68 -16
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/statusline-watcher.d.ts +1 -1
- package/dist/statusline-watcher.js +3 -2
- package/dist/statusline-watcher.js.map +1 -1
- package/dist/tmux-manager.d.ts +2 -0
- package/dist/tmux-manager.js +9 -0
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-archiver.d.ts +1 -1
- package/dist/topic-commands.d.ts +5 -1
- package/dist/topic-commands.js +72 -12
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-monitor.js +2 -0
- package/dist/transcript-monitor.js.map +1 -1
- package/dist/types.d.ts +2 -0
- package/dist/view-api.d.ts +1 -1
- package/dist/web-api.d.ts +2 -1
- package/dist/web-api.js +21 -14
- package/dist/web-api.js.map +1 -1
- package/package.json +1 -1
package/dist/daemon.js
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { join, dirname, basename, resolve } from "node:path";
|
|
2
2
|
import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync, chmodSync } from "node:fs";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
|
-
import { randomBytes } from "node:crypto";
|
|
4
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
5
5
|
import { EventEmitter } from "node:events";
|
|
6
|
-
import { createLogger } from "./logger.js";
|
|
7
6
|
import { TmuxManager } from "./tmux-manager.js";
|
|
8
7
|
import { TranscriptMonitor } from "./transcript-monitor.js";
|
|
9
8
|
import { ContextGuardian } from "./context-guardian.js";
|
|
@@ -21,6 +20,149 @@ const CROSS_INSTANCE_TOOLS = new Set(["send_to_instance", "list_instances", "sta
|
|
|
21
20
|
const SCHEDULE_TOOLS = new Set(["create_schedule", "list_schedules", "update_schedule", "delete_schedule"]);
|
|
22
21
|
const DECISION_TOOLS = new Set(["post_decision", "list_decisions", "update_decision"]);
|
|
23
22
|
const TASK_TOOL = "task";
|
|
23
|
+
export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
|
|
24
|
+
export const DEFAULT_STATE_POLL_INTERVAL_MS = 5_000;
|
|
25
|
+
/** Headless idle timer used by the daemon and unit tests. */
|
|
26
|
+
export class AutoPauseController {
|
|
27
|
+
thresholdMs;
|
|
28
|
+
idleSince = null;
|
|
29
|
+
pausedAt = null;
|
|
30
|
+
constructor(thresholdMs) {
|
|
31
|
+
this.thresholdMs = thresholdMs;
|
|
32
|
+
}
|
|
33
|
+
observe(state, now = Date.now()) {
|
|
34
|
+
if (this.pausedAt !== null || this.thresholdMs <= 0)
|
|
35
|
+
return false;
|
|
36
|
+
if (state !== "idle") {
|
|
37
|
+
this.idleSince = null;
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
this.idleSince ??= now;
|
|
41
|
+
return now - this.idleSince >= this.thresholdMs;
|
|
42
|
+
}
|
|
43
|
+
markPaused(now = Date.now()) {
|
|
44
|
+
this.pausedAt = now;
|
|
45
|
+
}
|
|
46
|
+
markAwake() {
|
|
47
|
+
this.pausedAt = null;
|
|
48
|
+
this.idleSince = null;
|
|
49
|
+
}
|
|
50
|
+
async wakeOnDeliver(wake) {
|
|
51
|
+
if (this.pausedAt === null)
|
|
52
|
+
return;
|
|
53
|
+
await wake();
|
|
54
|
+
this.markAwake();
|
|
55
|
+
}
|
|
56
|
+
get isPaused() { return this.pausedAt !== null; }
|
|
57
|
+
get lastPausedAt() { return this.pausedAt; }
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Headless state machine for pane-based execution state detection.
|
|
61
|
+
*
|
|
62
|
+
* Pane motion wins over a ready match because several backends keep their ready
|
|
63
|
+
* marker in a persistent header/footer while generating. A stable ready pane is
|
|
64
|
+
* idle; changing content is working; stable non-ready content eventually sticks.
|
|
65
|
+
*/
|
|
66
|
+
export class PaneStateMachine {
|
|
67
|
+
stuckTimeoutMs;
|
|
68
|
+
readyPattern;
|
|
69
|
+
lastPaneHash = null;
|
|
70
|
+
lastPaneChangeAt;
|
|
71
|
+
lastObservedAt;
|
|
72
|
+
stateChangedAt;
|
|
73
|
+
currentState = "idle";
|
|
74
|
+
constructor(readyPattern, stuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS, now = Date.now()) {
|
|
75
|
+
this.stuckTimeoutMs = stuckTimeoutMs;
|
|
76
|
+
// Stateful g/y regexes mutate lastIndex and can alternate true/false across
|
|
77
|
+
// polls. State detection must be deterministic for identical pane content.
|
|
78
|
+
this.readyPattern = new RegExp(readyPattern.source, readyPattern.flags.replace(/[gy]/g, ""));
|
|
79
|
+
this.lastPaneChangeAt = now;
|
|
80
|
+
this.lastObservedAt = now;
|
|
81
|
+
this.stateChangedAt = now;
|
|
82
|
+
}
|
|
83
|
+
observe(pane, now = Date.now()) {
|
|
84
|
+
const paneHash = createHash("sha256").update(pane).digest("hex");
|
|
85
|
+
const firstObservation = this.lastPaneHash === null;
|
|
86
|
+
const paneChanged = this.lastPaneHash !== paneHash;
|
|
87
|
+
if (paneChanged) {
|
|
88
|
+
this.lastPaneHash = paneHash;
|
|
89
|
+
this.lastPaneChangeAt = now;
|
|
90
|
+
}
|
|
91
|
+
this.lastObservedAt = now;
|
|
92
|
+
const ready = this.readyPattern.test(pane);
|
|
93
|
+
const nextState = firstObservation
|
|
94
|
+
? ready ? "idle" : "working"
|
|
95
|
+
: paneChanged
|
|
96
|
+
? "working"
|
|
97
|
+
: ready
|
|
98
|
+
? "idle"
|
|
99
|
+
: now - this.lastPaneChangeAt >= this.stuckTimeoutMs
|
|
100
|
+
? "stuck"
|
|
101
|
+
: "working";
|
|
102
|
+
if (nextState !== this.currentState) {
|
|
103
|
+
this.currentState = nextState;
|
|
104
|
+
this.stateChangedAt = now;
|
|
105
|
+
}
|
|
106
|
+
return this.snapshot(now);
|
|
107
|
+
}
|
|
108
|
+
snapshot(now = Date.now()) {
|
|
109
|
+
return {
|
|
110
|
+
state: this.currentState,
|
|
111
|
+
unchangedForMs: Math.max(0, now - this.lastPaneChangeAt),
|
|
112
|
+
observedAt: this.lastObservedAt,
|
|
113
|
+
stateChangedAt: this.stateChangedAt,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
/** Tracks whether an inbound arrived after the most recent confirmed idle prompt. */
|
|
118
|
+
export class PendingWorkTracker {
|
|
119
|
+
lastInboundAt = 0;
|
|
120
|
+
lastIdleAt;
|
|
121
|
+
sequence = 0;
|
|
122
|
+
lastInboundOrder = 0;
|
|
123
|
+
lastIdleOrder = 0;
|
|
124
|
+
constructor(now = Date.now()) {
|
|
125
|
+
this.lastIdleAt = now;
|
|
126
|
+
}
|
|
127
|
+
recordInbound(now = Date.now()) {
|
|
128
|
+
this.lastInboundAt = now;
|
|
129
|
+
this.lastInboundOrder = ++this.sequence;
|
|
130
|
+
}
|
|
131
|
+
recordIdle(now = Date.now()) {
|
|
132
|
+
// An async pane poll can finish after a newer inbound. Do not let its stale
|
|
133
|
+
// observation clear work which had not arrived when the pane was captured.
|
|
134
|
+
if (now < this.lastInboundAt)
|
|
135
|
+
return;
|
|
136
|
+
this.lastIdleAt = now;
|
|
137
|
+
this.lastIdleOrder = ++this.sequence;
|
|
138
|
+
}
|
|
139
|
+
hasPendingWork() {
|
|
140
|
+
return this.lastInboundOrder > this.lastIdleOrder;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
/** Redact likely credentials and control sequences before pane text reaches logs. */
|
|
144
|
+
export function sanitizePaneTail(pane, lineCount = 5) {
|
|
145
|
+
const secretAssignment = /\b(token|secret|password|passwd|api[_-]?key|authorization)\b\s*[:=]\s*\S+/gi;
|
|
146
|
+
const bearer = /\bBearer\s+\S+/gi;
|
|
147
|
+
const knownToken = /\b(?:sk-[A-Za-z0-9_-]+|ghp_[A-Za-z0-9]+|github_pat_[A-Za-z0-9_]+|AKIA[A-Z0-9]{16})\b/g;
|
|
148
|
+
const jwt = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g;
|
|
149
|
+
const opaqueSecret = /\b[A-Za-z0-9_+/=-]{32,}\b/g;
|
|
150
|
+
const lines = pane
|
|
151
|
+
.replace(/\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g, "")
|
|
152
|
+
.split(/\r?\n/);
|
|
153
|
+
while (lines.length > 0 && /^\s*$/.test(lines[lines.length - 1]))
|
|
154
|
+
lines.pop();
|
|
155
|
+
return lines
|
|
156
|
+
.slice(-lineCount)
|
|
157
|
+
.map(line => line
|
|
158
|
+
.replace(bearer, "Bearer [REDACTED]")
|
|
159
|
+
.replace(secretAssignment, "$1=[REDACTED]")
|
|
160
|
+
.replace(knownToken, "[REDACTED]")
|
|
161
|
+
.replace(jwt, "[REDACTED]")
|
|
162
|
+
.replace(opaqueSecret, "[REDACTED]")
|
|
163
|
+
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "")
|
|
164
|
+
.slice(0, 200));
|
|
165
|
+
}
|
|
24
166
|
export class Daemon extends EventEmitter {
|
|
25
167
|
name;
|
|
26
168
|
config;
|
|
@@ -68,6 +210,15 @@ export class Daemon extends EventEmitter {
|
|
|
68
210
|
rotationStartedAt = 0;
|
|
69
211
|
preRotationContextPct = 0;
|
|
70
212
|
hangDetector = null;
|
|
213
|
+
instanceState = "idle";
|
|
214
|
+
instanceStateMachine = null;
|
|
215
|
+
pendingWork = new PendingWorkTracker();
|
|
216
|
+
instanceStateMonitorTimer = null;
|
|
217
|
+
statePollInFlight = false;
|
|
218
|
+
autoPauseController;
|
|
219
|
+
pauseRequested = false;
|
|
220
|
+
pauseWakeState = "active";
|
|
221
|
+
pauseWakeTransition = null;
|
|
71
222
|
// Model failover: override model on next spawn when rate-limited
|
|
72
223
|
modelOverride;
|
|
73
224
|
// Context rotation v3: ring buffers for daemon-side snapshot
|
|
@@ -87,12 +238,17 @@ export class Daemon extends EventEmitter {
|
|
|
87
238
|
pasteQueueDepth = 0;
|
|
88
239
|
// PTY error pattern monitoring
|
|
89
240
|
errorMonitorTimer = null;
|
|
241
|
+
/** Prevent in-flight monitor callbacks from re-arming after a pause. */
|
|
242
|
+
runtimeMonitorsFrozen = false;
|
|
90
243
|
errorWaitingForRecovery = false; // true = error detected, waiting for ready pattern
|
|
91
244
|
errorDetectedAt = 0;
|
|
92
|
-
/** Whether this instance is in an error state (
|
|
245
|
+
/** Whether this instance is in an abnormal error state (auto-pause is normal). */
|
|
93
246
|
get isErrorState() {
|
|
94
|
-
return this.errorWaitingForRecovery || this.healthCheckPaused || Daemon.tmuxServerPaused;
|
|
247
|
+
return this.errorWaitingForRecovery || (this.healthCheckPaused && !this.isPaused) || Daemon.tmuxServerPaused;
|
|
95
248
|
}
|
|
249
|
+
get isPaused() { return this.pauseWakeState !== "active"; }
|
|
250
|
+
get lastPausedAt() { return this.autoPauseController.lastPausedAt; }
|
|
251
|
+
getPauseWakeState() { return this.pauseWakeState; }
|
|
96
252
|
/** Whether this instance is in a crash loop (3+ consecutive crashes). */
|
|
97
253
|
get isCrashLoop() {
|
|
98
254
|
return this.crashCount >= 3;
|
|
@@ -110,7 +266,7 @@ export class Daemon extends EventEmitter {
|
|
|
110
266
|
// still registers as new (prevents the old hash-dedup's permanent suppression).
|
|
111
267
|
lastErrorCount = new Map();
|
|
112
268
|
lastDetectedErrorType = null;
|
|
113
|
-
constructor(name, config, instanceDir, topicMode = false, backend, controlClient) {
|
|
269
|
+
constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger) {
|
|
114
270
|
super();
|
|
115
271
|
this.name = name;
|
|
116
272
|
this.config = config;
|
|
@@ -118,13 +274,23 @@ export class Daemon extends EventEmitter {
|
|
|
118
274
|
this.topicMode = topicMode;
|
|
119
275
|
this.backend = backend;
|
|
120
276
|
this.controlClient = controlClient;
|
|
121
|
-
|
|
277
|
+
if (!rootLogger)
|
|
278
|
+
throw new Error("Daemon requires a shared root logger");
|
|
279
|
+
this.logger = rootLogger.child({ instance: name }, { level: config.log_level });
|
|
122
280
|
this.tmuxSessionName = getTmuxSession();
|
|
123
281
|
this.messageBus = new MessageBus();
|
|
124
282
|
this.messageBus.setLogger(this.logger);
|
|
283
|
+
const autoPauseMinutes = typeof config.auto_pause_after === "number" ? config.auto_pause_after : 0; // default: disabled
|
|
284
|
+
this.autoPauseController = new AutoPauseController(Math.max(0, autoPauseMinutes) * 60_000);
|
|
125
285
|
}
|
|
126
286
|
async start() {
|
|
127
287
|
mkdirSync(this.instanceDir, { recursive: true });
|
|
288
|
+
// A daemon restart performs a normal CLI start, so any persisted auto-pause
|
|
289
|
+
// marker from the previous daemon is stale.
|
|
290
|
+
try {
|
|
291
|
+
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
292
|
+
}
|
|
293
|
+
catch { }
|
|
128
294
|
writeFileSync(join(this.instanceDir, "daemon.pid"), String(process.pid));
|
|
129
295
|
this.logger.info(`Starting ${this.name}`);
|
|
130
296
|
// P1: Read crash state from previous run — skip resume if last run was a crash loop
|
|
@@ -229,13 +395,18 @@ export class Daemon extends EventEmitter {
|
|
|
229
395
|
// Fleet manager routed a message to us (topic mode)
|
|
230
396
|
const meta = msg.meta;
|
|
231
397
|
const targetSession = msg.targetSession;
|
|
232
|
-
this.
|
|
398
|
+
void this.wake().then(() => {
|
|
399
|
+
this.pushChannelMessage(msg.content, meta, targetSession);
|
|
400
|
+
}).catch(err => {
|
|
401
|
+
this.logger.error({ err: err.message }, "Wake failed for inbound delivery");
|
|
402
|
+
});
|
|
233
403
|
}
|
|
234
404
|
else if (msg.type === "raw_paste") {
|
|
235
405
|
// Paste raw text directly to CLI without [user:] wrapping.
|
|
236
406
|
if (this.tmux) {
|
|
237
407
|
const rawText = msg.content;
|
|
238
408
|
this.pasteLock = this.pasteLock.then(async () => {
|
|
409
|
+
await this.wake();
|
|
239
410
|
await this.deliverMessage(rawText);
|
|
240
411
|
this.logger.debug({ text: rawText.slice(0, 100) }, "Raw paste delivered");
|
|
241
412
|
}).catch(err => {
|
|
@@ -246,12 +417,25 @@ export class Daemon extends EventEmitter {
|
|
|
246
417
|
else if (msg.type === "fleet_schedule_trigger") {
|
|
247
418
|
const payload = msg.payload;
|
|
248
419
|
const meta = msg.meta;
|
|
249
|
-
this.pushChannelMessage(payload.message, meta)
|
|
420
|
+
void this.wake().then(() => this.pushChannelMessage(payload.message, meta)).catch(err => {
|
|
421
|
+
this.logger.error({ err: err.message }, "Wake failed for scheduled delivery");
|
|
422
|
+
});
|
|
250
423
|
}
|
|
251
424
|
else if (msg.type === "fleet_tool_status_ack") {
|
|
252
425
|
// Fleet manager sent us the messageId for our tool status message
|
|
253
426
|
this.toolStatusMessageId = msg.messageId;
|
|
254
427
|
}
|
|
428
|
+
else if (msg.type === "query_instance_state") {
|
|
429
|
+
const snapshot = this.getInstanceStateSnapshot();
|
|
430
|
+
this.ipcServer?.send(socket, {
|
|
431
|
+
type: "instance_state_response",
|
|
432
|
+
requestId: msg.requestId,
|
|
433
|
+
instanceName: this.name,
|
|
434
|
+
...snapshot,
|
|
435
|
+
state: this.isPaused ? "paused" : snapshot.state,
|
|
436
|
+
pausedAt: this.lastPausedAt,
|
|
437
|
+
});
|
|
438
|
+
}
|
|
255
439
|
});
|
|
256
440
|
// 2. Tmux — ensure session, create window if not alive
|
|
257
441
|
await TmuxManager.ensureSession(this.tmuxSessionName);
|
|
@@ -356,9 +540,13 @@ export class Daemon extends EventEmitter {
|
|
|
356
540
|
this.recordRecentEvent({ type: "assistant_text", preview: text.slice(0, 100) });
|
|
357
541
|
});
|
|
358
542
|
this.transcriptMonitor.startPolling();
|
|
359
|
-
//
|
|
360
|
-
|
|
361
|
-
|
|
543
|
+
// HangDetector remains the fleet-manager notification event bridge. Its
|
|
544
|
+
// legacy silence timer is intentionally not started: pane state transitions
|
|
545
|
+
// below are now the sole source of hang events.
|
|
546
|
+
const hangConfig = this.config.hang_detector;
|
|
547
|
+
if (hangConfig?.enabled !== false) {
|
|
548
|
+
this.hangDetector = new HangDetector(hangConfig?.timeout_minutes ?? 10);
|
|
549
|
+
}
|
|
362
550
|
// 8. Context guardian
|
|
363
551
|
const statusFile = join(this.instanceDir, "statusline.json");
|
|
364
552
|
this.guardian = new ContextGuardian(this.config.context_guardian, this.logger, statusFile);
|
|
@@ -382,14 +570,22 @@ export class Daemon extends EventEmitter {
|
|
|
382
570
|
if (!this.config.lightweight) {
|
|
383
571
|
this.startErrorMonitor();
|
|
384
572
|
}
|
|
573
|
+
this.startInstanceStateMonitor();
|
|
385
574
|
this.logger.info(`${this.name} ready`);
|
|
386
575
|
}
|
|
387
576
|
startHealthCheck() {
|
|
577
|
+
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
578
|
+
return;
|
|
388
579
|
const { max_retries, backoff, reset_after } = this.config.restart_policy;
|
|
389
580
|
if (max_retries <= 0)
|
|
390
581
|
return; // restart disabled
|
|
391
582
|
const scheduleNext = () => {
|
|
583
|
+
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
584
|
+
return;
|
|
392
585
|
this.healthCheckTimer = setTimeout(async () => {
|
|
586
|
+
this.healthCheckTimer = null;
|
|
587
|
+
if (this.runtimeMonitorsFrozen)
|
|
588
|
+
return;
|
|
393
589
|
// Instance directory removed externally (e.g. `rm -rf ~/.agend/instances/<name>`).
|
|
394
590
|
// Stop the loop permanently — otherwise every tick triggers a respawn, whose
|
|
395
591
|
// writeRotationSnapshot fails with ENOENT and gets caught as "Failed to respawn",
|
|
@@ -407,6 +603,12 @@ export class Daemon extends EventEmitter {
|
|
|
407
603
|
// Human-readable backend label for logs (e.g. "claude", "kiro-cli")
|
|
408
604
|
const cliLabel = this.backend?.binaryName ?? "CLI";
|
|
409
605
|
let paneStatus = await this.tmux.getPaneStatus();
|
|
606
|
+
// Auto-pause intentionally exits the pane process. A health tick that
|
|
607
|
+
// began just before pause must not classify that exit as a crash.
|
|
608
|
+
if (this.isPaused || this.pauseWakeState === "waking") {
|
|
609
|
+
scheduleNext();
|
|
610
|
+
return;
|
|
611
|
+
}
|
|
410
612
|
if (paneStatus?.alive) {
|
|
411
613
|
scheduleNext();
|
|
412
614
|
return;
|
|
@@ -634,6 +836,8 @@ export class Daemon extends EventEmitter {
|
|
|
634
836
|
* (ready pattern visible), it goes back to monitoring for new errors.
|
|
635
837
|
*/
|
|
636
838
|
startErrorMonitor() {
|
|
839
|
+
if (this.runtimeMonitorsFrozen || this.errorMonitorTimer)
|
|
840
|
+
return;
|
|
637
841
|
const patterns = this.backend?.getErrorPatterns?.() ?? [];
|
|
638
842
|
const dialogs = this.backend?.getRuntimeDialogs?.() ?? [];
|
|
639
843
|
if (!patterns.length && !dialogs.length)
|
|
@@ -753,22 +957,13 @@ export class Daemon extends EventEmitter {
|
|
|
753
957
|
}
|
|
754
958
|
async stop() {
|
|
755
959
|
this.logger.info("Stopping daemon instance");
|
|
756
|
-
|
|
757
|
-
clearTimeout(this.healthCheckTimer);
|
|
758
|
-
this.healthCheckTimer = null;
|
|
759
|
-
}
|
|
760
|
-
if (this.errorMonitorTimer) {
|
|
761
|
-
clearInterval(this.errorMonitorTimer);
|
|
762
|
-
this.errorMonitorTimer = null;
|
|
763
|
-
}
|
|
960
|
+
this.freezeRuntimeMonitors();
|
|
764
961
|
if (this.toolStatusDebounce) {
|
|
765
962
|
clearTimeout(this.toolStatusDebounce);
|
|
766
963
|
this.toolStatusDebounce = null;
|
|
767
964
|
}
|
|
768
965
|
this.pendingIpcRequests.clear();
|
|
769
966
|
this.hangDetector?.stop();
|
|
770
|
-
this.transcriptMonitor?.stop();
|
|
771
|
-
this.guardian?.stop();
|
|
772
967
|
if (this.adapter)
|
|
773
968
|
await this.adapter.stop();
|
|
774
969
|
// Notify MCP servers of graceful shutdown (prevents reconnect attempts)
|
|
@@ -827,10 +1022,270 @@ export class Daemon extends EventEmitter {
|
|
|
827
1022
|
catch (e) {
|
|
828
1023
|
this.logger.debug({ err: e }, "Failed to remove PID file");
|
|
829
1024
|
}
|
|
1025
|
+
try {
|
|
1026
|
+
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
1027
|
+
}
|
|
1028
|
+
catch { }
|
|
830
1029
|
}
|
|
831
1030
|
getHangDetector() {
|
|
832
1031
|
return this.hangDetector;
|
|
833
1032
|
}
|
|
1033
|
+
getInstanceState() {
|
|
1034
|
+
return this.isPaused ? "paused" : this.instanceState;
|
|
1035
|
+
}
|
|
1036
|
+
getInstanceStateSnapshot() {
|
|
1037
|
+
return this.instanceStateMachine?.snapshot() ?? {
|
|
1038
|
+
state: this.instanceState,
|
|
1039
|
+
unchangedForMs: 0,
|
|
1040
|
+
observedAt: Date.now(),
|
|
1041
|
+
stateChangedAt: Date.now(),
|
|
1042
|
+
};
|
|
1043
|
+
}
|
|
1044
|
+
/** Gracefully stop the CLI while keeping its remain-on-exit tmux window. */
|
|
1045
|
+
async pause() {
|
|
1046
|
+
if (this.pauseWakeState === "paused")
|
|
1047
|
+
return;
|
|
1048
|
+
if (this.pauseWakeState === "pausing" || this.pauseWakeState === "waking") {
|
|
1049
|
+
await this.pauseWakeTransition;
|
|
1050
|
+
if (this.getPauseWakeState() !== "active")
|
|
1051
|
+
return;
|
|
1052
|
+
}
|
|
1053
|
+
if (this.instanceState !== "idle" || this.pasteQueueDepth > 0) {
|
|
1054
|
+
this.pauseRequested = false;
|
|
1055
|
+
return;
|
|
1056
|
+
}
|
|
1057
|
+
this.pauseWakeState = "pausing";
|
|
1058
|
+
this.healthCheckPaused = true;
|
|
1059
|
+
this.freezeRuntimeMonitors();
|
|
1060
|
+
const transition = (async () => {
|
|
1061
|
+
try {
|
|
1062
|
+
this.saveSessionId();
|
|
1063
|
+
const quitCmd = this.backend?.getQuitCommand();
|
|
1064
|
+
if (quitCmd && this.tmux) {
|
|
1065
|
+
await this.tmux.sendKeys(quitCmd);
|
|
1066
|
+
await new Promise(r => setTimeout(r, 150));
|
|
1067
|
+
await this.tmux.sendSpecialKey("Enter");
|
|
1068
|
+
}
|
|
1069
|
+
let exited = false;
|
|
1070
|
+
for (let i = 0; i < 15; i++) {
|
|
1071
|
+
await new Promise(r => setTimeout(r, 200));
|
|
1072
|
+
const status = await this.tmux?.getPaneStatus();
|
|
1073
|
+
if (status && !status.alive) {
|
|
1074
|
+
exited = true;
|
|
1075
|
+
break;
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
if (!exited) {
|
|
1079
|
+
await this.killProcessTree("SIGTERM");
|
|
1080
|
+
await new Promise(r => setTimeout(r, 1_000));
|
|
1081
|
+
const status = await this.tmux?.getPaneStatus();
|
|
1082
|
+
if (status?.alive) {
|
|
1083
|
+
await this.killProcessTree("SIGKILL");
|
|
1084
|
+
await new Promise(r => setTimeout(r, 200));
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
const finalStatus = await this.tmux?.getPaneStatus();
|
|
1088
|
+
if (!finalStatus || finalStatus.alive) {
|
|
1089
|
+
throw new Error("Auto-pause could not stop the CLI while preserving its tmux window");
|
|
1090
|
+
}
|
|
1091
|
+
this.pauseWakeState = "paused";
|
|
1092
|
+
this.autoPauseController.markPaused();
|
|
1093
|
+
writeFileSync(join(this.instanceDir, "paused-state.json"), JSON.stringify({
|
|
1094
|
+
paused_at: this.lastPausedAt,
|
|
1095
|
+
}));
|
|
1096
|
+
this.logger.info({ pausedAt: this.lastPausedAt }, "Instance auto-paused");
|
|
1097
|
+
this.ipcServer?.broadcast({
|
|
1098
|
+
type: "instance_state", instanceName: this.name, state: "paused", pausedAt: this.lastPausedAt,
|
|
1099
|
+
});
|
|
1100
|
+
this.emit("auto_paused", { name: this.name, pausedAt: this.lastPausedAt });
|
|
1101
|
+
}
|
|
1102
|
+
catch (err) {
|
|
1103
|
+
this.pauseWakeState = "active";
|
|
1104
|
+
this.healthCheckPaused = false;
|
|
1105
|
+
this.pauseRequested = false;
|
|
1106
|
+
this.resumeRuntimeMonitors();
|
|
1107
|
+
throw err;
|
|
1108
|
+
}
|
|
1109
|
+
})();
|
|
1110
|
+
this.pauseWakeTransition = transition;
|
|
1111
|
+
try {
|
|
1112
|
+
await transition;
|
|
1113
|
+
}
|
|
1114
|
+
finally {
|
|
1115
|
+
if (this.pauseWakeTransition === transition)
|
|
1116
|
+
this.pauseWakeTransition = null;
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
/** Respawn the CLI in the preserved window and block until its prompt is ready. */
|
|
1120
|
+
async wake(timeoutMs = 30_000) {
|
|
1121
|
+
if (this.pauseWakeState === "active")
|
|
1122
|
+
return;
|
|
1123
|
+
if (this.pauseWakeState === "pausing")
|
|
1124
|
+
await this.pauseWakeTransition;
|
|
1125
|
+
if (this.getPauseWakeState() === "active")
|
|
1126
|
+
return;
|
|
1127
|
+
if (this.pauseWakeState === "waking") {
|
|
1128
|
+
await this.pauseWakeTransition;
|
|
1129
|
+
return;
|
|
1130
|
+
}
|
|
1131
|
+
this.pauseWakeState = "waking";
|
|
1132
|
+
this.spawning = true;
|
|
1133
|
+
const transition = this.autoPauseController.wakeOnDeliver(async () => {
|
|
1134
|
+
let timeout;
|
|
1135
|
+
try {
|
|
1136
|
+
const ready = await Promise.race([
|
|
1137
|
+
this.trySpawn(true, timeoutMs),
|
|
1138
|
+
new Promise(resolve => { timeout = setTimeout(() => resolve(false), timeoutMs); }),
|
|
1139
|
+
]);
|
|
1140
|
+
if (!ready)
|
|
1141
|
+
throw new Error(`Wake timed out before CLI became ready (${timeoutMs}ms)`);
|
|
1142
|
+
}
|
|
1143
|
+
finally {
|
|
1144
|
+
if (timeout)
|
|
1145
|
+
clearTimeout(timeout);
|
|
1146
|
+
}
|
|
1147
|
+
});
|
|
1148
|
+
this.pauseWakeTransition = transition;
|
|
1149
|
+
try {
|
|
1150
|
+
await transition;
|
|
1151
|
+
this.pauseWakeState = "active";
|
|
1152
|
+
this.healthCheckPaused = false;
|
|
1153
|
+
this.pauseRequested = false;
|
|
1154
|
+
try {
|
|
1155
|
+
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
1156
|
+
}
|
|
1157
|
+
catch { }
|
|
1158
|
+
this.transcriptMonitor?.resetOffset();
|
|
1159
|
+
this.resumeRuntimeMonitors();
|
|
1160
|
+
this.logger.info("Instance auto-woke");
|
|
1161
|
+
this.ipcServer?.broadcast({
|
|
1162
|
+
type: "instance_state", instanceName: this.name, state: this.instanceState, pausedAt: null,
|
|
1163
|
+
});
|
|
1164
|
+
this.emit("auto_woke", { name: this.name });
|
|
1165
|
+
}
|
|
1166
|
+
catch (err) {
|
|
1167
|
+
this.pauseWakeState = "paused";
|
|
1168
|
+
this.healthCheckPaused = true;
|
|
1169
|
+
this.logger.error({ err: err.message }, "Instance wake failed");
|
|
1170
|
+
throw err;
|
|
1171
|
+
}
|
|
1172
|
+
finally {
|
|
1173
|
+
this.spawning = false;
|
|
1174
|
+
if (this.pauseWakeTransition === transition)
|
|
1175
|
+
this.pauseWakeTransition = null;
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
startInstanceStateMonitor() {
|
|
1179
|
+
if (this.runtimeMonitorsFrozen || !this.tmux || !this.backend || this.instanceStateMonitorTimer)
|
|
1180
|
+
return;
|
|
1181
|
+
const rawConfig = this.config.hang_detector;
|
|
1182
|
+
const timeoutMinutes = rawConfig?.timeout_minutes;
|
|
1183
|
+
const stuckTimeoutMs = typeof timeoutMinutes === "number" && timeoutMinutes > 0
|
|
1184
|
+
? timeoutMinutes * 60_000
|
|
1185
|
+
: DEFAULT_STUCK_TIMEOUT_MS;
|
|
1186
|
+
const pollIntervalMs = typeof rawConfig?.poll_interval_ms === "number" && rawConfig.poll_interval_ms > 0
|
|
1187
|
+
? rawConfig.poll_interval_ms
|
|
1188
|
+
: DEFAULT_STATE_POLL_INTERVAL_MS;
|
|
1189
|
+
const readyPattern = this.backend.getReadyPattern();
|
|
1190
|
+
this.instanceStateMachine = new PaneStateMachine(readyPattern, stuckTimeoutMs);
|
|
1191
|
+
const poll = async () => {
|
|
1192
|
+
if (!this.tmux || this.spawning || this.statePollInFlight)
|
|
1193
|
+
return;
|
|
1194
|
+
this.statePollInFlight = true;
|
|
1195
|
+
try {
|
|
1196
|
+
const paneStatus = await this.tmux.getPaneStatus();
|
|
1197
|
+
if (!paneStatus?.alive)
|
|
1198
|
+
return;
|
|
1199
|
+
const pane = await this.tmux.capturePane();
|
|
1200
|
+
const previous = this.instanceState;
|
|
1201
|
+
const snapshot = this.instanceStateMachine.observe(pane);
|
|
1202
|
+
this.instanceState = snapshot.state;
|
|
1203
|
+
// Only a transition back to idle completes pending work. Repeated idle
|
|
1204
|
+
// polls between enqueue and paste must not clear a newly-recorded inbound.
|
|
1205
|
+
if (snapshot.state === "idle" && previous !== "idle") {
|
|
1206
|
+
this.pendingWork.recordIdle(snapshot.observedAt);
|
|
1207
|
+
}
|
|
1208
|
+
if (snapshot.state !== previous) {
|
|
1209
|
+
this.logger.info({
|
|
1210
|
+
previousState: previous,
|
|
1211
|
+
state: snapshot.state,
|
|
1212
|
+
unchangedForMs: snapshot.unchangedForMs,
|
|
1213
|
+
}, "Instance execution state changed");
|
|
1214
|
+
this.emit("instance_state", { name: this.name, ...snapshot });
|
|
1215
|
+
this.ipcServer?.broadcast({ type: "instance_state", instanceName: this.name, ...snapshot });
|
|
1216
|
+
// Emit exactly once per transition into stuck. Further notifications
|
|
1217
|
+
// require observable progress (working) or a ready prompt (idle) first.
|
|
1218
|
+
if (snapshot.state === "stuck") {
|
|
1219
|
+
this.handleStuckTransition(pane, snapshot, readyPattern);
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
if (snapshot.state !== "idle")
|
|
1223
|
+
this.pauseRequested = false;
|
|
1224
|
+
if (!this.pauseRequested && this.pasteQueueDepth === 0 && this.autoPauseController.observe(snapshot.state)) {
|
|
1225
|
+
this.pauseRequested = true;
|
|
1226
|
+
this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
|
|
1227
|
+
}
|
|
1228
|
+
}
|
|
1229
|
+
catch (err) {
|
|
1230
|
+
// A pane can disappear between status and capture during restart. Keep
|
|
1231
|
+
// the last known state and retry on the next poll.
|
|
1232
|
+
this.logger.debug({ err: err.message }, "Instance state poll failed");
|
|
1233
|
+
}
|
|
1234
|
+
finally {
|
|
1235
|
+
this.statePollInFlight = false;
|
|
1236
|
+
}
|
|
1237
|
+
};
|
|
1238
|
+
void poll();
|
|
1239
|
+
this.instanceStateMonitorTimer = setInterval(() => { void poll(); }, pollIntervalMs);
|
|
1240
|
+
}
|
|
1241
|
+
handleStuckTransition(pane, snapshot, readyPattern) {
|
|
1242
|
+
const deterministicReadyPattern = new RegExp(readyPattern.source, readyPattern.flags.replace(/[gy]/g, ""));
|
|
1243
|
+
const diagnostic = {
|
|
1244
|
+
backend: this.backend?.binaryName ?? this.config.backend ?? "unknown",
|
|
1245
|
+
paneTail: sanitizePaneTail(pane),
|
|
1246
|
+
readyPattern: readyPattern.toString(),
|
|
1247
|
+
readyMatched: deterministicReadyPattern.test(pane),
|
|
1248
|
+
unchangedForMs: snapshot.unchangedForMs,
|
|
1249
|
+
pendingWork: this.pendingWork.hasPendingWork(),
|
|
1250
|
+
};
|
|
1251
|
+
if (!diagnostic.pendingWork) {
|
|
1252
|
+
this.logger.debug(diagnostic, "Suppressing stuck notification without pending work");
|
|
1253
|
+
return;
|
|
1254
|
+
}
|
|
1255
|
+
this.logger.warn(diagnostic, "Instance pane stuck with pending work");
|
|
1256
|
+
this.hangDetector?.emit("hang", { unchangedForMs: snapshot.unchangedForMs });
|
|
1257
|
+
}
|
|
1258
|
+
/** Stop every runtime poller/watcher while preserving IPC and daemon state. */
|
|
1259
|
+
freezeRuntimeMonitors() {
|
|
1260
|
+
this.runtimeMonitorsFrozen = true;
|
|
1261
|
+
if (this.healthCheckTimer) {
|
|
1262
|
+
clearTimeout(this.healthCheckTimer);
|
|
1263
|
+
this.healthCheckTimer = null;
|
|
1264
|
+
}
|
|
1265
|
+
if (this.errorMonitorTimer) {
|
|
1266
|
+
clearInterval(this.errorMonitorTimer);
|
|
1267
|
+
this.errorMonitorTimer = null;
|
|
1268
|
+
}
|
|
1269
|
+
if (this.instanceStateMonitorTimer) {
|
|
1270
|
+
clearInterval(this.instanceStateMonitorTimer);
|
|
1271
|
+
this.instanceStateMonitorTimer = null;
|
|
1272
|
+
}
|
|
1273
|
+
this.transcriptMonitor?.stop();
|
|
1274
|
+
this.guardian?.stop();
|
|
1275
|
+
}
|
|
1276
|
+
/** Restore the same monitor objects after wake without adding event listeners. */
|
|
1277
|
+
resumeRuntimeMonitors() {
|
|
1278
|
+
if (!this.runtimeMonitorsFrozen)
|
|
1279
|
+
return;
|
|
1280
|
+
this.runtimeMonitorsFrozen = false;
|
|
1281
|
+
this.startHealthCheck();
|
|
1282
|
+
if (!this.config.lightweight) {
|
|
1283
|
+
this.transcriptMonitor?.startPolling();
|
|
1284
|
+
this.guardian?.startWatching();
|
|
1285
|
+
this.startErrorMonitor();
|
|
1286
|
+
}
|
|
1287
|
+
this.startInstanceStateMonitor();
|
|
1288
|
+
}
|
|
834
1289
|
getMessageBus() {
|
|
835
1290
|
return this.messageBus;
|
|
836
1291
|
}
|
|
@@ -915,6 +1370,7 @@ export class Daemon extends EventEmitter {
|
|
|
915
1370
|
this.pendingInstructionsUpdate = undefined;
|
|
916
1371
|
}
|
|
917
1372
|
this.hangDetector?.recordInbound();
|
|
1373
|
+
this.pendingWork.recordInbound();
|
|
918
1374
|
// v3: record user messages for rotation snapshot
|
|
919
1375
|
this.recordRecentUserMessage(content, meta);
|
|
920
1376
|
// Format message with metadata prefix for the agent
|
|
@@ -1488,14 +1944,14 @@ export class Daemon extends EventEmitter {
|
|
|
1488
1944
|
return resumedSuccessfully;
|
|
1489
1945
|
}
|
|
1490
1946
|
/** Kill the entire process tree of the current tmux pane (CLI + MCP server). */
|
|
1491
|
-
async killProcessTree() {
|
|
1947
|
+
async killProcessTree(signal = "SIGTERM") {
|
|
1492
1948
|
if (!this.tmux)
|
|
1493
1949
|
return;
|
|
1494
1950
|
try {
|
|
1495
1951
|
const pid = await TmuxManager.getPanePid(this.tmuxSessionName, this.tmux.getWindowId());
|
|
1496
1952
|
if (pid) {
|
|
1497
|
-
process.kill(-pid,
|
|
1498
|
-
this.logger.debug({ pid }, "Killed process group");
|
|
1953
|
+
process.kill(-pid, signal);
|
|
1954
|
+
this.logger.debug({ pid, signal }, "Killed process group");
|
|
1499
1955
|
}
|
|
1500
1956
|
}
|
|
1501
1957
|
catch { /* process group may not exist or already dead */ }
|
|
@@ -1506,7 +1962,7 @@ export class Daemon extends EventEmitter {
|
|
|
1506
1962
|
* Handles confirmation dialogs (trust folder, bypass permissions).
|
|
1507
1963
|
* Returns true if CLI is ready, false if it failed or got stuck.
|
|
1508
1964
|
*/
|
|
1509
|
-
async trySpawn() {
|
|
1965
|
+
async trySpawn(reuseWindow = false, startupTimeoutMs) {
|
|
1510
1966
|
const backendConfig = this.buildBackendConfig();
|
|
1511
1967
|
// Compare freshly-built instructions against the last value the agent was
|
|
1512
1968
|
// told about. Computed for ALL backends (not gated by
|
|
@@ -1569,16 +2025,29 @@ export class Daemon extends EventEmitter {
|
|
|
1569
2025
|
const cmd = `${envPrefix} ` + this.backend.buildCommand(backendConfig);
|
|
1570
2026
|
// Ensure tmux session exists (may have been destroyed if all windows died)
|
|
1571
2027
|
await TmuxManager.ensureSession(this.tmuxSessionName);
|
|
1572
|
-
|
|
2028
|
+
let windowId;
|
|
2029
|
+
if (reuseWindow) {
|
|
2030
|
+
this.controlClient?.unregisterWindow(this.tmux.getWindowId());
|
|
2031
|
+
await this.tmux.respawnWindow(cmd, resolvedCwd);
|
|
2032
|
+
windowId = this.tmux.getWindowId();
|
|
2033
|
+
}
|
|
2034
|
+
else {
|
|
2035
|
+
windowId = await this.tmux.createWindow(cmd, resolvedCwd, this.name);
|
|
2036
|
+
}
|
|
1573
2037
|
writeFileSync(join(this.instanceDir, "window-id"), windowId);
|
|
1574
2038
|
// Enable remain-on-exit to capture exit codes on crash
|
|
1575
2039
|
await this.tmux.setRemainOnExit().catch(err => {
|
|
1576
2040
|
this.logger.warn({ err }, "Failed to set remain-on-exit — exit codes will not be captured");
|
|
1577
2041
|
});
|
|
2042
|
+
if (reuseWindow && !this.config.lightweight) {
|
|
2043
|
+
await this.tmux.pipeOutput(join(this.instanceDir, "output.log")).catch(err => {
|
|
2044
|
+
this.logger.warn({ err }, "Failed to restore pipe-pane after wake");
|
|
2045
|
+
});
|
|
2046
|
+
}
|
|
1578
2047
|
// Register with control client and wait for output + idle
|
|
1579
2048
|
await this.controlClient?.registerWindow(windowId);
|
|
1580
2049
|
if (this.controlClient) {
|
|
1581
|
-
const total = this.config.startup_timeout_ms ?? 25_000;
|
|
2050
|
+
const total = startupTimeoutMs ?? this.config.startup_timeout_ms ?? 25_000;
|
|
1582
2051
|
const outputTimeout = Math.round(total * 0.6);
|
|
1583
2052
|
const idleTimeout = total - outputTimeout;
|
|
1584
2053
|
const hasOutput = await this.controlClient.waitForOutput(windowId, outputTimeout);
|