@songsid/agend 2.1.0-beta.1 → 2.1.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/antigravity.js +3 -1
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/codex.js +3 -1
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/kiro.js +3 -2
- package/dist/backend/kiro.js.map +1 -1
- package/dist/cli.js +43 -6
- package/dist/cli.js.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/context-guardian.d.ts +2 -0
- package/dist/context-guardian.js +10 -2
- package/dist/context-guardian.js.map +1 -1
- package/dist/daemon-entry.js +2 -1
- package/dist/daemon-entry.js.map +1 -1
- package/dist/daemon.d.ts +50 -3
- package/dist/daemon.js +356 -31
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-context.d.ts +2 -1
- package/dist/fleet-manager.d.ts +10 -2
- package/dist/fleet-manager.js +125 -56
- package/dist/fleet-manager.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +6 -1
- package/dist/instance-lifecycle.js +45 -3
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/logger.js +6 -3
- package/dist/logger.js.map +1 -1
- package/dist/outbound-handlers.d.ts +5 -0
- package/dist/outbound-handlers.js +68 -16
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/statusline-watcher.d.ts +1 -1
- package/dist/statusline-watcher.js +3 -2
- package/dist/statusline-watcher.js.map +1 -1
- package/dist/tmux-manager.d.ts +2 -0
- package/dist/tmux-manager.js +9 -0
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-archiver.d.ts +1 -1
- package/dist/topic-commands.d.ts +5 -1
- package/dist/topic-commands.js +60 -12
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-monitor.js +2 -0
- package/dist/transcript-monitor.js.map +1 -1
- package/dist/types.d.ts +2 -0
- package/dist/view-api.d.ts +1 -1
- package/dist/web-api.d.ts +2 -1
- package/dist/web-api.js +21 -14
- package/dist/web-api.js.map +1 -1
- package/package.json +1 -1
package/dist/daemon.js
CHANGED
|
@@ -3,7 +3,6 @@ import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync,
|
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import { createHash, randomBytes } from "node:crypto";
|
|
5
5
|
import { EventEmitter } from "node:events";
|
|
6
|
-
import { createLogger } from "./logger.js";
|
|
7
6
|
import { TmuxManager } from "./tmux-manager.js";
|
|
8
7
|
import { TranscriptMonitor } from "./transcript-monitor.js";
|
|
9
8
|
import { ContextGuardian } from "./context-guardian.js";
|
|
@@ -23,6 +22,40 @@ const DECISION_TOOLS = new Set(["post_decision", "list_decisions", "update_decis
|
|
|
23
22
|
const TASK_TOOL = "task";
|
|
24
23
|
export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
|
|
25
24
|
export const DEFAULT_STATE_POLL_INTERVAL_MS = 5_000;
|
|
25
|
+
/** Headless idle timer used by the daemon and unit tests. */
|
|
26
|
+
export class AutoPauseController {
|
|
27
|
+
thresholdMs;
|
|
28
|
+
idleSince = null;
|
|
29
|
+
pausedAt = null;
|
|
30
|
+
constructor(thresholdMs) {
|
|
31
|
+
this.thresholdMs = thresholdMs;
|
|
32
|
+
}
|
|
33
|
+
observe(state, now = Date.now()) {
|
|
34
|
+
if (this.pausedAt !== null || this.thresholdMs <= 0)
|
|
35
|
+
return false;
|
|
36
|
+
if (state !== "idle") {
|
|
37
|
+
this.idleSince = null;
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
this.idleSince ??= now;
|
|
41
|
+
return now - this.idleSince >= this.thresholdMs;
|
|
42
|
+
}
|
|
43
|
+
markPaused(now = Date.now()) {
|
|
44
|
+
this.pausedAt = now;
|
|
45
|
+
}
|
|
46
|
+
markAwake() {
|
|
47
|
+
this.pausedAt = null;
|
|
48
|
+
this.idleSince = null;
|
|
49
|
+
}
|
|
50
|
+
async wakeOnDeliver(wake) {
|
|
51
|
+
if (this.pausedAt === null)
|
|
52
|
+
return;
|
|
53
|
+
await wake();
|
|
54
|
+
this.markAwake();
|
|
55
|
+
}
|
|
56
|
+
get isPaused() { return this.pausedAt !== null; }
|
|
57
|
+
get lastPausedAt() { return this.pausedAt; }
|
|
58
|
+
}
|
|
26
59
|
/**
|
|
27
60
|
* Headless state machine for pane-based execution state detection.
|
|
28
61
|
*
|
|
@@ -81,6 +114,55 @@ export class PaneStateMachine {
|
|
|
81
114
|
};
|
|
82
115
|
}
|
|
83
116
|
}
|
|
117
|
+
/** Tracks whether an inbound arrived after the most recent confirmed idle prompt. */
|
|
118
|
+
export class PendingWorkTracker {
|
|
119
|
+
lastInboundAt = 0;
|
|
120
|
+
lastIdleAt;
|
|
121
|
+
sequence = 0;
|
|
122
|
+
lastInboundOrder = 0;
|
|
123
|
+
lastIdleOrder = 0;
|
|
124
|
+
constructor(now = Date.now()) {
|
|
125
|
+
this.lastIdleAt = now;
|
|
126
|
+
}
|
|
127
|
+
recordInbound(now = Date.now()) {
|
|
128
|
+
this.lastInboundAt = now;
|
|
129
|
+
this.lastInboundOrder = ++this.sequence;
|
|
130
|
+
}
|
|
131
|
+
recordIdle(now = Date.now()) {
|
|
132
|
+
// An async pane poll can finish after a newer inbound. Do not let its stale
|
|
133
|
+
// observation clear work which had not arrived when the pane was captured.
|
|
134
|
+
if (now < this.lastInboundAt)
|
|
135
|
+
return;
|
|
136
|
+
this.lastIdleAt = now;
|
|
137
|
+
this.lastIdleOrder = ++this.sequence;
|
|
138
|
+
}
|
|
139
|
+
hasPendingWork() {
|
|
140
|
+
return this.lastInboundOrder > this.lastIdleOrder;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
/** Redact likely credentials and control sequences before pane text reaches logs. */
|
|
144
|
+
export function sanitizePaneTail(pane, lineCount = 5) {
|
|
145
|
+
const secretAssignment = /\b(token|secret|password|passwd|api[_-]?key|authorization)\b\s*[:=]\s*\S+/gi;
|
|
146
|
+
const bearer = /\bBearer\s+\S+/gi;
|
|
147
|
+
const knownToken = /\b(?:sk-[A-Za-z0-9_-]+|ghp_[A-Za-z0-9]+|github_pat_[A-Za-z0-9_]+|AKIA[A-Z0-9]{16})\b/g;
|
|
148
|
+
const jwt = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g;
|
|
149
|
+
const opaqueSecret = /\b[A-Za-z0-9_+/=-]{32,}\b/g;
|
|
150
|
+
const lines = pane
|
|
151
|
+
.replace(/\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g, "")
|
|
152
|
+
.split(/\r?\n/);
|
|
153
|
+
while (lines.length > 0 && /^\s*$/.test(lines[lines.length - 1]))
|
|
154
|
+
lines.pop();
|
|
155
|
+
return lines
|
|
156
|
+
.slice(-lineCount)
|
|
157
|
+
.map(line => line
|
|
158
|
+
.replace(bearer, "Bearer [REDACTED]")
|
|
159
|
+
.replace(secretAssignment, "$1=[REDACTED]")
|
|
160
|
+
.replace(knownToken, "[REDACTED]")
|
|
161
|
+
.replace(jwt, "[REDACTED]")
|
|
162
|
+
.replace(opaqueSecret, "[REDACTED]")
|
|
163
|
+
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "")
|
|
164
|
+
.slice(0, 200));
|
|
165
|
+
}
|
|
84
166
|
export class Daemon extends EventEmitter {
|
|
85
167
|
name;
|
|
86
168
|
config;
|
|
@@ -130,8 +212,13 @@ export class Daemon extends EventEmitter {
|
|
|
130
212
|
hangDetector = null;
|
|
131
213
|
instanceState = "idle";
|
|
132
214
|
instanceStateMachine = null;
|
|
215
|
+
pendingWork = new PendingWorkTracker();
|
|
133
216
|
instanceStateMonitorTimer = null;
|
|
134
217
|
statePollInFlight = false;
|
|
218
|
+
autoPauseController;
|
|
219
|
+
pauseRequested = false;
|
|
220
|
+
pauseWakeState = "active";
|
|
221
|
+
pauseWakeTransition = null;
|
|
135
222
|
// Model failover: override model on next spawn when rate-limited
|
|
136
223
|
modelOverride;
|
|
137
224
|
// Context rotation v3: ring buffers for daemon-side snapshot
|
|
@@ -151,12 +238,17 @@ export class Daemon extends EventEmitter {
|
|
|
151
238
|
pasteQueueDepth = 0;
|
|
152
239
|
// PTY error pattern monitoring
|
|
153
240
|
errorMonitorTimer = null;
|
|
241
|
+
/** Prevent in-flight monitor callbacks from re-arming after a pause. */
|
|
242
|
+
runtimeMonitorsFrozen = false;
|
|
154
243
|
errorWaitingForRecovery = false; // true = error detected, waiting for ready pattern
|
|
155
244
|
errorDetectedAt = 0;
|
|
156
|
-
/** Whether this instance is in an error state (
|
|
245
|
+
/** Whether this instance is in an abnormal error state (auto-pause is normal). */
|
|
157
246
|
get isErrorState() {
|
|
158
|
-
return this.errorWaitingForRecovery || this.healthCheckPaused || Daemon.tmuxServerPaused;
|
|
247
|
+
return this.errorWaitingForRecovery || (this.healthCheckPaused && !this.isPaused) || Daemon.tmuxServerPaused;
|
|
159
248
|
}
|
|
249
|
+
get isPaused() { return this.pauseWakeState !== "active"; }
|
|
250
|
+
get lastPausedAt() { return this.autoPauseController.lastPausedAt; }
|
|
251
|
+
getPauseWakeState() { return this.pauseWakeState; }
|
|
160
252
|
/** Whether this instance is in a crash loop (3+ consecutive crashes). */
|
|
161
253
|
get isCrashLoop() {
|
|
162
254
|
return this.crashCount >= 3;
|
|
@@ -174,7 +266,7 @@ export class Daemon extends EventEmitter {
|
|
|
174
266
|
// still registers as new (prevents the old hash-dedup's permanent suppression).
|
|
175
267
|
lastErrorCount = new Map();
|
|
176
268
|
lastDetectedErrorType = null;
|
|
177
|
-
constructor(name, config, instanceDir, topicMode = false, backend, controlClient) {
|
|
269
|
+
constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger) {
|
|
178
270
|
super();
|
|
179
271
|
this.name = name;
|
|
180
272
|
this.config = config;
|
|
@@ -182,13 +274,23 @@ export class Daemon extends EventEmitter {
|
|
|
182
274
|
this.topicMode = topicMode;
|
|
183
275
|
this.backend = backend;
|
|
184
276
|
this.controlClient = controlClient;
|
|
185
|
-
|
|
277
|
+
if (!rootLogger)
|
|
278
|
+
throw new Error("Daemon requires a shared root logger");
|
|
279
|
+
this.logger = rootLogger.child({ instance: name }, { level: config.log_level });
|
|
186
280
|
this.tmuxSessionName = getTmuxSession();
|
|
187
281
|
this.messageBus = new MessageBus();
|
|
188
282
|
this.messageBus.setLogger(this.logger);
|
|
283
|
+
const autoPauseMinutes = typeof config.auto_pause_after === "number" ? config.auto_pause_after : 0; // default: disabled
|
|
284
|
+
this.autoPauseController = new AutoPauseController(Math.max(0, autoPauseMinutes) * 60_000);
|
|
189
285
|
}
|
|
190
286
|
async start() {
|
|
191
287
|
mkdirSync(this.instanceDir, { recursive: true });
|
|
288
|
+
// A daemon restart performs a normal CLI start, so any persisted auto-pause
|
|
289
|
+
// marker from the previous daemon is stale.
|
|
290
|
+
try {
|
|
291
|
+
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
292
|
+
}
|
|
293
|
+
catch { }
|
|
192
294
|
writeFileSync(join(this.instanceDir, "daemon.pid"), String(process.pid));
|
|
193
295
|
this.logger.info(`Starting ${this.name}`);
|
|
194
296
|
// P1: Read crash state from previous run — skip resume if last run was a crash loop
|
|
@@ -293,13 +395,18 @@ export class Daemon extends EventEmitter {
|
|
|
293
395
|
// Fleet manager routed a message to us (topic mode)
|
|
294
396
|
const meta = msg.meta;
|
|
295
397
|
const targetSession = msg.targetSession;
|
|
296
|
-
this.
|
|
398
|
+
void this.wake().then(() => {
|
|
399
|
+
this.pushChannelMessage(msg.content, meta, targetSession);
|
|
400
|
+
}).catch(err => {
|
|
401
|
+
this.logger.error({ err: err.message }, "Wake failed for inbound delivery");
|
|
402
|
+
});
|
|
297
403
|
}
|
|
298
404
|
else if (msg.type === "raw_paste") {
|
|
299
405
|
// Paste raw text directly to CLI without [user:] wrapping.
|
|
300
406
|
if (this.tmux) {
|
|
301
407
|
const rawText = msg.content;
|
|
302
408
|
this.pasteLock = this.pasteLock.then(async () => {
|
|
409
|
+
await this.wake();
|
|
303
410
|
await this.deliverMessage(rawText);
|
|
304
411
|
this.logger.debug({ text: rawText.slice(0, 100) }, "Raw paste delivered");
|
|
305
412
|
}).catch(err => {
|
|
@@ -310,7 +417,9 @@ export class Daemon extends EventEmitter {
|
|
|
310
417
|
else if (msg.type === "fleet_schedule_trigger") {
|
|
311
418
|
const payload = msg.payload;
|
|
312
419
|
const meta = msg.meta;
|
|
313
|
-
this.pushChannelMessage(payload.message, meta)
|
|
420
|
+
void this.wake().then(() => this.pushChannelMessage(payload.message, meta)).catch(err => {
|
|
421
|
+
this.logger.error({ err: err.message }, "Wake failed for scheduled delivery");
|
|
422
|
+
});
|
|
314
423
|
}
|
|
315
424
|
else if (msg.type === "fleet_tool_status_ack") {
|
|
316
425
|
// Fleet manager sent us the messageId for our tool status message
|
|
@@ -323,6 +432,8 @@ export class Daemon extends EventEmitter {
|
|
|
323
432
|
requestId: msg.requestId,
|
|
324
433
|
instanceName: this.name,
|
|
325
434
|
...snapshot,
|
|
435
|
+
state: this.isPaused ? "paused" : snapshot.state,
|
|
436
|
+
pausedAt: this.lastPausedAt,
|
|
326
437
|
});
|
|
327
438
|
}
|
|
328
439
|
});
|
|
@@ -463,11 +574,18 @@ export class Daemon extends EventEmitter {
|
|
|
463
574
|
this.logger.info(`${this.name} ready`);
|
|
464
575
|
}
|
|
465
576
|
startHealthCheck() {
|
|
577
|
+
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
578
|
+
return;
|
|
466
579
|
const { max_retries, backoff, reset_after } = this.config.restart_policy;
|
|
467
580
|
if (max_retries <= 0)
|
|
468
581
|
return; // restart disabled
|
|
469
582
|
const scheduleNext = () => {
|
|
583
|
+
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
584
|
+
return;
|
|
470
585
|
this.healthCheckTimer = setTimeout(async () => {
|
|
586
|
+
this.healthCheckTimer = null;
|
|
587
|
+
if (this.runtimeMonitorsFrozen)
|
|
588
|
+
return;
|
|
471
589
|
// Instance directory removed externally (e.g. `rm -rf ~/.agend/instances/<name>`).
|
|
472
590
|
// Stop the loop permanently — otherwise every tick triggers a respawn, whose
|
|
473
591
|
// writeRotationSnapshot fails with ENOENT and gets caught as "Failed to respawn",
|
|
@@ -485,6 +603,12 @@ export class Daemon extends EventEmitter {
|
|
|
485
603
|
// Human-readable backend label for logs (e.g. "claude", "kiro-cli")
|
|
486
604
|
const cliLabel = this.backend?.binaryName ?? "CLI";
|
|
487
605
|
let paneStatus = await this.tmux.getPaneStatus();
|
|
606
|
+
// Auto-pause intentionally exits the pane process. A health tick that
|
|
607
|
+
// began just before pause must not classify that exit as a crash.
|
|
608
|
+
if (this.isPaused || this.pauseWakeState === "waking") {
|
|
609
|
+
scheduleNext();
|
|
610
|
+
return;
|
|
611
|
+
}
|
|
488
612
|
if (paneStatus?.alive) {
|
|
489
613
|
scheduleNext();
|
|
490
614
|
return;
|
|
@@ -712,6 +836,8 @@ export class Daemon extends EventEmitter {
|
|
|
712
836
|
* (ready pattern visible), it goes back to monitoring for new errors.
|
|
713
837
|
*/
|
|
714
838
|
startErrorMonitor() {
|
|
839
|
+
if (this.runtimeMonitorsFrozen || this.errorMonitorTimer)
|
|
840
|
+
return;
|
|
715
841
|
const patterns = this.backend?.getErrorPatterns?.() ?? [];
|
|
716
842
|
const dialogs = this.backend?.getRuntimeDialogs?.() ?? [];
|
|
717
843
|
if (!patterns.length && !dialogs.length)
|
|
@@ -831,26 +957,13 @@ export class Daemon extends EventEmitter {
|
|
|
831
957
|
}
|
|
832
958
|
async stop() {
|
|
833
959
|
this.logger.info("Stopping daemon instance");
|
|
834
|
-
|
|
835
|
-
clearTimeout(this.healthCheckTimer);
|
|
836
|
-
this.healthCheckTimer = null;
|
|
837
|
-
}
|
|
838
|
-
if (this.errorMonitorTimer) {
|
|
839
|
-
clearInterval(this.errorMonitorTimer);
|
|
840
|
-
this.errorMonitorTimer = null;
|
|
841
|
-
}
|
|
842
|
-
if (this.instanceStateMonitorTimer) {
|
|
843
|
-
clearInterval(this.instanceStateMonitorTimer);
|
|
844
|
-
this.instanceStateMonitorTimer = null;
|
|
845
|
-
}
|
|
960
|
+
this.freezeRuntimeMonitors();
|
|
846
961
|
if (this.toolStatusDebounce) {
|
|
847
962
|
clearTimeout(this.toolStatusDebounce);
|
|
848
963
|
this.toolStatusDebounce = null;
|
|
849
964
|
}
|
|
850
965
|
this.pendingIpcRequests.clear();
|
|
851
966
|
this.hangDetector?.stop();
|
|
852
|
-
this.transcriptMonitor?.stop();
|
|
853
|
-
this.guardian?.stop();
|
|
854
967
|
if (this.adapter)
|
|
855
968
|
await this.adapter.stop();
|
|
856
969
|
// Notify MCP servers of graceful shutdown (prevents reconnect attempts)
|
|
@@ -909,12 +1022,16 @@ export class Daemon extends EventEmitter {
|
|
|
909
1022
|
catch (e) {
|
|
910
1023
|
this.logger.debug({ err: e }, "Failed to remove PID file");
|
|
911
1024
|
}
|
|
1025
|
+
try {
|
|
1026
|
+
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
1027
|
+
}
|
|
1028
|
+
catch { }
|
|
912
1029
|
}
|
|
913
1030
|
getHangDetector() {
|
|
914
1031
|
return this.hangDetector;
|
|
915
1032
|
}
|
|
916
1033
|
getInstanceState() {
|
|
917
|
-
return this.instanceState;
|
|
1034
|
+
return this.isPaused ? "paused" : this.instanceState;
|
|
918
1035
|
}
|
|
919
1036
|
getInstanceStateSnapshot() {
|
|
920
1037
|
return this.instanceStateMachine?.snapshot() ?? {
|
|
@@ -924,8 +1041,142 @@ export class Daemon extends EventEmitter {
|
|
|
924
1041
|
stateChangedAt: Date.now(),
|
|
925
1042
|
};
|
|
926
1043
|
}
|
|
1044
|
+
/** Gracefully stop the CLI while keeping its remain-on-exit tmux window. */
|
|
1045
|
+
async pause() {
|
|
1046
|
+
if (this.pauseWakeState === "paused")
|
|
1047
|
+
return;
|
|
1048
|
+
if (this.pauseWakeState === "pausing" || this.pauseWakeState === "waking") {
|
|
1049
|
+
await this.pauseWakeTransition;
|
|
1050
|
+
if (this.getPauseWakeState() !== "active")
|
|
1051
|
+
return;
|
|
1052
|
+
}
|
|
1053
|
+
if (this.instanceState !== "idle" || this.pasteQueueDepth > 0) {
|
|
1054
|
+
this.pauseRequested = false;
|
|
1055
|
+
return;
|
|
1056
|
+
}
|
|
1057
|
+
this.pauseWakeState = "pausing";
|
|
1058
|
+
this.healthCheckPaused = true;
|
|
1059
|
+
this.freezeRuntimeMonitors();
|
|
1060
|
+
const transition = (async () => {
|
|
1061
|
+
try {
|
|
1062
|
+
this.saveSessionId();
|
|
1063
|
+
const quitCmd = this.backend?.getQuitCommand();
|
|
1064
|
+
if (quitCmd && this.tmux) {
|
|
1065
|
+
await this.tmux.sendKeys(quitCmd);
|
|
1066
|
+
await new Promise(r => setTimeout(r, 150));
|
|
1067
|
+
await this.tmux.sendSpecialKey("Enter");
|
|
1068
|
+
}
|
|
1069
|
+
let exited = false;
|
|
1070
|
+
for (let i = 0; i < 15; i++) {
|
|
1071
|
+
await new Promise(r => setTimeout(r, 200));
|
|
1072
|
+
const status = await this.tmux?.getPaneStatus();
|
|
1073
|
+
if (status && !status.alive) {
|
|
1074
|
+
exited = true;
|
|
1075
|
+
break;
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
if (!exited) {
|
|
1079
|
+
await this.killProcessTree("SIGTERM");
|
|
1080
|
+
await new Promise(r => setTimeout(r, 1_000));
|
|
1081
|
+
const status = await this.tmux?.getPaneStatus();
|
|
1082
|
+
if (status?.alive) {
|
|
1083
|
+
await this.killProcessTree("SIGKILL");
|
|
1084
|
+
await new Promise(r => setTimeout(r, 200));
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
const finalStatus = await this.tmux?.getPaneStatus();
|
|
1088
|
+
if (!finalStatus || finalStatus.alive) {
|
|
1089
|
+
throw new Error("Auto-pause could not stop the CLI while preserving its tmux window");
|
|
1090
|
+
}
|
|
1091
|
+
this.pauseWakeState = "paused";
|
|
1092
|
+
this.autoPauseController.markPaused();
|
|
1093
|
+
writeFileSync(join(this.instanceDir, "paused-state.json"), JSON.stringify({
|
|
1094
|
+
paused_at: this.lastPausedAt,
|
|
1095
|
+
}));
|
|
1096
|
+
this.logger.info({ pausedAt: this.lastPausedAt }, "Instance auto-paused");
|
|
1097
|
+
this.ipcServer?.broadcast({
|
|
1098
|
+
type: "instance_state", instanceName: this.name, state: "paused", pausedAt: this.lastPausedAt,
|
|
1099
|
+
});
|
|
1100
|
+
this.emit("auto_paused", { name: this.name, pausedAt: this.lastPausedAt });
|
|
1101
|
+
}
|
|
1102
|
+
catch (err) {
|
|
1103
|
+
this.pauseWakeState = "active";
|
|
1104
|
+
this.healthCheckPaused = false;
|
|
1105
|
+
this.pauseRequested = false;
|
|
1106
|
+
this.resumeRuntimeMonitors();
|
|
1107
|
+
throw err;
|
|
1108
|
+
}
|
|
1109
|
+
})();
|
|
1110
|
+
this.pauseWakeTransition = transition;
|
|
1111
|
+
try {
|
|
1112
|
+
await transition;
|
|
1113
|
+
}
|
|
1114
|
+
finally {
|
|
1115
|
+
if (this.pauseWakeTransition === transition)
|
|
1116
|
+
this.pauseWakeTransition = null;
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
/** Respawn the CLI in the preserved window and block until its prompt is ready. */
|
|
1120
|
+
async wake(timeoutMs = 30_000) {
|
|
1121
|
+
if (this.pauseWakeState === "active")
|
|
1122
|
+
return;
|
|
1123
|
+
if (this.pauseWakeState === "pausing")
|
|
1124
|
+
await this.pauseWakeTransition;
|
|
1125
|
+
if (this.getPauseWakeState() === "active")
|
|
1126
|
+
return;
|
|
1127
|
+
if (this.pauseWakeState === "waking") {
|
|
1128
|
+
await this.pauseWakeTransition;
|
|
1129
|
+
return;
|
|
1130
|
+
}
|
|
1131
|
+
this.pauseWakeState = "waking";
|
|
1132
|
+
this.spawning = true;
|
|
1133
|
+
const transition = this.autoPauseController.wakeOnDeliver(async () => {
|
|
1134
|
+
let timeout;
|
|
1135
|
+
try {
|
|
1136
|
+
const ready = await Promise.race([
|
|
1137
|
+
this.trySpawn(true, timeoutMs),
|
|
1138
|
+
new Promise(resolve => { timeout = setTimeout(() => resolve(false), timeoutMs); }),
|
|
1139
|
+
]);
|
|
1140
|
+
if (!ready)
|
|
1141
|
+
throw new Error(`Wake timed out before CLI became ready (${timeoutMs}ms)`);
|
|
1142
|
+
}
|
|
1143
|
+
finally {
|
|
1144
|
+
if (timeout)
|
|
1145
|
+
clearTimeout(timeout);
|
|
1146
|
+
}
|
|
1147
|
+
});
|
|
1148
|
+
this.pauseWakeTransition = transition;
|
|
1149
|
+
try {
|
|
1150
|
+
await transition;
|
|
1151
|
+
this.pauseWakeState = "active";
|
|
1152
|
+
this.healthCheckPaused = false;
|
|
1153
|
+
this.pauseRequested = false;
|
|
1154
|
+
try {
|
|
1155
|
+
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
1156
|
+
}
|
|
1157
|
+
catch { }
|
|
1158
|
+
this.transcriptMonitor?.resetOffset();
|
|
1159
|
+
this.resumeRuntimeMonitors();
|
|
1160
|
+
this.logger.info("Instance auto-woke");
|
|
1161
|
+
this.ipcServer?.broadcast({
|
|
1162
|
+
type: "instance_state", instanceName: this.name, state: this.instanceState, pausedAt: null,
|
|
1163
|
+
});
|
|
1164
|
+
this.emit("auto_woke", { name: this.name });
|
|
1165
|
+
}
|
|
1166
|
+
catch (err) {
|
|
1167
|
+
this.pauseWakeState = "paused";
|
|
1168
|
+
this.healthCheckPaused = true;
|
|
1169
|
+
this.logger.error({ err: err.message }, "Instance wake failed");
|
|
1170
|
+
throw err;
|
|
1171
|
+
}
|
|
1172
|
+
finally {
|
|
1173
|
+
this.spawning = false;
|
|
1174
|
+
if (this.pauseWakeTransition === transition)
|
|
1175
|
+
this.pauseWakeTransition = null;
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
927
1178
|
startInstanceStateMonitor() {
|
|
928
|
-
if (!this.tmux || !this.backend || this.instanceStateMonitorTimer)
|
|
1179
|
+
if (this.runtimeMonitorsFrozen || !this.tmux || !this.backend || this.instanceStateMonitorTimer)
|
|
929
1180
|
return;
|
|
930
1181
|
const rawConfig = this.config.hang_detector;
|
|
931
1182
|
const timeoutMinutes = rawConfig?.timeout_minutes;
|
|
@@ -935,7 +1186,8 @@ export class Daemon extends EventEmitter {
|
|
|
935
1186
|
const pollIntervalMs = typeof rawConfig?.poll_interval_ms === "number" && rawConfig.poll_interval_ms > 0
|
|
936
1187
|
? rawConfig.poll_interval_ms
|
|
937
1188
|
: DEFAULT_STATE_POLL_INTERVAL_MS;
|
|
938
|
-
|
|
1189
|
+
const readyPattern = this.backend.getReadyPattern();
|
|
1190
|
+
this.instanceStateMachine = new PaneStateMachine(readyPattern, stuckTimeoutMs);
|
|
939
1191
|
const poll = async () => {
|
|
940
1192
|
if (!this.tmux || this.spawning || this.statePollInFlight)
|
|
941
1193
|
return;
|
|
@@ -948,6 +1200,11 @@ export class Daemon extends EventEmitter {
|
|
|
948
1200
|
const previous = this.instanceState;
|
|
949
1201
|
const snapshot = this.instanceStateMachine.observe(pane);
|
|
950
1202
|
this.instanceState = snapshot.state;
|
|
1203
|
+
// Only a transition back to idle completes pending work. Repeated idle
|
|
1204
|
+
// polls between enqueue and paste must not clear a newly-recorded inbound.
|
|
1205
|
+
if (snapshot.state === "idle" && previous !== "idle") {
|
|
1206
|
+
this.pendingWork.recordIdle(snapshot.observedAt);
|
|
1207
|
+
}
|
|
951
1208
|
if (snapshot.state !== previous) {
|
|
952
1209
|
this.logger.info({
|
|
953
1210
|
previousState: previous,
|
|
@@ -959,9 +1216,15 @@ export class Daemon extends EventEmitter {
|
|
|
959
1216
|
// Emit exactly once per transition into stuck. Further notifications
|
|
960
1217
|
// require observable progress (working) or a ready prompt (idle) first.
|
|
961
1218
|
if (snapshot.state === "stuck") {
|
|
962
|
-
this.
|
|
1219
|
+
this.handleStuckTransition(pane, snapshot, readyPattern);
|
|
963
1220
|
}
|
|
964
1221
|
}
|
|
1222
|
+
if (snapshot.state !== "idle")
|
|
1223
|
+
this.pauseRequested = false;
|
|
1224
|
+
if (!this.pauseRequested && this.pasteQueueDepth === 0 && this.autoPauseController.observe(snapshot.state)) {
|
|
1225
|
+
this.pauseRequested = true;
|
|
1226
|
+
this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
|
|
1227
|
+
}
|
|
965
1228
|
}
|
|
966
1229
|
catch (err) {
|
|
967
1230
|
// A pane can disappear between status and capture during restart. Keep
|
|
@@ -975,6 +1238,54 @@ export class Daemon extends EventEmitter {
|
|
|
975
1238
|
void poll();
|
|
976
1239
|
this.instanceStateMonitorTimer = setInterval(() => { void poll(); }, pollIntervalMs);
|
|
977
1240
|
}
|
|
1241
|
+
handleStuckTransition(pane, snapshot, readyPattern) {
|
|
1242
|
+
const deterministicReadyPattern = new RegExp(readyPattern.source, readyPattern.flags.replace(/[gy]/g, ""));
|
|
1243
|
+
const diagnostic = {
|
|
1244
|
+
backend: this.backend?.binaryName ?? this.config.backend ?? "unknown",
|
|
1245
|
+
paneTail: sanitizePaneTail(pane),
|
|
1246
|
+
readyPattern: readyPattern.toString(),
|
|
1247
|
+
readyMatched: deterministicReadyPattern.test(pane),
|
|
1248
|
+
unchangedForMs: snapshot.unchangedForMs,
|
|
1249
|
+
pendingWork: this.pendingWork.hasPendingWork(),
|
|
1250
|
+
};
|
|
1251
|
+
if (!diagnostic.pendingWork) {
|
|
1252
|
+
this.logger.debug(diagnostic, "Suppressing stuck notification without pending work");
|
|
1253
|
+
return;
|
|
1254
|
+
}
|
|
1255
|
+
this.logger.warn(diagnostic, "Instance pane stuck with pending work");
|
|
1256
|
+
this.hangDetector?.emit("hang", { unchangedForMs: snapshot.unchangedForMs });
|
|
1257
|
+
}
|
|
1258
|
+
/** Stop every runtime poller/watcher while preserving IPC and daemon state. */
|
|
1259
|
+
freezeRuntimeMonitors() {
|
|
1260
|
+
this.runtimeMonitorsFrozen = true;
|
|
1261
|
+
if (this.healthCheckTimer) {
|
|
1262
|
+
clearTimeout(this.healthCheckTimer);
|
|
1263
|
+
this.healthCheckTimer = null;
|
|
1264
|
+
}
|
|
1265
|
+
if (this.errorMonitorTimer) {
|
|
1266
|
+
clearInterval(this.errorMonitorTimer);
|
|
1267
|
+
this.errorMonitorTimer = null;
|
|
1268
|
+
}
|
|
1269
|
+
if (this.instanceStateMonitorTimer) {
|
|
1270
|
+
clearInterval(this.instanceStateMonitorTimer);
|
|
1271
|
+
this.instanceStateMonitorTimer = null;
|
|
1272
|
+
}
|
|
1273
|
+
this.transcriptMonitor?.stop();
|
|
1274
|
+
this.guardian?.stop();
|
|
1275
|
+
}
|
|
1276
|
+
/** Restore the same monitor objects after wake without adding event listeners. */
|
|
1277
|
+
resumeRuntimeMonitors() {
|
|
1278
|
+
if (!this.runtimeMonitorsFrozen)
|
|
1279
|
+
return;
|
|
1280
|
+
this.runtimeMonitorsFrozen = false;
|
|
1281
|
+
this.startHealthCheck();
|
|
1282
|
+
if (!this.config.lightweight) {
|
|
1283
|
+
this.transcriptMonitor?.startPolling();
|
|
1284
|
+
this.guardian?.startWatching();
|
|
1285
|
+
this.startErrorMonitor();
|
|
1286
|
+
}
|
|
1287
|
+
this.startInstanceStateMonitor();
|
|
1288
|
+
}
|
|
978
1289
|
getMessageBus() {
|
|
979
1290
|
return this.messageBus;
|
|
980
1291
|
}
|
|
@@ -1059,6 +1370,7 @@ export class Daemon extends EventEmitter {
|
|
|
1059
1370
|
this.pendingInstructionsUpdate = undefined;
|
|
1060
1371
|
}
|
|
1061
1372
|
this.hangDetector?.recordInbound();
|
|
1373
|
+
this.pendingWork.recordInbound();
|
|
1062
1374
|
// v3: record user messages for rotation snapshot
|
|
1063
1375
|
this.recordRecentUserMessage(content, meta);
|
|
1064
1376
|
// Format message with metadata prefix for the agent
|
|
@@ -1632,14 +1944,14 @@ export class Daemon extends EventEmitter {
|
|
|
1632
1944
|
return resumedSuccessfully;
|
|
1633
1945
|
}
|
|
1634
1946
|
/** Kill the entire process tree of the current tmux pane (CLI + MCP server). */
|
|
1635
|
-
async killProcessTree() {
|
|
1947
|
+
async killProcessTree(signal = "SIGTERM") {
|
|
1636
1948
|
if (!this.tmux)
|
|
1637
1949
|
return;
|
|
1638
1950
|
try {
|
|
1639
1951
|
const pid = await TmuxManager.getPanePid(this.tmuxSessionName, this.tmux.getWindowId());
|
|
1640
1952
|
if (pid) {
|
|
1641
|
-
process.kill(-pid,
|
|
1642
|
-
this.logger.debug({ pid }, "Killed process group");
|
|
1953
|
+
process.kill(-pid, signal);
|
|
1954
|
+
this.logger.debug({ pid, signal }, "Killed process group");
|
|
1643
1955
|
}
|
|
1644
1956
|
}
|
|
1645
1957
|
catch { /* process group may not exist or already dead */ }
|
|
@@ -1650,7 +1962,7 @@ export class Daemon extends EventEmitter {
|
|
|
1650
1962
|
* Handles confirmation dialogs (trust folder, bypass permissions).
|
|
1651
1963
|
* Returns true if CLI is ready, false if it failed or got stuck.
|
|
1652
1964
|
*/
|
|
1653
|
-
async trySpawn() {
|
|
1965
|
+
async trySpawn(reuseWindow = false, startupTimeoutMs) {
|
|
1654
1966
|
const backendConfig = this.buildBackendConfig();
|
|
1655
1967
|
// Compare freshly-built instructions against the last value the agent was
|
|
1656
1968
|
// told about. Computed for ALL backends (not gated by
|
|
@@ -1713,16 +2025,29 @@ export class Daemon extends EventEmitter {
|
|
|
1713
2025
|
const cmd = `${envPrefix} ` + this.backend.buildCommand(backendConfig);
|
|
1714
2026
|
// Ensure tmux session exists (may have been destroyed if all windows died)
|
|
1715
2027
|
await TmuxManager.ensureSession(this.tmuxSessionName);
|
|
1716
|
-
|
|
2028
|
+
let windowId;
|
|
2029
|
+
if (reuseWindow) {
|
|
2030
|
+
this.controlClient?.unregisterWindow(this.tmux.getWindowId());
|
|
2031
|
+
await this.tmux.respawnWindow(cmd, resolvedCwd);
|
|
2032
|
+
windowId = this.tmux.getWindowId();
|
|
2033
|
+
}
|
|
2034
|
+
else {
|
|
2035
|
+
windowId = await this.tmux.createWindow(cmd, resolvedCwd, this.name);
|
|
2036
|
+
}
|
|
1717
2037
|
writeFileSync(join(this.instanceDir, "window-id"), windowId);
|
|
1718
2038
|
// Enable remain-on-exit to capture exit codes on crash
|
|
1719
2039
|
await this.tmux.setRemainOnExit().catch(err => {
|
|
1720
2040
|
this.logger.warn({ err }, "Failed to set remain-on-exit — exit codes will not be captured");
|
|
1721
2041
|
});
|
|
2042
|
+
if (reuseWindow && !this.config.lightweight) {
|
|
2043
|
+
await this.tmux.pipeOutput(join(this.instanceDir, "output.log")).catch(err => {
|
|
2044
|
+
this.logger.warn({ err }, "Failed to restore pipe-pane after wake");
|
|
2045
|
+
});
|
|
2046
|
+
}
|
|
1722
2047
|
// Register with control client and wait for output + idle
|
|
1723
2048
|
await this.controlClient?.registerWindow(windowId);
|
|
1724
2049
|
if (this.controlClient) {
|
|
1725
|
-
const total = this.config.startup_timeout_ms ?? 25_000;
|
|
2050
|
+
const total = startupTimeoutMs ?? this.config.startup_timeout_ms ?? 25_000;
|
|
1726
2051
|
const outputTimeout = Math.round(total * 0.6);
|
|
1727
2052
|
const idleTimeout = total - outputTimeout;
|
|
1728
2053
|
const hasOutput = await this.controlClient.waitForOutput(windowId, outputTimeout);
|