@songsid/agend 2.1.0-beta.2 → 2.1.0-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/README.zh-TW.md +2 -1
- package/dist/access-path.js +3 -3
- package/dist/access-path.js.map +1 -1
- package/dist/agent-endpoint.d.ts +8 -0
- package/dist/agent-endpoint.js +36 -8
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.js +3 -2
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/factory.js +5 -1
- package/dist/backend/factory.js.map +1 -1
- package/dist/backend/grok.d.ts +41 -0
- package/dist/backend/grok.js +256 -0
- package/dist/backend/grok.js.map +1 -0
- package/dist/backend/kiro.js +3 -2
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +12 -2
- package/dist/backend/types.js +1 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/classic-channel-manager.js +1 -1
- package/dist/cli.js +56 -6
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +50 -1
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +3 -1
- package/dist/config.js +13 -0
- package/dist/config.js.map +1 -1
- package/dist/context-guardian.d.ts +2 -0
- package/dist/context-guardian.js +10 -2
- package/dist/context-guardian.js.map +1 -1
- package/dist/daemon-entry.js +2 -1
- package/dist/daemon-entry.js.map +1 -1
- package/dist/daemon.d.ts +51 -3
- package/dist/daemon.js +380 -36
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-context.d.ts +2 -1
- package/dist/fleet-manager.d.ts +23 -5
- package/dist/fleet-manager.js +229 -123
- package/dist/fleet-manager.js.map +1 -1
- package/dist/general-knowledge/skills/fleet-config/SKILL.md +1 -1
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +1 -0
- package/dist/instance-lifecycle.d.ts +6 -1
- package/dist/instance-lifecycle.js +46 -3
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/logger.d.ts +6 -0
- package/dist/logger.js +29 -10
- package/dist/logger.js.map +1 -1
- package/dist/outbound-handlers.d.ts +5 -0
- package/dist/outbound-handlers.js +68 -16
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +1 -0
- package/dist/outbound-schemas.js +1 -1
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/settings-api.d.ts +17 -2
- package/dist/settings-api.js +105 -8
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +8 -0
- package/dist/setup-wizard.js.map +1 -1
- package/dist/statusline-watcher.d.ts +1 -1
- package/dist/statusline-watcher.js +3 -2
- package/dist/statusline-watcher.js.map +1 -1
- package/dist/tmux-manager.d.ts +3 -1
- package/dist/tmux-manager.js +9 -0
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-archiver.d.ts +1 -1
- package/dist/topic-commands.d.ts +14 -1
- package/dist/topic-commands.js +96 -25
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-monitor.js +2 -0
- package/dist/transcript-monitor.js.map +1 -1
- package/dist/types.d.ts +18 -1
- package/dist/ui/dashboard.html +2 -2
- package/dist/ui/settings.html +251 -40
- package/dist/view-api.d.ts +1 -1
- package/dist/web-api.d.ts +2 -1
- package/dist/web-api.js +24 -15
- package/dist/web-api.js.map +1 -1
- package/package.json +2 -1
package/dist/daemon.js
CHANGED
|
@@ -3,7 +3,6 @@ import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync,
|
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import { createHash, randomBytes } from "node:crypto";
|
|
5
5
|
import { EventEmitter } from "node:events";
|
|
6
|
-
import { createLogger } from "./logger.js";
|
|
7
6
|
import { TmuxManager } from "./tmux-manager.js";
|
|
8
7
|
import { TranscriptMonitor } from "./transcript-monitor.js";
|
|
9
8
|
import { ContextGuardian } from "./context-guardian.js";
|
|
@@ -23,6 +22,40 @@ const DECISION_TOOLS = new Set(["post_decision", "list_decisions", "update_decis
|
|
|
23
22
|
const TASK_TOOL = "task";
|
|
24
23
|
export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
|
|
25
24
|
export const DEFAULT_STATE_POLL_INTERVAL_MS = 5_000;
|
|
25
|
+
/** Headless idle timer used by the daemon and unit tests. */
|
|
26
|
+
export class AutoPauseController {
|
|
27
|
+
thresholdMs;
|
|
28
|
+
idleSince = null;
|
|
29
|
+
pausedAt = null;
|
|
30
|
+
constructor(thresholdMs) {
|
|
31
|
+
this.thresholdMs = thresholdMs;
|
|
32
|
+
}
|
|
33
|
+
observe(state, now = Date.now()) {
|
|
34
|
+
if (this.pausedAt !== null || this.thresholdMs <= 0)
|
|
35
|
+
return false;
|
|
36
|
+
if (state !== "idle") {
|
|
37
|
+
this.idleSince = null;
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
this.idleSince ??= now;
|
|
41
|
+
return now - this.idleSince >= this.thresholdMs;
|
|
42
|
+
}
|
|
43
|
+
markPaused(now = Date.now()) {
|
|
44
|
+
this.pausedAt = now;
|
|
45
|
+
}
|
|
46
|
+
markAwake() {
|
|
47
|
+
this.pausedAt = null;
|
|
48
|
+
this.idleSince = null;
|
|
49
|
+
}
|
|
50
|
+
async wakeOnDeliver(wake) {
|
|
51
|
+
if (this.pausedAt === null)
|
|
52
|
+
return;
|
|
53
|
+
await wake();
|
|
54
|
+
this.markAwake();
|
|
55
|
+
}
|
|
56
|
+
get isPaused() { return this.pausedAt !== null; }
|
|
57
|
+
get lastPausedAt() { return this.pausedAt; }
|
|
58
|
+
}
|
|
26
59
|
/**
|
|
27
60
|
* Headless state machine for pane-based execution state detection.
|
|
28
61
|
*
|
|
@@ -81,6 +114,55 @@ export class PaneStateMachine {
|
|
|
81
114
|
};
|
|
82
115
|
}
|
|
83
116
|
}
|
|
117
|
+
/** Tracks whether an inbound arrived after the most recent confirmed idle prompt. */
|
|
118
|
+
export class PendingWorkTracker {
|
|
119
|
+
lastInboundAt = 0;
|
|
120
|
+
lastIdleAt;
|
|
121
|
+
sequence = 0;
|
|
122
|
+
lastInboundOrder = 0;
|
|
123
|
+
lastIdleOrder = 0;
|
|
124
|
+
constructor(now = Date.now()) {
|
|
125
|
+
this.lastIdleAt = now;
|
|
126
|
+
}
|
|
127
|
+
recordInbound(now = Date.now()) {
|
|
128
|
+
this.lastInboundAt = now;
|
|
129
|
+
this.lastInboundOrder = ++this.sequence;
|
|
130
|
+
}
|
|
131
|
+
recordIdle(now = Date.now()) {
|
|
132
|
+
// An async pane poll can finish after a newer inbound. Do not let its stale
|
|
133
|
+
// observation clear work which had not arrived when the pane was captured.
|
|
134
|
+
if (now < this.lastInboundAt)
|
|
135
|
+
return;
|
|
136
|
+
this.lastIdleAt = now;
|
|
137
|
+
this.lastIdleOrder = ++this.sequence;
|
|
138
|
+
}
|
|
139
|
+
hasPendingWork() {
|
|
140
|
+
return this.lastInboundOrder > this.lastIdleOrder;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
/** Redact likely credentials and control sequences before pane text reaches logs. */
|
|
144
|
+
export function sanitizePaneTail(pane, lineCount = 5) {
|
|
145
|
+
const secretAssignment = /\b(token|secret|password|passwd|api[_-]?key|authorization)\b\s*[:=]\s*\S+/gi;
|
|
146
|
+
const bearer = /\bBearer\s+\S+/gi;
|
|
147
|
+
const knownToken = /\b(?:sk-[A-Za-z0-9_-]+|ghp_[A-Za-z0-9]+|github_pat_[A-Za-z0-9_]+|AKIA[A-Z0-9]{16})\b/g;
|
|
148
|
+
const jwt = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g;
|
|
149
|
+
const opaqueSecret = /\b[A-Za-z0-9_+/=-]{32,}\b/g;
|
|
150
|
+
const lines = pane
|
|
151
|
+
.replace(/\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g, "")
|
|
152
|
+
.split(/\r?\n/);
|
|
153
|
+
while (lines.length > 0 && /^\s*$/.test(lines[lines.length - 1]))
|
|
154
|
+
lines.pop();
|
|
155
|
+
return lines
|
|
156
|
+
.slice(-lineCount)
|
|
157
|
+
.map(line => line
|
|
158
|
+
.replace(bearer, "Bearer [REDACTED]")
|
|
159
|
+
.replace(secretAssignment, "$1=[REDACTED]")
|
|
160
|
+
.replace(knownToken, "[REDACTED]")
|
|
161
|
+
.replace(jwt, "[REDACTED]")
|
|
162
|
+
.replace(opaqueSecret, "[REDACTED]")
|
|
163
|
+
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "")
|
|
164
|
+
.slice(0, 200));
|
|
165
|
+
}
|
|
84
166
|
export class Daemon extends EventEmitter {
|
|
85
167
|
name;
|
|
86
168
|
config;
|
|
@@ -101,6 +183,7 @@ export class Daemon extends EventEmitter {
|
|
|
101
183
|
// Track chatId/threadId from inbound messages for automatic outbound routing
|
|
102
184
|
lastChatId;
|
|
103
185
|
lastThreadId;
|
|
186
|
+
lastAdapterId;
|
|
104
187
|
// Pending ack: react 🫡 on first transcript activity after receiving a message
|
|
105
188
|
pendingAckMessage = null;
|
|
106
189
|
// Tool status tracking for channel adapter
|
|
@@ -130,8 +213,13 @@ export class Daemon extends EventEmitter {
|
|
|
130
213
|
hangDetector = null;
|
|
131
214
|
instanceState = "idle";
|
|
132
215
|
instanceStateMachine = null;
|
|
216
|
+
pendingWork = new PendingWorkTracker();
|
|
133
217
|
instanceStateMonitorTimer = null;
|
|
134
218
|
statePollInFlight = false;
|
|
219
|
+
autoPauseController;
|
|
220
|
+
pauseRequested = false;
|
|
221
|
+
pauseWakeState = "active";
|
|
222
|
+
pauseWakeTransition = null;
|
|
135
223
|
// Model failover: override model on next spawn when rate-limited
|
|
136
224
|
modelOverride;
|
|
137
225
|
// Context rotation v3: ring buffers for daemon-side snapshot
|
|
@@ -151,12 +239,17 @@ export class Daemon extends EventEmitter {
|
|
|
151
239
|
pasteQueueDepth = 0;
|
|
152
240
|
// PTY error pattern monitoring
|
|
153
241
|
errorMonitorTimer = null;
|
|
242
|
+
/** Prevent in-flight monitor callbacks from re-arming after a pause. */
|
|
243
|
+
runtimeMonitorsFrozen = false;
|
|
154
244
|
errorWaitingForRecovery = false; // true = error detected, waiting for ready pattern
|
|
155
245
|
errorDetectedAt = 0;
|
|
156
|
-
/** Whether this instance is in an error state (
|
|
246
|
+
/** Whether this instance is in an abnormal error state (auto-pause is normal). */
|
|
157
247
|
get isErrorState() {
|
|
158
|
-
return this.errorWaitingForRecovery || this.healthCheckPaused || Daemon.tmuxServerPaused;
|
|
248
|
+
return this.errorWaitingForRecovery || (this.healthCheckPaused && !this.isPaused) || Daemon.tmuxServerPaused;
|
|
159
249
|
}
|
|
250
|
+
get isPaused() { return this.pauseWakeState !== "active"; }
|
|
251
|
+
get lastPausedAt() { return this.autoPauseController.lastPausedAt; }
|
|
252
|
+
getPauseWakeState() { return this.pauseWakeState; }
|
|
160
253
|
/** Whether this instance is in a crash loop (3+ consecutive crashes). */
|
|
161
254
|
get isCrashLoop() {
|
|
162
255
|
return this.crashCount >= 3;
|
|
@@ -174,7 +267,7 @@ export class Daemon extends EventEmitter {
|
|
|
174
267
|
// still registers as new (prevents the old hash-dedup's permanent suppression).
|
|
175
268
|
lastErrorCount = new Map();
|
|
176
269
|
lastDetectedErrorType = null;
|
|
177
|
-
constructor(name, config, instanceDir, topicMode = false, backend, controlClient) {
|
|
270
|
+
constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger) {
|
|
178
271
|
super();
|
|
179
272
|
this.name = name;
|
|
180
273
|
this.config = config;
|
|
@@ -182,13 +275,23 @@ export class Daemon extends EventEmitter {
|
|
|
182
275
|
this.topicMode = topicMode;
|
|
183
276
|
this.backend = backend;
|
|
184
277
|
this.controlClient = controlClient;
|
|
185
|
-
|
|
278
|
+
if (!rootLogger)
|
|
279
|
+
throw new Error("Daemon requires a shared root logger");
|
|
280
|
+
this.logger = rootLogger.child({ instance: name }, { level: config.log_level });
|
|
186
281
|
this.tmuxSessionName = getTmuxSession();
|
|
187
282
|
this.messageBus = new MessageBus();
|
|
188
283
|
this.messageBus.setLogger(this.logger);
|
|
284
|
+
const autoPauseMinutes = typeof config.auto_pause_after === "number" ? config.auto_pause_after : 0; // default: disabled
|
|
285
|
+
this.autoPauseController = new AutoPauseController(Math.max(0, autoPauseMinutes) * 60_000);
|
|
189
286
|
}
|
|
190
287
|
async start() {
|
|
191
288
|
mkdirSync(this.instanceDir, { recursive: true });
|
|
289
|
+
// A daemon restart performs a normal CLI start, so any persisted auto-pause
|
|
290
|
+
// marker from the previous daemon is stale.
|
|
291
|
+
try {
|
|
292
|
+
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
293
|
+
}
|
|
294
|
+
catch { }
|
|
192
295
|
writeFileSync(join(this.instanceDir, "daemon.pid"), String(process.pid));
|
|
193
296
|
this.logger.info(`Starting ${this.name}`);
|
|
194
297
|
// P1: Read crash state from previous run — skip resume if last run was a crash loop
|
|
@@ -214,6 +317,7 @@ export class Daemon extends EventEmitter {
|
|
|
214
317
|
if (saved.chatId) {
|
|
215
318
|
this.lastChatId = saved.chatId;
|
|
216
319
|
this.lastThreadId = saved.threadId || undefined;
|
|
320
|
+
this.lastAdapterId = saved.adapterId || undefined;
|
|
217
321
|
}
|
|
218
322
|
}
|
|
219
323
|
}
|
|
@@ -293,13 +397,18 @@ export class Daemon extends EventEmitter {
|
|
|
293
397
|
// Fleet manager routed a message to us (topic mode)
|
|
294
398
|
const meta = msg.meta;
|
|
295
399
|
const targetSession = msg.targetSession;
|
|
296
|
-
this.
|
|
400
|
+
void this.wake().then(() => {
|
|
401
|
+
this.pushChannelMessage(msg.content, meta, targetSession);
|
|
402
|
+
}).catch(err => {
|
|
403
|
+
this.logger.error({ err: err.message }, "Wake failed for inbound delivery");
|
|
404
|
+
});
|
|
297
405
|
}
|
|
298
406
|
else if (msg.type === "raw_paste") {
|
|
299
407
|
// Paste raw text directly to CLI without [user:] wrapping.
|
|
300
408
|
if (this.tmux) {
|
|
301
409
|
const rawText = msg.content;
|
|
302
410
|
this.pasteLock = this.pasteLock.then(async () => {
|
|
411
|
+
await this.wake();
|
|
303
412
|
await this.deliverMessage(rawText);
|
|
304
413
|
this.logger.debug({ text: rawText.slice(0, 100) }, "Raw paste delivered");
|
|
305
414
|
}).catch(err => {
|
|
@@ -310,7 +419,9 @@ export class Daemon extends EventEmitter {
|
|
|
310
419
|
else if (msg.type === "fleet_schedule_trigger") {
|
|
311
420
|
const payload = msg.payload;
|
|
312
421
|
const meta = msg.meta;
|
|
313
|
-
this.pushChannelMessage(payload.message, meta)
|
|
422
|
+
void this.wake().then(() => this.pushChannelMessage(payload.message, meta)).catch(err => {
|
|
423
|
+
this.logger.error({ err: err.message }, "Wake failed for scheduled delivery");
|
|
424
|
+
});
|
|
314
425
|
}
|
|
315
426
|
else if (msg.type === "fleet_tool_status_ack") {
|
|
316
427
|
// Fleet manager sent us the messageId for our tool status message
|
|
@@ -323,6 +434,8 @@ export class Daemon extends EventEmitter {
|
|
|
323
434
|
requestId: msg.requestId,
|
|
324
435
|
instanceName: this.name,
|
|
325
436
|
...snapshot,
|
|
437
|
+
state: this.isPaused ? "paused" : snapshot.state,
|
|
438
|
+
pausedAt: this.lastPausedAt,
|
|
326
439
|
});
|
|
327
440
|
}
|
|
328
441
|
});
|
|
@@ -463,11 +576,18 @@ export class Daemon extends EventEmitter {
|
|
|
463
576
|
this.logger.info(`${this.name} ready`);
|
|
464
577
|
}
|
|
465
578
|
startHealthCheck() {
|
|
579
|
+
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
580
|
+
return;
|
|
466
581
|
const { max_retries, backoff, reset_after } = this.config.restart_policy;
|
|
467
582
|
if (max_retries <= 0)
|
|
468
583
|
return; // restart disabled
|
|
469
584
|
const scheduleNext = () => {
|
|
585
|
+
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
586
|
+
return;
|
|
470
587
|
this.healthCheckTimer = setTimeout(async () => {
|
|
588
|
+
this.healthCheckTimer = null;
|
|
589
|
+
if (this.runtimeMonitorsFrozen)
|
|
590
|
+
return;
|
|
471
591
|
// Instance directory removed externally (e.g. `rm -rf ~/.agend/instances/<name>`).
|
|
472
592
|
// Stop the loop permanently — otherwise every tick triggers a respawn, whose
|
|
473
593
|
// writeRotationSnapshot fails with ENOENT and gets caught as "Failed to respawn",
|
|
@@ -485,6 +605,12 @@ export class Daemon extends EventEmitter {
|
|
|
485
605
|
// Human-readable backend label for logs (e.g. "claude", "kiro-cli")
|
|
486
606
|
const cliLabel = this.backend?.binaryName ?? "CLI";
|
|
487
607
|
let paneStatus = await this.tmux.getPaneStatus();
|
|
608
|
+
// Auto-pause intentionally exits the pane process. A health tick that
|
|
609
|
+
// began just before pause must not classify that exit as a crash.
|
|
610
|
+
if (this.isPaused || this.pauseWakeState === "waking") {
|
|
611
|
+
scheduleNext();
|
|
612
|
+
return;
|
|
613
|
+
}
|
|
488
614
|
if (paneStatus?.alive) {
|
|
489
615
|
scheduleNext();
|
|
490
616
|
return;
|
|
@@ -712,6 +838,8 @@ export class Daemon extends EventEmitter {
|
|
|
712
838
|
* (ready pattern visible), it goes back to monitoring for new errors.
|
|
713
839
|
*/
|
|
714
840
|
startErrorMonitor() {
|
|
841
|
+
if (this.runtimeMonitorsFrozen || this.errorMonitorTimer)
|
|
842
|
+
return;
|
|
715
843
|
const patterns = this.backend?.getErrorPatterns?.() ?? [];
|
|
716
844
|
const dialogs = this.backend?.getRuntimeDialogs?.() ?? [];
|
|
717
845
|
if (!patterns.length && !dialogs.length)
|
|
@@ -831,26 +959,13 @@ export class Daemon extends EventEmitter {
|
|
|
831
959
|
}
|
|
832
960
|
async stop() {
|
|
833
961
|
this.logger.info("Stopping daemon instance");
|
|
834
|
-
|
|
835
|
-
clearTimeout(this.healthCheckTimer);
|
|
836
|
-
this.healthCheckTimer = null;
|
|
837
|
-
}
|
|
838
|
-
if (this.errorMonitorTimer) {
|
|
839
|
-
clearInterval(this.errorMonitorTimer);
|
|
840
|
-
this.errorMonitorTimer = null;
|
|
841
|
-
}
|
|
842
|
-
if (this.instanceStateMonitorTimer) {
|
|
843
|
-
clearInterval(this.instanceStateMonitorTimer);
|
|
844
|
-
this.instanceStateMonitorTimer = null;
|
|
845
|
-
}
|
|
962
|
+
this.freezeRuntimeMonitors();
|
|
846
963
|
if (this.toolStatusDebounce) {
|
|
847
964
|
clearTimeout(this.toolStatusDebounce);
|
|
848
965
|
this.toolStatusDebounce = null;
|
|
849
966
|
}
|
|
850
967
|
this.pendingIpcRequests.clear();
|
|
851
968
|
this.hangDetector?.stop();
|
|
852
|
-
this.transcriptMonitor?.stop();
|
|
853
|
-
this.guardian?.stop();
|
|
854
969
|
if (this.adapter)
|
|
855
970
|
await this.adapter.stop();
|
|
856
971
|
// Notify MCP servers of graceful shutdown (prevents reconnect attempts)
|
|
@@ -862,12 +977,19 @@ export class Daemon extends EventEmitter {
|
|
|
862
977
|
this.healthCheckPaused = true;
|
|
863
978
|
let killed = false;
|
|
864
979
|
const quitCmd = this.backend?.getQuitCommand();
|
|
980
|
+
const quitKey = this.backend?.getQuitKey?.();
|
|
865
981
|
if (quitCmd) {
|
|
866
982
|
await this.tmux.sendKeys(quitCmd);
|
|
867
983
|
// Delay before Enter to prevent tmux server race when multiple
|
|
868
984
|
// instances stop in parallel (same pattern as pasteText).
|
|
869
985
|
await new Promise(r => setTimeout(r, 150));
|
|
870
986
|
await this.tmux.sendSpecialKey("Enter");
|
|
987
|
+
}
|
|
988
|
+
else if (quitKey) {
|
|
989
|
+
// Some CLIs quit via a key chord (e.g. grok Ctrl+Q), not a typed command.
|
|
990
|
+
await this.tmux.sendSpecialKey(quitKey);
|
|
991
|
+
}
|
|
992
|
+
if (quitCmd || quitKey) {
|
|
871
993
|
// Wait up to 3s for graceful exit, polling every 200ms. A healthy CLI
|
|
872
994
|
// exits within ~1s; a longer wait just delays the force-kill fallback.
|
|
873
995
|
for (let i = 0; i < 15; i++) {
|
|
@@ -909,12 +1031,16 @@ export class Daemon extends EventEmitter {
|
|
|
909
1031
|
catch (e) {
|
|
910
1032
|
this.logger.debug({ err: e }, "Failed to remove PID file");
|
|
911
1033
|
}
|
|
1034
|
+
try {
|
|
1035
|
+
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
1036
|
+
}
|
|
1037
|
+
catch { }
|
|
912
1038
|
}
|
|
913
1039
|
getHangDetector() {
|
|
914
1040
|
return this.hangDetector;
|
|
915
1041
|
}
|
|
916
1042
|
getInstanceState() {
|
|
917
|
-
return this.instanceState;
|
|
1043
|
+
return this.isPaused ? "paused" : this.instanceState;
|
|
918
1044
|
}
|
|
919
1045
|
getInstanceStateSnapshot() {
|
|
920
1046
|
return this.instanceStateMachine?.snapshot() ?? {
|
|
@@ -924,8 +1050,149 @@ export class Daemon extends EventEmitter {
|
|
|
924
1050
|
stateChangedAt: Date.now(),
|
|
925
1051
|
};
|
|
926
1052
|
}
|
|
1053
|
+
/** Gracefully stop the CLI while keeping its remain-on-exit tmux window. */
|
|
1054
|
+
async pause() {
|
|
1055
|
+
if (this.pauseWakeState === "paused")
|
|
1056
|
+
return;
|
|
1057
|
+
if (this.pauseWakeState === "pausing" || this.pauseWakeState === "waking") {
|
|
1058
|
+
await this.pauseWakeTransition;
|
|
1059
|
+
if (this.getPauseWakeState() !== "active")
|
|
1060
|
+
return;
|
|
1061
|
+
}
|
|
1062
|
+
if (this.instanceState !== "idle" || this.pasteQueueDepth > 0) {
|
|
1063
|
+
this.pauseRequested = false;
|
|
1064
|
+
return;
|
|
1065
|
+
}
|
|
1066
|
+
this.pauseWakeState = "pausing";
|
|
1067
|
+
this.healthCheckPaused = true;
|
|
1068
|
+
this.freezeRuntimeMonitors();
|
|
1069
|
+
const transition = (async () => {
|
|
1070
|
+
try {
|
|
1071
|
+
this.saveSessionId();
|
|
1072
|
+
const quitCmd = this.backend?.getQuitCommand();
|
|
1073
|
+
const quitKey = this.backend?.getQuitKey?.();
|
|
1074
|
+
if (this.tmux) {
|
|
1075
|
+
if (quitCmd) {
|
|
1076
|
+
await this.tmux.sendKeys(quitCmd);
|
|
1077
|
+
await new Promise(r => setTimeout(r, 150));
|
|
1078
|
+
await this.tmux.sendSpecialKey("Enter");
|
|
1079
|
+
}
|
|
1080
|
+
else if (quitKey) {
|
|
1081
|
+
// Key-chord quit (e.g. grok Ctrl+Q) — no typed command to send.
|
|
1082
|
+
await this.tmux.sendSpecialKey(quitKey);
|
|
1083
|
+
}
|
|
1084
|
+
}
|
|
1085
|
+
let exited = false;
|
|
1086
|
+
for (let i = 0; i < 15; i++) {
|
|
1087
|
+
await new Promise(r => setTimeout(r, 200));
|
|
1088
|
+
const status = await this.tmux?.getPaneStatus();
|
|
1089
|
+
if (status && !status.alive) {
|
|
1090
|
+
exited = true;
|
|
1091
|
+
break;
|
|
1092
|
+
}
|
|
1093
|
+
}
|
|
1094
|
+
if (!exited) {
|
|
1095
|
+
await this.killProcessTree("SIGTERM");
|
|
1096
|
+
await new Promise(r => setTimeout(r, 1_000));
|
|
1097
|
+
const status = await this.tmux?.getPaneStatus();
|
|
1098
|
+
if (status?.alive) {
|
|
1099
|
+
await this.killProcessTree("SIGKILL");
|
|
1100
|
+
await new Promise(r => setTimeout(r, 200));
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
const finalStatus = await this.tmux?.getPaneStatus();
|
|
1104
|
+
if (!finalStatus || finalStatus.alive) {
|
|
1105
|
+
throw new Error("Auto-pause could not stop the CLI while preserving its tmux window");
|
|
1106
|
+
}
|
|
1107
|
+
this.pauseWakeState = "paused";
|
|
1108
|
+
this.autoPauseController.markPaused();
|
|
1109
|
+
writeFileSync(join(this.instanceDir, "paused-state.json"), JSON.stringify({
|
|
1110
|
+
paused_at: this.lastPausedAt,
|
|
1111
|
+
}));
|
|
1112
|
+
this.logger.info({ pausedAt: this.lastPausedAt }, "Instance auto-paused");
|
|
1113
|
+
this.ipcServer?.broadcast({
|
|
1114
|
+
type: "instance_state", instanceName: this.name, state: "paused", pausedAt: this.lastPausedAt,
|
|
1115
|
+
});
|
|
1116
|
+
this.emit("auto_paused", { name: this.name, pausedAt: this.lastPausedAt });
|
|
1117
|
+
}
|
|
1118
|
+
catch (err) {
|
|
1119
|
+
this.pauseWakeState = "active";
|
|
1120
|
+
this.healthCheckPaused = false;
|
|
1121
|
+
this.pauseRequested = false;
|
|
1122
|
+
this.resumeRuntimeMonitors();
|
|
1123
|
+
throw err;
|
|
1124
|
+
}
|
|
1125
|
+
})();
|
|
1126
|
+
this.pauseWakeTransition = transition;
|
|
1127
|
+
try {
|
|
1128
|
+
await transition;
|
|
1129
|
+
}
|
|
1130
|
+
finally {
|
|
1131
|
+
if (this.pauseWakeTransition === transition)
|
|
1132
|
+
this.pauseWakeTransition = null;
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
/** Respawn the CLI in the preserved window and block until its prompt is ready. */
|
|
1136
|
+
async wake(timeoutMs = 30_000) {
|
|
1137
|
+
if (this.pauseWakeState === "active")
|
|
1138
|
+
return;
|
|
1139
|
+
if (this.pauseWakeState === "pausing")
|
|
1140
|
+
await this.pauseWakeTransition;
|
|
1141
|
+
if (this.getPauseWakeState() === "active")
|
|
1142
|
+
return;
|
|
1143
|
+
if (this.pauseWakeState === "waking") {
|
|
1144
|
+
await this.pauseWakeTransition;
|
|
1145
|
+
return;
|
|
1146
|
+
}
|
|
1147
|
+
this.pauseWakeState = "waking";
|
|
1148
|
+
this.spawning = true;
|
|
1149
|
+
const transition = this.autoPauseController.wakeOnDeliver(async () => {
|
|
1150
|
+
let timeout;
|
|
1151
|
+
try {
|
|
1152
|
+
const ready = await Promise.race([
|
|
1153
|
+
this.trySpawn(true, timeoutMs),
|
|
1154
|
+
new Promise(resolve => { timeout = setTimeout(() => resolve(false), timeoutMs); }),
|
|
1155
|
+
]);
|
|
1156
|
+
if (!ready)
|
|
1157
|
+
throw new Error(`Wake timed out before CLI became ready (${timeoutMs}ms)`);
|
|
1158
|
+
}
|
|
1159
|
+
finally {
|
|
1160
|
+
if (timeout)
|
|
1161
|
+
clearTimeout(timeout);
|
|
1162
|
+
}
|
|
1163
|
+
});
|
|
1164
|
+
this.pauseWakeTransition = transition;
|
|
1165
|
+
try {
|
|
1166
|
+
await transition;
|
|
1167
|
+
this.pauseWakeState = "active";
|
|
1168
|
+
this.healthCheckPaused = false;
|
|
1169
|
+
this.pauseRequested = false;
|
|
1170
|
+
try {
|
|
1171
|
+
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
1172
|
+
}
|
|
1173
|
+
catch { }
|
|
1174
|
+
this.transcriptMonitor?.resetOffset();
|
|
1175
|
+
this.resumeRuntimeMonitors();
|
|
1176
|
+
this.logger.info("Instance auto-woke");
|
|
1177
|
+
this.ipcServer?.broadcast({
|
|
1178
|
+
type: "instance_state", instanceName: this.name, state: this.instanceState, pausedAt: null,
|
|
1179
|
+
});
|
|
1180
|
+
this.emit("auto_woke", { name: this.name });
|
|
1181
|
+
}
|
|
1182
|
+
catch (err) {
|
|
1183
|
+
this.pauseWakeState = "paused";
|
|
1184
|
+
this.healthCheckPaused = true;
|
|
1185
|
+
this.logger.error({ err: err.message }, "Instance wake failed");
|
|
1186
|
+
throw err;
|
|
1187
|
+
}
|
|
1188
|
+
finally {
|
|
1189
|
+
this.spawning = false;
|
|
1190
|
+
if (this.pauseWakeTransition === transition)
|
|
1191
|
+
this.pauseWakeTransition = null;
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
927
1194
|
startInstanceStateMonitor() {
|
|
928
|
-
if (!this.tmux || !this.backend || this.instanceStateMonitorTimer)
|
|
1195
|
+
if (this.runtimeMonitorsFrozen || !this.tmux || !this.backend || this.instanceStateMonitorTimer)
|
|
929
1196
|
return;
|
|
930
1197
|
const rawConfig = this.config.hang_detector;
|
|
931
1198
|
const timeoutMinutes = rawConfig?.timeout_minutes;
|
|
@@ -935,7 +1202,8 @@ export class Daemon extends EventEmitter {
|
|
|
935
1202
|
const pollIntervalMs = typeof rawConfig?.poll_interval_ms === "number" && rawConfig.poll_interval_ms > 0
|
|
936
1203
|
? rawConfig.poll_interval_ms
|
|
937
1204
|
: DEFAULT_STATE_POLL_INTERVAL_MS;
|
|
938
|
-
|
|
1205
|
+
const readyPattern = this.backend.getReadyPattern();
|
|
1206
|
+
this.instanceStateMachine = new PaneStateMachine(readyPattern, stuckTimeoutMs);
|
|
939
1207
|
const poll = async () => {
|
|
940
1208
|
if (!this.tmux || this.spawning || this.statePollInFlight)
|
|
941
1209
|
return;
|
|
@@ -948,6 +1216,11 @@ export class Daemon extends EventEmitter {
|
|
|
948
1216
|
const previous = this.instanceState;
|
|
949
1217
|
const snapshot = this.instanceStateMachine.observe(pane);
|
|
950
1218
|
this.instanceState = snapshot.state;
|
|
1219
|
+
// Only a transition back to idle completes pending work. Repeated idle
|
|
1220
|
+
// polls between enqueue and paste must not clear a newly-recorded inbound.
|
|
1221
|
+
if (snapshot.state === "idle" && previous !== "idle") {
|
|
1222
|
+
this.pendingWork.recordIdle(snapshot.observedAt);
|
|
1223
|
+
}
|
|
951
1224
|
if (snapshot.state !== previous) {
|
|
952
1225
|
this.logger.info({
|
|
953
1226
|
previousState: previous,
|
|
@@ -959,9 +1232,15 @@ export class Daemon extends EventEmitter {
|
|
|
959
1232
|
// Emit exactly once per transition into stuck. Further notifications
|
|
960
1233
|
// require observable progress (working) or a ready prompt (idle) first.
|
|
961
1234
|
if (snapshot.state === "stuck") {
|
|
962
|
-
this.
|
|
1235
|
+
this.handleStuckTransition(pane, snapshot, readyPattern);
|
|
963
1236
|
}
|
|
964
1237
|
}
|
|
1238
|
+
if (snapshot.state !== "idle")
|
|
1239
|
+
this.pauseRequested = false;
|
|
1240
|
+
if (!this.pauseRequested && this.pasteQueueDepth === 0 && this.autoPauseController.observe(snapshot.state)) {
|
|
1241
|
+
this.pauseRequested = true;
|
|
1242
|
+
this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
|
|
1243
|
+
}
|
|
965
1244
|
}
|
|
966
1245
|
catch (err) {
|
|
967
1246
|
// A pane can disappear between status and capture during restart. Keep
|
|
@@ -975,6 +1254,54 @@ export class Daemon extends EventEmitter {
|
|
|
975
1254
|
void poll();
|
|
976
1255
|
this.instanceStateMonitorTimer = setInterval(() => { void poll(); }, pollIntervalMs);
|
|
977
1256
|
}
|
|
1257
|
+
handleStuckTransition(pane, snapshot, readyPattern) {
|
|
1258
|
+
const deterministicReadyPattern = new RegExp(readyPattern.source, readyPattern.flags.replace(/[gy]/g, ""));
|
|
1259
|
+
const diagnostic = {
|
|
1260
|
+
backend: this.backend?.binaryName ?? this.config.backend ?? "unknown",
|
|
1261
|
+
paneTail: sanitizePaneTail(pane),
|
|
1262
|
+
readyPattern: readyPattern.toString(),
|
|
1263
|
+
readyMatched: deterministicReadyPattern.test(pane),
|
|
1264
|
+
unchangedForMs: snapshot.unchangedForMs,
|
|
1265
|
+
pendingWork: this.pendingWork.hasPendingWork(),
|
|
1266
|
+
};
|
|
1267
|
+
if (!diagnostic.pendingWork) {
|
|
1268
|
+
this.logger.debug(diagnostic, "Suppressing stuck notification without pending work");
|
|
1269
|
+
return;
|
|
1270
|
+
}
|
|
1271
|
+
this.logger.warn(diagnostic, "Instance pane stuck with pending work");
|
|
1272
|
+
this.hangDetector?.emit("hang", { unchangedForMs: snapshot.unchangedForMs });
|
|
1273
|
+
}
|
|
1274
|
+
/** Stop every runtime poller/watcher while preserving IPC and daemon state. */
|
|
1275
|
+
freezeRuntimeMonitors() {
|
|
1276
|
+
this.runtimeMonitorsFrozen = true;
|
|
1277
|
+
if (this.healthCheckTimer) {
|
|
1278
|
+
clearTimeout(this.healthCheckTimer);
|
|
1279
|
+
this.healthCheckTimer = null;
|
|
1280
|
+
}
|
|
1281
|
+
if (this.errorMonitorTimer) {
|
|
1282
|
+
clearInterval(this.errorMonitorTimer);
|
|
1283
|
+
this.errorMonitorTimer = null;
|
|
1284
|
+
}
|
|
1285
|
+
if (this.instanceStateMonitorTimer) {
|
|
1286
|
+
clearInterval(this.instanceStateMonitorTimer);
|
|
1287
|
+
this.instanceStateMonitorTimer = null;
|
|
1288
|
+
}
|
|
1289
|
+
this.transcriptMonitor?.stop();
|
|
1290
|
+
this.guardian?.stop();
|
|
1291
|
+
}
|
|
1292
|
+
/** Restore the same monitor objects after wake without adding event listeners. */
|
|
1293
|
+
resumeRuntimeMonitors() {
|
|
1294
|
+
if (!this.runtimeMonitorsFrozen)
|
|
1295
|
+
return;
|
|
1296
|
+
this.runtimeMonitorsFrozen = false;
|
|
1297
|
+
this.startHealthCheck();
|
|
1298
|
+
if (!this.config.lightweight) {
|
|
1299
|
+
this.transcriptMonitor?.startPolling();
|
|
1300
|
+
this.guardian?.startWatching();
|
|
1301
|
+
this.startErrorMonitor();
|
|
1302
|
+
}
|
|
1303
|
+
this.startInstanceStateMonitor();
|
|
1304
|
+
}
|
|
978
1305
|
getMessageBus() {
|
|
979
1306
|
return this.messageBus;
|
|
980
1307
|
}
|
|
@@ -1053,12 +1380,13 @@ export class Daemon extends EventEmitter {
|
|
|
1053
1380
|
// Remember (and persist) the reply target. Only real channel messages have a
|
|
1054
1381
|
// non-empty chat_id; cross-instance messages have chat_id="" and must NOT
|
|
1055
1382
|
// overwrite it (their reply would otherwise go nowhere).
|
|
1056
|
-
this.updateLastChat(meta.chat_id, meta.thread_id);
|
|
1383
|
+
this.updateLastChat(meta.chat_id, meta.thread_id, meta.adapter_id);
|
|
1057
1384
|
if (this.pendingInstructionsUpdate) {
|
|
1058
1385
|
writeFileSync(join(this.instanceDir, "prev-instructions"), this.pendingInstructionsUpdate);
|
|
1059
1386
|
this.pendingInstructionsUpdate = undefined;
|
|
1060
1387
|
}
|
|
1061
1388
|
this.hangDetector?.recordInbound();
|
|
1389
|
+
this.pendingWork.recordInbound();
|
|
1062
1390
|
// v3: record user messages for rotation snapshot
|
|
1063
1391
|
this.recordRecentUserMessage(content, meta);
|
|
1064
1392
|
// Format message with metadata prefix for the agent
|
|
@@ -1632,14 +1960,14 @@ export class Daemon extends EventEmitter {
|
|
|
1632
1960
|
return resumedSuccessfully;
|
|
1633
1961
|
}
|
|
1634
1962
|
/** Kill the entire process tree of the current tmux pane (CLI + MCP server). */
|
|
1635
|
-
async killProcessTree() {
|
|
1963
|
+
async killProcessTree(signal = "SIGTERM") {
|
|
1636
1964
|
if (!this.tmux)
|
|
1637
1965
|
return;
|
|
1638
1966
|
try {
|
|
1639
1967
|
const pid = await TmuxManager.getPanePid(this.tmuxSessionName, this.tmux.getWindowId());
|
|
1640
1968
|
if (pid) {
|
|
1641
|
-
process.kill(-pid,
|
|
1642
|
-
this.logger.debug({ pid }, "Killed process group");
|
|
1969
|
+
process.kill(-pid, signal);
|
|
1970
|
+
this.logger.debug({ pid, signal }, "Killed process group");
|
|
1643
1971
|
}
|
|
1644
1972
|
}
|
|
1645
1973
|
catch { /* process group may not exist or already dead */ }
|
|
@@ -1650,7 +1978,7 @@ export class Daemon extends EventEmitter {
|
|
|
1650
1978
|
* Handles confirmation dialogs (trust folder, bypass permissions).
|
|
1651
1979
|
* Returns true if CLI is ready, false if it failed or got stuck.
|
|
1652
1980
|
*/
|
|
1653
|
-
async trySpawn() {
|
|
1981
|
+
async trySpawn(reuseWindow = false, startupTimeoutMs) {
|
|
1654
1982
|
const backendConfig = this.buildBackendConfig();
|
|
1655
1983
|
// Compare freshly-built instructions against the last value the agent was
|
|
1656
1984
|
// told about. Computed for ALL backends (not gated by
|
|
@@ -1713,16 +2041,29 @@ export class Daemon extends EventEmitter {
|
|
|
1713
2041
|
const cmd = `${envPrefix} ` + this.backend.buildCommand(backendConfig);
|
|
1714
2042
|
// Ensure tmux session exists (may have been destroyed if all windows died)
|
|
1715
2043
|
await TmuxManager.ensureSession(this.tmuxSessionName);
|
|
1716
|
-
|
|
2044
|
+
let windowId;
|
|
2045
|
+
if (reuseWindow) {
|
|
2046
|
+
this.controlClient?.unregisterWindow(this.tmux.getWindowId());
|
|
2047
|
+
await this.tmux.respawnWindow(cmd, resolvedCwd);
|
|
2048
|
+
windowId = this.tmux.getWindowId();
|
|
2049
|
+
}
|
|
2050
|
+
else {
|
|
2051
|
+
windowId = await this.tmux.createWindow(cmd, resolvedCwd, this.name);
|
|
2052
|
+
}
|
|
1717
2053
|
writeFileSync(join(this.instanceDir, "window-id"), windowId);
|
|
1718
2054
|
// Enable remain-on-exit to capture exit codes on crash
|
|
1719
2055
|
await this.tmux.setRemainOnExit().catch(err => {
|
|
1720
2056
|
this.logger.warn({ err }, "Failed to set remain-on-exit — exit codes will not be captured");
|
|
1721
2057
|
});
|
|
2058
|
+
if (reuseWindow && !this.config.lightweight) {
|
|
2059
|
+
await this.tmux.pipeOutput(join(this.instanceDir, "output.log")).catch(err => {
|
|
2060
|
+
this.logger.warn({ err }, "Failed to restore pipe-pane after wake");
|
|
2061
|
+
});
|
|
2062
|
+
}
|
|
1722
2063
|
// Register with control client and wait for output + idle
|
|
1723
2064
|
await this.controlClient?.registerWindow(windowId);
|
|
1724
2065
|
if (this.controlClient) {
|
|
1725
|
-
const total = this.config.startup_timeout_ms ?? 25_000;
|
|
2066
|
+
const total = startupTimeoutMs ?? this.config.startup_timeout_ms ?? 25_000;
|
|
1726
2067
|
const outputTimeout = Math.round(total * 0.6);
|
|
1727
2068
|
const idleTimeout = total - outputTimeout;
|
|
1728
2069
|
const hasOutput = await this.controlClient.waitForOutput(windowId, outputTimeout);
|
|
@@ -1812,14 +2153,17 @@ export class Daemon extends EventEmitter {
|
|
|
1812
2153
|
* messages) so it never overwrites a real channel target. Persisted to
|
|
1813
2154
|
* last-chat.json so the reply target survives a restart (see start()).
|
|
1814
2155
|
*/
|
|
1815
|
-
updateLastChat(chatId, threadId) {
|
|
2156
|
+
updateLastChat(chatId, threadId, adapterId) {
|
|
1816
2157
|
if (!chatId)
|
|
1817
2158
|
return;
|
|
1818
2159
|
this.lastChatId = chatId;
|
|
1819
|
-
|
|
1820
|
-
|
|
2160
|
+
// An unthreaded inbound must clear a previous topic rather than leaking it
|
|
2161
|
+
// into the next reply target.
|
|
2162
|
+
this.lastThreadId = threadId || undefined;
|
|
2163
|
+
if (adapterId)
|
|
2164
|
+
this.lastAdapterId = adapterId;
|
|
1821
2165
|
try {
|
|
1822
|
-
writeFileSync(join(this.instanceDir, "last-chat.json"), JSON.stringify({ chatId: this.lastChatId, threadId: this.lastThreadId }));
|
|
2166
|
+
writeFileSync(join(this.instanceDir, "last-chat.json"), JSON.stringify({ chatId: this.lastChatId, threadId: this.lastThreadId, adapterId: this.lastAdapterId }));
|
|
1823
2167
|
}
|
|
1824
2168
|
catch { /* best effort */ }
|
|
1825
2169
|
}
|