@songsid/agend 2.1.0-beta.18 → 2.1.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -1
- package/dist/access-path.js +3 -3
- package/dist/access-path.js.map +1 -1
- package/dist/agent-endpoint.d.ts +0 -8
- package/dist/agent-endpoint.js +8 -36
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.js +2 -3
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/factory.js +1 -5
- package/dist/backend/factory.js.map +1 -1
- package/dist/backend/kiro.js +2 -3
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +2 -12
- package/dist/backend/types.js +0 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/cli.js +6 -56
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +1 -50
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +1 -3
- package/dist/config.js +0 -13
- package/dist/config.js.map +1 -1
- package/dist/context-guardian.d.ts +0 -2
- package/dist/context-guardian.js +2 -10
- package/dist/context-guardian.js.map +1 -1
- package/dist/daemon-entry.js +1 -2
- package/dist/daemon-entry.js.map +1 -1
- package/dist/daemon.d.ts +3 -51
- package/dist/daemon.js +36 -380
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-context.d.ts +1 -2
- package/dist/fleet-manager.d.ts +5 -23
- package/dist/fleet-manager.js +123 -229
- package/dist/fleet-manager.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +1 -6
- package/dist/instance-lifecycle.js +3 -45
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/logger.d.ts +0 -6
- package/dist/logger.js +10 -29
- package/dist/logger.js.map +1 -1
- package/dist/outbound-handlers.d.ts +0 -5
- package/dist/outbound-handlers.js +16 -68
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/settings-api.d.ts +2 -17
- package/dist/settings-api.js +8 -105
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +0 -8
- package/dist/setup-wizard.js.map +1 -1
- package/dist/statusline-watcher.d.ts +1 -1
- package/dist/statusline-watcher.js +2 -3
- package/dist/statusline-watcher.js.map +1 -1
- package/dist/tmux-manager.d.ts +1 -3
- package/dist/tmux-manager.js +0 -9
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-archiver.d.ts +1 -1
- package/dist/topic-commands.d.ts +1 -14
- package/dist/topic-commands.js +25 -96
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-monitor.js +0 -2
- package/dist/transcript-monitor.js.map +1 -1
- package/dist/types.d.ts +1 -18
- package/dist/ui/dashboard.html +2 -2
- package/dist/ui/settings.html +40 -251
- package/dist/view-api.d.ts +1 -1
- package/dist/web-api.d.ts +1 -2
- package/dist/web-api.js +14 -23
- package/dist/web-api.js.map +1 -1
- package/package.json +1 -2
- package/dist/backend/grok.d.ts +0 -40
- package/dist/backend/grok.js +0 -213
- package/dist/backend/grok.js.map +0 -1
package/dist/daemon.js
CHANGED
|
@@ -3,6 +3,7 @@ import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync,
|
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import { createHash, randomBytes } from "node:crypto";
|
|
5
5
|
import { EventEmitter } from "node:events";
|
|
6
|
+
import { createLogger } from "./logger.js";
|
|
6
7
|
import { TmuxManager } from "./tmux-manager.js";
|
|
7
8
|
import { TranscriptMonitor } from "./transcript-monitor.js";
|
|
8
9
|
import { ContextGuardian } from "./context-guardian.js";
|
|
@@ -22,40 +23,6 @@ const DECISION_TOOLS = new Set(["post_decision", "list_decisions", "update_decis
|
|
|
22
23
|
const TASK_TOOL = "task";
|
|
23
24
|
export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
|
|
24
25
|
export const DEFAULT_STATE_POLL_INTERVAL_MS = 5_000;
|
|
25
|
-
/** Headless idle timer used by the daemon and unit tests. */
|
|
26
|
-
export class AutoPauseController {
|
|
27
|
-
thresholdMs;
|
|
28
|
-
idleSince = null;
|
|
29
|
-
pausedAt = null;
|
|
30
|
-
constructor(thresholdMs) {
|
|
31
|
-
this.thresholdMs = thresholdMs;
|
|
32
|
-
}
|
|
33
|
-
observe(state, now = Date.now()) {
|
|
34
|
-
if (this.pausedAt !== null || this.thresholdMs <= 0)
|
|
35
|
-
return false;
|
|
36
|
-
if (state !== "idle") {
|
|
37
|
-
this.idleSince = null;
|
|
38
|
-
return false;
|
|
39
|
-
}
|
|
40
|
-
this.idleSince ??= now;
|
|
41
|
-
return now - this.idleSince >= this.thresholdMs;
|
|
42
|
-
}
|
|
43
|
-
markPaused(now = Date.now()) {
|
|
44
|
-
this.pausedAt = now;
|
|
45
|
-
}
|
|
46
|
-
markAwake() {
|
|
47
|
-
this.pausedAt = null;
|
|
48
|
-
this.idleSince = null;
|
|
49
|
-
}
|
|
50
|
-
async wakeOnDeliver(wake) {
|
|
51
|
-
if (this.pausedAt === null)
|
|
52
|
-
return;
|
|
53
|
-
await wake();
|
|
54
|
-
this.markAwake();
|
|
55
|
-
}
|
|
56
|
-
get isPaused() { return this.pausedAt !== null; }
|
|
57
|
-
get lastPausedAt() { return this.pausedAt; }
|
|
58
|
-
}
|
|
59
26
|
/**
|
|
60
27
|
* Headless state machine for pane-based execution state detection.
|
|
61
28
|
*
|
|
@@ -114,55 +81,6 @@ export class PaneStateMachine {
|
|
|
114
81
|
};
|
|
115
82
|
}
|
|
116
83
|
}
|
|
117
|
-
/** Tracks whether an inbound arrived after the most recent confirmed idle prompt. */
|
|
118
|
-
export class PendingWorkTracker {
|
|
119
|
-
lastInboundAt = 0;
|
|
120
|
-
lastIdleAt;
|
|
121
|
-
sequence = 0;
|
|
122
|
-
lastInboundOrder = 0;
|
|
123
|
-
lastIdleOrder = 0;
|
|
124
|
-
constructor(now = Date.now()) {
|
|
125
|
-
this.lastIdleAt = now;
|
|
126
|
-
}
|
|
127
|
-
recordInbound(now = Date.now()) {
|
|
128
|
-
this.lastInboundAt = now;
|
|
129
|
-
this.lastInboundOrder = ++this.sequence;
|
|
130
|
-
}
|
|
131
|
-
recordIdle(now = Date.now()) {
|
|
132
|
-
// An async pane poll can finish after a newer inbound. Do not let its stale
|
|
133
|
-
// observation clear work which had not arrived when the pane was captured.
|
|
134
|
-
if (now < this.lastInboundAt)
|
|
135
|
-
return;
|
|
136
|
-
this.lastIdleAt = now;
|
|
137
|
-
this.lastIdleOrder = ++this.sequence;
|
|
138
|
-
}
|
|
139
|
-
hasPendingWork() {
|
|
140
|
-
return this.lastInboundOrder > this.lastIdleOrder;
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
/** Redact likely credentials and control sequences before pane text reaches logs. */
|
|
144
|
-
export function sanitizePaneTail(pane, lineCount = 5) {
|
|
145
|
-
const secretAssignment = /\b(token|secret|password|passwd|api[_-]?key|authorization)\b\s*[:=]\s*\S+/gi;
|
|
146
|
-
const bearer = /\bBearer\s+\S+/gi;
|
|
147
|
-
const knownToken = /\b(?:sk-[A-Za-z0-9_-]+|ghp_[A-Za-z0-9]+|github_pat_[A-Za-z0-9_]+|AKIA[A-Z0-9]{16})\b/g;
|
|
148
|
-
const jwt = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g;
|
|
149
|
-
const opaqueSecret = /\b[A-Za-z0-9_+/=-]{32,}\b/g;
|
|
150
|
-
const lines = pane
|
|
151
|
-
.replace(/\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g, "")
|
|
152
|
-
.split(/\r?\n/);
|
|
153
|
-
while (lines.length > 0 && /^\s*$/.test(lines[lines.length - 1]))
|
|
154
|
-
lines.pop();
|
|
155
|
-
return lines
|
|
156
|
-
.slice(-lineCount)
|
|
157
|
-
.map(line => line
|
|
158
|
-
.replace(bearer, "Bearer [REDACTED]")
|
|
159
|
-
.replace(secretAssignment, "$1=[REDACTED]")
|
|
160
|
-
.replace(knownToken, "[REDACTED]")
|
|
161
|
-
.replace(jwt, "[REDACTED]")
|
|
162
|
-
.replace(opaqueSecret, "[REDACTED]")
|
|
163
|
-
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "")
|
|
164
|
-
.slice(0, 200));
|
|
165
|
-
}
|
|
166
84
|
export class Daemon extends EventEmitter {
|
|
167
85
|
name;
|
|
168
86
|
config;
|
|
@@ -183,7 +101,6 @@ export class Daemon extends EventEmitter {
|
|
|
183
101
|
// Track chatId/threadId from inbound messages for automatic outbound routing
|
|
184
102
|
lastChatId;
|
|
185
103
|
lastThreadId;
|
|
186
|
-
lastAdapterId;
|
|
187
104
|
// Pending ack: react 🫡 on first transcript activity after receiving a message
|
|
188
105
|
pendingAckMessage = null;
|
|
189
106
|
// Tool status tracking for channel adapter
|
|
@@ -213,13 +130,8 @@ export class Daemon extends EventEmitter {
|
|
|
213
130
|
hangDetector = null;
|
|
214
131
|
instanceState = "idle";
|
|
215
132
|
instanceStateMachine = null;
|
|
216
|
-
pendingWork = new PendingWorkTracker();
|
|
217
133
|
instanceStateMonitorTimer = null;
|
|
218
134
|
statePollInFlight = false;
|
|
219
|
-
autoPauseController;
|
|
220
|
-
pauseRequested = false;
|
|
221
|
-
pauseWakeState = "active";
|
|
222
|
-
pauseWakeTransition = null;
|
|
223
135
|
// Model failover: override model on next spawn when rate-limited
|
|
224
136
|
modelOverride;
|
|
225
137
|
// Context rotation v3: ring buffers for daemon-side snapshot
|
|
@@ -239,17 +151,12 @@ export class Daemon extends EventEmitter {
|
|
|
239
151
|
pasteQueueDepth = 0;
|
|
240
152
|
// PTY error pattern monitoring
|
|
241
153
|
errorMonitorTimer = null;
|
|
242
|
-
/** Prevent in-flight monitor callbacks from re-arming after a pause. */
|
|
243
|
-
runtimeMonitorsFrozen = false;
|
|
244
154
|
errorWaitingForRecovery = false; // true = error detected, waiting for ready pattern
|
|
245
155
|
errorDetectedAt = 0;
|
|
246
|
-
/** Whether this instance is in an
|
|
156
|
+
/** Whether this instance is in an error state (rate-limited, paused, or crash loop). */
|
|
247
157
|
get isErrorState() {
|
|
248
|
-
return this.errorWaitingForRecovery ||
|
|
158
|
+
return this.errorWaitingForRecovery || this.healthCheckPaused || Daemon.tmuxServerPaused;
|
|
249
159
|
}
|
|
250
|
-
get isPaused() { return this.pauseWakeState !== "active"; }
|
|
251
|
-
get lastPausedAt() { return this.autoPauseController.lastPausedAt; }
|
|
252
|
-
getPauseWakeState() { return this.pauseWakeState; }
|
|
253
160
|
/** Whether this instance is in a crash loop (3+ consecutive crashes). */
|
|
254
161
|
get isCrashLoop() {
|
|
255
162
|
return this.crashCount >= 3;
|
|
@@ -267,7 +174,7 @@ export class Daemon extends EventEmitter {
|
|
|
267
174
|
// still registers as new (prevents the old hash-dedup's permanent suppression).
|
|
268
175
|
lastErrorCount = new Map();
|
|
269
176
|
lastDetectedErrorType = null;
|
|
270
|
-
constructor(name, config, instanceDir, topicMode = false, backend, controlClient
|
|
177
|
+
constructor(name, config, instanceDir, topicMode = false, backend, controlClient) {
|
|
271
178
|
super();
|
|
272
179
|
this.name = name;
|
|
273
180
|
this.config = config;
|
|
@@ -275,23 +182,13 @@ export class Daemon extends EventEmitter {
|
|
|
275
182
|
this.topicMode = topicMode;
|
|
276
183
|
this.backend = backend;
|
|
277
184
|
this.controlClient = controlClient;
|
|
278
|
-
|
|
279
|
-
throw new Error("Daemon requires a shared root logger");
|
|
280
|
-
this.logger = rootLogger.child({ instance: name }, { level: config.log_level });
|
|
185
|
+
this.logger = createLogger(config.log_level);
|
|
281
186
|
this.tmuxSessionName = getTmuxSession();
|
|
282
187
|
this.messageBus = new MessageBus();
|
|
283
188
|
this.messageBus.setLogger(this.logger);
|
|
284
|
-
const autoPauseMinutes = typeof config.auto_pause_after === "number" ? config.auto_pause_after : 0; // default: disabled
|
|
285
|
-
this.autoPauseController = new AutoPauseController(Math.max(0, autoPauseMinutes) * 60_000);
|
|
286
189
|
}
|
|
287
190
|
async start() {
|
|
288
191
|
mkdirSync(this.instanceDir, { recursive: true });
|
|
289
|
-
// A daemon restart performs a normal CLI start, so any persisted auto-pause
|
|
290
|
-
// marker from the previous daemon is stale.
|
|
291
|
-
try {
|
|
292
|
-
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
293
|
-
}
|
|
294
|
-
catch { }
|
|
295
192
|
writeFileSync(join(this.instanceDir, "daemon.pid"), String(process.pid));
|
|
296
193
|
this.logger.info(`Starting ${this.name}`);
|
|
297
194
|
// P1: Read crash state from previous run — skip resume if last run was a crash loop
|
|
@@ -317,7 +214,6 @@ export class Daemon extends EventEmitter {
|
|
|
317
214
|
if (saved.chatId) {
|
|
318
215
|
this.lastChatId = saved.chatId;
|
|
319
216
|
this.lastThreadId = saved.threadId || undefined;
|
|
320
|
-
this.lastAdapterId = saved.adapterId || undefined;
|
|
321
217
|
}
|
|
322
218
|
}
|
|
323
219
|
}
|
|
@@ -397,18 +293,13 @@ export class Daemon extends EventEmitter {
|
|
|
397
293
|
// Fleet manager routed a message to us (topic mode)
|
|
398
294
|
const meta = msg.meta;
|
|
399
295
|
const targetSession = msg.targetSession;
|
|
400
|
-
|
|
401
|
-
this.pushChannelMessage(msg.content, meta, targetSession);
|
|
402
|
-
}).catch(err => {
|
|
403
|
-
this.logger.error({ err: err.message }, "Wake failed for inbound delivery");
|
|
404
|
-
});
|
|
296
|
+
this.pushChannelMessage(msg.content, meta, targetSession);
|
|
405
297
|
}
|
|
406
298
|
else if (msg.type === "raw_paste") {
|
|
407
299
|
// Paste raw text directly to CLI without [user:] wrapping.
|
|
408
300
|
if (this.tmux) {
|
|
409
301
|
const rawText = msg.content;
|
|
410
302
|
this.pasteLock = this.pasteLock.then(async () => {
|
|
411
|
-
await this.wake();
|
|
412
303
|
await this.deliverMessage(rawText);
|
|
413
304
|
this.logger.debug({ text: rawText.slice(0, 100) }, "Raw paste delivered");
|
|
414
305
|
}).catch(err => {
|
|
@@ -419,9 +310,7 @@ export class Daemon extends EventEmitter {
|
|
|
419
310
|
else if (msg.type === "fleet_schedule_trigger") {
|
|
420
311
|
const payload = msg.payload;
|
|
421
312
|
const meta = msg.meta;
|
|
422
|
-
|
|
423
|
-
this.logger.error({ err: err.message }, "Wake failed for scheduled delivery");
|
|
424
|
-
});
|
|
313
|
+
this.pushChannelMessage(payload.message, meta);
|
|
425
314
|
}
|
|
426
315
|
else if (msg.type === "fleet_tool_status_ack") {
|
|
427
316
|
// Fleet manager sent us the messageId for our tool status message
|
|
@@ -434,8 +323,6 @@ export class Daemon extends EventEmitter {
|
|
|
434
323
|
requestId: msg.requestId,
|
|
435
324
|
instanceName: this.name,
|
|
436
325
|
...snapshot,
|
|
437
|
-
state: this.isPaused ? "paused" : snapshot.state,
|
|
438
|
-
pausedAt: this.lastPausedAt,
|
|
439
326
|
});
|
|
440
327
|
}
|
|
441
328
|
});
|
|
@@ -576,18 +463,11 @@ export class Daemon extends EventEmitter {
|
|
|
576
463
|
this.logger.info(`${this.name} ready`);
|
|
577
464
|
}
|
|
578
465
|
startHealthCheck() {
|
|
579
|
-
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
580
|
-
return;
|
|
581
466
|
const { max_retries, backoff, reset_after } = this.config.restart_policy;
|
|
582
467
|
if (max_retries <= 0)
|
|
583
468
|
return; // restart disabled
|
|
584
469
|
const scheduleNext = () => {
|
|
585
|
-
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
586
|
-
return;
|
|
587
470
|
this.healthCheckTimer = setTimeout(async () => {
|
|
588
|
-
this.healthCheckTimer = null;
|
|
589
|
-
if (this.runtimeMonitorsFrozen)
|
|
590
|
-
return;
|
|
591
471
|
// Instance directory removed externally (e.g. `rm -rf ~/.agend/instances/<name>`).
|
|
592
472
|
// Stop the loop permanently — otherwise every tick triggers a respawn, whose
|
|
593
473
|
// writeRotationSnapshot fails with ENOENT and gets caught as "Failed to respawn",
|
|
@@ -605,12 +485,6 @@ export class Daemon extends EventEmitter {
|
|
|
605
485
|
// Human-readable backend label for logs (e.g. "claude", "kiro-cli")
|
|
606
486
|
const cliLabel = this.backend?.binaryName ?? "CLI";
|
|
607
487
|
let paneStatus = await this.tmux.getPaneStatus();
|
|
608
|
-
// Auto-pause intentionally exits the pane process. A health tick that
|
|
609
|
-
// began just before pause must not classify that exit as a crash.
|
|
610
|
-
if (this.isPaused || this.pauseWakeState === "waking") {
|
|
611
|
-
scheduleNext();
|
|
612
|
-
return;
|
|
613
|
-
}
|
|
614
488
|
if (paneStatus?.alive) {
|
|
615
489
|
scheduleNext();
|
|
616
490
|
return;
|
|
@@ -838,8 +712,6 @@ export class Daemon extends EventEmitter {
|
|
|
838
712
|
* (ready pattern visible), it goes back to monitoring for new errors.
|
|
839
713
|
*/
|
|
840
714
|
startErrorMonitor() {
|
|
841
|
-
if (this.runtimeMonitorsFrozen || this.errorMonitorTimer)
|
|
842
|
-
return;
|
|
843
715
|
const patterns = this.backend?.getErrorPatterns?.() ?? [];
|
|
844
716
|
const dialogs = this.backend?.getRuntimeDialogs?.() ?? [];
|
|
845
717
|
if (!patterns.length && !dialogs.length)
|
|
@@ -959,13 +831,26 @@ export class Daemon extends EventEmitter {
|
|
|
959
831
|
}
|
|
960
832
|
async stop() {
|
|
961
833
|
this.logger.info("Stopping daemon instance");
|
|
962
|
-
this.
|
|
834
|
+
if (this.healthCheckTimer) {
|
|
835
|
+
clearTimeout(this.healthCheckTimer);
|
|
836
|
+
this.healthCheckTimer = null;
|
|
837
|
+
}
|
|
838
|
+
if (this.errorMonitorTimer) {
|
|
839
|
+
clearInterval(this.errorMonitorTimer);
|
|
840
|
+
this.errorMonitorTimer = null;
|
|
841
|
+
}
|
|
842
|
+
if (this.instanceStateMonitorTimer) {
|
|
843
|
+
clearInterval(this.instanceStateMonitorTimer);
|
|
844
|
+
this.instanceStateMonitorTimer = null;
|
|
845
|
+
}
|
|
963
846
|
if (this.toolStatusDebounce) {
|
|
964
847
|
clearTimeout(this.toolStatusDebounce);
|
|
965
848
|
this.toolStatusDebounce = null;
|
|
966
849
|
}
|
|
967
850
|
this.pendingIpcRequests.clear();
|
|
968
851
|
this.hangDetector?.stop();
|
|
852
|
+
this.transcriptMonitor?.stop();
|
|
853
|
+
this.guardian?.stop();
|
|
969
854
|
if (this.adapter)
|
|
970
855
|
await this.adapter.stop();
|
|
971
856
|
// Notify MCP servers of graceful shutdown (prevents reconnect attempts)
|
|
@@ -977,19 +862,12 @@ export class Daemon extends EventEmitter {
|
|
|
977
862
|
this.healthCheckPaused = true;
|
|
978
863
|
let killed = false;
|
|
979
864
|
const quitCmd = this.backend?.getQuitCommand();
|
|
980
|
-
const quitKey = this.backend?.getQuitKey?.();
|
|
981
865
|
if (quitCmd) {
|
|
982
866
|
await this.tmux.sendKeys(quitCmd);
|
|
983
867
|
// Delay before Enter to prevent tmux server race when multiple
|
|
984
868
|
// instances stop in parallel (same pattern as pasteText).
|
|
985
869
|
await new Promise(r => setTimeout(r, 150));
|
|
986
870
|
await this.tmux.sendSpecialKey("Enter");
|
|
987
|
-
}
|
|
988
|
-
else if (quitKey) {
|
|
989
|
-
// Some CLIs quit via a key chord (e.g. grok Ctrl+Q), not a typed command.
|
|
990
|
-
await this.tmux.sendSpecialKey(quitKey);
|
|
991
|
-
}
|
|
992
|
-
if (quitCmd || quitKey) {
|
|
993
871
|
// Wait up to 3s for graceful exit, polling every 200ms. A healthy CLI
|
|
994
872
|
// exits within ~1s; a longer wait just delays the force-kill fallback.
|
|
995
873
|
for (let i = 0; i < 15; i++) {
|
|
@@ -1031,16 +909,12 @@ export class Daemon extends EventEmitter {
|
|
|
1031
909
|
catch (e) {
|
|
1032
910
|
this.logger.debug({ err: e }, "Failed to remove PID file");
|
|
1033
911
|
}
|
|
1034
|
-
try {
|
|
1035
|
-
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
1036
|
-
}
|
|
1037
|
-
catch { }
|
|
1038
912
|
}
|
|
1039
913
|
getHangDetector() {
|
|
1040
914
|
return this.hangDetector;
|
|
1041
915
|
}
|
|
1042
916
|
getInstanceState() {
|
|
1043
|
-
return this.
|
|
917
|
+
return this.instanceState;
|
|
1044
918
|
}
|
|
1045
919
|
getInstanceStateSnapshot() {
|
|
1046
920
|
return this.instanceStateMachine?.snapshot() ?? {
|
|
@@ -1050,149 +924,8 @@ export class Daemon extends EventEmitter {
|
|
|
1050
924
|
stateChangedAt: Date.now(),
|
|
1051
925
|
};
|
|
1052
926
|
}
|
|
1053
|
-
/** Gracefully stop the CLI while keeping its remain-on-exit tmux window. */
|
|
1054
|
-
async pause() {
|
|
1055
|
-
if (this.pauseWakeState === "paused")
|
|
1056
|
-
return;
|
|
1057
|
-
if (this.pauseWakeState === "pausing" || this.pauseWakeState === "waking") {
|
|
1058
|
-
await this.pauseWakeTransition;
|
|
1059
|
-
if (this.getPauseWakeState() !== "active")
|
|
1060
|
-
return;
|
|
1061
|
-
}
|
|
1062
|
-
if (this.instanceState !== "idle" || this.pasteQueueDepth > 0) {
|
|
1063
|
-
this.pauseRequested = false;
|
|
1064
|
-
return;
|
|
1065
|
-
}
|
|
1066
|
-
this.pauseWakeState = "pausing";
|
|
1067
|
-
this.healthCheckPaused = true;
|
|
1068
|
-
this.freezeRuntimeMonitors();
|
|
1069
|
-
const transition = (async () => {
|
|
1070
|
-
try {
|
|
1071
|
-
this.saveSessionId();
|
|
1072
|
-
const quitCmd = this.backend?.getQuitCommand();
|
|
1073
|
-
const quitKey = this.backend?.getQuitKey?.();
|
|
1074
|
-
if (this.tmux) {
|
|
1075
|
-
if (quitCmd) {
|
|
1076
|
-
await this.tmux.sendKeys(quitCmd);
|
|
1077
|
-
await new Promise(r => setTimeout(r, 150));
|
|
1078
|
-
await this.tmux.sendSpecialKey("Enter");
|
|
1079
|
-
}
|
|
1080
|
-
else if (quitKey) {
|
|
1081
|
-
// Key-chord quit (e.g. grok Ctrl+Q) — no typed command to send.
|
|
1082
|
-
await this.tmux.sendSpecialKey(quitKey);
|
|
1083
|
-
}
|
|
1084
|
-
}
|
|
1085
|
-
let exited = false;
|
|
1086
|
-
for (let i = 0; i < 15; i++) {
|
|
1087
|
-
await new Promise(r => setTimeout(r, 200));
|
|
1088
|
-
const status = await this.tmux?.getPaneStatus();
|
|
1089
|
-
if (status && !status.alive) {
|
|
1090
|
-
exited = true;
|
|
1091
|
-
break;
|
|
1092
|
-
}
|
|
1093
|
-
}
|
|
1094
|
-
if (!exited) {
|
|
1095
|
-
await this.killProcessTree("SIGTERM");
|
|
1096
|
-
await new Promise(r => setTimeout(r, 1_000));
|
|
1097
|
-
const status = await this.tmux?.getPaneStatus();
|
|
1098
|
-
if (status?.alive) {
|
|
1099
|
-
await this.killProcessTree("SIGKILL");
|
|
1100
|
-
await new Promise(r => setTimeout(r, 200));
|
|
1101
|
-
}
|
|
1102
|
-
}
|
|
1103
|
-
const finalStatus = await this.tmux?.getPaneStatus();
|
|
1104
|
-
if (!finalStatus || finalStatus.alive) {
|
|
1105
|
-
throw new Error("Auto-pause could not stop the CLI while preserving its tmux window");
|
|
1106
|
-
}
|
|
1107
|
-
this.pauseWakeState = "paused";
|
|
1108
|
-
this.autoPauseController.markPaused();
|
|
1109
|
-
writeFileSync(join(this.instanceDir, "paused-state.json"), JSON.stringify({
|
|
1110
|
-
paused_at: this.lastPausedAt,
|
|
1111
|
-
}));
|
|
1112
|
-
this.logger.info({ pausedAt: this.lastPausedAt }, "Instance auto-paused");
|
|
1113
|
-
this.ipcServer?.broadcast({
|
|
1114
|
-
type: "instance_state", instanceName: this.name, state: "paused", pausedAt: this.lastPausedAt,
|
|
1115
|
-
});
|
|
1116
|
-
this.emit("auto_paused", { name: this.name, pausedAt: this.lastPausedAt });
|
|
1117
|
-
}
|
|
1118
|
-
catch (err) {
|
|
1119
|
-
this.pauseWakeState = "active";
|
|
1120
|
-
this.healthCheckPaused = false;
|
|
1121
|
-
this.pauseRequested = false;
|
|
1122
|
-
this.resumeRuntimeMonitors();
|
|
1123
|
-
throw err;
|
|
1124
|
-
}
|
|
1125
|
-
})();
|
|
1126
|
-
this.pauseWakeTransition = transition;
|
|
1127
|
-
try {
|
|
1128
|
-
await transition;
|
|
1129
|
-
}
|
|
1130
|
-
finally {
|
|
1131
|
-
if (this.pauseWakeTransition === transition)
|
|
1132
|
-
this.pauseWakeTransition = null;
|
|
1133
|
-
}
|
|
1134
|
-
}
|
|
1135
|
-
/** Respawn the CLI in the preserved window and block until its prompt is ready. */
|
|
1136
|
-
async wake(timeoutMs = 30_000) {
|
|
1137
|
-
if (this.pauseWakeState === "active")
|
|
1138
|
-
return;
|
|
1139
|
-
if (this.pauseWakeState === "pausing")
|
|
1140
|
-
await this.pauseWakeTransition;
|
|
1141
|
-
if (this.getPauseWakeState() === "active")
|
|
1142
|
-
return;
|
|
1143
|
-
if (this.pauseWakeState === "waking") {
|
|
1144
|
-
await this.pauseWakeTransition;
|
|
1145
|
-
return;
|
|
1146
|
-
}
|
|
1147
|
-
this.pauseWakeState = "waking";
|
|
1148
|
-
this.spawning = true;
|
|
1149
|
-
const transition = this.autoPauseController.wakeOnDeliver(async () => {
|
|
1150
|
-
let timeout;
|
|
1151
|
-
try {
|
|
1152
|
-
const ready = await Promise.race([
|
|
1153
|
-
this.trySpawn(true, timeoutMs),
|
|
1154
|
-
new Promise(resolve => { timeout = setTimeout(() => resolve(false), timeoutMs); }),
|
|
1155
|
-
]);
|
|
1156
|
-
if (!ready)
|
|
1157
|
-
throw new Error(`Wake timed out before CLI became ready (${timeoutMs}ms)`);
|
|
1158
|
-
}
|
|
1159
|
-
finally {
|
|
1160
|
-
if (timeout)
|
|
1161
|
-
clearTimeout(timeout);
|
|
1162
|
-
}
|
|
1163
|
-
});
|
|
1164
|
-
this.pauseWakeTransition = transition;
|
|
1165
|
-
try {
|
|
1166
|
-
await transition;
|
|
1167
|
-
this.pauseWakeState = "active";
|
|
1168
|
-
this.healthCheckPaused = false;
|
|
1169
|
-
this.pauseRequested = false;
|
|
1170
|
-
try {
|
|
1171
|
-
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
1172
|
-
}
|
|
1173
|
-
catch { }
|
|
1174
|
-
this.transcriptMonitor?.resetOffset();
|
|
1175
|
-
this.resumeRuntimeMonitors();
|
|
1176
|
-
this.logger.info("Instance auto-woke");
|
|
1177
|
-
this.ipcServer?.broadcast({
|
|
1178
|
-
type: "instance_state", instanceName: this.name, state: this.instanceState, pausedAt: null,
|
|
1179
|
-
});
|
|
1180
|
-
this.emit("auto_woke", { name: this.name });
|
|
1181
|
-
}
|
|
1182
|
-
catch (err) {
|
|
1183
|
-
this.pauseWakeState = "paused";
|
|
1184
|
-
this.healthCheckPaused = true;
|
|
1185
|
-
this.logger.error({ err: err.message }, "Instance wake failed");
|
|
1186
|
-
throw err;
|
|
1187
|
-
}
|
|
1188
|
-
finally {
|
|
1189
|
-
this.spawning = false;
|
|
1190
|
-
if (this.pauseWakeTransition === transition)
|
|
1191
|
-
this.pauseWakeTransition = null;
|
|
1192
|
-
}
|
|
1193
|
-
}
|
|
1194
927
|
startInstanceStateMonitor() {
|
|
1195
|
-
if (
|
|
928
|
+
if (!this.tmux || !this.backend || this.instanceStateMonitorTimer)
|
|
1196
929
|
return;
|
|
1197
930
|
const rawConfig = this.config.hang_detector;
|
|
1198
931
|
const timeoutMinutes = rawConfig?.timeout_minutes;
|
|
@@ -1202,8 +935,7 @@ export class Daemon extends EventEmitter {
|
|
|
1202
935
|
const pollIntervalMs = typeof rawConfig?.poll_interval_ms === "number" && rawConfig.poll_interval_ms > 0
|
|
1203
936
|
? rawConfig.poll_interval_ms
|
|
1204
937
|
: DEFAULT_STATE_POLL_INTERVAL_MS;
|
|
1205
|
-
|
|
1206
|
-
this.instanceStateMachine = new PaneStateMachine(readyPattern, stuckTimeoutMs);
|
|
938
|
+
this.instanceStateMachine = new PaneStateMachine(this.backend.getReadyPattern(), stuckTimeoutMs);
|
|
1207
939
|
const poll = async () => {
|
|
1208
940
|
if (!this.tmux || this.spawning || this.statePollInFlight)
|
|
1209
941
|
return;
|
|
@@ -1216,11 +948,6 @@ export class Daemon extends EventEmitter {
|
|
|
1216
948
|
const previous = this.instanceState;
|
|
1217
949
|
const snapshot = this.instanceStateMachine.observe(pane);
|
|
1218
950
|
this.instanceState = snapshot.state;
|
|
1219
|
-
// Only a transition back to idle completes pending work. Repeated idle
|
|
1220
|
-
// polls between enqueue and paste must not clear a newly-recorded inbound.
|
|
1221
|
-
if (snapshot.state === "idle" && previous !== "idle") {
|
|
1222
|
-
this.pendingWork.recordIdle(snapshot.observedAt);
|
|
1223
|
-
}
|
|
1224
951
|
if (snapshot.state !== previous) {
|
|
1225
952
|
this.logger.info({
|
|
1226
953
|
previousState: previous,
|
|
@@ -1232,15 +959,9 @@ export class Daemon extends EventEmitter {
|
|
|
1232
959
|
// Emit exactly once per transition into stuck. Further notifications
|
|
1233
960
|
// require observable progress (working) or a ready prompt (idle) first.
|
|
1234
961
|
if (snapshot.state === "stuck") {
|
|
1235
|
-
this.
|
|
962
|
+
this.hangDetector?.emit("hang");
|
|
1236
963
|
}
|
|
1237
964
|
}
|
|
1238
|
-
if (snapshot.state !== "idle")
|
|
1239
|
-
this.pauseRequested = false;
|
|
1240
|
-
if (!this.pauseRequested && this.pasteQueueDepth === 0 && this.autoPauseController.observe(snapshot.state)) {
|
|
1241
|
-
this.pauseRequested = true;
|
|
1242
|
-
this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
|
|
1243
|
-
}
|
|
1244
965
|
}
|
|
1245
966
|
catch (err) {
|
|
1246
967
|
// A pane can disappear between status and capture during restart. Keep
|
|
@@ -1254,54 +975,6 @@ export class Daemon extends EventEmitter {
|
|
|
1254
975
|
void poll();
|
|
1255
976
|
this.instanceStateMonitorTimer = setInterval(() => { void poll(); }, pollIntervalMs);
|
|
1256
977
|
}
|
|
1257
|
-
handleStuckTransition(pane, snapshot, readyPattern) {
|
|
1258
|
-
const deterministicReadyPattern = new RegExp(readyPattern.source, readyPattern.flags.replace(/[gy]/g, ""));
|
|
1259
|
-
const diagnostic = {
|
|
1260
|
-
backend: this.backend?.binaryName ?? this.config.backend ?? "unknown",
|
|
1261
|
-
paneTail: sanitizePaneTail(pane),
|
|
1262
|
-
readyPattern: readyPattern.toString(),
|
|
1263
|
-
readyMatched: deterministicReadyPattern.test(pane),
|
|
1264
|
-
unchangedForMs: snapshot.unchangedForMs,
|
|
1265
|
-
pendingWork: this.pendingWork.hasPendingWork(),
|
|
1266
|
-
};
|
|
1267
|
-
if (!diagnostic.pendingWork) {
|
|
1268
|
-
this.logger.debug(diagnostic, "Suppressing stuck notification without pending work");
|
|
1269
|
-
return;
|
|
1270
|
-
}
|
|
1271
|
-
this.logger.warn(diagnostic, "Instance pane stuck with pending work");
|
|
1272
|
-
this.hangDetector?.emit("hang", { unchangedForMs: snapshot.unchangedForMs });
|
|
1273
|
-
}
|
|
1274
|
-
/** Stop every runtime poller/watcher while preserving IPC and daemon state. */
|
|
1275
|
-
freezeRuntimeMonitors() {
|
|
1276
|
-
this.runtimeMonitorsFrozen = true;
|
|
1277
|
-
if (this.healthCheckTimer) {
|
|
1278
|
-
clearTimeout(this.healthCheckTimer);
|
|
1279
|
-
this.healthCheckTimer = null;
|
|
1280
|
-
}
|
|
1281
|
-
if (this.errorMonitorTimer) {
|
|
1282
|
-
clearInterval(this.errorMonitorTimer);
|
|
1283
|
-
this.errorMonitorTimer = null;
|
|
1284
|
-
}
|
|
1285
|
-
if (this.instanceStateMonitorTimer) {
|
|
1286
|
-
clearInterval(this.instanceStateMonitorTimer);
|
|
1287
|
-
this.instanceStateMonitorTimer = null;
|
|
1288
|
-
}
|
|
1289
|
-
this.transcriptMonitor?.stop();
|
|
1290
|
-
this.guardian?.stop();
|
|
1291
|
-
}
|
|
1292
|
-
/** Restore the same monitor objects after wake without adding event listeners. */
|
|
1293
|
-
resumeRuntimeMonitors() {
|
|
1294
|
-
if (!this.runtimeMonitorsFrozen)
|
|
1295
|
-
return;
|
|
1296
|
-
this.runtimeMonitorsFrozen = false;
|
|
1297
|
-
this.startHealthCheck();
|
|
1298
|
-
if (!this.config.lightweight) {
|
|
1299
|
-
this.transcriptMonitor?.startPolling();
|
|
1300
|
-
this.guardian?.startWatching();
|
|
1301
|
-
this.startErrorMonitor();
|
|
1302
|
-
}
|
|
1303
|
-
this.startInstanceStateMonitor();
|
|
1304
|
-
}
|
|
1305
978
|
getMessageBus() {
|
|
1306
979
|
return this.messageBus;
|
|
1307
980
|
}
|
|
@@ -1380,13 +1053,12 @@ export class Daemon extends EventEmitter {
|
|
|
1380
1053
|
// Remember (and persist) the reply target. Only real channel messages have a
|
|
1381
1054
|
// non-empty chat_id; cross-instance messages have chat_id="" and must NOT
|
|
1382
1055
|
// overwrite it (their reply would otherwise go nowhere).
|
|
1383
|
-
this.updateLastChat(meta.chat_id, meta.thread_id
|
|
1056
|
+
this.updateLastChat(meta.chat_id, meta.thread_id);
|
|
1384
1057
|
if (this.pendingInstructionsUpdate) {
|
|
1385
1058
|
writeFileSync(join(this.instanceDir, "prev-instructions"), this.pendingInstructionsUpdate);
|
|
1386
1059
|
this.pendingInstructionsUpdate = undefined;
|
|
1387
1060
|
}
|
|
1388
1061
|
this.hangDetector?.recordInbound();
|
|
1389
|
-
this.pendingWork.recordInbound();
|
|
1390
1062
|
// v3: record user messages for rotation snapshot
|
|
1391
1063
|
this.recordRecentUserMessage(content, meta);
|
|
1392
1064
|
// Format message with metadata prefix for the agent
|
|
@@ -1960,14 +1632,14 @@ export class Daemon extends EventEmitter {
|
|
|
1960
1632
|
return resumedSuccessfully;
|
|
1961
1633
|
}
|
|
1962
1634
|
/** Kill the entire process tree of the current tmux pane (CLI + MCP server). */
|
|
1963
|
-
async killProcessTree(
|
|
1635
|
+
async killProcessTree() {
|
|
1964
1636
|
if (!this.tmux)
|
|
1965
1637
|
return;
|
|
1966
1638
|
try {
|
|
1967
1639
|
const pid = await TmuxManager.getPanePid(this.tmuxSessionName, this.tmux.getWindowId());
|
|
1968
1640
|
if (pid) {
|
|
1969
|
-
process.kill(-pid,
|
|
1970
|
-
this.logger.debug({ pid
|
|
1641
|
+
process.kill(-pid, "SIGTERM");
|
|
1642
|
+
this.logger.debug({ pid }, "Killed process group");
|
|
1971
1643
|
}
|
|
1972
1644
|
}
|
|
1973
1645
|
catch { /* process group may not exist or already dead */ }
|
|
@@ -1978,7 +1650,7 @@ export class Daemon extends EventEmitter {
|
|
|
1978
1650
|
* Handles confirmation dialogs (trust folder, bypass permissions).
|
|
1979
1651
|
* Returns true if CLI is ready, false if it failed or got stuck.
|
|
1980
1652
|
*/
|
|
1981
|
-
async trySpawn(
|
|
1653
|
+
async trySpawn() {
|
|
1982
1654
|
const backendConfig = this.buildBackendConfig();
|
|
1983
1655
|
// Compare freshly-built instructions against the last value the agent was
|
|
1984
1656
|
// told about. Computed for ALL backends (not gated by
|
|
@@ -2041,29 +1713,16 @@ export class Daemon extends EventEmitter {
|
|
|
2041
1713
|
const cmd = `${envPrefix} ` + this.backend.buildCommand(backendConfig);
|
|
2042
1714
|
// Ensure tmux session exists (may have been destroyed if all windows died)
|
|
2043
1715
|
await TmuxManager.ensureSession(this.tmuxSessionName);
|
|
2044
|
-
|
|
2045
|
-
if (reuseWindow) {
|
|
2046
|
-
this.controlClient?.unregisterWindow(this.tmux.getWindowId());
|
|
2047
|
-
await this.tmux.respawnWindow(cmd, resolvedCwd);
|
|
2048
|
-
windowId = this.tmux.getWindowId();
|
|
2049
|
-
}
|
|
2050
|
-
else {
|
|
2051
|
-
windowId = await this.tmux.createWindow(cmd, resolvedCwd, this.name);
|
|
2052
|
-
}
|
|
1716
|
+
const windowId = await this.tmux.createWindow(cmd, resolvedCwd, this.name);
|
|
2053
1717
|
writeFileSync(join(this.instanceDir, "window-id"), windowId);
|
|
2054
1718
|
// Enable remain-on-exit to capture exit codes on crash
|
|
2055
1719
|
await this.tmux.setRemainOnExit().catch(err => {
|
|
2056
1720
|
this.logger.warn({ err }, "Failed to set remain-on-exit — exit codes will not be captured");
|
|
2057
1721
|
});
|
|
2058
|
-
if (reuseWindow && !this.config.lightweight) {
|
|
2059
|
-
await this.tmux.pipeOutput(join(this.instanceDir, "output.log")).catch(err => {
|
|
2060
|
-
this.logger.warn({ err }, "Failed to restore pipe-pane after wake");
|
|
2061
|
-
});
|
|
2062
|
-
}
|
|
2063
1722
|
// Register with control client and wait for output + idle
|
|
2064
1723
|
await this.controlClient?.registerWindow(windowId);
|
|
2065
1724
|
if (this.controlClient) {
|
|
2066
|
-
const total =
|
|
1725
|
+
const total = this.config.startup_timeout_ms ?? 25_000;
|
|
2067
1726
|
const outputTimeout = Math.round(total * 0.6);
|
|
2068
1727
|
const idleTimeout = total - outputTimeout;
|
|
2069
1728
|
const hasOutput = await this.controlClient.waitForOutput(windowId, outputTimeout);
|
|
@@ -2153,17 +1812,14 @@ export class Daemon extends EventEmitter {
|
|
|
2153
1812
|
* messages) so it never overwrites a real channel target. Persisted to
|
|
2154
1813
|
* last-chat.json so the reply target survives a restart (see start()).
|
|
2155
1814
|
*/
|
|
2156
|
-
updateLastChat(chatId, threadId
|
|
1815
|
+
updateLastChat(chatId, threadId) {
|
|
2157
1816
|
if (!chatId)
|
|
2158
1817
|
return;
|
|
2159
1818
|
this.lastChatId = chatId;
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
this.lastThreadId = threadId || undefined;
|
|
2163
|
-
if (adapterId)
|
|
2164
|
-
this.lastAdapterId = adapterId;
|
|
1819
|
+
if (threadId)
|
|
1820
|
+
this.lastThreadId = threadId;
|
|
2165
1821
|
try {
|
|
2166
|
-
writeFileSync(join(this.instanceDir, "last-chat.json"), JSON.stringify({ chatId: this.lastChatId, threadId: this.lastThreadId
|
|
1822
|
+
writeFileSync(join(this.instanceDir, "last-chat.json"), JSON.stringify({ chatId: this.lastChatId, threadId: this.lastThreadId }));
|
|
2167
1823
|
}
|
|
2168
1824
|
catch { /* best effort */ }
|
|
2169
1825
|
}
|