@songsid/agend 2.1.0-beta.3 → 2.1.0-beta.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/README.zh-TW.md +2 -1
- package/dist/access-path.js +3 -3
- package/dist/access-path.js.map +1 -1
- package/dist/agent-cli.js +1 -1
- package/dist/agent-cli.js.map +1 -1
- package/dist/agent-endpoint.d.ts +8 -0
- package/dist/agent-endpoint.js +36 -8
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.js +3 -2
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.js +4 -4
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/factory.js +4 -1
- package/dist/backend/factory.js.map +1 -1
- package/dist/backend/grok.d.ts +51 -0
- package/dist/backend/grok.js +282 -0
- package/dist/backend/grok.js.map +1 -0
- package/dist/backend/kiro.js +3 -2
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +12 -2
- package/dist/backend/types.js +1 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +20 -0
- package/dist/channel/adapters/discord.js +85 -11
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.js +4 -1
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/mcp-tools.js +17 -1
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/classic-channel-manager.d.ts +14 -1
- package/dist/classic-channel-manager.js +65 -4
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +108 -16
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +68 -1
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +3 -1
- package/dist/config.js +13 -1
- package/dist/config.js.map +1 -1
- package/dist/context-guardian.d.ts +2 -0
- package/dist/context-guardian.js +10 -2
- package/dist/context-guardian.js.map +1 -1
- package/dist/daemon-entry.js +2 -1
- package/dist/daemon-entry.js.map +1 -1
- package/dist/daemon.d.ts +76 -6
- package/dist/daemon.js +451 -113
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-context.d.ts +4 -0
- package/dist/fleet-manager.d.ts +88 -7
- package/dist/fleet-manager.js +712 -129
- package/dist/fleet-manager.js.map +1 -1
- package/dist/general-knowledge/skills/fleet-config/SKILL.md +1 -1
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +1 -0
- package/dist/instance-lifecycle.d.ts +11 -1
- package/dist/instance-lifecycle.js +64 -22
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/locale.js +26 -2
- package/dist/locale.js.map +1 -1
- package/dist/logger.d.ts +6 -0
- package/dist/logger.js +29 -10
- package/dist/logger.js.map +1 -1
- package/dist/outbound-handlers.d.ts +3 -0
- package/dist/outbound-handlers.js +203 -2
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +40 -1
- package/dist/outbound-schemas.js +36 -1
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pause-marker.d.ts +5 -0
- package/dist/pause-marker.js +36 -0
- package/dist/pause-marker.js.map +1 -0
- package/dist/settings-api.d.ts +20 -2
- package/dist/settings-api.js +213 -9
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +8 -0
- package/dist/setup-wizard.js.map +1 -1
- package/dist/statusline-watcher.d.ts +1 -1
- package/dist/statusline-watcher.js +3 -2
- package/dist/statusline-watcher.js.map +1 -1
- package/dist/tmux-control.d.ts +7 -0
- package/dist/tmux-control.js +22 -1
- package/dist/tmux-control.js.map +1 -1
- package/dist/tmux-manager.d.ts +1 -1
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-commands.d.ts +19 -1
- package/dist/topic-commands.js +132 -23
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-monitor.js +2 -0
- package/dist/transcript-monitor.js.map +1 -1
- package/dist/types.d.ts +23 -1
- package/dist/ui/dashboard.html +2 -2
- package/dist/ui/settings.html +453 -74
- package/dist/web-api.d.ts +1 -0
- package/dist/web-api.js +24 -15
- package/dist/web-api.js.map +1 -1
- package/dist/workflow-templates/default.md +0 -4
- package/package.json +2 -1
package/dist/daemon.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { join, dirname, basename, resolve } from "node:path";
|
|
2
|
-
import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync, chmodSync } from "node:fs";
|
|
2
|
+
import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync, chmodSync, renameSync } from "node:fs";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import { createHash, randomBytes } from "node:crypto";
|
|
5
5
|
import { EventEmitter } from "node:events";
|
|
6
|
-
import {
|
|
6
|
+
import { clearPausedMarker, writePausedMarker } from "./pause-marker.js";
|
|
7
7
|
import { TmuxManager } from "./tmux-manager.js";
|
|
8
8
|
import { TranscriptMonitor } from "./transcript-monitor.js";
|
|
9
9
|
import { ContextGuardian } from "./context-guardian.js";
|
|
@@ -21,32 +21,71 @@ const CROSS_INSTANCE_TOOLS = new Set(["send_to_instance", "list_instances", "sta
|
|
|
21
21
|
const SCHEDULE_TOOLS = new Set(["create_schedule", "list_schedules", "update_schedule", "delete_schedule"]);
|
|
22
22
|
const DECISION_TOOLS = new Set(["post_decision", "list_decisions", "update_decision"]);
|
|
23
23
|
const TASK_TOOL = "task";
|
|
24
|
+
/** Point a resumed CLI at its one backend-native instruction source. */
|
|
25
|
+
export function buildInstructionReloadNotice(binaryName, instanceName, instanceDir) {
|
|
26
|
+
const source = binaryName === "codex" || binaryName === "grok"
|
|
27
|
+
? "AGENTS.md"
|
|
28
|
+
: binaryName === "kiro-cli"
|
|
29
|
+
? `.kiro/steering/agend-${instanceName}.md`
|
|
30
|
+
: binaryName === "agy"
|
|
31
|
+
? ".agents/agents.md"
|
|
32
|
+
: binaryName === "gemini"
|
|
33
|
+
? "GEMINI.md"
|
|
34
|
+
: join(instanceDir, "fleet-instructions.md");
|
|
35
|
+
return `[system] Your AgEnD instructions have been updated. Reload only ${source}; do not scan other instruction directories. Do not reply to this message.`;
|
|
36
|
+
}
|
|
24
37
|
export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
|
|
25
|
-
export const
|
|
26
|
-
|
|
38
|
+
export const DEFAULT_STATE_IDLE_DEBOUNCE_MS = 2_000;
|
|
39
|
+
export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
|
|
40
|
+
/** @deprecated State detection is event-driven; this now aliases the safety sweep. */
|
|
41
|
+
export const DEFAULT_STATE_POLL_INTERVAL_MS = DEFAULT_STATE_SAFETY_SWEEP_MS;
|
|
42
|
+
const LAST_INBOUND_FILE = "last-inbound-at";
|
|
43
|
+
/** Read the last real channel inbound timestamp persisted across daemon restarts. */
|
|
44
|
+
export function readLastInboundAt(instanceDir, now = Date.now()) {
|
|
45
|
+
try {
|
|
46
|
+
const value = Number(readFileSync(join(instanceDir, LAST_INBOUND_FILE), "utf-8").trim());
|
|
47
|
+
return Number.isFinite(value) && value >= 0 && value <= now ? value : null;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** Atomically persist the last real channel inbound timestamp. */
|
|
54
|
+
export function writeLastInboundAt(instanceDir, timestamp) {
|
|
55
|
+
mkdirSync(instanceDir, { recursive: true });
|
|
56
|
+
const target = join(instanceDir, LAST_INBOUND_FILE);
|
|
57
|
+
const temp = `${target}.${process.pid}.tmp`;
|
|
58
|
+
writeFileSync(temp, String(timestamp));
|
|
59
|
+
renameSync(temp, target);
|
|
60
|
+
}
|
|
61
|
+
/** Headless inactivity timer used by the daemon and unit tests. */
|
|
27
62
|
export class AutoPauseController {
|
|
28
63
|
thresholdMs;
|
|
29
|
-
|
|
64
|
+
lastActivityAt;
|
|
30
65
|
pausedAt = null;
|
|
31
|
-
constructor(thresholdMs) {
|
|
66
|
+
constructor(thresholdMs, lastActivityAt = Date.now()) {
|
|
32
67
|
this.thresholdMs = thresholdMs;
|
|
68
|
+
this.lastActivityAt = lastActivityAt;
|
|
69
|
+
}
|
|
70
|
+
recordActivity(now = Date.now()) {
|
|
71
|
+
this.lastActivityAt = now;
|
|
33
72
|
}
|
|
34
73
|
observe(state, now = Date.now()) {
|
|
35
74
|
if (this.pausedAt !== null || this.thresholdMs <= 0)
|
|
36
75
|
return false;
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
this.idleSince ??= now;
|
|
42
|
-
return now - this.idleSince >= this.thresholdMs;
|
|
76
|
+
// Inactivity is based on the last user inbound, not on how long this daemon
|
|
77
|
+
// has observed the idle pane. This preserves pause eligibility across fleet
|
|
78
|
+
// restarts while still ensuring an actively working instance is never paused.
|
|
79
|
+
return state === "idle" && now - this.lastActivityAt >= this.thresholdMs;
|
|
43
80
|
}
|
|
44
81
|
markPaused(now = Date.now()) {
|
|
45
82
|
this.pausedAt = now;
|
|
46
83
|
}
|
|
47
|
-
markAwake() {
|
|
84
|
+
markAwake(now = Date.now()) {
|
|
48
85
|
this.pausedAt = null;
|
|
49
|
-
|
|
86
|
+
// A deliberate wake is activity in its own right and must provide a fresh
|
|
87
|
+
// window for the wake-before-deliver facade to enqueue its inbound message.
|
|
88
|
+
this.lastActivityAt = now;
|
|
50
89
|
}
|
|
51
90
|
async wakeOnDeliver(wake) {
|
|
52
91
|
if (this.pausedAt === null)
|
|
@@ -106,6 +145,16 @@ export class PaneStateMachine {
|
|
|
106
145
|
}
|
|
107
146
|
return this.snapshot(now);
|
|
108
147
|
}
|
|
148
|
+
/** Record pane motion from tmux control mode without capturing pane content. */
|
|
149
|
+
recordOutput(now = Date.now()) {
|
|
150
|
+
this.lastPaneChangeAt = now;
|
|
151
|
+
this.lastObservedAt = now;
|
|
152
|
+
if (this.currentState !== "working") {
|
|
153
|
+
this.currentState = "working";
|
|
154
|
+
this.stateChangedAt = now;
|
|
155
|
+
}
|
|
156
|
+
return this.snapshot(now);
|
|
157
|
+
}
|
|
109
158
|
snapshot(now = Date.now()) {
|
|
110
159
|
return {
|
|
111
160
|
state: this.currentState,
|
|
@@ -115,6 +164,79 @@ export class PaneStateMachine {
|
|
|
115
164
|
};
|
|
116
165
|
}
|
|
117
166
|
}
|
|
167
|
+
/** Tracks whether an inbound arrived after the most recent confirmed idle prompt. */
|
|
168
|
+
export class PendingWorkTracker {
|
|
169
|
+
lastInboundAt = 0;
|
|
170
|
+
lastIdleAt;
|
|
171
|
+
sequence = 0;
|
|
172
|
+
lastInboundOrder = 0;
|
|
173
|
+
lastIdleOrder = 0;
|
|
174
|
+
constructor(now = Date.now()) {
|
|
175
|
+
this.lastIdleAt = now;
|
|
176
|
+
}
|
|
177
|
+
recordInbound(now = Date.now()) {
|
|
178
|
+
this.lastInboundAt = now;
|
|
179
|
+
this.lastInboundOrder = ++this.sequence;
|
|
180
|
+
}
|
|
181
|
+
recordIdle(now = Date.now()) {
|
|
182
|
+
// An async pane poll can finish after a newer inbound. Do not let its stale
|
|
183
|
+
// observation clear work which had not arrived when the pane was captured.
|
|
184
|
+
if (now < this.lastInboundAt)
|
|
185
|
+
return;
|
|
186
|
+
this.lastIdleAt = now;
|
|
187
|
+
this.lastIdleOrder = ++this.sequence;
|
|
188
|
+
}
|
|
189
|
+
hasPendingWork() {
|
|
190
|
+
return this.lastInboundOrder > this.lastIdleOrder;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
const NORMAL_ENTER_SETTLE_MS = 500;
|
|
194
|
+
const FIRST_ENTER_SETTLE_MS = 1_750;
|
|
195
|
+
const FIRST_DELIVERY_WINDOW_MS = 5_000;
|
|
196
|
+
/**
|
|
197
|
+
* One-shot timing gate for the first paste after a CLI reaches its ready
|
|
198
|
+
* prompt. Recent tmux versions expose a short interval where the prompt is
|
|
199
|
+
* visible but the TUI can still swallow Enter while completing its redraw.
|
|
200
|
+
*/
|
|
201
|
+
export class FirstDeliveryDelay {
|
|
202
|
+
readyAt = 0;
|
|
203
|
+
pending = false;
|
|
204
|
+
recordReady(now = Date.now()) {
|
|
205
|
+
this.readyAt = now;
|
|
206
|
+
this.pending = true;
|
|
207
|
+
}
|
|
208
|
+
consume(now = Date.now()) {
|
|
209
|
+
if (!this.pending)
|
|
210
|
+
return NORMAL_ENTER_SETTLE_MS;
|
|
211
|
+
this.pending = false;
|
|
212
|
+
return this.readyAt > 0 && now - this.readyAt < FIRST_DELIVERY_WINDOW_MS
|
|
213
|
+
? FIRST_ENTER_SETTLE_MS
|
|
214
|
+
: NORMAL_ENTER_SETTLE_MS;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
/** Redact likely credentials and control sequences before pane text reaches logs. */
|
|
218
|
+
export function sanitizePaneTail(pane, lineCount = 5) {
|
|
219
|
+
const secretAssignment = /\b(token|secret|password|passwd|api[_-]?key|authorization)\b\s*[:=]\s*\S+/gi;
|
|
220
|
+
const bearer = /\bBearer\s+\S+/gi;
|
|
221
|
+
const knownToken = /\b(?:sk-[A-Za-z0-9_-]+|ghp_[A-Za-z0-9]+|github_pat_[A-Za-z0-9_]+|AKIA[A-Z0-9]{16})\b/g;
|
|
222
|
+
const jwt = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g;
|
|
223
|
+
const opaqueSecret = /\b[A-Za-z0-9_+/=-]{32,}\b/g;
|
|
224
|
+
const lines = pane
|
|
225
|
+
.replace(/\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g, "")
|
|
226
|
+
.split(/\r?\n/);
|
|
227
|
+
while (lines.length > 0 && /^\s*$/.test(lines[lines.length - 1]))
|
|
228
|
+
lines.pop();
|
|
229
|
+
return lines
|
|
230
|
+
.slice(-lineCount)
|
|
231
|
+
.map(line => line
|
|
232
|
+
.replace(bearer, "Bearer [REDACTED]")
|
|
233
|
+
.replace(secretAssignment, "$1=[REDACTED]")
|
|
234
|
+
.replace(knownToken, "[REDACTED]")
|
|
235
|
+
.replace(jwt, "[REDACTED]")
|
|
236
|
+
.replace(opaqueSecret, "[REDACTED]")
|
|
237
|
+
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "")
|
|
238
|
+
.slice(0, 200));
|
|
239
|
+
}
|
|
118
240
|
export class Daemon extends EventEmitter {
|
|
119
241
|
name;
|
|
120
242
|
config;
|
|
@@ -135,6 +257,7 @@ export class Daemon extends EventEmitter {
|
|
|
135
257
|
// Track chatId/threadId from inbound messages for automatic outbound routing
|
|
136
258
|
lastChatId;
|
|
137
259
|
lastThreadId;
|
|
260
|
+
lastAdapterId;
|
|
138
261
|
// Pending ack: react 🫡 on first transcript activity after receiving a message
|
|
139
262
|
pendingAckMessage = null;
|
|
140
263
|
// Tool status tracking for channel adapter
|
|
@@ -164,7 +287,19 @@ export class Daemon extends EventEmitter {
|
|
|
164
287
|
hangDetector = null;
|
|
165
288
|
instanceState = "idle";
|
|
166
289
|
instanceStateMachine = null;
|
|
290
|
+
pendingWork = new PendingWorkTracker();
|
|
291
|
+
/** Fallback safety sweep used only when no shared tmux control client exists. */
|
|
167
292
|
instanceStateMonitorTimer = null;
|
|
293
|
+
instanceStateIdleTimer = null;
|
|
294
|
+
instanceStateStuckTimer = null;
|
|
295
|
+
instanceStateOutputListener = null;
|
|
296
|
+
instanceStateOutputEventName = null;
|
|
297
|
+
instanceStateSafetyListener = null;
|
|
298
|
+
instanceStateLastOutputAt = 0;
|
|
299
|
+
instanceStateIdleDebounceMs = DEFAULT_STATE_IDLE_DEBOUNCE_MS;
|
|
300
|
+
instanceStateStuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS;
|
|
301
|
+
instanceStateReadyPattern = null;
|
|
302
|
+
instanceStateMonitorActive = false;
|
|
168
303
|
statePollInFlight = false;
|
|
169
304
|
autoPauseController;
|
|
170
305
|
pauseRequested = false;
|
|
@@ -187,8 +322,11 @@ export class Daemon extends EventEmitter {
|
|
|
187
322
|
warmupNeeded = false;
|
|
188
323
|
lastBuiltInstructions = "";
|
|
189
324
|
pasteQueueDepth = 0;
|
|
325
|
+
firstDeliveryDelay = new FirstDeliveryDelay();
|
|
190
326
|
// PTY error pattern monitoring
|
|
191
327
|
errorMonitorTimer = null;
|
|
328
|
+
/** Prevent in-flight monitor callbacks from re-arming after a pause. */
|
|
329
|
+
runtimeMonitorsFrozen = false;
|
|
192
330
|
errorWaitingForRecovery = false; // true = error detected, waiting for ready pattern
|
|
193
331
|
errorDetectedAt = 0;
|
|
194
332
|
/** Whether this instance is in an abnormal error state (auto-pause is normal). */
|
|
@@ -202,6 +340,10 @@ export class Daemon extends EventEmitter {
|
|
|
202
340
|
get isCrashLoop() {
|
|
203
341
|
return this.crashCount >= 3;
|
|
204
342
|
}
|
|
343
|
+
/** The most recent error type detected by the error monitor (e.g. "rate_limit", "auth_error"). */
|
|
344
|
+
get lastErrorType() {
|
|
345
|
+
return this.lastDetectedErrorType;
|
|
346
|
+
}
|
|
205
347
|
lastFailoverAt = 0; // cooldown: prevent repeated failover triggers
|
|
206
348
|
static FAILOVER_COOLDOWN_MS = 5 * 60_000; // 5 minutes
|
|
207
349
|
lastErrorNotifiedAt = new Map(); // per-type cooldown for all actions
|
|
@@ -215,7 +357,7 @@ export class Daemon extends EventEmitter {
|
|
|
215
357
|
// still registers as new (prevents the old hash-dedup's permanent suppression).
|
|
216
358
|
lastErrorCount = new Map();
|
|
217
359
|
lastDetectedErrorType = null;
|
|
218
|
-
constructor(name, config, instanceDir, topicMode = false, backend, controlClient) {
|
|
360
|
+
constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger) {
|
|
219
361
|
super();
|
|
220
362
|
this.name = name;
|
|
221
363
|
this.config = config;
|
|
@@ -223,21 +365,17 @@ export class Daemon extends EventEmitter {
|
|
|
223
365
|
this.topicMode = topicMode;
|
|
224
366
|
this.backend = backend;
|
|
225
367
|
this.controlClient = controlClient;
|
|
226
|
-
|
|
368
|
+
if (!rootLogger)
|
|
369
|
+
throw new Error("Daemon requires a shared root logger");
|
|
370
|
+
this.logger = rootLogger.child({ instance: name }, { level: config.log_level });
|
|
227
371
|
this.tmuxSessionName = getTmuxSession();
|
|
228
372
|
this.messageBus = new MessageBus();
|
|
229
373
|
this.messageBus.setLogger(this.logger);
|
|
230
|
-
const autoPauseMinutes = typeof config.auto_pause_after === "number" ? config.auto_pause_after :
|
|
231
|
-
this.autoPauseController = new AutoPauseController(Math.max(0, autoPauseMinutes) * 60_000);
|
|
374
|
+
const autoPauseMinutes = typeof config.auto_pause_after === "number" ? config.auto_pause_after : 0; // default: disabled
|
|
375
|
+
this.autoPauseController = new AutoPauseController(Math.max(0, autoPauseMinutes) * 60_000, readLastInboundAt(instanceDir) ?? Date.now());
|
|
232
376
|
}
|
|
233
377
|
async start() {
|
|
234
378
|
mkdirSync(this.instanceDir, { recursive: true });
|
|
235
|
-
// A daemon restart performs a normal CLI start, so any persisted auto-pause
|
|
236
|
-
// marker from the previous daemon is stale.
|
|
237
|
-
try {
|
|
238
|
-
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
239
|
-
}
|
|
240
|
-
catch { }
|
|
241
379
|
writeFileSync(join(this.instanceDir, "daemon.pid"), String(process.pid));
|
|
242
380
|
this.logger.info(`Starting ${this.name}`);
|
|
243
381
|
// P1: Read crash state from previous run — skip resume if last run was a crash loop
|
|
@@ -263,6 +401,7 @@ export class Daemon extends EventEmitter {
|
|
|
263
401
|
if (saved.chatId) {
|
|
264
402
|
this.lastChatId = saved.chatId;
|
|
265
403
|
this.lastThreadId = saved.threadId || undefined;
|
|
404
|
+
this.lastAdapterId = saved.adapterId || undefined;
|
|
266
405
|
}
|
|
267
406
|
}
|
|
268
407
|
}
|
|
@@ -443,7 +582,7 @@ export class Daemon extends EventEmitter {
|
|
|
443
582
|
else {
|
|
444
583
|
await new Promise(r => setTimeout(r, 5000));
|
|
445
584
|
}
|
|
446
|
-
await this.tmux?.pasteText(
|
|
585
|
+
await this.tmux?.pasteText(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir));
|
|
447
586
|
// Record the value the agent has now been told about so the next
|
|
448
587
|
// unchanged restart skips the reload.
|
|
449
588
|
try {
|
|
@@ -521,11 +660,18 @@ export class Daemon extends EventEmitter {
|
|
|
521
660
|
this.logger.info(`${this.name} ready`);
|
|
522
661
|
}
|
|
523
662
|
startHealthCheck() {
|
|
663
|
+
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
664
|
+
return;
|
|
524
665
|
const { max_retries, backoff, reset_after } = this.config.restart_policy;
|
|
525
666
|
if (max_retries <= 0)
|
|
526
667
|
return; // restart disabled
|
|
527
668
|
const scheduleNext = () => {
|
|
669
|
+
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
670
|
+
return;
|
|
528
671
|
this.healthCheckTimer = setTimeout(async () => {
|
|
672
|
+
this.healthCheckTimer = null;
|
|
673
|
+
if (this.runtimeMonitorsFrozen)
|
|
674
|
+
return;
|
|
529
675
|
// Instance directory removed externally (e.g. `rm -rf ~/.agend/instances/<name>`).
|
|
530
676
|
// Stop the loop permanently — otherwise every tick triggers a respawn, whose
|
|
531
677
|
// writeRotationSnapshot fails with ENOENT and gets caught as "Failed to respawn",
|
|
@@ -776,6 +922,8 @@ export class Daemon extends EventEmitter {
|
|
|
776
922
|
* (ready pattern visible), it goes back to monitoring for new errors.
|
|
777
923
|
*/
|
|
778
924
|
startErrorMonitor() {
|
|
925
|
+
if (this.runtimeMonitorsFrozen || this.errorMonitorTimer)
|
|
926
|
+
return;
|
|
779
927
|
const patterns = this.backend?.getErrorPatterns?.() ?? [];
|
|
780
928
|
const dialogs = this.backend?.getRuntimeDialogs?.() ?? [];
|
|
781
929
|
if (!patterns.length && !dialogs.length)
|
|
@@ -895,26 +1043,13 @@ export class Daemon extends EventEmitter {
|
|
|
895
1043
|
}
|
|
896
1044
|
async stop() {
|
|
897
1045
|
this.logger.info("Stopping daemon instance");
|
|
898
|
-
|
|
899
|
-
clearTimeout(this.healthCheckTimer);
|
|
900
|
-
this.healthCheckTimer = null;
|
|
901
|
-
}
|
|
902
|
-
if (this.errorMonitorTimer) {
|
|
903
|
-
clearInterval(this.errorMonitorTimer);
|
|
904
|
-
this.errorMonitorTimer = null;
|
|
905
|
-
}
|
|
906
|
-
if (this.instanceStateMonitorTimer) {
|
|
907
|
-
clearInterval(this.instanceStateMonitorTimer);
|
|
908
|
-
this.instanceStateMonitorTimer = null;
|
|
909
|
-
}
|
|
1046
|
+
this.freezeRuntimeMonitors();
|
|
910
1047
|
if (this.toolStatusDebounce) {
|
|
911
1048
|
clearTimeout(this.toolStatusDebounce);
|
|
912
1049
|
this.toolStatusDebounce = null;
|
|
913
1050
|
}
|
|
914
1051
|
this.pendingIpcRequests.clear();
|
|
915
1052
|
this.hangDetector?.stop();
|
|
916
|
-
this.transcriptMonitor?.stop();
|
|
917
|
-
this.guardian?.stop();
|
|
918
1053
|
if (this.adapter)
|
|
919
1054
|
await this.adapter.stop();
|
|
920
1055
|
// Notify MCP servers of graceful shutdown (prevents reconnect attempts)
|
|
@@ -926,12 +1061,19 @@ export class Daemon extends EventEmitter {
|
|
|
926
1061
|
this.healthCheckPaused = true;
|
|
927
1062
|
let killed = false;
|
|
928
1063
|
const quitCmd = this.backend?.getQuitCommand();
|
|
1064
|
+
const quitKey = this.backend?.getQuitKey?.();
|
|
929
1065
|
if (quitCmd) {
|
|
930
1066
|
await this.tmux.sendKeys(quitCmd);
|
|
931
1067
|
// Delay before Enter to prevent tmux server race when multiple
|
|
932
1068
|
// instances stop in parallel (same pattern as pasteText).
|
|
933
1069
|
await new Promise(r => setTimeout(r, 150));
|
|
934
1070
|
await this.tmux.sendSpecialKey("Enter");
|
|
1071
|
+
}
|
|
1072
|
+
else if (quitKey) {
|
|
1073
|
+
// Some CLIs quit via a key chord (e.g. grok Ctrl+Q), not a typed command.
|
|
1074
|
+
await this.tmux.sendSpecialKey(quitKey);
|
|
1075
|
+
}
|
|
1076
|
+
if (quitCmd || quitKey) {
|
|
935
1077
|
// Wait up to 3s for graceful exit, polling every 200ms. A healthy CLI
|
|
936
1078
|
// exits within ~1s; a longer wait just delays the force-kill fallback.
|
|
937
1079
|
for (let i = 0; i < 15; i++) {
|
|
@@ -973,10 +1115,6 @@ export class Daemon extends EventEmitter {
|
|
|
973
1115
|
catch (e) {
|
|
974
1116
|
this.logger.debug({ err: e }, "Failed to remove PID file");
|
|
975
1117
|
}
|
|
976
|
-
try {
|
|
977
|
-
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
978
|
-
}
|
|
979
|
-
catch { }
|
|
980
1118
|
}
|
|
981
1119
|
getHangDetector() {
|
|
982
1120
|
return this.hangDetector;
|
|
@@ -1007,14 +1145,22 @@ export class Daemon extends EventEmitter {
|
|
|
1007
1145
|
}
|
|
1008
1146
|
this.pauseWakeState = "pausing";
|
|
1009
1147
|
this.healthCheckPaused = true;
|
|
1148
|
+
this.freezeRuntimeMonitors();
|
|
1010
1149
|
const transition = (async () => {
|
|
1011
1150
|
try {
|
|
1012
1151
|
this.saveSessionId();
|
|
1013
1152
|
const quitCmd = this.backend?.getQuitCommand();
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1153
|
+
const quitKey = this.backend?.getQuitKey?.();
|
|
1154
|
+
if (this.tmux) {
|
|
1155
|
+
if (quitCmd) {
|
|
1156
|
+
await this.tmux.sendKeys(quitCmd);
|
|
1157
|
+
await new Promise(r => setTimeout(r, 150));
|
|
1158
|
+
await this.tmux.sendSpecialKey("Enter");
|
|
1159
|
+
}
|
|
1160
|
+
else if (quitKey) {
|
|
1161
|
+
// Key-chord quit (e.g. grok Ctrl+Q) — no typed command to send.
|
|
1162
|
+
await this.tmux.sendSpecialKey(quitKey);
|
|
1163
|
+
}
|
|
1018
1164
|
}
|
|
1019
1165
|
let exited = false;
|
|
1020
1166
|
for (let i = 0; i < 15; i++) {
|
|
@@ -1040,9 +1186,7 @@ export class Daemon extends EventEmitter {
|
|
|
1040
1186
|
}
|
|
1041
1187
|
this.pauseWakeState = "paused";
|
|
1042
1188
|
this.autoPauseController.markPaused();
|
|
1043
|
-
|
|
1044
|
-
paused_at: this.lastPausedAt,
|
|
1045
|
-
}));
|
|
1189
|
+
writePausedMarker(this.instanceDir, this.lastPausedAt ?? Date.now());
|
|
1046
1190
|
this.logger.info({ pausedAt: this.lastPausedAt }, "Instance auto-paused");
|
|
1047
1191
|
this.ipcServer?.broadcast({
|
|
1048
1192
|
type: "instance_state", instanceName: this.name, state: "paused", pausedAt: this.lastPausedAt,
|
|
@@ -1053,6 +1197,7 @@ export class Daemon extends EventEmitter {
|
|
|
1053
1197
|
this.pauseWakeState = "active";
|
|
1054
1198
|
this.healthCheckPaused = false;
|
|
1055
1199
|
this.pauseRequested = false;
|
|
1200
|
+
this.resumeRuntimeMonitors();
|
|
1056
1201
|
throw err;
|
|
1057
1202
|
}
|
|
1058
1203
|
})();
|
|
@@ -1100,11 +1245,9 @@ export class Daemon extends EventEmitter {
|
|
|
1100
1245
|
this.pauseWakeState = "active";
|
|
1101
1246
|
this.healthCheckPaused = false;
|
|
1102
1247
|
this.pauseRequested = false;
|
|
1103
|
-
|
|
1104
|
-
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
1105
|
-
}
|
|
1106
|
-
catch { }
|
|
1248
|
+
clearPausedMarker(this.instanceDir);
|
|
1107
1249
|
this.transcriptMonitor?.resetOffset();
|
|
1250
|
+
this.resumeRuntimeMonitors();
|
|
1108
1251
|
this.logger.info("Instance auto-woke");
|
|
1109
1252
|
this.ipcServer?.broadcast({
|
|
1110
1253
|
type: "instance_state", instanceName: this.name, state: this.instanceState, pausedAt: null,
|
|
@@ -1123,62 +1266,229 @@ export class Daemon extends EventEmitter {
|
|
|
1123
1266
|
this.pauseWakeTransition = null;
|
|
1124
1267
|
}
|
|
1125
1268
|
}
|
|
1269
|
+
applyInstanceStateSnapshot(snapshot, pane) {
|
|
1270
|
+
const previous = this.instanceState;
|
|
1271
|
+
this.instanceState = snapshot.state;
|
|
1272
|
+
// Only a transition back to idle completes pending work. Repeated idle
|
|
1273
|
+
// observations between enqueue and paste must not clear a newer inbound.
|
|
1274
|
+
if (snapshot.state === "idle" && previous !== "idle") {
|
|
1275
|
+
this.pendingWork.recordIdle(snapshot.observedAt);
|
|
1276
|
+
}
|
|
1277
|
+
if (snapshot.state !== previous) {
|
|
1278
|
+
this.logger.info({
|
|
1279
|
+
previousState: previous,
|
|
1280
|
+
state: snapshot.state,
|
|
1281
|
+
unchangedForMs: snapshot.unchangedForMs,
|
|
1282
|
+
}, "Instance execution state changed");
|
|
1283
|
+
this.emit("instance_state", { name: this.name, ...snapshot });
|
|
1284
|
+
this.ipcServer?.broadcast({ type: "instance_state", instanceName: this.name, ...snapshot });
|
|
1285
|
+
if (snapshot.state === "stuck" && pane && this.instanceStateReadyPattern) {
|
|
1286
|
+
this.handleStuckTransition(pane, snapshot, this.instanceStateReadyPattern);
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
if (snapshot.state !== "idle")
|
|
1290
|
+
this.pauseRequested = false;
|
|
1291
|
+
if (!this.pauseRequested && this.pasteQueueDepth === 0 && this.autoPauseController.observe(snapshot.state)) {
|
|
1292
|
+
this.pauseRequested = true;
|
|
1293
|
+
this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
clearInstanceStateIdleTimer() {
|
|
1297
|
+
if (this.instanceStateIdleTimer)
|
|
1298
|
+
clearTimeout(this.instanceStateIdleTimer);
|
|
1299
|
+
this.instanceStateIdleTimer = null;
|
|
1300
|
+
}
|
|
1301
|
+
clearInstanceStateStuckTimer() {
|
|
1302
|
+
if (this.instanceStateStuckTimer)
|
|
1303
|
+
clearTimeout(this.instanceStateStuckTimer);
|
|
1304
|
+
this.instanceStateStuckTimer = null;
|
|
1305
|
+
}
|
|
1306
|
+
scheduleInstanceStateStuckDeadline(changeAt) {
|
|
1307
|
+
// Keep one deadline timer per instance. High-volume TUIs can emit hundreds
|
|
1308
|
+
// of %output records per second; replacing a long timeout for every chunk
|
|
1309
|
+
// would trade tmux subprocess CPU for timer churn.
|
|
1310
|
+
if (this.instanceStateStuckTimer)
|
|
1311
|
+
return;
|
|
1312
|
+
const delay = Math.max(0, changeAt + this.instanceStateStuckTimeoutMs - Date.now());
|
|
1313
|
+
this.instanceStateStuckTimer = setTimeout(() => {
|
|
1314
|
+
this.instanceStateStuckTimer = null;
|
|
1315
|
+
const latestChangeAt = this.instanceStateLastOutputAt || changeAt;
|
|
1316
|
+
if (latestChangeAt + this.instanceStateStuckTimeoutMs > Date.now()) {
|
|
1317
|
+
this.scheduleInstanceStateStuckDeadline(latestChangeAt);
|
|
1318
|
+
return;
|
|
1319
|
+
}
|
|
1320
|
+
void this.captureAndEvaluateInstanceState("stuck_deadline", this.instanceStateLastOutputAt);
|
|
1321
|
+
}, delay);
|
|
1322
|
+
}
|
|
1323
|
+
scheduleInstanceStateIdleCapture() {
|
|
1324
|
+
if (this.instanceStateIdleTimer)
|
|
1325
|
+
return;
|
|
1326
|
+
const expectedOutputAt = this.instanceStateLastOutputAt;
|
|
1327
|
+
const delay = Math.max(0, expectedOutputAt + this.instanceStateIdleDebounceMs - Date.now());
|
|
1328
|
+
this.instanceStateIdleTimer = setTimeout(() => {
|
|
1329
|
+
this.instanceStateIdleTimer = null;
|
|
1330
|
+
const latestOutputAt = this.instanceStateLastOutputAt;
|
|
1331
|
+
if (latestOutputAt > expectedOutputAt
|
|
1332
|
+
&& latestOutputAt + this.instanceStateIdleDebounceMs > Date.now()) {
|
|
1333
|
+
this.scheduleInstanceStateIdleCapture();
|
|
1334
|
+
return;
|
|
1335
|
+
}
|
|
1336
|
+
void this.captureAndEvaluateInstanceState("idle_debounce", latestOutputAt);
|
|
1337
|
+
}, delay);
|
|
1338
|
+
}
|
|
1339
|
+
async captureAndEvaluateInstanceState(reason, expectedOutputAt = 0) {
|
|
1340
|
+
if (!this.instanceStateMonitorActive || this.runtimeMonitorsFrozen || !this.tmux || !this.instanceStateMachine || this.spawning)
|
|
1341
|
+
return;
|
|
1342
|
+
if (this.statePollInFlight) {
|
|
1343
|
+
if (reason === "idle_debounce")
|
|
1344
|
+
this.scheduleInstanceStateIdleCapture();
|
|
1345
|
+
return;
|
|
1346
|
+
}
|
|
1347
|
+
this.statePollInFlight = true;
|
|
1348
|
+
const captureStartedAt = Date.now();
|
|
1349
|
+
try {
|
|
1350
|
+
const pane = await this.tmux.capturePane();
|
|
1351
|
+
// Output received while capture-pane was in flight makes this snapshot
|
|
1352
|
+
// stale. Its output handler has already armed a new debounce.
|
|
1353
|
+
if ((expectedOutputAt > 0 && this.instanceStateLastOutputAt > expectedOutputAt)
|
|
1354
|
+
|| (this.instanceStateLastOutputAt > 0 && this.instanceStateLastOutputAt >= captureStartedAt))
|
|
1355
|
+
return;
|
|
1356
|
+
const observedChangeAt = expectedOutputAt || captureStartedAt;
|
|
1357
|
+
this.instanceStateMachine.observe(pane, observedChangeAt);
|
|
1358
|
+
const snapshot = this.instanceStateMachine.observe(pane, Date.now());
|
|
1359
|
+
this.applyInstanceStateSnapshot(snapshot, pane);
|
|
1360
|
+
if (snapshot.state === "idle") {
|
|
1361
|
+
this.clearInstanceStateStuckTimer();
|
|
1362
|
+
}
|
|
1363
|
+
else if (snapshot.state === "working") {
|
|
1364
|
+
// If control mode missed the pane change, the safety capture becomes
|
|
1365
|
+
// the new progress timestamp and re-arms both deadlines.
|
|
1366
|
+
if (!expectedOutputAt)
|
|
1367
|
+
this.instanceStateLastOutputAt = captureStartedAt;
|
|
1368
|
+
this.scheduleInstanceStateStuckDeadline(this.instanceStateLastOutputAt || captureStartedAt);
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
catch (err) {
|
|
1372
|
+
this.logger.debug({ err: err.message, reason }, "Instance state capture failed");
|
|
1373
|
+
}
|
|
1374
|
+
finally {
|
|
1375
|
+
this.statePollInFlight = false;
|
|
1376
|
+
}
|
|
1377
|
+
}
|
|
1378
|
+
handleInstancePaneOutput(event) {
|
|
1379
|
+
if (!this.instanceStateMonitorActive || this.runtimeMonitorsFrozen)
|
|
1380
|
+
return;
|
|
1381
|
+
const windowId = this.tmux?.getWindowId();
|
|
1382
|
+
if (!windowId || event.windowId !== windowId || !this.instanceStateMachine)
|
|
1383
|
+
return;
|
|
1384
|
+
this.instanceStateLastOutputAt = event.at;
|
|
1385
|
+
const snapshot = this.instanceStateMachine.recordOutput(event.at);
|
|
1386
|
+
this.applyInstanceStateSnapshot(snapshot);
|
|
1387
|
+
this.scheduleInstanceStateIdleCapture();
|
|
1388
|
+
this.scheduleInstanceStateStuckDeadline(event.at);
|
|
1389
|
+
}
|
|
1390
|
+
bindInstanceStateOutputListener(windowId) {
|
|
1391
|
+
if (!this.controlClient || !this.instanceStateMonitorActive)
|
|
1392
|
+
return;
|
|
1393
|
+
if (!this.instanceStateOutputListener) {
|
|
1394
|
+
this.instanceStateOutputListener = event => this.handleInstancePaneOutput(event);
|
|
1395
|
+
}
|
|
1396
|
+
if (this.instanceStateOutputEventName) {
|
|
1397
|
+
this.controlClient.removeListener(this.instanceStateOutputEventName, this.instanceStateOutputListener);
|
|
1398
|
+
}
|
|
1399
|
+
this.instanceStateOutputEventName = `output:${windowId}`;
|
|
1400
|
+
this.controlClient.on(this.instanceStateOutputEventName, this.instanceStateOutputListener);
|
|
1401
|
+
}
|
|
1126
1402
|
startInstanceStateMonitor() {
|
|
1127
|
-
if (!this.tmux || !this.backend || this.
|
|
1403
|
+
if (this.runtimeMonitorsFrozen || !this.tmux || !this.backend || this.instanceStateMonitorActive)
|
|
1128
1404
|
return;
|
|
1129
1405
|
const rawConfig = this.config.hang_detector;
|
|
1130
1406
|
const timeoutMinutes = rawConfig?.timeout_minutes;
|
|
1131
|
-
|
|
1407
|
+
this.instanceStateStuckTimeoutMs = typeof timeoutMinutes === "number" && timeoutMinutes > 0
|
|
1132
1408
|
? timeoutMinutes * 60_000
|
|
1133
1409
|
: DEFAULT_STUCK_TIMEOUT_MS;
|
|
1134
|
-
|
|
1135
|
-
? rawConfig.
|
|
1136
|
-
:
|
|
1137
|
-
this.
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1410
|
+
this.instanceStateIdleDebounceMs = typeof rawConfig?.idle_debounce_ms === "number" && rawConfig.idle_debounce_ms >= 0
|
|
1411
|
+
? rawConfig.idle_debounce_ms
|
|
1412
|
+
: DEFAULT_STATE_IDLE_DEBOUNCE_MS;
|
|
1413
|
+
this.instanceStateReadyPattern = this.backend.getReadyPattern();
|
|
1414
|
+
this.instanceStateMachine = new PaneStateMachine(this.instanceStateReadyPattern, this.instanceStateStuckTimeoutMs);
|
|
1415
|
+
this.instanceStateLastOutputAt = 0;
|
|
1416
|
+
this.instanceStateMonitorActive = true;
|
|
1417
|
+
if (this.controlClient) {
|
|
1418
|
+
this.instanceStateSafetyListener = () => { void this.captureAndEvaluateInstanceState("safety_sweep"); };
|
|
1419
|
+
this.bindInstanceStateOutputListener(this.tmux.getWindowId());
|
|
1420
|
+
this.controlClient.on("safety_sweep", this.instanceStateSafetyListener);
|
|
1421
|
+
}
|
|
1422
|
+
else {
|
|
1423
|
+
// Standalone/test fallback. Fleet production uses the one shared control
|
|
1424
|
+
// client and therefore one fleet-level safety sweep instead of N timers.
|
|
1425
|
+
this.instanceStateMonitorTimer = setInterval(() => {
|
|
1426
|
+
void this.captureAndEvaluateInstanceState("safety_sweep");
|
|
1427
|
+
}, DEFAULT_STATE_SAFETY_SWEEP_MS);
|
|
1428
|
+
}
|
|
1429
|
+
void this.captureAndEvaluateInstanceState("initial");
|
|
1430
|
+
}
|
|
1431
|
+
stopInstanceStateMonitor() {
|
|
1432
|
+
this.instanceStateMonitorActive = false;
|
|
1433
|
+
this.clearInstanceStateIdleTimer();
|
|
1434
|
+
this.clearInstanceStateStuckTimer();
|
|
1435
|
+
if (this.instanceStateMonitorTimer)
|
|
1436
|
+
clearInterval(this.instanceStateMonitorTimer);
|
|
1437
|
+
this.instanceStateMonitorTimer = null;
|
|
1438
|
+
if (this.controlClient && this.instanceStateOutputListener && this.instanceStateOutputEventName) {
|
|
1439
|
+
this.controlClient.removeListener(this.instanceStateOutputEventName, this.instanceStateOutputListener);
|
|
1440
|
+
}
|
|
1441
|
+
if (this.controlClient && this.instanceStateSafetyListener) {
|
|
1442
|
+
this.controlClient.removeListener("safety_sweep", this.instanceStateSafetyListener);
|
|
1443
|
+
}
|
|
1444
|
+
this.instanceStateOutputListener = null;
|
|
1445
|
+
this.instanceStateOutputEventName = null;
|
|
1446
|
+
this.instanceStateSafetyListener = null;
|
|
1447
|
+
}
|
|
1448
|
+
handleStuckTransition(pane, snapshot, readyPattern) {
|
|
1449
|
+
const deterministicReadyPattern = new RegExp(readyPattern.source, readyPattern.flags.replace(/[gy]/g, ""));
|
|
1450
|
+
const diagnostic = {
|
|
1451
|
+
backend: this.backend?.binaryName ?? this.config.backend ?? "unknown",
|
|
1452
|
+
paneTail: sanitizePaneTail(pane),
|
|
1453
|
+
readyPattern: readyPattern.toString(),
|
|
1454
|
+
readyMatched: deterministicReadyPattern.test(pane),
|
|
1455
|
+
unchangedForMs: snapshot.unchangedForMs,
|
|
1456
|
+
pendingWork: this.pendingWork.hasPendingWork(),
|
|
1179
1457
|
};
|
|
1180
|
-
|
|
1181
|
-
|
|
1458
|
+
if (!diagnostic.pendingWork) {
|
|
1459
|
+
this.logger.debug(diagnostic, "Suppressing stuck notification without pending work");
|
|
1460
|
+
return;
|
|
1461
|
+
}
|
|
1462
|
+
this.logger.warn(diagnostic, "Instance pane stuck with pending work");
|
|
1463
|
+
this.hangDetector?.emit("hang", { unchangedForMs: snapshot.unchangedForMs });
|
|
1464
|
+
}
|
|
1465
|
+
/** Stop every runtime poller/watcher while preserving IPC and daemon state. */
|
|
1466
|
+
freezeRuntimeMonitors() {
|
|
1467
|
+
this.runtimeMonitorsFrozen = true;
|
|
1468
|
+
if (this.healthCheckTimer) {
|
|
1469
|
+
clearTimeout(this.healthCheckTimer);
|
|
1470
|
+
this.healthCheckTimer = null;
|
|
1471
|
+
}
|
|
1472
|
+
if (this.errorMonitorTimer) {
|
|
1473
|
+
clearInterval(this.errorMonitorTimer);
|
|
1474
|
+
this.errorMonitorTimer = null;
|
|
1475
|
+
}
|
|
1476
|
+
this.stopInstanceStateMonitor();
|
|
1477
|
+
this.transcriptMonitor?.stop();
|
|
1478
|
+
this.guardian?.stop();
|
|
1479
|
+
}
|
|
1480
|
+
/** Restore the same monitor objects after wake without adding event listeners. */
|
|
1481
|
+
resumeRuntimeMonitors() {
|
|
1482
|
+
if (!this.runtimeMonitorsFrozen)
|
|
1483
|
+
return;
|
|
1484
|
+
this.runtimeMonitorsFrozen = false;
|
|
1485
|
+
this.startHealthCheck();
|
|
1486
|
+
if (!this.config.lightweight) {
|
|
1487
|
+
this.transcriptMonitor?.startPolling();
|
|
1488
|
+
this.guardian?.startWatching();
|
|
1489
|
+
this.startErrorMonitor();
|
|
1490
|
+
}
|
|
1491
|
+
this.startInstanceStateMonitor();
|
|
1182
1492
|
}
|
|
1183
1493
|
getMessageBus() {
|
|
1184
1494
|
return this.messageBus;
|
|
@@ -1258,12 +1568,23 @@ export class Daemon extends EventEmitter {
|
|
|
1258
1568
|
// Remember (and persist) the reply target. Only real channel messages have a
|
|
1259
1569
|
// non-empty chat_id; cross-instance messages have chat_id="" and must NOT
|
|
1260
1570
|
// overwrite it (their reply would otherwise go nowhere).
|
|
1261
|
-
this.updateLastChat(meta.chat_id, meta.thread_id);
|
|
1571
|
+
this.updateLastChat(meta.chat_id, meta.thread_id, meta.adapter_id);
|
|
1572
|
+
if (meta.chat_id) {
|
|
1573
|
+
const inboundAt = Date.now();
|
|
1574
|
+
this.autoPauseController.recordActivity(inboundAt);
|
|
1575
|
+
try {
|
|
1576
|
+
writeLastInboundAt(this.instanceDir, inboundAt);
|
|
1577
|
+
}
|
|
1578
|
+
catch (err) {
|
|
1579
|
+
this.logger.warn({ err: err.message }, "Failed to persist last inbound timestamp");
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1262
1582
|
if (this.pendingInstructionsUpdate) {
|
|
1263
1583
|
writeFileSync(join(this.instanceDir, "prev-instructions"), this.pendingInstructionsUpdate);
|
|
1264
1584
|
this.pendingInstructionsUpdate = undefined;
|
|
1265
1585
|
}
|
|
1266
1586
|
this.hangDetector?.recordInbound();
|
|
1587
|
+
this.pendingWork.recordInbound();
|
|
1267
1588
|
// v3: record user messages for rotation snapshot
|
|
1268
1589
|
this.recordRecentUserMessage(content, meta);
|
|
1269
1590
|
// Format message with metadata prefix for the agent
|
|
@@ -1282,7 +1603,10 @@ export class Daemon extends EventEmitter {
|
|
|
1282
1603
|
}
|
|
1283
1604
|
let formatted;
|
|
1284
1605
|
if (fromInstance) {
|
|
1285
|
-
|
|
1606
|
+
// #77: show the sender's display name for readability, keeping the machine
|
|
1607
|
+
// instance name in parens so the recipient's send_to_instance target is valid.
|
|
1608
|
+
const fromLabel = meta.from_display ? `${meta.from_display} (${fromInstance})` : fromInstance;
|
|
1609
|
+
formatted = `[from:${fromLabel}] ${content}\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)`;
|
|
1286
1610
|
}
|
|
1287
1611
|
else {
|
|
1288
1612
|
const via = meta.source ? ` via ${meta.source}` : "";
|
|
@@ -1313,7 +1637,7 @@ export class Daemon extends EventEmitter {
|
|
|
1313
1637
|
}
|
|
1314
1638
|
if (this.pendingInstructionsNotice) {
|
|
1315
1639
|
this.pendingInstructionsNotice = false;
|
|
1316
|
-
await this.deliverMessage(
|
|
1640
|
+
await this.deliverMessage(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir));
|
|
1317
1641
|
}
|
|
1318
1642
|
const status = (chatId && messageId)
|
|
1319
1643
|
? { chatId: meta.thread_id || chatId, messageId }
|
|
@@ -1370,8 +1694,14 @@ export class Daemon extends EventEmitter {
|
|
|
1370
1694
|
await new Promise(r => setTimeout(r, 2000));
|
|
1371
1695
|
continue;
|
|
1372
1696
|
}
|
|
1373
|
-
// Settle the bracketed paste, then submit.
|
|
1374
|
-
|
|
1697
|
+
// Settle the bracketed paste, then submit. The first delivery immediately
|
|
1698
|
+
// after ready gets a longer one-shot delay because the TUI may still be
|
|
1699
|
+
// completing its final redraw even though the prompt already matched.
|
|
1700
|
+
const enterSettleMs = this.firstDeliveryDelay.consume();
|
|
1701
|
+
if (enterSettleMs > NORMAL_ENTER_SETTLE_MS) {
|
|
1702
|
+
this.logger.debug({ enterSettleMs }, "First delivery after ready — extending Enter settle delay");
|
|
1703
|
+
}
|
|
1704
|
+
await new Promise(r => setTimeout(r, enterSettleMs));
|
|
1375
1705
|
const enterAt = Date.now();
|
|
1376
1706
|
await this.tmux.sendSpecialKey("Enter");
|
|
1377
1707
|
if (status)
|
|
@@ -1414,6 +1744,7 @@ export class Daemon extends EventEmitter {
|
|
|
1414
1744
|
this.tmux = new TmuxManager(this.tmuxSessionName, match.id);
|
|
1415
1745
|
writeFileSync(join(this.instanceDir, "window-id"), match.id);
|
|
1416
1746
|
await this.controlClient?.registerWindow(match.id);
|
|
1747
|
+
this.bindInstanceStateOutputListener(match.id);
|
|
1417
1748
|
this.logger.info({ windowId: match.id }, "Recovered window ID for message delivery");
|
|
1418
1749
|
return match.id;
|
|
1419
1750
|
}
|
|
@@ -1939,6 +2270,7 @@ export class Daemon extends EventEmitter {
|
|
|
1939
2270
|
}
|
|
1940
2271
|
// Register with control client and wait for output + idle
|
|
1941
2272
|
await this.controlClient?.registerWindow(windowId);
|
|
2273
|
+
this.bindInstanceStateOutputListener(windowId);
|
|
1942
2274
|
if (this.controlClient) {
|
|
1943
2275
|
const total = startupTimeoutMs ?? this.config.startup_timeout_ms ?? 25_000;
|
|
1944
2276
|
const outputTimeout = Math.round(total * 0.6);
|
|
@@ -1963,7 +2295,10 @@ export class Daemon extends EventEmitter {
|
|
|
1963
2295
|
// but a startup crash would already be caught by waitForOutput/waitForIdle above.
|
|
1964
2296
|
if (!await this.tmux.isWindowAlive())
|
|
1965
2297
|
return false;
|
|
1966
|
-
|
|
2298
|
+
const ready = await this.dismissDialogsUntilReady(3);
|
|
2299
|
+
if (ready)
|
|
2300
|
+
this.firstDeliveryDelay.recordReady();
|
|
2301
|
+
return ready;
|
|
1967
2302
|
}
|
|
1968
2303
|
/**
|
|
1969
2304
|
* Repeatedly check pane content, dismiss any confirmation dialogs,
|
|
@@ -2030,14 +2365,17 @@ export class Daemon extends EventEmitter {
|
|
|
2030
2365
|
* messages) so it never overwrites a real channel target. Persisted to
|
|
2031
2366
|
* last-chat.json so the reply target survives a restart (see start()).
|
|
2032
2367
|
*/
|
|
2033
|
-
updateLastChat(chatId, threadId) {
|
|
2368
|
+
updateLastChat(chatId, threadId, adapterId) {
|
|
2034
2369
|
if (!chatId)
|
|
2035
2370
|
return;
|
|
2036
2371
|
this.lastChatId = chatId;
|
|
2037
|
-
|
|
2038
|
-
|
|
2372
|
+
// An unthreaded inbound must clear a previous topic rather than leaking it
|
|
2373
|
+
// into the next reply target.
|
|
2374
|
+
this.lastThreadId = threadId || undefined;
|
|
2375
|
+
if (adapterId)
|
|
2376
|
+
this.lastAdapterId = adapterId;
|
|
2039
2377
|
try {
|
|
2040
|
-
writeFileSync(join(this.instanceDir, "last-chat.json"), JSON.stringify({ chatId: this.lastChatId, threadId: this.lastThreadId }));
|
|
2378
|
+
writeFileSync(join(this.instanceDir, "last-chat.json"), JSON.stringify({ chatId: this.lastChatId, threadId: this.lastThreadId, adapterId: this.lastAdapterId }));
|
|
2041
2379
|
}
|
|
2042
2380
|
catch { /* best effort */ }
|
|
2043
2381
|
}
|