@songsid/agend 2.1.0-beta.4 → 2.1.0-beta.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/README.zh-TW.md +2 -1
- package/dist/access-path.js +3 -3
- package/dist/access-path.js.map +1 -1
- package/dist/agent-cli.js +1 -1
- package/dist/agent-cli.js.map +1 -1
- package/dist/agent-endpoint.d.ts +8 -0
- package/dist/agent-endpoint.js +36 -8
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +2 -0
- package/dist/backend/antigravity.js +30 -3
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +2 -0
- package/dist/backend/claude-code.js +16 -4
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/factory.js +4 -1
- package/dist/backend/factory.js.map +1 -1
- package/dist/backend/grok.d.ts +53 -0
- package/dist/backend/grok.js +302 -0
- package/dist/backend/grok.js.map +1 -0
- package/dist/backend/kiro.d.ts +2 -0
- package/dist/backend/kiro.js +32 -3
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +36 -3
- package/dist/backend/types.js +1 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +20 -0
- package/dist/channel/adapters/discord.js +89 -11
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.js +4 -1
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/mcp-tools.js +18 -2
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/chat-export.js +5 -2
- package/dist/chat-export.js.map +1 -1
- package/dist/classic-channel-manager.d.ts +16 -1
- package/dist/classic-channel-manager.js +76 -4
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +116 -19
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +70 -1
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +3 -1
- package/dist/config.js +14 -1
- package/dist/config.js.map +1 -1
- package/dist/context-guardian.d.ts +2 -0
- package/dist/context-guardian.js +10 -2
- package/dist/context-guardian.js.map +1 -1
- package/dist/daemon-entry.js +2 -1
- package/dist/daemon-entry.js.map +1 -1
- package/dist/daemon.d.ts +91 -6
- package/dist/daemon.js +602 -186
- package/dist/daemon.js.map +1 -1
- package/dist/daily-summary.js +7 -2
- package/dist/daily-summary.js.map +1 -1
- package/dist/fleet-context.d.ts +6 -0
- package/dist/fleet-manager.d.ts +113 -8
- package/dist/fleet-manager.js +972 -142
- package/dist/fleet-manager.js.map +1 -1
- package/dist/fleet-system-prompt.js +1 -1
- package/dist/general-knowledge/skills/fleet-config/SKILL.md +19 -13
- package/dist/general-knowledge/skills/fleet-health/SKILL.md +30 -51
- package/dist/general-knowledge/skills/fleet-restart/SKILL.md +1 -1
- package/dist/general-knowledge/skills/instance-lifecycle/SKILL.md +19 -13
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +1 -5
- package/dist/general-knowledge/skills/scheduling/SKILL.md +41 -0
- package/dist/general-knowledge/skills/session-management/SKILL.md +29 -116
- package/dist/instance-lifecycle.d.ts +15 -1
- package/dist/instance-lifecycle.js +71 -22
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/locale.js +28 -2
- package/dist/locale.js.map +1 -1
- package/dist/logger.d.ts +6 -0
- package/dist/logger.js +28 -9
- package/dist/logger.js.map +1 -1
- package/dist/outbound-handlers.d.ts +3 -0
- package/dist/outbound-handlers.js +224 -11
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +44 -2
- package/dist/outbound-schemas.js +50 -3
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pause-marker.d.ts +5 -0
- package/dist/pause-marker.js +36 -0
- package/dist/pause-marker.js.map +1 -0
- package/dist/scheduler/db.d.ts +8 -0
- package/dist/scheduler/db.js +74 -5
- package/dist/scheduler/db.js.map +1 -1
- package/dist/scheduler/db.test.js +50 -0
- package/dist/scheduler/db.test.js.map +1 -1
- package/dist/scheduler/scheduler.d.ts +6 -0
- package/dist/scheduler/scheduler.js +99 -16
- package/dist/scheduler/scheduler.js.map +1 -1
- package/dist/scheduler/scheduler.test.js +131 -1
- package/dist/scheduler/scheduler.test.js.map +1 -1
- package/dist/scheduler/types.d.ts +9 -3
- package/dist/scheduler/types.js.map +1 -1
- package/dist/settings-api.d.ts +20 -2
- package/dist/settings-api.js +213 -9
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +8 -0
- package/dist/setup-wizard.js.map +1 -1
- package/dist/statusline-watcher.d.ts +1 -1
- package/dist/statusline-watcher.js +3 -2
- package/dist/statusline-watcher.js.map +1 -1
- package/dist/tmux-control.d.ts +7 -0
- package/dist/tmux-control.js +22 -1
- package/dist/tmux-control.js.map +1 -1
- package/dist/tmux-manager.d.ts +1 -1
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-commands.d.ts +19 -1
- package/dist/topic-commands.js +148 -23
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-monitor.js +2 -0
- package/dist/transcript-monitor.js.map +1 -1
- package/dist/types.d.ts +29 -1
- package/dist/tz-utils.d.ts +26 -0
- package/dist/tz-utils.js +46 -0
- package/dist/tz-utils.js.map +1 -0
- package/dist/ui/dashboard.html +3 -3
- package/dist/ui/settings.html +453 -74
- package/dist/web-api.d.ts +1 -0
- package/dist/web-api.js +24 -15
- package/dist/web-api.js.map +1 -1
- package/dist/workflow-templates/default.md +0 -4
- package/package.json +2 -1
package/dist/daemon.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { join, dirname, basename, resolve } from "node:path";
|
|
2
|
-
import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync, chmodSync } from "node:fs";
|
|
2
|
+
import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync, chmodSync, renameSync } from "node:fs";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import { createHash, randomBytes } from "node:crypto";
|
|
5
5
|
import { EventEmitter } from "node:events";
|
|
6
|
-
import {
|
|
6
|
+
import { clearPausedMarker, writePausedMarker } from "./pause-marker.js";
|
|
7
7
|
import { TmuxManager } from "./tmux-manager.js";
|
|
8
8
|
import { TranscriptMonitor } from "./transcript-monitor.js";
|
|
9
9
|
import { ContextGuardian } from "./context-guardian.js";
|
|
@@ -21,32 +21,71 @@ const CROSS_INSTANCE_TOOLS = new Set(["send_to_instance", "list_instances", "sta
|
|
|
21
21
|
const SCHEDULE_TOOLS = new Set(["create_schedule", "list_schedules", "update_schedule", "delete_schedule"]);
|
|
22
22
|
const DECISION_TOOLS = new Set(["post_decision", "list_decisions", "update_decision"]);
|
|
23
23
|
const TASK_TOOL = "task";
|
|
24
|
+
/** Point a resumed CLI at its one backend-native instruction source. */
|
|
25
|
+
export function buildInstructionReloadNotice(binaryName, instanceName, instanceDir) {
|
|
26
|
+
const source = binaryName === "codex" || binaryName === "grok"
|
|
27
|
+
? "AGENTS.md"
|
|
28
|
+
: binaryName === "kiro-cli"
|
|
29
|
+
? `.kiro/steering/agend-${instanceName}.md`
|
|
30
|
+
: binaryName === "agy"
|
|
31
|
+
? ".agents/agents.md"
|
|
32
|
+
: binaryName === "gemini"
|
|
33
|
+
? "GEMINI.md"
|
|
34
|
+
: join(instanceDir, "fleet-instructions.md");
|
|
35
|
+
return `[system] Your AgEnD instructions have been updated. Reload only ${source}; do not scan other instruction directories. Do not reply to this message.`;
|
|
36
|
+
}
|
|
24
37
|
export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
|
|
25
|
-
export const
|
|
26
|
-
|
|
38
|
+
export const DEFAULT_STATE_IDLE_DEBOUNCE_MS = 2_000;
|
|
39
|
+
export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
|
|
40
|
+
/** @deprecated State detection is event-driven; this now aliases the safety sweep. */
|
|
41
|
+
export const DEFAULT_STATE_POLL_INTERVAL_MS = DEFAULT_STATE_SAFETY_SWEEP_MS;
|
|
42
|
+
const LAST_INBOUND_FILE = "last-inbound-at";
|
|
43
|
+
/** Read the last real channel inbound timestamp persisted across daemon restarts. */
|
|
44
|
+
export function readLastInboundAt(instanceDir, now = Date.now()) {
|
|
45
|
+
try {
|
|
46
|
+
const value = Number(readFileSync(join(instanceDir, LAST_INBOUND_FILE), "utf-8").trim());
|
|
47
|
+
return Number.isFinite(value) && value >= 0 && value <= now ? value : null;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** Atomically persist the last real channel inbound timestamp. */
|
|
54
|
+
export function writeLastInboundAt(instanceDir, timestamp) {
|
|
55
|
+
mkdirSync(instanceDir, { recursive: true });
|
|
56
|
+
const target = join(instanceDir, LAST_INBOUND_FILE);
|
|
57
|
+
const temp = `${target}.${process.pid}.tmp`;
|
|
58
|
+
writeFileSync(temp, String(timestamp));
|
|
59
|
+
renameSync(temp, target);
|
|
60
|
+
}
|
|
61
|
+
/** Headless inactivity timer used by the daemon and unit tests. */
|
|
27
62
|
export class AutoPauseController {
|
|
28
63
|
thresholdMs;
|
|
29
|
-
|
|
64
|
+
lastActivityAt;
|
|
30
65
|
pausedAt = null;
|
|
31
|
-
constructor(thresholdMs) {
|
|
66
|
+
constructor(thresholdMs, lastActivityAt = Date.now()) {
|
|
32
67
|
this.thresholdMs = thresholdMs;
|
|
68
|
+
this.lastActivityAt = lastActivityAt;
|
|
69
|
+
}
|
|
70
|
+
recordActivity(now = Date.now()) {
|
|
71
|
+
this.lastActivityAt = now;
|
|
33
72
|
}
|
|
34
73
|
observe(state, now = Date.now()) {
|
|
35
74
|
if (this.pausedAt !== null || this.thresholdMs <= 0)
|
|
36
75
|
return false;
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
this.idleSince ??= now;
|
|
42
|
-
return now - this.idleSince >= this.thresholdMs;
|
|
76
|
+
// Inactivity is based on the last user inbound, not on how long this daemon
|
|
77
|
+
// has observed the idle pane. This preserves pause eligibility across fleet
|
|
78
|
+
// restarts while still ensuring an actively working instance is never paused.
|
|
79
|
+
return state === "idle" && now - this.lastActivityAt >= this.thresholdMs;
|
|
43
80
|
}
|
|
44
81
|
markPaused(now = Date.now()) {
|
|
45
82
|
this.pausedAt = now;
|
|
46
83
|
}
|
|
47
|
-
markAwake() {
|
|
84
|
+
markAwake(now = Date.now()) {
|
|
48
85
|
this.pausedAt = null;
|
|
49
|
-
|
|
86
|
+
// A deliberate wake is activity in its own right and must provide a fresh
|
|
87
|
+
// window for the wake-before-deliver facade to enqueue its inbound message.
|
|
88
|
+
this.lastActivityAt = now;
|
|
50
89
|
}
|
|
51
90
|
async wakeOnDeliver(wake) {
|
|
52
91
|
if (this.pausedAt === null)
|
|
@@ -106,6 +145,16 @@ export class PaneStateMachine {
|
|
|
106
145
|
}
|
|
107
146
|
return this.snapshot(now);
|
|
108
147
|
}
|
|
148
|
+
/** Record pane motion from tmux control mode without capturing pane content. */
|
|
149
|
+
recordOutput(now = Date.now()) {
|
|
150
|
+
this.lastPaneChangeAt = now;
|
|
151
|
+
this.lastObservedAt = now;
|
|
152
|
+
if (this.currentState !== "working") {
|
|
153
|
+
this.currentState = "working";
|
|
154
|
+
this.stateChangedAt = now;
|
|
155
|
+
}
|
|
156
|
+
return this.snapshot(now);
|
|
157
|
+
}
|
|
109
158
|
snapshot(now = Date.now()) {
|
|
110
159
|
return {
|
|
111
160
|
state: this.currentState,
|
|
@@ -115,6 +164,79 @@ export class PaneStateMachine {
|
|
|
115
164
|
};
|
|
116
165
|
}
|
|
117
166
|
}
|
|
167
|
+
/** Tracks whether an inbound arrived after the most recent confirmed idle prompt. */
|
|
168
|
+
export class PendingWorkTracker {
|
|
169
|
+
lastInboundAt = 0;
|
|
170
|
+
lastIdleAt;
|
|
171
|
+
sequence = 0;
|
|
172
|
+
lastInboundOrder = 0;
|
|
173
|
+
lastIdleOrder = 0;
|
|
174
|
+
constructor(now = Date.now()) {
|
|
175
|
+
this.lastIdleAt = now;
|
|
176
|
+
}
|
|
177
|
+
recordInbound(now = Date.now()) {
|
|
178
|
+
this.lastInboundAt = now;
|
|
179
|
+
this.lastInboundOrder = ++this.sequence;
|
|
180
|
+
}
|
|
181
|
+
recordIdle(now = Date.now()) {
|
|
182
|
+
// An async pane poll can finish after a newer inbound. Do not let its stale
|
|
183
|
+
// observation clear work which had not arrived when the pane was captured.
|
|
184
|
+
if (now < this.lastInboundAt)
|
|
185
|
+
return;
|
|
186
|
+
this.lastIdleAt = now;
|
|
187
|
+
this.lastIdleOrder = ++this.sequence;
|
|
188
|
+
}
|
|
189
|
+
hasPendingWork() {
|
|
190
|
+
return this.lastInboundOrder > this.lastIdleOrder;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
const NORMAL_ENTER_SETTLE_MS = 500;
|
|
194
|
+
const FIRST_ENTER_SETTLE_MS = 1_750;
|
|
195
|
+
const FIRST_DELIVERY_WINDOW_MS = 5_000;
|
|
196
|
+
/**
|
|
197
|
+
* One-shot timing gate for the first paste after a CLI reaches its ready
|
|
198
|
+
* prompt. Recent tmux versions expose a short interval where the prompt is
|
|
199
|
+
* visible but the TUI can still swallow Enter while completing its redraw.
|
|
200
|
+
*/
|
|
201
|
+
export class FirstDeliveryDelay {
|
|
202
|
+
readyAt = 0;
|
|
203
|
+
pending = false;
|
|
204
|
+
recordReady(now = Date.now()) {
|
|
205
|
+
this.readyAt = now;
|
|
206
|
+
this.pending = true;
|
|
207
|
+
}
|
|
208
|
+
consume(now = Date.now()) {
|
|
209
|
+
if (!this.pending)
|
|
210
|
+
return NORMAL_ENTER_SETTLE_MS;
|
|
211
|
+
this.pending = false;
|
|
212
|
+
return this.readyAt > 0 && now - this.readyAt < FIRST_DELIVERY_WINDOW_MS
|
|
213
|
+
? FIRST_ENTER_SETTLE_MS
|
|
214
|
+
: NORMAL_ENTER_SETTLE_MS;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
/** Redact likely credentials and control sequences before pane text reaches logs. */
|
|
218
|
+
export function sanitizePaneTail(pane, lineCount = 5) {
|
|
219
|
+
const secretAssignment = /\b(token|secret|password|passwd|api[_-]?key|authorization)\b\s*[:=]\s*\S+/gi;
|
|
220
|
+
const bearer = /\bBearer\s+\S+/gi;
|
|
221
|
+
const knownToken = /\b(?:sk-[A-Za-z0-9_-]+|ghp_[A-Za-z0-9]+|github_pat_[A-Za-z0-9_]+|AKIA[A-Z0-9]{16})\b/g;
|
|
222
|
+
const jwt = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g;
|
|
223
|
+
const opaqueSecret = /\b[A-Za-z0-9_+/=-]{32,}\b/g;
|
|
224
|
+
const lines = pane
|
|
225
|
+
.replace(/\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g, "")
|
|
226
|
+
.split(/\r?\n/);
|
|
227
|
+
while (lines.length > 0 && /^\s*$/.test(lines[lines.length - 1]))
|
|
228
|
+
lines.pop();
|
|
229
|
+
return lines
|
|
230
|
+
.slice(-lineCount)
|
|
231
|
+
.map(line => line
|
|
232
|
+
.replace(bearer, "Bearer [REDACTED]")
|
|
233
|
+
.replace(secretAssignment, "$1=[REDACTED]")
|
|
234
|
+
.replace(knownToken, "[REDACTED]")
|
|
235
|
+
.replace(jwt, "[REDACTED]")
|
|
236
|
+
.replace(opaqueSecret, "[REDACTED]")
|
|
237
|
+
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "")
|
|
238
|
+
.slice(0, 200));
|
|
239
|
+
}
|
|
118
240
|
export class Daemon extends EventEmitter {
|
|
119
241
|
name;
|
|
120
242
|
config;
|
|
@@ -135,6 +257,7 @@ export class Daemon extends EventEmitter {
|
|
|
135
257
|
// Track chatId/threadId from inbound messages for automatic outbound routing
|
|
136
258
|
lastChatId;
|
|
137
259
|
lastThreadId;
|
|
260
|
+
lastAdapterId;
|
|
138
261
|
// Pending ack: react 🫡 on first transcript activity after receiving a message
|
|
139
262
|
pendingAckMessage = null;
|
|
140
263
|
// Tool status tracking for channel adapter
|
|
@@ -153,6 +276,8 @@ export class Daemon extends EventEmitter {
|
|
|
153
276
|
lastSpawnAt = 0;
|
|
154
277
|
crashTimestamps = [];
|
|
155
278
|
healthCheckPaused = false;
|
|
279
|
+
/** CLI pane availability, independent from the daemon process and tri-state. */
|
|
280
|
+
processStatus = "running";
|
|
156
281
|
spawning = false;
|
|
157
282
|
skipResume = false;
|
|
158
283
|
backgroundSessionRecoveryAttempted = false;
|
|
@@ -164,7 +289,19 @@ export class Daemon extends EventEmitter {
|
|
|
164
289
|
hangDetector = null;
|
|
165
290
|
instanceState = "idle";
|
|
166
291
|
instanceStateMachine = null;
|
|
292
|
+
pendingWork = new PendingWorkTracker();
|
|
293
|
+
/** Fallback safety sweep used only when no shared tmux control client exists. */
|
|
167
294
|
instanceStateMonitorTimer = null;
|
|
295
|
+
instanceStateIdleTimer = null;
|
|
296
|
+
instanceStateStuckTimer = null;
|
|
297
|
+
instanceStateOutputListener = null;
|
|
298
|
+
instanceStateOutputEventName = null;
|
|
299
|
+
instanceStateSafetyListener = null;
|
|
300
|
+
instanceStateLastOutputAt = 0;
|
|
301
|
+
instanceStateIdleDebounceMs = DEFAULT_STATE_IDLE_DEBOUNCE_MS;
|
|
302
|
+
instanceStateStuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS;
|
|
303
|
+
instanceStateReadyPattern = null;
|
|
304
|
+
instanceStateMonitorActive = false;
|
|
168
305
|
statePollInFlight = false;
|
|
169
306
|
autoPauseController;
|
|
170
307
|
pauseRequested = false;
|
|
@@ -187,10 +324,15 @@ export class Daemon extends EventEmitter {
|
|
|
187
324
|
warmupNeeded = false;
|
|
188
325
|
lastBuiltInstructions = "";
|
|
189
326
|
pasteQueueDepth = 0;
|
|
327
|
+
firstDeliveryDelay = new FirstDeliveryDelay();
|
|
190
328
|
// PTY error pattern monitoring
|
|
191
329
|
errorMonitorTimer = null;
|
|
330
|
+
/** Prevent in-flight monitor callbacks from re-arming after a pause. */
|
|
331
|
+
runtimeMonitorsFrozen = false;
|
|
192
332
|
errorWaitingForRecovery = false; // true = error detected, waiting for ready pattern
|
|
193
333
|
errorDetectedAt = 0;
|
|
334
|
+
errorRecoveryDeadlineAt = 0;
|
|
335
|
+
activeErrorPatternKey = null;
|
|
194
336
|
/** Whether this instance is in an abnormal error state (auto-pause is normal). */
|
|
195
337
|
get isErrorState() {
|
|
196
338
|
return this.errorWaitingForRecovery || (this.healthCheckPaused && !this.isPaused) || Daemon.tmuxServerPaused;
|
|
@@ -202,11 +344,16 @@ export class Daemon extends EventEmitter {
|
|
|
202
344
|
get isCrashLoop() {
|
|
203
345
|
return this.crashCount >= 3;
|
|
204
346
|
}
|
|
347
|
+
/** The most recent error type detected by the error monitor (e.g. "rate_limit", "auth_error"). */
|
|
348
|
+
get lastErrorType() {
|
|
349
|
+
return this.lastDetectedErrorType;
|
|
350
|
+
}
|
|
205
351
|
lastFailoverAt = 0; // cooldown: prevent repeated failover triggers
|
|
206
352
|
static FAILOVER_COOLDOWN_MS = 5 * 60_000; // 5 minutes
|
|
207
|
-
lastErrorNotifiedAt = new Map(); // per-
|
|
353
|
+
lastErrorNotifiedAt = new Map(); // per-pattern cooldown for all actions
|
|
208
354
|
static ERROR_COOLDOWN_MS = 5 * 60_000;
|
|
209
|
-
|
|
355
|
+
static ERROR_RECOVERY_TIMEOUT_MS = 5 * 60_000;
|
|
356
|
+
// Count-based dedup: per error pattern, the number of occurrences already
|
|
210
357
|
// accounted for. A scan counts occurrences across the WHOLE pane; count > this
|
|
211
358
|
// baseline means a NEW error appeared. On recovery we absorb the current count
|
|
212
359
|
// (not reset to 0) so the just-handled error doesn't re-trigger, while a later
|
|
@@ -215,7 +362,16 @@ export class Daemon extends EventEmitter {
|
|
|
215
362
|
// still registers as new (prevents the old hash-dedup's permanent suppression).
|
|
216
363
|
lastErrorCount = new Map();
|
|
217
364
|
lastDetectedErrorType = null;
|
|
218
|
-
|
|
365
|
+
static errorPatternKey(ep) {
|
|
366
|
+
return `${ep.type}:${ep.pattern.source}`;
|
|
367
|
+
}
|
|
368
|
+
clearErrorRecoveryGate() {
|
|
369
|
+
this.errorWaitingForRecovery = false;
|
|
370
|
+
this.errorDetectedAt = 0;
|
|
371
|
+
this.errorRecoveryDeadlineAt = 0;
|
|
372
|
+
this.activeErrorPatternKey = null;
|
|
373
|
+
}
|
|
374
|
+
constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger) {
|
|
219
375
|
super();
|
|
220
376
|
this.name = name;
|
|
221
377
|
this.config = config;
|
|
@@ -223,21 +379,20 @@ export class Daemon extends EventEmitter {
|
|
|
223
379
|
this.topicMode = topicMode;
|
|
224
380
|
this.backend = backend;
|
|
225
381
|
this.controlClient = controlClient;
|
|
226
|
-
|
|
382
|
+
if (!rootLogger)
|
|
383
|
+
throw new Error("Daemon requires a shared root logger");
|
|
384
|
+
this.logger = rootLogger.child({ instance: name }, { level: config.log_level });
|
|
227
385
|
this.tmuxSessionName = getTmuxSession();
|
|
228
386
|
this.messageBus = new MessageBus();
|
|
229
387
|
this.messageBus.setLogger(this.logger);
|
|
230
|
-
|
|
231
|
-
|
|
388
|
+
// General is the dispatcher — it must stay warm to route messages, so it is
|
|
389
|
+
// never auto-paused regardless of auto_pause_after.
|
|
390
|
+
const isGeneral = config.general_topic === true || name === "general";
|
|
391
|
+
const autoPauseMinutes = isGeneral ? 0 : (typeof config.auto_pause_after === "number" ? config.auto_pause_after : 0); // default: disabled
|
|
392
|
+
this.autoPauseController = new AutoPauseController(Math.max(0, autoPauseMinutes) * 60_000, readLastInboundAt(instanceDir) ?? Date.now());
|
|
232
393
|
}
|
|
233
394
|
async start() {
|
|
234
395
|
mkdirSync(this.instanceDir, { recursive: true });
|
|
235
|
-
// A daemon restart performs a normal CLI start, so any persisted auto-pause
|
|
236
|
-
// marker from the previous daemon is stale.
|
|
237
|
-
try {
|
|
238
|
-
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
239
|
-
}
|
|
240
|
-
catch { }
|
|
241
396
|
writeFileSync(join(this.instanceDir, "daemon.pid"), String(process.pid));
|
|
242
397
|
this.logger.info(`Starting ${this.name}`);
|
|
243
398
|
// P1: Read crash state from previous run — skip resume if last run was a crash loop
|
|
@@ -263,6 +418,7 @@ export class Daemon extends EventEmitter {
|
|
|
263
418
|
if (saved.chatId) {
|
|
264
419
|
this.lastChatId = saved.chatId;
|
|
265
420
|
this.lastThreadId = saved.threadId || undefined;
|
|
421
|
+
this.lastAdapterId = saved.adapterId || undefined;
|
|
266
422
|
}
|
|
267
423
|
}
|
|
268
424
|
}
|
|
@@ -380,6 +536,7 @@ export class Daemon extends EventEmitter {
|
|
|
380
536
|
instanceName: this.name,
|
|
381
537
|
...snapshot,
|
|
382
538
|
state: this.isPaused ? "paused" : snapshot.state,
|
|
539
|
+
processStatus: this.processStatus,
|
|
383
540
|
pausedAt: this.lastPausedAt,
|
|
384
541
|
});
|
|
385
542
|
}
|
|
@@ -443,7 +600,7 @@ export class Daemon extends EventEmitter {
|
|
|
443
600
|
else {
|
|
444
601
|
await new Promise(r => setTimeout(r, 5000));
|
|
445
602
|
}
|
|
446
|
-
await this.tmux?.pasteText(
|
|
603
|
+
await this.tmux?.pasteText(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir));
|
|
447
604
|
// Record the value the agent has now been told about so the next
|
|
448
605
|
// unchanged restart skips the reload.
|
|
449
606
|
try {
|
|
@@ -521,11 +678,20 @@ export class Daemon extends EventEmitter {
|
|
|
521
678
|
this.logger.info(`${this.name} ready`);
|
|
522
679
|
}
|
|
523
680
|
startHealthCheck() {
|
|
681
|
+
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
682
|
+
return;
|
|
524
683
|
const { max_retries, backoff, reset_after } = this.config.restart_policy;
|
|
525
|
-
|
|
526
|
-
|
|
684
|
+
// Liveness monitoring remains active when automatic restart is disabled;
|
|
685
|
+
// otherwise a dead pane leaves its last idle snapshot cached forever.
|
|
686
|
+
const configuredInterval = this.config.restart_policy.health_check_interval_ms ?? 30_000;
|
|
687
|
+
const healthCheckIntervalMs = Math.min(60_000, configuredInterval > 0 ? configuredInterval : 30_000);
|
|
527
688
|
const scheduleNext = () => {
|
|
689
|
+
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
690
|
+
return;
|
|
528
691
|
this.healthCheckTimer = setTimeout(async () => {
|
|
692
|
+
this.healthCheckTimer = null;
|
|
693
|
+
if (this.runtimeMonitorsFrozen)
|
|
694
|
+
return;
|
|
529
695
|
// Instance directory removed externally (e.g. `rm -rf ~/.agend/instances/<name>`).
|
|
530
696
|
// Stop the loop permanently — otherwise every tick triggers a respawn, whose
|
|
531
697
|
// writeRotationSnapshot fails with ENOENT and gets caught as "Failed to respawn",
|
|
@@ -573,11 +739,13 @@ export class Daemon extends EventEmitter {
|
|
|
573
739
|
this.logger.debug({ exitCode }, `[health] pane exited with code: ${exitCode}`);
|
|
574
740
|
// Normal exit (e.g. user Ctrl+C or /exit) — no crash, no respawn
|
|
575
741
|
if (paneStatus && exitCode === 0) {
|
|
742
|
+
this.setProcessStatus("stopped");
|
|
576
743
|
this.logger.info("CLI exited normally (code 0) — pausing health check");
|
|
577
744
|
await this.tmux.killWindow();
|
|
578
745
|
this.healthCheckPaused = true;
|
|
579
746
|
return;
|
|
580
747
|
}
|
|
748
|
+
this.setProcessStatus("crashed");
|
|
581
749
|
// Distinguish tmux server crash from single window crash.
|
|
582
750
|
// nullReason records *why* getPaneStatus returned null (for diagnosing
|
|
583
751
|
// whether this was a real window loss or a transient query failure).
|
|
@@ -655,6 +823,7 @@ export class Daemon extends EventEmitter {
|
|
|
655
823
|
await new Promise(r => setTimeout(r, 2_000));
|
|
656
824
|
try {
|
|
657
825
|
await this.spawnClaudeWindow();
|
|
826
|
+
this.setProcessStatus("running");
|
|
658
827
|
this.logger.info("Recovered from background session conflict");
|
|
659
828
|
this.emit("crash_respawn", this.name);
|
|
660
829
|
}
|
|
@@ -680,6 +849,11 @@ export class Daemon extends EventEmitter {
|
|
|
680
849
|
}
|
|
681
850
|
// Append to crash history
|
|
682
851
|
this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: nullReason });
|
|
852
|
+
if (max_retries <= 0) {
|
|
853
|
+
this.healthCheckPaused = true;
|
|
854
|
+
this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
|
|
855
|
+
return;
|
|
856
|
+
}
|
|
683
857
|
// Detect rapid crash: sliding window — 3+ crashes in 5 minutes
|
|
684
858
|
this.crashTimestamps.push(Date.now());
|
|
685
859
|
const crashWindowMs = 5 * 60_000;
|
|
@@ -755,6 +929,7 @@ export class Daemon extends EventEmitter {
|
|
|
755
929
|
}
|
|
756
930
|
catch { /* may not exist */ }
|
|
757
931
|
}
|
|
932
|
+
this.setProcessStatus("running");
|
|
758
933
|
this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
|
|
759
934
|
this.emit("crash_respawn", this.name);
|
|
760
935
|
}
|
|
@@ -762,10 +937,39 @@ export class Daemon extends EventEmitter {
|
|
|
762
937
|
this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
|
|
763
938
|
}
|
|
764
939
|
scheduleNext();
|
|
765
|
-
},
|
|
940
|
+
}, healthCheckIntervalMs);
|
|
766
941
|
};
|
|
767
942
|
scheduleNext();
|
|
768
943
|
}
|
|
944
|
+
/**
|
|
945
|
+
* Publish pane-process availability separately from idle/working/stuck.
|
|
946
|
+
* A dead remain-on-exit pane still contains the old ready prompt, so allowing
|
|
947
|
+
* the pane monitor to capture it would re-create a false idle state.
|
|
948
|
+
*/
|
|
949
|
+
setProcessStatus(status) {
|
|
950
|
+
if (this.processStatus === status)
|
|
951
|
+
return;
|
|
952
|
+
this.processStatus = status;
|
|
953
|
+
if (status === "running") {
|
|
954
|
+
// A successful crash respawn is a new pane generation. It is ready by
|
|
955
|
+
// this point, so an error gate inherited from the dead pane must not keep
|
|
956
|
+
// the new process from being monitored.
|
|
957
|
+
this.clearErrorRecoveryGate();
|
|
958
|
+
this.startInstanceStateMonitor();
|
|
959
|
+
}
|
|
960
|
+
else {
|
|
961
|
+
this.stopInstanceStateMonitor();
|
|
962
|
+
// Force the next ready capture to emit a fresh transition after respawn.
|
|
963
|
+
this.instanceState = "working";
|
|
964
|
+
}
|
|
965
|
+
this.ipcServer?.broadcast({
|
|
966
|
+
type: "instance_process_state",
|
|
967
|
+
instanceName: this.name,
|
|
968
|
+
status,
|
|
969
|
+
observedAt: Date.now(),
|
|
970
|
+
});
|
|
971
|
+
this.emit("instance_process_state", { name: this.name, status });
|
|
972
|
+
}
|
|
769
973
|
/**
|
|
770
974
|
* Periodically scan PTY output for backend-defined error patterns.
|
|
771
975
|
*
|
|
@@ -776,6 +980,8 @@ export class Daemon extends EventEmitter {
|
|
|
776
980
|
* (ready pattern visible), it goes back to monitoring for new errors.
|
|
777
981
|
*/
|
|
778
982
|
startErrorMonitor() {
|
|
983
|
+
if (this.runtimeMonitorsFrozen || this.errorMonitorTimer)
|
|
984
|
+
return;
|
|
779
985
|
const patterns = this.backend?.getErrorPatterns?.() ?? [];
|
|
780
986
|
const dialogs = this.backend?.getRuntimeDialogs?.() ?? [];
|
|
781
987
|
if (!patterns.length && !dialogs.length)
|
|
@@ -793,13 +999,6 @@ export class Daemon extends EventEmitter {
|
|
|
793
999
|
if (!alive)
|
|
794
1000
|
return;
|
|
795
1001
|
const pane = await this.tmux.capturePane();
|
|
796
|
-
// Count occurrences of a pattern across the WHOLE pane (not just the text
|
|
797
|
-
// after the last ready prompt — a fast recovery can put "ready" AFTER the
|
|
798
|
-
// error line, which the old scanText approach missed).
|
|
799
|
-
const countMatches = (pattern) => {
|
|
800
|
-
const g = new RegExp(pattern.source, pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g");
|
|
801
|
-
return (pane.match(g) || []).length;
|
|
802
|
-
};
|
|
803
1002
|
// Auto-dismiss runtime dialogs (e.g. Codex rate limit model switch)
|
|
804
1003
|
for (const dialog of dialogs) {
|
|
805
1004
|
if (!dialog.pattern.test(pane))
|
|
@@ -817,73 +1016,96 @@ export class Daemon extends EventEmitter {
|
|
|
817
1016
|
}
|
|
818
1017
|
return; // Dialog dismissed, skip error checks this cycle
|
|
819
1018
|
}
|
|
820
|
-
|
|
821
|
-
if (this.errorWaitingForRecovery) {
|
|
822
|
-
if (readyPattern.test(pane)) {
|
|
823
|
-
const downtime = Math.round((Date.now() - this.errorDetectedAt) / 1000);
|
|
824
|
-
// Absorb the current count of the just-handled error type so it isn't
|
|
825
|
-
// re-triggered; a later NEW occurrence pushes the count higher again.
|
|
826
|
-
if (this.lastDetectedErrorType) {
|
|
827
|
-
const ep = patterns.find(p => p.type === this.lastDetectedErrorType);
|
|
828
|
-
if (ep)
|
|
829
|
-
this.lastErrorCount.set(ep.type, countMatches(ep.pattern));
|
|
830
|
-
}
|
|
831
|
-
this.errorWaitingForRecovery = false;
|
|
832
|
-
this.errorDetectedAt = 0;
|
|
833
|
-
this.logger.info({ downtime_s: downtime }, "PTY error recovered — agent is ready again");
|
|
834
|
-
this.emit("pty_recovered", { name: this.name, downtime_s: downtime });
|
|
835
|
-
}
|
|
836
|
-
return; // Don't check for errors while waiting for recovery
|
|
837
|
-
}
|
|
838
|
-
// State: monitoring — count-based new-error detection over the full pane
|
|
839
|
-
for (const ep of patterns) {
|
|
840
|
-
const count = countMatches(ep.pattern);
|
|
841
|
-
const seen = this.lastErrorCount.get(ep.type) ?? 0;
|
|
842
|
-
if (count <= seen) {
|
|
843
|
-
// Occurrences scrolled out of the capture buffer → lower the baseline
|
|
844
|
-
// so a future re-occurrence still counts as new (no permanent suppress).
|
|
845
|
-
if (count < seen)
|
|
846
|
-
this.lastErrorCount.set(ep.type, count);
|
|
847
|
-
continue;
|
|
848
|
-
}
|
|
849
|
-
// count > seen → a NEW occurrence of this error appeared.
|
|
850
|
-
// Cooldown: 2nd-layer guard so the same type isn't re-notified within
|
|
851
|
-
// the window. Leave the count unconsumed so it fires once cooldown ends.
|
|
852
|
-
if (!ep.skipCooldown) {
|
|
853
|
-
const lastNotified = this.lastErrorNotifiedAt.get(ep.type) ?? 0;
|
|
854
|
-
if (Date.now() - lastNotified < Daemon.ERROR_COOLDOWN_MS) {
|
|
855
|
-
this.logger.debug({ errorType: ep.type }, "PTY error suppressed (cooldown active)");
|
|
856
|
-
break;
|
|
857
|
-
}
|
|
858
|
-
}
|
|
859
|
-
if (ep.action === "failover" && Date.now() - this.lastFailoverAt < Daemon.FAILOVER_COOLDOWN_MS) {
|
|
860
|
-
this.logger.debug({ errorType: ep.type }, "PTY error suppressed (failover cooldown active)");
|
|
861
|
-
break;
|
|
862
|
-
}
|
|
863
|
-
this.lastErrorCount.set(ep.type, count);
|
|
864
|
-
// skipRecoveryWait: this error self-recovers (e.g. a timeout — Kiro is
|
|
865
|
-
// back at its prompt immediately). Its ready-pattern only matches the
|
|
866
|
-
// startup banner, so entering "waiting for recovery" would never clear
|
|
867
|
-
// and would block ALL future error detection. Just absorb the baseline
|
|
868
|
-
// (done above) and keep monitoring so the next occurrence still fires.
|
|
869
|
-
if (!ep.skipRecoveryWait) {
|
|
870
|
-
this.errorWaitingForRecovery = true;
|
|
871
|
-
this.errorDetectedAt = Date.now();
|
|
872
|
-
this.lastDetectedErrorType = ep.type;
|
|
873
|
-
}
|
|
874
|
-
this.lastErrorNotifiedAt.set(ep.type, Date.now());
|
|
875
|
-
if (ep.action === "failover")
|
|
876
|
-
this.lastFailoverAt = Date.now();
|
|
877
|
-
this.logger.warn({ errorType: ep.type, action: ep.action }, `PTY error detected: ${ep.message}`);
|
|
878
|
-
this.emit("pty_error", { name: this.name, ...ep });
|
|
879
|
-
break; // Only handle first new error per scan
|
|
880
|
-
}
|
|
1019
|
+
this.evaluateErrorPatterns(pane, patterns, readyPattern);
|
|
881
1020
|
}
|
|
882
1021
|
catch {
|
|
883
1022
|
// capturePane can fail if window is transitioning — ignore
|
|
884
1023
|
}
|
|
885
1024
|
}, 5_000); // Check every 5 seconds (runtime dialogs need fast response)
|
|
886
1025
|
}
|
|
1026
|
+
/** Evaluate one pane snapshot. Kept synchronous so state-machine edges are unit-testable. */
|
|
1027
|
+
evaluateErrorPatterns(pane, patterns, readyPattern, now = Date.now()) {
|
|
1028
|
+
// Count occurrences across the WHOLE pane (not just text after the last
|
|
1029
|
+
// ready prompt). Clone with `g` so stateful backend regexes cannot leak
|
|
1030
|
+
// lastIndex between monitor cycles.
|
|
1031
|
+
const countMatches = (pattern) => {
|
|
1032
|
+
const flags = pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g";
|
|
1033
|
+
return (pane.match(new RegExp(pattern.source, flags)) || []).length;
|
|
1034
|
+
};
|
|
1035
|
+
// State: waiting for recovery. A missing/outdated ready pattern must not
|
|
1036
|
+
// suppress every future error forever, so the gate has a hard deadline.
|
|
1037
|
+
if (this.errorWaitingForRecovery) {
|
|
1038
|
+
if (readyPattern.test(pane)) {
|
|
1039
|
+
const downtime = Math.round((now - this.errorDetectedAt) / 1000);
|
|
1040
|
+
const active = patterns.find(ep => Daemon.errorPatternKey(ep) === this.activeErrorPatternKey);
|
|
1041
|
+
if (active) {
|
|
1042
|
+
this.lastErrorCount.set(Daemon.errorPatternKey(active), countMatches(active.pattern));
|
|
1043
|
+
}
|
|
1044
|
+
this.clearErrorRecoveryGate();
|
|
1045
|
+
this.logger.info({ downtime_s: downtime }, "PTY error recovered — agent is ready again");
|
|
1046
|
+
this.emit("pty_recovered", { name: this.name, downtime_s: downtime });
|
|
1047
|
+
return;
|
|
1048
|
+
}
|
|
1049
|
+
if (now <= this.errorRecoveryDeadlineAt)
|
|
1050
|
+
return;
|
|
1051
|
+
// Re-arm the active occurrence as a reminder. Without lowering its
|
|
1052
|
+
// baseline, clearing the gate alone would still leave count === seen and
|
|
1053
|
+
// could never satisfy the promised post-timeout re-notification.
|
|
1054
|
+
const active = patterns.find(ep => Daemon.errorPatternKey(ep) === this.activeErrorPatternKey);
|
|
1055
|
+
if (active) {
|
|
1056
|
+
const key = Daemon.errorPatternKey(active);
|
|
1057
|
+
const count = countMatches(active.pattern);
|
|
1058
|
+
if (count > 0)
|
|
1059
|
+
this.lastErrorCount.set(key, count - 1);
|
|
1060
|
+
}
|
|
1061
|
+
this.logger.warn({ errorType: this.lastDetectedErrorType }, "PTY error recovery deadline expired — resuming error scan");
|
|
1062
|
+
this.clearErrorRecoveryGate();
|
|
1063
|
+
}
|
|
1064
|
+
// State: monitoring — count-based new-error detection over the full pane.
|
|
1065
|
+
for (const ep of patterns) {
|
|
1066
|
+
const key = Daemon.errorPatternKey(ep);
|
|
1067
|
+
const count = countMatches(ep.pattern);
|
|
1068
|
+
const seen = this.lastErrorCount.get(key) ?? 0;
|
|
1069
|
+
if (count <= seen) {
|
|
1070
|
+
// Occurrences scrolled out of the capture buffer → lower the baseline
|
|
1071
|
+
// so a future re-occurrence still counts as new (no permanent suppress).
|
|
1072
|
+
if (count < seen)
|
|
1073
|
+
this.lastErrorCount.set(key, count);
|
|
1074
|
+
continue;
|
|
1075
|
+
}
|
|
1076
|
+
// count > seen → a NEW occurrence of this exact pattern appeared.
|
|
1077
|
+
// Leave a cooldown-suppressed count unconsumed so it can fire once the
|
|
1078
|
+
// cooldown expires, but continue scanning unrelated patterns this cycle.
|
|
1079
|
+
if (!ep.skipCooldown) {
|
|
1080
|
+
const lastNotified = this.lastErrorNotifiedAt.get(key) ?? 0;
|
|
1081
|
+
if (now - lastNotified < Daemon.ERROR_COOLDOWN_MS) {
|
|
1082
|
+
this.logger.debug({ errorType: ep.type }, "PTY error suppressed (cooldown active)");
|
|
1083
|
+
continue;
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
if (ep.action === "failover" && now - this.lastFailoverAt < Daemon.FAILOVER_COOLDOWN_MS) {
|
|
1087
|
+
this.logger.debug({ errorType: ep.type }, "PTY error suppressed (failover cooldown active)");
|
|
1088
|
+
continue;
|
|
1089
|
+
}
|
|
1090
|
+
this.lastErrorCount.set(key, count);
|
|
1091
|
+
// skipRecoveryWait: this error self-recovers (e.g. a timeout — Kiro is
|
|
1092
|
+
// back at its prompt immediately). Keep monitoring so the next occurrence
|
|
1093
|
+
// can fire without waiting for a possibly startup-only ready pattern.
|
|
1094
|
+
if (!ep.skipRecoveryWait) {
|
|
1095
|
+
this.errorWaitingForRecovery = true;
|
|
1096
|
+
this.errorDetectedAt = now;
|
|
1097
|
+
this.errorRecoveryDeadlineAt = now + Daemon.ERROR_RECOVERY_TIMEOUT_MS;
|
|
1098
|
+
this.activeErrorPatternKey = key;
|
|
1099
|
+
this.lastDetectedErrorType = ep.type;
|
|
1100
|
+
}
|
|
1101
|
+
this.lastErrorNotifiedAt.set(key, now);
|
|
1102
|
+
if (ep.action === "failover")
|
|
1103
|
+
this.lastFailoverAt = now;
|
|
1104
|
+
this.logger.warn({ errorType: ep.type, action: ep.action }, `PTY error detected: ${ep.message}`);
|
|
1105
|
+
this.emit("pty_error", { name: this.name, ...ep });
|
|
1106
|
+
break; // Only handle first unsuppressed new error per scan
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
887
1109
|
/**
|
|
888
1110
|
* Interrupt the CLI's current generation (cancel button / `/cancel`).
|
|
889
1111
|
* Direct tmux key event (not a paste) so it registers as the interrupt key.
|
|
@@ -895,26 +1117,13 @@ export class Daemon extends EventEmitter {
|
|
|
895
1117
|
}
|
|
896
1118
|
async stop() {
|
|
897
1119
|
this.logger.info("Stopping daemon instance");
|
|
898
|
-
|
|
899
|
-
clearTimeout(this.healthCheckTimer);
|
|
900
|
-
this.healthCheckTimer = null;
|
|
901
|
-
}
|
|
902
|
-
if (this.errorMonitorTimer) {
|
|
903
|
-
clearInterval(this.errorMonitorTimer);
|
|
904
|
-
this.errorMonitorTimer = null;
|
|
905
|
-
}
|
|
906
|
-
if (this.instanceStateMonitorTimer) {
|
|
907
|
-
clearInterval(this.instanceStateMonitorTimer);
|
|
908
|
-
this.instanceStateMonitorTimer = null;
|
|
909
|
-
}
|
|
1120
|
+
this.freezeRuntimeMonitors();
|
|
910
1121
|
if (this.toolStatusDebounce) {
|
|
911
1122
|
clearTimeout(this.toolStatusDebounce);
|
|
912
1123
|
this.toolStatusDebounce = null;
|
|
913
1124
|
}
|
|
914
1125
|
this.pendingIpcRequests.clear();
|
|
915
1126
|
this.hangDetector?.stop();
|
|
916
|
-
this.transcriptMonitor?.stop();
|
|
917
|
-
this.guardian?.stop();
|
|
918
1127
|
if (this.adapter)
|
|
919
1128
|
await this.adapter.stop();
|
|
920
1129
|
// Notify MCP servers of graceful shutdown (prevents reconnect attempts)
|
|
@@ -926,12 +1135,19 @@ export class Daemon extends EventEmitter {
|
|
|
926
1135
|
this.healthCheckPaused = true;
|
|
927
1136
|
let killed = false;
|
|
928
1137
|
const quitCmd = this.backend?.getQuitCommand();
|
|
1138
|
+
const quitKey = this.backend?.getQuitKey?.();
|
|
929
1139
|
if (quitCmd) {
|
|
930
1140
|
await this.tmux.sendKeys(quitCmd);
|
|
931
1141
|
// Delay before Enter to prevent tmux server race when multiple
|
|
932
1142
|
// instances stop in parallel (same pattern as pasteText).
|
|
933
1143
|
await new Promise(r => setTimeout(r, 150));
|
|
934
1144
|
await this.tmux.sendSpecialKey("Enter");
|
|
1145
|
+
}
|
|
1146
|
+
else if (quitKey) {
|
|
1147
|
+
// Some CLIs quit via a key chord (e.g. grok Ctrl+Q), not a typed command.
|
|
1148
|
+
await this.tmux.sendSpecialKey(quitKey);
|
|
1149
|
+
}
|
|
1150
|
+
if (quitCmd || quitKey) {
|
|
935
1151
|
// Wait up to 3s for graceful exit, polling every 200ms. A healthy CLI
|
|
936
1152
|
// exits within ~1s; a longer wait just delays the force-kill fallback.
|
|
937
1153
|
for (let i = 0; i < 15; i++) {
|
|
@@ -973,10 +1189,6 @@ export class Daemon extends EventEmitter {
|
|
|
973
1189
|
catch (e) {
|
|
974
1190
|
this.logger.debug({ err: e }, "Failed to remove PID file");
|
|
975
1191
|
}
|
|
976
|
-
try {
|
|
977
|
-
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
978
|
-
}
|
|
979
|
-
catch { }
|
|
980
1192
|
}
|
|
981
1193
|
getHangDetector() {
|
|
982
1194
|
return this.hangDetector;
|
|
@@ -1007,14 +1219,22 @@ export class Daemon extends EventEmitter {
|
|
|
1007
1219
|
}
|
|
1008
1220
|
this.pauseWakeState = "pausing";
|
|
1009
1221
|
this.healthCheckPaused = true;
|
|
1222
|
+
this.freezeRuntimeMonitors();
|
|
1010
1223
|
const transition = (async () => {
|
|
1011
1224
|
try {
|
|
1012
1225
|
this.saveSessionId();
|
|
1013
1226
|
const quitCmd = this.backend?.getQuitCommand();
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1227
|
+
const quitKey = this.backend?.getQuitKey?.();
|
|
1228
|
+
if (this.tmux) {
|
|
1229
|
+
if (quitCmd) {
|
|
1230
|
+
await this.tmux.sendKeys(quitCmd);
|
|
1231
|
+
await new Promise(r => setTimeout(r, 150));
|
|
1232
|
+
await this.tmux.sendSpecialKey("Enter");
|
|
1233
|
+
}
|
|
1234
|
+
else if (quitKey) {
|
|
1235
|
+
// Key-chord quit (e.g. grok Ctrl+Q) — no typed command to send.
|
|
1236
|
+
await this.tmux.sendSpecialKey(quitKey);
|
|
1237
|
+
}
|
|
1018
1238
|
}
|
|
1019
1239
|
let exited = false;
|
|
1020
1240
|
for (let i = 0; i < 15; i++) {
|
|
@@ -1040,9 +1260,7 @@ export class Daemon extends EventEmitter {
|
|
|
1040
1260
|
}
|
|
1041
1261
|
this.pauseWakeState = "paused";
|
|
1042
1262
|
this.autoPauseController.markPaused();
|
|
1043
|
-
|
|
1044
|
-
paused_at: this.lastPausedAt,
|
|
1045
|
-
}));
|
|
1263
|
+
writePausedMarker(this.instanceDir, this.lastPausedAt ?? Date.now());
|
|
1046
1264
|
this.logger.info({ pausedAt: this.lastPausedAt }, "Instance auto-paused");
|
|
1047
1265
|
this.ipcServer?.broadcast({
|
|
1048
1266
|
type: "instance_state", instanceName: this.name, state: "paused", pausedAt: this.lastPausedAt,
|
|
@@ -1053,6 +1271,7 @@ export class Daemon extends EventEmitter {
|
|
|
1053
1271
|
this.pauseWakeState = "active";
|
|
1054
1272
|
this.healthCheckPaused = false;
|
|
1055
1273
|
this.pauseRequested = false;
|
|
1274
|
+
this.resumeRuntimeMonitors();
|
|
1056
1275
|
throw err;
|
|
1057
1276
|
}
|
|
1058
1277
|
})();
|
|
@@ -1100,11 +1319,12 @@ export class Daemon extends EventEmitter {
|
|
|
1100
1319
|
this.pauseWakeState = "active";
|
|
1101
1320
|
this.healthCheckPaused = false;
|
|
1102
1321
|
this.pauseRequested = false;
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1322
|
+
// trySpawn resolved only after the new CLI reached its ready prompt.
|
|
1323
|
+
// Discard any recovery gate retained while monitors were frozen.
|
|
1324
|
+
this.clearErrorRecoveryGate();
|
|
1325
|
+
clearPausedMarker(this.instanceDir);
|
|
1107
1326
|
this.transcriptMonitor?.resetOffset();
|
|
1327
|
+
this.resumeRuntimeMonitors();
|
|
1108
1328
|
this.logger.info("Instance auto-woke");
|
|
1109
1329
|
this.ipcServer?.broadcast({
|
|
1110
1330
|
type: "instance_state", instanceName: this.name, state: this.instanceState, pausedAt: null,
|
|
@@ -1123,62 +1343,229 @@ export class Daemon extends EventEmitter {
|
|
|
1123
1343
|
this.pauseWakeTransition = null;
|
|
1124
1344
|
}
|
|
1125
1345
|
}
|
|
1346
|
+
applyInstanceStateSnapshot(snapshot, pane) {
|
|
1347
|
+
const previous = this.instanceState;
|
|
1348
|
+
this.instanceState = snapshot.state;
|
|
1349
|
+
// Only a transition back to idle completes pending work. Repeated idle
|
|
1350
|
+
// observations between enqueue and paste must not clear a newer inbound.
|
|
1351
|
+
if (snapshot.state === "idle" && previous !== "idle") {
|
|
1352
|
+
this.pendingWork.recordIdle(snapshot.observedAt);
|
|
1353
|
+
}
|
|
1354
|
+
if (snapshot.state !== previous) {
|
|
1355
|
+
this.logger.info({
|
|
1356
|
+
previousState: previous,
|
|
1357
|
+
state: snapshot.state,
|
|
1358
|
+
unchangedForMs: snapshot.unchangedForMs,
|
|
1359
|
+
}, "Instance execution state changed");
|
|
1360
|
+
this.emit("instance_state", { name: this.name, ...snapshot });
|
|
1361
|
+
this.ipcServer?.broadcast({ type: "instance_state", instanceName: this.name, ...snapshot });
|
|
1362
|
+
if (snapshot.state === "stuck" && pane && this.instanceStateReadyPattern) {
|
|
1363
|
+
this.handleStuckTransition(pane, snapshot, this.instanceStateReadyPattern);
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
if (snapshot.state !== "idle")
|
|
1367
|
+
this.pauseRequested = false;
|
|
1368
|
+
if (!this.pauseRequested && this.pasteQueueDepth === 0 && this.autoPauseController.observe(snapshot.state)) {
|
|
1369
|
+
this.pauseRequested = true;
|
|
1370
|
+
this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
|
|
1371
|
+
}
|
|
1372
|
+
}
|
|
1373
|
+
clearInstanceStateIdleTimer() {
|
|
1374
|
+
if (this.instanceStateIdleTimer)
|
|
1375
|
+
clearTimeout(this.instanceStateIdleTimer);
|
|
1376
|
+
this.instanceStateIdleTimer = null;
|
|
1377
|
+
}
|
|
1378
|
+
clearInstanceStateStuckTimer() {
|
|
1379
|
+
if (this.instanceStateStuckTimer)
|
|
1380
|
+
clearTimeout(this.instanceStateStuckTimer);
|
|
1381
|
+
this.instanceStateStuckTimer = null;
|
|
1382
|
+
}
|
|
1383
|
+
scheduleInstanceStateStuckDeadline(changeAt) {
|
|
1384
|
+
// Keep one deadline timer per instance. High-volume TUIs can emit hundreds
|
|
1385
|
+
// of %output records per second; replacing a long timeout for every chunk
|
|
1386
|
+
// would trade tmux subprocess CPU for timer churn.
|
|
1387
|
+
if (this.instanceStateStuckTimer)
|
|
1388
|
+
return;
|
|
1389
|
+
const delay = Math.max(0, changeAt + this.instanceStateStuckTimeoutMs - Date.now());
|
|
1390
|
+
this.instanceStateStuckTimer = setTimeout(() => {
|
|
1391
|
+
this.instanceStateStuckTimer = null;
|
|
1392
|
+
const latestChangeAt = this.instanceStateLastOutputAt || changeAt;
|
|
1393
|
+
if (latestChangeAt + this.instanceStateStuckTimeoutMs > Date.now()) {
|
|
1394
|
+
this.scheduleInstanceStateStuckDeadline(latestChangeAt);
|
|
1395
|
+
return;
|
|
1396
|
+
}
|
|
1397
|
+
void this.captureAndEvaluateInstanceState("stuck_deadline", this.instanceStateLastOutputAt);
|
|
1398
|
+
}, delay);
|
|
1399
|
+
}
|
|
1400
|
+
scheduleInstanceStateIdleCapture() {
|
|
1401
|
+
if (this.instanceStateIdleTimer)
|
|
1402
|
+
return;
|
|
1403
|
+
const expectedOutputAt = this.instanceStateLastOutputAt;
|
|
1404
|
+
const delay = Math.max(0, expectedOutputAt + this.instanceStateIdleDebounceMs - Date.now());
|
|
1405
|
+
this.instanceStateIdleTimer = setTimeout(() => {
|
|
1406
|
+
this.instanceStateIdleTimer = null;
|
|
1407
|
+
const latestOutputAt = this.instanceStateLastOutputAt;
|
|
1408
|
+
if (latestOutputAt > expectedOutputAt
|
|
1409
|
+
&& latestOutputAt + this.instanceStateIdleDebounceMs > Date.now()) {
|
|
1410
|
+
this.scheduleInstanceStateIdleCapture();
|
|
1411
|
+
return;
|
|
1412
|
+
}
|
|
1413
|
+
void this.captureAndEvaluateInstanceState("idle_debounce", latestOutputAt);
|
|
1414
|
+
}, delay);
|
|
1415
|
+
}
|
|
1416
|
+
async captureAndEvaluateInstanceState(reason, expectedOutputAt = 0) {
|
|
1417
|
+
if (!this.instanceStateMonitorActive || this.runtimeMonitorsFrozen || !this.tmux || !this.instanceStateMachine || this.spawning)
|
|
1418
|
+
return;
|
|
1419
|
+
if (this.statePollInFlight) {
|
|
1420
|
+
if (reason === "idle_debounce")
|
|
1421
|
+
this.scheduleInstanceStateIdleCapture();
|
|
1422
|
+
return;
|
|
1423
|
+
}
|
|
1424
|
+
this.statePollInFlight = true;
|
|
1425
|
+
const captureStartedAt = Date.now();
|
|
1426
|
+
try {
|
|
1427
|
+
const pane = await this.tmux.capturePane();
|
|
1428
|
+
// Output received while capture-pane was in flight makes this snapshot
|
|
1429
|
+
// stale. Its output handler has already armed a new debounce.
|
|
1430
|
+
if ((expectedOutputAt > 0 && this.instanceStateLastOutputAt > expectedOutputAt)
|
|
1431
|
+
|| (this.instanceStateLastOutputAt > 0 && this.instanceStateLastOutputAt >= captureStartedAt))
|
|
1432
|
+
return;
|
|
1433
|
+
const observedChangeAt = expectedOutputAt || captureStartedAt;
|
|
1434
|
+
this.instanceStateMachine.observe(pane, observedChangeAt);
|
|
1435
|
+
const snapshot = this.instanceStateMachine.observe(pane, Date.now());
|
|
1436
|
+
this.applyInstanceStateSnapshot(snapshot, pane);
|
|
1437
|
+
if (snapshot.state === "idle") {
|
|
1438
|
+
this.clearInstanceStateStuckTimer();
|
|
1439
|
+
}
|
|
1440
|
+
else if (snapshot.state === "working") {
|
|
1441
|
+
// If control mode missed the pane change, the safety capture becomes
|
|
1442
|
+
// the new progress timestamp and re-arms both deadlines.
|
|
1443
|
+
if (!expectedOutputAt)
|
|
1444
|
+
this.instanceStateLastOutputAt = captureStartedAt;
|
|
1445
|
+
this.scheduleInstanceStateStuckDeadline(this.instanceStateLastOutputAt || captureStartedAt);
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
catch (err) {
|
|
1449
|
+
this.logger.debug({ err: err.message, reason }, "Instance state capture failed");
|
|
1450
|
+
}
|
|
1451
|
+
finally {
|
|
1452
|
+
this.statePollInFlight = false;
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
handleInstancePaneOutput(event) {
|
|
1456
|
+
if (!this.instanceStateMonitorActive || this.runtimeMonitorsFrozen)
|
|
1457
|
+
return;
|
|
1458
|
+
const windowId = this.tmux?.getWindowId();
|
|
1459
|
+
if (!windowId || event.windowId !== windowId || !this.instanceStateMachine)
|
|
1460
|
+
return;
|
|
1461
|
+
this.instanceStateLastOutputAt = event.at;
|
|
1462
|
+
const snapshot = this.instanceStateMachine.recordOutput(event.at);
|
|
1463
|
+
this.applyInstanceStateSnapshot(snapshot);
|
|
1464
|
+
this.scheduleInstanceStateIdleCapture();
|
|
1465
|
+
this.scheduleInstanceStateStuckDeadline(event.at);
|
|
1466
|
+
}
|
|
1467
|
+
bindInstanceStateOutputListener(windowId) {
|
|
1468
|
+
if (!this.controlClient || !this.instanceStateMonitorActive)
|
|
1469
|
+
return;
|
|
1470
|
+
if (!this.instanceStateOutputListener) {
|
|
1471
|
+
this.instanceStateOutputListener = event => this.handleInstancePaneOutput(event);
|
|
1472
|
+
}
|
|
1473
|
+
if (this.instanceStateOutputEventName) {
|
|
1474
|
+
this.controlClient.removeListener(this.instanceStateOutputEventName, this.instanceStateOutputListener);
|
|
1475
|
+
}
|
|
1476
|
+
this.instanceStateOutputEventName = `output:${windowId}`;
|
|
1477
|
+
this.controlClient.on(this.instanceStateOutputEventName, this.instanceStateOutputListener);
|
|
1478
|
+
}
|
|
1126
1479
|
startInstanceStateMonitor() {
|
|
1127
|
-
if (!this.tmux || !this.backend || this.
|
|
1480
|
+
if (this.runtimeMonitorsFrozen || !this.tmux || !this.backend || this.instanceStateMonitorActive)
|
|
1128
1481
|
return;
|
|
1129
1482
|
const rawConfig = this.config.hang_detector;
|
|
1130
1483
|
const timeoutMinutes = rawConfig?.timeout_minutes;
|
|
1131
|
-
|
|
1484
|
+
this.instanceStateStuckTimeoutMs = typeof timeoutMinutes === "number" && timeoutMinutes > 0
|
|
1132
1485
|
? timeoutMinutes * 60_000
|
|
1133
1486
|
: DEFAULT_STUCK_TIMEOUT_MS;
|
|
1134
|
-
|
|
1135
|
-
? rawConfig.
|
|
1136
|
-
:
|
|
1137
|
-
this.
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1487
|
+
this.instanceStateIdleDebounceMs = typeof rawConfig?.idle_debounce_ms === "number" && rawConfig.idle_debounce_ms >= 0
|
|
1488
|
+
? rawConfig.idle_debounce_ms
|
|
1489
|
+
: DEFAULT_STATE_IDLE_DEBOUNCE_MS;
|
|
1490
|
+
this.instanceStateReadyPattern = this.backend.getReadyPattern();
|
|
1491
|
+
this.instanceStateMachine = new PaneStateMachine(this.instanceStateReadyPattern, this.instanceStateStuckTimeoutMs);
|
|
1492
|
+
this.instanceStateLastOutputAt = 0;
|
|
1493
|
+
this.instanceStateMonitorActive = true;
|
|
1494
|
+
if (this.controlClient) {
|
|
1495
|
+
this.instanceStateSafetyListener = () => { void this.captureAndEvaluateInstanceState("safety_sweep"); };
|
|
1496
|
+
this.bindInstanceStateOutputListener(this.tmux.getWindowId());
|
|
1497
|
+
this.controlClient.on("safety_sweep", this.instanceStateSafetyListener);
|
|
1498
|
+
}
|
|
1499
|
+
else {
|
|
1500
|
+
// Standalone/test fallback. Fleet production uses the one shared control
|
|
1501
|
+
// client and therefore one fleet-level safety sweep instead of N timers.
|
|
1502
|
+
this.instanceStateMonitorTimer = setInterval(() => {
|
|
1503
|
+
void this.captureAndEvaluateInstanceState("safety_sweep");
|
|
1504
|
+
}, DEFAULT_STATE_SAFETY_SWEEP_MS);
|
|
1505
|
+
}
|
|
1506
|
+
void this.captureAndEvaluateInstanceState("initial");
|
|
1507
|
+
}
|
|
1508
|
+
stopInstanceStateMonitor() {
|
|
1509
|
+
this.instanceStateMonitorActive = false;
|
|
1510
|
+
this.clearInstanceStateIdleTimer();
|
|
1511
|
+
this.clearInstanceStateStuckTimer();
|
|
1512
|
+
if (this.instanceStateMonitorTimer)
|
|
1513
|
+
clearInterval(this.instanceStateMonitorTimer);
|
|
1514
|
+
this.instanceStateMonitorTimer = null;
|
|
1515
|
+
if (this.controlClient && this.instanceStateOutputListener && this.instanceStateOutputEventName) {
|
|
1516
|
+
this.controlClient.removeListener(this.instanceStateOutputEventName, this.instanceStateOutputListener);
|
|
1517
|
+
}
|
|
1518
|
+
if (this.controlClient && this.instanceStateSafetyListener) {
|
|
1519
|
+
this.controlClient.removeListener("safety_sweep", this.instanceStateSafetyListener);
|
|
1520
|
+
}
|
|
1521
|
+
this.instanceStateOutputListener = null;
|
|
1522
|
+
this.instanceStateOutputEventName = null;
|
|
1523
|
+
this.instanceStateSafetyListener = null;
|
|
1524
|
+
}
|
|
1525
|
+
handleStuckTransition(pane, snapshot, readyPattern) {
|
|
1526
|
+
const deterministicReadyPattern = new RegExp(readyPattern.source, readyPattern.flags.replace(/[gy]/g, ""));
|
|
1527
|
+
const diagnostic = {
|
|
1528
|
+
backend: this.backend?.binaryName ?? this.config.backend ?? "unknown",
|
|
1529
|
+
paneTail: sanitizePaneTail(pane),
|
|
1530
|
+
readyPattern: readyPattern.toString(),
|
|
1531
|
+
readyMatched: deterministicReadyPattern.test(pane),
|
|
1532
|
+
unchangedForMs: snapshot.unchangedForMs,
|
|
1533
|
+
pendingWork: this.pendingWork.hasPendingWork(),
|
|
1179
1534
|
};
|
|
1180
|
-
|
|
1181
|
-
|
|
1535
|
+
if (!diagnostic.pendingWork) {
|
|
1536
|
+
this.logger.debug(diagnostic, "Suppressing stuck notification without pending work");
|
|
1537
|
+
return;
|
|
1538
|
+
}
|
|
1539
|
+
this.logger.warn(diagnostic, "Instance pane stuck with pending work");
|
|
1540
|
+
this.hangDetector?.emit("hang", { unchangedForMs: snapshot.unchangedForMs });
|
|
1541
|
+
}
|
|
1542
|
+
/** Stop every runtime poller/watcher while preserving IPC and daemon state. */
|
|
1543
|
+
freezeRuntimeMonitors() {
|
|
1544
|
+
this.runtimeMonitorsFrozen = true;
|
|
1545
|
+
if (this.healthCheckTimer) {
|
|
1546
|
+
clearTimeout(this.healthCheckTimer);
|
|
1547
|
+
this.healthCheckTimer = null;
|
|
1548
|
+
}
|
|
1549
|
+
if (this.errorMonitorTimer) {
|
|
1550
|
+
clearInterval(this.errorMonitorTimer);
|
|
1551
|
+
this.errorMonitorTimer = null;
|
|
1552
|
+
}
|
|
1553
|
+
this.stopInstanceStateMonitor();
|
|
1554
|
+
this.transcriptMonitor?.stop();
|
|
1555
|
+
this.guardian?.stop();
|
|
1556
|
+
}
|
|
1557
|
+
/** Restore the same monitor objects after wake without adding event listeners. */
|
|
1558
|
+
resumeRuntimeMonitors() {
|
|
1559
|
+
if (!this.runtimeMonitorsFrozen)
|
|
1560
|
+
return;
|
|
1561
|
+
this.runtimeMonitorsFrozen = false;
|
|
1562
|
+
this.startHealthCheck();
|
|
1563
|
+
if (!this.config.lightweight) {
|
|
1564
|
+
this.transcriptMonitor?.startPolling();
|
|
1565
|
+
this.guardian?.startWatching();
|
|
1566
|
+
this.startErrorMonitor();
|
|
1567
|
+
}
|
|
1568
|
+
this.startInstanceStateMonitor();
|
|
1182
1569
|
}
|
|
1183
1570
|
getMessageBus() {
|
|
1184
1571
|
return this.messageBus;
|
|
@@ -1258,12 +1645,23 @@ export class Daemon extends EventEmitter {
|
|
|
1258
1645
|
// Remember (and persist) the reply target. Only real channel messages have a
|
|
1259
1646
|
// non-empty chat_id; cross-instance messages have chat_id="" and must NOT
|
|
1260
1647
|
// overwrite it (their reply would otherwise go nowhere).
|
|
1261
|
-
this.updateLastChat(meta.chat_id, meta.thread_id);
|
|
1648
|
+
this.updateLastChat(meta.chat_id, meta.thread_id, meta.adapter_id);
|
|
1649
|
+
if (meta.chat_id) {
|
|
1650
|
+
const inboundAt = Date.now();
|
|
1651
|
+
this.autoPauseController.recordActivity(inboundAt);
|
|
1652
|
+
try {
|
|
1653
|
+
writeLastInboundAt(this.instanceDir, inboundAt);
|
|
1654
|
+
}
|
|
1655
|
+
catch (err) {
|
|
1656
|
+
this.logger.warn({ err: err.message }, "Failed to persist last inbound timestamp");
|
|
1657
|
+
}
|
|
1658
|
+
}
|
|
1262
1659
|
if (this.pendingInstructionsUpdate) {
|
|
1263
1660
|
writeFileSync(join(this.instanceDir, "prev-instructions"), this.pendingInstructionsUpdate);
|
|
1264
1661
|
this.pendingInstructionsUpdate = undefined;
|
|
1265
1662
|
}
|
|
1266
1663
|
this.hangDetector?.recordInbound();
|
|
1664
|
+
this.pendingWork.recordInbound();
|
|
1267
1665
|
// v3: record user messages for rotation snapshot
|
|
1268
1666
|
this.recordRecentUserMessage(content, meta);
|
|
1269
1667
|
// Format message with metadata prefix for the agent
|
|
@@ -1282,7 +1680,10 @@ export class Daemon extends EventEmitter {
|
|
|
1282
1680
|
}
|
|
1283
1681
|
let formatted;
|
|
1284
1682
|
if (fromInstance) {
|
|
1285
|
-
|
|
1683
|
+
// #77: show the sender's display name for readability, keeping the machine
|
|
1684
|
+
// instance name in parens so the recipient's send_to_instance target is valid.
|
|
1685
|
+
const fromLabel = meta.from_display ? `${meta.from_display} (${fromInstance})` : fromInstance;
|
|
1686
|
+
formatted = `[from:${fromLabel}] ${content}\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)`;
|
|
1286
1687
|
}
|
|
1287
1688
|
else {
|
|
1288
1689
|
const via = meta.source ? ` via ${meta.source}` : "";
|
|
@@ -1313,7 +1714,7 @@ export class Daemon extends EventEmitter {
|
|
|
1313
1714
|
}
|
|
1314
1715
|
if (this.pendingInstructionsNotice) {
|
|
1315
1716
|
this.pendingInstructionsNotice = false;
|
|
1316
|
-
await this.deliverMessage(
|
|
1717
|
+
await this.deliverMessage(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir));
|
|
1317
1718
|
}
|
|
1318
1719
|
const status = (chatId && messageId)
|
|
1319
1720
|
? { chatId: meta.thread_id || chatId, messageId }
|
|
@@ -1370,8 +1771,14 @@ export class Daemon extends EventEmitter {
|
|
|
1370
1771
|
await new Promise(r => setTimeout(r, 2000));
|
|
1371
1772
|
continue;
|
|
1372
1773
|
}
|
|
1373
|
-
// Settle the bracketed paste, then submit.
|
|
1374
|
-
|
|
1774
|
+
// Settle the bracketed paste, then submit. The first delivery immediately
|
|
1775
|
+
// after ready gets a longer one-shot delay because the TUI may still be
|
|
1776
|
+
// completing its final redraw even though the prompt already matched.
|
|
1777
|
+
const enterSettleMs = this.firstDeliveryDelay.consume();
|
|
1778
|
+
if (enterSettleMs > NORMAL_ENTER_SETTLE_MS) {
|
|
1779
|
+
this.logger.debug({ enterSettleMs }, "First delivery after ready — extending Enter settle delay");
|
|
1780
|
+
}
|
|
1781
|
+
await new Promise(r => setTimeout(r, enterSettleMs));
|
|
1375
1782
|
const enterAt = Date.now();
|
|
1376
1783
|
await this.tmux.sendSpecialKey("Enter");
|
|
1377
1784
|
if (status)
|
|
@@ -1414,6 +1821,7 @@ export class Daemon extends EventEmitter {
|
|
|
1414
1821
|
this.tmux = new TmuxManager(this.tmuxSessionName, match.id);
|
|
1415
1822
|
writeFileSync(join(this.instanceDir, "window-id"), match.id);
|
|
1416
1823
|
await this.controlClient?.registerWindow(match.id);
|
|
1824
|
+
this.bindInstanceStateOutputListener(match.id);
|
|
1417
1825
|
this.logger.info({ windowId: match.id }, "Recovered window ID for message delivery");
|
|
1418
1826
|
return match.id;
|
|
1419
1827
|
}
|
|
@@ -1760,6 +2168,7 @@ export class Daemon extends EventEmitter {
|
|
|
1760
2168
|
},
|
|
1761
2169
|
skipPermissions: this.config.skipPermissions,
|
|
1762
2170
|
model: this.modelOverride ?? this.config.model,
|
|
2171
|
+
kiroUi: this.config.kiro_ui,
|
|
1763
2172
|
skipResume: this.skipResume,
|
|
1764
2173
|
instructions,
|
|
1765
2174
|
agentMode: isCliMode ? "cli" : "mcp",
|
|
@@ -1939,6 +2348,7 @@ export class Daemon extends EventEmitter {
|
|
|
1939
2348
|
}
|
|
1940
2349
|
// Register with control client and wait for output + idle
|
|
1941
2350
|
await this.controlClient?.registerWindow(windowId);
|
|
2351
|
+
this.bindInstanceStateOutputListener(windowId);
|
|
1942
2352
|
if (this.controlClient) {
|
|
1943
2353
|
const total = startupTimeoutMs ?? this.config.startup_timeout_ms ?? 25_000;
|
|
1944
2354
|
const outputTimeout = Math.round(total * 0.6);
|
|
@@ -1963,7 +2373,10 @@ export class Daemon extends EventEmitter {
|
|
|
1963
2373
|
// but a startup crash would already be caught by waitForOutput/waitForIdle above.
|
|
1964
2374
|
if (!await this.tmux.isWindowAlive())
|
|
1965
2375
|
return false;
|
|
1966
|
-
|
|
2376
|
+
const ready = await this.dismissDialogsUntilReady(3);
|
|
2377
|
+
if (ready)
|
|
2378
|
+
this.firstDeliveryDelay.recordReady();
|
|
2379
|
+
return ready;
|
|
1967
2380
|
}
|
|
1968
2381
|
/**
|
|
1969
2382
|
* Repeatedly check pane content, dismiss any confirmation dialogs,
|
|
@@ -2030,14 +2443,17 @@ export class Daemon extends EventEmitter {
|
|
|
2030
2443
|
* messages) so it never overwrites a real channel target. Persisted to
|
|
2031
2444
|
* last-chat.json so the reply target survives a restart (see start()).
|
|
2032
2445
|
*/
|
|
2033
|
-
updateLastChat(chatId, threadId) {
|
|
2446
|
+
updateLastChat(chatId, threadId, adapterId) {
|
|
2034
2447
|
if (!chatId)
|
|
2035
2448
|
return;
|
|
2036
2449
|
this.lastChatId = chatId;
|
|
2037
|
-
|
|
2038
|
-
|
|
2450
|
+
// An unthreaded inbound must clear a previous topic rather than leaking it
|
|
2451
|
+
// into the next reply target.
|
|
2452
|
+
this.lastThreadId = threadId || undefined;
|
|
2453
|
+
if (adapterId)
|
|
2454
|
+
this.lastAdapterId = adapterId;
|
|
2039
2455
|
try {
|
|
2040
|
-
writeFileSync(join(this.instanceDir, "last-chat.json"), JSON.stringify({ chatId: this.lastChatId, threadId: this.lastThreadId }));
|
|
2456
|
+
writeFileSync(join(this.instanceDir, "last-chat.json"), JSON.stringify({ chatId: this.lastChatId, threadId: this.lastThreadId, adapterId: this.lastAdapterId }));
|
|
2041
2457
|
}
|
|
2042
2458
|
catch { /* best effort */ }
|
|
2043
2459
|
}
|