@songsid/agend 2.1.0-beta.9 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/README.zh-TW.md +2 -1
- package/dist/access-path.js +3 -3
- package/dist/access-path.js.map +1 -1
- package/dist/agent-cli.js +1 -1
- package/dist/agent-cli.js.map +1 -1
- package/dist/agent-endpoint.d.ts +8 -0
- package/dist/agent-endpoint.js +36 -8
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +8 -0
- package/dist/backend/antigravity.js +43 -1
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +6 -0
- package/dist/backend/claude-code.js +20 -4
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +12 -1
- package/dist/backend/codex.js +55 -1
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/factory.js +4 -1
- package/dist/backend/factory.js.map +1 -1
- package/dist/backend/grok.d.ts +59 -0
- package/dist/backend/grok.js +329 -0
- package/dist/backend/grok.js.map +1 -0
- package/dist/backend/kiro.d.ts +6 -0
- package/dist/backend/kiro.js +38 -3
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/opencode.d.ts +5 -0
- package/dist/backend/opencode.js +18 -0
- package/dist/backend/opencode.js.map +1 -1
- package/dist/backend/types.d.ts +59 -3
- package/dist/backend/types.js +15 -2
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +20 -0
- package/dist/channel/adapters/discord.js +88 -11
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.js +4 -1
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/mcp-tools.js +18 -2
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/chat-export.js +5 -2
- package/dist/chat-export.js.map +1 -1
- package/dist/classic-channel-manager.d.ts +16 -1
- package/dist/classic-channel-manager.js +76 -4
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +82 -16
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +70 -1
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +3 -1
- package/dist/config.js +14 -1
- package/dist/config.js.map +1 -1
- package/dist/daemon.d.ts +67 -5
- package/dist/daemon.js +489 -180
- package/dist/daemon.js.map +1 -1
- package/dist/daily-summary.js +7 -2
- package/dist/daily-summary.js.map +1 -1
- package/dist/fleet-context.d.ts +11 -0
- package/dist/fleet-manager.d.ts +128 -7
- package/dist/fleet-manager.js +1116 -140
- package/dist/fleet-manager.js.map +1 -1
- package/dist/fleet-system-prompt.js +1 -1
- package/dist/general-knowledge/skills/fleet-config/SKILL.md +19 -13
- package/dist/general-knowledge/skills/fleet-health/SKILL.md +30 -51
- package/dist/general-knowledge/skills/fleet-restart/SKILL.md +1 -1
- package/dist/general-knowledge/skills/instance-lifecycle/SKILL.md +19 -13
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +1 -5
- package/dist/general-knowledge/skills/scheduling/SKILL.md +41 -0
- package/dist/general-knowledge/skills/session-management/SKILL.md +29 -116
- package/dist/instance-lifecycle.d.ts +13 -0
- package/dist/instance-lifecycle.js +57 -18
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/locale.js +28 -2
- package/dist/locale.js.map +1 -1
- package/dist/logger.d.ts +6 -0
- package/dist/logger.js +23 -7
- package/dist/logger.js.map +1 -1
- package/dist/outbound-handlers.js +199 -11
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +44 -2
- package/dist/outbound-schemas.js +50 -3
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pause-marker.d.ts +5 -0
- package/dist/pause-marker.js +36 -0
- package/dist/pause-marker.js.map +1 -0
- package/dist/scheduler/db.d.ts +8 -0
- package/dist/scheduler/db.js +74 -5
- package/dist/scheduler/db.js.map +1 -1
- package/dist/scheduler/db.test.js +50 -0
- package/dist/scheduler/db.test.js.map +1 -1
- package/dist/scheduler/scheduler.d.ts +6 -0
- package/dist/scheduler/scheduler.js +99 -16
- package/dist/scheduler/scheduler.js.map +1 -1
- package/dist/scheduler/scheduler.test.js +131 -1
- package/dist/scheduler/scheduler.test.js.map +1 -1
- package/dist/scheduler/types.d.ts +9 -3
- package/dist/scheduler/types.js.map +1 -1
- package/dist/settings-api.d.ts +20 -2
- package/dist/settings-api.js +213 -9
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +8 -0
- package/dist/setup-wizard.js.map +1 -1
- package/dist/tmux-control.d.ts +7 -0
- package/dist/tmux-control.js +22 -1
- package/dist/tmux-control.js.map +1 -1
- package/dist/tmux-manager.d.ts +1 -1
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-commands.d.ts +14 -0
- package/dist/topic-commands.js +107 -15
- package/dist/topic-commands.js.map +1 -1
- package/dist/types.d.ts +29 -1
- package/dist/tz-utils.d.ts +26 -0
- package/dist/tz-utils.js +46 -0
- package/dist/tz-utils.js.map +1 -0
- package/dist/ui/dashboard.html +3 -3
- package/dist/ui/settings.html +453 -74
- package/dist/web-api.js +3 -1
- package/dist/web-api.js.map +1 -1
- package/dist/workflow-templates/default.md +0 -4
- package/package.json +2 -1
package/dist/daemon.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { join, dirname, basename, resolve } from "node:path";
|
|
2
|
-
import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync, chmodSync } from "node:fs";
|
|
2
|
+
import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync, chmodSync, renameSync } from "node:fs";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import { createHash, randomBytes } from "node:crypto";
|
|
5
5
|
import { EventEmitter } from "node:events";
|
|
6
|
+
import { clearPausedMarker, writePausedMarker } from "./pause-marker.js";
|
|
6
7
|
import { TmuxManager } from "./tmux-manager.js";
|
|
7
8
|
import { TranscriptMonitor } from "./transcript-monitor.js";
|
|
8
9
|
import { ContextGuardian } from "./context-guardian.js";
|
|
@@ -20,32 +21,71 @@ const CROSS_INSTANCE_TOOLS = new Set(["send_to_instance", "list_instances", "sta
|
|
|
20
21
|
const SCHEDULE_TOOLS = new Set(["create_schedule", "list_schedules", "update_schedule", "delete_schedule"]);
|
|
21
22
|
const DECISION_TOOLS = new Set(["post_decision", "list_decisions", "update_decision"]);
|
|
22
23
|
const TASK_TOOL = "task";
|
|
24
|
+
/** Point a resumed CLI at its one backend-native instruction source. */
|
|
25
|
+
export function buildInstructionReloadNotice(binaryName, instanceName, instanceDir) {
|
|
26
|
+
const source = binaryName === "codex" || binaryName === "grok"
|
|
27
|
+
? "AGENTS.md"
|
|
28
|
+
: binaryName === "kiro-cli"
|
|
29
|
+
? `.kiro/steering/agend-${instanceName}.md`
|
|
30
|
+
: binaryName === "agy"
|
|
31
|
+
? ".agents/agents.md"
|
|
32
|
+
: binaryName === "gemini"
|
|
33
|
+
? "GEMINI.md"
|
|
34
|
+
: join(instanceDir, "fleet-instructions.md");
|
|
35
|
+
return `[system] Your AgEnD instructions have been updated. Reload only ${source}; do not scan other instruction directories. Do not reply to this message.`;
|
|
36
|
+
}
|
|
23
37
|
export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
|
|
24
|
-
export const
|
|
25
|
-
|
|
38
|
+
export const DEFAULT_STATE_IDLE_DEBOUNCE_MS = 2_000;
|
|
39
|
+
export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
|
|
40
|
+
/** @deprecated State detection is event-driven; this now aliases the safety sweep. */
|
|
41
|
+
export const DEFAULT_STATE_POLL_INTERVAL_MS = DEFAULT_STATE_SAFETY_SWEEP_MS;
|
|
42
|
+
const LAST_INBOUND_FILE = "last-inbound-at";
|
|
43
|
+
/** Read the last real channel inbound timestamp persisted across daemon restarts. */
|
|
44
|
+
export function readLastInboundAt(instanceDir, now = Date.now()) {
|
|
45
|
+
try {
|
|
46
|
+
const value = Number(readFileSync(join(instanceDir, LAST_INBOUND_FILE), "utf-8").trim());
|
|
47
|
+
return Number.isFinite(value) && value >= 0 && value <= now ? value : null;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** Atomically persist the last real channel inbound timestamp. */
|
|
54
|
+
export function writeLastInboundAt(instanceDir, timestamp) {
|
|
55
|
+
mkdirSync(instanceDir, { recursive: true });
|
|
56
|
+
const target = join(instanceDir, LAST_INBOUND_FILE);
|
|
57
|
+
const temp = `${target}.${process.pid}.tmp`;
|
|
58
|
+
writeFileSync(temp, String(timestamp));
|
|
59
|
+
renameSync(temp, target);
|
|
60
|
+
}
|
|
61
|
+
/** Headless inactivity timer used by the daemon and unit tests. */
|
|
26
62
|
export class AutoPauseController {
|
|
27
63
|
thresholdMs;
|
|
28
|
-
|
|
64
|
+
lastActivityAt;
|
|
29
65
|
pausedAt = null;
|
|
30
|
-
constructor(thresholdMs) {
|
|
66
|
+
constructor(thresholdMs, lastActivityAt = Date.now()) {
|
|
31
67
|
this.thresholdMs = thresholdMs;
|
|
68
|
+
this.lastActivityAt = lastActivityAt;
|
|
69
|
+
}
|
|
70
|
+
recordActivity(now = Date.now()) {
|
|
71
|
+
this.lastActivityAt = now;
|
|
32
72
|
}
|
|
33
73
|
observe(state, now = Date.now()) {
|
|
34
74
|
if (this.pausedAt !== null || this.thresholdMs <= 0)
|
|
35
75
|
return false;
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
this.idleSince ??= now;
|
|
41
|
-
return now - this.idleSince >= this.thresholdMs;
|
|
76
|
+
// Inactivity is based on the last user inbound, not on how long this daemon
|
|
77
|
+
// has observed the idle pane. This preserves pause eligibility across fleet
|
|
78
|
+
// restarts while still ensuring an actively working instance is never paused.
|
|
79
|
+
return state === "idle" && now - this.lastActivityAt >= this.thresholdMs;
|
|
42
80
|
}
|
|
43
81
|
markPaused(now = Date.now()) {
|
|
44
82
|
this.pausedAt = now;
|
|
45
83
|
}
|
|
46
|
-
markAwake() {
|
|
84
|
+
markAwake(now = Date.now()) {
|
|
47
85
|
this.pausedAt = null;
|
|
48
|
-
|
|
86
|
+
// A deliberate wake is activity in its own right and must provide a fresh
|
|
87
|
+
// window for the wake-before-deliver facade to enqueue its inbound message.
|
|
88
|
+
this.lastActivityAt = now;
|
|
49
89
|
}
|
|
50
90
|
async wakeOnDeliver(wake) {
|
|
51
91
|
if (this.pausedAt === null)
|
|
@@ -105,6 +145,16 @@ export class PaneStateMachine {
|
|
|
105
145
|
}
|
|
106
146
|
return this.snapshot(now);
|
|
107
147
|
}
|
|
148
|
+
/** Record pane motion from tmux control mode without capturing pane content. */
|
|
149
|
+
recordOutput(now = Date.now()) {
|
|
150
|
+
this.lastPaneChangeAt = now;
|
|
151
|
+
this.lastObservedAt = now;
|
|
152
|
+
if (this.currentState !== "working") {
|
|
153
|
+
this.currentState = "working";
|
|
154
|
+
this.stateChangedAt = now;
|
|
155
|
+
}
|
|
156
|
+
return this.snapshot(now);
|
|
157
|
+
}
|
|
108
158
|
snapshot(now = Date.now()) {
|
|
109
159
|
return {
|
|
110
160
|
state: this.currentState,
|
|
@@ -140,6 +190,30 @@ export class PendingWorkTracker {
|
|
|
140
190
|
return this.lastInboundOrder > this.lastIdleOrder;
|
|
141
191
|
}
|
|
142
192
|
}
|
|
193
|
+
const NORMAL_ENTER_SETTLE_MS = 500;
|
|
194
|
+
const FIRST_ENTER_SETTLE_MS = 1_750;
|
|
195
|
+
const FIRST_DELIVERY_WINDOW_MS = 5_000;
|
|
196
|
+
/**
|
|
197
|
+
* One-shot timing gate for the first paste after a CLI reaches its ready
|
|
198
|
+
* prompt. Recent tmux versions expose a short interval where the prompt is
|
|
199
|
+
* visible but the TUI can still swallow Enter while completing its redraw.
|
|
200
|
+
*/
|
|
201
|
+
export class FirstDeliveryDelay {
|
|
202
|
+
readyAt = 0;
|
|
203
|
+
pending = false;
|
|
204
|
+
recordReady(now = Date.now()) {
|
|
205
|
+
this.readyAt = now;
|
|
206
|
+
this.pending = true;
|
|
207
|
+
}
|
|
208
|
+
consume(now = Date.now()) {
|
|
209
|
+
if (!this.pending)
|
|
210
|
+
return NORMAL_ENTER_SETTLE_MS;
|
|
211
|
+
this.pending = false;
|
|
212
|
+
return this.readyAt > 0 && now - this.readyAt < FIRST_DELIVERY_WINDOW_MS
|
|
213
|
+
? FIRST_ENTER_SETTLE_MS
|
|
214
|
+
: NORMAL_ENTER_SETTLE_MS;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
143
217
|
/** Redact likely credentials and control sequences before pane text reaches logs. */
|
|
144
218
|
export function sanitizePaneTail(pane, lineCount = 5) {
|
|
145
219
|
const secretAssignment = /\b(token|secret|password|passwd|api[_-]?key|authorization)\b\s*[:=]\s*\S+/gi;
|
|
@@ -183,6 +257,7 @@ export class Daemon extends EventEmitter {
|
|
|
183
257
|
// Track chatId/threadId from inbound messages for automatic outbound routing
|
|
184
258
|
lastChatId;
|
|
185
259
|
lastThreadId;
|
|
260
|
+
lastAdapterId;
|
|
186
261
|
// Pending ack: react 🫡 on first transcript activity after receiving a message
|
|
187
262
|
pendingAckMessage = null;
|
|
188
263
|
// Tool status tracking for channel adapter
|
|
@@ -201,6 +276,8 @@ export class Daemon extends EventEmitter {
|
|
|
201
276
|
lastSpawnAt = 0;
|
|
202
277
|
crashTimestamps = [];
|
|
203
278
|
healthCheckPaused = false;
|
|
279
|
+
/** CLI pane availability, independent from the daemon process and tri-state. */
|
|
280
|
+
processStatus = "running";
|
|
204
281
|
spawning = false;
|
|
205
282
|
skipResume = false;
|
|
206
283
|
backgroundSessionRecoveryAttempted = false;
|
|
@@ -213,7 +290,18 @@ export class Daemon extends EventEmitter {
|
|
|
213
290
|
instanceState = "idle";
|
|
214
291
|
instanceStateMachine = null;
|
|
215
292
|
pendingWork = new PendingWorkTracker();
|
|
293
|
+
/** Fallback safety sweep used only when no shared tmux control client exists. */
|
|
216
294
|
instanceStateMonitorTimer = null;
|
|
295
|
+
instanceStateIdleTimer = null;
|
|
296
|
+
instanceStateStuckTimer = null;
|
|
297
|
+
instanceStateOutputListener = null;
|
|
298
|
+
instanceStateOutputEventName = null;
|
|
299
|
+
instanceStateSafetyListener = null;
|
|
300
|
+
instanceStateLastOutputAt = 0;
|
|
301
|
+
instanceStateIdleDebounceMs = DEFAULT_STATE_IDLE_DEBOUNCE_MS;
|
|
302
|
+
instanceStateStuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS;
|
|
303
|
+
instanceStateReadyPattern = null;
|
|
304
|
+
instanceStateMonitorActive = false;
|
|
217
305
|
statePollInFlight = false;
|
|
218
306
|
autoPauseController;
|
|
219
307
|
pauseRequested = false;
|
|
@@ -236,12 +324,15 @@ export class Daemon extends EventEmitter {
|
|
|
236
324
|
warmupNeeded = false;
|
|
237
325
|
lastBuiltInstructions = "";
|
|
238
326
|
pasteQueueDepth = 0;
|
|
327
|
+
firstDeliveryDelay = new FirstDeliveryDelay();
|
|
239
328
|
// PTY error pattern monitoring
|
|
240
329
|
errorMonitorTimer = null;
|
|
241
330
|
/** Prevent in-flight monitor callbacks from re-arming after a pause. */
|
|
242
331
|
runtimeMonitorsFrozen = false;
|
|
243
332
|
errorWaitingForRecovery = false; // true = error detected, waiting for ready pattern
|
|
244
333
|
errorDetectedAt = 0;
|
|
334
|
+
errorRecoveryDeadlineAt = 0;
|
|
335
|
+
activeErrorPatternKey = null;
|
|
245
336
|
/** Whether this instance is in an abnormal error state (auto-pause is normal). */
|
|
246
337
|
get isErrorState() {
|
|
247
338
|
return this.errorWaitingForRecovery || (this.healthCheckPaused && !this.isPaused) || Daemon.tmuxServerPaused;
|
|
@@ -253,11 +344,16 @@ export class Daemon extends EventEmitter {
|
|
|
253
344
|
get isCrashLoop() {
|
|
254
345
|
return this.crashCount >= 3;
|
|
255
346
|
}
|
|
347
|
+
/** The most recent error type detected by the error monitor (e.g. "rate_limit", "auth_error"). */
|
|
348
|
+
get lastErrorType() {
|
|
349
|
+
return this.lastDetectedErrorType;
|
|
350
|
+
}
|
|
256
351
|
lastFailoverAt = 0; // cooldown: prevent repeated failover triggers
|
|
257
352
|
static FAILOVER_COOLDOWN_MS = 5 * 60_000; // 5 minutes
|
|
258
|
-
lastErrorNotifiedAt = new Map(); // per-
|
|
353
|
+
lastErrorNotifiedAt = new Map(); // per-pattern cooldown for all actions
|
|
259
354
|
static ERROR_COOLDOWN_MS = 5 * 60_000;
|
|
260
|
-
|
|
355
|
+
static ERROR_RECOVERY_TIMEOUT_MS = 5 * 60_000;
|
|
356
|
+
// Count-based dedup: per error pattern, the number of occurrences already
|
|
261
357
|
// accounted for. A scan counts occurrences across the WHOLE pane; count > this
|
|
262
358
|
// baseline means a NEW error appeared. On recovery we absorb the current count
|
|
263
359
|
// (not reset to 0) so the just-handled error doesn't re-trigger, while a later
|
|
@@ -266,6 +362,15 @@ export class Daemon extends EventEmitter {
|
|
|
266
362
|
// still registers as new (prevents the old hash-dedup's permanent suppression).
|
|
267
363
|
lastErrorCount = new Map();
|
|
268
364
|
lastDetectedErrorType = null;
|
|
365
|
+
static errorPatternKey(ep) {
|
|
366
|
+
return `${ep.type}:${ep.pattern.source}`;
|
|
367
|
+
}
|
|
368
|
+
clearErrorRecoveryGate() {
|
|
369
|
+
this.errorWaitingForRecovery = false;
|
|
370
|
+
this.errorDetectedAt = 0;
|
|
371
|
+
this.errorRecoveryDeadlineAt = 0;
|
|
372
|
+
this.activeErrorPatternKey = null;
|
|
373
|
+
}
|
|
269
374
|
constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger) {
|
|
270
375
|
super();
|
|
271
376
|
this.name = name;
|
|
@@ -280,17 +385,14 @@ export class Daemon extends EventEmitter {
|
|
|
280
385
|
this.tmuxSessionName = getTmuxSession();
|
|
281
386
|
this.messageBus = new MessageBus();
|
|
282
387
|
this.messageBus.setLogger(this.logger);
|
|
283
|
-
|
|
284
|
-
|
|
388
|
+
// General is the dispatcher — it must stay warm to route messages, so it is
|
|
389
|
+
// never auto-paused regardless of auto_pause_after.
|
|
390
|
+
const isGeneral = config.general_topic === true || name === "general";
|
|
391
|
+
const autoPauseMinutes = isGeneral ? 0 : (typeof config.auto_pause_after === "number" ? config.auto_pause_after : 0); // default: disabled
|
|
392
|
+
this.autoPauseController = new AutoPauseController(Math.max(0, autoPauseMinutes) * 60_000, readLastInboundAt(instanceDir) ?? Date.now());
|
|
285
393
|
}
|
|
286
394
|
async start() {
|
|
287
395
|
mkdirSync(this.instanceDir, { recursive: true });
|
|
288
|
-
// A daemon restart performs a normal CLI start, so any persisted auto-pause
|
|
289
|
-
// marker from the previous daemon is stale.
|
|
290
|
-
try {
|
|
291
|
-
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
292
|
-
}
|
|
293
|
-
catch { }
|
|
294
396
|
writeFileSync(join(this.instanceDir, "daemon.pid"), String(process.pid));
|
|
295
397
|
this.logger.info(`Starting ${this.name}`);
|
|
296
398
|
// P1: Read crash state from previous run — skip resume if last run was a crash loop
|
|
@@ -316,6 +418,7 @@ export class Daemon extends EventEmitter {
|
|
|
316
418
|
if (saved.chatId) {
|
|
317
419
|
this.lastChatId = saved.chatId;
|
|
318
420
|
this.lastThreadId = saved.threadId || undefined;
|
|
421
|
+
this.lastAdapterId = saved.adapterId || undefined;
|
|
319
422
|
}
|
|
320
423
|
}
|
|
321
424
|
}
|
|
@@ -433,6 +536,7 @@ export class Daemon extends EventEmitter {
|
|
|
433
536
|
instanceName: this.name,
|
|
434
537
|
...snapshot,
|
|
435
538
|
state: this.isPaused ? "paused" : snapshot.state,
|
|
539
|
+
processStatus: this.processStatus,
|
|
436
540
|
pausedAt: this.lastPausedAt,
|
|
437
541
|
});
|
|
438
542
|
}
|
|
@@ -496,7 +600,7 @@ export class Daemon extends EventEmitter {
|
|
|
496
600
|
else {
|
|
497
601
|
await new Promise(r => setTimeout(r, 5000));
|
|
498
602
|
}
|
|
499
|
-
await this.tmux?.pasteText(
|
|
603
|
+
await this.tmux?.pasteText(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir));
|
|
500
604
|
// Record the value the agent has now been told about so the next
|
|
501
605
|
// unchanged restart skips the reload.
|
|
502
606
|
try {
|
|
@@ -577,8 +681,10 @@ export class Daemon extends EventEmitter {
|
|
|
577
681
|
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
578
682
|
return;
|
|
579
683
|
const { max_retries, backoff, reset_after } = this.config.restart_policy;
|
|
580
|
-
|
|
581
|
-
|
|
684
|
+
// Liveness monitoring remains active when automatic restart is disabled;
|
|
685
|
+
// otherwise a dead pane leaves its last idle snapshot cached forever.
|
|
686
|
+
const configuredInterval = this.config.restart_policy.health_check_interval_ms ?? 30_000;
|
|
687
|
+
const healthCheckIntervalMs = Math.min(60_000, configuredInterval > 0 ? configuredInterval : 30_000);
|
|
582
688
|
const scheduleNext = () => {
|
|
583
689
|
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
584
690
|
return;
|
|
@@ -633,11 +739,13 @@ export class Daemon extends EventEmitter {
|
|
|
633
739
|
this.logger.debug({ exitCode }, `[health] pane exited with code: ${exitCode}`);
|
|
634
740
|
// Normal exit (e.g. user Ctrl+C or /exit) — no crash, no respawn
|
|
635
741
|
if (paneStatus && exitCode === 0) {
|
|
742
|
+
this.setProcessStatus("stopped");
|
|
636
743
|
this.logger.info("CLI exited normally (code 0) — pausing health check");
|
|
637
744
|
await this.tmux.killWindow();
|
|
638
745
|
this.healthCheckPaused = true;
|
|
639
746
|
return;
|
|
640
747
|
}
|
|
748
|
+
this.setProcessStatus("crashed");
|
|
641
749
|
// Distinguish tmux server crash from single window crash.
|
|
642
750
|
// nullReason records *why* getPaneStatus returned null (for diagnosing
|
|
643
751
|
// whether this was a real window loss or a transient query failure).
|
|
@@ -715,6 +823,7 @@ export class Daemon extends EventEmitter {
|
|
|
715
823
|
await new Promise(r => setTimeout(r, 2_000));
|
|
716
824
|
try {
|
|
717
825
|
await this.spawnClaudeWindow();
|
|
826
|
+
this.setProcessStatus("running");
|
|
718
827
|
this.logger.info("Recovered from background session conflict");
|
|
719
828
|
this.emit("crash_respawn", this.name);
|
|
720
829
|
}
|
|
@@ -740,6 +849,11 @@ export class Daemon extends EventEmitter {
|
|
|
740
849
|
}
|
|
741
850
|
// Append to crash history
|
|
742
851
|
this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: nullReason });
|
|
852
|
+
if (max_retries <= 0) {
|
|
853
|
+
this.healthCheckPaused = true;
|
|
854
|
+
this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
|
|
855
|
+
return;
|
|
856
|
+
}
|
|
743
857
|
// Detect rapid crash: sliding window — 3+ crashes in 5 minutes
|
|
744
858
|
this.crashTimestamps.push(Date.now());
|
|
745
859
|
const crashWindowMs = 5 * 60_000;
|
|
@@ -815,6 +929,7 @@ export class Daemon extends EventEmitter {
|
|
|
815
929
|
}
|
|
816
930
|
catch { /* may not exist */ }
|
|
817
931
|
}
|
|
932
|
+
this.setProcessStatus("running");
|
|
818
933
|
this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
|
|
819
934
|
this.emit("crash_respawn", this.name);
|
|
820
935
|
}
|
|
@@ -822,10 +937,39 @@ export class Daemon extends EventEmitter {
|
|
|
822
937
|
this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
|
|
823
938
|
}
|
|
824
939
|
scheduleNext();
|
|
825
|
-
},
|
|
940
|
+
}, healthCheckIntervalMs);
|
|
826
941
|
};
|
|
827
942
|
scheduleNext();
|
|
828
943
|
}
|
|
944
|
+
/**
|
|
945
|
+
* Publish pane-process availability separately from idle/working/stuck.
|
|
946
|
+
* A dead remain-on-exit pane still contains the old ready prompt, so allowing
|
|
947
|
+
* the pane monitor to capture it would re-create a false idle state.
|
|
948
|
+
*/
|
|
949
|
+
setProcessStatus(status) {
|
|
950
|
+
if (this.processStatus === status)
|
|
951
|
+
return;
|
|
952
|
+
this.processStatus = status;
|
|
953
|
+
if (status === "running") {
|
|
954
|
+
// A successful crash respawn is a new pane generation. It is ready by
|
|
955
|
+
// this point, so an error gate inherited from the dead pane must not keep
|
|
956
|
+
// the new process from being monitored.
|
|
957
|
+
this.clearErrorRecoveryGate();
|
|
958
|
+
this.startInstanceStateMonitor();
|
|
959
|
+
}
|
|
960
|
+
else {
|
|
961
|
+
this.stopInstanceStateMonitor();
|
|
962
|
+
// Force the next ready capture to emit a fresh transition after respawn.
|
|
963
|
+
this.instanceState = "working";
|
|
964
|
+
}
|
|
965
|
+
this.ipcServer?.broadcast({
|
|
966
|
+
type: "instance_process_state",
|
|
967
|
+
instanceName: this.name,
|
|
968
|
+
status,
|
|
969
|
+
observedAt: Date.now(),
|
|
970
|
+
});
|
|
971
|
+
this.emit("instance_process_state", { name: this.name, status });
|
|
972
|
+
}
|
|
829
973
|
/**
|
|
830
974
|
* Periodically scan PTY output for backend-defined error patterns.
|
|
831
975
|
*
|
|
@@ -855,13 +999,6 @@ export class Daemon extends EventEmitter {
|
|
|
855
999
|
if (!alive)
|
|
856
1000
|
return;
|
|
857
1001
|
const pane = await this.tmux.capturePane();
|
|
858
|
-
// Count occurrences of a pattern across the WHOLE pane (not just the text
|
|
859
|
-
// after the last ready prompt — a fast recovery can put "ready" AFTER the
|
|
860
|
-
// error line, which the old scanText approach missed).
|
|
861
|
-
const countMatches = (pattern) => {
|
|
862
|
-
const g = new RegExp(pattern.source, pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g");
|
|
863
|
-
return (pane.match(g) || []).length;
|
|
864
|
-
};
|
|
865
1002
|
// Auto-dismiss runtime dialogs (e.g. Codex rate limit model switch)
|
|
866
1003
|
for (const dialog of dialogs) {
|
|
867
1004
|
if (!dialog.pattern.test(pane))
|
|
@@ -879,73 +1016,96 @@ export class Daemon extends EventEmitter {
|
|
|
879
1016
|
}
|
|
880
1017
|
return; // Dialog dismissed, skip error checks this cycle
|
|
881
1018
|
}
|
|
882
|
-
|
|
883
|
-
if (this.errorWaitingForRecovery) {
|
|
884
|
-
if (readyPattern.test(pane)) {
|
|
885
|
-
const downtime = Math.round((Date.now() - this.errorDetectedAt) / 1000);
|
|
886
|
-
// Absorb the current count of the just-handled error type so it isn't
|
|
887
|
-
// re-triggered; a later NEW occurrence pushes the count higher again.
|
|
888
|
-
if (this.lastDetectedErrorType) {
|
|
889
|
-
const ep = patterns.find(p => p.type === this.lastDetectedErrorType);
|
|
890
|
-
if (ep)
|
|
891
|
-
this.lastErrorCount.set(ep.type, countMatches(ep.pattern));
|
|
892
|
-
}
|
|
893
|
-
this.errorWaitingForRecovery = false;
|
|
894
|
-
this.errorDetectedAt = 0;
|
|
895
|
-
this.logger.info({ downtime_s: downtime }, "PTY error recovered — agent is ready again");
|
|
896
|
-
this.emit("pty_recovered", { name: this.name, downtime_s: downtime });
|
|
897
|
-
}
|
|
898
|
-
return; // Don't check for errors while waiting for recovery
|
|
899
|
-
}
|
|
900
|
-
// State: monitoring — count-based new-error detection over the full pane
|
|
901
|
-
for (const ep of patterns) {
|
|
902
|
-
const count = countMatches(ep.pattern);
|
|
903
|
-
const seen = this.lastErrorCount.get(ep.type) ?? 0;
|
|
904
|
-
if (count <= seen) {
|
|
905
|
-
// Occurrences scrolled out of the capture buffer → lower the baseline
|
|
906
|
-
// so a future re-occurrence still counts as new (no permanent suppress).
|
|
907
|
-
if (count < seen)
|
|
908
|
-
this.lastErrorCount.set(ep.type, count);
|
|
909
|
-
continue;
|
|
910
|
-
}
|
|
911
|
-
// count > seen → a NEW occurrence of this error appeared.
|
|
912
|
-
// Cooldown: 2nd-layer guard so the same type isn't re-notified within
|
|
913
|
-
// the window. Leave the count unconsumed so it fires once cooldown ends.
|
|
914
|
-
if (!ep.skipCooldown) {
|
|
915
|
-
const lastNotified = this.lastErrorNotifiedAt.get(ep.type) ?? 0;
|
|
916
|
-
if (Date.now() - lastNotified < Daemon.ERROR_COOLDOWN_MS) {
|
|
917
|
-
this.logger.debug({ errorType: ep.type }, "PTY error suppressed (cooldown active)");
|
|
918
|
-
break;
|
|
919
|
-
}
|
|
920
|
-
}
|
|
921
|
-
if (ep.action === "failover" && Date.now() - this.lastFailoverAt < Daemon.FAILOVER_COOLDOWN_MS) {
|
|
922
|
-
this.logger.debug({ errorType: ep.type }, "PTY error suppressed (failover cooldown active)");
|
|
923
|
-
break;
|
|
924
|
-
}
|
|
925
|
-
this.lastErrorCount.set(ep.type, count);
|
|
926
|
-
// skipRecoveryWait: this error self-recovers (e.g. a timeout — Kiro is
|
|
927
|
-
// back at its prompt immediately). Its ready-pattern only matches the
|
|
928
|
-
// startup banner, so entering "waiting for recovery" would never clear
|
|
929
|
-
// and would block ALL future error detection. Just absorb the baseline
|
|
930
|
-
// (done above) and keep monitoring so the next occurrence still fires.
|
|
931
|
-
if (!ep.skipRecoveryWait) {
|
|
932
|
-
this.errorWaitingForRecovery = true;
|
|
933
|
-
this.errorDetectedAt = Date.now();
|
|
934
|
-
this.lastDetectedErrorType = ep.type;
|
|
935
|
-
}
|
|
936
|
-
this.lastErrorNotifiedAt.set(ep.type, Date.now());
|
|
937
|
-
if (ep.action === "failover")
|
|
938
|
-
this.lastFailoverAt = Date.now();
|
|
939
|
-
this.logger.warn({ errorType: ep.type, action: ep.action }, `PTY error detected: ${ep.message}`);
|
|
940
|
-
this.emit("pty_error", { name: this.name, ...ep });
|
|
941
|
-
break; // Only handle first new error per scan
|
|
942
|
-
}
|
|
1019
|
+
this.evaluateErrorPatterns(pane, patterns, readyPattern);
|
|
943
1020
|
}
|
|
944
1021
|
catch {
|
|
945
1022
|
// capturePane can fail if window is transitioning — ignore
|
|
946
1023
|
}
|
|
947
1024
|
}, 5_000); // Check every 5 seconds (runtime dialogs need fast response)
|
|
948
1025
|
}
|
|
1026
|
+
/** Evaluate one pane snapshot. Kept synchronous so state-machine edges are unit-testable. */
|
|
1027
|
+
evaluateErrorPatterns(pane, patterns, readyPattern, now = Date.now()) {
|
|
1028
|
+
// Count occurrences across the WHOLE pane (not just text after the last
|
|
1029
|
+
// ready prompt). Clone with `g` so stateful backend regexes cannot leak
|
|
1030
|
+
// lastIndex between monitor cycles.
|
|
1031
|
+
const countMatches = (pattern) => {
|
|
1032
|
+
const flags = pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g";
|
|
1033
|
+
return (pane.match(new RegExp(pattern.source, flags)) || []).length;
|
|
1034
|
+
};
|
|
1035
|
+
// State: waiting for recovery. A missing/outdated ready pattern must not
|
|
1036
|
+
// suppress every future error forever, so the gate has a hard deadline.
|
|
1037
|
+
if (this.errorWaitingForRecovery) {
|
|
1038
|
+
if (readyPattern.test(pane)) {
|
|
1039
|
+
const downtime = Math.round((now - this.errorDetectedAt) / 1000);
|
|
1040
|
+
const active = patterns.find(ep => Daemon.errorPatternKey(ep) === this.activeErrorPatternKey);
|
|
1041
|
+
if (active) {
|
|
1042
|
+
this.lastErrorCount.set(Daemon.errorPatternKey(active), countMatches(active.pattern));
|
|
1043
|
+
}
|
|
1044
|
+
this.clearErrorRecoveryGate();
|
|
1045
|
+
this.logger.info({ downtime_s: downtime }, "PTY error recovered — agent is ready again");
|
|
1046
|
+
this.emit("pty_recovered", { name: this.name, downtime_s: downtime });
|
|
1047
|
+
return;
|
|
1048
|
+
}
|
|
1049
|
+
if (now <= this.errorRecoveryDeadlineAt)
|
|
1050
|
+
return;
|
|
1051
|
+
// Re-arm the active occurrence as a reminder. Without lowering its
|
|
1052
|
+
// baseline, clearing the gate alone would still leave count === seen and
|
|
1053
|
+
// could never satisfy the promised post-timeout re-notification.
|
|
1054
|
+
const active = patterns.find(ep => Daemon.errorPatternKey(ep) === this.activeErrorPatternKey);
|
|
1055
|
+
if (active) {
|
|
1056
|
+
const key = Daemon.errorPatternKey(active);
|
|
1057
|
+
const count = countMatches(active.pattern);
|
|
1058
|
+
if (count > 0)
|
|
1059
|
+
this.lastErrorCount.set(key, count - 1);
|
|
1060
|
+
}
|
|
1061
|
+
this.logger.warn({ errorType: this.lastDetectedErrorType }, "PTY error recovery deadline expired — resuming error scan");
|
|
1062
|
+
this.clearErrorRecoveryGate();
|
|
1063
|
+
}
|
|
1064
|
+
// State: monitoring — count-based new-error detection over the full pane.
|
|
1065
|
+
for (const ep of patterns) {
|
|
1066
|
+
const key = Daemon.errorPatternKey(ep);
|
|
1067
|
+
const count = countMatches(ep.pattern);
|
|
1068
|
+
const seen = this.lastErrorCount.get(key) ?? 0;
|
|
1069
|
+
if (count <= seen) {
|
|
1070
|
+
// Occurrences scrolled out of the capture buffer → lower the baseline
|
|
1071
|
+
// so a future re-occurrence still counts as new (no permanent suppress).
|
|
1072
|
+
if (count < seen)
|
|
1073
|
+
this.lastErrorCount.set(key, count);
|
|
1074
|
+
continue;
|
|
1075
|
+
}
|
|
1076
|
+
// count > seen → a NEW occurrence of this exact pattern appeared.
|
|
1077
|
+
// Leave a cooldown-suppressed count unconsumed so it can fire once the
|
|
1078
|
+
// cooldown expires, but continue scanning unrelated patterns this cycle.
|
|
1079
|
+
if (!ep.skipCooldown) {
|
|
1080
|
+
const lastNotified = this.lastErrorNotifiedAt.get(key) ?? 0;
|
|
1081
|
+
if (now - lastNotified < Daemon.ERROR_COOLDOWN_MS) {
|
|
1082
|
+
this.logger.debug({ errorType: ep.type }, "PTY error suppressed (cooldown active)");
|
|
1083
|
+
continue;
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
if (ep.action === "failover" && now - this.lastFailoverAt < Daemon.FAILOVER_COOLDOWN_MS) {
|
|
1087
|
+
this.logger.debug({ errorType: ep.type }, "PTY error suppressed (failover cooldown active)");
|
|
1088
|
+
continue;
|
|
1089
|
+
}
|
|
1090
|
+
this.lastErrorCount.set(key, count);
|
|
1091
|
+
// skipRecoveryWait: this error self-recovers (e.g. a timeout — Kiro is
|
|
1092
|
+
// back at its prompt immediately). Keep monitoring so the next occurrence
|
|
1093
|
+
// can fire without waiting for a possibly startup-only ready pattern.
|
|
1094
|
+
if (!ep.skipRecoveryWait) {
|
|
1095
|
+
this.errorWaitingForRecovery = true;
|
|
1096
|
+
this.errorDetectedAt = now;
|
|
1097
|
+
this.errorRecoveryDeadlineAt = now + Daemon.ERROR_RECOVERY_TIMEOUT_MS;
|
|
1098
|
+
this.activeErrorPatternKey = key;
|
|
1099
|
+
this.lastDetectedErrorType = ep.type;
|
|
1100
|
+
}
|
|
1101
|
+
this.lastErrorNotifiedAt.set(key, now);
|
|
1102
|
+
if (ep.action === "failover")
|
|
1103
|
+
this.lastFailoverAt = now;
|
|
1104
|
+
this.logger.warn({ errorType: ep.type, action: ep.action }, `PTY error detected: ${ep.message}`);
|
|
1105
|
+
this.emit("pty_error", { name: this.name, ...ep });
|
|
1106
|
+
break; // Only handle first unsuppressed new error per scan
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
949
1109
|
/**
|
|
950
1110
|
* Interrupt the CLI's current generation (cancel button / `/cancel`).
|
|
951
1111
|
* Direct tmux key event (not a paste) so it registers as the interrupt key.
|
|
@@ -975,12 +1135,19 @@ export class Daemon extends EventEmitter {
|
|
|
975
1135
|
this.healthCheckPaused = true;
|
|
976
1136
|
let killed = false;
|
|
977
1137
|
const quitCmd = this.backend?.getQuitCommand();
|
|
1138
|
+
const quitKey = this.backend?.getQuitKey?.();
|
|
978
1139
|
if (quitCmd) {
|
|
979
1140
|
await this.tmux.sendKeys(quitCmd);
|
|
980
1141
|
// Delay before Enter to prevent tmux server race when multiple
|
|
981
1142
|
// instances stop in parallel (same pattern as pasteText).
|
|
982
1143
|
await new Promise(r => setTimeout(r, 150));
|
|
983
1144
|
await this.tmux.sendSpecialKey("Enter");
|
|
1145
|
+
}
|
|
1146
|
+
else if (quitKey) {
|
|
1147
|
+
// Some CLIs quit via a key chord (e.g. grok Ctrl+Q), not a typed command.
|
|
1148
|
+
await this.tmux.sendSpecialKey(quitKey);
|
|
1149
|
+
}
|
|
1150
|
+
if (quitCmd || quitKey) {
|
|
984
1151
|
// Wait up to 3s for graceful exit, polling every 200ms. A healthy CLI
|
|
985
1152
|
// exits within ~1s; a longer wait just delays the force-kill fallback.
|
|
986
1153
|
for (let i = 0; i < 15; i++) {
|
|
@@ -1022,10 +1189,6 @@ export class Daemon extends EventEmitter {
|
|
|
1022
1189
|
catch (e) {
|
|
1023
1190
|
this.logger.debug({ err: e }, "Failed to remove PID file");
|
|
1024
1191
|
}
|
|
1025
|
-
try {
|
|
1026
|
-
unlinkSync(join(this.instanceDir, "paused-state.json"));
|
|
1027
|
-
}
|
|
1028
|
-
catch { }
|
|
1029
1192
|
}
|
|
1030
1193
|
getHangDetector() {
|
|
1031
1194
|
return this.hangDetector;
|
|
@@ -1061,10 +1224,17 @@ export class Daemon extends EventEmitter {
|
|
|
1061
1224
|
try {
|
|
1062
1225
|
this.saveSessionId();
|
|
1063
1226
|
const quitCmd = this.backend?.getQuitCommand();
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1227
|
+
const quitKey = this.backend?.getQuitKey?.();
|
|
1228
|
+
if (this.tmux) {
|
|
1229
|
+
if (quitCmd) {
|
|
1230
|
+
await this.tmux.sendKeys(quitCmd);
|
|
1231
|
+
await new Promise(r => setTimeout(r, 150));
|
|
1232
|
+
await this.tmux.sendSpecialKey("Enter");
|
|
1233
|
+
}
|
|
1234
|
+
else if (quitKey) {
|
|
1235
|
+
// Key-chord quit (e.g. grok Ctrl+Q) — no typed command to send.
|
|
1236
|
+
await this.tmux.sendSpecialKey(quitKey);
|
|
1237
|
+
}
|
|
1068
1238
|
}
|
|
1069
1239
|
let exited = false;
|
|
1070
1240
|
for (let i = 0; i < 15; i++) {
|
|
@@ -1090,9 +1260,7 @@ export class Daemon extends EventEmitter {
|
|
|
1090
1260
|
}
|
|
1091
1261
|
this.pauseWakeState = "paused";
|
|
1092
1262
|
this.autoPauseController.markPaused();
|
|
1093
|
-
|
|
1094
|
-
paused_at: this.lastPausedAt,
|
|
1095
|
-
}));
|
|
1263
|
+
writePausedMarker(this.instanceDir, this.lastPausedAt ?? Date.now());
|
|
1096
1264
|
this.logger.info({ pausedAt: this.lastPausedAt }, "Instance auto-paused");
|
|
1097
1265
|
this.ipcServer?.broadcast({
|
|
1098
1266
|
type: "instance_state", instanceName: this.name, state: "paused", pausedAt: this.lastPausedAt,
|
|
@@ -1151,10 +1319,10 @@ export class Daemon extends EventEmitter {
|
|
|
1151
1319
|
this.pauseWakeState = "active";
|
|
1152
1320
|
this.healthCheckPaused = false;
|
|
1153
1321
|
this.pauseRequested = false;
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1322
|
+
// trySpawn resolved only after the new CLI reached its ready prompt.
|
|
1323
|
+
// Discard any recovery gate retained while monitors were frozen.
|
|
1324
|
+
this.clearErrorRecoveryGate();
|
|
1325
|
+
clearPausedMarker(this.instanceDir);
|
|
1158
1326
|
this.transcriptMonitor?.resetOffset();
|
|
1159
1327
|
this.resumeRuntimeMonitors();
|
|
1160
1328
|
this.logger.info("Instance auto-woke");
|
|
@@ -1175,68 +1343,184 @@ export class Daemon extends EventEmitter {
|
|
|
1175
1343
|
this.pauseWakeTransition = null;
|
|
1176
1344
|
}
|
|
1177
1345
|
}
|
|
1346
|
+
applyInstanceStateSnapshot(snapshot, pane) {
|
|
1347
|
+
const previous = this.instanceState;
|
|
1348
|
+
this.instanceState = snapshot.state;
|
|
1349
|
+
// Only a transition back to idle completes pending work. Repeated idle
|
|
1350
|
+
// observations between enqueue and paste must not clear a newer inbound.
|
|
1351
|
+
if (snapshot.state === "idle" && previous !== "idle") {
|
|
1352
|
+
this.pendingWork.recordIdle(snapshot.observedAt);
|
|
1353
|
+
}
|
|
1354
|
+
if (snapshot.state !== previous) {
|
|
1355
|
+
this.logger.info({
|
|
1356
|
+
previousState: previous,
|
|
1357
|
+
state: snapshot.state,
|
|
1358
|
+
unchangedForMs: snapshot.unchangedForMs,
|
|
1359
|
+
}, "Instance execution state changed");
|
|
1360
|
+
this.emit("instance_state", { name: this.name, ...snapshot });
|
|
1361
|
+
this.ipcServer?.broadcast({ type: "instance_state", instanceName: this.name, ...snapshot });
|
|
1362
|
+
if (snapshot.state === "stuck" && pane && this.instanceStateReadyPattern) {
|
|
1363
|
+
this.handleStuckTransition(pane, snapshot, this.instanceStateReadyPattern);
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
if (snapshot.state !== "idle")
|
|
1367
|
+
this.pauseRequested = false;
|
|
1368
|
+
if (!this.pauseRequested && this.pasteQueueDepth === 0 && this.autoPauseController.observe(snapshot.state)) {
|
|
1369
|
+
this.pauseRequested = true;
|
|
1370
|
+
this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
|
|
1371
|
+
}
|
|
1372
|
+
}
|
|
1373
|
+
clearInstanceStateIdleTimer() {
|
|
1374
|
+
if (this.instanceStateIdleTimer)
|
|
1375
|
+
clearTimeout(this.instanceStateIdleTimer);
|
|
1376
|
+
this.instanceStateIdleTimer = null;
|
|
1377
|
+
}
|
|
1378
|
+
clearInstanceStateStuckTimer() {
|
|
1379
|
+
if (this.instanceStateStuckTimer)
|
|
1380
|
+
clearTimeout(this.instanceStateStuckTimer);
|
|
1381
|
+
this.instanceStateStuckTimer = null;
|
|
1382
|
+
}
|
|
1383
|
+
scheduleInstanceStateStuckDeadline(changeAt) {
|
|
1384
|
+
// Keep one deadline timer per instance. High-volume TUIs can emit hundreds
|
|
1385
|
+
// of %output records per second; replacing a long timeout for every chunk
|
|
1386
|
+
// would trade tmux subprocess CPU for timer churn.
|
|
1387
|
+
if (this.instanceStateStuckTimer)
|
|
1388
|
+
return;
|
|
1389
|
+
const delay = Math.max(0, changeAt + this.instanceStateStuckTimeoutMs - Date.now());
|
|
1390
|
+
this.instanceStateStuckTimer = setTimeout(() => {
|
|
1391
|
+
this.instanceStateStuckTimer = null;
|
|
1392
|
+
const latestChangeAt = this.instanceStateLastOutputAt || changeAt;
|
|
1393
|
+
if (latestChangeAt + this.instanceStateStuckTimeoutMs > Date.now()) {
|
|
1394
|
+
this.scheduleInstanceStateStuckDeadline(latestChangeAt);
|
|
1395
|
+
return;
|
|
1396
|
+
}
|
|
1397
|
+
void this.captureAndEvaluateInstanceState("stuck_deadline", this.instanceStateLastOutputAt);
|
|
1398
|
+
}, delay);
|
|
1399
|
+
}
|
|
1400
|
+
scheduleInstanceStateIdleCapture() {
|
|
1401
|
+
if (this.instanceStateIdleTimer)
|
|
1402
|
+
return;
|
|
1403
|
+
const expectedOutputAt = this.instanceStateLastOutputAt;
|
|
1404
|
+
const delay = Math.max(0, expectedOutputAt + this.instanceStateIdleDebounceMs - Date.now());
|
|
1405
|
+
this.instanceStateIdleTimer = setTimeout(() => {
|
|
1406
|
+
this.instanceStateIdleTimer = null;
|
|
1407
|
+
const latestOutputAt = this.instanceStateLastOutputAt;
|
|
1408
|
+
if (latestOutputAt > expectedOutputAt
|
|
1409
|
+
&& latestOutputAt + this.instanceStateIdleDebounceMs > Date.now()) {
|
|
1410
|
+
this.scheduleInstanceStateIdleCapture();
|
|
1411
|
+
return;
|
|
1412
|
+
}
|
|
1413
|
+
void this.captureAndEvaluateInstanceState("idle_debounce", latestOutputAt);
|
|
1414
|
+
}, delay);
|
|
1415
|
+
}
|
|
1416
|
+
async captureAndEvaluateInstanceState(reason, expectedOutputAt = 0) {
|
|
1417
|
+
if (!this.instanceStateMonitorActive || this.runtimeMonitorsFrozen || !this.tmux || !this.instanceStateMachine || this.spawning)
|
|
1418
|
+
return;
|
|
1419
|
+
if (this.statePollInFlight) {
|
|
1420
|
+
if (reason === "idle_debounce")
|
|
1421
|
+
this.scheduleInstanceStateIdleCapture();
|
|
1422
|
+
return;
|
|
1423
|
+
}
|
|
1424
|
+
this.statePollInFlight = true;
|
|
1425
|
+
const captureStartedAt = Date.now();
|
|
1426
|
+
try {
|
|
1427
|
+
const pane = await this.tmux.capturePane();
|
|
1428
|
+
// Output received while capture-pane was in flight makes this snapshot
|
|
1429
|
+
// stale. Its output handler has already armed a new debounce.
|
|
1430
|
+
if ((expectedOutputAt > 0 && this.instanceStateLastOutputAt > expectedOutputAt)
|
|
1431
|
+
|| (this.instanceStateLastOutputAt > 0 && this.instanceStateLastOutputAt >= captureStartedAt))
|
|
1432
|
+
return;
|
|
1433
|
+
const observedChangeAt = expectedOutputAt || captureStartedAt;
|
|
1434
|
+
this.instanceStateMachine.observe(pane, observedChangeAt);
|
|
1435
|
+
const snapshot = this.instanceStateMachine.observe(pane, Date.now());
|
|
1436
|
+
this.applyInstanceStateSnapshot(snapshot, pane);
|
|
1437
|
+
if (snapshot.state === "idle") {
|
|
1438
|
+
this.clearInstanceStateStuckTimer();
|
|
1439
|
+
}
|
|
1440
|
+
else if (snapshot.state === "working") {
|
|
1441
|
+
// If control mode missed the pane change, the safety capture becomes
|
|
1442
|
+
// the new progress timestamp and re-arms both deadlines.
|
|
1443
|
+
if (!expectedOutputAt)
|
|
1444
|
+
this.instanceStateLastOutputAt = captureStartedAt;
|
|
1445
|
+
this.scheduleInstanceStateStuckDeadline(this.instanceStateLastOutputAt || captureStartedAt);
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
catch (err) {
|
|
1449
|
+
this.logger.debug({ err: err.message, reason }, "Instance state capture failed");
|
|
1450
|
+
}
|
|
1451
|
+
finally {
|
|
1452
|
+
this.statePollInFlight = false;
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
handleInstancePaneOutput(event) {
|
|
1456
|
+
if (!this.instanceStateMonitorActive || this.runtimeMonitorsFrozen)
|
|
1457
|
+
return;
|
|
1458
|
+
const windowId = this.tmux?.getWindowId();
|
|
1459
|
+
if (!windowId || event.windowId !== windowId || !this.instanceStateMachine)
|
|
1460
|
+
return;
|
|
1461
|
+
this.instanceStateLastOutputAt = event.at;
|
|
1462
|
+
const snapshot = this.instanceStateMachine.recordOutput(event.at);
|
|
1463
|
+
this.applyInstanceStateSnapshot(snapshot);
|
|
1464
|
+
this.scheduleInstanceStateIdleCapture();
|
|
1465
|
+
this.scheduleInstanceStateStuckDeadline(event.at);
|
|
1466
|
+
}
|
|
1467
|
+
bindInstanceStateOutputListener(windowId) {
|
|
1468
|
+
if (!this.controlClient || !this.instanceStateMonitorActive)
|
|
1469
|
+
return;
|
|
1470
|
+
if (!this.instanceStateOutputListener) {
|
|
1471
|
+
this.instanceStateOutputListener = event => this.handleInstancePaneOutput(event);
|
|
1472
|
+
}
|
|
1473
|
+
if (this.instanceStateOutputEventName) {
|
|
1474
|
+
this.controlClient.removeListener(this.instanceStateOutputEventName, this.instanceStateOutputListener);
|
|
1475
|
+
}
|
|
1476
|
+
this.instanceStateOutputEventName = `output:${windowId}`;
|
|
1477
|
+
this.controlClient.on(this.instanceStateOutputEventName, this.instanceStateOutputListener);
|
|
1478
|
+
}
|
|
1178
1479
|
startInstanceStateMonitor() {
|
|
1179
|
-
if (this.runtimeMonitorsFrozen || !this.tmux || !this.backend || this.
|
|
1480
|
+
if (this.runtimeMonitorsFrozen || !this.tmux || !this.backend || this.instanceStateMonitorActive)
|
|
1180
1481
|
return;
|
|
1181
1482
|
const rawConfig = this.config.hang_detector;
|
|
1182
1483
|
const timeoutMinutes = rawConfig?.timeout_minutes;
|
|
1183
|
-
|
|
1484
|
+
this.instanceStateStuckTimeoutMs = typeof timeoutMinutes === "number" && timeoutMinutes > 0
|
|
1184
1485
|
? timeoutMinutes * 60_000
|
|
1185
1486
|
: DEFAULT_STUCK_TIMEOUT_MS;
|
|
1186
|
-
|
|
1187
|
-
? rawConfig.
|
|
1188
|
-
:
|
|
1189
|
-
|
|
1190
|
-
this.instanceStateMachine = new PaneStateMachine(
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
this.
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
this.
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
this.pauseRequested = false;
|
|
1224
|
-
if (!this.pauseRequested && this.pasteQueueDepth === 0 && this.autoPauseController.observe(snapshot.state)) {
|
|
1225
|
-
this.pauseRequested = true;
|
|
1226
|
-
this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
|
|
1227
|
-
}
|
|
1228
|
-
}
|
|
1229
|
-
catch (err) {
|
|
1230
|
-
// A pane can disappear between status and capture during restart. Keep
|
|
1231
|
-
// the last known state and retry on the next poll.
|
|
1232
|
-
this.logger.debug({ err: err.message }, "Instance state poll failed");
|
|
1233
|
-
}
|
|
1234
|
-
finally {
|
|
1235
|
-
this.statePollInFlight = false;
|
|
1236
|
-
}
|
|
1237
|
-
};
|
|
1238
|
-
void poll();
|
|
1239
|
-
this.instanceStateMonitorTimer = setInterval(() => { void poll(); }, pollIntervalMs);
|
|
1487
|
+
this.instanceStateIdleDebounceMs = typeof rawConfig?.idle_debounce_ms === "number" && rawConfig.idle_debounce_ms >= 0
|
|
1488
|
+
? rawConfig.idle_debounce_ms
|
|
1489
|
+
: DEFAULT_STATE_IDLE_DEBOUNCE_MS;
|
|
1490
|
+
this.instanceStateReadyPattern = this.backend.getReadyPattern();
|
|
1491
|
+
this.instanceStateMachine = new PaneStateMachine(this.instanceStateReadyPattern, this.instanceStateStuckTimeoutMs);
|
|
1492
|
+
this.instanceStateLastOutputAt = 0;
|
|
1493
|
+
this.instanceStateMonitorActive = true;
|
|
1494
|
+
if (this.controlClient) {
|
|
1495
|
+
this.instanceStateSafetyListener = () => { void this.captureAndEvaluateInstanceState("safety_sweep"); };
|
|
1496
|
+
this.bindInstanceStateOutputListener(this.tmux.getWindowId());
|
|
1497
|
+
this.controlClient.on("safety_sweep", this.instanceStateSafetyListener);
|
|
1498
|
+
}
|
|
1499
|
+
else {
|
|
1500
|
+
// Standalone/test fallback. Fleet production uses the one shared control
|
|
1501
|
+
// client and therefore one fleet-level safety sweep instead of N timers.
|
|
1502
|
+
this.instanceStateMonitorTimer = setInterval(() => {
|
|
1503
|
+
void this.captureAndEvaluateInstanceState("safety_sweep");
|
|
1504
|
+
}, DEFAULT_STATE_SAFETY_SWEEP_MS);
|
|
1505
|
+
}
|
|
1506
|
+
void this.captureAndEvaluateInstanceState("initial");
|
|
1507
|
+
}
|
|
1508
|
+
stopInstanceStateMonitor() {
|
|
1509
|
+
this.instanceStateMonitorActive = false;
|
|
1510
|
+
this.clearInstanceStateIdleTimer();
|
|
1511
|
+
this.clearInstanceStateStuckTimer();
|
|
1512
|
+
if (this.instanceStateMonitorTimer)
|
|
1513
|
+
clearInterval(this.instanceStateMonitorTimer);
|
|
1514
|
+
this.instanceStateMonitorTimer = null;
|
|
1515
|
+
if (this.controlClient && this.instanceStateOutputListener && this.instanceStateOutputEventName) {
|
|
1516
|
+
this.controlClient.removeListener(this.instanceStateOutputEventName, this.instanceStateOutputListener);
|
|
1517
|
+
}
|
|
1518
|
+
if (this.controlClient && this.instanceStateSafetyListener) {
|
|
1519
|
+
this.controlClient.removeListener("safety_sweep", this.instanceStateSafetyListener);
|
|
1520
|
+
}
|
|
1521
|
+
this.instanceStateOutputListener = null;
|
|
1522
|
+
this.instanceStateOutputEventName = null;
|
|
1523
|
+
this.instanceStateSafetyListener = null;
|
|
1240
1524
|
}
|
|
1241
1525
|
handleStuckTransition(pane, snapshot, readyPattern) {
|
|
1242
1526
|
const deterministicReadyPattern = new RegExp(readyPattern.source, readyPattern.flags.replace(/[gy]/g, ""));
|
|
@@ -1266,10 +1550,7 @@ export class Daemon extends EventEmitter {
|
|
|
1266
1550
|
clearInterval(this.errorMonitorTimer);
|
|
1267
1551
|
this.errorMonitorTimer = null;
|
|
1268
1552
|
}
|
|
1269
|
-
|
|
1270
|
-
clearInterval(this.instanceStateMonitorTimer);
|
|
1271
|
-
this.instanceStateMonitorTimer = null;
|
|
1272
|
-
}
|
|
1553
|
+
this.stopInstanceStateMonitor();
|
|
1273
1554
|
this.transcriptMonitor?.stop();
|
|
1274
1555
|
this.guardian?.stop();
|
|
1275
1556
|
}
|
|
@@ -1364,7 +1645,17 @@ export class Daemon extends EventEmitter {
|
|
|
1364
1645
|
// Remember (and persist) the reply target. Only real channel messages have a
|
|
1365
1646
|
// non-empty chat_id; cross-instance messages have chat_id="" and must NOT
|
|
1366
1647
|
// overwrite it (their reply would otherwise go nowhere).
|
|
1367
|
-
this.updateLastChat(meta.chat_id, meta.thread_id);
|
|
1648
|
+
this.updateLastChat(meta.chat_id, meta.thread_id, meta.adapter_id);
|
|
1649
|
+
if (meta.chat_id) {
|
|
1650
|
+
const inboundAt = Date.now();
|
|
1651
|
+
this.autoPauseController.recordActivity(inboundAt);
|
|
1652
|
+
try {
|
|
1653
|
+
writeLastInboundAt(this.instanceDir, inboundAt);
|
|
1654
|
+
}
|
|
1655
|
+
catch (err) {
|
|
1656
|
+
this.logger.warn({ err: err.message }, "Failed to persist last inbound timestamp");
|
|
1657
|
+
}
|
|
1658
|
+
}
|
|
1368
1659
|
if (this.pendingInstructionsUpdate) {
|
|
1369
1660
|
writeFileSync(join(this.instanceDir, "prev-instructions"), this.pendingInstructionsUpdate);
|
|
1370
1661
|
this.pendingInstructionsUpdate = undefined;
|
|
@@ -1389,7 +1680,10 @@ export class Daemon extends EventEmitter {
|
|
|
1389
1680
|
}
|
|
1390
1681
|
let formatted;
|
|
1391
1682
|
if (fromInstance) {
|
|
1392
|
-
|
|
1683
|
+
// #77: show the sender's display name for readability, keeping the machine
|
|
1684
|
+
// instance name in parens so the recipient's send_to_instance target is valid.
|
|
1685
|
+
const fromLabel = meta.from_display ? `${meta.from_display} (${fromInstance})` : fromInstance;
|
|
1686
|
+
formatted = `[from:${fromLabel}] ${content}\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)`;
|
|
1393
1687
|
}
|
|
1394
1688
|
else {
|
|
1395
1689
|
const via = meta.source ? ` via ${meta.source}` : "";
|
|
@@ -1420,7 +1714,7 @@ export class Daemon extends EventEmitter {
|
|
|
1420
1714
|
}
|
|
1421
1715
|
if (this.pendingInstructionsNotice) {
|
|
1422
1716
|
this.pendingInstructionsNotice = false;
|
|
1423
|
-
await this.deliverMessage(
|
|
1717
|
+
await this.deliverMessage(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir));
|
|
1424
1718
|
}
|
|
1425
1719
|
const status = (chatId && messageId)
|
|
1426
1720
|
? { chatId: meta.thread_id || chatId, messageId }
|
|
@@ -1477,8 +1771,14 @@ export class Daemon extends EventEmitter {
|
|
|
1477
1771
|
await new Promise(r => setTimeout(r, 2000));
|
|
1478
1772
|
continue;
|
|
1479
1773
|
}
|
|
1480
|
-
// Settle the bracketed paste, then submit.
|
|
1481
|
-
|
|
1774
|
+
// Settle the bracketed paste, then submit. The first delivery immediately
|
|
1775
|
+
// after ready gets a longer one-shot delay because the TUI may still be
|
|
1776
|
+
// completing its final redraw even though the prompt already matched.
|
|
1777
|
+
const enterSettleMs = this.firstDeliveryDelay.consume();
|
|
1778
|
+
if (enterSettleMs > NORMAL_ENTER_SETTLE_MS) {
|
|
1779
|
+
this.logger.debug({ enterSettleMs }, "First delivery after ready — extending Enter settle delay");
|
|
1780
|
+
}
|
|
1781
|
+
await new Promise(r => setTimeout(r, enterSettleMs));
|
|
1482
1782
|
const enterAt = Date.now();
|
|
1483
1783
|
await this.tmux.sendSpecialKey("Enter");
|
|
1484
1784
|
if (status)
|
|
@@ -1521,6 +1821,7 @@ export class Daemon extends EventEmitter {
|
|
|
1521
1821
|
this.tmux = new TmuxManager(this.tmuxSessionName, match.id);
|
|
1522
1822
|
writeFileSync(join(this.instanceDir, "window-id"), match.id);
|
|
1523
1823
|
await this.controlClient?.registerWindow(match.id);
|
|
1824
|
+
this.bindInstanceStateOutputListener(match.id);
|
|
1524
1825
|
this.logger.info({ windowId: match.id }, "Recovered window ID for message delivery");
|
|
1525
1826
|
return match.id;
|
|
1526
1827
|
}
|
|
@@ -1867,6 +2168,7 @@ export class Daemon extends EventEmitter {
|
|
|
1867
2168
|
},
|
|
1868
2169
|
skipPermissions: this.config.skipPermissions,
|
|
1869
2170
|
model: this.modelOverride ?? this.config.model,
|
|
2171
|
+
kiroUi: this.config.kiro_ui,
|
|
1870
2172
|
skipResume: this.skipResume,
|
|
1871
2173
|
instructions,
|
|
1872
2174
|
agentMode: isCliMode ? "cli" : "mcp",
|
|
@@ -2046,6 +2348,7 @@ export class Daemon extends EventEmitter {
|
|
|
2046
2348
|
}
|
|
2047
2349
|
// Register with control client and wait for output + idle
|
|
2048
2350
|
await this.controlClient?.registerWindow(windowId);
|
|
2351
|
+
this.bindInstanceStateOutputListener(windowId);
|
|
2049
2352
|
if (this.controlClient) {
|
|
2050
2353
|
const total = startupTimeoutMs ?? this.config.startup_timeout_ms ?? 25_000;
|
|
2051
2354
|
const outputTimeout = Math.round(total * 0.6);
|
|
@@ -2070,7 +2373,10 @@ export class Daemon extends EventEmitter {
|
|
|
2070
2373
|
// but a startup crash would already be caught by waitForOutput/waitForIdle above.
|
|
2071
2374
|
if (!await this.tmux.isWindowAlive())
|
|
2072
2375
|
return false;
|
|
2073
|
-
|
|
2376
|
+
const ready = await this.dismissDialogsUntilReady(3);
|
|
2377
|
+
if (ready)
|
|
2378
|
+
this.firstDeliveryDelay.recordReady();
|
|
2379
|
+
return ready;
|
|
2074
2380
|
}
|
|
2075
2381
|
/**
|
|
2076
2382
|
* Repeatedly check pane content, dismiss any confirmation dialogs,
|
|
@@ -2137,14 +2443,17 @@ export class Daemon extends EventEmitter {
|
|
|
2137
2443
|
* messages) so it never overwrites a real channel target. Persisted to
|
|
2138
2444
|
* last-chat.json so the reply target survives a restart (see start()).
|
|
2139
2445
|
*/
|
|
2140
|
-
updateLastChat(chatId, threadId) {
|
|
2446
|
+
updateLastChat(chatId, threadId, adapterId) {
|
|
2141
2447
|
if (!chatId)
|
|
2142
2448
|
return;
|
|
2143
2449
|
this.lastChatId = chatId;
|
|
2144
|
-
|
|
2145
|
-
|
|
2450
|
+
// An unthreaded inbound must clear a previous topic rather than leaking it
|
|
2451
|
+
// into the next reply target.
|
|
2452
|
+
this.lastThreadId = threadId || undefined;
|
|
2453
|
+
if (adapterId)
|
|
2454
|
+
this.lastAdapterId = adapterId;
|
|
2146
2455
|
try {
|
|
2147
|
-
writeFileSync(join(this.instanceDir, "last-chat.json"), JSON.stringify({ chatId: this.lastChatId, threadId: this.lastThreadId }));
|
|
2456
|
+
writeFileSync(join(this.instanceDir, "last-chat.json"), JSON.stringify({ chatId: this.lastChatId, threadId: this.lastThreadId, adapterId: this.lastAdapterId }));
|
|
2148
2457
|
}
|
|
2149
2458
|
catch { /* best effort */ }
|
|
2150
2459
|
}
|