@songsid/agend 2.1.1-beta.17 → 2.1.1-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/claude-code.d.ts +37 -0
- package/dist/backend/claude-code.js +40 -1
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/kiro.d.ts +20 -0
- package/dist/backend/kiro.js +30 -0
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +27 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +15 -0
- package/dist/channel/adapters/discord.js +81 -1
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +21 -0
- package/dist/channel/adapters/telegram.js +82 -0
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/ipc-bridge.d.ts +9 -1
- package/dist/channel/ipc-bridge.js +12 -3
- package/dist/channel/ipc-bridge.js.map +1 -1
- package/dist/channel/types.d.ts +29 -0
- package/dist/daemon.d.ts +101 -5
- package/dist/daemon.js +304 -50
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-manager.d.ts +92 -1
- package/dist/fleet-manager.js +330 -19
- package/dist/fleet-manager.js.map +1 -1
- package/dist/hang-detector.d.ts +19 -13
- package/dist/hang-detector.js +19 -49
- package/dist/hang-detector.js.map +1 -1
- package/dist/pane-write-lock.d.ts +48 -0
- package/dist/pane-write-lock.js +73 -0
- package/dist/pane-write-lock.js.map +1 -0
- package/dist/tmux-control.d.ts +48 -1
- package/dist/tmux-control.js +79 -6
- package/dist/tmux-control.js.map +1 -1
- package/package.json +1 -1
- package/dist/channel/tool-tracker.d.ts +0 -13
- package/dist/channel/tool-tracker.js +0 -58
- package/dist/channel/tool-tracker.js.map +0 -1
- package/dist/daemon-entry.d.ts +0 -1
- package/dist/daemon-entry.js +0 -30
- package/dist/daemon-entry.js.map +0 -1
package/dist/daemon.js
CHANGED
|
@@ -16,6 +16,7 @@ import { shellQuote } from "./backend/types.js";
|
|
|
16
16
|
import { getTmuxSession } from "./config.js";
|
|
17
17
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
18
18
|
import { HangDetector } from "./hang-detector.js";
|
|
19
|
+
import { PaneWriteLock } from "./pane-write-lock.js";
|
|
19
20
|
import { buildFleetInstructions } from "./instructions.js";
|
|
20
21
|
const __filename = fileURLToPath(import.meta.url);
|
|
21
22
|
const __dirname = dirname(__filename);
|
|
@@ -40,8 +41,6 @@ export function buildInstructionReloadNotice(binaryName, instanceName, instanceD
|
|
|
40
41
|
export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
|
|
41
42
|
export const DEFAULT_STATE_IDLE_DEBOUNCE_MS = 2_000;
|
|
42
43
|
export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
|
|
43
|
-
/** @deprecated State detection is event-driven; this now aliases the safety sweep. */
|
|
44
|
-
export const DEFAULT_STATE_POLL_INTERVAL_MS = DEFAULT_STATE_SAFETY_SWEEP_MS;
|
|
45
44
|
const LAST_INBOUND_FILE = "last-inbound-at";
|
|
46
45
|
/** Minimum gap between "health check is failing" notifications for one instance. */
|
|
47
46
|
const HEALTH_ERROR_NOTIFY_INTERVAL_MS = 10 * 60_000;
|
|
@@ -186,24 +185,47 @@ export class AutoPauseController {
|
|
|
186
185
|
* Pane motion wins over a ready match because several backends keep their ready
|
|
187
186
|
* marker in a persistent header/footer while generating. A stable ready pane is
|
|
188
187
|
* idle; changing content is working; stable non-ready content eventually sticks.
|
|
188
|
+
*
|
|
189
|
+
* ## Why an optional busy pattern exists
|
|
190
|
+
*
|
|
191
|
+
* "Motion wins" degrades badly when a backend's ready marker is *always* on
|
|
192
|
+
* screen. `stuck` requires `!ready`, so for such a backend the state can only ever
|
|
193
|
+
* be idle or working — the stuck edge, and with it `handleStuckTransition` and the
|
|
194
|
+
* hang notification, is unreachable. A frozen CLI is then reported as idle, which
|
|
195
|
+
* also clears pending work: the message the user sent is quietly booked as done.
|
|
196
|
+
*
|
|
197
|
+
* claude-code was exactly this case (see ClaudeCodeBackend.getReadyPattern). A
|
|
198
|
+
* backend that can point at a marker meaning "generating right now" supplies
|
|
199
|
+
* `getBusyPattern()`, and that match vetoes ready regardless of what the ready
|
|
200
|
+
* pattern says.
|
|
189
201
|
*/
|
|
190
202
|
export class PaneStateMachine {
|
|
191
203
|
stuckTimeoutMs;
|
|
192
204
|
readyPattern;
|
|
205
|
+
busyPattern;
|
|
193
206
|
lastPaneHash = null;
|
|
194
207
|
lastPaneChangeAt;
|
|
195
208
|
lastObservedAt;
|
|
196
209
|
stateChangedAt;
|
|
197
210
|
currentState = "idle";
|
|
198
|
-
constructor(readyPattern, stuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS, now = Date.now()) {
|
|
211
|
+
constructor(readyPattern, stuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS, now = Date.now(), busyPattern) {
|
|
199
212
|
this.stuckTimeoutMs = stuckTimeoutMs;
|
|
200
213
|
// Stateful g/y regexes mutate lastIndex and can alternate true/false across
|
|
201
214
|
// polls. State detection must be deterministic for identical pane content.
|
|
202
215
|
this.readyPattern = new RegExp(readyPattern.source, readyPattern.flags.replace(/[gy]/g, ""));
|
|
216
|
+
this.busyPattern = busyPattern
|
|
217
|
+
? new RegExp(busyPattern.source, busyPattern.flags.replace(/[gy]/g, ""))
|
|
218
|
+
: null;
|
|
203
219
|
this.lastPaneChangeAt = now;
|
|
204
220
|
this.lastObservedAt = now;
|
|
205
221
|
this.stateChangedAt = now;
|
|
206
222
|
}
|
|
223
|
+
/** Ready, unless the backend can positively see that it is still generating. */
|
|
224
|
+
isReady(pane) {
|
|
225
|
+
if (this.busyPattern?.test(pane))
|
|
226
|
+
return false;
|
|
227
|
+
return this.readyPattern.test(pane);
|
|
228
|
+
}
|
|
207
229
|
observe(pane, now = Date.now()) {
|
|
208
230
|
const paneHash = createHash("sha256").update(pane).digest("hex");
|
|
209
231
|
const firstObservation = this.lastPaneHash === null;
|
|
@@ -213,7 +235,7 @@ export class PaneStateMachine {
|
|
|
213
235
|
this.lastPaneChangeAt = now;
|
|
214
236
|
}
|
|
215
237
|
this.lastObservedAt = now;
|
|
216
|
-
const ready = this.
|
|
238
|
+
const ready = this.isReady(pane);
|
|
217
239
|
const nextState = firstObservation
|
|
218
240
|
? ready ? "idle" : "working"
|
|
219
241
|
: paneChanged
|
|
@@ -279,6 +301,21 @@ const FIRST_ENTER_SETTLE_MS = 1_750;
|
|
|
279
301
|
const FIRST_DELIVERY_WINDOW_MS = 5_000;
|
|
280
302
|
/** After busy native-queue paste+Enter, wait before checking the pane for silent loss. */
|
|
281
303
|
const NATIVE_QUEUE_PASTE_VERIFY_MS = 2_000;
|
|
304
|
+
/** Enter-confirmation poll: 10 × 200ms ≈ 2s of observed silence before giving up. */
|
|
305
|
+
const CONFIRM_BUSY_POLLS = 10;
|
|
306
|
+
const CONFIRM_BUSY_POLL_MS = 200;
|
|
307
|
+
/**
|
|
308
|
+
* Ceiling on the whole confirmation, including time re-earned after a control-mode
|
|
309
|
+
* reconnect. Without it a reconnect loop could restart the poll indefinitely and
|
|
310
|
+
* hold the pane write lock along with it.
|
|
311
|
+
*/
|
|
312
|
+
const CONFIRM_BUSY_MAX_WAIT_MS = 10_000;
|
|
313
|
+
/**
|
|
314
|
+
* How long a delivery waits for an in-flight spawn. Comfortably past the default
|
|
315
|
+
* 25s startup timeout plus dialog dismissal; past it we fall back to the old
|
|
316
|
+
* behaviour rather than holding a message indefinitely.
|
|
317
|
+
*/
|
|
318
|
+
const SPAWN_SETTLE_MAX_WAIT_MS = 60_000;
|
|
282
319
|
/**
|
|
283
320
|
* One-shot timing gate for the first paste after a CLI reaches its ready
|
|
284
321
|
* prompt. Recent tmux versions expose a short interval where the prompt is
|
|
@@ -337,7 +374,6 @@ export class Daemon extends EventEmitter {
|
|
|
337
374
|
ipcServer = null;
|
|
338
375
|
messageBus;
|
|
339
376
|
transcriptMonitor = null;
|
|
340
|
-
toolTracker = null;
|
|
341
377
|
guardian = null;
|
|
342
378
|
adapter = null;
|
|
343
379
|
pendingIpcRequests = new Map();
|
|
@@ -348,6 +384,8 @@ export class Daemon extends EventEmitter {
|
|
|
348
384
|
// Pending ack: react 🫡 on first transcript activity after receiving a message
|
|
349
385
|
pendingAckMessage = null;
|
|
350
386
|
// Tool status tracking for channel adapter
|
|
387
|
+
/** Last activity published to the fleet manager; null when nothing is running. */
|
|
388
|
+
currentActivity = null;
|
|
351
389
|
toolStatusMessageId = null;
|
|
352
390
|
toolStatusLines = [];
|
|
353
391
|
toolStatusDebounce = null;
|
|
@@ -367,6 +405,15 @@ export class Daemon extends EventEmitter {
|
|
|
367
405
|
/** CLI pane availability, independent from the daemon process and tri-state. */
|
|
368
406
|
processStatus = "running";
|
|
369
407
|
spawning = false;
|
|
408
|
+
/**
|
|
409
|
+
* Resolves when the in-flight spawn finishes; null when none is running.
|
|
410
|
+
*
|
|
411
|
+
* `spawning` alone can only be polled. Delivery needs to *wait*, and a boolean
|
|
412
|
+
* poll would either busy-loop or race the flag being cleared.
|
|
413
|
+
*/
|
|
414
|
+
spawnSettled = null;
|
|
415
|
+
resolveSpawnSettled = null;
|
|
416
|
+
spawnDepth = 0;
|
|
370
417
|
skipResume = false;
|
|
371
418
|
backgroundSessionRecoveryAttempted = false;
|
|
372
419
|
/** Whether the last spawn started a fresh session (not resumed). */
|
|
@@ -389,6 +436,7 @@ export class Daemon extends EventEmitter {
|
|
|
389
436
|
instanceStateIdleDebounceMs = DEFAULT_STATE_IDLE_DEBOUNCE_MS;
|
|
390
437
|
instanceStateStuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS;
|
|
391
438
|
instanceStateReadyPattern = null;
|
|
439
|
+
instanceStateBusyPattern = null;
|
|
392
440
|
instanceStateMonitorActive = false;
|
|
393
441
|
statePollInFlight = false;
|
|
394
442
|
autoPauseController;
|
|
@@ -409,7 +457,13 @@ export class Daemon extends EventEmitter {
|
|
|
409
457
|
recentEvents = [];
|
|
410
458
|
recentToolActivity = [];
|
|
411
459
|
snapshotConsumed = false;
|
|
460
|
+
/** Orders inbound channel messages against each other (queue-depth accounting
|
|
461
|
+
* and ⏳/👀/✅ reactions live here). It does NOT cover the other writers that
|
|
462
|
+
* reach the pane — {@link paneWriteLock} does. */
|
|
412
463
|
pasteLock = Promise.resolve();
|
|
464
|
+
/** Mutual exclusion for *every* write into the pane, whichever subsystem it
|
|
465
|
+
* comes from. See PaneWriteLock for why interleaving is destructive. */
|
|
466
|
+
paneWriteLock = new PaneWriteLock();
|
|
413
467
|
pendingInstructionsUpdate;
|
|
414
468
|
pendingInstructionsNotice = false;
|
|
415
469
|
// Whether the warmup steering-reload notice should be injected after spawn.
|
|
@@ -552,9 +606,6 @@ export class Daemon extends EventEmitter {
|
|
|
552
606
|
if ((type === "fleet_schedule_response" || type === "fleet_outbound_response" || type === "fleet_decision_response" || type === "fleet_task_response" || type === "fleet_display_name_response" || type === "fleet_description_response") && msg.fleetRequestId) {
|
|
553
607
|
key = String(msg.fleetRequestId);
|
|
554
608
|
}
|
|
555
|
-
else if (type === "fleet_outbound_response" && msg.requestId != null) {
|
|
556
|
-
key = `fleet_out_${msg.requestId}`;
|
|
557
|
-
}
|
|
558
609
|
if (key && this.pendingIpcRequests.has(key)) {
|
|
559
610
|
const handler = this.pendingIpcRequests.get(key);
|
|
560
611
|
this.pendingIpcRequests.delete(key);
|
|
@@ -603,6 +654,14 @@ export class Daemon extends EventEmitter {
|
|
|
603
654
|
// Fleet manager routed a message to us (topic mode)
|
|
604
655
|
const meta = msg.meta;
|
|
605
656
|
const targetSession = msg.targetSession;
|
|
657
|
+
// `no_wake` marks low-value signals (a non-approval reaction, #408): worth
|
|
658
|
+
// showing an instance that is already awake, never worth resuming a paused
|
|
659
|
+
// one and spending a turn on. Dropped rather than queued, so a paused
|
|
660
|
+
// instance does not surface a pile of stale emoji on its next wake.
|
|
661
|
+
if (meta?.no_wake === "true" && this.isPaused) {
|
|
662
|
+
this.logger.debug({ preview: String(msg.content).slice(0, 60) }, "Dropping no_wake message for paused instance");
|
|
663
|
+
return;
|
|
664
|
+
}
|
|
606
665
|
void this.wake().then(() => {
|
|
607
666
|
this.pushChannelMessage(msg.content, meta, targetSession);
|
|
608
667
|
}).catch(err => {
|
|
@@ -705,7 +764,12 @@ export class Daemon extends EventEmitter {
|
|
|
705
764
|
else {
|
|
706
765
|
await new Promise(r => setTimeout(r, 5000));
|
|
707
766
|
}
|
|
708
|
-
|
|
767
|
+
// This path only runs when pasteQueueDepth > 0 — i.e. exactly when a real
|
|
768
|
+
// delivery is already in flight or queued. Without the lock the notice and
|
|
769
|
+
// that delivery race into the same pane.
|
|
770
|
+
await this.paneWriteLock.run(async () => {
|
|
771
|
+
await this.tmux?.pasteText(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir));
|
|
772
|
+
});
|
|
709
773
|
// Record the value the agent has now been told about so the next
|
|
710
774
|
// unchanged restart skips the reload.
|
|
711
775
|
try {
|
|
@@ -738,27 +802,31 @@ export class Daemon extends EventEmitter {
|
|
|
738
802
|
this.transcriptMonitor.on("tool_use", (name, input) => {
|
|
739
803
|
this.logger.debug({ tool: name }, "Tool use");
|
|
740
804
|
ackIfPending();
|
|
741
|
-
this.hangDetector?.recordActivity();
|
|
742
805
|
this.recordRecentEvent({ type: "tool_use", name, preview: this.summarizeTool(name, input) });
|
|
743
806
|
this.recordRecentToolActivity(this.summarizeTool(name, input));
|
|
807
|
+
this.publishActivity(this.summarizeTool(name, input));
|
|
744
808
|
});
|
|
745
809
|
this.transcriptMonitor.on("tool_result", (name, _output) => {
|
|
746
|
-
this.hangDetector?.recordActivity();
|
|
747
810
|
this.recordRecentEvent({ type: "tool_result", name });
|
|
811
|
+
// The tool finished; whatever comes next has not started yet. Better to
|
|
812
|
+
// show only elapsed time than to leave a stale "Bash: npm test" on screen.
|
|
813
|
+
this.publishActivity(null);
|
|
748
814
|
});
|
|
749
815
|
this.transcriptMonitor.on("assistant_text", (text) => {
|
|
750
816
|
this.logger.debug({ text: text.slice(0, 200) }, "Claude response");
|
|
751
817
|
ackIfPending();
|
|
752
|
-
this.hangDetector?.recordActivity();
|
|
753
818
|
this.recordRecentEvent({ type: "assistant_text", preview: text.slice(0, 100) });
|
|
754
819
|
});
|
|
755
820
|
this.transcriptMonitor.startPolling();
|
|
756
|
-
// HangDetector
|
|
757
|
-
//
|
|
758
|
-
//
|
|
821
|
+
// HangDetector is the notification bridge to the fleet manager; pane state
|
|
822
|
+
// transitions below are the sole source of hang events (see hang-detector.ts).
|
|
823
|
+
// `hang_detector.enabled: false` opts out of the notification entirely.
|
|
824
|
+
// NOTE: `hang_detector.timeout_minutes` is NOT read here — the stuck timeout
|
|
825
|
+
// the pane monitor actually uses is resolved separately below. The two have
|
|
826
|
+
// long been separate; documented rather than silently ignored.
|
|
759
827
|
const hangConfig = this.config.hang_detector;
|
|
760
828
|
if (hangConfig?.enabled !== false) {
|
|
761
|
-
this.hangDetector = new HangDetector(
|
|
829
|
+
this.hangDetector = new HangDetector();
|
|
762
830
|
}
|
|
763
831
|
// 8. Context guardian
|
|
764
832
|
const statusFile = join(this.instanceDir, "statusline.json");
|
|
@@ -766,7 +834,6 @@ export class Daemon extends EventEmitter {
|
|
|
766
834
|
this.guardian.startWatching();
|
|
767
835
|
this.guardian.on("status_update", () => {
|
|
768
836
|
this.saveSessionId();
|
|
769
|
-
this.hangDetector?.recordStatuslineUpdate();
|
|
770
837
|
});
|
|
771
838
|
// Context rotation removed: all CLI backends have built-in auto-compact.
|
|
772
839
|
// Crash recovery (health check + respawn with snapshot) is retained below.
|
|
@@ -1185,6 +1252,7 @@ export class Daemon extends EventEmitter {
|
|
|
1185
1252
|
if (!this.backend)
|
|
1186
1253
|
return; // lightweight mode has no backend
|
|
1187
1254
|
const readyPattern = this.backend.getReadyPattern();
|
|
1255
|
+
const busyPattern = this.backend.getBusyPattern?.() ?? null;
|
|
1188
1256
|
this.errorMonitorTimer = setInterval(async () => {
|
|
1189
1257
|
if (!this.tmux || this.spawning)
|
|
1190
1258
|
return;
|
|
@@ -1197,20 +1265,30 @@ export class Daemon extends EventEmitter {
|
|
|
1197
1265
|
for (const dialog of dialogs) {
|
|
1198
1266
|
if (!dialog.pattern.test(pane))
|
|
1199
1267
|
continue;
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1268
|
+
// These keys go straight into the pane. Sent while a message delivery is
|
|
1269
|
+
// mid-transaction, an `Escape` wipes the pasted text and an `Enter`
|
|
1270
|
+
// submits it half-composed — the user sees a message that vanished. Skip
|
|
1271
|
+
// (not queue) when the pane is busy: this poller runs every 5s, and the
|
|
1272
|
+
// dialog will still be on screen next tick.
|
|
1273
|
+
const dismissed = await this.paneWriteLock.tryRun(async () => {
|
|
1274
|
+
this.logger.info(`Auto-dismissing runtime dialog: ${dialog.description}`);
|
|
1275
|
+
const SPECIAL_KEYS = new Set(["Up", "Down", "Enter", "Escape", "Right", "Left"]);
|
|
1276
|
+
for (const key of dialog.keys) {
|
|
1277
|
+
if (SPECIAL_KEYS.has(key)) {
|
|
1278
|
+
await this.tmux.sendSpecialKey(key);
|
|
1279
|
+
}
|
|
1280
|
+
else {
|
|
1281
|
+
await this.tmux.pasteText(key);
|
|
1282
|
+
}
|
|
1283
|
+
await new Promise(r => setTimeout(r, 200));
|
|
1208
1284
|
}
|
|
1209
|
-
|
|
1285
|
+
});
|
|
1286
|
+
if (!dismissed) {
|
|
1287
|
+
this.logger.debug({ dialog: dialog.description }, "Dialog dismissal deferred — pane write in flight");
|
|
1210
1288
|
}
|
|
1211
|
-
return; // Dialog
|
|
1289
|
+
return; // Dialog handled (or deliberately deferred): skip error checks this cycle
|
|
1212
1290
|
}
|
|
1213
|
-
this.evaluateErrorPatterns(pane, patterns, readyPattern);
|
|
1291
|
+
this.evaluateErrorPatterns(pane, patterns, readyPattern, Date.now(), busyPattern);
|
|
1214
1292
|
}
|
|
1215
1293
|
catch {
|
|
1216
1294
|
// capturePane can fail if window is transitioning — ignore
|
|
@@ -1218,7 +1296,7 @@ export class Daemon extends EventEmitter {
|
|
|
1218
1296
|
}, 5_000); // Check every 5 seconds (runtime dialogs need fast response)
|
|
1219
1297
|
}
|
|
1220
1298
|
/** Evaluate one pane snapshot. Kept synchronous so state-machine edges are unit-testable. */
|
|
1221
|
-
evaluateErrorPatterns(pane, patterns, readyPattern, now = Date.now()) {
|
|
1299
|
+
evaluateErrorPatterns(pane, patterns, readyPattern, now = Date.now(), busyPattern = null) {
|
|
1222
1300
|
// Count occurrences across the WHOLE pane (not just text after the last
|
|
1223
1301
|
// ready prompt). Clone with `g` so stateful backend regexes cannot leak
|
|
1224
1302
|
// lastIndex between monitor cycles.
|
|
@@ -1226,10 +1304,15 @@ export class Daemon extends EventEmitter {
|
|
|
1226
1304
|
const flags = pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g";
|
|
1227
1305
|
return (pane.match(new RegExp(pattern.source, flags)) || []).length;
|
|
1228
1306
|
};
|
|
1307
|
+
// Same veto as the state machine: a backend whose ready marker is permanently
|
|
1308
|
+
// on screen would otherwise "recover" on the very first tick after the error,
|
|
1309
|
+
// rebaselining the occurrence count while the error is still displayed — one
|
|
1310
|
+
// notification, a false recovery log, then silence.
|
|
1311
|
+
const looksReady = () => !busyPattern?.test(pane) && readyPattern.test(pane);
|
|
1229
1312
|
// State: waiting for recovery. A missing/outdated ready pattern must not
|
|
1230
1313
|
// suppress every future error forever, so the gate has a hard deadline.
|
|
1231
1314
|
if (this.errorWaitingForRecovery) {
|
|
1232
|
-
if (
|
|
1315
|
+
if (looksReady()) {
|
|
1233
1316
|
const downtime = Math.round((now - this.errorDetectedAt) / 1000);
|
|
1234
1317
|
const active = patterns.find(ep => Daemon.errorPatternKey(ep) === this.activeErrorPatternKey);
|
|
1235
1318
|
if (active) {
|
|
@@ -1328,6 +1411,12 @@ export class Daemon extends EventEmitter {
|
|
|
1328
1411
|
* Interrupt the CLI's current generation (cancel button / `/cancel`).
|
|
1329
1412
|
* Direct tmux key event (not a paste) so it registers as the interrupt key.
|
|
1330
1413
|
* kiro-cli interrupts on Ctrl+C; the others (claude-code, codex, …) on Escape.
|
|
1414
|
+
*
|
|
1415
|
+
* Deliberately NOT taken through `paneWriteLock`. Cancel is the user's way out
|
|
1416
|
+
* of a pane that is busy or wedged — queueing it behind the very delivery chain
|
|
1417
|
+
* it exists to unblock would make the button useless exactly when it is needed.
|
|
1418
|
+
* The cost is accepted: an Escape landing between a paste and its Enter discards
|
|
1419
|
+
* that message, which is what the user asked for anyway.
|
|
1331
1420
|
*/
|
|
1332
1421
|
async sendEscape() {
|
|
1333
1422
|
const cancelKey = this.backend?.getCancelKey() ?? "Escape";
|
|
@@ -1365,7 +1454,6 @@ export class Daemon extends EventEmitter {
|
|
|
1365
1454
|
this.toolStatusDebounce = null;
|
|
1366
1455
|
}
|
|
1367
1456
|
this.pendingIpcRequests.clear();
|
|
1368
|
-
this.hangDetector?.stop();
|
|
1369
1457
|
if (this.adapter)
|
|
1370
1458
|
await this.adapter.stop();
|
|
1371
1459
|
// Notify MCP servers of graceful shutdown (prevents reconnect attempts)
|
|
@@ -1392,7 +1480,12 @@ export class Daemon extends EventEmitter {
|
|
|
1392
1480
|
if (!killed)
|
|
1393
1481
|
this.logger.warn("CLI did not exit gracefully within 3s, force killing window");
|
|
1394
1482
|
// Always kill window — remain-on-exit keeps dead panes around after CLI exits
|
|
1483
|
+
const stoppedWindowId = this.tmux.getWindowId();
|
|
1395
1484
|
await this.tmux.killWindow();
|
|
1485
|
+
// The control client is shared across the fleet and outlives this daemon, so
|
|
1486
|
+
// a registration left behind here is re-resolved on every reconnect forever.
|
|
1487
|
+
if (stoppedWindowId)
|
|
1488
|
+
this.controlClient?.unregisterWindow(stoppedWindowId);
|
|
1396
1489
|
const windowIdFile = join(this.instanceDir, "window-id");
|
|
1397
1490
|
try {
|
|
1398
1491
|
unlinkSync(windowIdFile);
|
|
@@ -1528,7 +1621,7 @@ export class Daemon extends EventEmitter {
|
|
|
1528
1621
|
return;
|
|
1529
1622
|
}
|
|
1530
1623
|
this.pauseWakeState = "waking";
|
|
1531
|
-
this.
|
|
1624
|
+
this.beginSpawn();
|
|
1532
1625
|
const transition = this.autoPauseController.wakeOnDeliver(async () => {
|
|
1533
1626
|
let timeout;
|
|
1534
1627
|
try {
|
|
@@ -1569,7 +1662,7 @@ export class Daemon extends EventEmitter {
|
|
|
1569
1662
|
throw err;
|
|
1570
1663
|
}
|
|
1571
1664
|
finally {
|
|
1572
|
-
this.
|
|
1665
|
+
this.endSpawn();
|
|
1573
1666
|
if (this.pauseWakeTransition === transition)
|
|
1574
1667
|
this.pauseWakeTransition = null;
|
|
1575
1668
|
}
|
|
@@ -1581,6 +1674,10 @@ export class Daemon extends EventEmitter {
|
|
|
1581
1674
|
// observations between enqueue and paste must not clear a newer inbound.
|
|
1582
1675
|
if (snapshot.state === "idle" && previous !== "idle") {
|
|
1583
1676
|
this.pendingWork.recordIdle(snapshot.observedAt);
|
|
1677
|
+
// The turn is over. A transcript can end on a tool_use with no matching
|
|
1678
|
+
// tool_result (interrupted, crashed, cancelled), which would otherwise leave
|
|
1679
|
+
// the last tool pinned to the progress line for the rest of the session.
|
|
1680
|
+
this.publishActivity(null);
|
|
1584
1681
|
}
|
|
1585
1682
|
if (snapshot.state !== previous) {
|
|
1586
1683
|
this.logger.info({
|
|
@@ -1671,6 +1768,13 @@ export class Daemon extends EventEmitter {
|
|
|
1671
1768
|
this.instanceStateMachine.observe(pane, observedChangeAt);
|
|
1672
1769
|
const snapshot = this.instanceStateMachine.observe(pane, Date.now());
|
|
1673
1770
|
this.applyInstanceStateSnapshot(snapshot, pane);
|
|
1771
|
+
// Backends without a transcript feed can still say what they are doing, if
|
|
1772
|
+
// their TUI names it. Free-riding on a capture that already happened: no
|
|
1773
|
+
// extra tmux call. Cadence is the capture cadence (idle debounce + the 60s
|
|
1774
|
+
// safety sweep), which matches the progress ticker's own 60s interval.
|
|
1775
|
+
if (this.backend?.getPaneActivity) {
|
|
1776
|
+
this.publishActivity(snapshot.state === "idle" ? null : this.backend.getPaneActivity(pane));
|
|
1777
|
+
}
|
|
1674
1778
|
if (snapshot.state === "idle") {
|
|
1675
1779
|
this.clearInstanceStateStuckTimer();
|
|
1676
1780
|
}
|
|
@@ -1725,7 +1829,8 @@ export class Daemon extends EventEmitter {
|
|
|
1725
1829
|
? rawConfig.idle_debounce_ms
|
|
1726
1830
|
: DEFAULT_STATE_IDLE_DEBOUNCE_MS;
|
|
1727
1831
|
this.instanceStateReadyPattern = this.backend.getReadyPattern();
|
|
1728
|
-
this.
|
|
1832
|
+
this.instanceStateBusyPattern = this.backend.getBusyPattern?.() ?? null;
|
|
1833
|
+
this.instanceStateMachine = new PaneStateMachine(this.instanceStateReadyPattern, this.instanceStateStuckTimeoutMs, Date.now(), this.instanceStateBusyPattern);
|
|
1729
1834
|
this.instanceStateLastOutputAt = 0;
|
|
1730
1835
|
this.instanceStateMonitorActive = true;
|
|
1731
1836
|
if (this.controlClient) {
|
|
@@ -1766,6 +1871,12 @@ export class Daemon extends EventEmitter {
|
|
|
1766
1871
|
paneTail: sanitizePaneTail(pane),
|
|
1767
1872
|
readyPattern: readyPattern.toString(),
|
|
1768
1873
|
readyMatched: deterministicReadyPattern.test(pane),
|
|
1874
|
+
// A pane that is stuck *while* the ready marker matches means the busy
|
|
1875
|
+
// pattern is what held it back — worth seeing when tuning either regex.
|
|
1876
|
+
busyPattern: this.instanceStateBusyPattern?.toString(),
|
|
1877
|
+
busyMatched: this.instanceStateBusyPattern
|
|
1878
|
+
? new RegExp(this.instanceStateBusyPattern.source, this.instanceStateBusyPattern.flags.replace(/[gy]/g, "")).test(pane)
|
|
1879
|
+
: undefined,
|
|
1769
1880
|
unchangedForMs: snapshot.unchangedForMs,
|
|
1770
1881
|
pendingWork: this.pendingWork.hasPendingWork(),
|
|
1771
1882
|
};
|
|
@@ -1808,6 +1919,28 @@ export class Daemon extends EventEmitter {
|
|
|
1808
1919
|
return this.messageBus;
|
|
1809
1920
|
}
|
|
1810
1921
|
// ── Tool status tracking ──────────────────────────────────────
|
|
1922
|
+
/**
|
|
1923
|
+
* Tell the fleet manager what this instance is doing right now, for the live
|
|
1924
|
+
* progress line on the cancel button.
|
|
1925
|
+
*
|
|
1926
|
+
* Purely cosmetic — nothing decides anything from it, so it is fine that only
|
|
1927
|
+
* backends with a transcript feed report at all (claude-code today). For the
|
|
1928
|
+
* rest the progress line keeps showing just elapsed time.
|
|
1929
|
+
*
|
|
1930
|
+
* Repeats are dropped: the ticker only edits the channel message when the text
|
|
1931
|
+
* changes, and a stream of identical broadcasts would defeat that.
|
|
1932
|
+
*/
|
|
1933
|
+
publishActivity(activity) {
|
|
1934
|
+
const next = activity && activity.trim() ? activity.trim() : null;
|
|
1935
|
+
if (next === this.currentActivity)
|
|
1936
|
+
return;
|
|
1937
|
+
this.currentActivity = next;
|
|
1938
|
+
this.ipcServer?.broadcast({
|
|
1939
|
+
type: "instance_activity",
|
|
1940
|
+
instanceName: this.name,
|
|
1941
|
+
activity: next,
|
|
1942
|
+
});
|
|
1943
|
+
}
|
|
1811
1944
|
summarizeTool(name, input) {
|
|
1812
1945
|
const inp = input;
|
|
1813
1946
|
if (!inp)
|
|
@@ -1897,7 +2030,6 @@ export class Daemon extends EventEmitter {
|
|
|
1897
2030
|
writeFileSync(join(this.instanceDir, "prev-instructions"), this.pendingInstructionsUpdate);
|
|
1898
2031
|
this.pendingInstructionsUpdate = undefined;
|
|
1899
2032
|
}
|
|
1900
|
-
this.hangDetector?.recordInbound();
|
|
1901
2033
|
this.pendingWork.recordInbound();
|
|
1902
2034
|
// v3: record user messages for rotation snapshot
|
|
1903
2035
|
this.recordRecentUserMessage(content, meta);
|
|
@@ -1997,6 +2129,8 @@ export class Daemon extends EventEmitter {
|
|
|
1997
2129
|
// Odd number of fences = unclosed. Remove all code fences from the message.
|
|
1998
2130
|
formatted = formatted.replace(/```/g, "");
|
|
1999
2131
|
}
|
|
2132
|
+
// Before anything reads the window id: a spawn in progress is about to change it.
|
|
2133
|
+
await this.waitForSpawnToSettle();
|
|
2000
2134
|
let windowId = this.getWindowId();
|
|
2001
2135
|
const supportsQueuedInput = this.backend?.supportsQueuedInput?.() === true;
|
|
2002
2136
|
let handingOffToNativeQueue = false;
|
|
@@ -2024,6 +2158,21 @@ export class Daemon extends EventEmitter {
|
|
|
2024
2158
|
}
|
|
2025
2159
|
}
|
|
2026
2160
|
}
|
|
2161
|
+
// Everything above is *waiting*; everything below *writes*. Only the write is
|
|
2162
|
+
// held under the pane lock — holding it across the idle wait (up to 30 min)
|
|
2163
|
+
// would starve the runtime-dialog dismisser, which is often the very thing
|
|
2164
|
+
// that would let the pane go idle again.
|
|
2165
|
+
return this.paneWriteLock.run(() => this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status));
|
|
2166
|
+
}
|
|
2167
|
+
/**
|
|
2168
|
+
* The write half of a delivery: paste → Enter → confirm, with retries.
|
|
2169
|
+
*
|
|
2170
|
+
* Always called under {@link paneWriteLock}. Split out from `deliverMessage`
|
|
2171
|
+
* precisely so the lock's scope is visible at the call site rather than being
|
|
2172
|
+
* an invariant maintained by comments.
|
|
2173
|
+
*/
|
|
2174
|
+
async writeMessageToPane(formatted, initialWindowId, handingOffToNativeQueue, status) {
|
|
2175
|
+
let windowId = initialWindowId;
|
|
2027
2176
|
// Bug A: paste with backoff. Transient failures are usually a stale window id
|
|
2028
2177
|
// after a crash/respawn — recover by name and retry (max 3 attempts, 2s apart).
|
|
2029
2178
|
const maxAttempts = 3;
|
|
@@ -2164,6 +2313,7 @@ export class Daemon extends EventEmitter {
|
|
|
2164
2313
|
}
|
|
2165
2314
|
/** Re-resolve this instance's tmux window by name (stale id after crash/respawn). */
|
|
2166
2315
|
async recoverWindow() {
|
|
2316
|
+
const previousWindowId = this.tmux?.getWindowId();
|
|
2167
2317
|
try {
|
|
2168
2318
|
const windows = await TmuxManager.listWindows(this.tmuxSessionName);
|
|
2169
2319
|
const match = windows.find(w => w.name === this.name);
|
|
@@ -2171,6 +2321,12 @@ export class Daemon extends EventEmitter {
|
|
|
2171
2321
|
return undefined;
|
|
2172
2322
|
this.tmux = new TmuxManager(this.tmuxSessionName, match.id, resolveTmuxLogicalSize(this.config.terminal));
|
|
2173
2323
|
writeFileSync(join(this.instanceDir, "window-id"), match.id);
|
|
2324
|
+
// The window we were talking to is gone; leaving it registered means the
|
|
2325
|
+
// control client re-resolves a dead id — one tmux subprocess — on every
|
|
2326
|
+
// reconnect, for the life of the fleet process.
|
|
2327
|
+
if (previousWindowId && previousWindowId !== match.id) {
|
|
2328
|
+
this.controlClient?.unregisterWindow(previousWindowId);
|
|
2329
|
+
}
|
|
2174
2330
|
await this.controlClient?.registerWindow(match.id);
|
|
2175
2331
|
this.bindInstanceStateOutputListener(match.id);
|
|
2176
2332
|
this.logger.info({ windowId: match.id }, "Recovered window ID for message delivery");
|
|
@@ -2181,12 +2337,38 @@ export class Daemon extends EventEmitter {
|
|
|
2181
2337
|
return undefined;
|
|
2182
2338
|
}
|
|
2183
2339
|
}
|
|
2184
|
-
/**
|
|
2340
|
+
/**
|
|
2341
|
+
* Poll up to ~2s (200ms × 10) for the pane to emit output after `since`.
|
|
2342
|
+
*
|
|
2343
|
+
* A `false` here is not "no output" — it is read as **"the message was pasted but
|
|
2344
|
+
* never submitted"**, which re-sends Enter and then reports ❌ to the user. So a
|
|
2345
|
+
* false negative costs a possible double submit and a failure notice for a
|
|
2346
|
+
* message that actually arrived.
|
|
2347
|
+
*
|
|
2348
|
+
* That is exactly what a control-mode reconnect used to produce. `connect()`
|
|
2349
|
+
* drops every output timestamp, so `hasOutputSince` answers `false` for a pane
|
|
2350
|
+
* that did react — the evidence was thrown away, not absent. Blind time is
|
|
2351
|
+
* therefore not counted against the budget: after a reset the poll restarts from
|
|
2352
|
+
* the moment observation resumed. A hard wall-clock cap keeps a reconnect loop
|
|
2353
|
+
* from extending this forever.
|
|
2354
|
+
*/
|
|
2185
2355
|
async confirmBusyAfterEnter(windowId, since) {
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2356
|
+
const client = this.controlClient;
|
|
2357
|
+
const hardDeadline = Date.now() + CONFIRM_BUSY_MAX_WAIT_MS;
|
|
2358
|
+
let observedFrom = since;
|
|
2359
|
+
let polls = 0;
|
|
2360
|
+
while (polls < CONFIRM_BUSY_POLLS) {
|
|
2361
|
+
await new Promise(r => setTimeout(r, CONFIRM_BUSY_POLL_MS));
|
|
2362
|
+
if (client.hasOutputSince(windowId, observedFrom))
|
|
2189
2363
|
return true;
|
|
2364
|
+
const resetAt = client.getObservationResetAt();
|
|
2365
|
+
if (resetAt > observedFrom && Date.now() < hardDeadline) {
|
|
2366
|
+
this.logger.debug({ windowId, resetAt }, "Control mode reconnected mid-confirmation — restarting the Enter check from when we could see again");
|
|
2367
|
+
observedFrom = resetAt;
|
|
2368
|
+
polls = 0;
|
|
2369
|
+
continue;
|
|
2370
|
+
}
|
|
2371
|
+
polls++;
|
|
2190
2372
|
}
|
|
2191
2373
|
return false;
|
|
2192
2374
|
}
|
|
@@ -2564,7 +2746,9 @@ export class Daemon extends EventEmitter {
|
|
|
2564
2746
|
// Small delay to let the CLI fully render its ready prompt
|
|
2565
2747
|
await new Promise(r => setTimeout(r, 1_000));
|
|
2566
2748
|
try {
|
|
2567
|
-
|
|
2749
|
+
// Messages can arrive during a restart and be queued on pasteLock before the
|
|
2750
|
+
// snapshot lands; both write to the pane, so both go through the same lock.
|
|
2751
|
+
await this.paneWriteLock.run(() => this.tmux.pasteText(`[system:session-snapshot]\n${snapshot}\n\nThis is a background context restore — do NOT reply to or acknowledge this message. Simply resume normal operation when the next user or instance message arrives.`));
|
|
2568
2752
|
this.logger.info("Injected session snapshot as first message");
|
|
2569
2753
|
this.emit("snapshot_injected", this.name);
|
|
2570
2754
|
}
|
|
@@ -2573,9 +2757,69 @@ export class Daemon extends EventEmitter {
|
|
|
2573
2757
|
this.emit("snapshot_failed", this.name);
|
|
2574
2758
|
}
|
|
2575
2759
|
}
|
|
2760
|
+
/**
|
|
2761
|
+
* Depth, not a flag: the wake path marks a spawn and then calls trySpawn, which
|
|
2762
|
+
* marks another. With a plain boolean the inner one's completion would clear the
|
|
2763
|
+
* latch while the outer spawn was still dismissing dialogs, releasing delivery
|
|
2764
|
+
* into exactly the window this exists to close.
|
|
2765
|
+
*/
|
|
2766
|
+
beginSpawn() {
|
|
2767
|
+
this.spawnDepth++;
|
|
2768
|
+
this.spawning = true;
|
|
2769
|
+
if (!this.spawnSettled) {
|
|
2770
|
+
this.spawnSettled = new Promise(resolve => { this.resolveSpawnSettled = resolve; });
|
|
2771
|
+
}
|
|
2772
|
+
}
|
|
2773
|
+
endSpawn() {
|
|
2774
|
+
this.spawnDepth = Math.max(0, this.spawnDepth - 1);
|
|
2775
|
+
if (this.spawnDepth > 0)
|
|
2776
|
+
return; // an enclosing spawn is still running
|
|
2777
|
+
this.spawning = false;
|
|
2778
|
+
const resolve = this.resolveSpawnSettled;
|
|
2779
|
+
this.spawnSettled = null;
|
|
2780
|
+
this.resolveSpawnSettled = null;
|
|
2781
|
+
resolve?.();
|
|
2782
|
+
}
|
|
2783
|
+
/**
|
|
2784
|
+
* Hold until the CLI has finished starting up.
|
|
2785
|
+
*
|
|
2786
|
+
* Startup is not a quiet period: `dismissDialogsUntilReady` is clicking through
|
|
2787
|
+
* trust prompts and session pickers. A pane showing a modal dialog produces no
|
|
2788
|
+
* output, so `waitUntilIdle` reports it idle and a queued message gets pasted
|
|
2789
|
+
* *into the dialog* — where the text is discarded and the Enter picks a menu
|
|
2790
|
+
* item. The paste and the Enter both "succeed", so this loses the message
|
|
2791
|
+
* without even a ❌.
|
|
2792
|
+
*
|
|
2793
|
+
* The pane write lock (#414) does not cover this: it serialises each key
|
|
2794
|
+
* sequence, but it is released between one dialog and the next.
|
|
2795
|
+
*
|
|
2796
|
+
* Bounded, and called BEFORE the pane lock is taken — waiting on the spawn while
|
|
2797
|
+
* holding the lock the spawn itself needs would deadlock.
|
|
2798
|
+
*/
|
|
2799
|
+
async waitForSpawnToSettle() {
|
|
2800
|
+
const settled = this.spawnSettled;
|
|
2801
|
+
if (!settled)
|
|
2802
|
+
return;
|
|
2803
|
+
this.logger.debug("Holding delivery until the CLI has finished starting up");
|
|
2804
|
+
let timer;
|
|
2805
|
+
const cap = new Promise(resolve => {
|
|
2806
|
+
timer = setTimeout(resolve, SPAWN_SETTLE_MAX_WAIT_MS);
|
|
2807
|
+
timer.unref?.();
|
|
2808
|
+
});
|
|
2809
|
+
try {
|
|
2810
|
+
await Promise.race([settled, cap]);
|
|
2811
|
+
}
|
|
2812
|
+
finally {
|
|
2813
|
+
if (timer)
|
|
2814
|
+
clearTimeout(timer);
|
|
2815
|
+
}
|
|
2816
|
+
if (this.spawning) {
|
|
2817
|
+
this.logger.warn("CLI still starting after the delivery hold — delivering anyway");
|
|
2818
|
+
}
|
|
2819
|
+
}
|
|
2576
2820
|
/** Spawn a CLI window. Returns true if --resume was used successfully. */
|
|
2577
2821
|
async spawnClaudeWindow() {
|
|
2578
|
-
this.
|
|
2822
|
+
this.beginSpawn();
|
|
2579
2823
|
let resumedSuccessfully = false;
|
|
2580
2824
|
try {
|
|
2581
2825
|
this.toolStatusLines = [];
|
|
@@ -2612,7 +2856,7 @@ export class Daemon extends EventEmitter {
|
|
|
2612
2856
|
this.backgroundSessionRecoveryAttempted = false;
|
|
2613
2857
|
}
|
|
2614
2858
|
finally {
|
|
2615
|
-
this.
|
|
2859
|
+
this.endSpawn();
|
|
2616
2860
|
}
|
|
2617
2861
|
return resumedSuccessfully;
|
|
2618
2862
|
}
|
|
@@ -2705,7 +2949,12 @@ export class Daemon extends EventEmitter {
|
|
|
2705
2949
|
windowId = this.tmux.getWindowId();
|
|
2706
2950
|
}
|
|
2707
2951
|
else {
|
|
2952
|
+
// A crash respawn makes a brand new window. The dead one stays registered
|
|
2953
|
+
// unless we say so — the respawn branch above already does.
|
|
2954
|
+
const retired = this.tmux.getWindowId();
|
|
2708
2955
|
windowId = await this.tmux.createWindow(cmd, resolvedCwd, this.name);
|
|
2956
|
+
if (retired && retired !== windowId)
|
|
2957
|
+
this.controlClient?.unregisterWindow(retired);
|
|
2709
2958
|
}
|
|
2710
2959
|
writeFileSync(join(this.instanceDir, "window-id"), windowId);
|
|
2711
2960
|
// Enable remain-on-exit to capture exit codes on crash
|
|
@@ -2771,15 +3020,20 @@ export class Daemon extends EventEmitter {
|
|
|
2771
3020
|
for (const dialog of startupDialogs) {
|
|
2772
3021
|
if (dialog.pattern.test(pane)) {
|
|
2773
3022
|
this.logger.debug(`Dismissing startup dialog: ${dialog.description}`);
|
|
2774
|
-
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
3023
|
+
// Restart is exactly when inbound messages pile up, and nothing gates
|
|
3024
|
+
// delivery on `spawning`. Take the pane lock for the key sequence so a
|
|
3025
|
+
// queued message cannot be pasted into a half-dismissed trust dialog.
|
|
3026
|
+
await this.paneWriteLock.run(async () => {
|
|
3027
|
+
for (const key of dialog.keys) {
|
|
3028
|
+
if (key === "Up" || key === "Down" || key === "Enter" || key === "Escape") {
|
|
3029
|
+
await this.tmux.sendSpecialKey(key);
|
|
3030
|
+
}
|
|
3031
|
+
else {
|
|
3032
|
+
await this.tmux.sendKeys(key);
|
|
3033
|
+
}
|
|
3034
|
+
await new Promise(r => setTimeout(r, 200));
|
|
2780
3035
|
}
|
|
2781
|
-
|
|
2782
|
-
}
|
|
3036
|
+
});
|
|
2783
3037
|
// Wait for next screen to render
|
|
2784
3038
|
if (this.controlClient) {
|
|
2785
3039
|
const wid = readFileSync(join(this.instanceDir, "window-id"), "utf-8").trim();
|