@songsid/agend 2.1.4-beta.3 → 2.1.4-beta.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh-TW.md +1 -1
- package/dist/agent-endpoint.js +1 -1
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +9 -5
- package/dist/backend/antigravity.js +155 -30
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +33 -21
- package/dist/backend/claude-code.js +103 -27
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +17 -0
- package/dist/backend/codex.js +123 -8
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/gemini-cli.js +2 -2
- package/dist/backend/gemini-cli.js.map +1 -1
- package/dist/backend/grok.js +2 -2
- package/dist/backend/grok.js.map +1 -1
- package/dist/backend/kiro.d.ts +1 -0
- package/dist/backend/kiro.js +17 -3
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/opencode.d.ts +27 -11
- package/dist/backend/opencode.js +50 -39
- package/dist/backend/opencode.js.map +1 -1
- package/dist/backend/types.d.ts +20 -0
- package/dist/backend/types.js +12 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.js +106 -19
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.js +5 -3
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/agy-mcp-launcher.d.ts +2 -0
- package/dist/channel/agy-mcp-launcher.js +28 -0
- package/dist/channel/agy-mcp-launcher.js.map +1 -0
- package/dist/channel/mcp-server.js +6 -20
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/channel/mcp-tools.js +4 -2
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/types.d.ts +1 -1
- package/dist/classic-channel-manager.d.ts +40 -3
- package/dist/classic-channel-manager.js +218 -31
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +148 -33
- package/dist/cli.js.map +1 -1
- package/dist/completion-install.d.ts +70 -0
- package/dist/completion-install.js +152 -0
- package/dist/completion-install.js.map +1 -0
- package/dist/completion.d.ts +7 -1
- package/dist/completion.js +14 -4
- package/dist/completion.js.map +1 -1
- package/dist/config-validator.js +38 -7
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +2 -0
- package/dist/config.js +10 -1
- package/dist/config.js.map +1 -1
- package/dist/daemon.d.ts +177 -5
- package/dist/daemon.js +771 -99
- package/dist/daemon.js.map +1 -1
- package/dist/doctor.d.ts +26 -0
- package/dist/doctor.js +216 -0
- package/dist/doctor.js.map +1 -0
- package/dist/fleet-context.d.ts +47 -0
- package/dist/fleet-manager.d.ts +251 -10
- package/dist/fleet-manager.js +2059 -387
- package/dist/fleet-manager.js.map +1 -1
- package/dist/fleet-yaml-slim.d.ts +10 -0
- package/dist/fleet-yaml-slim.js +59 -0
- package/dist/fleet-yaml-slim.js.map +1 -0
- package/dist/general-knowledge/skills/backend-providers/SKILL.md +114 -0
- package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +3 -1
- package/dist/general-knowledge/skills/delegation-playbook/SKILL.md +52 -0
- package/dist/general-knowledge/skills/development-workflow/SKILL.md +28 -0
- package/dist/general-knowledge/skills/fleet-config/SKILL.md +1 -0
- package/dist/general-knowledge/skills/fleet-health/SKILL.md +1 -0
- package/dist/general-knowledge/skills/fleet-restart/SKILL.md +1 -0
- package/dist/general-knowledge/skills/instance-lifecycle/SKILL.md +1 -0
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +64 -16
- package/dist/general-knowledge/skills/multi-channel/SKILL.md +1 -0
- package/dist/general-knowledge/skills/scheduling/SKILL.md +1 -0
- package/dist/general-knowledge/skills/session-management/SKILL.md +1 -0
- package/dist/general-knowledge/skills/tui-effort/SKILL.md +1 -0
- package/dist/general-knowledge/skills/worker-collaboration/SKILL.md +30 -0
- package/dist/instance-lifecycle.d.ts +45 -0
- package/dist/instance-lifecycle.js +150 -5
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instructions.d.ts +12 -0
- package/dist/instructions.js +35 -1
- package/dist/instructions.js.map +1 -1
- package/dist/locale.js +750 -288
- package/dist/locale.js.map +1 -1
- package/dist/login-flows.d.ts +85 -0
- package/dist/login-flows.js +159 -0
- package/dist/login-flows.js.map +1 -0
- package/dist/login-manager.d.ts +63 -0
- package/dist/login-manager.js +134 -0
- package/dist/login-manager.js.map +1 -0
- package/dist/outbound-handlers.d.ts +8 -0
- package/dist/outbound-handlers.js +44 -5
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +5 -0
- package/dist/outbound-schemas.js +7 -0
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/quickstart.js +25 -0
- package/dist/quickstart.js.map +1 -1
- package/dist/restart-progress.d.ts +2 -0
- package/dist/restart-progress.js +3 -0
- package/dist/restart-progress.js.map +1 -1
- package/dist/scheduler/db.d.ts +12 -0
- package/dist/scheduler/db.js +59 -0
- package/dist/scheduler/db.js.map +1 -1
- package/dist/service-installer.d.ts +11 -0
- package/dist/service-installer.js +84 -18
- package/dist/service-installer.js.map +1 -1
- package/dist/settings-api.js +1 -1
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +2 -2
- package/dist/setup-wizard.js.map +1 -1
- package/dist/tips.d.ts +44 -0
- package/dist/tips.js +355 -0
- package/dist/tips.js.map +1 -0
- package/dist/tmux-manager.d.ts +6 -0
- package/dist/tmux-manager.js +12 -0
- package/dist/tmux-manager.js.map +1 -1
- package/dist/tool-progress.d.ts +40 -0
- package/dist/tool-progress.js +289 -0
- package/dist/tool-progress.js.map +1 -0
- package/dist/topic-commands.d.ts +36 -4
- package/dist/topic-commands.js +322 -64
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-monitor.d.ts +15 -2
- package/dist/transcript-monitor.js +63 -17
- package/dist/transcript-monitor.js.map +1 -1
- package/dist/transcript-sources.d.ts +131 -0
- package/dist/transcript-sources.js +580 -0
- package/dist/transcript-sources.js.map +1 -0
- package/dist/types.d.ts +11 -0
- package/dist/ui/dashboard.html +55 -32
- package/dist/ui/settings.html +120 -51
- package/dist/ui/view.html +23 -15
- package/dist/usage/providers.d.ts +7 -0
- package/dist/usage/providers.js +18 -1
- package/dist/usage/providers.js.map +1 -1
- package/dist/usage/usage-api.d.ts +8 -2
- package/dist/usage/usage-api.js +24 -3
- package/dist/usage/usage-api.js.map +1 -1
- package/dist/workflow-templates/default.md +2 -1
- package/package.json +1 -1
package/dist/daemon.js
CHANGED
|
@@ -8,6 +8,9 @@ import { mcpServerState } from "./mcp-liveness.js";
|
|
|
8
8
|
import { clearPausedMarker, writePausedMarker } from "./pause-marker.js";
|
|
9
9
|
import { TmuxManager, resolveTmuxLogicalSize } from "./tmux-manager.js";
|
|
10
10
|
import { TranscriptMonitor } from "./transcript-monitor.js";
|
|
11
|
+
import { createTranscriptSource } from "./transcript-sources.js";
|
|
12
|
+
import { LOGIN_FLOWS } from "./login-flows.js";
|
|
13
|
+
import { ProgressAccumulator, summarizeProgress } from "./tool-progress.js";
|
|
11
14
|
import { ContextGuardian } from "./context-guardian.js";
|
|
12
15
|
import { IpcServer } from "./channel/ipc-bridge.js";
|
|
13
16
|
import { daemonBudgetMs } from "./channel/ipc-timeouts.js";
|
|
@@ -45,6 +48,10 @@ export function buildInstructionReloadNotice(binaryName, instanceName, instanceD
|
|
|
45
48
|
export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
|
|
46
49
|
export const DEFAULT_STATE_IDLE_DEBOUNCE_MS = 2_000;
|
|
47
50
|
export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
|
|
51
|
+
/** Coalesce one cosmetic-redraw burst before comparing the visible pane. */
|
|
52
|
+
export const PERIODIC_REDRAW_PROBE_MS = 25;
|
|
53
|
+
/** A foreground server/port-forward should hand control back or be acknowledged. */
|
|
54
|
+
export const DEFAULT_BLOCKING_PROCESS_GRACE_MS = 2 * 60_000;
|
|
48
55
|
const LAST_INBOUND_FILE = "last-inbound-at";
|
|
49
56
|
/** Minimum gap between "health check is failing" notifications for one instance. */
|
|
50
57
|
const HEALTH_ERROR_NOTIFY_INTERVAL_MS = 10 * 60_000;
|
|
@@ -157,6 +164,11 @@ export class AutoPauseController {
|
|
|
157
164
|
recordActivity(now = Date.now()) {
|
|
158
165
|
this.lastActivityAt = now;
|
|
159
166
|
}
|
|
167
|
+
/** Apply a new idle threshold and start a fresh countdown from this update. */
|
|
168
|
+
reconfigure(thresholdMs, now = Date.now()) {
|
|
169
|
+
this.thresholdMs = Math.max(0, thresholdMs);
|
|
170
|
+
this.lastActivityAt = now;
|
|
171
|
+
}
|
|
160
172
|
observe(state, now = Date.now()) {
|
|
161
173
|
if (this.pausedAt !== null || this.thresholdMs <= 0)
|
|
162
174
|
return false;
|
|
@@ -279,9 +291,11 @@ export class PaneStateMachine {
|
|
|
279
291
|
* working" and a finished turn would not be seen as idle until the next 60s
|
|
280
292
|
* safety sweep. Output timestamps are the honest liveness signal here, and
|
|
281
293
|
* the caller has them.
|
|
294
|
+
* @param opts.forceBusy the backend has positively identified a still-running
|
|
295
|
+
* foreground tool even though its TUI keeps an old ready footer visible.
|
|
282
296
|
*/
|
|
283
297
|
observe(pane, now = Date.now(), opts = {}) {
|
|
284
|
-
const { settled = false, changeAt = now } = opts;
|
|
298
|
+
const { settled = false, changeAt = now, forceBusy = false } = opts;
|
|
285
299
|
const paneHash = createHash("sha256").update(pane).digest("hex");
|
|
286
300
|
const firstObservation = this.lastPaneHash === null;
|
|
287
301
|
const paneChanged = this.lastPaneHash !== paneHash;
|
|
@@ -290,7 +304,10 @@ export class PaneStateMachine {
|
|
|
290
304
|
this.lastPaneChangeAt = changeAt;
|
|
291
305
|
}
|
|
292
306
|
this.lastObservedAt = now;
|
|
293
|
-
|
|
307
|
+
// Some TUIs keep their ready footer visible while a foreground tool owns
|
|
308
|
+
// stdin. Backends can positively identify that tool from the pane; in that
|
|
309
|
+
// case the old prompt must not clear pending work or retire Cancel.
|
|
310
|
+
const ready = !forceBusy && this.isReady(pane);
|
|
294
311
|
const nextState = firstObservation
|
|
295
312
|
? ready ? "idle" : "working"
|
|
296
313
|
: paneChanged && !settled
|
|
@@ -591,6 +608,69 @@ export class InteractivePromptDetector {
|
|
|
591
608
|
this.notifiedSignature = null;
|
|
592
609
|
}
|
|
593
610
|
}
|
|
611
|
+
const BLOCKING_PROCESS_PATTERNS = [
|
|
612
|
+
// kubectl port-forward (including a backgrounded child whose inherited
|
|
613
|
+
// stdout keeps the parent shell tool open).
|
|
614
|
+
/^\s*(?:Forwarding from\s+(?:127\.0\.0\.1|localhost|\[::1\]):\d+\s+->\s+\d+|Handling connection for\s+\d+)\s*$/im,
|
|
615
|
+
// Common development servers. Keep these line-shaped and require an address
|
|
616
|
+
// or port so prose such as "the server is running" does not arm the detector.
|
|
617
|
+
/^\s*(?:(?:INFO:\s*)?Uvicorn running on|Serving HTTP on|Listening on|Server (?:is )?(?:listening|running) (?:at|on)|Application startup complete[^\n]*(?:port|https?:\/\/))[^\n]*(?:https?:\/\/|\b(?:localhost|127\.0\.0\.1|0\.0\.0\.0|\[::\]|port)\b)[^\n]*$/im,
|
|
618
|
+
];
|
|
619
|
+
/**
|
|
620
|
+
* Detect a foreground shell tool which has become a long-lived server.
|
|
621
|
+
*
|
|
622
|
+
* Unlike the generic pane-stuck clock, this clock deliberately survives new
|
|
623
|
+
* output: access logs and `Handling connection` lines are exactly why a
|
|
624
|
+
* foreground server can remain blocked forever without ever looking silent.
|
|
625
|
+
* Once a server marker is seen, the still-running tool identity is the stable
|
|
626
|
+
* signal; the marker may scroll out of the 32-row pane.
|
|
627
|
+
*/
|
|
628
|
+
export class BlockingProcessDetector {
|
|
629
|
+
graceMs;
|
|
630
|
+
activity = null;
|
|
631
|
+
evidence = null;
|
|
632
|
+
detectedAt = 0;
|
|
633
|
+
notified = false;
|
|
634
|
+
constructor(graceMs = DEFAULT_BLOCKING_PROCESS_GRACE_MS) {
|
|
635
|
+
this.graceMs = graceMs;
|
|
636
|
+
}
|
|
637
|
+
observe(pane, activity, now = Date.now()) {
|
|
638
|
+
const shellActivity = activity && /^(?:shell|bash|exec_command|terminal|command)(?::|$)/i.test(activity);
|
|
639
|
+
if (!shellActivity) {
|
|
640
|
+
this.reset();
|
|
641
|
+
return null;
|
|
642
|
+
}
|
|
643
|
+
if (activity !== this.activity) {
|
|
644
|
+
this.reset();
|
|
645
|
+
this.activity = activity;
|
|
646
|
+
}
|
|
647
|
+
if (!this.evidence) {
|
|
648
|
+
const tail = sanitizePaneTail(pane, 40).join("\n");
|
|
649
|
+
for (const pattern of BLOCKING_PROCESS_PATTERNS) {
|
|
650
|
+
const match = tail.match(pattern);
|
|
651
|
+
if (!match)
|
|
652
|
+
continue;
|
|
653
|
+
this.evidence = match[0].trim().slice(0, 200);
|
|
654
|
+
this.detectedAt = now;
|
|
655
|
+
break;
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
if (!this.evidence || this.notified || now - this.detectedAt < this.graceMs)
|
|
659
|
+
return null;
|
|
660
|
+
this.notified = true;
|
|
661
|
+
return {
|
|
662
|
+
activity,
|
|
663
|
+
evidence: this.evidence,
|
|
664
|
+
blockedForMs: now - this.detectedAt,
|
|
665
|
+
};
|
|
666
|
+
}
|
|
667
|
+
reset() {
|
|
668
|
+
this.activity = null;
|
|
669
|
+
this.evidence = null;
|
|
670
|
+
this.detectedAt = 0;
|
|
671
|
+
this.notified = false;
|
|
672
|
+
}
|
|
673
|
+
}
|
|
594
674
|
export class Daemon extends EventEmitter {
|
|
595
675
|
name;
|
|
596
676
|
config;
|
|
@@ -655,6 +735,7 @@ export class Daemon extends EventEmitter {
|
|
|
655
735
|
/** Fallback safety sweep used only when no shared tmux control client exists. */
|
|
656
736
|
instanceStateMonitorTimer = null;
|
|
657
737
|
instanceStateIdleTimer = null;
|
|
738
|
+
instanceStateOutputProbeTimer = null;
|
|
658
739
|
instanceStateStuckTimer = null;
|
|
659
740
|
instanceStateOutputListener = null;
|
|
660
741
|
instanceStateOutputEventName = null;
|
|
@@ -665,6 +746,7 @@ export class Daemon extends EventEmitter {
|
|
|
665
746
|
instanceStateReadyPattern = null;
|
|
666
747
|
instanceStateBusyPattern = null;
|
|
667
748
|
instanceStateMonitorActive = false;
|
|
749
|
+
sessionCheckpointWarningEmitted = false;
|
|
668
750
|
statePollInFlight = false;
|
|
669
751
|
// ── Dead-MCP proxy reply: per-turn evidence ─────────────────────────────
|
|
670
752
|
// Set when an inbound message lands in the CLI; cleared on the idle edge that
|
|
@@ -684,6 +766,24 @@ export class Daemon extends EventEmitter {
|
|
|
684
766
|
* would silently no-op exactly when we most need to stop feeding the CLI.
|
|
685
767
|
*/
|
|
686
768
|
pausePending = false;
|
|
769
|
+
/**
|
|
770
|
+
* An auth failure was detected and nothing has resolved it yet.
|
|
771
|
+
*
|
|
772
|
+
* Separate from `pausePending` because it must outlive it: the deferred pause
|
|
773
|
+
* clears `pausePending` as soon as it fires, but if that pause is refused the
|
|
774
|
+
* instance is still auth-broken and must not resume raising alarms.
|
|
775
|
+
*/
|
|
776
|
+
authFailureUnresolved = false;
|
|
777
|
+
/** One login-screen auth report per spawn — the screen persists across polls. */
|
|
778
|
+
loginScreenReported = false;
|
|
779
|
+
/** model_error seen mid-turn: notify only if it survives to the idle screen. */
|
|
780
|
+
pendingModelErrorKey = null;
|
|
781
|
+
/**
|
|
782
|
+
* Let the next pause() proceed from a stuck pane. Set only for the auth-deferred
|
|
783
|
+
* pause — see the pausePending consumption site for why waiting for idle there
|
|
784
|
+
* is waiting forever.
|
|
785
|
+
*/
|
|
786
|
+
pauseAllowStuck = false;
|
|
687
787
|
pauseWakeState = "active";
|
|
688
788
|
pauseWakeTransition = null;
|
|
689
789
|
// Model failover: override model on next spawn when rate-limited
|
|
@@ -697,6 +797,12 @@ export class Daemon extends EventEmitter {
|
|
|
697
797
|
* and ⏳/👀/✅ reactions live here). It does NOT cover the other writers that
|
|
698
798
|
* reach the pane — {@link paneWriteLock} does. */
|
|
699
799
|
pasteLock = Promise.resolve();
|
|
800
|
+
/**
|
|
801
|
+
* Fleet cancellation epoch. Queue entries retain the epoch they arrived with;
|
|
802
|
+
* advancing it invalidates only work received before the cancel, never a new
|
|
803
|
+
* user message which arrives immediately afterwards.
|
|
804
|
+
*/
|
|
805
|
+
deliveryEpoch = 0;
|
|
700
806
|
/** Mutual exclusion for *every* write into the pane, whichever subsystem it
|
|
701
807
|
* comes from. See PaneWriteLock for why interleaving is destructive. */
|
|
702
808
|
paneWriteLock = new PaneWriteLock();
|
|
@@ -710,9 +816,21 @@ export class Daemon extends EventEmitter {
|
|
|
710
816
|
lastBuiltInstructions = "";
|
|
711
817
|
pasteQueueDepth = 0;
|
|
712
818
|
firstDeliveryDelay = new FirstDeliveryDelay();
|
|
819
|
+
/** Orders /steer pastes against each other. Deliberately NOT pasteLock:
|
|
820
|
+
* a steer must not queue behind the normal deliveries it exists to
|
|
821
|
+
* overtake. Pane-level exclusion still comes from paneWriteLock inside
|
|
822
|
+
* deliverMessage, so a steer and a normal delivery can never interleave
|
|
823
|
+
* their PTY writes — the steer just doesn't wait for idle. */
|
|
824
|
+
steerLock = Promise.resolve();
|
|
825
|
+
/** Tool lines shown in the channel processing bubble for the current turn. */
|
|
826
|
+
turnProgress = new ProgressAccumulator();
|
|
827
|
+
lastProgressBroadcast = "";
|
|
828
|
+
progressBroadcastTimer = null;
|
|
829
|
+
lastProgressBroadcastAt = 0;
|
|
713
830
|
// PTY error pattern monitoring
|
|
714
831
|
errorMonitorTimer = null;
|
|
715
832
|
interactivePromptDetector = new InteractivePromptDetector();
|
|
833
|
+
blockingProcessDetector = new BlockingProcessDetector();
|
|
716
834
|
/** Same 5-min gate the error monitor uses, so a dead MCP server alerts once. */
|
|
717
835
|
static MCP_DEATH_COOLDOWN_MS = 5 * 60_000;
|
|
718
836
|
lastMcpDeathNotifiedAt = 0;
|
|
@@ -910,43 +1028,38 @@ export class Daemon extends EventEmitter {
|
|
|
910
1028
|
// Fleet manager routed a message to us (topic mode)
|
|
911
1029
|
const meta = msg.meta;
|
|
912
1030
|
const targetSession = msg.targetSession;
|
|
1031
|
+
const deliveryEpoch = this.captureDeliveryEpoch(msg.delivery_epoch);
|
|
913
1032
|
void this.wake().then(() => {
|
|
914
|
-
this.pushChannelMessage(msg.content, meta, targetSession);
|
|
1033
|
+
this.pushChannelMessage(msg.content, meta, targetSession, deliveryEpoch);
|
|
915
1034
|
}).catch(err => {
|
|
916
1035
|
this.logger.error({ err: err.message }, "Wake failed for inbound delivery");
|
|
917
1036
|
});
|
|
918
1037
|
}
|
|
919
1038
|
else if (msg.type === "raw_paste") {
|
|
920
1039
|
// Paste raw text directly to CLI without [user:] wrapping.
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
1040
|
+
this.queueRawPaste(msg.content, this.captureDeliveryEpoch(msg.delivery_epoch));
|
|
1041
|
+
}
|
|
1042
|
+
else if (msg.type === "config_update") {
|
|
1043
|
+
this.applyConfigUpdate(msg.config);
|
|
1044
|
+
}
|
|
1045
|
+
else if (msg.type === "steer") {
|
|
1046
|
+
const meta = (msg.meta ?? {});
|
|
1047
|
+
this.steerMessage(msg.content, meta, this.captureDeliveryEpoch(msg.delivery_epoch));
|
|
1048
|
+
}
|
|
1049
|
+
else if (msg.type === "btw") {
|
|
1050
|
+
const meta = (msg.meta ?? {});
|
|
1051
|
+
this.btwMessage(msg.content, meta, this.captureDeliveryEpoch(msg.delivery_epoch));
|
|
931
1052
|
}
|
|
932
1053
|
else if (msg.type === "fleet_schedule_trigger") {
|
|
933
1054
|
const payload = msg.payload;
|
|
934
1055
|
const meta = msg.meta;
|
|
935
|
-
|
|
1056
|
+
const deliveryEpoch = this.captureDeliveryEpoch(msg.delivery_epoch);
|
|
1057
|
+
void this.wake().then(() => this.pushChannelMessage(payload.message, meta, undefined, deliveryEpoch)).catch(err => {
|
|
936
1058
|
this.logger.error({ err: err.message }, "Wake failed for scheduled delivery");
|
|
937
1059
|
});
|
|
938
1060
|
}
|
|
939
1061
|
else if (msg.type === "query_instance_state") {
|
|
940
|
-
|
|
941
|
-
this.ipcServer?.send(socket, {
|
|
942
|
-
type: "instance_state_response",
|
|
943
|
-
requestId: msg.requestId,
|
|
944
|
-
instanceName: this.name,
|
|
945
|
-
...snapshot,
|
|
946
|
-
state: this.isPaused ? "paused" : snapshot.state,
|
|
947
|
-
processStatus: this.processStatus,
|
|
948
|
-
pausedAt: this.lastPausedAt,
|
|
949
|
-
});
|
|
1062
|
+
void this.respondToInstanceStateQuery(msg, socket);
|
|
950
1063
|
}
|
|
951
1064
|
});
|
|
952
1065
|
// 2. Tmux — ensure session, create window if not alive
|
|
@@ -1032,8 +1145,11 @@ export class Daemon extends EventEmitter {
|
|
|
1032
1145
|
const outputLog = join(this.instanceDir, "output.log");
|
|
1033
1146
|
rotateLogIfNeeded(outputLog);
|
|
1034
1147
|
await this.tmux.pipeOutput(outputLog).catch(() => { });
|
|
1035
|
-
// 4. Transcript monitor
|
|
1036
|
-
|
|
1148
|
+
// 4. Transcript monitor. claude-code is handled inside the monitor
|
|
1149
|
+
// (statusline transcript); codex/kiro/opencode read their CLI's own
|
|
1150
|
+
// conversation store via a pluggable source. Backends with no known
|
|
1151
|
+
// source stay inert exactly as before.
|
|
1152
|
+
this.transcriptMonitor = new TranscriptMonitor(this.instanceDir, this.logger, createTranscriptSource(this.config.backend ?? "claude-code", this.config.working_directory));
|
|
1037
1153
|
// 5. Wire transcript events
|
|
1038
1154
|
const ackIfPending = () => {
|
|
1039
1155
|
if (!this.pendingAckMessage || !this.adapter)
|
|
@@ -1049,6 +1165,7 @@ export class Daemon extends EventEmitter {
|
|
|
1049
1165
|
this.recordRecentEvent({ type: "tool_use", name, preview: this.summarizeTool(name, input) });
|
|
1050
1166
|
this.recordRecentToolActivity(this.summarizeTool(name, input));
|
|
1051
1167
|
this.publishActivity(this.summarizeTool(name, input));
|
|
1168
|
+
this.recordToolProgress(name, input);
|
|
1052
1169
|
});
|
|
1053
1170
|
this.transcriptMonitor.on("tool_result", (name, _output) => {
|
|
1054
1171
|
this.recordRecentEvent({ type: "tool_result", name });
|
|
@@ -1129,9 +1246,13 @@ export class Daemon extends EventEmitter {
|
|
|
1129
1246
|
return;
|
|
1130
1247
|
this.mcpDeathNotifiedForPid = status.pid;
|
|
1131
1248
|
this.lastMcpDeathNotifiedAt = Date.now();
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1249
|
+
// An auth-broken CLI cannot keep an MCP server alive, and restarting it
|
|
1250
|
+
// only loops back to the sign-in screen — hold the auto-restart until a
|
|
1251
|
+
// re-login (which restarts the instance and replaces this daemon anyway).
|
|
1252
|
+
const authSuspected = this.authFailureUnresolved;
|
|
1253
|
+
const autoRestart = this.config.mcp_auto_restart !== false && !authSuspected;
|
|
1254
|
+
this.logger.error({ pid: status.pid, autoRestart, authSuspected }, "MCP server process is gone — instance has no agend tools");
|
|
1255
|
+
this.emit("mcp_died", { name: this.name, pid: status.pid, autoRestart, authSuspected });
|
|
1135
1256
|
if (autoRestart)
|
|
1136
1257
|
this.armMcpRestartWhenIdle();
|
|
1137
1258
|
}
|
|
@@ -1357,6 +1478,17 @@ export class Daemon extends EventEmitter {
|
|
|
1357
1478
|
catch (err) {
|
|
1358
1479
|
this.logger.error({ err: err.message }, "Recovery from background session conflict failed");
|
|
1359
1480
|
}
|
|
1481
|
+
// Keep the loop alive: every health-check exit must either
|
|
1482
|
+
// scheduleNext() or deliberately set healthCheckPaused. This one
|
|
1483
|
+
// did neither, so a recovered instance ran UNMONITORED (while
|
|
1484
|
+
// isHealthCheckEffectivelyPaused still said monitoring was on)
|
|
1485
|
+
// until the next pause→wake cycle or fleet restart. The next tick
|
|
1486
|
+
// does the right thing in both outcomes: a healthy new window
|
|
1487
|
+
// passes the alive check, and a failed recovery hits the dead
|
|
1488
|
+
// window again with backgroundSessionRecoveryAttempted set — so
|
|
1489
|
+
// it falls through to the normal crash handling (crash counting,
|
|
1490
|
+
// backoff, supervision_ended when retries are exhausted).
|
|
1491
|
+
scheduleNext();
|
|
1360
1492
|
return; // Don't count as crash
|
|
1361
1493
|
}
|
|
1362
1494
|
// Already attempted recovery — fall through to normal crash handling
|
|
@@ -1573,6 +1705,25 @@ export class Daemon extends EventEmitter {
|
|
|
1573
1705
|
// A prompt is not an error and must not enter the PTY recovery gate.
|
|
1574
1706
|
// Continue scanning real errors in this same snapshot.
|
|
1575
1707
|
}
|
|
1708
|
+
// If a backend can derive activity from this exact pane, trust its
|
|
1709
|
+
// explicit null (tool completed) instead of falling back to a possibly
|
|
1710
|
+
// stale transcript activity from the previous scan.
|
|
1711
|
+
const paneActivity = this.backend?.getPaneActivity
|
|
1712
|
+
? this.backend.getPaneActivity(pane)
|
|
1713
|
+
: this.currentActivity;
|
|
1714
|
+
const hasPendingWork = this.pendingWork.hasPendingWork();
|
|
1715
|
+
if (!hasPendingWork)
|
|
1716
|
+
this.blockingProcessDetector.reset();
|
|
1717
|
+
const blockingProcess = hasPendingWork
|
|
1718
|
+
? this.blockingProcessDetector.observe(pane, paneActivity, Date.now())
|
|
1719
|
+
: null;
|
|
1720
|
+
if (blockingProcess) {
|
|
1721
|
+
// Reuse the hang-notification bridge: it offers explicit Restart/Wait
|
|
1722
|
+
// choices and, unlike pty_error, does not retire a still-useful Cancel
|
|
1723
|
+
// button merely because a foreground process owns stdin.
|
|
1724
|
+
this.logger.warn(blockingProcess, "Foreground process is blocking the agent input loop");
|
|
1725
|
+
this.hangDetector?.emit("hang", { unchangedForMs: blockingProcess.blockedForMs });
|
|
1726
|
+
}
|
|
1576
1727
|
// Auto-dismiss runtime dialogs (e.g. Codex rate limit model switch)
|
|
1577
1728
|
for (const dialog of dialogs) {
|
|
1578
1729
|
if (!dialog.pattern.test(pane))
|
|
@@ -1645,6 +1796,7 @@ export class Daemon extends EventEmitter {
|
|
|
1645
1796
|
if (seen > 0)
|
|
1646
1797
|
this.lastErrorCount.set(Daemon.errorPatternKey(ep), seen);
|
|
1647
1798
|
}
|
|
1799
|
+
this.authFailureUnresolved = false;
|
|
1648
1800
|
this.clearErrorRecoveryGate();
|
|
1649
1801
|
this.logger.info({ downtime_s: downtime }, "PTY error recovered — agent is ready again");
|
|
1650
1802
|
this.emit("pty_recovered", { name: this.name, downtime_s: downtime });
|
|
@@ -1665,6 +1817,19 @@ export class Daemon extends EventEmitter {
|
|
|
1665
1817
|
this.logger.warn({ errorType: this.lastDetectedErrorType }, "PTY error recovery deadline expired — resuming error scan");
|
|
1666
1818
|
this.clearErrorRecoveryGate();
|
|
1667
1819
|
}
|
|
1820
|
+
// A model_error deferred mid-turn: the turn is over (or the pane left
|
|
1821
|
+
// "working"), so decide now — still on screen means the CLI gave up and
|
|
1822
|
+
// the user must act; gone means a retry got past it and nobody needs to know.
|
|
1823
|
+
if (this.pendingModelErrorKey && this.instanceState !== "working") {
|
|
1824
|
+
const key = this.pendingModelErrorKey;
|
|
1825
|
+
this.pendingModelErrorKey = null;
|
|
1826
|
+
const ep = patterns.find(candidate => Daemon.errorPatternKey(candidate) === key);
|
|
1827
|
+
if (ep && ep.pattern.test(pane)) {
|
|
1828
|
+
this.emitErrorPattern(ep, key, pane, now);
|
|
1829
|
+
return;
|
|
1830
|
+
}
|
|
1831
|
+
this.logger.debug({ errorKey: key }, "Deferred model_error cleared before idle — dropped");
|
|
1832
|
+
}
|
|
1668
1833
|
// State: monitoring — count-based new-error detection over the full pane.
|
|
1669
1834
|
for (const ep of patterns) {
|
|
1670
1835
|
const key = Daemon.errorPatternKey(ep);
|
|
@@ -1692,25 +1857,49 @@ export class Daemon extends EventEmitter {
|
|
|
1692
1857
|
continue;
|
|
1693
1858
|
}
|
|
1694
1859
|
this.lastErrorCount.set(key, count);
|
|
1695
|
-
//
|
|
1696
|
-
//
|
|
1697
|
-
//
|
|
1698
|
-
if (
|
|
1699
|
-
this.
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
this.
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
this.
|
|
1860
|
+
// Kiro's `auto` model picks a concrete model per turn and reroutes around
|
|
1861
|
+
// a temporarily unavailable one on its own — telling the user to run
|
|
1862
|
+
// /model would ask them to fix something the CLI is already handling.
|
|
1863
|
+
if (ep.type === "model_error" && ep.action === "notify" && this.config.model === "auto") {
|
|
1864
|
+
this.logger.debug({ errorType: ep.type }, "model_error suppressed — model 'auto' reroutes on its own");
|
|
1865
|
+
continue;
|
|
1866
|
+
}
|
|
1867
|
+
// A mid-turn model error is often a transient the CLI retries past. Hold
|
|
1868
|
+
// the notification until the turn ends and notify only if the error is
|
|
1869
|
+
// still on the idle screen (see the deferred check above).
|
|
1870
|
+
if (ep.type === "model_error" && ep.action === "notify" && this.instanceState === "working") {
|
|
1871
|
+
this.pendingModelErrorKey = key;
|
|
1872
|
+
this.logger.debug({ errorType: ep.type }, "model_error deferred — instance is mid-turn, re-checking at idle");
|
|
1873
|
+
continue;
|
|
1874
|
+
}
|
|
1875
|
+
this.emitErrorPattern(ep, key, pane, now);
|
|
1711
1876
|
break; // Only handle first unsuppressed new error per scan
|
|
1712
1877
|
}
|
|
1713
1878
|
}
|
|
1879
|
+
/** Arm the recovery gates and emit one detected error pattern. */
|
|
1880
|
+
emitErrorPattern(ep, key, pane, now) {
|
|
1881
|
+
// skipRecoveryWait: this error self-recovers (e.g. a timeout — Kiro is
|
|
1882
|
+
// back at its prompt immediately). Keep monitoring so the next occurrence
|
|
1883
|
+
// can fire without waiting for a possibly startup-only ready pattern.
|
|
1884
|
+
if (!ep.skipRecoveryWait) {
|
|
1885
|
+
this.errorWaitingForRecovery = true;
|
|
1886
|
+
this.errorDetectedAt = now;
|
|
1887
|
+
this.errorRecoveryDeadlineAt = now + Daemon.ERROR_RECOVERY_TIMEOUT_MS;
|
|
1888
|
+
this.activeErrorPatternKey = key;
|
|
1889
|
+
this.lastDetectedErrorType = ep.type;
|
|
1890
|
+
}
|
|
1891
|
+
this.lastErrorNotifiedAt.set(key, now);
|
|
1892
|
+
if (ep.action === "failover")
|
|
1893
|
+
this.lastFailoverAt = now;
|
|
1894
|
+
const message = this.resolveErrorMessage(pane, ep);
|
|
1895
|
+
// Remember an auth failure for as long as it is unresolved. An auth-broken
|
|
1896
|
+
// CLI cannot finish the turn it is holding, so every other detector in this
|
|
1897
|
+
// scan would otherwise keep reporting the same dead instance forever.
|
|
1898
|
+
if (ep.type === "auth_error")
|
|
1899
|
+
this.authFailureUnresolved = true;
|
|
1900
|
+
this.logger.warn({ errorType: ep.type, action: ep.action }, `PTY error detected: ${message}`);
|
|
1901
|
+
this.emit("pty_error", { name: this.name, ...ep, message });
|
|
1902
|
+
}
|
|
1714
1903
|
/**
|
|
1715
1904
|
* Notification text for a detected pattern. `formatMessage` patterns build it
|
|
1716
1905
|
* from the match so the user gets the specifics (e.g. which period and what
|
|
@@ -1749,6 +1938,46 @@ export class Daemon extends EventEmitter {
|
|
|
1749
1938
|
const cancelKey = this.backend?.getCancelKey() ?? "Escape";
|
|
1750
1939
|
await this.tmux?.sendSpecialKey(cancelKey);
|
|
1751
1940
|
}
|
|
1941
|
+
/**
|
|
1942
|
+
* Drop every not-yet-started pane delivery which predates a user cancel.
|
|
1943
|
+
* Promise chains cannot be removed, so entries become generation-checked
|
|
1944
|
+
* no-ops. An entry already past its check is allowed to finish its current PTY
|
|
1945
|
+
* transaction; the interrupt key still cancels the CLI generation.
|
|
1946
|
+
*/
|
|
1947
|
+
clearPendingDeliveries(fleetEpoch) {
|
|
1948
|
+
this.deliveryEpoch = fleetEpoch === undefined
|
|
1949
|
+
? this.deliveryEpoch + 1
|
|
1950
|
+
: Math.max(this.deliveryEpoch, fleetEpoch);
|
|
1951
|
+
this.logger.info({ deliveryEpoch: this.deliveryEpoch }, "Pending delivery queue cleared by user cancel");
|
|
1952
|
+
}
|
|
1953
|
+
captureDeliveryEpoch(value) {
|
|
1954
|
+
const epoch = typeof value === "number" && Number.isSafeInteger(value) && value >= 0
|
|
1955
|
+
? value
|
|
1956
|
+
: this.deliveryEpoch;
|
|
1957
|
+
if (epoch > this.deliveryEpoch)
|
|
1958
|
+
this.deliveryEpoch = epoch;
|
|
1959
|
+
return epoch;
|
|
1960
|
+
}
|
|
1961
|
+
isDeliveryEpochCurrent(epoch) {
|
|
1962
|
+
return epoch === this.deliveryEpoch;
|
|
1963
|
+
}
|
|
1964
|
+
queueRawPaste(rawText, deliveryEpoch = this.deliveryEpoch) {
|
|
1965
|
+
if (!this.tmux || !this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
1966
|
+
return;
|
|
1967
|
+
this.pasteLock = this.pasteLock.then(async () => {
|
|
1968
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
1969
|
+
this.logger.info("Pending raw delivery dropped by user cancel");
|
|
1970
|
+
return;
|
|
1971
|
+
}
|
|
1972
|
+
await this.wake();
|
|
1973
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
1974
|
+
return;
|
|
1975
|
+
await this.deliverMessage(rawText, undefined, { deliveryEpoch });
|
|
1976
|
+
this.logger.debug({ text: rawText.slice(0, 100) }, "Raw paste delivered");
|
|
1977
|
+
}).catch(err => {
|
|
1978
|
+
this.logger.warn({ err: err.message }, "raw_paste delivery error");
|
|
1979
|
+
});
|
|
1980
|
+
}
|
|
1752
1981
|
/** Send the backend-specific graceful quit command/key sequence. */
|
|
1753
1982
|
async sendQuitSequence() {
|
|
1754
1983
|
if (!this.tmux || !this.backend)
|
|
@@ -1850,6 +2079,33 @@ export class Daemon extends EventEmitter {
|
|
|
1850
2079
|
stateChangedAt: Date.now(),
|
|
1851
2080
|
};
|
|
1852
2081
|
}
|
|
2082
|
+
/**
|
|
2083
|
+
* Answer a FleetManager state query. Most callers only need the cached state,
|
|
2084
|
+
* but lifecycle decisions such as post-reply Cancel retirement need an
|
|
2085
|
+
* authoritative pane observation: after startup the last transition can stay
|
|
2086
|
+
* "working" even though the CLI has since settled without emitting another
|
|
2087
|
+
* control-mode output record.
|
|
2088
|
+
*/
|
|
2089
|
+
async respondToInstanceStateQuery(msg, socket) {
|
|
2090
|
+
try {
|
|
2091
|
+
if (msg.refresh === true) {
|
|
2092
|
+
await this.captureAndEvaluateInstanceState("state_query");
|
|
2093
|
+
}
|
|
2094
|
+
const snapshot = this.getInstanceStateSnapshot();
|
|
2095
|
+
this.ipcServer?.send(socket, {
|
|
2096
|
+
type: "instance_state_response",
|
|
2097
|
+
requestId: msg.requestId,
|
|
2098
|
+
instanceName: this.name,
|
|
2099
|
+
...snapshot,
|
|
2100
|
+
state: this.isPaused ? "paused" : snapshot.state,
|
|
2101
|
+
processStatus: this.processStatus,
|
|
2102
|
+
pausedAt: this.lastPausedAt,
|
|
2103
|
+
});
|
|
2104
|
+
}
|
|
2105
|
+
catch (err) {
|
|
2106
|
+
this.logger.debug({ err: err.message }, "Instance state query failed");
|
|
2107
|
+
}
|
|
2108
|
+
}
|
|
1853
2109
|
/** Gracefully stop the CLI while keeping its remain-on-exit tmux window. */
|
|
1854
2110
|
/**
|
|
1855
2111
|
* Mark the instance to pause as soon as its pane is idle. Used for auth
|
|
@@ -1869,7 +2125,14 @@ export class Daemon extends EventEmitter {
|
|
|
1869
2125
|
if (this.getPauseWakeState() !== "active")
|
|
1870
2126
|
return;
|
|
1871
2127
|
}
|
|
1872
|
-
|
|
2128
|
+
// `pauseAllowStuck` is consumed here, not merely read: it authorises exactly
|
|
2129
|
+
// one pause from a stuck pane (the auth-deferred one) and must never leak
|
|
2130
|
+
// into an ordinary idle-timeout pause of a busy instance.
|
|
2131
|
+
const allowStuck = this.pauseAllowStuck;
|
|
2132
|
+
this.pauseAllowStuck = false;
|
|
2133
|
+
const pausableState = this.instanceState === "idle"
|
|
2134
|
+
|| (allowStuck && this.instanceState === "stuck");
|
|
2135
|
+
if (!pausableState || this.pasteQueueDepth > 0) {
|
|
1873
2136
|
this.pauseRequested = false;
|
|
1874
2137
|
return;
|
|
1875
2138
|
}
|
|
@@ -1933,6 +2196,8 @@ export class Daemon extends EventEmitter {
|
|
|
1933
2196
|
// An explicit wake (e.g. after the user re-logs in) cancels a deferred
|
|
1934
2197
|
// auth pause — otherwise the instance would pause again the moment it idles.
|
|
1935
2198
|
this.pausePending = false;
|
|
2199
|
+
this.authFailureUnresolved = false;
|
|
2200
|
+
this.loginScreenReported = false;
|
|
1936
2201
|
if (this.pauseWakeState === "active")
|
|
1937
2202
|
return;
|
|
1938
2203
|
if (this.pauseWakeState === "pausing")
|
|
@@ -1993,6 +2258,14 @@ export class Daemon extends EventEmitter {
|
|
|
1993
2258
|
applyInstanceStateSnapshot(snapshot, pane) {
|
|
1994
2259
|
const previous = this.instanceState;
|
|
1995
2260
|
this.instanceState = snapshot.state;
|
|
2261
|
+
// OpenCode creates its session lazily on the first submitted message.
|
|
2262
|
+
// Waiting until stop/pause to persist that id loses resume state when the
|
|
2263
|
+
// fleet is SIGKILLed or the host reboots.
|
|
2264
|
+
// Idle observations are safe checkpoints; the 60s safety sweep also gives
|
|
2265
|
+
// us a bounded retry if session creation produced no visible state edge.
|
|
2266
|
+
if (snapshot.state === "idle" && this.backend?.binaryName === "opencode") {
|
|
2267
|
+
this.saveSessionId();
|
|
2268
|
+
}
|
|
1996
2269
|
// Only a transition back to idle completes pending work. Repeated idle
|
|
1997
2270
|
// observations between enqueue and paste must not clear a newer inbound.
|
|
1998
2271
|
if (snapshot.state === "idle" && previous !== "idle") {
|
|
@@ -2001,6 +2274,7 @@ export class Daemon extends EventEmitter {
|
|
|
2001
2274
|
// tool_result (interrupted, crashed, cancelled), which would otherwise leave
|
|
2002
2275
|
// the last tool pinned to the progress line for the rest of the session.
|
|
2003
2276
|
this.publishActivity(null);
|
|
2277
|
+
this.resetToolProgress();
|
|
2004
2278
|
// Must run before the mcpRestartPending branch below: the pane text is the
|
|
2005
2279
|
// only copy of the answer, and the revival restart is about to clear it.
|
|
2006
2280
|
this.maybeProxyReplyOnTurnEnd(pane);
|
|
@@ -2020,8 +2294,13 @@ export class Daemon extends EventEmitter {
|
|
|
2020
2294
|
if (snapshot.state !== "idle")
|
|
2021
2295
|
this.pauseRequested = false;
|
|
2022
2296
|
// A pause deferred by an auth failure: retry the moment the pane settles.
|
|
2023
|
-
|
|
2297
|
+
// "stuck" counts as settled here, and that is the whole point — an expired
|
|
2298
|
+
// session cannot complete the turn it is holding, so the pane never returns
|
|
2299
|
+
// to idle and waiting for idle meant waiting forever. The instance stayed
|
|
2300
|
+
// live, kept its warm slot, and re-raised a hang alert on every scan.
|
|
2301
|
+
if (this.pausePending && (snapshot.state === "idle" || snapshot.state === "stuck") && this.pasteQueueDepth === 0) {
|
|
2024
2302
|
this.pausePending = false;
|
|
2303
|
+
this.pauseAllowStuck = snapshot.state === "stuck";
|
|
2025
2304
|
this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
|
|
2026
2305
|
return;
|
|
2027
2306
|
}
|
|
@@ -2148,6 +2427,25 @@ export class Daemon extends EventEmitter {
|
|
|
2148
2427
|
clearTimeout(this.instanceStateIdleTimer);
|
|
2149
2428
|
this.instanceStateIdleTimer = null;
|
|
2150
2429
|
}
|
|
2430
|
+
clearInstanceStateOutputProbeTimer() {
|
|
2431
|
+
if (this.instanceStateOutputProbeTimer)
|
|
2432
|
+
clearTimeout(this.instanceStateOutputProbeTimer);
|
|
2433
|
+
this.instanceStateOutputProbeTimer = null;
|
|
2434
|
+
}
|
|
2435
|
+
/**
|
|
2436
|
+
* Periodic status-line hooks can generate `%output` without changing a single
|
|
2437
|
+
* visible cell. Probe the pane once per burst: a changed capture is real work;
|
|
2438
|
+
* an identical ready capture remains idle and emits no state edge.
|
|
2439
|
+
*/
|
|
2440
|
+
scheduleInstanceStateOutputProbe() {
|
|
2441
|
+
if (this.instanceStateOutputProbeTimer)
|
|
2442
|
+
return;
|
|
2443
|
+
this.instanceStateOutputProbeTimer = setTimeout(() => {
|
|
2444
|
+
this.instanceStateOutputProbeTimer = null;
|
|
2445
|
+
void this.captureAndEvaluateInstanceState("output_probe", this.instanceStateLastOutputAt);
|
|
2446
|
+
}, PERIODIC_REDRAW_PROBE_MS);
|
|
2447
|
+
this.instanceStateOutputProbeTimer.unref?.();
|
|
2448
|
+
}
|
|
2151
2449
|
clearInstanceStateStuckTimer() {
|
|
2152
2450
|
if (this.instanceStateStuckTimer)
|
|
2153
2451
|
clearTimeout(this.instanceStateStuckTimer);
|
|
@@ -2192,6 +2490,8 @@ export class Daemon extends EventEmitter {
|
|
|
2192
2490
|
if (this.statePollInFlight) {
|
|
2193
2491
|
if (reason === "idle_debounce")
|
|
2194
2492
|
this.scheduleInstanceStateIdleCapture();
|
|
2493
|
+
if (reason === "output_probe")
|
|
2494
|
+
this.scheduleInstanceStateOutputProbe();
|
|
2195
2495
|
return;
|
|
2196
2496
|
}
|
|
2197
2497
|
this.statePollInFlight = true;
|
|
@@ -2200,9 +2500,19 @@ export class Daemon extends EventEmitter {
|
|
|
2200
2500
|
const pane = await this.tmux.capturePane();
|
|
2201
2501
|
// Output received while capture-pane was in flight makes this snapshot
|
|
2202
2502
|
// stale. Its output handler has already armed a new debounce.
|
|
2203
|
-
|
|
2204
|
-
|| (this.instanceStateLastOutputAt > 0 && this.instanceStateLastOutputAt >= captureStartedAt)
|
|
2503
|
+
const outputMovedDuringCapture = (expectedOutputAt > 0 && this.instanceStateLastOutputAt > expectedOutputAt)
|
|
2504
|
+
|| (this.instanceStateLastOutputAt > 0 && this.instanceStateLastOutputAt >= captureStartedAt);
|
|
2505
|
+
if (outputMovedDuringCapture) {
|
|
2506
|
+
if (reason === "output_probe") {
|
|
2507
|
+
// A second burst overtook the probe. Do not bless its stale capture as
|
|
2508
|
+
// idle: continuous output is the safer working signal, while the next
|
|
2509
|
+
// quiet probe/debounce can still prove that it was only repainting.
|
|
2510
|
+
this.applyInstanceStateSnapshot(this.instanceStateMachine.recordOutput(this.instanceStateLastOutputAt));
|
|
2511
|
+
this.scheduleInstanceStateIdleCapture();
|
|
2512
|
+
this.scheduleInstanceStateStuckDeadline(this.instanceStateLastOutputAt);
|
|
2513
|
+
}
|
|
2205
2514
|
return;
|
|
2515
|
+
}
|
|
2206
2516
|
const observedChangeAt = expectedOutputAt || captureStartedAt;
|
|
2207
2517
|
// One observation per capture. This used to call observe() twice with the
|
|
2208
2518
|
// same pane: the second call always saw an unchanged pane, so the
|
|
@@ -2211,6 +2521,7 @@ export class Daemon extends EventEmitter {
|
|
|
2211
2521
|
// mismatched spinner frame was enough to report idle mid-turn.
|
|
2212
2522
|
const settled = this.instanceStateLastOutputAt === 0
|
|
2213
2523
|
|| captureStartedAt - this.instanceStateLastOutputAt >= this.instanceStateIdleDebounceMs;
|
|
2524
|
+
const paneActivity = this.backend?.getPaneActivity?.(pane) ?? null;
|
|
2214
2525
|
// The two times are genuinely different and both matter: the content
|
|
2215
2526
|
// changed when tmux reported output (observedChangeAt), but idle/stuck are
|
|
2216
2527
|
// decisions about *now*. The old double-observe expressed that by calling
|
|
@@ -2218,6 +2529,7 @@ export class Daemon extends EventEmitter {
|
|
|
2218
2529
|
const snapshot = this.instanceStateMachine.observe(pane, Date.now(), {
|
|
2219
2530
|
settled,
|
|
2220
2531
|
changeAt: observedChangeAt,
|
|
2532
|
+
forceBusy: paneActivity !== null,
|
|
2221
2533
|
});
|
|
2222
2534
|
this.applyInstanceStateSnapshot(snapshot, pane);
|
|
2223
2535
|
// Backends without a transcript feed can still say what they are doing, if
|
|
@@ -2225,7 +2537,7 @@ export class Daemon extends EventEmitter {
|
|
|
2225
2537
|
// extra tmux call. Cadence is the capture cadence (idle debounce + the 60s
|
|
2226
2538
|
// safety sweep), which matches the progress ticker's own 60s interval.
|
|
2227
2539
|
if (this.backend?.getPaneActivity) {
|
|
2228
|
-
this.publishActivity(snapshot.state === "idle" ? null :
|
|
2540
|
+
this.publishActivity(snapshot.state === "idle" ? null : paneActivity);
|
|
2229
2541
|
}
|
|
2230
2542
|
if (snapshot.state === "idle") {
|
|
2231
2543
|
this.clearInstanceStateStuckTimer();
|
|
@@ -2252,6 +2564,14 @@ export class Daemon extends EventEmitter {
|
|
|
2252
2564
|
if (!windowId || event.windowId !== windowId || !this.instanceStateMachine)
|
|
2253
2565
|
return;
|
|
2254
2566
|
this.instanceStateLastOutputAt = event.at;
|
|
2567
|
+
if (this.backend?.hasPeriodicPaneRedraw?.() === true && this.instanceState === "idle") {
|
|
2568
|
+
// Do not emit idle→working solely because agy repainted an identical
|
|
2569
|
+
// footer. The short capture probe below still observes genuine pane
|
|
2570
|
+
// changes promptly and moves the state to working.
|
|
2571
|
+
this.scheduleInstanceStateOutputProbe();
|
|
2572
|
+
this.scheduleInstanceStateIdleCapture();
|
|
2573
|
+
return;
|
|
2574
|
+
}
|
|
2255
2575
|
const snapshot = this.instanceStateMachine.recordOutput(event.at);
|
|
2256
2576
|
this.applyInstanceStateSnapshot(snapshot);
|
|
2257
2577
|
this.scheduleInstanceStateIdleCapture();
|
|
@@ -2302,6 +2622,7 @@ export class Daemon extends EventEmitter {
|
|
|
2302
2622
|
stopInstanceStateMonitor() {
|
|
2303
2623
|
this.instanceStateMonitorActive = false;
|
|
2304
2624
|
this.clearInstanceStateIdleTimer();
|
|
2625
|
+
this.clearInstanceStateOutputProbeTimer();
|
|
2305
2626
|
this.clearInstanceStateStuckTimer();
|
|
2306
2627
|
if (this.instanceStateMonitorTimer)
|
|
2307
2628
|
clearInterval(this.instanceStateMonitorTimer);
|
|
@@ -2336,6 +2657,15 @@ export class Daemon extends EventEmitter {
|
|
|
2336
2657
|
this.logger.debug(diagnostic, "Suppressing stuck notification without pending work");
|
|
2337
2658
|
return;
|
|
2338
2659
|
}
|
|
2660
|
+
// An auth-broken instance IS stuck, and stays stuck: it cannot finish the
|
|
2661
|
+
// turn it is holding, so pendingWork never clears and this fired on every
|
|
2662
|
+
// scan forever. The notification is also useless — it offers Restart/Wait
|
|
2663
|
+
// and asks General for help, but only the user re-logging in fixes this, and
|
|
2664
|
+
// they have already been told exactly that by the auth alert.
|
|
2665
|
+
if (this.authFailureUnresolved) {
|
|
2666
|
+
this.logger.debug(diagnostic, "Suppressing stuck notification — unresolved auth failure (re-login required)");
|
|
2667
|
+
return;
|
|
2668
|
+
}
|
|
2339
2669
|
this.logger.warn(diagnostic, "Instance pane stuck with pending work");
|
|
2340
2670
|
this.hangDetector?.emit("hang", { unchangedForMs: snapshot.unchangedForMs });
|
|
2341
2671
|
}
|
|
@@ -2355,6 +2685,7 @@ export class Daemon extends EventEmitter {
|
|
|
2355
2685
|
// fire into a stopped daemon).
|
|
2356
2686
|
this.clearMcpRestartRequest();
|
|
2357
2687
|
this.interactivePromptDetector.reset();
|
|
2688
|
+
this.blockingProcessDetector.reset();
|
|
2358
2689
|
this.stopInstanceStateMonitor();
|
|
2359
2690
|
this.transcriptMonitor?.stop();
|
|
2360
2691
|
this.guardian?.stop();
|
|
@@ -2380,9 +2711,9 @@ export class Daemon extends EventEmitter {
|
|
|
2380
2711
|
* Tell the fleet manager what this instance is doing right now, for the live
|
|
2381
2712
|
* progress line on the cancel button.
|
|
2382
2713
|
*
|
|
2383
|
-
*
|
|
2384
|
-
*
|
|
2385
|
-
*
|
|
2714
|
+
* Primarily cosmetic. The foreground-process detector also uses a current
|
|
2715
|
+
* shell activity as one of several positive signals, but never makes a state
|
|
2716
|
+
* decision from an arbitrary activity label alone.
|
|
2386
2717
|
*
|
|
2387
2718
|
* Repeats are dropped: the ticker only edits the channel message when the text
|
|
2388
2719
|
* changes, and a stream of identical broadcasts would defeat that.
|
|
@@ -2398,6 +2729,141 @@ export class Daemon extends EventEmitter {
|
|
|
2398
2729
|
activity: next,
|
|
2399
2730
|
});
|
|
2400
2731
|
}
|
|
2732
|
+
/** Effective tool_progress level, hardened against junk config values.
|
|
2733
|
+
* Off unless explicitly enabled: an update must never make bubbles start
|
|
2734
|
+
* listing tool activity the operator did not ask to broadcast. */
|
|
2735
|
+
toolProgressLevel() {
|
|
2736
|
+
const raw = this.config.tool_progress;
|
|
2737
|
+
return raw === "standard" || raw === "verbose" ? raw : "off";
|
|
2738
|
+
}
|
|
2739
|
+
/**
|
|
2740
|
+
* Apply a tool-progress setting without restarting the CLI session.
|
|
2741
|
+
*
|
|
2742
|
+
* Drop the current accumulator at the boundary: verbose entries may contain
|
|
2743
|
+
* command previews which must not survive a downgrade to standard/off, and a
|
|
2744
|
+
* newly enabled level should start with a clean, internally consistent list.
|
|
2745
|
+
*/
|
|
2746
|
+
updateToolProgress(level) {
|
|
2747
|
+
const previous = this.toolProgressLevel();
|
|
2748
|
+
this.config.tool_progress = level;
|
|
2749
|
+
if (this.toolProgressLevel() !== previous)
|
|
2750
|
+
this.resetToolProgress();
|
|
2751
|
+
}
|
|
2752
|
+
/** Snapshot the configuration actually owned by this live daemon. */
|
|
2753
|
+
getConfigSnapshot() {
|
|
2754
|
+
return structuredClone(this.config);
|
|
2755
|
+
}
|
|
2756
|
+
/**
|
|
2757
|
+
* Apply the whitelisted hot configuration carried by FleetManager IPC.
|
|
2758
|
+
* null removes an optional value; invalid/unlisted fields are ignored rather
|
|
2759
|
+
* than allowing another client on the per-instance socket to mutate cold
|
|
2760
|
+
* process/session settings.
|
|
2761
|
+
*/
|
|
2762
|
+
applyConfigUpdate(value) {
|
|
2763
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
2764
|
+
return;
|
|
2765
|
+
const update = value;
|
|
2766
|
+
if (update.tool_progress === null || ["off", "standard", "verbose"].includes(String(update.tool_progress))) {
|
|
2767
|
+
this.updateToolProgress(update.tool_progress === null ? undefined : update.tool_progress);
|
|
2768
|
+
}
|
|
2769
|
+
if (typeof update.mcp_proxy_reply === "boolean")
|
|
2770
|
+
this.config.mcp_proxy_reply = update.mcp_proxy_reply;
|
|
2771
|
+
else if (update.mcp_proxy_reply === null)
|
|
2772
|
+
delete this.config.mcp_proxy_reply;
|
|
2773
|
+
if (typeof update.warm_cap === "number" && Number.isInteger(update.warm_cap) && update.warm_cap >= 0)
|
|
2774
|
+
this.config.warm_cap = update.warm_cap;
|
|
2775
|
+
else if (update.warm_cap === null)
|
|
2776
|
+
delete this.config.warm_cap;
|
|
2777
|
+
const nextAutoPause = typeof update.auto_pause_after === "number"
|
|
2778
|
+
&& Number.isFinite(update.auto_pause_after)
|
|
2779
|
+
&& update.auto_pause_after >= 0
|
|
2780
|
+
? update.auto_pause_after
|
|
2781
|
+
: update.auto_pause_after === null ? 0 : undefined;
|
|
2782
|
+
if (nextAutoPause !== undefined) {
|
|
2783
|
+
const previousAutoPause = typeof this.config.auto_pause_after === "number" ? this.config.auto_pause_after : 0;
|
|
2784
|
+
if (update.auto_pause_after === null)
|
|
2785
|
+
delete this.config.auto_pause_after;
|
|
2786
|
+
else
|
|
2787
|
+
this.config.auto_pause_after = nextAutoPause;
|
|
2788
|
+
if (nextAutoPause !== previousAutoPause) {
|
|
2789
|
+
const isGeneral = this.config.general_topic === true || this.name === "general";
|
|
2790
|
+
this.autoPauseController.reconfigure((isGeneral ? 0 : nextAutoPause) * 60_000);
|
|
2791
|
+
}
|
|
2792
|
+
}
|
|
2793
|
+
for (const key of ["display_name", "description"]) {
|
|
2794
|
+
if (typeof update[key] === "string")
|
|
2795
|
+
this.config[key] = update[key];
|
|
2796
|
+
else if (update[key] === null)
|
|
2797
|
+
delete this.config[key];
|
|
2798
|
+
}
|
|
2799
|
+
if (Array.isArray(update.tags) && update.tags.every(tag => typeof tag === "string"))
|
|
2800
|
+
this.config.tags = [...update.tags];
|
|
2801
|
+
else if (update.tags === null)
|
|
2802
|
+
delete this.config.tags;
|
|
2803
|
+
if (["trace", "debug", "info", "warn", "error"].includes(String(update.log_level))) {
|
|
2804
|
+
const level = update.log_level;
|
|
2805
|
+
if (this.config.log_level !== level) {
|
|
2806
|
+
this.config.log_level = level;
|
|
2807
|
+
this.logger.level = level;
|
|
2808
|
+
}
|
|
2809
|
+
}
|
|
2810
|
+
}
|
|
2811
|
+
/**
|
|
2812
|
+
* Accumulate one semantic progress line for the channel bubble and schedule
|
|
2813
|
+
* a coalesced broadcast. Separate from publishActivity on purpose: that one
|
|
2814
|
+
* is the single-line statusline detail (terse, operator-facing), this is the
|
|
2815
|
+
* multi-line channel list (semantic, argument-free at `standard`) — see
|
|
2816
|
+
* tool-progress.ts for why the two labellers must not merge.
|
|
2817
|
+
*/
|
|
2818
|
+
recordToolProgress(name, input) {
|
|
2819
|
+
const level = this.toolProgressLevel();
|
|
2820
|
+
if (level === "off")
|
|
2821
|
+
return;
|
|
2822
|
+
if (this.turnProgress.add(summarizeProgress(name, input, level))) {
|
|
2823
|
+
this.scheduleProgressBroadcast();
|
|
2824
|
+
}
|
|
2825
|
+
}
|
|
2826
|
+
/** Coalesce progress broadcasts: at most one per 3s, trailing edge kept. */
|
|
2827
|
+
scheduleProgressBroadcast() {
|
|
2828
|
+
const MIN_INTERVAL_MS = 3_000;
|
|
2829
|
+
const since = Date.now() - this.lastProgressBroadcastAt;
|
|
2830
|
+
if (since >= MIN_INTERVAL_MS) {
|
|
2831
|
+
this.broadcastToolProgress();
|
|
2832
|
+
return;
|
|
2833
|
+
}
|
|
2834
|
+
if (this.progressBroadcastTimer)
|
|
2835
|
+
return;
|
|
2836
|
+
this.progressBroadcastTimer = setTimeout(() => {
|
|
2837
|
+
this.progressBroadcastTimer = null;
|
|
2838
|
+
this.broadcastToolProgress();
|
|
2839
|
+
}, MIN_INTERVAL_MS - since);
|
|
2840
|
+
this.progressBroadcastTimer.unref?.();
|
|
2841
|
+
}
|
|
2842
|
+
broadcastToolProgress() {
|
|
2843
|
+
const rendered = this.turnProgress.render();
|
|
2844
|
+
if (rendered === this.lastProgressBroadcast)
|
|
2845
|
+
return;
|
|
2846
|
+
this.lastProgressBroadcast = rendered;
|
|
2847
|
+
this.lastProgressBroadcastAt = Date.now();
|
|
2848
|
+
this.ipcServer?.broadcast({
|
|
2849
|
+
type: "instance_progress",
|
|
2850
|
+
instanceName: this.name,
|
|
2851
|
+
progress: rendered,
|
|
2852
|
+
});
|
|
2853
|
+
}
|
|
2854
|
+
/** New turn (or turn over): drop the list and tell the fleet to clear it. */
|
|
2855
|
+
resetToolProgress() {
|
|
2856
|
+
if (this.progressBroadcastTimer) {
|
|
2857
|
+
clearTimeout(this.progressBroadcastTimer);
|
|
2858
|
+
this.progressBroadcastTimer = null;
|
|
2859
|
+
}
|
|
2860
|
+
this.turnProgress.reset();
|
|
2861
|
+
if (this.lastProgressBroadcast !== "") {
|
|
2862
|
+
this.lastProgressBroadcast = "";
|
|
2863
|
+
this.lastProgressBroadcastAt = Date.now();
|
|
2864
|
+
this.ipcServer?.broadcast({ type: "instance_progress", instanceName: this.name, progress: "" });
|
|
2865
|
+
}
|
|
2866
|
+
}
|
|
2401
2867
|
/**
|
|
2402
2868
|
* Options for every system-initiated paste (startup notice, session snapshot,
|
|
2403
2869
|
* runtime-dialog keys).
|
|
@@ -2433,12 +2899,130 @@ export class Daemon extends EventEmitter {
|
|
|
2433
2899
|
return ""; // skip channel tools
|
|
2434
2900
|
return name;
|
|
2435
2901
|
}
|
|
2902
|
+
/**
|
|
2903
|
+
* The one place an inbound message grows its metadata wrapper ([user:]/
|
|
2904
|
+
* [from:] prefix, pending reactions, handoff metadata, reply instructions).
|
|
2905
|
+
* Both the normal queued path (pushChannelMessage) and /steer go through
|
|
2906
|
+
* here — a steered message must read EXACTLY like a queued one to the
|
|
2907
|
+
* agent, or the two drift apart in what the agent is told about replying.
|
|
2908
|
+
*/
|
|
2909
|
+
formatInboundMessage(content, meta) {
|
|
2910
|
+
const user = meta.user || "unknown";
|
|
2911
|
+
const fromInstance = meta.from_instance;
|
|
2912
|
+
let formatted;
|
|
2913
|
+
if (fromInstance) {
|
|
2914
|
+
// #77: show the sender's display name for readability, keeping the machine
|
|
2915
|
+
// instance name in parens so the recipient's send_to_instance target is valid.
|
|
2916
|
+
const fromLabel = meta.from_display ? `${meta.from_display} (${fromInstance})` : fromInstance;
|
|
2917
|
+
formatted = `[from:${fromLabel}] ${content}`;
|
|
2918
|
+
formatted += renderHandoffMetadata(meta);
|
|
2919
|
+
// A delegated task that requires a reply must not read like a chatty FYI —
|
|
2920
|
+
// the "you may stay silent" line is for the latter only.
|
|
2921
|
+
formatted += meta.requires_reply === "true"
|
|
2922
|
+
? "\n(A reply IS required: use report_result with the correlation_id above — or send_to_instance. Not direct text.)"
|
|
2923
|
+
: "\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)";
|
|
2924
|
+
}
|
|
2925
|
+
else {
|
|
2926
|
+
const via = meta.source ? ` via ${meta.source}` : "";
|
|
2927
|
+
const idTag = meta.user_id ? `, id:${meta.user_id}` : "";
|
|
2928
|
+
formatted = `[user:${user}${via}${idTag}] ${content}`;
|
|
2929
|
+
// Reactions queued since the last real message (#432). One leading line of
|
|
2930
|
+
// context, present only when something is pending — a reaction no longer
|
|
2931
|
+
// costs a turn of its own.
|
|
2932
|
+
if (meta.pending_reactions) {
|
|
2933
|
+
formatted = `[Recent reactions: ${meta.pending_reactions}]\n${formatted}`;
|
|
2934
|
+
}
|
|
2935
|
+
formatted += renderHandoffMetadata(meta);
|
|
2936
|
+
formatted += "\n(Reply using the reply tool — do NOT respond with direct text)";
|
|
2937
|
+
}
|
|
2938
|
+
if (meta.reply_to_text) {
|
|
2939
|
+
formatted += `\n(reply_to: "${meta.reply_to_text}")`;
|
|
2940
|
+
}
|
|
2941
|
+
return formatted;
|
|
2942
|
+
}
|
|
2943
|
+
/**
|
|
2944
|
+
* /steer: interject into the CURRENT turn instead of queueing for idle.
|
|
2945
|
+
*
|
|
2946
|
+
* Differences from pushChannelMessage, and nothing else:
|
|
2947
|
+
* - serialized on steerLock, not pasteLock — it must overtake, not queue
|
|
2948
|
+
* - deliverMessage runs with { steer: true }, which takes the busy branch
|
|
2949
|
+
* that pastes immediately (the codex native-queue transaction) instead of
|
|
2950
|
+
* waiting for idle. Verification and silent-loss fallback are the ones
|
|
2951
|
+
* that path already has: pane-capture visibility check, then one
|
|
2952
|
+
* idle-gated redelivery — so on a TUI that swallows busy input the steer
|
|
2953
|
+
* degrades to "next message after this turn", never silently vanishes.
|
|
2954
|
+
*
|
|
2955
|
+
* The steered text goes through formatInboundMessage so the agent sees a
|
|
2956
|
+
* normal inbound message, with a steering notice prepended for context.
|
|
2957
|
+
*/
|
|
2958
|
+
steerMessage(content, meta, deliveryEpoch = this.deliveryEpoch) {
|
|
2959
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
2960
|
+
return;
|
|
2961
|
+
this.updateLastChat(meta.chat_id, meta.thread_id, meta.adapter_id);
|
|
2962
|
+
this.pendingWork.recordInbound();
|
|
2963
|
+
this.recordRecentUserMessage(content, meta);
|
|
2964
|
+
const formatted = "[STEERING — mid-task course correction from the user. Fold this into the CURRENT work.]\n"
|
|
2965
|
+
+ this.formatInboundMessage(content, meta);
|
|
2966
|
+
const chatId = meta.chat_id;
|
|
2967
|
+
const messageId = meta.message_id;
|
|
2968
|
+
const status = (chatId && messageId)
|
|
2969
|
+
? { chatId: meta.thread_id || chatId, messageId }
|
|
2970
|
+
: undefined;
|
|
2971
|
+
this.steerLock = this.steerLock.then(async () => {
|
|
2972
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
2973
|
+
return;
|
|
2974
|
+
await this.wake();
|
|
2975
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
2976
|
+
return;
|
|
2977
|
+
if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch })) {
|
|
2978
|
+
this.markTurnStarted(meta, formatted);
|
|
2979
|
+
}
|
|
2980
|
+
}).catch(err => {
|
|
2981
|
+
this.logger.warn({ err: err.message }, "steer delivery error");
|
|
2982
|
+
});
|
|
2983
|
+
}
|
|
2984
|
+
/**
|
|
2985
|
+
* /btw: submit a separately-labelled side question immediately while asking
|
|
2986
|
+
* the agent not to change the active task. Claude's native `/btw` opens a
|
|
2987
|
+
* branchAndResume panel whose answer never reaches the channel or transcript,
|
|
2988
|
+
* so channel delivery intentionally uses a normal wrapped inbound instead.
|
|
2989
|
+
*/
|
|
2990
|
+
btwMessage(content, meta, deliveryEpoch = this.deliveryEpoch) {
|
|
2991
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
2992
|
+
return;
|
|
2993
|
+
this.updateLastChat(meta.chat_id, meta.thread_id, meta.adapter_id);
|
|
2994
|
+
this.pendingWork.recordInbound();
|
|
2995
|
+
this.recordRecentUserMessage(content, meta);
|
|
2996
|
+
const formatted = "[BTW — side question from the user. Answer it separately without changing or interrupting the CURRENT work.]\n"
|
|
2997
|
+
+ this.formatInboundMessage(content, meta);
|
|
2998
|
+
const chatId = meta.chat_id;
|
|
2999
|
+
const messageId = meta.message_id;
|
|
3000
|
+
const status = (chatId && messageId)
|
|
3001
|
+
? { chatId: meta.thread_id || chatId, messageId }
|
|
3002
|
+
: undefined;
|
|
3003
|
+
this.steerLock = this.steerLock.then(async () => {
|
|
3004
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3005
|
+
return;
|
|
3006
|
+
await this.wake();
|
|
3007
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3008
|
+
return;
|
|
3009
|
+
if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch })) {
|
|
3010
|
+
this.markTurnStarted(meta, formatted);
|
|
3011
|
+
}
|
|
3012
|
+
}).catch(err => {
|
|
3013
|
+
this.logger.warn({ err: err.message }, "btw delivery error");
|
|
3014
|
+
});
|
|
3015
|
+
}
|
|
2436
3016
|
/**
|
|
2437
3017
|
* Push an inbound channel message to a specific MCP session.
|
|
2438
3018
|
* If targetSession is provided, only send to the matching socket.
|
|
2439
3019
|
* Otherwise send to the instance's own session (this.name).
|
|
2440
3020
|
*/
|
|
2441
|
-
pushChannelMessage(content, meta, _targetSession) {
|
|
3021
|
+
pushChannelMessage(content, meta, _targetSession, deliveryEpoch = this.deliveryEpoch) {
|
|
3022
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
3023
|
+
this.logger.info("Pending channel delivery dropped by user cancel");
|
|
3024
|
+
return;
|
|
3025
|
+
}
|
|
2442
3026
|
if (!this.tmux) {
|
|
2443
3027
|
this.logger.warn("Cannot push channel message: tmux not running");
|
|
2444
3028
|
return;
|
|
@@ -2472,46 +3056,19 @@ export class Daemon extends EventEmitter {
|
|
|
2472
3056
|
const rawText = content.slice(5);
|
|
2473
3057
|
this.logger.info({ user }, "Raw paste from topic mode user");
|
|
2474
3058
|
this.pasteLock = this.pasteLock.then(async () => {
|
|
2475
|
-
if (
|
|
3059
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3060
|
+
return;
|
|
3061
|
+
if (await this.deliverMessage(rawText, undefined, { deliveryEpoch }))
|
|
2476
3062
|
this.markTurnStarted(meta, rawText);
|
|
2477
3063
|
}).catch(err => {
|
|
2478
3064
|
this.logger.warn({ err: err.message }, "pasteLock raw delivery error");
|
|
2479
3065
|
});
|
|
2480
3066
|
return;
|
|
2481
3067
|
}
|
|
2482
|
-
|
|
2483
|
-
if (fromInstance) {
|
|
2484
|
-
// #77: show the sender's display name for readability, keeping the machine
|
|
2485
|
-
// instance name in parens so the recipient's send_to_instance target is valid.
|
|
2486
|
-
const fromLabel = meta.from_display ? `${meta.from_display} (${fromInstance})` : fromInstance;
|
|
2487
|
-
formatted = `[from:${fromLabel}] ${content}`;
|
|
2488
|
-
formatted += renderHandoffMetadata(meta);
|
|
2489
|
-
// A delegated task that requires a reply must not read like a chatty FYI —
|
|
2490
|
-
// the "you may stay silent" line is for the latter only.
|
|
2491
|
-
formatted += meta.requires_reply === "true"
|
|
2492
|
-
? "\n(A reply IS required: use report_result with the correlation_id above — or send_to_instance. Not direct text.)"
|
|
2493
|
-
: "\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)";
|
|
2494
|
-
}
|
|
2495
|
-
else {
|
|
2496
|
-
const via = meta.source ? ` via ${meta.source}` : "";
|
|
2497
|
-
const idTag = meta.user_id ? `, id:${meta.user_id}` : "";
|
|
2498
|
-
formatted = `[user:${user}${via}${idTag}] ${content}`;
|
|
2499
|
-
// Reactions queued since the last real message (#432). One leading line of
|
|
2500
|
-
// context, present only when something is pending — a reaction no longer
|
|
2501
|
-
// costs a turn of its own.
|
|
2502
|
-
if (meta.pending_reactions) {
|
|
2503
|
-
formatted = `[Recent reactions: ${meta.pending_reactions}]\n${formatted}`;
|
|
2504
|
-
}
|
|
2505
|
-
formatted += renderHandoffMetadata(meta);
|
|
2506
|
-
formatted += "\n(Reply using the reply tool — do NOT respond with direct text)";
|
|
2507
|
-
}
|
|
2508
|
-
if (meta.reply_to_text) {
|
|
2509
|
-
formatted += `\n(reply_to: "${meta.reply_to_text}")`;
|
|
2510
|
-
}
|
|
3068
|
+
const formatted = this.formatInboundMessage(content, meta);
|
|
2511
3069
|
// Serialize deliveries: each message waits for the previous to complete,
|
|
2512
3070
|
// and each waits for the CLI to be idle before pasting. Messages are never
|
|
2513
|
-
// dropped for age —
|
|
2514
|
-
// (the user can press Cancel to interrupt and let the queue drain sooner).
|
|
3071
|
+
// dropped for age — only an explicit user Cancel invalidates queued work.
|
|
2515
3072
|
const chatId = meta.chat_id;
|
|
2516
3073
|
const messageId = meta.message_id;
|
|
2517
3074
|
const wasQueued = this.pasteQueueDepth > 0;
|
|
@@ -2524,17 +3081,28 @@ export class Daemon extends EventEmitter {
|
|
|
2524
3081
|
}
|
|
2525
3082
|
this.pasteLock = this.pasteLock.then(async () => {
|
|
2526
3083
|
try {
|
|
3084
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
3085
|
+
this.logger.info("Pending channel delivery dropped by user cancel");
|
|
3086
|
+
return;
|
|
3087
|
+
}
|
|
2527
3088
|
if (this.config.pre_task_command) {
|
|
2528
|
-
await this.deliverMessage(this.config.pre_task_command);
|
|
3089
|
+
await this.deliverMessage(this.config.pre_task_command, undefined, { deliveryEpoch });
|
|
3090
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3091
|
+
return;
|
|
2529
3092
|
}
|
|
2530
3093
|
if (this.pendingInstructionsNotice) {
|
|
2531
3094
|
this.pendingInstructionsNotice = false;
|
|
2532
|
-
await this.deliverMessage(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir));
|
|
3095
|
+
await this.deliverMessage(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir), undefined, { deliveryEpoch });
|
|
3096
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3097
|
+
return;
|
|
2533
3098
|
}
|
|
2534
3099
|
const status = (chatId && messageId)
|
|
2535
3100
|
? { chatId: meta.thread_id || chatId, messageId }
|
|
2536
3101
|
: undefined;
|
|
2537
|
-
|
|
3102
|
+
// A fresh delivery begins a fresh turn — its bubble must not inherit
|
|
3103
|
+
// the previous turn's tool list.
|
|
3104
|
+
this.resetToolProgress();
|
|
3105
|
+
if (await this.deliverMessage(formatted, status, { deliveryEpoch }))
|
|
2538
3106
|
this.markTurnStarted(meta, formatted);
|
|
2539
3107
|
}
|
|
2540
3108
|
finally {
|
|
@@ -2561,7 +3129,11 @@ export class Daemon extends EventEmitter {
|
|
|
2561
3129
|
* immediately and own the application-level queue themselves. The pasteLock remains
|
|
2562
3130
|
* serial in both cases so separate PTY writes can never overlap.
|
|
2563
3131
|
*/
|
|
2564
|
-
async deliverMessage(formatted, status) {
|
|
3132
|
+
async deliverMessage(formatted, status, opts) {
|
|
3133
|
+
const cancelled = () => opts?.deliveryEpoch !== undefined
|
|
3134
|
+
&& !this.isDeliveryEpochCurrent(opts.deliveryEpoch);
|
|
3135
|
+
if (cancelled())
|
|
3136
|
+
return false;
|
|
2565
3137
|
// Sanitize unclosed code fences — they cause CLI to wait for closure on Enter
|
|
2566
3138
|
const fenceCount = (formatted.match(/```/g) || []).length;
|
|
2567
3139
|
if (fenceCount % 2 !== 0) {
|
|
@@ -2570,22 +3142,31 @@ export class Daemon extends EventEmitter {
|
|
|
2570
3142
|
}
|
|
2571
3143
|
// Before anything reads the window id: a spawn in progress is about to change it.
|
|
2572
3144
|
await this.waitForSpawnToSettle();
|
|
3145
|
+
if (cancelled())
|
|
3146
|
+
return false;
|
|
2573
3147
|
let windowId = this.getWindowId();
|
|
2574
3148
|
const supportsQueuedInput = this.backend?.supportsQueuedInput?.() === true;
|
|
2575
3149
|
let handingOffToNativeQueue = false;
|
|
2576
3150
|
// If the CLI is busy, either hand the complete submission to its native input
|
|
2577
3151
|
// queue or wait for idle. Native queue support is an explicit backend capability:
|
|
2578
3152
|
// normal Enter input has steering/interrupt semantics in several other CLIs.
|
|
2579
|
-
if (windowId && this.controlClient && !this.
|
|
3153
|
+
if (windowId && this.controlClient && !this.isPaneIdleForDelivery(windowId)) {
|
|
2580
3154
|
if (status)
|
|
2581
3155
|
this.emit("message_queued", status);
|
|
2582
|
-
if (supportsQueuedInput) {
|
|
3156
|
+
if (supportsQueuedInput || opts?.steer) {
|
|
3157
|
+
// Native queue (codex), or an explicit /steer: hand the complete
|
|
3158
|
+
// paste+Enter transaction to the busy CLI now. For steer this is the
|
|
3159
|
+
// point — the user asked to interject, and the transaction's
|
|
3160
|
+
// visibility check + idle-gated fallback catch TUIs that swallow
|
|
3161
|
+
// busy input (see steerMessage).
|
|
2583
3162
|
handingOffToNativeQueue = true;
|
|
2584
|
-
this.logger.debug("CLI busy — handing message to backend-native input queue");
|
|
3163
|
+
this.logger.debug(opts?.steer ? "CLI busy — steering into the running turn" : "CLI busy — handing message to backend-native input queue");
|
|
2585
3164
|
}
|
|
2586
3165
|
else {
|
|
2587
3166
|
this.logger.debug("CLI busy — queuing message until idle");
|
|
2588
|
-
const becameIdle = await this.
|
|
3167
|
+
const becameIdle = await this.waitForPaneIdleForDelivery(windowId);
|
|
3168
|
+
if (cancelled())
|
|
3169
|
+
return false;
|
|
2589
3170
|
if (!becameIdle) {
|
|
2590
3171
|
// The pane never freed up. Report the failure instead of pasting into a
|
|
2591
3172
|
// wedged CLI (where the text would sit unsubmitted and the next message
|
|
@@ -2601,7 +3182,54 @@ export class Daemon extends EventEmitter {
|
|
|
2601
3182
|
// held under the pane lock — holding it across the idle wait (up to 30 min)
|
|
2602
3183
|
// would starve the runtime-dialog dismisser, which is often the very thing
|
|
2603
3184
|
// that would let the pane go idle again.
|
|
2604
|
-
return this.paneWriteLock.run(() =>
|
|
3185
|
+
return this.paneWriteLock.run(async () => {
|
|
3186
|
+
if (cancelled())
|
|
3187
|
+
return false;
|
|
3188
|
+
return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status);
|
|
3189
|
+
});
|
|
3190
|
+
}
|
|
3191
|
+
/**
|
|
3192
|
+
* A raw `%output` silence check is authoritative for ordinary TUIs. It is not
|
|
3193
|
+
* for a backend such as agy whose status-line hook periodically redraws an
|
|
3194
|
+
* identical screen. In that case trust an idle pane-state snapshot only when
|
|
3195
|
+
* it was captured after both the latest output and the latest control-client
|
|
3196
|
+
* observation reset.
|
|
3197
|
+
*/
|
|
3198
|
+
isPaneIdleForDelivery(windowId) {
|
|
3199
|
+
if (!this.controlClient)
|
|
3200
|
+
return true;
|
|
3201
|
+
if (this.controlClient.isIdle(windowId))
|
|
3202
|
+
return true;
|
|
3203
|
+
if (this.backend?.hasPeriodicPaneRedraw?.() !== true || this.instanceState !== "idle")
|
|
3204
|
+
return false;
|
|
3205
|
+
const lastOutputAt = this.controlClient.getLastOutputAt(windowId);
|
|
3206
|
+
if (lastOutputAt === undefined || !this.instanceStateMachine)
|
|
3207
|
+
return false;
|
|
3208
|
+
const observedAt = this.instanceStateMachine.snapshot().observedAt;
|
|
3209
|
+
return observedAt >= lastOutputAt
|
|
3210
|
+
&& observedAt >= this.controlClient.getObservationResetAt();
|
|
3211
|
+
}
|
|
3212
|
+
waitForPaneIdleForDelivery(windowId, timeoutMs = 30 * 60_000) {
|
|
3213
|
+
if (!this.controlClient)
|
|
3214
|
+
return Promise.resolve(true);
|
|
3215
|
+
if (this.backend?.hasPeriodicPaneRedraw?.() !== true) {
|
|
3216
|
+
return this.controlClient.waitUntilIdle(windowId, timeoutMs);
|
|
3217
|
+
}
|
|
3218
|
+
if (this.isPaneIdleForDelivery(windowId))
|
|
3219
|
+
return Promise.resolve(true);
|
|
3220
|
+
return new Promise(resolve => {
|
|
3221
|
+
const check = setInterval(() => {
|
|
3222
|
+
if (!this.isPaneIdleForDelivery(windowId))
|
|
3223
|
+
return;
|
|
3224
|
+
clearInterval(check);
|
|
3225
|
+
clearTimeout(timer);
|
|
3226
|
+
resolve(true);
|
|
3227
|
+
}, 50);
|
|
3228
|
+
const timer = setTimeout(() => {
|
|
3229
|
+
clearInterval(check);
|
|
3230
|
+
resolve(false);
|
|
3231
|
+
}, timeoutMs);
|
|
3232
|
+
});
|
|
2605
3233
|
}
|
|
2606
3234
|
/**
|
|
2607
3235
|
* The write half of a delivery: paste → Enter → confirm, with retries.
|
|
@@ -3097,7 +3725,10 @@ export class Daemon extends EventEmitter {
|
|
|
3097
3725
|
}
|
|
3098
3726
|
/** Build config object for the CLI backend */
|
|
3099
3727
|
buildBackendConfig() {
|
|
3100
|
-
|
|
3728
|
+
// Antigravity 1.1.17 has no per-workspace MCP config, but its global MCP
|
|
3729
|
+
// children inherit the CLI process environment. The backend installs one
|
|
3730
|
+
// environment-selecting launcher, so MCP is now the safe default here too.
|
|
3731
|
+
const isCliMode = this.config.agent_mode === "cli";
|
|
3101
3732
|
const sockPath = join(this.instanceDir, "channel.sock");
|
|
3102
3733
|
let serverJs = join(__dirname, "channel", "mcp-server.js");
|
|
3103
3734
|
if (!existsSync(serverJs)) {
|
|
@@ -3234,6 +3865,8 @@ export class Daemon extends EventEmitter {
|
|
|
3234
3865
|
});
|
|
3235
3866
|
}
|
|
3236
3867
|
const agentPort = parseInt(process.env.AGEND_PORT ?? "19280", 10);
|
|
3868
|
+
const backendName = this.config.backend ?? "claude-code";
|
|
3869
|
+
const backendOptions = this.config.backend_options?.[backendName];
|
|
3237
3870
|
return {
|
|
3238
3871
|
workingDirectory: this.config.working_directory,
|
|
3239
3872
|
instanceDir: this.instanceDir,
|
|
@@ -3253,6 +3886,7 @@ export class Daemon extends EventEmitter {
|
|
|
3253
3886
|
instructions,
|
|
3254
3887
|
agentMode: isCliMode ? "cli" : "mcp",
|
|
3255
3888
|
agentPort: isCliMode ? agentPort : undefined,
|
|
3889
|
+
backendOptions,
|
|
3256
3890
|
};
|
|
3257
3891
|
}
|
|
3258
3892
|
/**
|
|
@@ -3580,6 +4214,24 @@ export class Daemon extends EventEmitter {
|
|
|
3580
4214
|
// CLI is ready (pattern defined by each backend)
|
|
3581
4215
|
if (this.backend.getReadyPattern().test(pane))
|
|
3582
4216
|
return true;
|
|
4217
|
+
// A CLI parked at its own sign-in screen is an AUTH incident: no dialog
|
|
4218
|
+
// key can dismiss it and no ready pattern will ever appear. Report it as
|
|
4219
|
+
// an auth error (the lifecycle double-checks with the token-free probe)
|
|
4220
|
+
// instead of letting it decay into crash/hang/MCP-died noise, and stop
|
|
4221
|
+
// the retry loop — only re-login (see /login) ends this state.
|
|
4222
|
+
const loginScreen = LOGIN_FLOWS[this.config.backend ?? "claude-code"]?.loginScreenPattern;
|
|
4223
|
+
if (loginScreen?.test(pane) && !this.loginScreenReported) {
|
|
4224
|
+
this.loginScreenReported = true;
|
|
4225
|
+
this.authFailureUnresolved = true;
|
|
4226
|
+
this.logger.warn({ backend: this.config.backend }, "CLI is waiting at its sign-in screen — reporting auth error");
|
|
4227
|
+
this.emit("pty_error", {
|
|
4228
|
+
name: this.name,
|
|
4229
|
+
type: "auth_error",
|
|
4230
|
+
action: "pause",
|
|
4231
|
+
message: "CLI is waiting at its sign-in screen — credentials are missing or expired",
|
|
4232
|
+
});
|
|
4233
|
+
return true;
|
|
4234
|
+
}
|
|
3583
4235
|
// Fatal: command not found (must match full phrase to avoid false positives
|
|
3584
4236
|
// like Kiro's "agent X not found, using default")
|
|
3585
4237
|
if (/command not found|: not found$/m.test(pane))
|
|
@@ -3616,9 +4268,29 @@ export class Daemon extends EventEmitter {
|
|
|
3616
4268
|
// back from statusline.json — that would re-arm --continue and re-loop.
|
|
3617
4269
|
if (this.skipResume)
|
|
3618
4270
|
return;
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
4271
|
+
try {
|
|
4272
|
+
const sid = this.backend?.getSessionId();
|
|
4273
|
+
if (!sid)
|
|
4274
|
+
return;
|
|
4275
|
+
const path = join(this.instanceDir, "session-id");
|
|
4276
|
+
// Idle observations happen after turns. Avoid needless writes (and mtime
|
|
4277
|
+
// churn) while still updating the marker when /new changes the session.
|
|
4278
|
+
try {
|
|
4279
|
+
if (readFileSync(path, "utf-8").trim() === sid)
|
|
4280
|
+
return;
|
|
4281
|
+
}
|
|
4282
|
+
catch { /* first checkpoint */ }
|
|
4283
|
+
writeFileSync(path, sid);
|
|
4284
|
+
this.sessionCheckpointWarningEmitted = false;
|
|
4285
|
+
this.logger.debug("Session id checkpointed");
|
|
4286
|
+
}
|
|
4287
|
+
catch (err) {
|
|
4288
|
+
// Session discovery is best-effort and must never break the pane state
|
|
4289
|
+
// monitor. Existing stop/pause checkpoints get another chance later.
|
|
4290
|
+
if (!this.sessionCheckpointWarningEmitted) {
|
|
4291
|
+
this.sessionCheckpointWarningEmitted = true;
|
|
4292
|
+
this.logger.warn({ err: err.message }, "Session id checkpoint failed");
|
|
4293
|
+
}
|
|
3622
4294
|
}
|
|
3623
4295
|
}
|
|
3624
4296
|
readContextPercentage() {
|