agent-relay-server 0.121.5 → 0.122.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/docs/openapi.json +1 -1
  2. package/package.json +6 -6
  3. package/public/assets/{MessageBubble-CLqxCG9b.js → MessageBubble-DRdUD-kW.js} +2 -2
  4. package/public/assets/{MessageBubble-CLqxCG9b.js.map → MessageBubble-DRdUD-kW.js.map} +1 -1
  5. package/public/assets/{PlanGraphPanel-BkkToFEb.js → PlanGraphPanel-siKMPCln.js} +2 -2
  6. package/public/assets/{PlanGraphPanel-BkkToFEb.js.map → PlanGraphPanel-siKMPCln.js.map} +1 -1
  7. package/public/assets/{activity-D-ocGQGs.js → activity-D3MNcSaV.js} +2 -2
  8. package/public/assets/{activity-D-ocGQGs.js.map → activity-D3MNcSaV.js.map} +1 -1
  9. package/public/assets/{agent-profiles-CRPgYTyb.js → agent-profiles-CKzttq6G.js} +2 -2
  10. package/public/assets/{agent-profiles-CRPgYTyb.js.map → agent-profiles-CKzttq6G.js.map} +1 -1
  11. package/public/assets/{agent-type-icon-CeEoMouY.js → agent-type-icon-0ZJRP1Vy.js} +2 -2
  12. package/public/assets/{agent-type-icon-CeEoMouY.js.map → agent-type-icon-0ZJRP1Vy.js.map} +1 -1
  13. package/public/assets/{agents-Chw5E_2q.js → agents-CuVj_rfG.js} +2 -2
  14. package/public/assets/{agents-Chw5E_2q.js.map → agents-CuVj_rfG.js.map} +1 -1
  15. package/public/assets/{analytics-BmNDG0hR.js → analytics-iob6C27d.js} +2 -2
  16. package/public/assets/{analytics-BmNDG0hR.js.map → analytics-iob6C27d.js.map} +1 -1
  17. package/public/assets/{automation-D7g90kTv.js → automation-B2ixxs4q.js} +2 -2
  18. package/public/assets/{automation-D7g90kTv.js.map → automation-B2ixxs4q.js.map} +1 -1
  19. package/public/assets/{branch-state-badge-Bv7DhLBZ.js → branch-state-badge-CafBJWKo.js} +2 -2
  20. package/public/assets/{branch-state-badge-Bv7DhLBZ.js.map → branch-state-badge-CafBJWKo.js.map} +1 -1
  21. package/public/assets/{channels-CLwPeEPi.js → channels-C-WsbL2H.js} +2 -2
  22. package/public/assets/{channels-CLwPeEPi.js.map → channels-C-WsbL2H.js.map} +1 -1
  23. package/public/assets/{chat-Cgj7VKzc.js → chat-DeoVTVob.js} +2 -2
  24. package/public/assets/{chat-Cgj7VKzc.js.map → chat-DeoVTVob.js.map} +1 -1
  25. package/public/assets/{connectors-Br6fiTny.js → connectors-C2Ac03LJ.js} +2 -2
  26. package/public/assets/{connectors-Br6fiTny.js.map → connectors-C2Ac03LJ.js.map} +1 -1
  27. package/public/assets/display-CJzSoTMq.js +3 -0
  28. package/public/assets/display-CJzSoTMq.js.map +1 -0
  29. package/public/assets/{formatted-body-CmkuD23M.js → formatted-body-35XBOY5L.js} +3 -3
  30. package/public/assets/{formatted-body-CmkuD23M.js.map → formatted-body-35XBOY5L.js.map} +1 -1
  31. package/public/assets/{formatted-body-impl-DbdFmbwW.js → formatted-body-impl-RmDxI5Ux.js} +2 -2
  32. package/public/assets/{formatted-body-impl-DbdFmbwW.js.map → formatted-body-impl-RmDxI5Ux.js.map} +1 -1
  33. package/public/assets/index-D0xMQQb_.js +25 -0
  34. package/public/assets/{index-D9OogaB5.js.map → index-D0xMQQb_.js.map} +1 -1
  35. package/public/assets/{insights-qvaPqWCV.js → insights-B643SFvm.js} +2 -2
  36. package/public/assets/{insights-qvaPqWCV.js.map → insights-B643SFvm.js.map} +1 -1
  37. package/public/assets/{integrations-DYEGSqq_.js → integrations-hFIHeCF4.js} +2 -2
  38. package/public/assets/{integrations-DYEGSqq_.js.map → integrations-hFIHeCF4.js.map} +1 -1
  39. package/public/assets/{maintenance-C0HIEB-J.js → maintenance-CURvdp_Z.js} +2 -2
  40. package/public/assets/{maintenance-C0HIEB-J.js.map → maintenance-CURvdp_Z.js.map} +1 -1
  41. package/public/assets/{managed-agents-DdS7aI38.js → managed-agents-DrZtdlSn.js} +2 -2
  42. package/public/assets/{managed-agents-DdS7aI38.js.map → managed-agents-DrZtdlSn.js.map} +1 -1
  43. package/public/assets/{markdown-preview-impl-BnXMH1z1.js → markdown-preview-impl-CVDO--ek.js} +2 -2
  44. package/public/assets/{markdown-preview-impl-BnXMH1z1.js.map → markdown-preview-impl-CVDO--ek.js.map} +1 -1
  45. package/public/assets/{memory-CSwx7bz8.js → memory-Bm7dt_Qf.js} +2 -2
  46. package/public/assets/{memory-CSwx7bz8.js.map → memory-Bm7dt_Qf.js.map} +1 -1
  47. package/public/assets/{messages-Be9CmvDv.js → messages-DTq4i2KZ.js} +2 -2
  48. package/public/assets/{messages-Be9CmvDv.js.map → messages-DTq4i2KZ.js.map} +1 -1
  49. package/public/assets/{orchestrators-BgtvvHvo.js → orchestrators-Dra318Ap.js} +2 -2
  50. package/public/assets/{orchestrators-BgtvvHvo.js.map → orchestrators-Dra318Ap.js.map} +1 -1
  51. package/public/assets/{overview-BvhbPWtF.js → overview-yr1Vmk64.js} +2 -2
  52. package/public/assets/{overview-BvhbPWtF.js.map → overview-yr1Vmk64.js.map} +1 -1
  53. package/public/assets/{pairs-CpXSOMx0.js → pairs-mnlKDBGp.js} +2 -2
  54. package/public/assets/{pairs-CpXSOMx0.js.map → pairs-mnlKDBGp.js.map} +1 -1
  55. package/public/assets/projects-Dzdf-gkK.js +2 -0
  56. package/public/assets/{projects-DdPNPFJI.js.map → projects-Dzdf-gkK.js.map} +1 -1
  57. package/public/assets/prompt-settings-ByL8SNCi.js +2 -0
  58. package/public/assets/{prompt-settings-BpZse7-X.js.map → prompt-settings-ByL8SNCi.js.map} +1 -1
  59. package/public/assets/{registry-pb1gTZEg.js → registry-CXWyOtpr.js} +2 -2
  60. package/public/assets/{registry-pb1gTZEg.js.map → registry-CXWyOtpr.js.map} +1 -1
  61. package/public/assets/{security-BgcX1n9D.js → security-0SBsc_4W.js} +2 -2
  62. package/public/assets/{security-BgcX1n9D.js.map → security-0SBsc_4W.js.map} +1 -1
  63. package/public/assets/{select-DZPVpKPv.js → select-Bw5z2KJ_.js} +2 -2
  64. package/public/assets/{select-DZPVpKPv.js.map → select-Bw5z2KJ_.js.map} +1 -1
  65. package/public/assets/{settings-hd5kzdRB.js → settings-78Cyq_tA.js} +4 -4
  66. package/public/assets/{settings-hd5kzdRB.js.map → settings-78Cyq_tA.js.map} +1 -1
  67. package/public/assets/{store-DKTLubBT.js → store-BPrVuKiv.js} +4 -4
  68. package/public/assets/store-BPrVuKiv.js.map +1 -0
  69. package/public/assets/{tasks-C5d2LU5E.js → tasks-Cg2MMeri.js} +2 -2
  70. package/public/assets/{tasks-C5d2LU5E.js.map → tasks-Cg2MMeri.js.map} +1 -1
  71. package/public/assets/teams-DWv6UwFV.js +2 -0
  72. package/public/assets/{teams-B8f2pGJe.js.map → teams-DWv6UwFV.js.map} +1 -1
  73. package/public/assets/{terminal-viewer-impl-CxAscH0U.js → terminal-viewer-impl-DBLDjbIg.js} +2 -2
  74. package/public/assets/{terminal-viewer-impl-CxAscH0U.js.map → terminal-viewer-impl-DBLDjbIg.js.map} +1 -1
  75. package/public/assets/{types-uVOXgvMT.js → types-BeJUSfDj.js} +2 -2
  76. package/public/assets/types-BeJUSfDj.js.map +1 -0
  77. package/public/assets/{user-avatar-Dw6mzWhv.js → user-avatar-BsOLyVJj.js} +2 -2
  78. package/public/assets/{user-avatar-Dw6mzWhv.js.map → user-avatar-BsOLyVJj.js.map} +1 -1
  79. package/public/assets/{work-queue-9imc7aNK.js → work-queue-DmwvnF6F.js} +2 -2
  80. package/public/assets/{work-queue-9imc7aNK.js.map → work-queue-DmwvnF6F.js.map} +1 -1
  81. package/public/assets/{workspaces-CLYc3yF3.js → workspaces-dIBivJN8.js} +2 -2
  82. package/public/assets/{workspaces-CLYc3yF3.js.map → workspaces-dIBivJN8.js.map} +1 -1
  83. package/public/index.html +6 -6
  84. package/runner/plugins/claude/.claude-plugin/plugin.json +1 -1
  85. package/runner/plugins/claude/monitors/relay-monitor.provisioned.mjs +44 -938
  86. package/runner/src/adapter.ts +53 -18
  87. package/runner/src/adapters/claude-delivery.ts +127 -0
  88. package/runner/src/adapters/claude-quota-harvest.ts +92 -0
  89. package/runner/src/adapters/claude-session-probe.ts +225 -0
  90. package/runner/src/adapters/claude-status-detectors.ts +147 -0
  91. package/runner/src/adapters/claude-transcript-tail.ts +128 -0
  92. package/runner/src/adapters/claude-transcript.ts +414 -0
  93. package/runner/src/adapters/claude.ts +672 -0
  94. package/runner/src/adapters/codex-agent-message-capture.ts +139 -0
  95. package/runner/src/adapters/codex-client.ts +279 -0
  96. package/runner/src/adapters/codex.ts +1345 -0
  97. package/runner/src/attachment-cache.ts +189 -0
  98. package/runner/src/busy-reconciler.ts +102 -0
  99. package/runner/src/claim-tracker.ts +140 -0
  100. package/runner/src/claude-prompt-gates.ts +268 -0
  101. package/runner/src/config.ts +1 -4
  102. package/runner/src/context-probe-state.ts +6 -0
  103. package/runner/src/continuation-archive.ts +179 -0
  104. package/runner/src/control-server.ts +639 -0
  105. package/runner/src/index.ts +396 -0
  106. package/runner/src/launch-assembly.ts +508 -0
  107. package/runner/src/liveness.ts +50 -0
  108. package/runner/src/logger.ts +99 -0
  109. package/runner/src/mcp-outbox.ts +120 -0
  110. package/runner/src/message-body-config/config.ts +3 -0
  111. package/runner/src/message-body-config.ts +1 -0
  112. package/runner/src/native-self-resume.ts +202 -0
  113. package/runner/src/outbox.ts +342 -0
  114. package/runner/src/process-meta.ts +9 -0
  115. package/runner/src/profile-home.ts +260 -0
  116. package/runner/src/profile-projection.ts +305 -0
  117. package/runner/src/providers.ts +20 -0
  118. package/runner/src/provisioning.ts +314 -0
  119. package/runner/src/quota.ts +40 -0
  120. package/runner/src/rate-limit.ts +189 -0
  121. package/runner/src/relay-injection-events.ts +102 -0
  122. package/runner/src/relay-instructions.ts +60 -0
  123. package/runner/src/relay-mcp-proxy.ts +383 -0
  124. package/runner/src/relay-mcp.ts +156 -0
  125. package/runner/src/reply-obligation-cache.ts +136 -0
  126. package/runner/src/response-capture-report.ts +63 -0
  127. package/runner/src/runner-core.ts +2409 -0
  128. package/runner/src/runner-helpers.ts +435 -0
  129. package/runner/src/runner-insights.ts +105 -0
  130. package/runner/src/runner.ts +14 -0
  131. package/runner/src/session-destroy.ts +22 -0
  132. package/runner/src/session-insights.ts +118 -0
  133. package/runner/src/session-scratch.ts +271 -0
  134. package/runner/src/template-resolver.ts +29 -0
  135. package/runner/src/version.ts +43 -0
  136. package/src/agent-card-projection.ts +5 -1
  137. package/src/agent-command-activity.ts +39 -0
  138. package/src/automations/reconcile.ts +2 -2
  139. package/src/cli/workspace.ts +3 -1
  140. package/src/config.ts +10 -1
  141. package/src/db/agents.ts +118 -37
  142. package/src/db/message-reads.ts +50 -0
  143. package/src/db/messages.ts +48 -9
  144. package/src/db/migrations.ts +54 -0
  145. package/src/db/provider-quota-burn.ts +96 -67
  146. package/src/db/provider-quotas.ts +29 -7
  147. package/src/db/scheduled-timers.ts +21 -0
  148. package/src/db/schema.ts +1 -0
  149. package/src/db/tasks.ts +33 -3
  150. package/src/db/work-evidence.ts +10 -7
  151. package/src/event-loop-lag.ts +15 -0
  152. package/src/gate-resolver.ts +19 -1
  153. package/src/maintenance/jobs.ts +23 -6
  154. package/src/managed-policy.ts +5 -12
  155. package/src/mcp-plan-tools.ts +13 -1
  156. package/src/project-instructions.ts +36 -2
  157. package/src/quota-advisory.ts +6 -4
  158. package/src/quota-band-transition.ts +23 -14
  159. package/src/routes/agent-sessions.ts +39 -7
  160. package/src/routes/agents.ts +3 -2
  161. package/src/routes/tasks.ts +16 -4
  162. package/src/routes/workspaces.ts +35 -10
  163. package/src/services/plan-graph.ts +27 -9
  164. package/src/services/register-agent.ts +12 -12
  165. package/src/services/send-message.ts +43 -15
  166. package/src/services/spawn-agent.ts +3 -0
  167. package/src/spawn-command.ts +8 -0
  168. package/src/spawn-targets.ts +19 -34
  169. package/src/sse.ts +25 -3
  170. package/src/workspace-merge.ts +25 -5
  171. package/src/workspace-orphans.ts +13 -6
  172. package/public/assets/display-exQDWum3.js +0 -3
  173. package/public/assets/display-exQDWum3.js.map +0 -1
  174. package/public/assets/index-D9OogaB5.js +0 -25
  175. package/public/assets/projects-DdPNPFJI.js +0 -2
  176. package/public/assets/prompt-settings-BpZse7-X.js +0 -2
  177. package/public/assets/store-DKTLubBT.js.map +0 -1
  178. package/public/assets/teams-B8f2pGJe.js +0 -2
  179. package/public/assets/types-uVOXgvMT.js.map +0 -1
@@ -0,0 +1,2409 @@
1
+ import { hostname } from "node:os";
2
+ import { mkdirSync, writeFileSync } from "node:fs";
3
+ import { readFile } from "node:fs/promises";
4
+ import { dirname, join } from "node:path";
5
+ import type { AgentLifecycle, AgentProfile, ContextState, Message, MessageSessionMeta, ProviderState, SendMessageInput, TaskStatusInput, WorkspaceMetadata } from "agent-relay-sdk";
6
+ import { errMessage, RelayBusClient, RelayHttpClient } from "agent-relay-sdk";
7
+ import { contextStateFromProbeMetrics, quotaStateFromProbeMetrics } from "agent-relay-sdk/context-probe";
8
+ import { isPidAlive } from "agent-relay-sdk/process-utils";
9
+ import { providerAttachmentText, providerMessageText } from "./adapter";
10
+ import { computeLivenessSignal, type LivenessInputs } from "./liveness";
11
+ import type { ManagedProcess, ProviderAdapter, ProviderConfig, ProviderPermissionDecision, ProviderPermissionDecisionInput, ProviderSessionEvent, ProviderStatusUpdate, RunnerSpawnConfig, SemanticStatus, TerminalAttachSpec } from "./adapter";
12
+ import { messagesWithCachedAttachments } from "./attachment-cache";
13
+ import { ClaimTracker } from "./claim-tracker";
14
+ import { ClaudeQuotaHarvest } from "./adapters/claude-quota-harvest";
15
+ import { readRunnerContextProbeState } from "./context-probe-state";
16
+ import { startControlServer, type ControlServer, type ResolvedPermissionPrompt } from "./control-server";
17
+ import { ReplyObligationCache, obligationRequiresExplicitReply, responseCaptureReplyRoute } from "./reply-obligation-cache";
18
+ import { Outbox, type OutboxRecord } from "./outbox";
19
+ import { extractLastAssistantTurn, extractLastAssistantTurnAfterEntry, countTranscriptEntries, extractFinalAssistantMessage, extractFinalAssistantMessageAfterEntry, extractHookAssistantMessage, extractLatestTurnSteps, stepDedupKeys, transcriptLooksComplete } from "./adapters/claude-transcript";
20
+ import { IncrementalClaudeTranscriptTail } from "./adapters/claude-transcript-tail";
21
+ import { getManifest } from "agent-relay-providers";
22
+ import { profileUsesProviderHostGlobals } from "./profile-home";
23
+ import { RELAY_MCP_TOKEN_ENV, relayMcpEndpoint, resolveSharedMcpUrl } from "./relay-mcp";
24
+ import { RelayMcpProxy } from "./relay-mcp-proxy";
25
+ import { runtimeMetadata } from "./version";
26
+ import { logger, parseLogLevel } from "./logger";
27
+ import { ensureSessionScratch, reapSessionLauncher, reapSessionScratch, sweepStaleLaunchers, sweepStaleSessions, type SessionScratchLayout } from "./session-scratch";
28
+ import { deliverBufferedMcpCall as deliverBufferedMcpOutboxCall, deliverRunnerOutboxEvent } from "./mcp-outbox";
29
+ import { RunnerInsights } from "./runner-insights";
30
+ import { BusyReconciler } from "./busy-reconciler";
31
+ import { publishCapturedResponse } from "./response-capture-report";
32
+ import { runnerLaunchInjectionSessionEvents, type RunnerRelayInjectionEvent } from "./relay-injection-events";
33
+ import { providerTerminalSession, providerTerminalSocket } from "./process-meta";
34
+ import { capsFromEnv, logLevelFromEnv, mcpProxyEnabledFromEnv, orchestratorBaseDirFromEnv, orchestratorUrlFromEnv, registrationTimeoutMsFromEnv, runnerInfoFileFromEnv, runnerLogFileFromEnv, runnerOutboxDirWithInfoFallback, sessionDebugEnabled, tagsFromEnv, workspaceModeFromEnv } from "./config";
35
+ import { boundaryReasonForCommand, PRE_DESTROY_TIMEOUT_MS, reasonExitsRunner, type LifecycleAction, type SessionDestroyReason } from "./session-destroy";
36
+ import { isNativeSelfResumeCompact, NativeSelfResumeTracker, recoverNativeSelfResumeExit } from "./native-self-resume";
37
+ import { initializeRunnerTemplates } from "./template-resolver";
38
+ import {
39
+ appliedAgentProfileMetadata,
40
+ commandTimeoutMs,
41
+ csvTags,
42
+ httpErrorKey,
43
+ isContextState,
44
+ isHttpAuthError,
45
+ isHttpStatusError,
46
+ lifecycleCapabilities,
47
+ providerExitDiagnostics,
48
+ providerStateFromActiveWork,
49
+ registerWithinDeadline,
50
+ relayBusUrl,
51
+ runnerAgentStatus,
52
+ runnerBusErrorAction,
53
+ runnerMessageMatches,
54
+ runnerMessageShouldDirtyObligations,
55
+ runnerShouldResolveProviderExit,
56
+ runnerShouldRestartUnexpectedProviderExit,
57
+ runtimeProviderCapabilities,
58
+ runtimeTokenRenewDelayMs,
59
+ runtimeTokenRenewTimerSchedule,
60
+ shouldLogDeliveryFailure,
61
+ stripRunnerClaimedGuidance,
62
+ taskIdFromMessage,
63
+ type ProbeModelInfo,
64
+ } from "./runner-helpers";
65
+
66
+ // Pre-destroy work is best-effort and must never hang teardown. Capping it keeps a slow
67
+ // transcript read or a wedged provider from stalling a shutdown the operator asked for.
68
+ // Bounded window to deliver the durable outbox before an exit-bound teardown (#183). Kept
69
+ // short so a wedged/down server can't stall an operator-requested shutdown for long; a
70
+ // row that still can't land is logged, not silently dropped.
71
+ const OUTBOX_FLUSH_TIMEOUT_MS = 3_000;
72
+
73
+ interface RunnerOptions {
74
+ provider: string;
75
+ model?: string;
76
+ effort?: string;
77
+ runnerId: string;
78
+ instanceId: string;
79
+ agentId?: string;
80
+ relayUrl: string;
81
+ token?: string;
82
+ tokenJti?: string;
83
+ tokenProfileId?: string;
84
+ tokenExpiresAt?: number;
85
+ rootTokenFallback?: boolean;
86
+ cwd: string;
87
+ headless: boolean;
88
+ approvalMode: string;
89
+ label?: string;
90
+ rig?: string;
91
+ profile?: string;
92
+ agentProfile?: AgentProfile;
93
+ prompt?: string;
94
+ systemPromptAppend?: string;
95
+ relayInjectionEvents?: RunnerRelayInjectionEvent[];
96
+ tags: string[];
97
+ capabilities: string[];
98
+ providerArgs: string[];
99
+ policyName?: string;
100
+ spawnRequestId?: string;
101
+ taskId?: number;
102
+ automationId?: string; automationRunId?: string;
103
+ workspace?: WorkspaceMetadata; lifecycle?: AgentLifecycle;
104
+ tmuxSession?: string;
105
+ startedAt: number;
106
+ providerConfig: ProviderConfig;
107
+ adapter: ProviderAdapter;
108
+ onProviderExit?: (code: number | null) => void;
109
+ exitProcessOnShutdown?: boolean;
110
+ }
111
+
112
+ interface ActiveTaskClaim {
113
+ messageId: number;
114
+ taskId: number;
115
+ observedProviderBusy: boolean;
116
+ }
117
+
118
+ const CLAIM_RENEW_INTERVAL_MS = 5 * 60 * 1000;
119
+ const HTTP_LIVENESS_INTERVAL_MS = 20_000;
120
+ // #650: how often to reconcile pid-bearing busy work against actual process liveness.
121
+ const BUSY_LIVENESS_RECONCILE_MS = 5_000;
122
+ const HTTP_LIVENESS_LOG_INTERVAL_MS = 5 * 60 * 1000;
123
+ const TOKEN_RENEW_RETRY_MS = 60_000;
124
+ // Debounce reactive token recovery so a burst of 401-ing calls in the same window
125
+ // triggers a single re-mint attempt, not one per failing request.
126
+ const REACTIVE_TOKEN_RECOVERY_DEBOUNCE_MS = 10_000;
127
+ const UNEXPECTED_EXIT_WINDOW_MS = 2 * 60 * 1000;
128
+ const RAPID_EXIT_MS = 30 * 1000;
129
+ const MAX_RAPID_UNEXPECTED_EXITS = 3;
130
+ const UNDETERMINED_PROVIDER_EXIT_CONFIRM_MS = 1_500;
131
+ const PROVIDER_EXIT_CONFIRM_POLL_MS = 250;
132
+ // A UserPromptSubmit echo matching a runner-injected prompt within this window is
133
+ // the same prompt arriving back from the provider — drop it to avoid a duplicate.
134
+ const PROMPT_ECHO_DEDUP_MS = 30_000;
135
+ // Relay-injected content (delivered messages, memory context) is wrapped with
136
+ // these markers; a UserPromptSubmit echo starting with one is a runner injection,
137
+ // not a human typing into the terminal, so it must not be mirrored as a prompt.
138
+ // `<task-notification>` is the harness wrapper for async monitor/task wake-ups
139
+ // (which embed a `[relay message #…]` deeper inside, defeating a plain prefix
140
+ // match) — it is emitted only by the harness, never typed by a human, so any
141
+ // echo starting with it is a system injection, not a user prompt (#289).
142
+ const RELAY_INJECTION_MARKERS = ["[relay message #", "[agent-relay", "<task-notification>"];
143
+ // #329: a cold isolated profile (first-run trust/onboarding/install gates) can take far longer
144
+ // than the warm-restart window to render an input-ready TUI, so the first prompt delivery gets a
145
+ // generous timeout. If it still misses, the prompt is re-attempted on the next provider ready
146
+ // signal (short timeout, since it just went idle), up to a bounded number of attempts.
147
+ const INITIAL_PROMPT_FIRST_READY_TIMEOUT_MS = 30_000;
148
+ const INITIAL_PROMPT_RETRY_READY_TIMEOUT_MS = 10_000;
149
+ const MAX_INITIAL_PROMPT_ATTEMPTS = 6;
150
+ // Reasoning tailer poll cadence (item 5). Coarse on purpose — reasoning is a
151
+ // discreet progress signal, not a token stream; each tick reads only appended
152
+ // transcript bytes.
153
+ const REASONING_POLL_MS = 1_200;
154
+ const SESSION_MIRROR_BATCH_MAX = 100;
155
+ interface RunnerTimelineEvent {
156
+ status: string;
157
+ id?: string;
158
+ timestamp: number;
159
+ title?: string;
160
+ body?: string;
161
+ icon?: string;
162
+ metadata?: Record<string, unknown>;
163
+ }
164
+
165
+ interface ReasoningTailState {
166
+ timer: ReturnType<typeof setInterval>; emittedNarrationKeys: Set<string>;
167
+ poll(): Promise<void>;
168
+ }
169
+
170
+ export class AgentRunner {
171
+ private readonly agentId: string;
172
+ private readonly claims = new ClaimTracker();
173
+ private readonly http: RelayHttpClient;
174
+ private readonly bus: RelayBusClient;
175
+ // Phase 2 (#196): the Stop hook reads reply obligations from this local snapshot, never
176
+ // from the server — so a slow server can no longer wedge a turn (the crux fix).
177
+ private readonly obligationCache: ReplyObligationCache;
178
+ // Phase 2 (#196): Runner→server append-log events (session turns, reasoning, prompts,
179
+ // insights, hook-fatal) go through this durable, disk-backed, timestamped queue instead of
180
+ // direct fire-and-forget HTTP — so nothing is lost across a server/Runner restart.
181
+ private readonly outbox: Outbox;
182
+ // #332: a fast-lane outbox for display-only session-mirror trace events. Routing them through
183
+ // their own FIFO queue stops one transient trace failure from head-of-line blocking real-message
184
+ // delivery (and vice versa); its backoff is capped low since a stale live-mirror frame is cheap.
185
+ private readonly sessionOutbox: Outbox;
186
+ private sessionBatchSeq = 0;
187
+ private readonly insights: RunnerInsights;
188
+ private readonly busyReconciler: BusyReconciler;
189
+ private readonly claudeQuotaHarvest: ClaudeQuotaHarvest;
190
+ private currentToken?: string;
191
+ private currentTokenJti?: string;
192
+ private currentTokenProfileId?: string;
193
+ private currentTokenExpiresAt?: number;
194
+ private control?: ControlServer;
195
+ // Stage 2 (#215): the local MCP endpoint the agent connects to, fronting the relay so the
196
+ // Runner owns reconnect/backoff + a durable buffer. Disabled via AGENT_RELAY_MCP_PROXY=0
197
+ // (then the agent connects to the relay directly, Stage-1 behaviour). The proxy secret is the
198
+ // bearer the agent presents to the localhost proxy — it decouples the agent from the rotating
199
+ // relay token (the proxy injects the live token relay-side).
200
+ private proxy?: RelayMcpProxy;
201
+ private mcpProxyEndpoint?: string;
202
+ private readonly mcpProxyEnabled: boolean;
203
+ private readonly mcpProxySecret: string;
204
+ private process?: ManagedProcess;
205
+ private stopped = false;
206
+ // #633 — set when the provider terminally exits but the runner stays alive (Claude native
207
+ // auto-compaction / manual-resume hold). Rides on the offline status frame so the server fires a
208
+ // terminal `agent.exited` event with a #636 diagnosis instead of the death going silent. Cleared
209
+ // on any genuine restart back to busy/idle.
210
+ private terminalProviderExit?: Record<string, unknown>;
211
+ private nativeSelfResume = new NativeSelfResumeTracker();
212
+ private exitCommandInProgress = false;
213
+ private restartInProgress = false;
214
+ // Set for the whole unexpected-exit restart, including the pre-restart backoff
215
+ // sleep, so a second exit during that window can't launch a parallel restart.
216
+ private restartPending = false;
217
+ private delivering = false;
218
+ private drainTimer?: Timer;
219
+ private claimRenewTimer?: Timer;
220
+ private httpLivenessTimer?: Timer;
221
+ private livenessReconcileTimer?: Timer;
222
+ private httpLivenessInFlight = false;
223
+ private httpLivenessAuthFailed = false;
224
+ private httpLivenessLastLog?: { key: string; at: number };
225
+ private tokenRenewTimer?: Timer;
226
+ private tokenRenewInFlight = false;
227
+ private tokenRenewLastLog?: { key: string; at: number };
228
+ private reactiveTokenRecoveryAt?: number;
229
+ private processStartedAt = 0;
230
+ private providerSessionId = crypto.randomUUID();
231
+ // Last transcript path seen this session — used by end-of-session Insights (#184)
232
+ // when the SessionEnd hook payload omits it.
233
+ private lastTranscriptPath?: string;
234
+ private lifecycleAction?: LifecycleAction;
235
+ // Coalesces concurrent pre-session-destroy runs (e.g. the shutdown bus command and the
236
+ // SessionEnd hook both fire for the same teardown) so the cursor isn't raced.
237
+ private preDestroyPromise?: Promise<void>;
238
+ private readonly unexpectedExitTimes: number[] = [];
239
+ private readonly pendingMessages = new Map<number, Message>();
240
+ private readonly activeTaskClaims = new Map<number, ActiveTaskClaim>();
241
+ private pendingTimelineEvent?: RunnerTimelineEvent;
242
+ private lastProviderEventAt?: number;
243
+ private pendingPromptMessageId?: number;
244
+ // #329: a spawn-time initial prompt whose first delivery timed out before the TUI was
245
+ // ready. Held (not dropped) for a ready-signal-driven re-attempt; cleared on success.
246
+ private pendingInitialPrompt?: string;
247
+ private deliveringInitialPrompt = false;
248
+ private initialPromptAttempts = 0;
249
+ // Session-mirror: a synthesized id grouping a turn's reasoning/tool steps and
250
+ // its final response. Set when a provider-turn starts, cleared when it ends.
251
+ private currentTurnId?: string;
252
+ private currentTurnStartedAt?: number;
253
+ private completedProviderTurns = 0;
254
+ // #435: high-water-mark for the pre-turn narrative flush. Counts how many
255
+ // valid JSONL entries were in the transcript at last pre-flush time so the
256
+ // Stop-hook capture (full mode) can skip what was already emitted.
257
+ private narrativeFlushEntryCount = 0;
258
+ // True while a turn that was already in flight is being compacted. Claude's PostCompact
259
+ // hook posts a single `idle`, but a mid-turn compaction RESUMES the turn afterward — so
260
+ // treating that idle as a turn end (flip idle, kill the reasoning tail) makes chat go dark
261
+ // until the next prompt. Set when `compacting` arrives with a turn running; cleared on the
262
+ // genuine end (a plain idle with no compaction timeline).
263
+ private compactionMidTurn = false;
264
+ // Prompt-echo dedup: a short, time-bounded queue of prompts the runner itself
265
+ // injected (chat box or initial prompt) that are still awaiting their matching
266
+ // UserPromptSubmit echo. A single slot dropped earlier entries when several prompts
267
+ // were injected before their echoes returned (rapid sends while the provider is busy
268
+ // and queues them) — the evicted ones then double-posted. Match consumes one entry.
269
+ private injectedPrompts: Array<{ text: string; at: number }> = [];
270
+ // Verbose session-mirror diagnostics (turn lifecycle, reconciler probes, tail
271
+ // emits) → a dedicated clean log. Always on for key transitions; AGENT_RELAY_SESSION_DEBUG=1 adds the high-frequency probe/emit lines.
272
+ private readonly sessionDebugVerbose = sessionDebugEnabled();
273
+ // Tracks whether the provider is in a legitimate blocked/approval state, so the
274
+ // busy reconciler doesn't mistake a permission prompt for a stuck-busy turn.
275
+ private providerBlocked = false;
276
+ private terminalFailure?: { reason: string; message: string; providerState?: ProviderState };
277
+ // #286: a usage/rate-limit hold. The turn truly ended (idle), but the agent is
278
+ // held until the limit resets, so this can't ride an active-work claim like the
279
+ // permission-blocked state does. publishStatus surfaces it as the agent's
280
+ // providerState; it clears when a new turn starts (the relay's resume message
281
+ // wakes one) so the blocked badge lifts on its own.
282
+ private rateLimitHold?: ProviderState;
283
+ private pendingRateLimitHold?: ProviderState;
284
+ // Reasoning tailer (item 5): streams the in-flight turn's reasoning/tool steps
285
+ // from the Claude transcript into chat as discreet session events.
286
+ private reasoningTail?: ReasoningTailState;
287
+ private scratch?: SessionScratchLayout;
288
+ // #417: for non-claude providers, tracks how many times each spawner obligation has
289
+ // been re-injected so a non-responsive agent can't spin in a re-injection loop.
290
+ private readonly pendingObligationInjections = new Map<number, number>();
291
+
292
+ constructor(private readonly options: RunnerOptions) {
293
+ this.agentId = options.agentId ?? options.runnerId;
294
+ // Bind the process-global logger to this agent. AGENT_RELAY_SESSION_DEBUG=1 is
295
+ // kept as a back-compat alias for the verbose probe/emit lines, now expressed
296
+ // as log level "debug" (AGENT_RELAY_LOG_LEVEL still wins when both are set).
297
+ logger.configure({
298
+ agentId: this.agentId,
299
+ headless: options.headless,
300
+ ...(this.sessionDebugVerbose && !parseLogLevel(logLevelFromEnv()) ? { level: "debug" as const } : {}),
301
+ });
302
+ this.currentToken = options.token;
303
+ this.currentTokenJti = options.tokenJti;
304
+ this.currentTokenProfileId = options.tokenProfileId;
305
+ this.currentTokenExpiresAt = options.tokenExpiresAt;
306
+ this.mcpProxyEnabled = mcpProxyEnabledFromEnv();
307
+ this.mcpProxySecret = crypto.randomUUID();
308
+ const runtime = runtimeMetadata(options.provider);
309
+ this.http = new RelayHttpClient({ baseUrl: options.relayUrl, token: this.currentToken });
310
+ this.claudeQuotaHarvest = new ClaudeQuotaHarvest({ agentId: this.agentId, provider: options.provider, http: () => this.http });
311
+ this.obligationCache = new ReplyObligationCache({ fetch: () => this.http.listReplyObligations(this.agentId) });
312
+ // Co-locate the durable outbox with the runner's runtime state (survives reboot) when the
313
+ // orchestrator told us where that is; otherwise the Outbox falls back to a temp dir.
314
+ const outboxDir = runnerOutboxDirWithInfoFallback();
315
+ this.outbox = new Outbox({ agentId: this.agentId, dir: outboxDir, send: (record) => this.deliverOutboxEvent(record) });
316
+ this.sessionOutbox = new Outbox({
317
+ agentId: `${this.agentId}-session`,
318
+ dir: outboxDir,
319
+ send: (record) => this.deliverOutboxEvent(record),
320
+ // Ephemeral trace frames: cap the per-row stall low and poison sooner than the main queue.
321
+ maxBackoffMs: 5_000,
322
+ maxAttempts: 6,
323
+ });
324
+ this.insights = new RunnerInsights({
325
+ agentId: this.agentId,
326
+ cwd: options.cwd,
327
+ workspace: options.workspace,
328
+ getOutbox: () => this.outbox,
329
+ getProviderSessionId: () => this.providerSessionId,
330
+ getProcess: () => this.process,
331
+ getAdapter: () => this.options.adapter,
332
+ getLastTranscriptPath: () => this.lastTranscriptPath,
333
+ sessionLog: (message) => this.sessionLog(message),
334
+ });
335
+ this.busyReconciler = new BusyReconciler({
336
+ isStopped: () => this.stopped,
337
+ hasProcess: () => Boolean(this.process),
338
+ currentStatus: () => this.claims.currentStatus(),
339
+ isProviderBlocked: () => this.providerBlocked,
340
+ hasProviderTurn: () => this.claims.activeWork().some((w) => w.kind === "provider-turn"),
341
+ canProbeActivity: () => Boolean(this.options.adapter.probeActivity),
342
+ probeActivity: () => this.process && this.options.adapter.probeActivity
343
+ ? this.options.adapter.probeActivity(this.process)
344
+ : undefined,
345
+ probePaneBusy: () => this.process && this.options.adapter.probePaneBusy
346
+ ? this.options.adapter.probePaneBusy(this.process)
347
+ : undefined,
348
+ clearProviderTurn: (reason) => this.forceClearProviderTurn(reason),
349
+ sessionDebug: (message) => this.sessionDebug(message),
350
+ });
351
+ this.bus = new RelayBusClient({
352
+ url: relayBusUrl(options.relayUrl),
353
+ role: "provider",
354
+ componentId: options.runnerId,
355
+ agentId: this.agentId,
356
+ instanceId: options.instanceId,
357
+ token: this.currentToken,
358
+ machine: hostname(),
359
+ capabilities: [
360
+ ...new Set([
361
+ ...options.providerConfig.defaultCapabilities,
362
+ ...options.capabilities,
363
+ ...csvTags(capsFromEnv()),
364
+ "lifecycle.shutdown.hard",
365
+ "lifecycle.restart.hard",
366
+ "lifecycle.status.semantic",
367
+ "lifecycle.heartbeat",
368
+ "lifecycle.reconnect.transport",
369
+ "commands.lifecycle",
370
+ "capabilities.report",
371
+ ]),
372
+ ],
373
+ tags: [...new Set([options.provider, ...csvTags(tagsFromEnv()), ...options.tags, ...options.providerConfig.defaultTags, ...(options.headless ? ["headless"] : [])])],
374
+ meta: {
375
+ ...runtime,
376
+ version: runtime.package.version,
377
+ provider: options.provider,
378
+ model: options.model ?? null,
379
+ effort: options.effort ?? null,
380
+ profile: options.profile ?? null,
381
+ // #557 — pass a spawn-shaped config so the projection describes the SAME assembled
382
+ // launch the adapter builds (env/controlPort are placeholders the pure assembler ignores).
383
+ agentProfile: options.agentProfile ? appliedAgentProfileMetadata(options.provider, options.agentProfile, { ...options, agentId: this.agentId, env: {}, controlPort: 0 } as RunnerSpawnConfig, options.providerConfig) : null,
384
+ runnerId: options.runnerId,
385
+ startedAt: options.startedAt,
386
+ tmuxSession: providerTerminalSession(this.process) ?? options.tmuxSession ?? null,
387
+ tmuxSocket: providerTerminalSocket(this.process) ?? null,
388
+ policyName: options.policyName ?? null,
389
+ spawnRequestId: options.spawnRequestId ?? null,
390
+ taskId: options.taskId ?? null,
391
+ automationId: options.automationId ?? null,
392
+ automationRunId: options.automationRunId ?? null,
393
+ workspace: options.workspace ?? null,
394
+ lifecycle: options.lifecycle ?? "persistent",
395
+ workspaceMode: options.workspace?.requestedMode ?? options.workspace?.mode ?? workspaceModeFromEnv() ?? null,
396
+ runnerManaged: true,
397
+ cwd: options.cwd,
398
+ approvalMode: options.approvalMode,
399
+ label: options.label ?? null,
400
+ auth: {
401
+ profileId: this.currentTokenProfileId ?? null,
402
+ jti: this.currentTokenJti ?? null,
403
+ expiresAt: this.currentTokenExpiresAt ?? null,
404
+ rootTokenFallback: options.rootTokenFallback === true,
405
+ },
406
+ lifecycleCapabilities: lifecycleCapabilities(),
407
+ providerCapabilities: runtimeProviderCapabilities(options),
408
+ },
409
+ heartbeatMeta: () => this.heartbeatRuntimeMeta(),
410
+ heartbeatIntervalMs: 30_000,
411
+ initialStatus: "idle",
412
+ });
413
+ }
414
+
415
+ get id(): string {
416
+ return this.agentId;
417
+ }
418
+
419
+ async run(): Promise<void> {
420
+ this.control = startControlServer({
421
+ onStatus: (status) => this.setProviderStatus(status),
422
+ onTerminalAttachSpec: () => this.terminalAttachSpec(),
423
+ // Hot-path-safe: answered instantly from the local snapshot, never a server
424
+ // round-trip. The snapshot is kept warm by the background refresh below (#196).
425
+ onReplyObligations: () => Promise.resolve(this.obligationCache.get()),
426
+ onSessionTurn: (input) => this.publishSessionTurn(input), onPermissionResolved: (input) => this.publishSessionEvent({ from: this.agentId, to: "user", body: input.body, session: { type: "prompt-resolution", origin: "provider", label: input.decisionLabel, ...(this.currentTurnId ? { turnId: this.currentTurnId } : {}), stepId: `prompt-resolution:${input.approvalId}`, promptResolution: input } }),
427
+ onUserPrompt: (input) => this.handleUserPrompt(input),
428
+ onSessionBoundary: (input) => this.handleSessionBoundary(input),
429
+ onHookFatal: (report) => this.reportHookFatal(report),
430
+ });
431
+ this.startMcpProxy();
432
+ this.writeRunnerInfoFile();
433
+ this.options.adapter.onStatusChange((status) => {
434
+ if (this.restartInProgress || this.restartPending) return;
435
+ const semanticStatus = typeof status === "string" ? status : status.status;
436
+ if (this.shouldRestartUnexpectedProviderExit(semanticStatus)) {
437
+ void this.restartUnexpectedProviderExit(semanticStatus);
438
+ return;
439
+ }
440
+ this.setProviderStatus(status);
441
+ if (runnerShouldResolveProviderExit(semanticStatus, this.exitCommandInProgress)) this.options.onProviderExit?.(semanticStatus === "offline" ? 0 : 1);
442
+ });
443
+ this.options.adapter.onSessionEvent?.((event) => { void this.publishProviderSessionEvent(event); });
444
+ this.bus.on("message.new", (message) => {
445
+ // A delivered message may create a new reply obligation — warm the snapshot so the
446
+ // next turn-end sees it without a hot-path server read.
447
+ const msg = message as Message;
448
+ if (runnerMessageShouldDirtyObligations(msg, this.agentId)) this.obligationCache.markDirty();
449
+ this.enqueueMessage(msg);
450
+ });
451
+ this.bus.on("command", (type, params, commandId, command) => {
452
+ void this.handleCommand(type, params, commandId, command);
453
+ });
454
+ this.bus.on("error", (code, message) => this.handleBusError(String(code), String(message)));
455
+ await registerWithinDeadline(this.bus, registrationTimeoutMsFromEnv(), this.options.relayUrl, (reason) => logger.fatal("register", reason)); await initializeRunnerTemplates(this.http);
456
+ this.obligationCache.start();
457
+ this.outbox.start();
458
+ this.sessionOutbox.start();
459
+ this.ensureScratch();
460
+ void this.sweepStaleScratch();
461
+ this.process = await this.spawnProvider();
462
+ this.writeRunnerInfoFile();
463
+ this.processStartedAt = Date.now();
464
+ this.publishStatus();
465
+ await this.deliverInitialPrompt();
466
+ await this.bootstrapUnreadMessages();
467
+ this.startClaimRenewer();
468
+ this.startHttpLiveness();
469
+ this.scheduleRuntimeTokenRenewal();
470
+ this.scheduleDrain();
471
+ }
472
+
473
+ async stop(): Promise<void> {
474
+ const alreadyStopped = this.stopped;
475
+ this.stopped = true;
476
+ const launcherScript = typeof this.process?.meta?.launcherScript === "string" ? this.process.meta.launcherScript : undefined; const tmuxSession = typeof this.process?.meta?.tmuxSession === "string" ? this.process.meta.tmuxSession : undefined;
477
+ reapSessionScratch({ agentId: this.agentId, cwd: this.options.cwd, fallbackBaseDir: orchestratorBaseDirFromEnv() });
478
+ reapSessionLauncher({ launcherScript, tmuxSession });
479
+ if (!alreadyStopped) await this.bus.statusAsync({ agentStatus: "offline", ready: false });
480
+ if (this.process && !alreadyStopped) {
481
+ await this.options.adapter.shutdown(this.process, {
482
+ graceful: true,
483
+ timeoutMs: this.options.providerConfig.headless.shutdownTimeoutMs,
484
+ });
485
+ }
486
+ if (this.drainTimer) clearTimeout(this.drainTimer);
487
+ this.drainTimer = undefined;
488
+ if (this.claimRenewTimer) clearInterval(this.claimRenewTimer);
489
+ this.claimRenewTimer = undefined;
490
+ if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
491
+ this.httpLivenessTimer = undefined;
492
+ if (this.tokenRenewTimer) clearTimeout(this.tokenRenewTimer);
493
+ this.tokenRenewTimer = undefined;
494
+ this.disarmBusyLivenessReconcile();
495
+ this.busyReconciler.disarm();
496
+ this.stopReasoningTail();
497
+ this.obligationCache.stop();
498
+ this.outbox.close(); this.sessionOutbox.close();
499
+ this.proxy?.stop();
500
+ this.control?.stop();
501
+ await this.bus.close();
502
+ }
503
+
504
+ // Start the local MCP proxy the agent connects to (Stage 2, #215). Forwards tool calls to the
505
+ // relay with the runner's LIVE token, buffers bufferable writes durably during a relay outage,
506
+ // and narrows the tool list to this agent's workspace context. Best-effort: if it can't bind,
507
+ // we fall back to a direct relay MCP connection (the agent env still works, no resilience).
508
+ private startMcpProxy(): void {
509
+ if (!this.mcpProxyEnabled) return;
510
+ try {
511
+ this.proxy = new RelayMcpProxy({
512
+ relayMcpEndpoint: relayMcpEndpoint(this.options.relayUrl),
513
+ getToken: () => this.currentToken,
514
+ authSecret: this.mcpProxySecret,
515
+ enqueueBuffered: (call) => {
516
+ this.outbox.enqueue({
517
+ kind: "mcp-tool-call",
518
+ payload: { tool: call.tool, arguments: call.arguments },
519
+ idempotencyKey: call.idempotencyKey,
520
+ });
521
+ },
522
+ initialContext: { isolatedWorktree: this.ownsIsolatedWorktree() },
523
+ });
524
+ this.mcpProxyEndpoint = this.proxy.start().url;
525
+ logger.info("mcp-proxy", `runner MCP proxy listening at ${this.mcpProxyEndpoint} (worktree=${this.ownsIsolatedWorktree()})`);
526
+ } catch (error) {
527
+ this.proxy = undefined;
528
+ this.mcpProxyEndpoint = undefined;
529
+ logger.warn("mcp-proxy", `failed to start MCP proxy; agent will connect to the relay directly: ${errMessage(error)}`);
530
+ }
531
+ }
532
+
533
+ private ownsIsolatedWorktree(): boolean {
534
+ const mode = this.options.workspace?.requestedMode ?? this.options.workspace?.mode ?? workspaceModeFromEnv();
535
+ return mode === "isolated";
536
+ }
537
+
538
+ private async spawnProvider(opts: { resumeId?: string; suppressPrompt?: boolean } = {}): Promise<ManagedProcess> {
539
+ this.providerSessionId = crypto.randomUUID();
540
+ this.lastTranscriptPath = undefined;
541
+ const includeProviderGlobals = profileUsesProviderHostGlobals(this.options);
542
+ const env = {
543
+ ...process.env as Record<string, string>,
544
+ ...(includeProviderGlobals ? this.options.providerConfig.env : {}),
545
+ AGENT_RELAY_RUNNER_PORT: String(this.control!.port),
546
+ AGENT_RELAY_RUNNER_ID: this.options.runnerId,
547
+ AGENT_RELAY_PROVIDER_SESSION_ID: this.providerSessionId,
548
+ AGENT_RELAY_ID: this.agentId,
549
+ // The repo-name project id the runner uses for insight observations. Exposed so
550
+ // agent-side signals (e.g. /introspect) tag the SAME project and aggregate together,
551
+ // instead of the agent's worktree cwd splitting one repo into many "projects".
552
+ AGENT_RELAY_PROJECT: this.insightProject(),
553
+ ...(this.scratch ? { AGENT_RELAY_SCRATCH_DIR: this.scratch.tmpDir } : {}),
554
+ AGENT_RELAY_URL: this.options.relayUrl,
555
+ AGENT_RELAY_APPROVAL: this.options.approvalMode,
556
+ ...(this.currentToken ? { AGENT_RELAY_TOKEN: this.currentToken } : {}),
557
+ // Dedicated, un-clobberable credential for the injected MCP endpoint. A rig's
558
+ // settings.json `env.AGENT_RELAY_TOKEN` would override the scoped token above at
559
+ // MCP-parse time → server-actor auth, no identity (#233). The MCP config references
560
+ // ${AGENT_RELAY_SESSION_TOKEN}, which rigs never set. See runner/src/relay-mcp.ts.
561
+ //
562
+ // Stage 2 (#215): when the proxy is active the agent connects to the LOCAL proxy, so this
563
+ // holds the per-session PROXY SECRET (not the relay token). The proxy injects the live
564
+ // relay token itself — the agent never holds it, and token rotation is invisible. With the
565
+ // proxy disabled this stays the scoped relay token (Stage-1 direct connection).
566
+ ...(this.proxy
567
+ ? { [RELAY_MCP_TOKEN_ENV]: this.mcpProxySecret }
568
+ : (this.currentToken ? { [RELAY_MCP_TOKEN_ENV]: this.currentToken } : {})),
569
+ ...(this.currentTokenJti ? { AGENT_RELAY_TOKEN_JTI: this.currentTokenJti } : {}),
570
+ ...(this.currentTokenProfileId ? { AGENT_RELAY_TOKEN_PROFILE: this.currentTokenProfileId } : {}),
571
+ ...(this.currentTokenExpiresAt ? { AGENT_RELAY_TOKEN_EXPIRES_AT: String(this.currentTokenExpiresAt) } : {}),
572
+ };
573
+ const config: RunnerSpawnConfig = {
574
+ provider: this.options.provider,
575
+ model: this.options.model,
576
+ effort: this.options.effort,
577
+ runnerId: this.options.runnerId,
578
+ instanceId: this.options.instanceId,
579
+ agentId: this.agentId,
580
+ relayUrl: this.options.relayUrl,
581
+ cwd: this.options.cwd,
582
+ headless: this.options.headless,
583
+ approvalMode: this.options.approvalMode,
584
+ ...(this.options.label ? { label: this.options.label } : {}),
585
+ ...(this.options.rig ? { rig: this.options.rig } : {}),
586
+ ...(this.options.profile ? { profile: this.options.profile } : {}),
587
+ ...(this.options.agentProfile ? { agentProfile: this.options.agentProfile } : {}),
588
+ ...(this.options.prompt && !opts.resumeId && !opts.suppressPrompt ? { prompt: this.options.prompt } : {}),
589
+ ...(this.options.systemPromptAppend ? { systemPromptAppend: this.options.systemPromptAppend } : {}),
590
+ ...(this.options.tmuxSession ? { tmuxSession: this.options.tmuxSession } : {}),
591
+ providerArgs: opts.resumeId ? [...this.options.providerArgs, "--resume", opts.resumeId] : this.options.providerArgs,
592
+ providerConfig: this.options.providerConfig,
593
+ env,
594
+ controlPort: this.control!.port,
595
+ // Stage 2 (#215): the MCP endpoint the agent's client should target — the runner-local
596
+ // proxy when active, undefined when disabled (adapters fall back to the direct relay URL).
597
+ ...(this.mcpProxyEndpoint ? { relayMcpEndpoint: this.mcpProxyEndpoint } : {}),
598
+ // #672: the shared host-listener URL the `callmux bridge` connects to (env → default).
599
+ // Always resolved; the assembler only emits the bridge when `agentProfile.sharedMcp` is on.
600
+ sharedMcpUrl: resolveSharedMcpUrl(),
601
+ monitor: {
602
+ deliver: (messages) => this.control!.deliverToMonitor(messages),
603
+ },
604
+ };
605
+ const launchSeededPrompt = this.options.adapter.seedsInitialPromptAtLaunch && !opts.resumeId && !opts.suppressPrompt ? this.options.prompt?.trim() : ""; if (launchSeededPrompt) this.recordInjectedPrompt(launchSeededPrompt);
606
+ const managedProcess = await this.options.adapter.spawn(config);
607
+ const injectionEvents = runnerLaunchInjectionSessionEvents({ carried: this.options.relayInjectionEvents, provider: this.options.provider, config, providerConfig: this.options.providerConfig, agentId: this.agentId, providerSessionId: this.providerSessionId });
608
+ for (const event of injectionEvents) {
609
+ this.publishSessionEvent(event);
610
+ }
611
+ return managedProcess;
612
+ }
613
+
614
+ private async terminalAttachSpec(): Promise<TerminalAttachSpec> {
615
+ if (!this.process) throw new Error("provider process is unavailable");
616
+ if (!this.options.adapter.terminalAttachSpec) throw new Error("provider does not support terminal attach");
617
+ return this.options.adapter.terminalAttachSpec(this.process);
618
+ }
619
+
620
+ private writeRunnerInfoFile(): void {
621
+ const file = runnerInfoFileFromEnv();
622
+ if (!file || !this.control) return;
623
+ try {
624
+ mkdirSync(dirname(file), { recursive: true });
625
+ writeFileSync(file, JSON.stringify({
626
+ agentId: this.agentId,
627
+ runnerId: this.options.runnerId,
628
+ provider: this.options.provider,
629
+ controlUrl: this.control.url,
630
+ pid: process.pid,
631
+ ...(providerTerminalSession(this.process) ? { tmuxSession: providerTerminalSession(this.process) } : {}),
632
+ ...(providerTerminalSocket(this.process) ? { tmuxSocket: providerTerminalSocket(this.process) } : {}),
633
+ startedAt: this.options.startedAt,
634
+ }, null, 2) + "\n", { mode: 0o600 });
635
+ } catch (error) {
636
+ logger.error("runner", `failed to write runner info file: ${error}`);
637
+ }
638
+ }
639
+
640
+ private enqueueMessage(message: Message): void {
641
+ if (!this.matchesMessage(message)) return;
642
+ if (message.kind === "session") return;
643
+ this.pendingMessages.set(message.id, message);
644
+ this.scheduleDrain();
645
+ }
646
+
647
+ private async bootstrapUnreadMessages(): Promise<void> {
648
+ try {
649
+ const messages = await this.http.pollMessages({ for: this.agentId, unread: true, limit: 100 });
650
+ for (const message of messages) this.enqueueMessage(message);
651
+ } catch (error) {
652
+ logger.error("runner", `inbox bootstrap failed: ${error}`);
653
+ }
654
+ }
655
+
656
+ private async deliverInitialPrompt(): Promise<void> {
657
+ const prompt = this.options.prompt?.trim();
658
+ if (!prompt || this.options.adapter.seedsInitialPromptAtLaunch) return;
659
+ await this.attemptInitialPromptDelivery(prompt, INITIAL_PROMPT_FIRST_READY_TIMEOUT_MS);
660
+ }
661
+
662
+ // Deliver the spawn-time first prompt, surviving a cold-start TUI that isn't input-ready yet
663
+ // (#329). Timeout stashes it for ready-signal retry and surfaces the stall on the timeline.
664
+ private async attemptInitialPromptDelivery(prompt: string, readyTimeoutMs: number): Promise<void> {
665
+ if (this.deliveringInitialPrompt || this.stopped || !this.process || !this.options.adapter.deliverInitialPrompt) return;
666
+ this.deliveringInitialPrompt = true;
667
+ const attempt = (this.initialPromptAttempts += 1);
668
+ try {
669
+ this.recordInjectedPrompt(prompt); await this.options.adapter.deliverInitialPrompt(this.process, prompt, { readyTimeoutMs });
670
+ this.pendingInitialPrompt = undefined;
671
+ } catch (error) {
672
+ const giveUp = attempt >= MAX_INITIAL_PROMPT_ATTEMPTS;
673
+ this.pendingInitialPrompt = giveUp ? undefined : prompt;
674
+ logger.error("runner", `initial prompt attempt ${attempt} failed${giveUp ? " (giving up)" : "; will retry on next ready signal"}: ${errMessage(error)}`);
675
+ this.publishRunnerTimelineEvent({ status: "prompt_failed", timestamp: Date.now(), title: "Initial prompt not delivered", body: errMessage(error), metadata: { attempts: attempt, terminal: giveUp } });
676
+ } finally {
677
+ this.deliveringInitialPrompt = false;
678
+ }
679
+ }
680
+
681
+ private scheduleDrain(delayMs = 0): void {
682
+ if (this.stopped || this.drainTimer) return;
683
+ this.drainTimer = setTimeout(() => {
684
+ this.drainTimer = undefined;
685
+ void this.drainMessages();
686
+ }, delayMs);
687
+ }
688
+
689
+ private async drainMessages(): Promise<void> {
690
+ if (this.stopped || this.delivering || this.pendingMessages.size === 0) return;
691
+ if (!this.process) {
692
+ this.scheduleDrain(500);
693
+ return;
694
+ }
695
+ this.delivering = true;
696
+ const messages = [...this.pendingMessages.values()].sort((a, b) => a.id - b.id);
697
+ this.pendingMessages.clear();
698
+ const deliverable: Message[] = [];
699
+ const providerAlreadyBusy = this.claims.reasons().includes("provider-turn");
700
+ for (const message of messages) {
701
+ let toDeliver = message;
702
+ if (message.claimable) {
703
+ const claimed = await this.http.claimMessageResult(message.id, this.agentId).catch(() => ({ ok: false, claimExpiresAt: undefined }));
704
+ if (!claimed.ok) continue;
705
+ this.claims.startClaim("message", String(message.id), claimed.claimExpiresAt);
706
+ const taskId = taskIdFromMessage(message);
707
+ if (taskId) {
708
+ this.activeTaskClaims.set(message.id, { messageId: message.id, taskId, observedProviderBusy: providerAlreadyBusy });
709
+ this.claims.startClaim("task", String(taskId), claimed.claimExpiresAt);
710
+ await this.updateTaskStatus(taskId, {
711
+ status: "in_progress",
712
+ agentId: this.agentId,
713
+ metadata: { messageId: message.id, completedBy: "runner" },
714
+ }).catch((error) => logger.error("task", `task ${taskId} in_progress update failed: ${error}`));
715
+ // Runner owns claim + status here; drop the server's self-claim instruction
716
+ // so the agent doesn't improvise a stray claim send (see stripRunnerClaimedGuidance).
717
+ toDeliver = { ...message, body: stripRunnerClaimedGuidance(message.body) };
718
+ }
719
+ }
720
+ deliverable.push(toDeliver);
721
+ }
722
+ if (deliverable.length === 0) {
723
+ this.delivering = false;
724
+ this.publishStatus();
725
+ return;
726
+ }
727
+ this.publishStatus();
728
+ let failed = false;
729
+ try {
730
+ const prepared = await messagesWithCachedAttachments(deliverable, this.http, {
731
+ agentId: this.agentId,
732
+ onError: (message) => logger.error("runner", message),
733
+ });
734
+ await this.options.adapter.deliver(this.process, prepared);
735
+ for (const message of deliverable) {
736
+ await this.http.recordMessageDeliveryAttempt(message.id, { agentId: this.agentId, status: "delivered" }).catch(() => {});
737
+ await this.http.markRead(message.id, this.agentId).catch(() => {});
738
+ if (!taskIdFromMessage(message)) this.claims.finishClaim("message", String(message.id));
739
+ }
740
+ } catch (error) {
741
+ failed = true;
742
+ if (shouldLogDeliveryFailure(error)) logger.warn("delivery", `message delivery failed: ${error}`);
743
+ for (const message of deliverable) {
744
+ this.clearActiveClaim(message);
745
+ this.pendingMessages.set(message.id, message);
746
+ }
747
+ } finally {
748
+ this.delivering = false;
749
+ this.publishStatus();
750
+ if (this.pendingMessages.size > 0) this.scheduleDrain(failed ? 1000 : 0);
751
+ }
752
+ }
753
+
754
+ private async handleCommand(type: string, params: Record<string, unknown>, commandId: string, command?: Record<string, unknown>): Promise<void> {
755
+ const target = typeof command?.target === "string" ? command.target : this.agentId;
756
+ if (target !== this.agentId && target !== this.options.runnerId) return;
757
+ if (type !== "agent.shutdown" && type !== "agent.restart" && type !== "agent.reconnect" && type !== "agent.kill" && type !== "agent.compact" && type !== "agent.clearContext" && type !== "agent.injectContext" && type !== "agent.permissionDecision" && type !== "agent.interrupt" && type !== "prompt.inject") return;
758
+
759
+ const exitAfterCommand = type === "agent.shutdown" || type === "agent.kill";
760
+ if (exitAfterCommand) this.exitCommandInProgress = true;
761
+ const nativeSelfResume = isNativeSelfResumeCompact(type, params)
762
+ ? this.nativeSelfResume.begin(commandId, params)
763
+ : undefined;
764
+ this.claims.startClaim("command", commandId);
765
+ try {
766
+ await this.updateCommand(commandId, "accepted");
767
+ await this.updateCommand(commandId, "running");
768
+ // Pre-session-destroy seam (#183): for destructive transitions, run end-of-session
769
+ // work (Insights capture, #184) BEFORE the invasive operation, surfaced as a
770
+ // non-addressable "finalizing" state so the agent isn't mistaken for merely busy.
771
+ const destroyReason = boundaryReasonForCommand(type);
772
+ if (destroyReason) await this.runPreSessionDestroy(destroyReason);
773
+ // Move from the transient finalizing window to the executing teardown state (or drop
774
+ // it entirely for compact/clear, which complete promptly once capture is done).
775
+ if (exitAfterCommand) this.lifecycleAction = type === "agent.kill" ? "killing" : "shutting-down";
776
+ else if (type === "agent.restart") this.lifecycleAction = "restarting";
777
+ else this.lifecycleAction = undefined;
778
+ this.publishStatus();
779
+ let providerResult: Record<string, unknown> | void = undefined;
780
+ if (type === "agent.restart") await this.restartProvider();
781
+ else if (type === "agent.reconnect") this.publishStatus();
782
+ else if (type === "agent.compact") {
783
+ if (!this.options.adapter.compact || !this.process) throw new Error("provider does not support compact");
784
+ providerResult = await this.options.adapter.compact(this.process, {
785
+ instructions: typeof params.instructions === "string" ? params.instructions : undefined,
786
+ });
787
+ if (nativeSelfResume) {
788
+ const handoff = await nativeSelfResume.promise;
789
+ providerResult = { ...(providerResult ?? {}), nativeSelfResume: handoff };
790
+ }
791
+ } else if (type === "agent.clearContext") {
792
+ if (!this.options.adapter.clearContext || !this.process) throw new Error("provider does not support clearContext");
793
+ providerResult = await this.options.adapter.clearContext(this.process);
794
+ } else if (type === "agent.interrupt") {
795
+ if (!this.options.adapter.interrupt || !this.process) throw new Error("provider does not support interrupt");
796
+ this.sessionLog("interrupt requested from dashboard");
797
+ providerResult = await this.options.adapter.interrupt(this.process);
798
+ this.busyReconciler.scheduleInterruptReconcile();
799
+ } else if (type === "agent.injectContext") {
800
+ if (!this.process) throw new Error("provider process is unavailable");
801
+ providerResult = await this.injectContext(params);
802
+ } else if (type === "agent.permissionDecision") {
803
+ providerResult = await this.respondToPermissionDecision(params);
804
+ } else if (type === "prompt.inject") {
805
+ providerResult = await this.injectPrompt(params);
806
+ } else await this.shutdownProvider(type === "agent.kill", commandTimeoutMs(params));
807
+ await this.updateCommand(commandId, "succeeded", {
808
+ action: type,
809
+ agentId: this.agentId,
810
+ runnerId: this.options.runnerId,
811
+ policyName: this.options.policyName,
812
+ spawnRequestId: this.options.spawnRequestId,
813
+ reason: typeof params.reason === "string" ? params.reason : undefined,
814
+ ...(providerResult ? { providerResult } : {}),
815
+ });
816
+ } catch (error) {
817
+ if (nativeSelfResume) this.nativeSelfResume.finish(nativeSelfResume);
818
+ await this.updateCommand(commandId, "failed", undefined, errMessage(error)).catch(() => {});
819
+ } finally {
820
+ this.claims.finishClaim("command", commandId);
821
+ if (exitAfterCommand) {
822
+ if (params.preserveRegistration === true) {
823
+ await this.http.setStatus(this.agentId, "offline", this.options.instanceId).catch(() => {});
824
+ } else {
825
+ await this.http.deleteAgent(this.agentId).catch(() => {});
826
+ }
827
+ if (this.options.exitProcessOnShutdown !== false) {
828
+ setTimeout(() => void this.stop().catch((error) => {
829
+ logger.error("lifecycle", `stop after command failed: ${error}`);
830
+ }).finally(() => process.exit(0)), 10);
831
+ }
832
+ } else if (!this.stopped) {
833
+ if (nativeSelfResume) this.nativeSelfResume.finish(nativeSelfResume);
834
+ this.lifecycleAction = undefined;
835
+ this.publishStatus();
836
+ }
837
+ }
838
+ }
839
+
840
+ private async injectContext(params: Record<string, unknown>): Promise<Record<string, unknown>> {
841
+ const content = typeof params.content === "string" ? params.content.trim() : "";
842
+ if (!content) throw new Error("content required");
843
+ const memoryIds = Array.isArray(params.memoryIds) ? params.memoryIds.filter((id): id is string => typeof id === "string") : [];
844
+ await this.options.adapter.deliver(this.process!, [{
845
+ id: 0,
846
+ from: "system",
847
+ to: this.agentId,
848
+ kind: "system",
849
+ subject: "Agent Relay memory context",
850
+ body: content,
851
+ payload: {
852
+ memoryInjection: true,
853
+ reason: typeof params.reason === "string" ? params.reason : undefined,
854
+ memoryIds,
855
+ },
856
+ readBy: [],
857
+ createdAt: Date.now(),
858
+ }]);
859
+ return { memoryIds, injectedMemoryCount: memoryIds.length };
860
+ }
861
+
862
+ private async respondToPermissionDecision(params: Record<string, unknown>): Promise<Record<string, unknown>> {
863
+ const approvalId = typeof params.approvalId === "string" ? params.approvalId : "";
864
+ const decision = typeof params.decision === "string" ? params.decision : "";
865
+ if (!approvalId) throw new Error("approvalId required");
866
+ if (decision !== "approve" && decision !== "approve-session" && decision !== "deny" && decision !== "abort" && decision !== "answer") {
867
+ throw new Error("decision must be approve, approve-session, deny, abort, or answer");
868
+ }
869
+ const answersRaw = params.answers;
870
+ const answers = answersRaw && typeof answersRaw === "object" && !Array.isArray(answersRaw)
871
+ ? Object.fromEntries(
872
+ Object.entries(answersRaw as Record<string, unknown>).filter(([, v]) => typeof v === "string") as [string, string][],
873
+ )
874
+ : undefined;
875
+ if (decision === "answer" && (!answers || Object.keys(answers).length === 0)) {
876
+ throw new Error("answers required for answer decision");
877
+ }
878
+ const input: ProviderPermissionDecisionInput = {
879
+ approvalId,
880
+ decision: decision as ProviderPermissionDecision,
881
+ ...(typeof params.reason === "string" ? { reason: params.reason } : {}),
882
+ ...(answers ? { answers } : {}),
883
+ };
884
+ if (this.control?.resolvePermissionDecision(input)) {
885
+ return { approvalId, decision, provider: this.options.provider };
886
+ }
887
+ if (!this.process) throw new Error("provider process is unavailable");
888
+ if (!this.options.adapter.respondToPermissionDecision) throw new Error("provider does not support permission decisions");
889
+ const result = await this.options.adapter.respondToPermissionDecision(this.process, input);
890
+ return { approvalId, decision, ...(result ? { providerResult: result } : {}) };
891
+ }
892
+
893
+ private async injectPrompt(params: Record<string, unknown>): Promise<Record<string, unknown>> {
894
+ const body = typeof params.body === "string" ? params.body : "";
895
+ if (!body) throw new Error("body required");
896
+ if (!this.process) throw new Error("provider process is unavailable");
897
+ if (!this.options.adapter.deliverInitialPrompt) throw new Error("provider does not support prompt injection");
898
+ const messageId = typeof params.messageId === "number" ? params.messageId : undefined;
899
+ const attachments = Array.isArray(params.attachments)
900
+ ? params.attachments.filter((item): item is Record<string, unknown> => item !== null && typeof item === "object" && !Array.isArray(item))
901
+ : [];
902
+ if (messageId) this.pendingPromptMessageId = messageId;
903
+ const prompt = await this.promptInjectionText(body, messageId, attachments);
904
+ // Mark so the matching UserPromptSubmit echo isn't double-posted: a chat-box
905
+ // prompt already created its own session message shown in the dashboard.
906
+ this.recordInjectedPrompt(prompt.trim());
907
+ await this.options.adapter.deliverInitialPrompt(this.process, prompt);
908
+ return { injected: true, messageId, attachmentCount: attachments.length };
909
+ }
910
+
911
+ private async promptInjectionText(body: string, messageId: number | undefined, attachments: Record<string, unknown>[]): Promise<string> {
912
+ if (!attachments.length) return body;
913
+ const messages = await messagesWithCachedAttachments([{
914
+ id: messageId ?? 0,
915
+ from: "user",
916
+ to: this.agentId,
917
+ kind: "session",
918
+ body,
919
+ payload: { attachments },
920
+ readBy: [],
921
+ createdAt: Date.now(),
922
+ }], this.http, {
923
+ agentId: this.agentId,
924
+ onError: (message) => logger.warn("attachments", message),
925
+ });
926
+ const attachmentText = providerAttachmentText(messages[0]!);
927
+ return attachmentText ? `${body}\n\n${attachmentText}` : body;
928
+ }
929
+
930
+ private async restartProvider(opts: { resumeId?: string; suppressPrompt?: boolean } = {}): Promise<void> {
931
+ this.restartInProgress = true;
932
+ try {
933
+ if (this.process) {
934
+ await this.options.adapter.shutdown(this.process, {
935
+ graceful: true,
936
+ timeoutMs: this.options.providerConfig.headless.shutdownTimeoutMs,
937
+ });
938
+ }
939
+ this.claims.clearTerminalStatus();
940
+ this.claims.clearWorkKind("provider-turn");
941
+ this.claims.clearWorkKind("subagent");
942
+ this.claims.clearWorkKind("background-script");
943
+ if (this.stopped) return;
944
+ this.process = await this.spawnProvider(opts);
945
+ this.processStartedAt = Date.now();
946
+ } finally {
947
+ this.restartInProgress = false;
948
+ }
949
+ }
950
+
951
+ private shouldRestartUnexpectedProviderExit(status: SemanticStatus): boolean {
952
+ return runnerShouldRestartUnexpectedProviderExit(status, {
953
+ exitCommandInProgress: this.exitCommandInProgress,
954
+ stopped: this.stopped,
955
+ restartInProgress: this.restartInProgress,
956
+ provider: this.options.provider,
957
+ headless: this.options.headless,
958
+ hasTerminalSession: typeof this.process?.meta?.tmuxSession === "string"
959
+ || (this.options.adapter.supportsUnexpectedExitRestart?.() ?? false),
960
+ });
961
+ }
962
+
963
+ private async restartUnexpectedProviderExit(status: SemanticStatus): Promise<void> {
964
+ if (this.restartPending) return;
965
+ this.restartPending = true;
966
+ try {
967
+ // Best-effort Insights capture for the segment that just ended in a crash (#183). This
968
+ // path has no controlled teardown, so without it crashed sessions silently drop their
969
+ // context-ratio datapoint. The process handle is still set (cleared later), so the
970
+ // Claude transcript is readable; the runner stays alive here (restart or offline), so the
971
+ // durable outbox drains normally — no flush needed.
972
+ await Promise.race([
973
+ this.captureContextRatio("crash"),
974
+ new Promise<void>((resolve) => setTimeout(resolve, PRE_DESTROY_TIMEOUT_MS)),
975
+ ]).catch((error) => this.sessionLog(`insights: crash capture failed: ${errMessage(error)}`));
976
+
977
+ const now = Date.now();
978
+ const runtimeMs = this.processStartedAt ? now - this.processStartedAt : Number.POSITIVE_INFINITY;
979
+ const recent = this.unexpectedExitTimes.filter((time) => now - time <= UNEXPECTED_EXIT_WINDOW_MS);
980
+ recent.push(now);
981
+ this.unexpectedExitTimes.splice(0, this.unexpectedExitTimes.length, ...recent);
982
+ const diagnostics = providerExitDiagnostics({ status, runtimeMs, processMeta: this.process?.meta, hasProcess: Boolean(this.process?.process), exitCommandInProgress: this.exitCommandInProgress, stopped: this.stopped, restartInProgress: this.restartInProgress, restartPending: this.restartPending, headless: this.options.headless, provider: this.options.provider, logFile: runnerLogFileFromEnv() });
983
+
984
+ this.publishRunnerTimelineEvent({
985
+ status: "provider.exit_detected",
986
+ id: `provider-exit-${this.providerSessionId}-${now}`,
987
+ timestamp: now,
988
+ title: "Provider exited",
989
+ body: `${this.options.provider} reported ${status} after ${Math.round(runtimeMs / 1000)}s`,
990
+ icon: "ti-plug-off",
991
+ metadata: {
992
+ eventType: "provider.exit_detected",
993
+ ...diagnostics,
994
+ },
995
+ });
996
+
997
+ const nativeSelfResume = this.nativeSelfResume.active;
998
+ if (nativeSelfResume && getManifest(this.options.provider)?.resume?.supported) {
999
+ await recoverNativeSelfResumeExit({ handoff: nativeSelfResume, tracker: this.nativeSelfResume, providerSessionId: this.providerSessionId, status, diagnostics, now, publishTimeline: (event) => this.publishRunnerTimelineEvent(event), setTerminalProviderExit: (marker) => { this.terminalProviderExit = marker; }, setProviderStatus: (update) => this.setProviderStatus(update), restartProvider: (resumeId) => this.restartProvider(resumeId ? { resumeId } : { suppressPrompt: true }), publishStatus: () => this.publishStatus(), scheduleDrain: () => this.scheduleDrain() });
1000
+ return;
1001
+ }
1002
+
1003
+ if (this.shouldStopUnexpectedProviderExit(diagnostics)) {
1004
+ const hasResumeId = typeof diagnostics.claudeResumeId === "string" && diagnostics.claudeResumeId.length > 0;
1005
+ if (!hasResumeId) {
1006
+ const recoveredStatus = await this.recoveredStatusAfterUndeterminedProviderExit();
1007
+ if (recoveredStatus) {
1008
+ logger.warn("lifecycle", `${this.options.provider} exit signal recovered as ${recoveredStatus}; suppressing terminal exit`);
1009
+ this.publishRunnerTimelineEvent({
1010
+ status: "provider.restart_decision",
1011
+ id: `provider-restart-decision-${this.providerSessionId}-${now}`,
1012
+ timestamp: Date.now(),
1013
+ title: "Provider exit suppressed",
1014
+ body: `${this.options.provider} reported ${recoveredStatus} during exit confirmation; runner kept the provider live.`,
1015
+ icon: "ti-player-play",
1016
+ metadata: {
1017
+ eventType: "provider.restart_decision",
1018
+ decision: "suppress-recovered",
1019
+ reason: "undetermined-provider-exit-recovered",
1020
+ recoveredStatus,
1021
+ confirmWindowMs: UNDETERMINED_PROVIDER_EXIT_CONFIRM_MS,
1022
+ ...diagnostics,
1023
+ },
1024
+ });
1025
+ this.setProviderStatus(recoveredStatus);
1026
+ return;
1027
+ }
1028
+ }
1029
+ if (!hasResumeId && this.shouldRestartRecoverableProviderExit(diagnostics)) {
1030
+ logger.warn("lifecycle", `${this.options.provider} exit signal is early/recoverable; allowing fresh restart`);
1031
+ } else {
1032
+ logger.warn("lifecycle", `${this.options.provider} exited; leaving agent offline for manual recovery`);
1033
+ this.publishRunnerTimelineEvent({
1034
+ status: "provider.restart_decision",
1035
+ id: `provider-restart-decision-${this.providerSessionId}-${now}`,
1036
+ timestamp: Date.now(),
1037
+ title: "Provider restart skipped",
1038
+ body: hasResumeId
1039
+ ? "Claude exited; runner will not auto-resume. Resume id captured for manual recovery."
1040
+ : "Claude exited; runner will not restart automatically.",
1041
+ icon: "ti-player-stop",
1042
+ metadata: {
1043
+ eventType: "provider.restart_decision",
1044
+ decision: "stop-surface",
1045
+ reason: hasResumeId ? "claude-exit-manual-resume-available" : "claude-exit-manual-intervention-required",
1046
+ ...diagnostics,
1047
+ },
1048
+ });
1049
+ this.process = undefined;
1050
+ // #633: the runner stays alive (manual-resume hold), so neither the orchestrator tmux-gone
1051
+ // path nor the heartbeat reaper will ever fire. Stamp the terminal marker so the SAME offline
1052
+ // status frame (published by setProviderStatus below) carries it; the server's bus status
1053
+ // handler turns that edge into a terminal `agent.exited` event with a #636 diagnosis.
1054
+ this.terminalProviderExit = {
1055
+ reason: hasResumeId ? "claude-exit-manual-resume-available" : "claude-exit-manual-intervention-required",
1056
+ ...diagnostics,
1057
+ };
1058
+ this.setProviderStatus({
1059
+ status,
1060
+ reason: "provider-turn",
1061
+ id: `provider-exit-${this.providerSessionId}`,
1062
+ clear: ["provider-turn", "subagent", "background-script"],
1063
+ });
1064
+ return;
1065
+ }
1066
+ }
1067
+
1068
+ if (runtimeMs < RAPID_EXIT_MS && recent.length > MAX_RAPID_UNEXPECTED_EXITS) {
1069
+ logger.error("lifecycle", `provider session exited ${recent.length} times within ${Math.round(UNEXPECTED_EXIT_WINDOW_MS / 1000)}s; giving up`);
1070
+ this.publishRunnerTimelineEvent({
1071
+ status: "provider.restart_decision",
1072
+ id: `provider-restart-decision-${this.providerSessionId}-${now}`,
1073
+ timestamp: Date.now(),
1074
+ title: "Provider restart skipped",
1075
+ body: `rapid unexpected exits exceeded ${MAX_RAPID_UNEXPECTED_EXITS}`,
1076
+ icon: "ti-alert-triangle",
1077
+ metadata: {
1078
+ eventType: "provider.restart_decision",
1079
+ decision: "give-up",
1080
+ reason: "rapid-unexpected-provider-exits",
1081
+ rapidExitCount: recent.length,
1082
+ rapidExitWindowMs: UNEXPECTED_EXIT_WINDOW_MS,
1083
+ maxRapidUnexpectedExits: MAX_RAPID_UNEXPECTED_EXITS,
1084
+ ...diagnostics,
1085
+ },
1086
+ });
1087
+ this.setProviderStatus(status);
1088
+ this.options.onProviderExit?.(0);
1089
+ return;
1090
+ }
1091
+
1092
+ const delayMs = Math.min(10_000, Math.max(500, 500 * recent.length));
1093
+ logger.warn("lifecycle", `provider session exited unexpectedly after ${Math.round(runtimeMs / 1000)}s; restarting in ${delayMs}ms`);
1094
+ this.publishRunnerTimelineEvent({
1095
+ status: "provider.restart_decision",
1096
+ id: `provider-restart-decision-${this.providerSessionId}-${now}`,
1097
+ timestamp: Date.now(),
1098
+ title: "Provider restart scheduled",
1099
+ body: `runner will start a fresh ${this.options.provider} provider in ${delayMs}ms`,
1100
+ icon: "ti-refresh",
1101
+ metadata: {
1102
+ eventType: "provider.restart_decision",
1103
+ decision: "restart-fresh",
1104
+ reason: "unexpected-headless-terminal-exit",
1105
+ delayMs,
1106
+ rapidExitCount: recent.length,
1107
+ rapidExitWindowMs: UNEXPECTED_EXIT_WINDOW_MS,
1108
+ ...diagnostics,
1109
+ },
1110
+ });
1111
+ await Bun.sleep(delayMs);
1112
+ if (this.stopped || this.exitCommandInProgress) return;
1113
+ try {
1114
+ await this.restartProvider();
1115
+ this.publishStatus();
1116
+ this.scheduleDrain();
1117
+ } catch (error) {
1118
+ logger.error("lifecycle", `provider restart after unexpected exit failed: ${error}`);
1119
+ this.setProviderStatus("error");
1120
+ this.options.onProviderExit?.(1);
1121
+ }
1122
+ } finally {
1123
+ this.restartPending = false;
1124
+ }
1125
+ }
1126
+
1127
+ private shouldStopUnexpectedProviderExit(diagnostics: Record<string, unknown>): boolean {
1128
+ return getManifest(this.options.provider)?.launch?.managedPolicyPrompt?.alwaysOn === "append-system-prompt" && diagnostics.exitCommandInProgress !== true;
1129
+ }
1130
+
1131
+ private shouldRestartRecoverableProviderExit(diagnostics: Record<string, unknown>): boolean {
1132
+ return diagnostics.exitSource === "tmux-session-ended"
1133
+ && typeof diagnostics.runtimeMs === "number"
1134
+ && diagnostics.runtimeMs >= 0
1135
+ && diagnostics.runtimeMs < RAPID_EXIT_MS;
1136
+ }
1137
+
1138
+ private async recoveredStatusAfterUndeterminedProviderExit(): Promise<"busy" | "idle" | undefined> {
1139
+ if (!this.process || !this.options.adapter.probeActivity) return undefined;
1140
+ const deadline = Date.now() + UNDETERMINED_PROVIDER_EXIT_CONFIRM_MS;
1141
+ do {
1142
+ const activity = await this.options.adapter.probeActivity(this.process).catch((error) => {
1143
+ this.sessionLog(`provider-exit confirmation probe failed: ${errMessage(error)}`);
1144
+ return "unknown" as const;
1145
+ });
1146
+ if (activity === "busy" || activity === "idle") return activity;
1147
+ if (Date.now() >= deadline) return undefined;
1148
+ await Bun.sleep(PROVIDER_EXIT_CONFIRM_POLL_MS);
1149
+ } while (!this.stopped && !this.exitCommandInProgress);
1150
+ return undefined;
1151
+ }
1152
+
1153
+ private async shutdownProvider(hard: boolean, timeoutMs = this.options.providerConfig.headless.shutdownTimeoutMs): Promise<void> {
1154
+ this.lifecycleAction = hard ? "killing" : "shutting-down";
1155
+ this.publishStatus();
1156
+ await this.bus.statusAsync({ agentStatus: "idle", ready: false, meta: { lifecycleAction: this.lifecycleAction, lifecycleActionAt: Date.now() } });
1157
+ if (this.process) {
1158
+ await this.options.adapter.shutdown(this.process, {
1159
+ graceful: !hard,
1160
+ timeoutMs,
1161
+ });
1162
+ }
1163
+ this.claims.setTerminalStatus("offline");
1164
+ this.publishStatus();
1165
+ this.stopped = true;
1166
+ }
1167
+
1168
+ private publishRunnerTimelineEvent(event: RunnerTimelineEvent): void {
1169
+ this.pendingTimelineEvent = {
1170
+ ...event,
1171
+ metadata: {
1172
+ source: "runner",
1173
+ provider: this.options.provider,
1174
+ runnerId: this.options.runnerId,
1175
+ agentId: this.agentId,
1176
+ policyName: this.options.policyName ?? null,
1177
+ spawnRequestId: this.options.spawnRequestId ?? null,
1178
+ label: this.options.label ?? null,
1179
+ providerSessionId: this.providerSessionId,
1180
+ ...(event.metadata ?? {}),
1181
+ },
1182
+ };
1183
+ this.publishStatus();
1184
+ }
1185
+
1186
+ private async updateCommand(commandId: string, status: string, result?: Record<string, unknown>, error?: string): Promise<void> {
1187
+ await this.bus.updateCommand(commandId, { status, ...(result ? { result } : {}), ...(error ? { error } : {}) });
1188
+ }
1189
+
1190
+ private handleBusError(code: string, message: string): void {
1191
+ const action = runnerBusErrorAction(code, this.stopped);
1192
+ if (action === "ignore") return;
1193
+ logger.error("bus", `bus error ${code}: ${message}`);
1194
+ if (action === "stop") {
1195
+ void this.stop().catch((error) => {
1196
+ logger.error("bus", `stop after bus error failed: ${error}`);
1197
+ }).finally(() => process.exit(0));
1198
+ }
1199
+ }
1200
+
1201
+ private setProviderStatus(update: ProviderStatusUpdate): void {
1202
+ const status = typeof update === "string" ? update : update.status;
1203
+ this.lastProviderEventAt = Date.now();
1204
+ const providerSessionId = typeof update === "string" ? undefined : update.providerSessionId;
1205
+ if (providerSessionId && providerSessionId !== this.providerSessionId) return;
1206
+ const reason = typeof update === "string" ? "provider-turn" : update.reason ?? "provider-turn";
1207
+ const id = typeof update === "string" ? reason : update.id ?? reason;
1208
+ if (typeof update !== "string" && update.timeline) {
1209
+ this.pendingTimelineEvent = {
1210
+ status: update.timeline.status,
1211
+ ...(update.timeline.id ? { id: update.timeline.id } : {}),
1212
+ timestamp: update.timeline.timestamp ?? Date.now(),
1213
+ ...(update.timeline.title ? { title: update.timeline.title } : {}),
1214
+ ...(update.timeline.body ? { body: update.timeline.body } : {}),
1215
+ ...(update.timeline.icon ? { icon: update.timeline.icon } : {}),
1216
+ ...(update.timeline.metadata ? { metadata: update.timeline.metadata } : {}),
1217
+ };
1218
+ }
1219
+ let deferredRateLimitHold = false;
1220
+ if (typeof update !== "string" && update.providerState) {
1221
+ const ps = update.providerState;
1222
+ // A rate-limit hold persists across the idle the turn ends on; any other
1223
+ // providerState supersedes it (clears a stale hold).
1224
+ if (ps.state === "blocked" && ps.reason === "rate_limit") {
1225
+ const paneRateLimitHold = typeof ps.source === "string" && ps.source.startsWith("claude-pane");
1226
+ if (status === "idle" && reason === "provider-turn" && paneRateLimitHold && this.claims.activeWork().some((work) => work.kind === "provider-turn")) {
1227
+ this.pendingRateLimitHold = update.providerState;
1228
+ this.providerBlocked = false;
1229
+ deferredRateLimitHold = true;
1230
+ this.sessionLog(`rate-limit hold deferred until active turn finishes (turn ${this.currentTurnId ?? "?"})`);
1231
+ } else {
1232
+ this.providerBlocked = true;
1233
+ this.pendingRateLimitHold = undefined;
1234
+ const fresh = !this.rateLimitHold;
1235
+ this.rateLimitHold = update.providerState;
1236
+ if (fresh) this.publishRateLimitNotice(update.providerState);
1237
+ }
1238
+ } else {
1239
+ this.providerBlocked = ps.state === "blocked";
1240
+ this.pendingRateLimitHold = undefined;
1241
+ this.rateLimitHold = undefined;
1242
+ }
1243
+ } else if (status === "idle") {
1244
+ this.providerBlocked = false;
1245
+ }
1246
+ // Forward progress (a real turn) lifts the hold so the blocked badge clears.
1247
+ if (status === "busy") this.rateLimitHold = undefined;
1248
+ if (deferredRateLimitHold) {
1249
+ if (typeof update !== "string") {
1250
+ for (const kind of update.clear ?? []) this.claims.clearWorkKind(kind);
1251
+ }
1252
+ this.publishStatus();
1253
+ return;
1254
+ }
1255
+ if (typeof update !== "string" && status === "error") {
1256
+ const terminalReason = typeof update.metadata?.terminalFailureReason === "string"
1257
+ ? update.metadata.terminalFailureReason
1258
+ : typeof update.providerState?.reason === "string"
1259
+ ? update.providerState.reason
1260
+ : "provider-error";
1261
+ const terminalMessage = typeof update.metadata?.terminalFailureMessage === "string"
1262
+ ? update.metadata.terminalFailureMessage
1263
+ : typeof update.providerState?.message === "string"
1264
+ ? update.providerState.message
1265
+ : "Provider reported an unrecoverable error.";
1266
+ this.terminalFailure = {
1267
+ reason: terminalReason,
1268
+ message: terminalMessage,
1269
+ ...(update.providerState ? { providerState: update.providerState } : {}),
1270
+ };
1271
+ } else if (status === "idle" || status === "busy") {
1272
+ this.terminalFailure = undefined;
1273
+ }
1274
+ // Compaction lifecycle (Claude PreCompact→`compacting` busy / PostCompact→`compacted`
1275
+ // idle). A `compacted` idle for a turn that predated compaction is a hook artifact — the
1276
+ // turn resumes — so it must NOT end the turn. Discriminate on whether a turn was running,
1277
+ // captured BEFORE the busy logic below mints a currentTurnId (it does for /compact at idle).
1278
+ const timelineStatus = typeof update !== "string" ? update.timeline?.status : undefined;
1279
+ if (timelineStatus === "compacting") {
1280
+ this.compactionMidTurn = this.currentTurnId !== undefined;
1281
+ } else if (timelineStatus === "compacted") {
1282
+ this.nativeSelfResume.noteCompacted(typeof update !== "string" && typeof update.timeline?.timestamp === "number" ? update.timeline.timestamp : Date.now());
1283
+ this.publishCompactionNotice();
1284
+ if (this.compactionMidTurn) {
1285
+ // Keep the turn (and its live reasoning tail) alive; the genuine Stop hook ends it.
1286
+ // pendingTimelineEvent (the marker) was set above and is consumed by publishStatus.
1287
+ this.sessionLog(`compaction completed mid-turn (turn ${this.currentTurnId ?? "?"}) — staying busy`);
1288
+ this.publishStatus();
1289
+ return;
1290
+ }
1291
+ }
1292
+ if (status === "busy" && reason === "provider-turn") {
1293
+ if (!this.currentTurnId) {
1294
+ this.currentTurnId = typeof update !== "string" && update.id ? update.id : crypto.randomUUID();
1295
+ this.currentTurnStartedAt = Date.now();
1296
+ this.compactionMidTurn = false;
1297
+ this.sessionLog(`turn started (turn ${this.currentTurnId})`);
1298
+ }
1299
+ this.busyReconciler.arm();
1300
+ } else if (status === "idle" && reason === "provider-turn") {
1301
+ if (this.currentTurnId) { this.completedProviderTurns += 1; this.sessionLog(`turn ended via provider idle (turn ${this.currentTurnId})`); }
1302
+ this.currentTurnId = undefined;
1303
+ this.currentTurnStartedAt = undefined;
1304
+ this.compactionMidTurn = false;
1305
+ this.busyReconciler.disarm();
1306
+ this.stopReasoningTail();
1307
+ // Providers with append-system-prompt policy use lifecycle hooks for obligation handling;
1308
+ // others need the runner to re-inject pending obligations on idle.
1309
+ if (getManifest(this.options.provider)?.launch?.managedPolicyPrompt?.alwaysOn !== "append-system-prompt") void this.reInjectPendingObligation();
1310
+ }
1311
+ // A genuine restart back to forward progress clears the terminal-exit marker so a later offline
1312
+ // doesn't re-announce a stale provider exit.
1313
+ if (status === "busy" || status === "idle") this.terminalProviderExit = undefined;
1314
+ if (status === "busy") {
1315
+ this.claims.clearTerminalStatus();
1316
+ // #650: liveness-track the pids a reporter explicitly supplied; otherwise, for a
1317
+ // provider-turn, fall back to the managed provider process itself. If that process
1318
+ // dies out-of-band with no exit event reaching the runner (crash/orphan/external
1319
+ // kill), the reconcile clears the stuck turn → idle. A dead pid means the turn is
1320
+ // definitively over, so this can never falsely clear a live turn (#653's hazard).
1321
+ const reportedPids = typeof update === "string" ? undefined : update.pids;
1322
+ const livenessPids = reportedPids?.length
1323
+ ? reportedPids
1324
+ : reason === "provider-turn" && typeof this.process?.pid === "number" && this.process.pid > 0
1325
+ ? [this.process.pid]
1326
+ : undefined;
1327
+ this.claims.startWork(reason, id, typeof update === "string" ? (livenessPids ? { pids: livenessPids } : {}) : {
1328
+ label: update.label,
1329
+ role: update.role,
1330
+ parentId: update.parentId,
1331
+ ...(livenessPids ? { pids: livenessPids } : {}),
1332
+ metadata: {
1333
+ ...(update.metadata ?? {}),
1334
+ ...(update.providerState ? { providerState: update.providerState } : {}),
1335
+ },
1336
+ });
1337
+ // A busy report with liveness-trackable pids arms the periodic reconcile (self-disarms
1338
+ // once no pid-bearing work remains, so idle/untracked agents pay nothing).
1339
+ if (livenessPids?.length) this.armBusyLivenessReconcile();
1340
+ if (reason === "provider-turn") {
1341
+ for (const claim of this.activeTaskClaims.values()) claim.observedProviderBusy = true;
1342
+ }
1343
+ } else if (status === "idle") {
1344
+ this.claims.clearTerminalStatus();
1345
+ this.claims.finishWork(reason, id);
1346
+ if (reason === "provider-turn") void this.completeObservedTaskClaims();
1347
+ }
1348
+ else if (status === "offline" || status === "error") this.claims.setTerminalStatus(status);
1349
+ if (typeof update !== "string") {
1350
+ for (const kind of update.clear ?? []) this.claims.clearWorkKind(kind);
1351
+ }
1352
+ // #329: the provider just went idle (ready). If a first initial-prompt delivery timed out on
1353
+ // a cold-start TUI, re-attempt now — the ready wait should resolve almost immediately.
1354
+ if (status === "idle" && this.pendingInitialPrompt && !this.deliveringInitialPrompt) {
1355
+ void this.attemptInitialPromptDelivery(this.pendingInitialPrompt, INITIAL_PROMPT_RETRY_READY_TIMEOUT_MS);
1356
+ }
1357
+ if (status === "idle" && this.pendingRateLimitHold && !this.claims.activeWork().some((work) => work.kind === "provider-turn")) {
1358
+ const hold = this.pendingRateLimitHold;
1359
+ this.pendingRateLimitHold = undefined;
1360
+ this.providerBlocked = true;
1361
+ const fresh = !this.rateLimitHold;
1362
+ this.rateLimitHold = hold;
1363
+ if (fresh) this.publishRateLimitNotice(hold);
1364
+ }
1365
+ if (status === "idle" && this.nativeSelfResume.active) {
1366
+ this.nativeSelfResume.resolve(this.nativeSelfResume.active, { resumed: true, via: "provider-idle" });
1367
+ }
1368
+ this.publishStatus();
1369
+ }
1370
+
1371
+ // #417: when a non-claude provider finishes a turn with a pending spawner obligation
1372
+ // still unresolved, re-inject the obligation as a synthetic message so providerMessageText
1373
+ // surfaces the specific `agent-relay /reply <id>` command. Claude handles this via the
1374
+ // stop.sh hook; every other provider needs this in-process nudge.
1375
+ private async reInjectPendingObligation(): Promise<void> {
1376
+ if (!this.process) return;
1377
+ // Filter to spawner obligations only — user obligations go through the existing codex
1378
+ // prompt-capture path (publishProviderSessionEvent) and the session mirror (#418), never
1379
+ // an explicit /reply. Shared predicate so this rule stays in lockstep with the Claude
1380
+ // Stop-hook nag exemption (control-server.replyObligationStopDecision).
1381
+ const obligations = this.obligationCache.get().filter(obligationRequiresExplicitReply);
1382
+ if (!obligations.length) return;
1383
+ const MAX_REINJECTIONS = 3;
1384
+ for (const obligation of obligations) {
1385
+ const count = this.pendingObligationInjections.get(obligation.messageId) ?? 0;
1386
+ if (count >= MAX_REINJECTIONS) continue;
1387
+ this.pendingObligationInjections.set(obligation.messageId, count + 1);
1388
+ this.sessionLog(`re-injecting pending reply obligation #${obligation.messageId} from ${obligation.from} (attempt ${count + 1}/${MAX_REINJECTIONS})`);
1389
+ const replayMessage = {
1390
+ id: obligation.messageId,
1391
+ from: obligation.from,
1392
+ to: this.agentId,
1393
+ kind: obligation.kind,
1394
+ ...(obligation.subject ? { subject: obligation.subject } : {}),
1395
+ body: obligation.bodyPreview,
1396
+ payload: {},
1397
+ readBy: [] as string[],
1398
+ createdAt: obligation.createdAt,
1399
+ };
1400
+ // Suppress the userMessage echo that providers (e.g. Codex) reflect back when
1401
+ // a turn is started with relay-formatted content — the echo body is exactly what
1402
+ // providerMessageText produces, which starts with "[relay message #" and is caught
1403
+ // by the RELAY_INJECTION_MARKERS prefix check, but record it explicitly as
1404
+ // defense-in-depth for adapters that might echo only the unformatted body text.
1405
+ this.recordInjectedPrompt(providerMessageText([replayMessage]));
1406
+ await this.options.adapter.deliver(this.process, [replayMessage]);
1407
+ // Kick off a cache refresh so a cleared obligation (agent replied) is detected
1408
+ // before the next idle fires — without this, the stale snapshot re-injects up to
1409
+ // MAX_REINJECTIONS times even after the agent has already /reply-ed.
1410
+ this.obligationCache.markDirty();
1411
+ // One obligation at a time — let the provider reply before nudging again.
1412
+ break;
1413
+ }
1414
+ }
1415
+
1416
+ // Session-mirror lane: capture the assistant turn from the Claude transcript and
1417
+ // post it as a "session" message so it shows in the dashboard chat with zero
1418
+ // agent tokens. Capture is UNCONDITIONAL — it no longer depends on a triggering
1419
+ // relay message existing, so turns started from the web terminal (which create
1420
+ // no relay message) are mirrored too. A reply obligation, when present, is still
1421
+ // used as replyTo so the Stop hook stops nagging the agent to /reply.
1422
+ //
1423
+ // isPreFlush: true (#435/#499) — mid-turn PreToolUse boundary. Drain the live
1424
+ // tail and force-flush session messages before showing the blocking control;
1425
+ // the entry cursor lets Stop capture skip already-emitted text.
1426
+ private async publishSessionTurn(input: { transcriptPath: string; lastAssistantMessage?: unknown; isPreFlush?: boolean }): Promise<void> {
1427
+ if (input.transcriptPath) this.lastTranscriptPath = input.transcriptPath;
1428
+
1429
+ if (input.isPreFlush) {
1430
+ if (!input.transcriptPath) return;
1431
+ let jsonl: string | undefined;
1432
+ try { jsonl = await readFile(input.transcriptPath, "utf8"); } catch { return; }
1433
+ if (!jsonl) return;
1434
+ const body = extractLastAssistantTurnAfterEntry(jsonl, this.narrativeFlushEntryCount);
1435
+ await this.drainReasoningTail();
1436
+ this.narrativeFlushEntryCount = countTranscriptEntries(jsonl);
1437
+ if (body && !this.preFlushBodyAlreadyMirroredByReasoningTail(jsonl, body)) {
1438
+ const turnId = this.currentTurnId;
1439
+ this.sessionLog(`pre-flush narrative for turn ${turnId ?? "?"} (${body.length} chars)`);
1440
+ await this.publishSessionEvent({ from: this.agentId, to: "user", body, session: { type: "response", origin: "provider", ...(turnId ? { turnId } : {}) } });
1441
+ }
1442
+ await this.sessionOutbox.flush(2_000).then((ok) => { if (!ok) this.sessionLog(`pre-flush session outbox incomplete before blocking control (${this.sessionOutbox.pendingCount()} pending)`); }, (error) => this.sessionLog(`pre-flush session outbox failed before blocking control: ${errMessage(error)}`));
1443
+ return;
1444
+ }
1445
+
1446
+ const turnId = this.currentTurnId;
1447
+ this.stopReasoningTail();
1448
+ // Optional correlation for threading + obligation clearing — never a capture gate.
1449
+ let replyToMessageId: number | undefined, replyTarget = "user";
1450
+ const pendingPrompt = this.pendingPromptMessageId;
1451
+ if (pendingPrompt) {
1452
+ replyToMessageId = pendingPrompt;
1453
+ this.pendingPromptMessageId = undefined;
1454
+ } else {
1455
+ // Correlation-only (threading + obligation clearing) — the local snapshot is fresh
1456
+ // enough and never blocks the response-capture path (#196).
1457
+ const route = responseCaptureReplyRoute(this.obligationCache.get(), this.currentTurnStartedAt);
1458
+ replyToMessageId = route?.replyToMessageId; replyTarget = route?.to ?? "user";
1459
+ }
1460
+
1461
+ // Grab and reset the pre-flush cursor before extraction: the cursor may be
1462
+ // non-zero when a PreToolUse hook flushed narrative earlier this turn (#435).
1463
+ // In "full" mode, skip those already-emitted entries so they don't duplicate.
1464
+ const flushCount = this.narrativeFlushEntryCount;
1465
+ this.narrativeFlushEntryCount = 0;
1466
+
1467
+ // The Stop hook can fire before the final assistant entry is flushed to disk.
1468
+ // Claude writes thinking and text as separate entries (both with end_turn), so
1469
+ // the transcript can "look complete" while the text entry is still pending.
1470
+ // Retry until both the transcript has an end_turn AND extraction yields text.
1471
+ let body = "";
1472
+ let jsonl: string | undefined;
1473
+ try { jsonl = await readFile(input.transcriptPath, "utf8"); } catch { jsonl = undefined; }
1474
+ if (jsonl !== undefined) {
1475
+ for (let attempt = 0; attempt < 5; attempt++) {
1476
+ if (attempt > 0) {
1477
+ await new Promise((r) => setTimeout(r, 100));
1478
+ try { jsonl = await readFile(input.transcriptPath, "utf8"); } catch { break; }
1479
+ }
1480
+ if (!transcriptLooksComplete(jsonl)) continue;
1481
+ // If a pre-flush already emitted entries 1..flushCount, resume after that mark to avoid double-emit (#435/#499).
1482
+ const extract = this.options.providerConfig.chatCaptureMode === "full"
1483
+ ? (j: string) => (flushCount > 0 ? extractLastAssistantTurnAfterEntry(j, flushCount) : extractLastAssistantTurn(j))
1484
+ : (j: string) => (flushCount > 0 ? extractFinalAssistantMessageAfterEntry(j, flushCount) : extractFinalAssistantMessage(j));
1485
+ body = extract(jsonl);
1486
+ if (body) break;
1487
+ }
1488
+ }
1489
+ // Fallback: last_assistant_message from the Stop hook payload, which bypasses
1490
+ // the transcript file race entirely.
1491
+ if (!body && input.lastAssistantMessage) {
1492
+ body = extractHookAssistantMessage(input.lastAssistantMessage);
1493
+ }
1494
+ // A pure tool-use turn with no closing text is fine to skip — its reasoning and
1495
+ // tool steps already carried the visibility into chat.
1496
+ if (!body) {
1497
+ this.sessionLog(`response capture: no closing text for turn ${turnId ?? "?"} (skipped)`);
1498
+ return;
1499
+ }
1500
+
1501
+ this.sessionLog(`response captured for turn ${turnId ?? "?"} (${body.length} chars${replyToMessageId ? `, replyTo #${replyToMessageId}` : ", no replyTo"})`);
1502
+ await publishCapturedResponse({ publishSessionEvent: (event) => this.publishSessionEvent(event), outbox: this.sessionOutbox, provider: this.options.provider, from: this.agentId, to: replyTarget, body, replyTo: replyToMessageId, session: { type: "response", origin: "provider", final: true, ...(turnId ? { turnId } : {}) } });
1503
+ // The agent's reply may have cleared an obligation — refresh the snapshot so the next
1504
+ // turn-end doesn't re-prompt for a message already answered (#196).
1505
+ if (replyToMessageId) this.obligationCache.markDirty();
1506
+ }
1507
+
1508
+ // Post one session-mirror event (prompt echo, assistant response, reasoning or
1509
+ // tool step) as a `kind: "session"` relay message tagged with payload.session so
1510
+ // the dashboard can render the live provider session faithfully. Display-only:
1511
+ // session messages are never delivered back into a provider.
1512
+ private publishSessionEvent(input: {
1513
+ from: string;
1514
+ to: string;
1515
+ body: string;
1516
+ session: MessageSessionMeta;
1517
+ replyTo?: number;
1518
+ }): void {
1519
+ // Durable, ordered, timestamped (#196): the actual POST happens in deliverOutboxEvent,
1520
+ // retried until it lands. occurredAt is stamped now so a queued event reports when it
1521
+ // truly happened, not when the server finally accepted it. Routed through the fast-lane
1522
+ // sessionOutbox (#332) so a transient trace failure can't head-of-line block real messages.
1523
+ // A stepId-bearing step (Codex tool running→completed, streamed reasoning/response) uses a
1524
+ // STABLE idempotency key so the server upserts the row in place instead of appending a dup.
1525
+ const stepId = input.session.stepId;
1526
+ this.sessionOutbox.enqueue({
1527
+ kind: "session-message",
1528
+ ...(stepId ? { idempotencyKey: `session-step:${input.from}:${input.session.turnId ?? ""}:${stepId}` } : {}),
1529
+ payload: {
1530
+ from: input.from,
1531
+ to: input.to,
1532
+ ...(input.replyTo ? { replyTo: input.replyTo } : {}),
1533
+ kind: "session",
1534
+ body: input.body,
1535
+ payload: { session: { provider: this.options.provider, ...input.session } },
1536
+ } satisfies SendMessageInput,
1537
+ });
1538
+ }
1539
+
1540
+ private publishSessionEventsBatch(inputs: Array<{
1541
+ from: string;
1542
+ to: string;
1543
+ body: string;
1544
+ session: MessageSessionMeta;
1545
+ replyTo?: number;
1546
+ }>): void {
1547
+ for (let i = 0; i < inputs.length; i += SESSION_MIRROR_BATCH_MAX) {
1548
+ const chunk = inputs.slice(i, i + SESSION_MIRROR_BATCH_MAX);
1549
+ if (chunk.length === 1) {
1550
+ this.publishSessionEvent(chunk[0]!);
1551
+ continue;
1552
+ }
1553
+ const messages = chunk.map((input) => {
1554
+ const stepId = input.session.stepId;
1555
+ return {
1556
+ from: input.from,
1557
+ to: input.to,
1558
+ ...(input.replyTo ? { replyTo: input.replyTo } : {}),
1559
+ kind: "session",
1560
+ body: input.body,
1561
+ ...(stepId ? { idempotencyKey: `session-step:${input.from}:${input.session.turnId ?? ""}:${stepId}` } : {}),
1562
+ payload: { session: { provider: this.options.provider, ...input.session } },
1563
+ } satisfies SendMessageInput;
1564
+ });
1565
+ this.sessionOutbox.enqueue({
1566
+ kind: "session-message-batch",
1567
+ idempotencyKey: `session-batch:${this.agentId}:${++this.sessionBatchSeq}:${messages.length}`,
1568
+ payload: { messages },
1569
+ });
1570
+ }
1571
+ }
1572
+
1573
+ // Map queued records to HTTP calls. Throw to retry, return to ack/delete.
1574
+ private async deliverOutboxEvent(record: OutboxRecord): Promise<void> {
1575
+ await deliverRunnerOutboxEvent({
1576
+ record,
1577
+ http: this.http,
1578
+ relayUrl: this.options.relayUrl,
1579
+ token: this.currentToken,
1580
+ updatePayload: (seq, payload) => this.outbox.updatePayload(seq, payload),
1581
+ sessionLog: (message) => this.sessionLog(message),
1582
+ recoverRuntimeTokenAfterAuthFailure: (source) => this.recoverRuntimeTokenAfterAuthFailure(source),
1583
+ });
1584
+ }
1585
+
1586
+ // Replay a buffered MCP tool call (Stage 2, #215) that the proxy queued while the relay was
1587
+ // unreachable. Kept as a method wrapper for tests; the concern lives in mcp-outbox.ts.
1588
+ private async deliverBufferedMcpCall(record: OutboxRecord): Promise<void> {
1589
+ await deliverBufferedMcpOutboxCall({
1590
+ record,
1591
+ relayUrl: this.options.relayUrl,
1592
+ token: this.currentToken,
1593
+ recoverRuntimeTokenAfterAuthFailure: (source) => this.recoverRuntimeTokenAfterAuthFailure(source),
1594
+ });
1595
+ }
1596
+
1597
+ // A hook reported an unhandled failure (#198 seam). Already logged FATAL by the control
1598
+ // server; here we additionally surface it durably to the server as a generic insight so
1599
+ // it shows up in observability rather than only in the per-agent log (#196).
1600
+ private reportHookFatal(report: { hook: string; error: string }): void {
1601
+ try {
1602
+ this.insights.queueHookFatal(report);
1603
+ } catch (error) {
1604
+ logger.error("outbox", `failed to queue hook-fatal report: ${errMessage(error)}`);
1605
+ }
1606
+ }
1607
+
1608
+ // A human typed a prompt directly into the provider (web terminal / TUI). Mirror
1609
+ // it into the dashboard chat so both surfaces stay in sync, and kick off reasoning
1610
+ // tailing for the turn. Skips prompts the runner itself injected (chat box, relay
1611
+ // deliveries) so those aren't double-posted.
1612
+ private async handleUserPrompt(input: { prompt: string; transcriptPath?: string }): Promise<void> {
1613
+ if (input.transcriptPath) this.lastTranscriptPath = input.transcriptPath;
1614
+ if (!this.currentTurnId) {
1615
+ this.currentTurnId = crypto.randomUUID();
1616
+ this.currentTurnStartedAt = Date.now();
1617
+ }
1618
+ const text = input.prompt.trim();
1619
+ if (text && !this.isRunnerInjectedPrompt(text)) {
1620
+ this.sessionLog(`prompt echoed from terminal (${text.length} chars)`);
1621
+ await this.publishSessionEvent({
1622
+ from: "user",
1623
+ to: this.agentId,
1624
+ body: text,
1625
+ session: { type: "prompt", origin: "terminal", turnId: this.currentTurnId },
1626
+ });
1627
+ } else if (text) {
1628
+ this.sessionDebug("user-prompt hook: skipped echo (runner-injected)");
1629
+ }
1630
+ if (input.transcriptPath) this.startReasoningTail(input.transcriptPath);
1631
+ }
1632
+
1633
+ // A provider lifecycle hook reported a session boundary (Claude PreCompact / SessionEnd
1634
+ // → control server). Normalize the raw provider reason to a SessionDestroyReason and run
1635
+ // the same pre-destroy seam the bus commands use. `clear`/`compact` continue the session;
1636
+ // anything else (logout, prompt_input_exit, other) is a real termination.
1637
+ private async handleSessionBoundary(input: { reason?: string; transcriptPath?: string }): Promise<void> {
1638
+ // Reason mapping is fail-safe-toward-termination: only the two known session-
1639
+ // CONTINUING reasons are special-cased; everything else (logout, prompt_input_exit,
1640
+ // other, AND any future reason) maps to "shutdown" → full pre-destroy capture.
1641
+ // ⚠ If Claude Code adds a new BENIGN/continuing boundary reason, add it here — until
1642
+ // then it will trigger a (harmless but wasteful) full context capture on a session
1643
+ // that isn't actually ending.
1644
+ const reason = input.reason === "compact" ? "compact"
1645
+ : input.reason === "clear" ? "clear"
1646
+ : "shutdown";
1647
+ await this.runPreSessionDestroy(reason, { transcriptPath: input.transcriptPath });
1648
+ // clear/compact CONTINUE the session — the finalizing window is transient. The bus-command
1649
+ // path (handleCommand) restores the addressable status in its finally; the hook path has no
1650
+ // such teardown, so without this the dashboard stays stuck on "wrapping up — messaging
1651
+ // paused" with the composer disabled forever. Only restore if we're still in the exact
1652
+ // finalizing state we published — a concurrent bus command (kill/restart/shutdown) may have
1653
+ // transitioned lifecycleAction since, and that must win. "shutdown" reasons end the session,
1654
+ // so their finalizing state is superseded by going offline; leave it be.
1655
+ if ((reason === "clear" || reason === "compact") && this.lifecycleAction === `finalizing-${reason}` && !this.stopped) {
1656
+ this.lifecycleAction = undefined;
1657
+ this.publishStatus();
1658
+ }
1659
+ }
1660
+
1661
+ // The pre-session-destroy seam (#183): the single place end-of-session work runs before
1662
+ // an invasive transition (compact/clear/restart/shutdown/kill). Best-effort and
1663
+ // time-boxed so it never hangs teardown; concurrent calls for the same teardown coalesce
1664
+ // (a shutdown bus command and the SessionEnd hook can both fire). During the window the
1665
+ // agent is published non-addressable so the operator sees "wrapping up", not "busy".
1666
+ private runPreSessionDestroy(reason: SessionDestroyReason, opts?: { transcriptPath?: string }): Promise<void> {
1667
+ if (this.preDestroyPromise) return this.preDestroyPromise;
1668
+ const run = (async () => {
1669
+ this.publishFinalizing(reason);
1670
+ try {
1671
+ await Promise.race([
1672
+ Promise.all([
1673
+ this.captureContextRatio(reason, opts),
1674
+ this.captureContinuationArchive(reason, opts),
1675
+ ]).then(() => undefined),
1676
+ new Promise<void>((resolve) => setTimeout(resolve, PRE_DESTROY_TIMEOUT_MS)),
1677
+ ]);
1678
+ } catch (error) {
1679
+ this.sessionLog(`pre-destroy capture failed: ${errMessage(error)}`);
1680
+ }
1681
+ // For exit-bound transitions the runner won't be alive afterward to drain the durable
1682
+ // outbox, so block (bounded) on delivering what capture just enqueued. This runs before
1683
+ // handleCommand's finally deletes the agent, so the runtime token is still valid here.
1684
+ if (reasonExitsRunner(reason)) {
1685
+ try {
1686
+ const delivered = await this.outbox.flush(OUTBOX_FLUSH_TIMEOUT_MS);
1687
+ if (!delivered) this.sessionLog(`insights: outbox flush incomplete before ${reason} (${this.outbox.pendingCount()} pending)`);
1688
+ } catch (error) {
1689
+ this.sessionLog(`insights: outbox flush failed: ${errMessage(error)}`);
1690
+ }
1691
+ }
1692
+ })();
1693
+ this.preDestroyPromise = run;
1694
+ void run.finally(() => { this.preDestroyPromise = undefined; });
1695
+ return run;
1696
+ }
1697
+
1698
+ // Publish the transient pre-destroy state: a non-offline status with ready:false (so the
1699
+ // agent drops out of isAgentOnline fan-out targeting without going "offline") plus a
1700
+ // finalizing-<reason> lifecycleAction the dashboard renders as "wrapping up" with the
1701
+ // composer disabled.
1702
+ private publishFinalizing(reason: SessionDestroyReason): void {
1703
+ this.lifecycleAction = `finalizing-${reason}`;
1704
+ void this.bus.statusAsync({ agentStatus: "busy", ready: false, meta: { lifecycleAction: this.lifecycleAction, lifecycleActionAt: Date.now() } });
1705
+ }
1706
+
1707
+ // Repo-name project id for insight observations, resolved once by the insights collaborator.
1708
+ private insightProject(): string {
1709
+ return this.insights.project();
1710
+ }
1711
+
1712
+ private async captureContinuationArchive(reason: SessionDestroyReason, opts?: { transcriptPath?: string }): Promise<void> {
1713
+ await this.insights.captureContinuationArchive(reason, opts);
1714
+ }
1715
+
1716
+ private async captureContextRatio(reason: SessionDestroyReason, opts?: { transcriptPath?: string }): Promise<void> {
1717
+ await this.insights.captureContextRatio(reason, opts);
1718
+ }
1719
+
1720
+ // Route a provider-emitted session event (Codex app-server) into the chat mirror.
1721
+ // Mirrors the same semantics as the Claude lane: prompts are echoed with dedup,
1722
+ // and a response is only auto-captured when the agent won't separately reply to a
1723
+ // relay obligation (so relay-triggered turns aren't double-posted).
1724
+ private async publishProviderSessionEvent(event: ProviderSessionEvent): Promise<void> {
1725
+ const body = event.body.trim();
1726
+ if (!body) return;
1727
+ const turnId = event.turnId ?? this.currentTurnId;
1728
+ if (event.type === "prompt") {
1729
+ if (this.isRunnerInjectedPrompt(body)) return;
1730
+ await this.publishSessionEvent({
1731
+ from: "user",
1732
+ to: this.agentId,
1733
+ body,
1734
+ session: { type: "prompt", origin: event.origin ?? "terminal", ...(turnId ? { turnId } : {}) },
1735
+ });
1736
+ return;
1737
+ }
1738
+ if (event.type === "response") {
1739
+ // Dashboard prompt injection is already answered by this captured App Server
1740
+ // response. User obligations still belong to the session mirror; spawner
1741
+ // obligations (#413) route to the spawn-parent with replyTo to clear the debt.
1742
+ let replyToMessageId: number | undefined, replyTarget = "user";
1743
+ const pendingPrompt = this.pendingPromptMessageId;
1744
+ if (pendingPrompt) {
1745
+ replyToMessageId = pendingPrompt;
1746
+ this.pendingPromptMessageId = undefined;
1747
+ } else {
1748
+ const route = responseCaptureReplyRoute(this.obligationCache.get(), this.currentTurnStartedAt, { suppressUser: true });
1749
+ if (!route) return; // The session mirror will satisfy the user obligation; don't double-post (#196).
1750
+ replyToMessageId = route.replyToMessageId; replyTarget = route.to;
1751
+ }
1752
+ await publishCapturedResponse({ publishSessionEvent: (sessionEvent) => this.publishSessionEvent(sessionEvent), outbox: this.sessionOutbox, provider: this.options.provider, from: this.agentId, to: replyTarget, body, replyTo: replyToMessageId, session: { type: "response", origin: event.origin ?? "provider", ...(turnId ? { turnId } : {}) } });
1753
+ if (replyToMessageId) this.obligationCache.markDirty();
1754
+ return;
1755
+ }
1756
+ if (this.options.providerConfig.reasoningCapture === false) return;
1757
+ await this.publishSessionEvent({
1758
+ from: this.agentId,
1759
+ to: "user",
1760
+ body,
1761
+ session: {
1762
+ type: event.type,
1763
+ origin: event.origin ?? "provider",
1764
+ ...(turnId ? { turnId } : {}),
1765
+ ...(event.label ? { label: event.label } : {}),
1766
+ ...(event.status ? { status: event.status } : {}),
1767
+ ...(event.streaming !== undefined ? { streaming: event.streaming } : {}),
1768
+ ...(event.stepId ? { stepId: event.stepId } : {}),
1769
+ ...(event.final ? { final: true } : {}),
1770
+ },
1771
+ });
1772
+ }
1773
+
1774
+ // Remember an injected prompt so its UserPromptSubmit echo can be suppressed. Prunes
1775
+ // expired entries first; a defensive length cap guards against echoes that never
1776
+ // arrive (e.g. the provider drops a queued prompt) so the queue can't grow unbounded.
1777
+ private recordInjectedPrompt(text: string): void {
1778
+ const now = Date.now();
1779
+ this.injectedPrompts = this.injectedPrompts.filter((p) => now - p.at < PROMPT_ECHO_DEDUP_MS);
1780
+ this.injectedPrompts.push({ text, at: now });
1781
+ if (this.injectedPrompts.length > 50) this.injectedPrompts.shift();
1782
+ }
1783
+
1784
+ private isRunnerInjectedPrompt(text: string): boolean {
1785
+ if (RELAY_INJECTION_MARKERS.some((marker) => text.startsWith(marker))) return true;
1786
+ const now = Date.now();
1787
+ this.injectedPrompts = this.injectedPrompts.filter((p) => now - p.at < PROMPT_ECHO_DEDUP_MS);
1788
+ const idx = this.injectedPrompts.findIndex((p) => p.text === text);
1789
+ if (idx !== -1) {
1790
+ this.injectedPrompts.splice(idx, 1); // consume one — identical repeats each match once
1791
+ return true;
1792
+ }
1793
+ return false;
1794
+ }
1795
+
1796
+ // Force-clear a stuck provider-turn claim directly. Unlike the idle status path
1797
+ // it does NOT depend on a matching claim id (the Stop hook keys busy as
1798
+ // provider-turn:provider-turn, but reconciliation has no specific id), and it
1799
+ // deliberately leaves the reasoning tail alone so a late clear can't truncate
1800
+ // a turn's activity stream.
1801
+ private forceClearProviderTurn(reason: string): void {
1802
+ if (!this.claims.activeWork().some((w) => w.kind === "provider-turn")) return;
1803
+ this.sessionLog(`force-clearing stuck provider-turn (${reason})`);
1804
+ this.claims.clearWorkKind("provider-turn");
1805
+ this.currentTurnId = undefined;
1806
+ this.compactionMidTurn = false;
1807
+ this.publishStatus();
1808
+ }
1809
+
1810
+ // #650: reconcile pid-bearing busy work against actual process liveness. A tracked
1811
+ // command/shell that dies out-of-band (killed by the operator, crashes, orphaned)
1812
+ // sends no exit event, so its busy marker would otherwise never clear and the agent
1813
+ // shows `busy` forever — blocking transient auto-reap and (for Claude) holding quota.
1814
+ // The timer self-disarms once no pid-bearing work remains; it is armed lazily when a
1815
+ // busy report carrying pids arrives, so idle/non-tracked agents pay nothing.
1816
+ private armBusyLivenessReconcile(): void {
1817
+ if (this.livenessReconcileTimer || this.stopped) return;
1818
+ this.livenessReconcileTimer = setInterval(() => this.runBusyLivenessReconcile(), BUSY_LIVENESS_RECONCILE_MS);
1819
+ }
1820
+
1821
+ private disarmBusyLivenessReconcile(): void {
1822
+ if (this.livenessReconcileTimer) clearInterval(this.livenessReconcileTimer);
1823
+ this.livenessReconcileTimer = undefined;
1824
+ }
1825
+
1826
+ private runBusyLivenessReconcile(): void {
1827
+ if (this.stopped) {
1828
+ this.disarmBusyLivenessReconcile();
1829
+ return;
1830
+ }
1831
+ const removed = this.claims.reconcileLiveness(isPidAlive);
1832
+ if (removed.length > 0) {
1833
+ this.sessionLog(`cleared ${removed.length} stuck busy work item(s) whose tracked process died out-of-band: ${removed.map((w) => `${w.kind}:${w.id}`).join(", ")}`);
1834
+ // A cleared provider-turn must also reset the turn bookkeeping so a later idle
1835
+ // doesn't double-end it and the reasoning tail stops.
1836
+ if (removed.some((w) => w.kind === "provider-turn")) {
1837
+ this.currentTurnId = undefined;
1838
+ this.currentTurnStartedAt = undefined;
1839
+ this.compactionMidTurn = false;
1840
+ this.busyReconciler.disarm();
1841
+ this.stopReasoningTail();
1842
+ }
1843
+ this.publishStatus();
1844
+ }
1845
+ if (!this.claims.hasLivenessTrackedWork()) this.disarmBusyLivenessReconcile();
1846
+ }
1847
+
1848
+ // --- Turn-step tailer (item 5) ------------------------------------------------------
1849
+ // Tail the in-flight turn's Claude transcript and surface new narration/reasoning/tool
1850
+ // steps as session events, in transcript order. `narration` (the agent's intermediate
1851
+ // text) is the primary visible content; reasoning visibility is a client-side toggle.
1852
+ // Coalesced and coarse; the final response still comes through publishSessionTurn.
1853
+ // `reasoningCapture: false` disables the whole live trace (server-side kill switch).
1854
+ private startReasoningTail(transcriptPath: string): void {
1855
+ if (this.options.providerConfig.reasoningCapture === false) return;
1856
+ this.stopReasoningTail();
1857
+ // Track emitted steps by content signature, not by index/count: the "latest
1858
+ // turn" window in the transcript can shrink/reset (a tool_result entry, a
1859
+ // mid-turn user line), and an index cursor would then either re-emit or stall
1860
+ // and drop the rest of the turn. A seen-set is idempotent under any reshuffle.
1861
+ // The signature is salted with each step's occurrence-within-window (stepDedupKeys)
1862
+ // so two identical steps in one turn — same tool, same input — both surface (#265).
1863
+ const seen = new Set<string>();
1864
+ const emittedNarrationKeys = new Set<string>();
1865
+ const transcriptTail = new IncrementalClaudeTranscriptTail();
1866
+ const turnIdAtStart = this.currentTurnId;
1867
+ // On the first poll the new prompt usually hasn't landed in the transcript yet,
1868
+ // so extractLatestTurnSteps still returns the PRIOR (completed) turn. Seed those
1869
+ // signatures as already-seen so we don't replay last turn's reasoning/tools as
1870
+ // this turn's activity. Once our prompt lands the window resets at the new user
1871
+ // boundary and genuinely-new steps emit normally. Only seed when the transcript
1872
+ // is complete (last entry is an end_turn assistant) — otherwise we're already
1873
+ // inside the new turn and those steps are legitimately ours.
1874
+ let seeded = false;
1875
+ let pendingPoll = Promise.resolve();
1876
+ const pollBody = async (): Promise<void> => {
1877
+ const result = await transcriptTail.poll(transcriptPath);
1878
+ if (!result.available) return;
1879
+ if (result.reset) { seen.clear(); emittedNarrationKeys.clear(); seeded = false; }
1880
+ if (!result.changed && (seeded || (!result.steps.length && !result.transcriptLooksComplete))) return;
1881
+ const keyed = stepDedupKeys(result.steps).map((sig, i) => ({ sig, step: result.steps[i]! }));
1882
+ if (!seeded) {
1883
+ seeded = true;
1884
+ if (result.transcriptLooksComplete) {
1885
+ for (const { sig } of keyed) seen.add(sig);
1886
+ }
1887
+ }
1888
+ const turnId = this.currentTurnId ?? turnIdAtStart;
1889
+ let emitted = 0;
1890
+ const batch: Array<{ from: string; to: string; body: string; session: MessageSessionMeta }> = [];
1891
+ for (const { sig, step } of keyed) {
1892
+ if (seen.has(sig)) continue;
1893
+ seen.add(sig);
1894
+ emitted += 1;
1895
+ batch.push({
1896
+ from: this.agentId,
1897
+ to: "user",
1898
+ body: step.text,
1899
+ session: { type: step.type, origin: "provider", ...(turnId ? { turnId } : {}), ...(step.label ? { label: step.label } : {}) },
1900
+ });
1901
+ if (step.type === "narration") emittedNarrationKeys.add(sig);
1902
+ }
1903
+ this.publishSessionEventsBatch(batch);
1904
+ if (emitted) this.sessionDebug(`reasoning tail emitted ${emitted} step(s) (turn ${turnId ?? "?"}, ${seen.size} total)`);
1905
+ };
1906
+ const poll = (): Promise<void> => {
1907
+ const next = pendingPoll.then(pollBody, pollBody);
1908
+ pendingPoll = next.catch(() => {});
1909
+ return next;
1910
+ };
1911
+ this.reasoningTail = { emittedNarrationKeys, poll, timer: setInterval(() => { void poll(); }, REASONING_POLL_MS) };
1912
+ this.sessionLog(`reasoning tail started (turn ${turnIdAtStart ?? "?"})`);
1913
+ void poll();
1914
+ }
1915
+
1916
+ private async drainReasoningTail(): Promise<void> {
1917
+ if (this.options.providerConfig.reasoningCapture === false) return;
1918
+ await this.reasoningTail?.poll();
1919
+ }
1920
+
1921
+ private preFlushBodyAlreadyMirroredByReasoningTail(jsonl: string, body: string): boolean {
1922
+ const tail = this.reasoningTail;
1923
+ if (!tail || !tail.emittedNarrationKeys.size) return false;
1924
+ const steps = extractLatestTurnSteps(jsonl);
1925
+ const keys = stepDedupKeys(steps);
1926
+ const mirrored = steps
1927
+ .map((step, index) => ({ step, key: keys[index]! }))
1928
+ .filter(({ step, key }) => step.type === "narration" && tail.emittedNarrationKeys.has(key))
1929
+ .map(({ step }) => step.text)
1930
+ .join("\n\n")
1931
+ .trim();
1932
+ const trimmed = body.trim();
1933
+ return mirrored === trimmed || mirrored.endsWith(`\n\n${trimmed}`);
1934
+ }
1935
+
1936
+ private stopReasoningTail(): void {
1937
+ if (this.reasoningTail) clearInterval(this.reasoningTail.timer);
1938
+ this.reasoningTail = undefined;
1939
+ }
1940
+
1941
+ // Mirror a discreet, durable "context compacted" marker into chat via the existing
1942
+ // session-mirror lane (not a parallel channel). `notice` renders as an inline timeline
1943
+ // marker (never a bubble) and survives reload, unlike the ephemeral timeline-status one.
1944
+ private publishCompactionNotice(): void {
1945
+ this.publishSessionEvent({
1946
+ from: this.agentId,
1947
+ to: "user",
1948
+ body: "🗜 Context compacted",
1949
+ session: { type: "notice", origin: "provider", label: "compacted", ...(this.currentTurnId ? { turnId: this.currentTurnId } : {}) },
1950
+ });
1951
+ }
1952
+
1953
+ // #286: a discreet, durable chat marker when a usage/rate-limit hold begins, via
1954
+ // the same session-mirror lane as the compaction notice. Outbound session event
1955
+ // (NOT an inbound message) so it shows in the dashboard chat WITHOUT waking a turn
1956
+ // — waking a still-limited agent would just re-fail. The relay sends the waking
1957
+ // resume message later, once the window has reset.
1958
+ private publishRateLimitNotice(providerState: ProviderState): void {
1959
+ const label = providerState.label || "usage limit reached";
1960
+ this.publishSessionEvent({
1961
+ from: this.agentId,
1962
+ to: "user",
1963
+ body: `⏳ ${label} — holding; agent-relay will auto-resume at reset.`,
1964
+ session: { type: "notice", origin: "provider", label: "rate-limit", ...(this.currentTurnId ? { turnId: this.currentTurnId } : {}) },
1965
+ });
1966
+ }
1967
+
1968
+ private publishStatus(): void {
1969
+ this.claims.expire();
1970
+ const status = this.claims.currentStatus();
1971
+ const agentStatus = runnerAgentStatus(status);
1972
+ const activeWork = this.claims.activeWork();
1973
+ const activeSubagents = activeWork.filter((item) => item.kind === "subagent");
1974
+ const terminalFailure = this.terminalFailure;
1975
+ const providerState = terminalFailure?.providerState ?? this.rateLimitHold ?? providerStateFromActiveWork(activeWork);
1976
+ this.bus.setSemanticStatus(status === "offline" || status === "error" ? "idle" : status);
1977
+ const timelineEvent = this.pendingTimelineEvent;
1978
+ this.pendingTimelineEvent = undefined;
1979
+ // §4.1 Runner↔provider liveness contract (#725 / #746): fold the raw transport/timeline/
1980
+ // token/work-kind signals into ONE normalized verdict the server reads instead of six raw
1981
+ // `meta` fields. The adapter owns what counts as progress for its provider (#713/#724);
1982
+ // mock adapters fall back to the shared fold.
1983
+ const livenessInputs: LivenessInputs = {
1984
+ transport: this.bus.transportState,
1985
+ ...(this.lastProviderEventAt !== undefined ? { lastProviderEventAt: this.lastProviderEventAt } : {}),
1986
+ ...(timelineEvent ? { timelineEvent } : {}),
1987
+ activeWork,
1988
+ };
1989
+ const liveness = this.options.adapter.liveness
1990
+ ? this.options.adapter.liveness(livenessInputs)
1991
+ : computeLivenessSignal(livenessInputs);
1992
+ this.bus.status({
1993
+ agentStatus,
1994
+ ready: agentStatus !== "offline" && !this.stopped,
1995
+ meta: {
1996
+ runnerId: this.options.runnerId,
1997
+ startedAt: this.options.startedAt,
1998
+ tmuxSession: providerTerminalSession(this.process) ?? this.options.tmuxSession ?? null,
1999
+ tmuxSocket: providerTerminalSocket(this.process) ?? null,
2000
+ policyName: this.options.policyName ?? null,
2001
+ spawnRequestId: this.options.spawnRequestId ?? null,
2002
+ taskId: this.options.taskId ?? null,
2003
+ automationId: this.options.automationId ?? null,
2004
+ automationRunId: this.options.automationRunId ?? null,
2005
+ workspace: this.options.workspace ?? null,
2006
+ lifecycle: this.options.lifecycle ?? "persistent", workspaceMode: this.options.workspace?.requestedMode ?? this.options.workspace?.mode ?? workspaceModeFromEnv() ?? null,
2007
+ lifecycleAction: this.lifecycleAction ?? null,
2008
+ profile: this.options.profile ?? null,
2009
+ ...(status === "error" ? { terminalStatus: "error" } : {}),
2010
+ ...(terminalFailure ? {
2011
+ lastError: terminalFailure.message,
2012
+ terminalFailureReason: terminalFailure.reason,
2013
+ terminalFailureMessage: terminalFailure.message,
2014
+ } : {}),
2015
+ // #633 — terminal provider-exit marker (runner alive, provider gone). Drives the server's
2016
+ // silent-death terminal event + #636 post-mortem.
2017
+ ...(this.terminalProviderExit ? {
2018
+ terminalProviderExit: this.terminalProviderExit,
2019
+ ...(typeof this.terminalProviderExit.claudeResumeId === "string"
2020
+ ? { lastError: `Claude provider exited; resume id ${this.terminalProviderExit.claudeResumeId} captured for manual recovery` }
2021
+ : { lastError: "Claude provider exited; manual intervention required" }),
2022
+ } : {}),
2023
+ busyReasons: this.claims.reasons(),
2024
+ completedProviderTurns: this.completedProviderTurns,
2025
+ activeWork,
2026
+ activeSubagents,
2027
+ activeSubagentCount: activeSubagents.length,
2028
+ ...(this.lastProviderEventAt !== undefined ? { lastProviderEventAt: this.lastProviderEventAt } : {}),
2029
+ providerState,
2030
+ auth: {
2031
+ profileId: this.currentTokenProfileId ?? null,
2032
+ jti: this.currentTokenJti ?? null,
2033
+ expiresAt: this.currentTokenExpiresAt ?? null,
2034
+ rootTokenFallback: this.options.rootTokenFallback === true,
2035
+ },
2036
+ ...(timelineEvent ? { timelineEvent } : {}),
2037
+ transport: this.bus.transportState,
2038
+ // #746 — normalized liveness verdict; the server reads ONLY this for stuck-busy.
2039
+ // The raw fields above stay for other consumers (timeline rendering, diagnostics).
2040
+ liveness,
2041
+ },
2042
+ });
2043
+ }
2044
+
2045
+ private startHttpLiveness(): void {
2046
+ if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
2047
+ void this.publishHttpLiveness();
2048
+ this.httpLivenessTimer = setInterval(() => {
2049
+ void this.publishHttpLiveness();
2050
+ }, HTTP_LIVENESS_INTERVAL_MS);
2051
+ }
2052
+
2053
+ private async publishHttpLiveness(): Promise<void> {
2054
+ if (this.stopped || this.httpLivenessInFlight || this.httpLivenessAuthFailed) return;
2055
+ this.httpLivenessInFlight = true;
2056
+ const status = this.claims.currentStatus();
2057
+ const agentStatus = runnerAgentStatus(status);
2058
+ try {
2059
+ await this.http.setStatus(this.agentId, agentStatus, this.options.instanceId);
2060
+ await this.http.setReady(this.agentId, agentStatus !== "offline", this.options.instanceId);
2061
+ await this.http.heartbeat(this.agentId, this.options.instanceId);
2062
+ await this.claudeQuotaHarvest.publish().catch((error) => this.logRunnerDiagnostic(`passive Claude quota publish failed: ${errMessage(error)}`));
2063
+ await this.bootstrapUnreadMessages();
2064
+ } catch (error) {
2065
+ if (!this.stopped) this.handleHttpLivenessFailure(error);
2066
+ } finally {
2067
+ this.httpLivenessInFlight = false;
2068
+ }
2069
+ }
2070
+
2071
+ private handleHttpLivenessFailure(error: unknown): void {
2072
+ const authFailed = isHttpAuthError(error);
2073
+ this.logHttpLivenessFailure(error, authFailed);
2074
+ if (!authFailed) return;
2075
+ this.httpLivenessAuthFailed = true;
2076
+ if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
2077
+ this.httpLivenessTimer = undefined;
2078
+ // A 401/403 here is the only timely signal that the token died — stopping the
2079
+ // liveness timer means there is no second chance, so recover from THIS failure.
2080
+ this.recoverRuntimeTokenAfterAuthFailure("http-liveness");
2081
+ }
2082
+
2083
+ // A definitive relay auth failure (401/403) means the runtime token is dead right
2084
+ // now — expired, or (the common case) revoked when the relay marked this agent
2085
+ // stale across its own restart/reconnect. The proactive renew timer is keyed to
2086
+ // TTL and structurally cannot catch a revocation, so the auth failure itself must
2087
+ // drive recovery. renewRuntimeToken() prefers an orchestrator re-mint, which heals
2088
+ // an expired token AND a token revoked-as-"stale" (the restart strand, #484) —
2089
+ // but NOT a hard kill (admin revoke / rotation / agent removal), which stays dead
2090
+ // by design. Debounced so a burst of failing calls re-mints once.
2091
+ private recoverRuntimeTokenAfterAuthFailure(source: string): void {
2092
+ if (this.stopped || this.tokenRenewInFlight) return;
2093
+ if (!this.isRuntimeTokenRenewable() && !this.canRemintViaOrchestrator()) return;
2094
+ const now = Date.now();
2095
+ if (this.reactiveTokenRecoveryAt && now - this.reactiveTokenRecoveryAt < REACTIVE_TOKEN_RECOVERY_DEBOUNCE_MS) return;
2096
+ this.reactiveTokenRecoveryAt = now;
2097
+ this.logRunnerDiagnostic(`[runner] relay auth failure on ${source}; recovering runtime token`);
2098
+ void this.renewRuntimeToken();
2099
+ }
2100
+
2101
+ private logHttpLivenessFailure(error: unknown, authFailed: boolean): void {
2102
+ const key = httpErrorKey(error);
2103
+ const now = Date.now();
2104
+ if (
2105
+ this.httpLivenessLastLog?.key === key &&
2106
+ now - this.httpLivenessLastLog.at < HTTP_LIVENESS_LOG_INTERVAL_MS
2107
+ ) return;
2108
+ this.httpLivenessLastLog = { key, at: now };
2109
+ const suffix = authFailed ? "auth failed; stopping HTTP liveness retries until restart" : String(error);
2110
+ this.logRunnerDiagnostic(`[runner] HTTP liveness update failed: ${suffix}`);
2111
+ }
2112
+
2113
+ // Runner operational diagnostics (HTTP liveness, token renewal failures). Routed
2114
+ // through the leveled logger at warn — see logger.ts. Kept as a thin wrapper so
2115
+ // the existing call sites and their `[runner]` framing stay put.
2116
+ private logRunnerDiagnostic(message: string): void {
2117
+ logger.warn("runner", message.replace(/^\[runner\]\s*/, ""));
2118
+ }
2119
+
2120
+ // Session-mirror diagnostics → the leveled logger (component "mirror"), written
2121
+ // to the dashboard-surfaced session-mirror-<agent>.log. Key transitions log at
2122
+ // info; the single place to look when chat/terminal sync misbehaves.
2123
+ private sessionLog(message: string): void {
2124
+ logger.info("mirror", message);
2125
+ }
2126
+
2127
+ // Verbose, high-frequency lines (per-probe, per-emit) — surfaced only at log
2128
+ // level "debug" (AGENT_RELAY_LOG_LEVEL=debug, or flip live via /log-level).
2129
+ private sessionDebug(message: string): void {
2130
+ logger.debug("mirror", message);
2131
+ }
2132
+
2133
+ private ensureScratch(): void {
2134
+ try {
2135
+ this.scratch = ensureSessionScratch({
2136
+ agentId: this.agentId,
2137
+ cwd: this.options.cwd,
2138
+ fallbackBaseDir: orchestratorBaseDirFromEnv(),
2139
+ });
2140
+ } catch (error) {
2141
+ this.logRunnerDiagnostic(`session scratch setup failed: ${errMessage(error)}`);
2142
+ }
2143
+ }
2144
+
2145
+ private async sweepStaleScratch(): Promise<void> {
2146
+ try {
2147
+ const agents = await this.http.listAgents().catch(() => [] as Awaited<ReturnType<RelayHttpClient["listAgents"]>>);
2148
+ const keep = new Set(agents.map((a) => a.id)); keep.add(this.agentId); const now = Date.now();
2149
+ const removed = sweepStaleSessions({
2150
+ cwd: this.options.cwd,
2151
+ fallbackBaseDir: orchestratorBaseDirFromEnv(),
2152
+ keepAgentIds: keep,
2153
+ now,
2154
+ });
2155
+ if (removed.length) this.logRunnerDiagnostic(`swept ${removed.length} stale session scratch dir(s)`);
2156
+ const removedLaunchers = sweepStaleLaunchers({ keepAgentIds: keep, now }); if (removedLaunchers.length) this.logRunnerDiagnostic(`swept ${removedLaunchers.length} stale launcher script(s)`);
2157
+ } catch {}
2158
+ }
2159
+
2160
+ private scheduleRuntimeTokenRenewal(delayMs?: number): void {
2161
+ if (this.tokenRenewTimer) clearTimeout(this.tokenRenewTimer);
2162
+ this.tokenRenewTimer = undefined;
2163
+ if (this.stopped) return;
2164
+ const canSelfRenew = this.isRuntimeTokenRenewable();
2165
+ const canRemint = this.canRemintViaOrchestrator();
2166
+ // Keep the renewal clock ticking as long as the session can recover its token
2167
+ // by EITHER path. Without the re-mint fallback an expired token would stop the
2168
+ // timer forever (the old deadlock that stranded live agents off the bus).
2169
+ if (!canSelfRenew && !canRemint) return;
2170
+ let computedDelay = delayMs;
2171
+ if (computedDelay === undefined) {
2172
+ computedDelay = canSelfRenew
2173
+ ? runtimeTokenRenewDelayMs(this.currentTokenExpiresAt!, Date.now())
2174
+ : TOKEN_RENEW_RETRY_MS; // expired but re-mintable → retry via orchestrator soon
2175
+ if (computedDelay === undefined) computedDelay = TOKEN_RENEW_RETRY_MS;
2176
+ }
2177
+ const schedule = runtimeTokenRenewTimerSchedule(computedDelay);
2178
+ if (!schedule) return;
2179
+ this.tokenRenewTimer = setTimeout(() => {
2180
+ this.tokenRenewTimer = undefined;
2181
+ if (!schedule.renew) {
2182
+ this.scheduleRuntimeTokenRenewal();
2183
+ return;
2184
+ }
2185
+ void this.renewRuntimeToken();
2186
+ }, schedule.delayMs);
2187
+ }
2188
+
2189
+ // Can the runner self-renew right now? Requires a non-expired runner-profile token
2190
+ // (the relay rejects renewal of an expired token).
2191
+ private isRuntimeTokenRenewable(): boolean {
2192
+ return Boolean(
2193
+ this.currentToken &&
2194
+ this.currentTokenExpiresAt &&
2195
+ this.currentTokenExpiresAt * 1000 > Date.now() &&
2196
+ (this.currentTokenProfileId === "provider-agent" || this.currentTokenProfileId === "provider-interactive"),
2197
+ );
2198
+ }
2199
+
2200
+ // Can the runner recover its token via the orchestrator? Works even when the token
2201
+ // is already expired — the orchestrator's standing credential is the authority.
2202
+ private canRemintViaOrchestrator(): boolean {
2203
+ return Boolean(
2204
+ orchestratorUrlFromEnv() &&
2205
+ this.currentToken &&
2206
+ (this.currentTokenProfileId === "provider-agent" || this.currentTokenProfileId === "provider-interactive"),
2207
+ );
2208
+ }
2209
+
2210
+ private async renewRuntimeToken(): Promise<void> {
2211
+ if (this.stopped || this.tokenRenewInFlight || !this.currentToken) return;
2212
+ this.tokenRenewInFlight = true;
2213
+ try {
2214
+ // Preferred path: self-renew directly against the relay while the token is
2215
+ // still valid. Cheapest and needs no orchestrator round-trip.
2216
+ if (this.isRuntimeTokenRenewable()) {
2217
+ try {
2218
+ const renewed = await this.http.renewRuntimeToken();
2219
+ this.applyRenewedToken(renewed.token, renewed.record, "runtime-token-renewed");
2220
+ return;
2221
+ } catch (error) {
2222
+ this.logRuntimeTokenRenewalFailure(error);
2223
+ // Relay unreachable or token rejected — fall through to orchestrator re-mint.
2224
+ }
2225
+ }
2226
+ // Recovery path: token expired, or self-renew failed. Ask the orchestrator —
2227
+ // it holds a long-lived credential and can mint a fresh runner token, so a
2228
+ // live session heals instead of being stranded off the bus.
2229
+ if (this.canRemintViaOrchestrator() && await this.remintViaOrchestrator()) return;
2230
+ this.pendingTimelineEvent = {
2231
+ status: "runtime-token-renewal-failed",
2232
+ timestamp: Date.now(),
2233
+ };
2234
+ this.publishStatus();
2235
+ this.scheduleRuntimeTokenRenewal(TOKEN_RENEW_RETRY_MS);
2236
+ } finally {
2237
+ this.tokenRenewInFlight = false;
2238
+ }
2239
+ }
2240
+
2241
+ // Apply a freshly issued token across every live surface — runner state, the
2242
+ // RunnerOptions bag (re-injected into the provider on respawn), the HTTP client,
2243
+ // the bus client — then force a bus handshake with the new token and reschedule.
2244
+ private applyRenewedToken(
2245
+ token: string,
2246
+ record: { jti: string; profileId?: string; expiresAt?: number },
2247
+ status: "runtime-token-renewed" | "runtime-token-reminted",
2248
+ ): void {
2249
+ this.currentToken = token;
2250
+ this.currentTokenJti = record.jti;
2251
+ this.currentTokenProfileId = record.profileId ?? this.currentTokenProfileId;
2252
+ this.currentTokenExpiresAt = record.expiresAt;
2253
+ this.options.token = token;
2254
+ this.options.tokenJti = record.jti;
2255
+ this.options.tokenProfileId = this.currentTokenProfileId;
2256
+ this.options.tokenExpiresAt = this.currentTokenExpiresAt;
2257
+ this.http.setToken(token);
2258
+ this.bus.setToken(token);
2259
+ // The proxy reads the token live via getToken(), so forwarding already uses the new one.
2260
+ // A re-mint can change scope (e.g. a profile change), so refresh the relay tool list and
2261
+ // emit tools/list_changed if the visible set changed (#215 — token-scope transition).
2262
+ void this.proxy?.refreshTools().catch(() => {});
2263
+ this.httpLivenessAuthFailed = false;
2264
+ this.reactiveTokenRecoveryAt = undefined;
2265
+ // An earlier auth failure may have stopped the liveness loop; restart it so the
2266
+ // agent reports live again on the fresh token. startHttpLiveness clears any
2267
+ // existing timer first, so this is safe on the normal (proactive) renew path too.
2268
+ this.startHttpLiveness();
2269
+ this.pendingTimelineEvent = { status, id: record.jti, timestamp: Date.now() };
2270
+ this.bus.reconnectTransport(status === "runtime-token-reminted" ? "runtime token re-minted" : "runtime token renewed");
2271
+ this.publishStatus();
2272
+ this.scheduleRuntimeTokenRenewal();
2273
+ }
2274
+
2275
+ // Recover the runtime token through the orchestrator. The runner proxies its own
2276
+ // (possibly expired) token; the orchestrator re-mints it via the relay using its
2277
+ // standing credential. Returns true on success.
2278
+ private async remintViaOrchestrator(): Promise<boolean> {
2279
+ const orchUrl = orchestratorUrlFromEnv();
2280
+ if (!orchUrl || !this.currentToken) return false;
2281
+ try {
2282
+ const res = await fetch(`${orchUrl.replace(/\/+$/, "")}/api/runtime-tokens/runner-renew`, {
2283
+ method: "POST",
2284
+ headers: { "Content-Type": "application/json" },
2285
+ body: JSON.stringify({ token: this.currentToken }),
2286
+ signal: AbortSignal.timeout(10_000),
2287
+ });
2288
+ if (!res.ok) return false;
2289
+ const renewed = await res.json() as { token?: string; record?: { jti: string; profileId?: string; expiresAt?: number } };
2290
+ if (!renewed?.token || !renewed.record) return false;
2291
+ this.applyRenewedToken(renewed.token, renewed.record, "runtime-token-reminted");
2292
+ this.logRunnerDiagnostic(`[runner] runtime token re-minted via orchestrator (jti ${renewed.record.jti})`);
2293
+ return true;
2294
+ } catch (error) {
2295
+ this.logRuntimeTokenRenewalFailure(error);
2296
+ return false;
2297
+ }
2298
+ }
2299
+
2300
+ private logRuntimeTokenRenewalFailure(error: unknown): void {
2301
+ const key = httpErrorKey(error);
2302
+ const now = Date.now();
2303
+ if (
2304
+ this.tokenRenewLastLog?.key === key &&
2305
+ now - this.tokenRenewLastLog.at < HTTP_LIVENESS_LOG_INTERVAL_MS
2306
+ ) return;
2307
+ this.tokenRenewLastLog = { key, at: now };
2308
+ this.logRunnerDiagnostic(`[runner] runtime token renewal failed: ${String(error)}`);
2309
+ }
2310
+
2311
+ private heartbeatRuntimeMeta(): Record<string, unknown> | undefined {
2312
+ const processContext = this.latestProcessContext();
2313
+ const probeMetrics = this.latestProbeMetrics();
2314
+ const probeContext = probeMetrics ? contextStateFromProbeMetrics(probeMetrics) : undefined;
2315
+ const context = processContext ?? probeContext;
2316
+ const quota = probeMetrics ? quotaStateFromProbeMetrics(probeMetrics) : undefined;
2317
+ const terminalSession = providerTerminalSession(this.process);
2318
+ const terminalSocket = providerTerminalSocket(this.process);
2319
+ const probeModel: ProbeModelInfo | undefined = probeMetrics?.model || probeMetrics?.effort
2320
+ ? { model: probeMetrics.model, effort: probeMetrics.effort }
2321
+ : undefined;
2322
+ const meta: Record<string, unknown> = {
2323
+ providerCapabilities: runtimeProviderCapabilities(this.options, context, probeModel, quota),
2324
+ ...(terminalSession ? { tmuxSession: terminalSession } : {}),
2325
+ ...(terminalSocket ? { tmuxSocket: terminalSocket } : {}),
2326
+ };
2327
+ if (context) meta.context = context;
2328
+ return meta;
2329
+ }
2330
+
2331
+ private latestProcessContext(): ContextState | undefined {
2332
+ const getContext = this.process?.meta?.getContext;
2333
+ if (typeof getContext === "function") {
2334
+ const context = getContext();
2335
+ if (isContextState(context)) return context;
2336
+ }
2337
+ const context = this.process?.meta?.context;
2338
+ return isContextState(context) ? context : undefined;
2339
+ }
2340
+
2341
+ private latestProbeMetrics() {
2342
+ return readRunnerContextProbeState(this.agentId);
2343
+ }
2344
+
2345
+ private matchesMessage(message: Message): boolean {
2346
+ return runnerMessageMatches(message, {
2347
+ agentId: this.agentId,
2348
+ label: this.options.label,
2349
+ provider: this.options.provider,
2350
+ tags: this.options.tags,
2351
+ capabilities: this.options.capabilities,
2352
+ defaultTags: this.options.providerConfig.defaultTags,
2353
+ defaultCapabilities: this.options.providerConfig.defaultCapabilities,
2354
+ allowBroadTargets: this.options.agentProfile?.relay?.context !== false,
2355
+ });
2356
+ }
2357
+
2358
+ private startClaimRenewer(): void {
2359
+ if (this.claimRenewTimer) return;
2360
+ this.claimRenewTimer = setInterval(() => {
2361
+ void this.renewActiveTaskClaims();
2362
+ }, CLAIM_RENEW_INTERVAL_MS);
2363
+ }
2364
+
2365
+ private async renewActiveTaskClaims(): Promise<void> {
2366
+ for (const claim of this.activeTaskClaims.values()) {
2367
+ const renewed = await this.http.renewMessageClaim(claim.messageId, this.agentId).catch(() => ({ ok: false, claimExpiresAt: undefined }));
2368
+ if (renewed.ok) {
2369
+ this.claims.startClaim("message", String(claim.messageId), renewed.claimExpiresAt);
2370
+ this.claims.startClaim("task", String(claim.taskId), renewed.claimExpiresAt);
2371
+ }
2372
+ }
2373
+ this.publishStatus();
2374
+ }
2375
+
2376
+ private async completeObservedTaskClaims(): Promise<void> {
2377
+ const completed = [...this.activeTaskClaims.values()].filter((claim) => claim.observedProviderBusy);
2378
+ for (const claim of completed) {
2379
+ const ok = await this.updateTaskStatus(claim.taskId, {
2380
+ status: "done",
2381
+ agentId: this.agentId,
2382
+ body: `Runner observed provider completion for message #${claim.messageId}`,
2383
+ metadata: { messageId: claim.messageId, completedBy: "runner" },
2384
+ })
2385
+ .then(() => true)
2386
+ .catch((error) => {
2387
+ logger.error("task", `task ${claim.taskId} completion update failed: ${error}`);
2388
+ return false;
2389
+ });
2390
+ if (!ok) continue;
2391
+ this.activeTaskClaims.delete(claim.messageId);
2392
+ this.claims.finishClaim("message", String(claim.messageId));
2393
+ this.claims.finishClaim("task", String(claim.taskId));
2394
+ }
2395
+ if (completed.length > 0) this.publishStatus();
2396
+ }
2397
+
2398
+ private async updateTaskStatus(taskId: number, input: TaskStatusInput): Promise<void> {
2399
+ await this.http.updateTaskStatus(taskId, input);
2400
+ }
2401
+
2402
+ private clearActiveClaim(message: Message): void {
2403
+ const taskId = taskIdFromMessage(message);
2404
+ this.claims.finishClaim("message", String(message.id));
2405
+ if (!taskId) return;
2406
+ this.activeTaskClaims.delete(message.id);
2407
+ this.claims.finishClaim("task", String(taskId));
2408
+ }
2409
+ }