agent-relay-runner 0.62.3 → 0.63.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/runner.ts CHANGED
@@ -1,2660 +1,14 @@
1
- import { hostname } from "node:os";
2
- import { closeSync, mkdirSync, openSync, readSync, statSync, writeFileSync } from "node:fs";
3
- import { readFile } from "node:fs/promises";
4
- import { dirname, join } from "node:path";
5
- import type { AgentLifecycle, AgentProfile, ContextState, Message, MessageSessionMeta, ProviderCapabilities, SendMessageInput, TaskStatusInput, WorkspaceMetadata } from "agent-relay-sdk";
6
- import { errMessage, RelayBusClient, RelayHttpClient } from "agent-relay-sdk";
7
- import { contextStateFromProbeMetrics, readContextProbeState } from "agent-relay-sdk/context-probe";
8
- import { providerAttachmentText, providerMessageText } from "./adapter";
9
- import type { ManagedProcess, ProviderAdapter, ProviderConfig, ProviderPermissionDecision, ProviderPermissionDecisionInput, ProviderSessionEvent, ProviderStatusUpdate, RunnerSpawnConfig, SemanticStatus, TerminalAttachSpec } from "./adapter";
10
- import { messagesWithCachedAttachments } from "./attachment-cache";
11
- import { resolveProjectName } from "./config";
12
- import { ClaimTracker } from "./claim-tracker";
13
- import { startControlServer, type ControlServer } from "./control-server";
14
- import { ReplyObligationCache, obligationRequiresExplicitReply } from "./reply-obligation-cache";
15
- import { Outbox, type OutboxRecord } from "./outbox";
16
- import { extractLastAssistantTurn, extractLastAssistantTurnAfterEntry, countTranscriptEntries, extractFinalAssistantMessage, extractHookAssistantMessage, extractLatestTurnSteps, stepDedupKeys, transcriptLooksComplete } from "./adapters/claude-transcript";
17
- import { computeContextRatio } from "./session-insights";
18
- import { agentProfileProjectionReport } from "./profile-projection";
19
- import { profileUsesHostProviderGlobals } from "./profile-home";
20
- import { RELAY_MCP_TOKEN_ENV, relayMcpEndpoint } from "./relay-mcp";
21
- import { RelayMcpProxy } from "./relay-mcp-proxy";
22
- import { runtimeMetadata } from "./version";
23
- import { logger, parseLogLevel } from "./logger";
24
- import { ensureSessionScratch, reapSessionScratch, sweepStaleSessions, type SessionScratchLayout } from "./session-scratch";
25
- import { boundContinuationArchiveSegment, deliverContinuationArchiveRecord } from "./continuation-archive";
26
-
27
- // A destructive session transition. The runner runs end-of-session work (Insights
28
- // capture, #183/#184) before the invasive operation and, during that window, presents a
29
- // distinct non-addressable lifecycle state. Bus commands and provider hooks (Claude
30
- // PreCompact / SessionEnd) both normalize to one of these.
31
- type SessionDestroyReason = "compact" | "clear" | "restart" | "shutdown" | "kill" | "crash";
32
-
33
- // Reasons after which the runner process won't survive to drain the durable outbox (and the
34
- // per-agent outbox is never reopened once the agent is gone). For these, pre-destroy must
35
- // block on delivery of the just-captured Insights datapoint, not just enqueue it (#183).
36
- // `restart` (bus command) deliberately excluded: the runner stays alive and drains normally.
37
- function reasonExitsRunner(reason: SessionDestroyReason): boolean {
38
- return reason === "shutdown" || reason === "kill" || reason === "crash";
39
- }
40
-
41
- // `finalizing-<reason>` is the transient pre-destroy window; the others are the executing
42
- // teardown states the dashboard already renders.
43
- type LifecycleAction =
44
- | "shutting-down" | "killing" | "restarting"
45
- | `finalizing-${SessionDestroyReason}`;
46
-
47
- // Pre-destroy work is best-effort and must never hang teardown. Capping it keeps a slow
48
- // transcript read or a wedged provider from stalling a shutdown the operator asked for.
49
- const PRE_DESTROY_TIMEOUT_MS = 4_000;
50
-
51
- // Bounded window to deliver the durable outbox before an exit-bound teardown (#183). Kept
52
- // short so a wedged/down server can't stall an operator-requested shutdown for long; a
53
- // row that still can't land is logged, not silently dropped.
54
- const OUTBOX_FLUSH_TIMEOUT_MS = 3_000;
55
-
56
- // Map a lifecycle bus command to its destructive boundary reason, or undefined for
57
- // non-destructive commands (interrupt, inject, reconnect, permission decisions).
58
- function boundaryReasonForCommand(type: string): SessionDestroyReason | undefined {
59
- switch (type) {
60
- case "agent.compact": return "compact";
61
- case "agent.clearContext": return "clear";
62
- case "agent.restart": return "restart";
63
- case "agent.shutdown": return "shutdown";
64
- case "agent.kill": return "kill";
65
- default: return undefined;
66
- }
67
- }
68
-
69
- interface RunnerOptions {
70
- provider: string;
71
- model?: string;
72
- effort?: string;
73
- runnerId: string;
74
- instanceId: string;
75
- agentId?: string;
76
- relayUrl: string;
77
- token?: string;
78
- tokenJti?: string;
79
- tokenProfileId?: string;
80
- tokenExpiresAt?: number;
81
- rootTokenFallback?: boolean;
82
- cwd: string;
83
- headless: boolean;
84
- approvalMode: string;
85
- label?: string;
86
- rig?: string;
87
- profile?: string;
88
- agentProfile?: AgentProfile;
89
- prompt?: string;
90
- systemPromptAppend?: string;
91
- tags: string[];
92
- capabilities: string[];
93
- providerArgs: string[];
94
- policyName?: string;
95
- spawnRequestId?: string;
96
- automationId?: string; automationRunId?: string;
97
- workspace?: WorkspaceMetadata; lifecycle?: AgentLifecycle;
98
- tmuxSession?: string;
99
- startedAt: number;
100
- providerConfig: ProviderConfig;
101
- adapter: ProviderAdapter;
102
- onProviderExit?: (code: number | null) => void;
103
- exitProcessOnShutdown?: boolean;
104
- }
105
-
106
- interface ActiveTaskClaim {
107
- messageId: number;
108
- taskId: number;
109
- observedProviderBusy: boolean;
110
- }
111
-
112
- const CLAIM_RENEW_INTERVAL_MS = 5 * 60 * 1000;
113
- const HTTP_LIVENESS_INTERVAL_MS = 20_000;
114
- const HTTP_LIVENESS_LOG_INTERVAL_MS = 5 * 60 * 1000;
115
- const TOKEN_RENEW_RETRY_MS = 60_000;
116
- // Debounce reactive token recovery so a burst of 401-ing calls in the same window
117
- // triggers a single re-mint attempt, not one per failing request.
118
- const REACTIVE_TOKEN_RECOVERY_DEBOUNCE_MS = 10_000;
119
- const UNEXPECTED_EXIT_WINDOW_MS = 2 * 60 * 1000;
120
- const RAPID_EXIT_MS = 30 * 1000;
121
- const MAX_RAPID_UNEXPECTED_EXITS = 3;
122
- const MAX_TIMER_DELAY_MS = 2_147_483_647;
123
- const LOG_TAIL_BYTES = 128 * 1024;
124
- // A UserPromptSubmit echo matching a runner-injected prompt within this window is
125
- // the same prompt arriving back from the provider — drop it to avoid a duplicate.
126
- const PROMPT_ECHO_DEDUP_MS = 30_000;
127
- // Busy reconciler: a conservative LAST-RESORT backstop for a turn that ended
128
- // without the provider's Stop hook clearing busy (e.g. ESC straight into the web
129
- // terminal). It must never fire during a live turn, so it requires a long,
130
- // unbroken idle streak — an active turn shows its working spinner well within
131
- // this window, which resets the streak. ~32s of uninterrupted idle = really done.
132
- const BUSY_RECONCILE_POLL_MS = 4_000;
133
- const BUSY_RECONCILE_IDLE_CONFIRM = 8;
134
- // When the reconciler never observed the provider busy this turn (a turn faster
135
- // than the 4s poll — common for short voice/autosend replies), it can't trust a
136
- // quick idle the way it does after seeing the spinner. But refusing forever wedged
137
- // fast turns in "busy" when the Stop hook's idle was lost (#199). So we still
138
- // force-clear, just after a much longer unbroken-idle window — an active turn would
139
- // have flashed its spinner into at least one of these probes and reset the streak.
140
- const BUSY_RECONCILE_IDLE_CONFIRM_NO_BUSY = 15;
141
- // After a dashboard interrupt, give the provider a moment to drop out of its turn,
142
- // then reconcile immediately so the user sees "stopped" without waiting for the backstop.
143
- const INTERRUPT_RECONCILE_DELAY_MS = 1_500;
144
- // Relay-injected content (delivered messages, memory context) is wrapped with
145
- // these markers; a UserPromptSubmit echo starting with one is a runner injection,
146
- // not a human typing into the terminal, so it must not be mirrored as a prompt.
147
- // `<task-notification>` is the harness wrapper for async monitor/task wake-ups
148
- // (which embed a `[relay message #…]` deeper inside, defeating a plain prefix
149
- // match) — it is emitted only by the harness, never typed by a human, so any
150
- // echo starting with it is a system injection, not a user prompt (#289).
151
- const RELAY_INJECTION_MARKERS = ["[relay message #", "[agent-relay", "<task-notification>"];
152
- // #329: a cold isolated profile (first-run trust/onboarding/install gates) can take far longer
153
- // than the warm-restart window to render an input-ready TUI, so the first prompt delivery gets a
154
- // generous timeout. If it still misses, the prompt is re-attempted on the next provider ready
155
- // signal (short timeout, since it just went idle), up to a bounded number of attempts.
156
- const INITIAL_PROMPT_FIRST_READY_TIMEOUT_MS = 30_000;
157
- const INITIAL_PROMPT_RETRY_READY_TIMEOUT_MS = 10_000;
158
- const MAX_INITIAL_PROMPT_ATTEMPTS = 6;
159
- // Reasoning tailer poll cadence (item 5). Coarse on purpose — reasoning is a
160
- // discreet progress signal, not a token stream, so ~1.2s keeps it light.
161
- const REASONING_POLL_MS = 1_200;
162
- const CLAUDE_RESUME_RE = /\bclaude\s+--resume\s+([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b/gi;
163
-
164
- interface RunnerTimelineEvent {
165
- status: string;
166
- id?: string;
167
- timestamp: number;
168
- title?: string;
169
- body?: string;
170
- icon?: string;
171
- metadata?: Record<string, unknown>;
172
- }
173
-
174
- export class AgentRunner {
175
- private readonly agentId: string;
176
- private readonly claims = new ClaimTracker();
177
- private readonly http: RelayHttpClient;
178
- private readonly bus: RelayBusClient;
179
- // Phase 2 (#196): the Stop hook reads reply obligations from this local snapshot, never
180
- // from the server — so a slow server can no longer wedge a turn (the crux fix).
181
- private readonly obligationCache: ReplyObligationCache;
182
- // Phase 2 (#196): Runner→server append-log events (session turns, reasoning, prompts,
183
- // insights, hook-fatal) go through this durable, disk-backed, timestamped queue instead of
184
- // direct fire-and-forget HTTP — so nothing is lost across a server/Runner restart.
185
- private readonly outbox: Outbox;
186
- // #332: a fast-lane outbox for display-only session-mirror trace events. Routing them through
187
- // their own FIFO queue stops one transient trace failure from head-of-line blocking real-message
188
- // delivery (and vice versa); its backoff is capped low since a stale live-mirror frame is cheap.
189
- private readonly sessionOutbox: Outbox;
190
- private currentToken?: string;
191
- private currentTokenJti?: string;
192
- private currentTokenProfileId?: string;
193
- private currentTokenExpiresAt?: number;
194
- private control?: ControlServer;
195
- // Stage 2 (#215): the local MCP endpoint the agent connects to, fronting the relay so the
196
- // Runner owns reconnect/backoff + a durable buffer. Disabled via AGENT_RELAY_MCP_PROXY=0
197
- // (then the agent connects to the relay directly, Stage-1 behaviour). The proxy secret is the
198
- // bearer the agent presents to the localhost proxy — it decouples the agent from the rotating
199
- // relay token (the proxy injects the live token relay-side).
200
- private proxy?: RelayMcpProxy;
201
- private mcpProxyEndpoint?: string;
202
- private readonly mcpProxyEnabled: boolean;
203
- private readonly mcpProxySecret: string;
204
- private process?: ManagedProcess;
205
- private stopped = false;
206
- private exitCommandInProgress = false;
207
- private restartInProgress = false;
208
- // Set for the whole unexpected-exit restart, including the pre-restart backoff
209
- // sleep, so a second exit during that window can't launch a parallel restart.
210
- private restartPending = false;
211
- private delivering = false;
212
- private drainTimer?: Timer;
213
- private claimRenewTimer?: Timer;
214
- private httpLivenessTimer?: Timer;
215
- private httpLivenessInFlight = false;
216
- private httpLivenessAuthFailed = false;
217
- private httpLivenessLastLog?: { key: string; at: number };
218
- private tokenRenewTimer?: Timer;
219
- private tokenRenewInFlight = false;
220
- private tokenRenewLastLog?: { key: string; at: number };
221
- private reactiveTokenRecoveryAt?: number;
222
- private processStartedAt = 0;
223
- private providerSessionId = crypto.randomUUID();
224
- // Last transcript path seen this session — used by end-of-session Insights (#184)
225
- // when the SessionEnd hook payload omits it.
226
- private lastTranscriptPath?: string;
227
- private lifecycleAction?: LifecycleAction;
228
- // #183/#184 per-segment cursor: how many of the current session's normalized events
229
- // have already been folded into an observation, and the key (transcript path / Codex
230
- // session) that count belongs to. A boundary captures only events since the last one,
231
- // so each datapoint is one work chunk between context resets; a key change or a shrink
232
- // (transcript rotated, Codex buffer trimmed) resets the cursor.
233
- private insightsObserved = 0;
234
- private insightsCursorKey = "";
235
- private archiveObservedChars = 0;
236
- private archiveCursorKey = "";
237
- // Memoized repo-name project id for insight observations (resolved once; involves a
238
- // git toplevel lookup for direct agents). Aggregates by repo, not per-session worktree.
239
- private insightProjectName?: string;
240
- // Coalesces concurrent pre-session-destroy runs (e.g. the shutdown bus command and the
241
- // SessionEnd hook both fire for the same teardown) so the cursor isn't raced.
242
- private preDestroyPromise?: Promise<void>;
243
- private readonly unexpectedExitTimes: number[] = [];
244
- private readonly pendingMessages = new Map<number, Message>();
245
- private readonly activeTaskClaims = new Map<number, ActiveTaskClaim>();
246
- private pendingTimelineEvent?: RunnerTimelineEvent;
247
- private pendingPromptMessageId?: number;
248
- // #329: a spawn-time initial prompt whose first delivery timed out before the TUI was
249
- // ready. Held (not dropped) for a ready-signal-driven re-attempt; cleared on success.
250
- private pendingInitialPrompt?: string;
251
- private deliveringInitialPrompt = false;
252
- private initialPromptAttempts = 0;
253
- // Session-mirror: a synthesized id grouping a turn's reasoning/tool steps and
254
- // its final response. Set when a provider-turn starts, cleared when it ends.
255
- private currentTurnId?: string;
256
- private currentTurnStartedAt?: number;
257
- // #435: high-water-mark for the pre-turn narrative flush. Counts how many
258
- // valid JSONL entries were in the transcript at last pre-flush time so the
259
- // Stop-hook capture (full mode) can skip what was already emitted.
260
- private narrativeFlushEntryCount = 0;
261
- // True while a turn that was already in flight is being compacted. Claude's PostCompact
262
- // hook posts a single `idle`, but a mid-turn compaction RESUMES the turn afterward — so
263
- // treating that idle as a turn end (flip idle, kill the reasoning tail) makes chat go dark
264
- // until the next prompt. Set when `compacting` arrives with a turn running; cleared on the
265
- // genuine end (a plain idle with no compaction timeline).
266
- private compactionMidTurn = false;
267
- // Prompt-echo dedup: a short, time-bounded queue of prompts the runner itself
268
- // injected (chat box or initial prompt) that are still awaiting their matching
269
- // UserPromptSubmit echo. A single slot dropped earlier entries when several prompts
270
- // were injected before their echoes returned (rapid sends while the provider is busy
271
- // and queues them) — the evicted ones then double-posted. Match consumes one entry.
272
- private injectedPrompts: Array<{ text: string; at: number }> = [];
273
- // Busy reconciler: consecutive idle probes observed while claims still say busy.
274
- private busyReconcileIdleStreak = 0;
275
- private busyReconcileTimer?: ReturnType<typeof setInterval>;
276
- // The reconciler only trusts an "idle" reading once it has seen the provider
277
- // actually busy this turn — so a flaky/always-idle probe can never false-clear.
278
- private busyReconcileSawBusy = false;
279
- // Verbose session-mirror diagnostics (turn lifecycle, reconciler probes, tail
280
- // emits) → a dedicated clean log. Always on for key transitions; AGENT_RELAY_SESSION_DEBUG=1 adds the high-frequency probe/emit lines.
281
- private readonly sessionDebugVerbose = process.env.AGENT_RELAY_SESSION_DEBUG === "1";
282
- // Tracks whether the provider is in a legitimate blocked/approval state, so the
283
- // busy reconciler doesn't mistake a permission prompt for a stuck-busy turn.
284
- private providerBlocked = false;
285
- private terminalFailure?: { reason: string; message: string; providerState?: Record<string, unknown> };
286
- // #286: a usage/rate-limit hold. The turn truly ended (idle), but the agent is
287
- // held until the limit resets, so this can't ride an active-work claim like the
288
- // permission-blocked state does. publishStatus surfaces it as the agent's
289
- // providerState; it clears when a new turn starts (the relay's resume message
290
- // wakes one) so the blocked badge lifts on its own.
291
- private rateLimitHold?: Record<string, unknown>;
292
- // Reasoning tailer (item 5): streams the in-flight turn's reasoning/tool steps
293
- // from the Claude transcript into chat as discreet session events.
294
- private reasoningTail?: { timer: ReturnType<typeof setInterval>; seen: Set<string> };
295
- private scratch?: SessionScratchLayout;
296
- // #417: for non-claude providers, tracks how many times each spawner obligation has
297
- // been re-injected so a non-responsive agent can't spin in a re-injection loop.
298
- private readonly pendingObligationInjections = new Map<number, number>();
299
-
300
- constructor(private readonly options: RunnerOptions) {
301
- this.agentId = options.agentId ?? options.runnerId;
302
- // Bind the process-global logger to this agent. AGENT_RELAY_SESSION_DEBUG=1 is
303
- // kept as a back-compat alias for the verbose probe/emit lines, now expressed
304
- // as log level "debug" (AGENT_RELAY_LOG_LEVEL still wins when both are set).
305
- logger.configure({
306
- agentId: this.agentId,
307
- headless: options.headless,
308
- ...(this.sessionDebugVerbose && !parseLogLevel(process.env.AGENT_RELAY_LOG_LEVEL) ? { level: "debug" as const } : {}),
309
- });
310
- this.currentToken = options.token;
311
- this.currentTokenJti = options.tokenJti;
312
- this.currentTokenProfileId = options.tokenProfileId;
313
- this.currentTokenExpiresAt = options.tokenExpiresAt;
314
- this.mcpProxyEnabled = !["0", "false", "off"].includes((process.env.AGENT_RELAY_MCP_PROXY ?? "").trim().toLowerCase());
315
- this.mcpProxySecret = crypto.randomUUID();
316
- const runtime = runtimeMetadata(options.provider);
317
- this.http = new RelayHttpClient({ baseUrl: options.relayUrl, token: this.currentToken });
318
- this.obligationCache = new ReplyObligationCache({ fetch: () => this.http.listReplyObligations(this.agentId) });
319
- // Co-locate the durable outbox with the runner's runtime state (survives reboot) when the
320
- // orchestrator told us where that is; otherwise the Outbox falls back to a temp dir.
321
- const outboxDir = process.env.AGENT_RELAY_RUNNER_OUTBOX_DIR
322
- ?? (process.env.AGENT_RELAY_RUNNER_INFO_FILE ? join(dirname(process.env.AGENT_RELAY_RUNNER_INFO_FILE), "outbox") : undefined);
323
- this.outbox = new Outbox({ agentId: this.agentId, dir: outboxDir, send: (record) => this.deliverOutboxEvent(record) });
324
- this.sessionOutbox = new Outbox({
325
- agentId: `${this.agentId}-session`,
326
- dir: outboxDir,
327
- send: (record) => this.deliverOutboxEvent(record),
328
- // Ephemeral trace frames: cap the per-row stall low and poison sooner than the main queue.
329
- maxBackoffMs: 5_000,
330
- maxAttempts: 6,
331
- });
332
- this.bus = new RelayBusClient({
333
- url: relayBusUrl(options.relayUrl),
334
- role: "provider",
335
- componentId: options.runnerId,
336
- agentId: this.agentId,
337
- instanceId: options.instanceId,
338
- token: this.currentToken,
339
- machine: hostname(),
340
- capabilities: [
341
- ...new Set([
342
- ...options.providerConfig.defaultCapabilities,
343
- ...options.capabilities,
344
- ...csvTags(process.env.AGENT_RELAY_CAPS),
345
- "lifecycle.shutdown.hard",
346
- "lifecycle.restart.hard",
347
- "lifecycle.status.semantic",
348
- "lifecycle.heartbeat",
349
- "lifecycle.reconnect.transport",
350
- "commands.lifecycle",
351
- "capabilities.report",
352
- ]),
353
- ],
354
- tags: [...new Set([options.provider, ...csvTags(process.env.AGENT_RELAY_TAGS), ...options.tags, ...options.providerConfig.defaultTags, ...(options.headless ? ["headless"] : [])])],
355
- meta: {
356
- ...runtime,
357
- version: runtime.package.version,
358
- provider: options.provider,
359
- model: options.model ?? null,
360
- effort: options.effort ?? null,
361
- profile: options.profile ?? null,
362
- agentProfile: options.agentProfile ? appliedAgentProfileMetadata(options.provider, options.agentProfile) : null,
363
- runnerId: options.runnerId,
364
- startedAt: options.startedAt,
365
- tmuxSession: this.providerTerminalSession() ?? options.tmuxSession ?? null,
366
- tmuxSocket: this.providerTerminalSocket() ?? null,
367
- policyName: options.policyName ?? null,
368
- spawnRequestId: options.spawnRequestId ?? null,
369
- automationId: options.automationId ?? null,
370
- automationRunId: options.automationRunId ?? null,
371
- workspace: options.workspace ?? null,
372
- lifecycle: options.lifecycle ?? "persistent",
373
- workspaceMode: options.workspace?.requestedMode ?? options.workspace?.mode ?? process.env.AGENT_RELAY_WORKSPACE_MODE ?? null,
374
- runnerManaged: true,
375
- cwd: options.cwd,
376
- approvalMode: options.approvalMode,
377
- label: options.label ?? null,
378
- auth: {
379
- profileId: this.currentTokenProfileId ?? null,
380
- jti: this.currentTokenJti ?? null,
381
- expiresAt: this.currentTokenExpiresAt ?? null,
382
- rootTokenFallback: options.rootTokenFallback === true,
383
- },
384
- lifecycleCapabilities: lifecycleCapabilities(),
385
- providerCapabilities: runtimeProviderCapabilities(options),
386
- },
387
- heartbeatMeta: () => this.heartbeatRuntimeMeta(),
388
- heartbeatIntervalMs: 30_000,
389
- initialStatus: "idle",
390
- });
391
- }
392
-
393
- get id(): string {
394
- return this.agentId;
395
- }
396
-
397
- async run(): Promise<void> {
398
- this.control = startControlServer({
399
- onStatus: (status) => this.setProviderStatus(status),
400
- onTerminalAttachSpec: () => this.terminalAttachSpec(),
401
- // Hot-path-safe: answered instantly from the local snapshot, never a server
402
- // round-trip. The snapshot is kept warm by the background refresh below (#196).
403
- onReplyObligations: () => Promise.resolve(this.obligationCache.get()),
404
- onSessionTurn: (input) => this.publishSessionTurn(input),
405
- onUserPrompt: (input) => this.handleUserPrompt(input),
406
- onSessionBoundary: (input) => this.handleSessionBoundary(input),
407
- onHookFatal: (report) => this.reportHookFatal(report),
408
- });
409
- this.startMcpProxy();
410
- this.writeRunnerInfoFile();
411
- this.options.adapter.onStatusChange((status) => {
412
- if (this.restartInProgress || this.restartPending) return;
413
- const semanticStatus = typeof status === "string" ? status : status.status;
414
- if (this.shouldRestartUnexpectedProviderExit(semanticStatus)) {
415
- void this.restartUnexpectedProviderExit(semanticStatus);
416
- return;
417
- }
418
- this.setProviderStatus(status);
419
- if (runnerShouldResolveProviderExit(semanticStatus, this.exitCommandInProgress)) this.options.onProviderExit?.(semanticStatus === "offline" ? 0 : 1);
420
- });
421
- this.options.adapter.onSessionEvent?.((event) => { void this.publishProviderSessionEvent(event); });
422
- this.bus.on("message.new", (message) => {
423
- // A delivered message may create a new reply obligation — warm the snapshot so the
424
- // next turn-end sees it without a hot-path server read.
425
- this.obligationCache.markDirty();
426
- this.enqueueMessage(message as Message);
427
- });
428
- this.bus.on("command", (type, params, commandId, command) => {
429
- void this.handleCommand(type, params, commandId, command);
430
- });
431
- this.bus.on("error", (code, message) => this.handleBusError(String(code), String(message)));
432
- await this.bus.connect();
433
- this.obligationCache.start();
434
- this.outbox.start();
435
- this.sessionOutbox.start();
436
- this.ensureScratch();
437
- void this.sweepStaleScratch();
438
- this.process = await this.spawnProvider();
439
- this.writeRunnerInfoFile();
440
- this.processStartedAt = Date.now();
441
- this.publishStatus();
442
- await this.deliverInitialPrompt();
443
- await this.bootstrapUnreadMessages();
444
- this.startClaimRenewer();
445
- this.startHttpLiveness();
446
- this.scheduleRuntimeTokenRenewal();
447
- this.scheduleDrain();
448
- }
449
-
450
- async stop(): Promise<void> {
451
- const alreadyStopped = this.stopped;
452
- this.stopped = true;
453
- reapSessionScratch({
454
- agentId: this.agentId,
455
- cwd: this.options.cwd,
456
- fallbackBaseDir: process.env.AGENT_RELAY_ORCHESTRATOR_BASE_DIR,
457
- });
458
- if (!alreadyStopped) await this.bus.statusAsync({ agentStatus: "offline", ready: false });
459
- if (this.process && !alreadyStopped) {
460
- await this.options.adapter.shutdown(this.process, {
461
- graceful: true,
462
- timeoutMs: this.options.providerConfig.headless.shutdownTimeoutMs,
463
- });
464
- }
465
- if (this.drainTimer) clearTimeout(this.drainTimer);
466
- this.drainTimer = undefined;
467
- if (this.claimRenewTimer) clearInterval(this.claimRenewTimer);
468
- this.claimRenewTimer = undefined;
469
- if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
470
- this.httpLivenessTimer = undefined;
471
- if (this.tokenRenewTimer) clearTimeout(this.tokenRenewTimer);
472
- this.tokenRenewTimer = undefined;
473
- this.disarmBusyReconciler();
474
- this.stopReasoningTail();
475
- this.obligationCache.stop();
476
- this.outbox.close();
477
- this.sessionOutbox.close();
478
- this.proxy?.stop();
479
- this.control?.stop();
480
- await this.bus.close();
481
- }
482
-
483
- // Start the local MCP proxy the agent connects to (Stage 2, #215). Forwards tool calls to the
484
- // relay with the runner's LIVE token, buffers bufferable writes durably during a relay outage,
485
- // and narrows the tool list to this agent's workspace context. Best-effort: if it can't bind,
486
- // we fall back to a direct relay MCP connection (the agent env still works, no resilience).
487
- private startMcpProxy(): void {
488
- if (!this.mcpProxyEnabled) return;
489
- try {
490
- this.proxy = new RelayMcpProxy({
491
- relayMcpEndpoint: relayMcpEndpoint(this.options.relayUrl),
492
- getToken: () => this.currentToken,
493
- authSecret: this.mcpProxySecret,
494
- enqueueBuffered: (call) => {
495
- this.outbox.enqueue({
496
- kind: "mcp-tool-call",
497
- payload: { tool: call.tool, arguments: call.arguments },
498
- idempotencyKey: call.idempotencyKey,
499
- });
500
- },
501
- initialContext: { isolatedWorktree: this.ownsIsolatedWorktree() },
502
- });
503
- this.mcpProxyEndpoint = this.proxy.start().url;
504
- logger.info("mcp-proxy", `runner MCP proxy listening at ${this.mcpProxyEndpoint} (worktree=${this.ownsIsolatedWorktree()})`);
505
- } catch (error) {
506
- this.proxy = undefined;
507
- this.mcpProxyEndpoint = undefined;
508
- logger.warn("mcp-proxy", `failed to start MCP proxy; agent will connect to the relay directly: ${errMessage(error)}`);
509
- }
510
- }
511
-
512
- private ownsIsolatedWorktree(): boolean {
513
- const mode = this.options.workspace?.requestedMode ?? this.options.workspace?.mode ?? process.env.AGENT_RELAY_WORKSPACE_MODE;
514
- return mode === "isolated";
515
- }
516
-
517
- private async spawnProvider(): Promise<ManagedProcess> {
518
- this.providerSessionId = crypto.randomUUID();
519
- this.lastTranscriptPath = undefined;
520
- const includeProviderGlobals = profileUsesHostProviderGlobals(this.options);
521
- const env = {
522
- ...process.env as Record<string, string>,
523
- ...(includeProviderGlobals ? this.options.providerConfig.env : {}),
524
- AGENT_RELAY_RUNNER_PORT: String(this.control!.port),
525
- AGENT_RELAY_RUNNER_ID: this.options.runnerId,
526
- AGENT_RELAY_PROVIDER_SESSION_ID: this.providerSessionId,
527
- AGENT_RELAY_ID: this.agentId,
528
- // The repo-name project id the runner uses for insight observations. Exposed so
529
- // agent-side signals (e.g. /introspect) tag the SAME project and aggregate together,
530
- // instead of the agent's worktree cwd splitting one repo into many "projects".
531
- AGENT_RELAY_PROJECT: this.insightProject(),
532
- ...(this.scratch ? { AGENT_RELAY_SCRATCH_DIR: this.scratch.tmpDir } : {}),
533
- AGENT_RELAY_URL: this.options.relayUrl,
534
- AGENT_RELAY_APPROVAL: this.options.approvalMode,
535
- ...(this.currentToken ? { AGENT_RELAY_TOKEN: this.currentToken } : {}),
536
- // Dedicated, un-clobberable credential for the injected MCP endpoint. A rig's
537
- // settings.json `env.AGENT_RELAY_TOKEN` would override the scoped token above at
538
- // MCP-parse time → server-actor auth, no identity (#233). The MCP config references
539
- // ${AGENT_RELAY_SESSION_TOKEN}, which rigs never set. See runner/src/relay-mcp.ts.
540
- //
541
- // Stage 2 (#215): when the proxy is active the agent connects to the LOCAL proxy, so this
542
- // holds the per-session PROXY SECRET (not the relay token). The proxy injects the live
543
- // relay token itself — the agent never holds it, and token rotation is invisible. With the
544
- // proxy disabled this stays the scoped relay token (Stage-1 direct connection).
545
- ...(this.proxy
546
- ? { [RELAY_MCP_TOKEN_ENV]: this.mcpProxySecret }
547
- : (this.currentToken ? { [RELAY_MCP_TOKEN_ENV]: this.currentToken } : {})),
548
- ...(this.currentTokenJti ? { AGENT_RELAY_TOKEN_JTI: this.currentTokenJti } : {}),
549
- ...(this.currentTokenProfileId ? { AGENT_RELAY_TOKEN_PROFILE: this.currentTokenProfileId } : {}),
550
- ...(this.currentTokenExpiresAt ? { AGENT_RELAY_TOKEN_EXPIRES_AT: String(this.currentTokenExpiresAt) } : {}),
551
- };
552
- const config: RunnerSpawnConfig = {
553
- provider: this.options.provider,
554
- model: this.options.model,
555
- effort: this.options.effort,
556
- runnerId: this.options.runnerId,
557
- instanceId: this.options.instanceId,
558
- agentId: this.agentId,
559
- relayUrl: this.options.relayUrl,
560
- cwd: this.options.cwd,
561
- headless: this.options.headless,
562
- approvalMode: this.options.approvalMode,
563
- ...(this.options.label ? { label: this.options.label } : {}),
564
- ...(this.options.rig ? { rig: this.options.rig } : {}),
565
- ...(this.options.profile ? { profile: this.options.profile } : {}),
566
- ...(this.options.agentProfile ? { agentProfile: this.options.agentProfile } : {}),
567
- ...(this.options.prompt ? { prompt: this.options.prompt } : {}),
568
- ...(this.options.systemPromptAppend ? { systemPromptAppend: this.options.systemPromptAppend } : {}),
569
- ...(this.options.tmuxSession ? { tmuxSession: this.options.tmuxSession } : {}),
570
- providerArgs: this.options.providerArgs,
571
- providerConfig: this.options.providerConfig,
572
- env,
573
- controlPort: this.control!.port,
574
- // Stage 2 (#215): the MCP endpoint the agent's client should target — the runner-local
575
- // proxy when active, undefined when disabled (adapters fall back to the direct relay URL).
576
- ...(this.mcpProxyEndpoint ? { relayMcpEndpoint: this.mcpProxyEndpoint } : {}),
577
- monitor: {
578
- deliver: (messages) => this.control!.deliverToMonitor(messages),
579
- },
580
- };
581
- const launchSeededPrompt = this.options.adapter.seedsInitialPromptAtLaunch ? this.options.prompt?.trim() : ""; if (launchSeededPrompt) this.recordInjectedPrompt(launchSeededPrompt);
582
- return this.options.adapter.spawn(config);
583
- }
584
-
585
- private async terminalAttachSpec(): Promise<TerminalAttachSpec> {
586
- if (!this.process) throw new Error("provider process is unavailable");
587
- if (!this.options.adapter.terminalAttachSpec) throw new Error("provider does not support terminal attach");
588
- return this.options.adapter.terminalAttachSpec(this.process);
589
- }
590
-
591
- private writeRunnerInfoFile(): void {
592
- const file = process.env.AGENT_RELAY_RUNNER_INFO_FILE;
593
- if (!file || !this.control) return;
594
- try {
595
- mkdirSync(dirname(file), { recursive: true });
596
- writeFileSync(file, JSON.stringify({
597
- agentId: this.agentId,
598
- runnerId: this.options.runnerId,
599
- provider: this.options.provider,
600
- controlUrl: this.control.url,
601
- pid: process.pid,
602
- ...(this.providerTerminalSession() ? { tmuxSession: this.providerTerminalSession() } : {}),
603
- ...(this.providerTerminalSocket() ? { tmuxSocket: this.providerTerminalSocket() } : {}),
604
- startedAt: this.options.startedAt,
605
- }, null, 2) + "\n", { mode: 0o600 });
606
- } catch (error) {
607
- logger.error("runner", `failed to write runner info file: ${error}`);
608
- }
609
- }
610
-
611
- private enqueueMessage(message: Message): void {
612
- if (!this.matchesMessage(message)) return;
613
- if (message.kind === "session") return;
614
- this.pendingMessages.set(message.id, message);
615
- this.scheduleDrain();
616
- }
617
-
618
- private async bootstrapUnreadMessages(): Promise<void> {
619
- try {
620
- const messages = await this.http.pollMessages({ for: this.agentId, unread: true, limit: 100 });
621
- for (const message of messages) this.enqueueMessage(message);
622
- } catch (error) {
623
- logger.error("runner", `inbox bootstrap failed: ${error}`);
624
- }
625
- }
626
-
627
- private async deliverInitialPrompt(): Promise<void> {
628
- const prompt = this.options.prompt?.trim();
629
- if (!prompt || this.options.adapter.seedsInitialPromptAtLaunch) return;
630
- await this.attemptInitialPromptDelivery(prompt, INITIAL_PROMPT_FIRST_READY_TIMEOUT_MS);
631
- }
632
-
633
- // Deliver the spawn-time first prompt, surviving a cold-start TUI that isn't input-ready yet
634
- // (#329). Timeout stashes it for ready-signal retry and surfaces the stall on the timeline.
635
- private async attemptInitialPromptDelivery(prompt: string, readyTimeoutMs: number): Promise<void> {
636
- if (this.deliveringInitialPrompt || this.stopped || !this.process || !this.options.adapter.deliverInitialPrompt) return;
637
- this.deliveringInitialPrompt = true;
638
- const attempt = (this.initialPromptAttempts += 1);
639
- try {
640
- this.recordInjectedPrompt(prompt); await this.options.adapter.deliverInitialPrompt(this.process, prompt, { readyTimeoutMs });
641
- this.pendingInitialPrompt = undefined;
642
- } catch (error) {
643
- const giveUp = attempt >= MAX_INITIAL_PROMPT_ATTEMPTS;
644
- this.pendingInitialPrompt = giveUp ? undefined : prompt;
645
- logger.error("runner", `initial prompt attempt ${attempt} failed${giveUp ? " (giving up)" : "; will retry on next ready signal"}: ${errMessage(error)}`);
646
- this.publishRunnerTimelineEvent({ status: "prompt_failed", timestamp: Date.now(), title: "Initial prompt not delivered", body: errMessage(error), metadata: { attempts: attempt, terminal: giveUp } });
647
- } finally {
648
- this.deliveringInitialPrompt = false;
649
- }
650
- }
651
-
652
- private scheduleDrain(delayMs = 0): void {
653
- if (this.stopped || this.drainTimer) return;
654
- this.drainTimer = setTimeout(() => {
655
- this.drainTimer = undefined;
656
- void this.drainMessages();
657
- }, delayMs);
658
- }
659
-
660
- private async drainMessages(): Promise<void> {
661
- if (this.stopped || this.delivering || this.pendingMessages.size === 0) return;
662
- if (!this.process) {
663
- this.scheduleDrain(500);
664
- return;
665
- }
666
- this.delivering = true;
667
- const messages = [...this.pendingMessages.values()].sort((a, b) => a.id - b.id);
668
- this.pendingMessages.clear();
669
- const deliverable: Message[] = [];
670
- const providerAlreadyBusy = this.claims.reasons().includes("provider-turn");
671
- for (const message of messages) {
672
- let toDeliver = message;
673
- if (message.claimable) {
674
- const claimed = await this.http.claimMessageResult(message.id, this.agentId).catch(() => ({ ok: false, claimExpiresAt: undefined }));
675
- if (!claimed.ok) continue;
676
- this.claims.startClaim("message", String(message.id), claimed.claimExpiresAt);
677
- const taskId = taskIdFromMessage(message);
678
- if (taskId) {
679
- this.activeTaskClaims.set(message.id, { messageId: message.id, taskId, observedProviderBusy: providerAlreadyBusy });
680
- this.claims.startClaim("task", String(taskId), claimed.claimExpiresAt);
681
- await this.updateTaskStatus(taskId, {
682
- status: "in_progress",
683
- agentId: this.agentId,
684
- metadata: { messageId: message.id, completedBy: "runner" },
685
- }).catch((error) => logger.error("task", `task ${taskId} in_progress update failed: ${error}`));
686
- // Runner owns claim + status here; drop the server's self-claim instruction
687
- // so the agent doesn't improvise a stray claim send (see stripRunnerClaimedGuidance).
688
- toDeliver = { ...message, body: stripRunnerClaimedGuidance(message.body) };
689
- }
690
- }
691
- deliverable.push(toDeliver);
692
- }
693
- if (deliverable.length === 0) {
694
- this.delivering = false;
695
- this.publishStatus();
696
- return;
697
- }
698
- this.publishStatus();
699
- let failed = false;
700
- try {
701
- const prepared = await messagesWithCachedAttachments(deliverable, this.http, {
702
- agentId: this.agentId,
703
- onError: (message) => logger.error("runner", message),
704
- });
705
- await this.options.adapter.deliver(this.process, prepared);
706
- for (const message of deliverable) {
707
- await this.http.markRead(message.id, this.agentId).catch(() => {});
708
- if (!taskIdFromMessage(message)) this.claims.finishClaim("message", String(message.id));
709
- }
710
- } catch (error) {
711
- failed = true;
712
- if (shouldLogDeliveryFailure(error)) logger.warn("delivery", `message delivery failed: ${error}`);
713
- for (const message of deliverable) {
714
- this.clearActiveClaim(message);
715
- this.pendingMessages.set(message.id, message);
716
- }
717
- } finally {
718
- this.delivering = false;
719
- this.publishStatus();
720
- if (this.pendingMessages.size > 0) this.scheduleDrain(failed ? 1000 : 0);
721
- }
722
- }
723
-
724
- private async handleCommand(type: string, params: Record<string, unknown>, commandId: string, command?: Record<string, unknown>): Promise<void> {
725
- const target = typeof command?.target === "string" ? command.target : this.agentId;
726
- if (target !== this.agentId && target !== this.options.runnerId) return;
727
- if (type !== "agent.shutdown" && type !== "agent.restart" && type !== "agent.reconnect" && type !== "agent.kill" && type !== "agent.compact" && type !== "agent.clearContext" && type !== "agent.injectContext" && type !== "agent.permissionDecision" && type !== "agent.interrupt" && type !== "prompt.inject") return;
728
-
729
- const exitAfterCommand = type === "agent.shutdown" || type === "agent.kill";
730
- if (exitAfterCommand) this.exitCommandInProgress = true;
731
- this.claims.startClaim("command", commandId);
732
- try {
733
- await this.updateCommand(commandId, "accepted");
734
- await this.updateCommand(commandId, "running");
735
- // Pre-session-destroy seam (#183): for destructive transitions, run end-of-session
736
- // work (Insights capture, #184) BEFORE the invasive operation, surfaced as a
737
- // non-addressable "finalizing" state so the agent isn't mistaken for merely busy.
738
- const destroyReason = boundaryReasonForCommand(type);
739
- if (destroyReason) await this.runPreSessionDestroy(destroyReason);
740
- // Move from the transient finalizing window to the executing teardown state (or drop
741
- // it entirely for compact/clear, which complete promptly once capture is done).
742
- if (exitAfterCommand) this.lifecycleAction = type === "agent.kill" ? "killing" : "shutting-down";
743
- else if (type === "agent.restart") this.lifecycleAction = "restarting";
744
- else this.lifecycleAction = undefined;
745
- this.publishStatus();
746
- let providerResult: Record<string, unknown> | void = undefined;
747
- if (type === "agent.restart") await this.restartProvider();
748
- else if (type === "agent.reconnect") this.publishStatus();
749
- else if (type === "agent.compact") {
750
- if (!this.options.adapter.compact || !this.process) throw new Error("provider does not support compact");
751
- providerResult = await this.options.adapter.compact(this.process, {
752
- instructions: typeof params.instructions === "string" ? params.instructions : undefined,
753
- });
754
- } else if (type === "agent.clearContext") {
755
- if (!this.options.adapter.clearContext || !this.process) throw new Error("provider does not support clearContext");
756
- providerResult = await this.options.adapter.clearContext(this.process);
757
- } else if (type === "agent.interrupt") {
758
- if (!this.options.adapter.interrupt || !this.process) throw new Error("provider does not support interrupt");
759
- this.sessionLog("interrupt requested from dashboard");
760
- providerResult = await this.options.adapter.interrupt(this.process);
761
- this.scheduleInterruptReconcile();
762
- } else if (type === "agent.injectContext") {
763
- if (!this.process) throw new Error("provider process is unavailable");
764
- providerResult = await this.injectContext(params);
765
- } else if (type === "agent.permissionDecision") {
766
- providerResult = await this.respondToPermissionDecision(params);
767
- } else if (type === "prompt.inject") {
768
- providerResult = await this.injectPrompt(params);
769
- } else await this.shutdownProvider(type === "agent.kill", commandTimeoutMs(params));
770
- await this.updateCommand(commandId, "succeeded", {
771
- action: type,
772
- agentId: this.agentId,
773
- runnerId: this.options.runnerId,
774
- policyName: this.options.policyName,
775
- spawnRequestId: this.options.spawnRequestId,
776
- reason: typeof params.reason === "string" ? params.reason : undefined,
777
- ...(providerResult ? { providerResult } : {}),
778
- });
779
- } catch (error) {
780
- await this.updateCommand(commandId, "failed", undefined, errMessage(error)).catch(() => {});
781
- } finally {
782
- this.claims.finishClaim("command", commandId);
783
- if (exitAfterCommand) {
784
- if (params.preserveRegistration === true) {
785
- await this.http.setStatus(this.agentId, "offline", this.options.instanceId).catch(() => {});
786
- } else {
787
- await this.http.deleteAgent(this.agentId).catch(() => {});
788
- }
789
- if (this.options.exitProcessOnShutdown !== false) {
790
- setTimeout(() => void this.stop().catch((error) => {
791
- logger.error("lifecycle", `stop after command failed: ${error}`);
792
- }).finally(() => process.exit(0)), 10);
793
- }
794
- } else if (!this.stopped) {
795
- this.lifecycleAction = undefined;
796
- this.publishStatus();
797
- }
798
- }
799
- }
800
-
801
- private async injectContext(params: Record<string, unknown>): Promise<Record<string, unknown>> {
802
- const content = typeof params.content === "string" ? params.content.trim() : "";
803
- if (!content) throw new Error("content required");
804
- const memoryIds = Array.isArray(params.memoryIds) ? params.memoryIds.filter((id): id is string => typeof id === "string") : [];
805
- await this.options.adapter.deliver(this.process!, [{
806
- id: 0,
807
- from: "system",
808
- to: this.agentId,
809
- kind: "system",
810
- subject: "Agent Relay memory context",
811
- body: content,
812
- payload: {
813
- memoryInjection: true,
814
- reason: typeof params.reason === "string" ? params.reason : undefined,
815
- memoryIds,
816
- },
817
- readBy: [],
818
- createdAt: Date.now(),
819
- }]);
820
- return { memoryIds, injectedMemoryCount: memoryIds.length };
821
- }
822
-
823
- private async respondToPermissionDecision(params: Record<string, unknown>): Promise<Record<string, unknown>> {
824
- const approvalId = typeof params.approvalId === "string" ? params.approvalId : "";
825
- const decision = typeof params.decision === "string" ? params.decision : "";
826
- if (!approvalId) throw new Error("approvalId required");
827
- if (decision !== "approve" && decision !== "approve-session" && decision !== "deny" && decision !== "abort" && decision !== "answer") {
828
- throw new Error("decision must be approve, approve-session, deny, abort, or answer");
829
- }
830
- const answersRaw = params.answers;
831
- const answers = answersRaw && typeof answersRaw === "object" && !Array.isArray(answersRaw)
832
- ? Object.fromEntries(
833
- Object.entries(answersRaw as Record<string, unknown>).filter(([, v]) => typeof v === "string") as [string, string][],
834
- )
835
- : undefined;
836
- if (decision === "answer" && (!answers || Object.keys(answers).length === 0)) {
837
- throw new Error("answers required for answer decision");
838
- }
839
- const input: ProviderPermissionDecisionInput = {
840
- approvalId,
841
- decision: decision as ProviderPermissionDecision,
842
- ...(typeof params.reason === "string" ? { reason: params.reason } : {}),
843
- ...(answers ? { answers } : {}),
844
- };
845
- if (this.control?.resolvePermissionDecision(input)) {
846
- return { approvalId, decision, provider: "claude" };
847
- }
848
- if (!this.process) throw new Error("provider process is unavailable");
849
- if (!this.options.adapter.respondToPermissionDecision) throw new Error("provider does not support permission decisions");
850
- const result = await this.options.adapter.respondToPermissionDecision(this.process, input);
851
- return { approvalId, decision, ...(result ? { providerResult: result } : {}) };
852
- }
853
-
854
- private async injectPrompt(params: Record<string, unknown>): Promise<Record<string, unknown>> {
855
- const body = typeof params.body === "string" ? params.body : "";
856
- if (!body) throw new Error("body required");
857
- if (!this.process) throw new Error("provider process is unavailable");
858
- if (!this.options.adapter.deliverInitialPrompt) throw new Error("provider does not support prompt injection");
859
- const messageId = typeof params.messageId === "number" ? params.messageId : undefined;
860
- const attachments = Array.isArray(params.attachments)
861
- ? params.attachments.filter((item): item is Record<string, unknown> => item !== null && typeof item === "object" && !Array.isArray(item))
862
- : [];
863
- if (messageId) this.pendingPromptMessageId = messageId;
864
- const prompt = await this.promptInjectionText(body, messageId, attachments);
865
- // Mark so the matching UserPromptSubmit echo isn't double-posted: a chat-box
866
- // prompt already created its own session message shown in the dashboard.
867
- this.recordInjectedPrompt(prompt.trim());
868
- await this.options.adapter.deliverInitialPrompt(this.process, prompt);
869
- return { injected: true, messageId, attachmentCount: attachments.length };
870
- }
871
-
872
- private async promptInjectionText(body: string, messageId: number | undefined, attachments: Record<string, unknown>[]): Promise<string> {
873
- if (!attachments.length) return body;
874
- const messages = await messagesWithCachedAttachments([{
875
- id: messageId ?? 0,
876
- from: "user",
877
- to: this.agentId,
878
- kind: "session",
879
- body,
880
- payload: { attachments },
881
- readBy: [],
882
- createdAt: Date.now(),
883
- }], this.http, {
884
- agentId: this.agentId,
885
- onError: (message) => logger.warn("attachments", message),
886
- });
887
- const attachmentText = providerAttachmentText(messages[0]!);
888
- return attachmentText ? `${body}\n\n${attachmentText}` : body;
889
- }
890
-
891
- private async restartProvider(): Promise<void> {
892
- this.restartInProgress = true;
893
- try {
894
- if (this.process) {
895
- await this.options.adapter.shutdown(this.process, {
896
- graceful: true,
897
- timeoutMs: this.options.providerConfig.headless.shutdownTimeoutMs,
898
- });
899
- }
900
- this.claims.clearTerminalStatus();
901
- this.claims.clearWorkKind("provider-turn");
902
- this.claims.clearWorkKind("subagent");
903
- if (this.stopped) return;
904
- this.process = await this.spawnProvider();
905
- this.processStartedAt = Date.now();
906
- } finally {
907
- this.restartInProgress = false;
908
- }
909
- }
910
-
911
- private shouldRestartUnexpectedProviderExit(status: SemanticStatus): boolean {
912
- return runnerShouldRestartUnexpectedProviderExit(status, {
913
- exitCommandInProgress: this.exitCommandInProgress,
914
- stopped: this.stopped,
915
- restartInProgress: this.restartInProgress,
916
- provider: this.options.provider,
917
- headless: this.options.headless,
918
- hasTerminalSession: typeof this.process?.meta?.tmuxSession === "string"
919
- || (this.options.adapter.supportsUnexpectedExitRestart?.() ?? false),
920
- });
921
- }
922
-
923
- private async restartUnexpectedProviderExit(status: SemanticStatus): Promise<void> {
924
- if (this.restartPending) return;
925
- this.restartPending = true;
926
- try {
927
- // Best-effort Insights capture for the segment that just ended in a crash (#183). This
928
- // path has no controlled teardown, so without it crashed sessions silently drop their
929
- // context-ratio datapoint. The process handle is still set (cleared later), so the
930
- // Claude transcript is readable; the runner stays alive here (restart or offline), so the
931
- // durable outbox drains normally — no flush needed.
932
- await Promise.race([
933
- this.captureContextRatio("crash"),
934
- new Promise<void>((resolve) => setTimeout(resolve, PRE_DESTROY_TIMEOUT_MS)),
935
- ]).catch((error) => this.sessionLog(`insights: crash capture failed: ${errMessage(error)}`));
936
-
937
- const now = Date.now();
938
- const runtimeMs = this.processStartedAt ? now - this.processStartedAt : Number.POSITIVE_INFINITY;
939
- const recent = this.unexpectedExitTimes.filter((time) => now - time <= UNEXPECTED_EXIT_WINDOW_MS);
940
- recent.push(now);
941
- this.unexpectedExitTimes.splice(0, this.unexpectedExitTimes.length, ...recent);
942
- const diagnostics = this.providerExitDiagnostics(status, runtimeMs);
943
-
944
- this.publishRunnerTimelineEvent({
945
- status: "provider.exit_detected",
946
- id: `provider-exit-${this.providerSessionId}-${now}`,
947
- timestamp: now,
948
- title: "Provider exited",
949
- body: `${this.options.provider} reported ${status} after ${Math.round(runtimeMs / 1000)}s`,
950
- icon: "ti-plug-off",
951
- metadata: {
952
- eventType: "provider.exit_detected",
953
- ...diagnostics,
954
- },
955
- });
956
-
957
- if (this.shouldStopUnexpectedProviderExit(diagnostics)) {
958
- const hasResumeId = typeof diagnostics.claudeResumeId === "string" && diagnostics.claudeResumeId.length > 0;
959
- logger.warn("lifecycle", `${this.options.provider} exited; leaving agent offline for manual recovery`);
960
- this.publishRunnerTimelineEvent({
961
- status: "provider.restart_decision",
962
- id: `provider-restart-decision-${this.providerSessionId}-${now}`,
963
- timestamp: Date.now(),
964
- title: "Provider restart skipped",
965
- body: hasResumeId
966
- ? "Claude exited; runner will not auto-resume. Resume id captured for manual recovery."
967
- : "Claude exited; runner will not restart automatically.",
968
- icon: "ti-player-stop",
969
- metadata: {
970
- eventType: "provider.restart_decision",
971
- decision: "stop-surface",
972
- reason: hasResumeId ? "claude-exit-manual-resume-available" : "claude-exit-manual-intervention-required",
973
- ...diagnostics,
974
- },
975
- });
976
- this.process = undefined;
977
- this.setProviderStatus({
978
- status,
979
- reason: "provider-turn",
980
- id: `provider-exit-${this.providerSessionId}`,
981
- clear: ["provider-turn", "subagent"],
982
- });
983
- return;
984
- }
985
-
986
- if (runtimeMs < RAPID_EXIT_MS && recent.length > MAX_RAPID_UNEXPECTED_EXITS) {
987
- logger.error("lifecycle", `provider session exited ${recent.length} times within ${Math.round(UNEXPECTED_EXIT_WINDOW_MS / 1000)}s; giving up`);
988
- this.publishRunnerTimelineEvent({
989
- status: "provider.restart_decision",
990
- id: `provider-restart-decision-${this.providerSessionId}-${now}`,
991
- timestamp: Date.now(),
992
- title: "Provider restart skipped",
993
- body: `rapid unexpected exits exceeded ${MAX_RAPID_UNEXPECTED_EXITS}`,
994
- icon: "ti-alert-triangle",
995
- metadata: {
996
- eventType: "provider.restart_decision",
997
- decision: "give-up",
998
- reason: "rapid-unexpected-provider-exits",
999
- rapidExitCount: recent.length,
1000
- rapidExitWindowMs: UNEXPECTED_EXIT_WINDOW_MS,
1001
- maxRapidUnexpectedExits: MAX_RAPID_UNEXPECTED_EXITS,
1002
- ...diagnostics,
1003
- },
1004
- });
1005
- this.setProviderStatus(status);
1006
- this.options.onProviderExit?.(0);
1007
- return;
1008
- }
1009
-
1010
- const delayMs = Math.min(10_000, Math.max(500, 500 * recent.length));
1011
- logger.warn("lifecycle", `provider session exited unexpectedly after ${Math.round(runtimeMs / 1000)}s; restarting in ${delayMs}ms`);
1012
- this.publishRunnerTimelineEvent({
1013
- status: "provider.restart_decision",
1014
- id: `provider-restart-decision-${this.providerSessionId}-${now}`,
1015
- timestamp: Date.now(),
1016
- title: "Provider restart scheduled",
1017
- body: `runner will start a fresh ${this.options.provider} provider in ${delayMs}ms`,
1018
- icon: "ti-refresh",
1019
- metadata: {
1020
- eventType: "provider.restart_decision",
1021
- decision: "restart-fresh",
1022
- reason: "unexpected-headless-terminal-exit",
1023
- delayMs,
1024
- rapidExitCount: recent.length,
1025
- rapidExitWindowMs: UNEXPECTED_EXIT_WINDOW_MS,
1026
- ...diagnostics,
1027
- },
1028
- });
1029
- await Bun.sleep(delayMs);
1030
- if (this.stopped || this.exitCommandInProgress) return;
1031
- try {
1032
- await this.restartProvider();
1033
- this.publishStatus();
1034
- this.scheduleDrain();
1035
- } catch (error) {
1036
- logger.error("lifecycle", `provider restart after unexpected exit failed: ${error}`);
1037
- this.setProviderStatus("error");
1038
- this.options.onProviderExit?.(1);
1039
- }
1040
- } finally {
1041
- this.restartPending = false;
1042
- }
1043
- }
1044
-
1045
- private shouldStopUnexpectedProviderExit(diagnostics: Record<string, unknown>): boolean {
1046
- return this.options.provider === "claude" && diagnostics.exitCommandInProgress !== true;
1047
- }
1048
-
1049
- private async shutdownProvider(hard: boolean, timeoutMs = this.options.providerConfig.headless.shutdownTimeoutMs): Promise<void> {
1050
- this.lifecycleAction = hard ? "killing" : "shutting-down";
1051
- this.publishStatus();
1052
- await this.bus.statusAsync({ agentStatus: "idle", ready: false, meta: { lifecycleAction: this.lifecycleAction, lifecycleActionAt: Date.now() } });
1053
- if (this.process) {
1054
- await this.options.adapter.shutdown(this.process, {
1055
- graceful: !hard,
1056
- timeoutMs,
1057
- });
1058
- }
1059
- this.claims.setTerminalStatus("offline");
1060
- this.publishStatus();
1061
- this.stopped = true;
1062
- }
1063
-
1064
- private publishRunnerTimelineEvent(event: RunnerTimelineEvent): void {
1065
- this.pendingTimelineEvent = {
1066
- ...event,
1067
- metadata: {
1068
- source: "runner",
1069
- provider: this.options.provider,
1070
- runnerId: this.options.runnerId,
1071
- agentId: this.agentId,
1072
- policyName: this.options.policyName ?? null,
1073
- spawnRequestId: this.options.spawnRequestId ?? null,
1074
- label: this.options.label ?? null,
1075
- providerSessionId: this.providerSessionId,
1076
- ...(event.metadata ?? {}),
1077
- },
1078
- };
1079
- this.publishStatus();
1080
- }
1081
-
1082
- private providerExitDiagnostics(status: SemanticStatus, runtimeMs: number): Record<string, unknown> {
1083
- const tmuxSession = typeof this.process?.meta?.tmuxSession === "string" ? this.process.meta.tmuxSession : undefined;
1084
- const tmuxSocket = typeof this.process?.meta?.tmuxSocket === "string" ? this.process.meta.tmuxSocket : undefined;
1085
- const exitSource = tmuxSession ? "tmux-session-ended" : this.process?.process ? "process-exit" : "provider-status";
1086
- const logFile = typeof process.env.AGENT_RELAY_LOG_FILE === "string" ? process.env.AGENT_RELAY_LOG_FILE : undefined;
1087
- const claudeResumeId = this.options.provider === "claude" && logFile ? latestClaudeResumeIdFromLogFile(logFile) : undefined;
1088
- return {
1089
- status,
1090
- runtimeMs: Number.isFinite(runtimeMs) ? runtimeMs : null,
1091
- exitSource,
1092
- exitCommandInProgress: this.exitCommandInProgress,
1093
- stopped: this.stopped,
1094
- restartInProgress: this.restartInProgress,
1095
- restartPending: this.restartPending,
1096
- headless: this.options.headless,
1097
- hasTerminalSession: Boolean(tmuxSession),
1098
- tmuxSession: tmuxSession ?? null,
1099
- tmuxSocket: tmuxSocket ?? null,
1100
- claudeResumeId: claudeResumeId ?? null,
1101
- };
1102
- }
1103
-
1104
- private async updateCommand(commandId: string, status: string, result?: Record<string, unknown>, error?: string): Promise<void> {
1105
- await this.bus.updateCommand(commandId, { status, ...(result ? { result } : {}), ...(error ? { error } : {}) });
1106
- }
1107
-
1108
- private handleBusError(code: string, message: string): void {
1109
- const action = runnerBusErrorAction(code, this.stopped);
1110
- if (action === "ignore") return;
1111
- logger.error("bus", `bus error ${code}: ${message}`);
1112
- if (action === "stop") {
1113
- void this.stop().catch((error) => {
1114
- logger.error("bus", `stop after bus error failed: ${error}`);
1115
- }).finally(() => process.exit(0));
1116
- }
1117
- }
1118
-
1119
- private setProviderStatus(update: ProviderStatusUpdate): void {
1120
- const status = typeof update === "string" ? update : update.status;
1121
- const providerSessionId = typeof update === "string" ? undefined : update.providerSessionId;
1122
- if (providerSessionId && providerSessionId !== this.providerSessionId) return;
1123
- const reason = typeof update === "string" ? "provider-turn" : update.reason ?? "provider-turn";
1124
- const id = typeof update === "string" ? reason : update.id ?? reason;
1125
- if (typeof update !== "string" && update.timeline) {
1126
- this.pendingTimelineEvent = {
1127
- status: update.timeline.status,
1128
- ...(update.timeline.id ? { id: update.timeline.id } : {}),
1129
- timestamp: update.timeline.timestamp ?? Date.now(),
1130
- ...(update.timeline.title ? { title: update.timeline.title } : {}),
1131
- ...(update.timeline.body ? { body: update.timeline.body } : {}),
1132
- ...(update.timeline.icon ? { icon: update.timeline.icon } : {}),
1133
- ...(update.timeline.metadata ? { metadata: update.timeline.metadata } : {}),
1134
- };
1135
- }
1136
- if (typeof update !== "string" && update.providerState) {
1137
- const ps = update.providerState as { state?: unknown; reason?: unknown };
1138
- this.providerBlocked = ps.state === "blocked";
1139
- // A rate-limit hold persists across the idle the turn ends on; any other
1140
- // providerState supersedes it (clears a stale hold).
1141
- if (ps.state === "blocked" && ps.reason === "rate_limit") {
1142
- const fresh = !this.rateLimitHold;
1143
- this.rateLimitHold = update.providerState;
1144
- if (fresh) this.publishRateLimitNotice(update.providerState);
1145
- } else {
1146
- this.rateLimitHold = undefined;
1147
- }
1148
- } else if (status === "idle") {
1149
- this.providerBlocked = false;
1150
- }
1151
- // Forward progress (a real turn) lifts the hold so the blocked badge clears.
1152
- if (status === "busy") this.rateLimitHold = undefined;
1153
- if (typeof update !== "string" && status === "error") {
1154
- const terminalReason = typeof update.metadata?.terminalFailureReason === "string"
1155
- ? update.metadata.terminalFailureReason
1156
- : typeof update.providerState?.reason === "string"
1157
- ? update.providerState.reason
1158
- : "provider-error";
1159
- const terminalMessage = typeof update.metadata?.terminalFailureMessage === "string"
1160
- ? update.metadata.terminalFailureMessage
1161
- : typeof update.providerState?.message === "string"
1162
- ? update.providerState.message
1163
- : "Provider reported an unrecoverable error.";
1164
- this.terminalFailure = {
1165
- reason: terminalReason,
1166
- message: terminalMessage,
1167
- ...(update.providerState ? { providerState: update.providerState } : {}),
1168
- };
1169
- } else if (status === "idle" || status === "busy") {
1170
- this.terminalFailure = undefined;
1171
- }
1172
- // Compaction lifecycle (Claude PreCompact→`compacting` busy / PostCompact→`compacted`
1173
- // idle). A `compacted` idle for a turn that predated compaction is a hook artifact — the
1174
- // turn resumes — so it must NOT end the turn. Discriminate on whether a turn was running,
1175
- // captured BEFORE the busy logic below mints a currentTurnId (it does for /compact at idle).
1176
- const timelineStatus = typeof update !== "string" ? update.timeline?.status : undefined;
1177
- if (timelineStatus === "compacting") {
1178
- this.compactionMidTurn = this.currentTurnId !== undefined;
1179
- } else if (timelineStatus === "compacted") {
1180
- this.publishCompactionNotice();
1181
- if (this.compactionMidTurn) {
1182
- // Keep the turn (and its live reasoning tail) alive; the genuine Stop hook ends it.
1183
- // pendingTimelineEvent (the marker) was set above and is consumed by publishStatus.
1184
- this.sessionLog(`compaction completed mid-turn (turn ${this.currentTurnId ?? "?"}) — staying busy`);
1185
- this.publishStatus();
1186
- return;
1187
- }
1188
- }
1189
- if (status === "busy" && reason === "provider-turn") {
1190
- if (!this.currentTurnId) {
1191
- this.currentTurnId = typeof update !== "string" && update.id ? update.id : crypto.randomUUID();
1192
- this.currentTurnStartedAt = Date.now();
1193
- this.compactionMidTurn = false;
1194
- this.sessionLog(`turn started (turn ${this.currentTurnId})`);
1195
- }
1196
- this.armBusyReconciler();
1197
- } else if (status === "idle" && reason === "provider-turn") {
1198
- if (this.currentTurnId) this.sessionLog(`turn ended via provider idle (turn ${this.currentTurnId})`);
1199
- this.currentTurnId = undefined;
1200
- this.currentTurnStartedAt = undefined;
1201
- this.compactionMidTurn = false;
1202
- this.disarmBusyReconciler();
1203
- this.stopReasoningTail();
1204
- // #417: claude enforces obligations via the stop.sh hook; for every other provider
1205
- // re-inject any pending spawner obligation so it surfaces the /reply command.
1206
- if (this.options.provider !== "claude") void this.reInjectPendingObligation();
1207
- }
1208
- if (status === "busy") {
1209
- this.claims.clearTerminalStatus();
1210
- this.claims.startWork(reason, id, typeof update === "string" ? {} : {
1211
- label: update.label,
1212
- role: update.role,
1213
- parentId: update.parentId,
1214
- metadata: {
1215
- ...(update.metadata ?? {}),
1216
- ...(update.providerState ? { providerState: update.providerState } : {}),
1217
- },
1218
- });
1219
- if (reason === "provider-turn") {
1220
- for (const claim of this.activeTaskClaims.values()) claim.observedProviderBusy = true;
1221
- }
1222
- } else if (status === "idle") {
1223
- this.claims.clearTerminalStatus();
1224
- this.claims.finishWork(reason, id);
1225
- if (reason === "provider-turn") void this.completeObservedTaskClaims();
1226
- }
1227
- else if (status === "offline" || status === "error") this.claims.setTerminalStatus(status);
1228
- if (typeof update !== "string") {
1229
- for (const kind of update.clear ?? []) this.claims.clearWorkKind(kind);
1230
- }
1231
- // #329: the provider just went idle (ready). If a first initial-prompt delivery timed out on
1232
- // a cold-start TUI, re-attempt now — the ready wait should resolve almost immediately.
1233
- if (status === "idle" && this.pendingInitialPrompt && !this.deliveringInitialPrompt) {
1234
- void this.attemptInitialPromptDelivery(this.pendingInitialPrompt, INITIAL_PROMPT_RETRY_READY_TIMEOUT_MS);
1235
- }
1236
- this.publishStatus();
1237
- }
1238
-
1239
- // #417: when a non-claude provider finishes a turn with a pending spawner obligation
1240
- // still unresolved, re-inject the obligation as a synthetic message so providerMessageText
1241
- // surfaces the specific `agent-relay /reply <id>` command. Claude handles this via the
1242
- // stop.sh hook; every other provider needs this in-process nudge.
1243
- private async reInjectPendingObligation(): Promise<void> {
1244
- if (!this.process) return;
1245
- // Filter to spawner obligations only — user obligations go through the existing codex
1246
- // prompt-capture path (publishProviderSessionEvent) and the session mirror (#418), never
1247
- // an explicit /reply. Shared predicate so this rule stays in lockstep with the Claude
1248
- // Stop-hook nag exemption (control-server.replyObligationStopDecision).
1249
- const obligations = this.obligationCache.get().filter(obligationRequiresExplicitReply);
1250
- if (!obligations.length) return;
1251
- const MAX_REINJECTIONS = 3;
1252
- for (const obligation of obligations) {
1253
- const count = this.pendingObligationInjections.get(obligation.messageId) ?? 0;
1254
- if (count >= MAX_REINJECTIONS) continue;
1255
- this.pendingObligationInjections.set(obligation.messageId, count + 1);
1256
- this.sessionLog(`re-injecting pending reply obligation #${obligation.messageId} from ${obligation.from} (attempt ${count + 1}/${MAX_REINJECTIONS})`);
1257
- const replayMessage = {
1258
- id: obligation.messageId,
1259
- from: obligation.from,
1260
- to: this.agentId,
1261
- kind: obligation.kind,
1262
- ...(obligation.subject ? { subject: obligation.subject } : {}),
1263
- body: obligation.bodyPreview,
1264
- payload: {},
1265
- readBy: [] as string[],
1266
- createdAt: obligation.createdAt,
1267
- };
1268
- // Suppress the userMessage echo that providers (e.g. Codex) reflect back when
1269
- // a turn is started with relay-formatted content — the echo body is exactly what
1270
- // providerMessageText produces, which starts with "[relay message #" and is caught
1271
- // by the RELAY_INJECTION_MARKERS prefix check, but record it explicitly as
1272
- // defense-in-depth for adapters that might echo only the unformatted body text.
1273
- this.recordInjectedPrompt(providerMessageText([replayMessage]));
1274
- await this.options.adapter.deliver(this.process, [replayMessage]);
1275
- // Kick off a cache refresh so a cleared obligation (agent replied) is detected
1276
- // before the next idle fires — without this, the stale snapshot re-injects up to
1277
- // MAX_REINJECTIONS times even after the agent has already /reply-ed.
1278
- this.obligationCache.markDirty();
1279
- // One obligation at a time — let the provider reply before nudging again.
1280
- break;
1281
- }
1282
- }
1283
-
1284
- // Session-mirror lane: capture the assistant turn from the Claude transcript and
1285
- // post it as a "session" message so it shows in the dashboard chat with zero
1286
- // agent tokens. Capture is UNCONDITIONAL — it no longer depends on a triggering
1287
- // relay message existing, so turns started from the web terminal (which create
1288
- // no relay message) are mirrored too. A reply obligation, when present, is still
1289
- // used as replyTo so the Stop hook stops nagging the agent to /reply.
1290
- //
1291
- // isPreFlush: true (#435) — mid-turn call from a PreToolUse hook (AskUserQuestion /
1292
- // ExitPlanMode / permission). Emits any un-mirrored narrative from the transcript
1293
- // tail immediately, before the interactive form appears in chat, then updates the
1294
- // entry-count cursor so the eventual Stop-hook call (full mode) skips it.
1295
- private async publishSessionTurn(input: { transcriptPath: string; lastAssistantMessage?: unknown; isPreFlush?: boolean }): Promise<void> {
1296
- if (input.transcriptPath) this.lastTranscriptPath = input.transcriptPath;
1297
-
1298
- if (input.isPreFlush) {
1299
- if (!input.transcriptPath) return;
1300
- let jsonl: string | undefined;
1301
- try { jsonl = await readFile(input.transcriptPath, "utf8"); } catch { return; }
1302
- if (!jsonl) return;
1303
- const body = extractLastAssistantTurnAfterEntry(jsonl, this.narrativeFlushEntryCount);
1304
- this.narrativeFlushEntryCount = countTranscriptEntries(jsonl);
1305
- if (!body) return;
1306
- const turnId = this.currentTurnId;
1307
- this.sessionLog(`pre-flush narrative for turn ${turnId ?? "?"} (${body.length} chars)`);
1308
- await this.publishSessionEvent({
1309
- from: this.agentId,
1310
- to: "user",
1311
- body,
1312
- session: { type: "response", origin: "provider", ...(turnId ? { turnId } : {}) },
1313
- });
1314
- return;
1315
- }
1316
-
1317
- const turnId = this.currentTurnId;
1318
- this.stopReasoningTail();
1319
- // Optional correlation for threading + obligation clearing — never a capture gate.
1320
- let replyToMessageId: number | undefined;
1321
- const pendingPrompt = this.pendingPromptMessageId;
1322
- if (pendingPrompt) {
1323
- replyToMessageId = pendingPrompt;
1324
- this.pendingPromptMessageId = undefined;
1325
- } else {
1326
- // Correlation-only (threading + obligation clearing) — the local snapshot is fresh
1327
- // enough and never blocks the response-capture path (#196).
1328
- const obligation = [...this.obligationCache.get()].reverse().find((o) => o.from === "user");
1329
- replyToMessageId = obligation?.messageId;
1330
- }
1331
-
1332
- // Grab and reset the pre-flush cursor before extraction: the cursor may be
1333
- // non-zero when a PreToolUse hook flushed narrative earlier this turn (#435).
1334
- // In "full" mode, skip those already-emitted entries so they don't duplicate.
1335
- const flushCount = this.narrativeFlushEntryCount;
1336
- this.narrativeFlushEntryCount = 0;
1337
-
1338
- // The Stop hook can fire before the final assistant entry is flushed to disk.
1339
- // Claude writes thinking and text as separate entries (both with end_turn), so
1340
- // the transcript can "look complete" while the text entry is still pending.
1341
- // Retry until both the transcript has an end_turn AND extraction yields text.
1342
- let body = "";
1343
- let jsonl: string | undefined;
1344
- try { jsonl = await readFile(input.transcriptPath, "utf8"); } catch { jsonl = undefined; }
1345
- if (jsonl !== undefined) {
1346
- for (let attempt = 0; attempt < 5; attempt++) {
1347
- if (attempt > 0) {
1348
- await new Promise((r) => setTimeout(r, 100));
1349
- try { jsonl = await readFile(input.transcriptPath, "utf8"); } catch { break; }
1350
- }
1351
- if (!transcriptLooksComplete(jsonl)) continue;
1352
- // Full mode: if a pre-flush already emitted entries 1..flushCount, only
1353
- // collect text from entries after that mark to avoid double-emit (#435).
1354
- const extract = this.options.providerConfig.chatCaptureMode === "full"
1355
- ? (j: string) => (flushCount > 0 ? extractLastAssistantTurnAfterEntry(j, flushCount) : extractLastAssistantTurn(j))
1356
- : extractFinalAssistantMessage;
1357
- body = extract(jsonl);
1358
- if (body) break;
1359
- }
1360
- }
1361
- // Fallback: last_assistant_message from the Stop hook payload, which bypasses
1362
- // the transcript file race entirely.
1363
- if (!body && input.lastAssistantMessage) {
1364
- body = extractHookAssistantMessage(input.lastAssistantMessage);
1365
- }
1366
- // A pure tool-use turn with no closing text is fine to skip — its reasoning and
1367
- // tool steps already carried the visibility into chat.
1368
- if (!body) {
1369
- this.sessionLog(`response capture: no closing text for turn ${turnId ?? "?"} (skipped)`);
1370
- return;
1371
- }
1372
-
1373
- this.sessionLog(`response captured for turn ${turnId ?? "?"} (${body.length} chars${replyToMessageId ? `, replyTo #${replyToMessageId}` : ", no replyTo"})`);
1374
- await this.publishSessionEvent({
1375
- from: this.agentId,
1376
- to: "user",
1377
- body,
1378
- ...(replyToMessageId ? { replyTo: replyToMessageId } : {}),
1379
- session: { type: "response", origin: "provider", ...(turnId ? { turnId } : {}) },
1380
- });
1381
- // The agent's reply may have cleared an obligation — refresh the snapshot so the next
1382
- // turn-end doesn't re-prompt for a message already answered (#196).
1383
- if (replyToMessageId) this.obligationCache.markDirty();
1384
- }
1385
-
1386
- // Post one session-mirror event (prompt echo, assistant response, reasoning or
1387
- // tool step) as a `kind: "session"` relay message tagged with payload.session so
1388
- // the dashboard can render the live provider session faithfully. Display-only:
1389
- // session messages are never delivered back into a provider.
1390
- private publishSessionEvent(input: {
1391
- from: string;
1392
- to: string;
1393
- body: string;
1394
- session: MessageSessionMeta;
1395
- replyTo?: number;
1396
- }): void {
1397
- // Durable, ordered, timestamped (#196): the actual POST happens in deliverOutboxEvent,
1398
- // retried until it lands. occurredAt is stamped now so a queued event reports when it
1399
- // truly happened, not when the server finally accepted it. Routed through the fast-lane
1400
- // sessionOutbox (#332) so a transient trace failure can't head-of-line block real messages.
1401
- // A stepId-bearing step (Codex tool running→completed, streamed reasoning/response) uses a
1402
- // STABLE idempotency key so the server upserts the row in place instead of appending a dup.
1403
- const stepId = input.session.stepId;
1404
- this.sessionOutbox.enqueue({
1405
- kind: "session-message",
1406
- ...(stepId ? { idempotencyKey: `session-step:${input.from}:${input.session.turnId ?? ""}:${stepId}` } : {}),
1407
- payload: {
1408
- from: input.from,
1409
- to: input.to,
1410
- ...(input.replyTo ? { replyTo: input.replyTo } : {}),
1411
- kind: "session",
1412
- body: input.body,
1413
- payload: { session: { provider: this.options.provider, ...input.session } },
1414
- } satisfies SendMessageInput,
1415
- });
1416
- }
1417
-
1418
- // Map queued records to HTTP calls. Throw to retry, return to ack/delete.
1419
- private async deliverOutboxEvent(record: OutboxRecord): Promise<void> {
1420
- try {
1421
- if (record.kind === "session-message") {
1422
- await this.http.sendMessage({
1423
- ...(record.payload as SendMessageInput),
1424
- occurredAt: record.occurredAt,
1425
- idempotencyKey: record.idempotencyKey,
1426
- });
1427
- return;
1428
- }
1429
- if (record.kind === "insight") {
1430
- await this.http.recordInsightObservation({
1431
- ...(record.payload as Parameters<RelayHttpClient["recordInsightObservation"]>[0]),
1432
- occurredAt: record.occurredAt,
1433
- });
1434
- return;
1435
- }
1436
- if (record.kind === "continuation-archive") {
1437
- await deliverContinuationArchiveRecord({ record, http: this.http, updatePayload: (seq, payload) => this.outbox.updatePayload(seq, payload), sessionLog: (message) => this.sessionLog(message) });
1438
- return;
1439
- }
1440
- if (record.kind === "mcp-tool-call") {
1441
- await this.deliverBufferedMcpCall(record);
1442
- return;
1443
- }
1444
- logger.warn("outbox", `dropping event with unknown kind: ${record.kind}`);
1445
- } catch (error) {
1446
- // 409 = the server intentionally rejected it (e.g. Insights/feature toggled off). That
1447
- // is a permanent "don't want this", not a transient failure — ack so it doesn't retry.
1448
- if (isHttpStatusError(error, 409)) return;
1449
- if (isHttpAuthError(error)) this.recoverRuntimeTokenAfterAuthFailure("outbox");
1450
- throw error; // transient (or auth, post-recovery) → let the outbox retry with backoff
1451
- }
1452
- }
1453
-
1454
- // Replay a buffered MCP tool call (Stage 2, #215) that the proxy queued while the relay was
1455
- // unreachable. POST it to the relay MCP endpoint with the LIVE token — same path the live call
1456
- // would have taken. Throw to retry (transient), return to ack (delivered or permanently
1457
- // rejected). The proxy stamped an idempotencyKey into the arguments so a retry that already
1458
- // landed server-side is deduped, not double-sent.
1459
- private async deliverBufferedMcpCall(record: OutboxRecord): Promise<void> {
1460
- const payload = record.payload as { tool: string; arguments: Record<string, unknown> };
1461
- const headers: Record<string, string> = { "content-type": "application/json" };
1462
- if (this.currentToken) headers.authorization = `Bearer ${this.currentToken}`;
1463
- const response = await fetch(relayMcpEndpoint(this.options.relayUrl), {
1464
- method: "POST",
1465
- headers,
1466
- body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: payload.tool, arguments: payload.arguments } }),
1467
- });
1468
- if (response.status === 401 || response.status === 403) {
1469
- this.recoverRuntimeTokenAfterAuthFailure("mcp-outbox");
1470
- throw new Error(`relay rejected buffered ${payload.tool} with ${response.status}`);
1471
- }
1472
- if (response.status >= 500) throw new Error(`relay ${response.status} on buffered ${payload.tool}`);
1473
- if (!response.ok) {
1474
- // A 4xx (e.g. target gone, validation) is a permanent rejection — retrying won't help.
1475
- // Ack so it doesn't block the queue, but log loudly: a queued write did not land.
1476
- const body = await response.text().catch(() => "");
1477
- logger.warn("mcp-outbox", `buffered ${payload.tool} permanently rejected (${response.status}); dropping: ${body.slice(0, 200)}`);
1478
- return;
1479
- }
1480
- // HTTP 200 but the JSON-RPC body may still carry a tool-level error. Those reflect the same
1481
- // permanent-rejection semantics (bad target, validation) — ack and log, don't loop.
1482
- const json = await response.json().catch(() => null) as { error?: { message?: string } } | null;
1483
- if (json?.error) {
1484
- logger.warn("mcp-outbox", `buffered ${payload.tool} returned a tool error; dropping: ${json.error.message ?? "(no detail)"}`);
1485
- }
1486
- }
1487
-
1488
- // A hook reported an unhandled failure (#198 seam). Already logged FATAL by the control
1489
- // server; here we additionally surface it durably to the server as a generic insight so
1490
- // it shows up in observability rather than only in the per-agent log (#196).
1491
- private reportHookFatal(report: { hook: string; error: string }): void {
1492
- try {
1493
- this.outbox.enqueue({
1494
- kind: "insight",
1495
- payload: {
1496
- sessionId: this.providerSessionId,
1497
- project: this.insightProject(),
1498
- agentId: this.agentId,
1499
- signal: "hook_fatal",
1500
- value: { hook: report.hook, error: report.error },
1501
- source: "server",
1502
- },
1503
- });
1504
- } catch (error) {
1505
- logger.error("outbox", `failed to queue hook-fatal report: ${errMessage(error)}`);
1506
- }
1507
- }
1508
-
1509
- // A human typed a prompt directly into the provider (web terminal / TUI). Mirror
1510
- // it into the dashboard chat so both surfaces stay in sync, and kick off reasoning
1511
- // tailing for the turn. Skips prompts the runner itself injected (chat box, relay
1512
- // deliveries) so those aren't double-posted.
1513
- private async handleUserPrompt(input: { prompt: string; transcriptPath?: string }): Promise<void> {
1514
- if (input.transcriptPath) this.lastTranscriptPath = input.transcriptPath;
1515
- if (!this.currentTurnId) {
1516
- this.currentTurnId = crypto.randomUUID();
1517
- this.currentTurnStartedAt = Date.now();
1518
- }
1519
- const text = input.prompt.trim();
1520
- if (text && !this.isRunnerInjectedPrompt(text)) {
1521
- this.sessionLog(`prompt echoed from terminal (${text.length} chars)`);
1522
- await this.publishSessionEvent({
1523
- from: "user",
1524
- to: this.agentId,
1525
- body: text,
1526
- session: { type: "prompt", origin: "terminal", turnId: this.currentTurnId },
1527
- });
1528
- } else if (text) {
1529
- this.sessionDebug("user-prompt hook: skipped echo (runner-injected)");
1530
- }
1531
- if (input.transcriptPath) this.startReasoningTail(input.transcriptPath);
1532
- }
1533
-
1534
- // A provider lifecycle hook reported a session boundary (Claude PreCompact / SessionEnd
1535
- // → control server). Normalize the raw provider reason to a SessionDestroyReason and run
1536
- // the same pre-destroy seam the bus commands use. `clear`/`compact` continue the session;
1537
- // anything else (logout, prompt_input_exit, other) is a real termination.
1538
- private async handleSessionBoundary(input: { reason?: string; transcriptPath?: string }): Promise<void> {
1539
- // Reason mapping is fail-safe-toward-termination: only the two known session-
1540
- // CONTINUING reasons are special-cased; everything else (logout, prompt_input_exit,
1541
- // other, AND any future reason) maps to "shutdown" → full pre-destroy capture.
1542
- // ⚠ If Claude Code adds a new BENIGN/continuing boundary reason, add it here — until
1543
- // then it will trigger a (harmless but wasteful) full context capture on a session
1544
- // that isn't actually ending.
1545
- const reason = input.reason === "compact" ? "compact"
1546
- : input.reason === "clear" ? "clear"
1547
- : "shutdown";
1548
- await this.runPreSessionDestroy(reason, { transcriptPath: input.transcriptPath });
1549
- // clear/compact CONTINUE the session — the finalizing window is transient. The bus-command
1550
- // path (handleCommand) restores the addressable status in its finally; the hook path has no
1551
- // such teardown, so without this the dashboard stays stuck on "wrapping up — messaging
1552
- // paused" with the composer disabled forever. Only restore if we're still in the exact
1553
- // finalizing state we published — a concurrent bus command (kill/restart/shutdown) may have
1554
- // transitioned lifecycleAction since, and that must win. "shutdown" reasons end the session,
1555
- // so their finalizing state is superseded by going offline; leave it be.
1556
- if ((reason === "clear" || reason === "compact") && this.lifecycleAction === `finalizing-${reason}` && !this.stopped) {
1557
- this.lifecycleAction = undefined;
1558
- this.publishStatus();
1559
- }
1560
- }
1561
-
1562
- // The pre-session-destroy seam (#183): the single place end-of-session work runs before
1563
- // an invasive transition (compact/clear/restart/shutdown/kill). Best-effort and
1564
- // time-boxed so it never hangs teardown; concurrent calls for the same teardown coalesce
1565
- // (a shutdown bus command and the SessionEnd hook can both fire). During the window the
1566
- // agent is published non-addressable so the operator sees "wrapping up", not "busy".
1567
- private runPreSessionDestroy(reason: SessionDestroyReason, opts?: { transcriptPath?: string }): Promise<void> {
1568
- if (this.preDestroyPromise) return this.preDestroyPromise;
1569
- const run = (async () => {
1570
- this.publishFinalizing(reason);
1571
- try {
1572
- await Promise.race([
1573
- Promise.all([
1574
- this.captureContextRatio(reason, opts),
1575
- this.captureContinuationArchive(reason, opts),
1576
- ]).then(() => undefined),
1577
- new Promise<void>((resolve) => setTimeout(resolve, PRE_DESTROY_TIMEOUT_MS)),
1578
- ]);
1579
- } catch (error) {
1580
- this.sessionLog(`pre-destroy capture failed: ${errMessage(error)}`);
1581
- }
1582
- // For exit-bound transitions the runner won't be alive afterward to drain the durable
1583
- // outbox, so block (bounded) on delivering what capture just enqueued. This runs before
1584
- // handleCommand's finally deletes the agent, so the runtime token is still valid here.
1585
- if (reasonExitsRunner(reason)) {
1586
- try {
1587
- const delivered = await this.outbox.flush(OUTBOX_FLUSH_TIMEOUT_MS);
1588
- if (!delivered) this.sessionLog(`insights: outbox flush incomplete before ${reason} (${this.outbox.pendingCount()} pending)`);
1589
- } catch (error) {
1590
- this.sessionLog(`insights: outbox flush failed: ${errMessage(error)}`);
1591
- }
1592
- }
1593
- })();
1594
- this.preDestroyPromise = run;
1595
- void run.finally(() => { this.preDestroyPromise = undefined; });
1596
- return run;
1597
- }
1598
-
1599
- // Publish the transient pre-destroy state: a non-offline status with ready:false (so the
1600
- // agent drops out of isAgentOnline fan-out targeting without going "offline") plus a
1601
- // finalizing-<reason> lifecycleAction the dashboard renders as "wrapping up" with the
1602
- // composer disabled.
1603
- private publishFinalizing(reason: SessionDestroyReason): void {
1604
- this.lifecycleAction = `finalizing-${reason}`;
1605
- void this.bus.statusAsync({ agentStatus: "busy", ready: false, meta: { lifecycleAction: this.lifecycleAction, lifecycleActionAt: Date.now() } });
1606
- }
1607
-
1608
- // Compute the #184 context-gathering ratio for the segment since the last boundary and
1609
- // queue it (durable outbox, #196). Provider-agnostic: the adapter normalizes its session
1610
- // into the shared SessionEvent stream; the math + classifier live in session-insights.ts.
1611
- // Per-segment via a runner-side cursor, so each datapoint is one work chunk between
1612
- // context resets. Mechanical, model-free → zero agent tokens, un-gameable.
1613
- // Repo-name project id for insight observations, resolved once. Aggregating by the raw
1614
- // cwd would split one repo across many "projects" (each isolated agent runs from a
1615
- // unique per-session worktree dir). See resolveProjectName.
1616
- private insightProject(): string {
1617
- return (this.insightProjectName ??= resolveProjectName(this.options.cwd, this.options.workspace));
1618
- }
1619
-
1620
- private async captureContinuationArchive(reason: SessionDestroyReason, opts?: { transcriptPath?: string }): Promise<void> {
1621
- const adapter = this.options.adapter;
1622
- if (!adapter.collectSessionArchiveSegment || !this.process) return;
1623
- const transcriptPath = opts?.transcriptPath ?? this.lastTranscriptPath;
1624
- const archive = await adapter.collectSessionArchiveSegment(this.process, { transcriptPath });
1625
- if (!archive) return;
1626
- const key = transcriptPath ?? `session:${this.providerSessionId}`;
1627
- if (key !== this.archiveCursorKey || archive.length < this.archiveObservedChars) {
1628
- this.archiveCursorKey = key;
1629
- this.archiveObservedChars = 0;
1630
- }
1631
- const segment = archive.slice(this.archiveObservedChars).trim();
1632
- this.archiveObservedChars = archive.length;
1633
- if (!segment) return;
1634
- const bounded = boundContinuationArchiveSegment(segment);
1635
- if (bounded.droppedBytes > 0) this.sessionLog(`continuation archive truncated at ${bounded.keptBytes} bytes; dropped ${bounded.droppedBytes} bytes (${reason})`);
1636
- this.outbox.enqueue({
1637
- kind: "continuation-archive",
1638
- payload: {
1639
- agentId: this.agentId,
1640
- segment: bounded.segment,
1641
- },
1642
- });
1643
- this.sessionLog(`continuation archive queued (${bounded.segment.length} chars, ${reason})`);
1644
- }
1645
-
1646
- private async captureContextRatio(reason: SessionDestroyReason, opts?: { transcriptPath?: string }): Promise<void> {
1647
- const adapter = this.options.adapter;
1648
- if (!adapter.collectSessionEvents || !this.process) return;
1649
- const transcriptPath = opts?.transcriptPath ?? this.lastTranscriptPath;
1650
- const events = await adapter.collectSessionEvents(this.process, { transcriptPath });
1651
- if (!events) return;
1652
- // Reset the cursor when the underlying log changed identity (transcript rotated on
1653
- // resume) or shrank (Codex buffer trimmed) — otherwise the slice would be wrong.
1654
- const key = transcriptPath ?? `session:${this.providerSessionId}`;
1655
- if (key !== this.insightsCursorKey || events.length < this.insightsObserved) {
1656
- this.insightsCursorKey = key;
1657
- this.insightsObserved = 0;
1658
- }
1659
- const segment = events.slice(this.insightsObserved);
1660
- this.insightsObserved = events.length;
1661
- const analysis = computeContextRatio(segment);
1662
- if (!analysis) return; // no tool calls this segment = nothing substantive to measure
1663
- this.outbox.enqueue({
1664
- kind: "insight",
1665
- payload: {
1666
- sessionId: this.providerSessionId,
1667
- project: this.insightProject(),
1668
- agentId: this.agentId,
1669
- signal: "context_ratio",
1670
- value: { ...analysis.metric, endReason: reason },
1671
- outcome: { ...analysis.outcome },
1672
- source: "server",
1673
- },
1674
- });
1675
- this.sessionLog(`insights: context_ratio ${analysis.metric.ratio.toFixed(2)} (${analysis.metric.gatheringCalls}/${analysis.metric.totalToolCalls} gathering, ${reason}) queued`);
1676
- }
1677
-
1678
- // Route a provider-emitted session event (Codex app-server) into the chat mirror.
1679
- // Mirrors the same semantics as the Claude lane: prompts are echoed with dedup,
1680
- // and a response is only auto-captured when the agent won't separately reply to a
1681
- // relay obligation (so relay-triggered turns aren't double-posted).
1682
- private async publishProviderSessionEvent(event: ProviderSessionEvent): Promise<void> {
1683
- const body = event.body.trim();
1684
- if (!body) return;
1685
- const turnId = event.turnId ?? this.currentTurnId;
1686
- if (event.type === "prompt") {
1687
- if (this.isRunnerInjectedPrompt(body)) return;
1688
- await this.publishSessionEvent({
1689
- from: "user",
1690
- to: this.agentId,
1691
- body,
1692
- session: { type: "prompt", origin: event.origin ?? "terminal", ...(turnId ? { turnId } : {}) },
1693
- });
1694
- return;
1695
- }
1696
- if (event.type === "response") {
1697
- // Dashboard prompt injection is already answered by this captured App Server
1698
- // response. Other Relay inbox obligations still belong to the agent's explicit
1699
- // reply path, so those keep suppressing auto-capture to avoid duplicates.
1700
- let replyToMessageId: number | undefined;
1701
- const pendingPrompt = this.pendingPromptMessageId;
1702
- if (pendingPrompt) {
1703
- replyToMessageId = pendingPrompt;
1704
- this.pendingPromptMessageId = undefined;
1705
- } else if (this.obligationCache.get().some((o) => o.from === "user" && this.obligationPredatesCurrentTurn(o))) {
1706
- // The agent will answer the relay obligation itself — don't double-post (#196).
1707
- return;
1708
- }
1709
- await this.publishSessionEvent({
1710
- from: this.agentId,
1711
- to: "user",
1712
- body,
1713
- ...(replyToMessageId ? { replyTo: replyToMessageId } : {}),
1714
- session: { type: "response", origin: event.origin ?? "provider", ...(turnId ? { turnId } : {}) },
1715
- });
1716
- if (replyToMessageId) this.obligationCache.markDirty();
1717
- return;
1718
- }
1719
- if (this.options.providerConfig.reasoningCapture === false) return;
1720
- await this.publishSessionEvent({
1721
- from: this.agentId,
1722
- to: "user",
1723
- body,
1724
- session: {
1725
- type: event.type,
1726
- origin: event.origin ?? "provider",
1727
- ...(turnId ? { turnId } : {}),
1728
- ...(event.label ? { label: event.label } : {}),
1729
- ...(event.status ? { status: event.status } : {}),
1730
- ...(event.streaming !== undefined ? { streaming: event.streaming } : {}),
1731
- ...(event.stepId ? { stepId: event.stepId } : {}),
1732
- },
1733
- });
1734
- }
1735
-
1736
- // Remember an injected prompt so its UserPromptSubmit echo can be suppressed. Prunes
1737
- // expired entries first; a defensive length cap guards against echoes that never
1738
- // arrive (e.g. the provider drops a queued prompt) so the queue can't grow unbounded.
1739
- private recordInjectedPrompt(text: string): void {
1740
- const now = Date.now();
1741
- this.injectedPrompts = this.injectedPrompts.filter((p) => now - p.at < PROMPT_ECHO_DEDUP_MS);
1742
- this.injectedPrompts.push({ text, at: now });
1743
- if (this.injectedPrompts.length > 50) this.injectedPrompts.shift();
1744
- }
1745
-
1746
- private isRunnerInjectedPrompt(text: string): boolean {
1747
- if (RELAY_INJECTION_MARKERS.some((marker) => text.startsWith(marker))) return true;
1748
- const now = Date.now();
1749
- this.injectedPrompts = this.injectedPrompts.filter((p) => now - p.at < PROMPT_ECHO_DEDUP_MS);
1750
- const idx = this.injectedPrompts.findIndex((p) => p.text === text);
1751
- if (idx !== -1) {
1752
- this.injectedPrompts.splice(idx, 1); // consume one — identical repeats each match once
1753
- return true;
1754
- }
1755
- return false;
1756
- }
1757
-
1758
- private obligationPredatesCurrentTurn(obligation: { createdAt: number }): boolean {
1759
- return this.currentTurnStartedAt === undefined || obligation.createdAt <= this.currentTurnStartedAt;
1760
- }
1761
-
1762
- // --- Busy-state reconciler (item 2) -------------------------------------------------
1763
- // A safety net for turns that end out of band (interrupted from the web terminal,
1764
- // a hook that never fired) where the runner would otherwise stay stuck "busy".
1765
- private armBusyReconciler(): void {
1766
- if (this.busyReconcileTimer || !this.options.adapter.probeActivity) return;
1767
- this.busyReconcileIdleStreak = 0;
1768
- this.busyReconcileSawBusy = false;
1769
- this.busyReconcileTimer = setInterval(() => { void this.runBusyReconcile(); }, BUSY_RECONCILE_POLL_MS);
1770
- }
1771
-
1772
- private disarmBusyReconciler(): void {
1773
- if (this.busyReconcileTimer) clearInterval(this.busyReconcileTimer);
1774
- this.busyReconcileTimer = undefined;
1775
- this.busyReconcileIdleStreak = 0;
1776
- this.busyReconcileSawBusy = false;
1777
- }
1778
-
1779
- private async runBusyReconcile(): Promise<void> {
1780
- if (this.stopped || !this.process || !this.options.adapter.probeActivity) { this.disarmBusyReconciler(); return; }
1781
- // Only act while the runner still believes a provider turn is in flight, and
1782
- // never override a legitimate approval/blocked state.
1783
- if (this.claims.currentStatus() !== "busy" || this.providerBlocked) { this.busyReconcileIdleStreak = 0; return; }
1784
- if (!this.claims.activeWork().some((w) => w.kind === "provider-turn")) { this.disarmBusyReconciler(); return; }
1785
- let activity: "busy" | "idle" | "unknown";
1786
- try { activity = await this.options.adapter.probeActivity(this.process); } catch { return; }
1787
- if (activity === "busy") this.busyReconcileSawBusy = true;
1788
- // Reset the streak on anything that isn't a confident idle.
1789
- if (activity !== "idle") {
1790
- this.busyReconcileIdleStreak = 0;
1791
- this.sessionDebug(`reconcile probe=${activity} sawBusy=${this.busyReconcileSawBusy} streak=${this.busyReconcileIdleStreak}`);
1792
- return;
1793
- }
1794
- this.busyReconcileIdleStreak += 1;
1795
- // Confirm faster once we've seen the spinner this turn; otherwise demand a much
1796
- // longer all-idle window before trusting it (rescues fast turns without
1797
- // false-clearing a live turn that simply hasn't flashed busy into a probe yet).
1798
- const confirm = this.busyReconcileSawBusy ? BUSY_RECONCILE_IDLE_CONFIRM : BUSY_RECONCILE_IDLE_CONFIRM_NO_BUSY;
1799
- this.sessionDebug(`reconcile probe=idle sawBusy=${this.busyReconcileSawBusy} streak=${this.busyReconcileIdleStreak}/${confirm}`);
1800
- if (this.busyReconcileIdleStreak < confirm) return;
1801
- this.disarmBusyReconciler();
1802
- this.forceClearProviderTurn(this.busyReconcileSawBusy ? "backstop reconciler" : "backstop reconciler (no-busy-observed)");
1803
- }
1804
-
1805
- // Force-clear a stuck provider-turn claim directly. Unlike the idle status path
1806
- // it does NOT depend on a matching claim id (the Stop hook keys busy as
1807
- // provider-turn:provider-turn, but reconciliation has no specific id), and it
1808
- // deliberately leaves the reasoning tail alone so a late clear can't truncate
1809
- // a turn's activity stream.
1810
- private forceClearProviderTurn(reason: string): void {
1811
- if (!this.claims.activeWork().some((w) => w.kind === "provider-turn")) return;
1812
- this.sessionLog(`force-clearing stuck provider-turn (${reason})`);
1813
- this.claims.clearWorkKind("provider-turn");
1814
- this.currentTurnId = undefined;
1815
- this.compactionMidTurn = false;
1816
- this.publishStatus();
1817
- }
1818
-
1819
- // After a dashboard interrupt, the provider should drop out of its turn; reconcile
1820
- // promptly so the busy indicator clears even if the Stop hook doesn't fire.
1821
- private scheduleInterruptReconcile(): void {
1822
- setTimeout(() => {
1823
- if (this.stopped || !this.process) return;
1824
- void (async () => {
1825
- if (this.claims.currentStatus() !== "busy" || this.providerBlocked) return;
1826
- let activity: "busy" | "idle" | "unknown" = "unknown";
1827
- try { if (this.options.adapter.probeActivity) activity = await this.options.adapter.probeActivity(this.process!); } catch { return; }
1828
- this.sessionDebug(`post-interrupt reconcile probe=${activity}`);
1829
- if (activity === "idle") this.forceClearProviderTurn("post-interrupt");
1830
- })();
1831
- }, INTERRUPT_RECONCILE_DELAY_MS);
1832
- }
1833
-
1834
- // --- Turn-step tailer (item 5) ------------------------------------------------------
1835
- // Tail the in-flight turn's Claude transcript and surface new narration/reasoning/tool
1836
- // steps as session events, in transcript order. `narration` (the agent's intermediate
1837
- // text) is the primary visible content; reasoning visibility is a client-side toggle.
1838
- // Coalesced and coarse; the final response still comes through publishSessionTurn.
1839
- // `reasoningCapture: false` disables the whole live trace (server-side kill switch).
1840
- private startReasoningTail(transcriptPath: string): void {
1841
- if (this.options.providerConfig.reasoningCapture === false) return;
1842
- this.stopReasoningTail();
1843
- // Track emitted steps by content signature, not by index/count: the "latest
1844
- // turn" window in the transcript can shrink/reset (a tool_result entry, a
1845
- // mid-turn user line), and an index cursor would then either re-emit or stall
1846
- // and drop the rest of the turn. A seen-set is idempotent under any reshuffle.
1847
- // The signature is salted with each step's occurrence-within-window (stepDedupKeys)
1848
- // so two identical steps in one turn — same tool, same input — both surface (#265).
1849
- const seen = new Set<string>();
1850
- const turnIdAtStart = this.currentTurnId;
1851
- // On the first poll the new prompt usually hasn't landed in the transcript yet,
1852
- // so extractLatestTurnSteps still returns the PRIOR (completed) turn. Seed those
1853
- // signatures as already-seen so we don't replay last turn's reasoning/tools as
1854
- // this turn's activity. Once our prompt lands the window resets at the new user
1855
- // boundary and genuinely-new steps emit normally. Only seed when the transcript
1856
- // is complete (last entry is an end_turn assistant) — otherwise we're already
1857
- // inside the new turn and those steps are legitimately ours.
1858
- let seeded = false;
1859
- const poll = async (): Promise<void> => {
1860
- let jsonl: string;
1861
- try { jsonl = await readFile(transcriptPath, "utf8"); } catch { return; }
1862
- let steps: ReturnType<typeof extractLatestTurnSteps>;
1863
- try { steps = extractLatestTurnSteps(jsonl); } catch { return; }
1864
- const keyed = stepDedupKeys(steps).map((sig, i) => ({ sig, step: steps[i]! }));
1865
- if (!seeded) {
1866
- seeded = true;
1867
- if (transcriptLooksComplete(jsonl)) {
1868
- for (const { sig } of keyed) seen.add(sig);
1869
- }
1870
- }
1871
- const turnId = this.currentTurnId ?? turnIdAtStart;
1872
- let emitted = 0;
1873
- for (const { sig, step } of keyed) {
1874
- if (seen.has(sig)) continue;
1875
- seen.add(sig);
1876
- emitted += 1;
1877
- void this.publishSessionEvent({
1878
- from: this.agentId,
1879
- to: "user",
1880
- body: step.text,
1881
- session: { type: step.type, origin: "provider", ...(turnId ? { turnId } : {}), ...(step.label ? { label: step.label } : {}) },
1882
- });
1883
- }
1884
- if (emitted) this.sessionDebug(`reasoning tail emitted ${emitted} step(s) (turn ${turnId ?? "?"}, ${seen.size} total)`);
1885
- };
1886
- this.reasoningTail = { seen, timer: setInterval(() => { void poll(); }, REASONING_POLL_MS) };
1887
- this.sessionLog(`reasoning tail started (turn ${turnIdAtStart ?? "?"})`);
1888
- void poll();
1889
- }
1890
-
1891
- private stopReasoningTail(): void {
1892
- if (this.reasoningTail) clearInterval(this.reasoningTail.timer);
1893
- this.reasoningTail = undefined;
1894
- }
1895
-
1896
- // Mirror a discreet, durable "context compacted" marker into chat via the existing
1897
- // session-mirror lane (not a parallel channel). `notice` renders as an inline timeline
1898
- // marker (never a bubble) and survives reload, unlike the ephemeral timeline-status one.
1899
- private publishCompactionNotice(): void {
1900
- this.publishSessionEvent({
1901
- from: this.agentId,
1902
- to: "user",
1903
- body: "🗜 Context compacted",
1904
- session: { type: "notice", origin: "provider", label: "compacted", ...(this.currentTurnId ? { turnId: this.currentTurnId } : {}) },
1905
- });
1906
- }
1907
-
1908
- // #286: a discreet, durable chat marker when a usage/rate-limit hold begins, via
1909
- // the same session-mirror lane as the compaction notice. Outbound session event
1910
- // (NOT an inbound message) so it shows in the dashboard chat WITHOUT waking a turn
1911
- // — waking a still-limited agent would just re-fail. The relay sends the waking
1912
- // resume message later, once the window has reset.
1913
- private publishRateLimitNotice(providerState: Record<string, unknown>): void {
1914
- const label = typeof providerState.label === "string" && providerState.label ? providerState.label : "usage limit reached";
1915
- this.publishSessionEvent({
1916
- from: this.agentId,
1917
- to: "user",
1918
- body: `⏳ ${label} — holding; agent-relay will auto-resume at reset.`,
1919
- session: { type: "notice", origin: "provider", label: "rate-limit", ...(this.currentTurnId ? { turnId: this.currentTurnId } : {}) },
1920
- });
1921
- }
1922
-
1923
- private publishStatus(): void {
1924
- this.claims.expire();
1925
- const status = this.claims.currentStatus();
1926
- const agentStatus = runnerAgentStatus(status);
1927
- const activeWork = this.claims.activeWork();
1928
- const activeSubagents = activeWork.filter((item) => item.kind === "subagent");
1929
- const terminalFailure = this.terminalFailure;
1930
- const providerState = terminalFailure?.providerState ?? this.rateLimitHold ?? providerStateFromActiveWork(activeWork);
1931
- this.bus.setSemanticStatus(status === "offline" || status === "error" ? "idle" : status);
1932
- const timelineEvent = this.pendingTimelineEvent;
1933
- this.pendingTimelineEvent = undefined;
1934
- this.bus.status({
1935
- agentStatus,
1936
- ready: agentStatus !== "offline" && !this.stopped,
1937
- meta: {
1938
- runnerId: this.options.runnerId,
1939
- startedAt: this.options.startedAt,
1940
- tmuxSession: this.providerTerminalSession() ?? this.options.tmuxSession ?? null,
1941
- tmuxSocket: this.providerTerminalSocket() ?? null,
1942
- policyName: this.options.policyName ?? null,
1943
- spawnRequestId: this.options.spawnRequestId ?? null,
1944
- automationId: this.options.automationId ?? null,
1945
- automationRunId: this.options.automationRunId ?? null,
1946
- workspace: this.options.workspace ?? null,
1947
- lifecycle: this.options.lifecycle ?? "persistent", workspaceMode: this.options.workspace?.requestedMode ?? this.options.workspace?.mode ?? process.env.AGENT_RELAY_WORKSPACE_MODE ?? null,
1948
- lifecycleAction: this.lifecycleAction ?? null,
1949
- profile: this.options.profile ?? null,
1950
- ...(status === "error" ? { terminalStatus: "error" } : {}),
1951
- ...(terminalFailure ? {
1952
- lastError: terminalFailure.message,
1953
- terminalFailureReason: terminalFailure.reason,
1954
- terminalFailureMessage: terminalFailure.message,
1955
- } : {}),
1956
- busyReasons: this.claims.reasons(),
1957
- activeWork,
1958
- activeSubagents,
1959
- activeSubagentCount: activeSubagents.length,
1960
- providerState,
1961
- auth: {
1962
- profileId: this.currentTokenProfileId ?? null,
1963
- jti: this.currentTokenJti ?? null,
1964
- expiresAt: this.currentTokenExpiresAt ?? null,
1965
- rootTokenFallback: this.options.rootTokenFallback === true,
1966
- },
1967
- ...(timelineEvent ? { timelineEvent } : {}),
1968
- transport: this.bus.transportState,
1969
- },
1970
- });
1971
- }
1972
-
1973
- private startHttpLiveness(): void {
1974
- if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
1975
- void this.publishHttpLiveness();
1976
- this.httpLivenessTimer = setInterval(() => {
1977
- void this.publishHttpLiveness();
1978
- }, HTTP_LIVENESS_INTERVAL_MS);
1979
- }
1980
-
1981
- private async publishHttpLiveness(): Promise<void> {
1982
- if (this.stopped || this.httpLivenessInFlight || this.httpLivenessAuthFailed) return;
1983
- this.httpLivenessInFlight = true;
1984
- const status = this.claims.currentStatus();
1985
- const agentStatus = runnerAgentStatus(status);
1986
- try {
1987
- await this.http.setStatus(this.agentId, agentStatus, this.options.instanceId);
1988
- await this.http.setReady(this.agentId, agentStatus !== "offline", this.options.instanceId);
1989
- await this.http.heartbeat(this.agentId, this.options.instanceId);
1990
- await this.bootstrapUnreadMessages();
1991
- } catch (error) {
1992
- if (!this.stopped) this.handleHttpLivenessFailure(error);
1993
- } finally {
1994
- this.httpLivenessInFlight = false;
1995
- }
1996
- }
1997
-
1998
- private handleHttpLivenessFailure(error: unknown): void {
1999
- const authFailed = isHttpAuthError(error);
2000
- this.logHttpLivenessFailure(error, authFailed);
2001
- if (!authFailed) return;
2002
- this.httpLivenessAuthFailed = true;
2003
- if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
2004
- this.httpLivenessTimer = undefined;
2005
- // A 401/403 here is the only timely signal that the token died — stopping the
2006
- // liveness timer means there is no second chance, so recover from THIS failure.
2007
- this.recoverRuntimeTokenAfterAuthFailure("http-liveness");
2008
- }
2009
-
2010
- // A definitive relay auth failure (401/403) means the runtime token is dead right
2011
- // now — expired, or (the common case) revoked when the relay marked this agent
2012
- // stale across its own restart/reconnect. The proactive renew timer is keyed to
2013
- // TTL and structurally cannot catch a revocation, so the auth failure itself must
2014
- // drive recovery. renewRuntimeToken() prefers an orchestrator re-mint, which heals
2015
- // an expired token AND a token revoked-as-"stale" (the restart strand, #484) —
2016
- // but NOT a hard kill (admin revoke / rotation / agent removal), which stays dead
2017
- // by design. Debounced so a burst of failing calls re-mints once.
2018
- private recoverRuntimeTokenAfterAuthFailure(source: string): void {
2019
- if (this.stopped || this.tokenRenewInFlight) return;
2020
- if (!this.isRuntimeTokenRenewable() && !this.canRemintViaOrchestrator()) return;
2021
- const now = Date.now();
2022
- if (this.reactiveTokenRecoveryAt && now - this.reactiveTokenRecoveryAt < REACTIVE_TOKEN_RECOVERY_DEBOUNCE_MS) return;
2023
- this.reactiveTokenRecoveryAt = now;
2024
- this.logRunnerDiagnostic(`[runner] relay auth failure on ${source}; recovering runtime token`);
2025
- void this.renewRuntimeToken();
2026
- }
2027
-
2028
- private logHttpLivenessFailure(error: unknown, authFailed: boolean): void {
2029
- const key = httpErrorKey(error);
2030
- const now = Date.now();
2031
- if (
2032
- this.httpLivenessLastLog?.key === key &&
2033
- now - this.httpLivenessLastLog.at < HTTP_LIVENESS_LOG_INTERVAL_MS
2034
- ) return;
2035
- this.httpLivenessLastLog = { key, at: now };
2036
- const suffix = authFailed ? "auth failed; stopping HTTP liveness retries until restart" : String(error);
2037
- this.logRunnerDiagnostic(`[runner] HTTP liveness update failed: ${suffix}`);
2038
- }
2039
-
2040
- // Runner operational diagnostics (HTTP liveness, token renewal failures). Routed
2041
- // through the leveled logger at warn — see logger.ts. Kept as a thin wrapper so
2042
- // the existing call sites and their `[runner]` framing stay put.
2043
- private logRunnerDiagnostic(message: string): void {
2044
- logger.warn("runner", message.replace(/^\[runner\]\s*/, ""));
2045
- }
2046
-
2047
- // Session-mirror diagnostics → the leveled logger (component "mirror"), written
2048
- // to the dashboard-surfaced session-mirror-<agent>.log. Key transitions log at
2049
- // info; the single place to look when chat/terminal sync misbehaves.
2050
- private sessionLog(message: string): void {
2051
- logger.info("mirror", message);
2052
- }
2053
-
2054
- // Verbose, high-frequency lines (per-probe, per-emit) — surfaced only at log
2055
- // level "debug" (AGENT_RELAY_LOG_LEVEL=debug, or flip live via /log-level).
2056
- private sessionDebug(message: string): void {
2057
- logger.debug("mirror", message);
2058
- }
2059
-
2060
- private ensureScratch(): void {
2061
- try {
2062
- this.scratch = ensureSessionScratch({
2063
- agentId: this.agentId,
2064
- cwd: this.options.cwd,
2065
- fallbackBaseDir: process.env.AGENT_RELAY_ORCHESTRATOR_BASE_DIR,
2066
- });
2067
- } catch (error) {
2068
- this.logRunnerDiagnostic(`session scratch setup failed: ${errMessage(error)}`);
2069
- }
2070
- }
2071
-
2072
- private async sweepStaleScratch(): Promise<void> {
2073
- try {
2074
- const agents = await this.http.listAgents().catch(() => [] as Awaited<ReturnType<RelayHttpClient["listAgents"]>>);
2075
- const keep = new Set(agents.map((a) => a.id));
2076
- keep.add(this.agentId);
2077
- const removed = sweepStaleSessions({
2078
- cwd: this.options.cwd,
2079
- fallbackBaseDir: process.env.AGENT_RELAY_ORCHESTRATOR_BASE_DIR,
2080
- keepAgentIds: keep,
2081
- now: Date.now(),
2082
- });
2083
- if (removed.length) this.logRunnerDiagnostic(`swept ${removed.length} stale session scratch dir(s)`);
2084
- } catch {}
2085
- }
2086
-
2087
- private scheduleRuntimeTokenRenewal(delayMs?: number): void {
2088
- if (this.tokenRenewTimer) clearTimeout(this.tokenRenewTimer);
2089
- this.tokenRenewTimer = undefined;
2090
- if (this.stopped) return;
2091
- const canSelfRenew = this.isRuntimeTokenRenewable();
2092
- const canRemint = this.canRemintViaOrchestrator();
2093
- // Keep the renewal clock ticking as long as the session can recover its token
2094
- // by EITHER path. Without the re-mint fallback an expired token would stop the
2095
- // timer forever (the old deadlock that stranded live agents off the bus).
2096
- if (!canSelfRenew && !canRemint) return;
2097
- let computedDelay = delayMs;
2098
- if (computedDelay === undefined) {
2099
- computedDelay = canSelfRenew
2100
- ? runtimeTokenRenewDelayMs(this.currentTokenExpiresAt!, Date.now())
2101
- : TOKEN_RENEW_RETRY_MS; // expired but re-mintable → retry via orchestrator soon
2102
- if (computedDelay === undefined) computedDelay = TOKEN_RENEW_RETRY_MS;
2103
- }
2104
- const schedule = runtimeTokenRenewTimerSchedule(computedDelay);
2105
- if (!schedule) return;
2106
- this.tokenRenewTimer = setTimeout(() => {
2107
- this.tokenRenewTimer = undefined;
2108
- if (!schedule.renew) {
2109
- this.scheduleRuntimeTokenRenewal();
2110
- return;
2111
- }
2112
- void this.renewRuntimeToken();
2113
- }, schedule.delayMs);
2114
- }
2115
-
2116
- // Can the runner self-renew right now? Requires a non-expired runner-profile token
2117
- // (the relay rejects renewal of an expired token).
2118
- private isRuntimeTokenRenewable(): boolean {
2119
- return Boolean(
2120
- this.currentToken &&
2121
- this.currentTokenExpiresAt &&
2122
- this.currentTokenExpiresAt * 1000 > Date.now() &&
2123
- (this.currentTokenProfileId === "provider-agent" || this.currentTokenProfileId === "provider-interactive"),
2124
- );
2125
- }
2126
-
2127
- // Can the runner recover its token via the orchestrator? Works even when the token
2128
- // is already expired — the orchestrator's standing credential is the authority.
2129
- private canRemintViaOrchestrator(): boolean {
2130
- return Boolean(
2131
- process.env.AGENT_RELAY_ORCHESTRATOR_URL &&
2132
- this.currentToken &&
2133
- (this.currentTokenProfileId === "provider-agent" || this.currentTokenProfileId === "provider-interactive"),
2134
- );
2135
- }
2136
-
2137
- private async renewRuntimeToken(): Promise<void> {
2138
- if (this.stopped || this.tokenRenewInFlight || !this.currentToken) return;
2139
- this.tokenRenewInFlight = true;
2140
- try {
2141
- // Preferred path: self-renew directly against the relay while the token is
2142
- // still valid. Cheapest and needs no orchestrator round-trip.
2143
- if (this.isRuntimeTokenRenewable()) {
2144
- try {
2145
- const renewed = await this.http.renewRuntimeToken();
2146
- this.applyRenewedToken(renewed.token, renewed.record, "runtime-token-renewed");
2147
- return;
2148
- } catch (error) {
2149
- this.logRuntimeTokenRenewalFailure(error);
2150
- // Relay unreachable or token rejected — fall through to orchestrator re-mint.
2151
- }
2152
- }
2153
- // Recovery path: token expired, or self-renew failed. Ask the orchestrator —
2154
- // it holds a long-lived credential and can mint a fresh runner token, so a
2155
- // live session heals instead of being stranded off the bus.
2156
- if (this.canRemintViaOrchestrator() && await this.remintViaOrchestrator()) return;
2157
- this.pendingTimelineEvent = {
2158
- status: "runtime-token-renewal-failed",
2159
- timestamp: Date.now(),
2160
- };
2161
- this.publishStatus();
2162
- this.scheduleRuntimeTokenRenewal(TOKEN_RENEW_RETRY_MS);
2163
- } finally {
2164
- this.tokenRenewInFlight = false;
2165
- }
2166
- }
2167
-
2168
- // Apply a freshly issued token across every live surface — runner state, the
2169
- // RunnerOptions bag (re-injected into the provider on respawn), the HTTP client,
2170
- // the bus client — then force a bus handshake with the new token and reschedule.
2171
- private applyRenewedToken(
2172
- token: string,
2173
- record: { jti: string; profileId?: string; expiresAt?: number },
2174
- status: "runtime-token-renewed" | "runtime-token-reminted",
2175
- ): void {
2176
- this.currentToken = token;
2177
- this.currentTokenJti = record.jti;
2178
- this.currentTokenProfileId = record.profileId ?? this.currentTokenProfileId;
2179
- this.currentTokenExpiresAt = record.expiresAt;
2180
- this.options.token = token;
2181
- this.options.tokenJti = record.jti;
2182
- this.options.tokenProfileId = this.currentTokenProfileId;
2183
- this.options.tokenExpiresAt = this.currentTokenExpiresAt;
2184
- this.http.setToken(token);
2185
- this.bus.setToken(token);
2186
- // The proxy reads the token live via getToken(), so forwarding already uses the new one.
2187
- // A re-mint can change scope (e.g. a profile change), so refresh the relay tool list and
2188
- // emit tools/list_changed if the visible set changed (#215 — token-scope transition).
2189
- void this.proxy?.refreshTools().catch(() => {});
2190
- this.httpLivenessAuthFailed = false;
2191
- this.reactiveTokenRecoveryAt = undefined;
2192
- // An earlier auth failure may have stopped the liveness loop; restart it so the
2193
- // agent reports live again on the fresh token. startHttpLiveness clears any
2194
- // existing timer first, so this is safe on the normal (proactive) renew path too.
2195
- this.startHttpLiveness();
2196
- this.pendingTimelineEvent = { status, id: record.jti, timestamp: Date.now() };
2197
- this.bus.reconnectTransport(status === "runtime-token-reminted" ? "runtime token re-minted" : "runtime token renewed");
2198
- this.publishStatus();
2199
- this.scheduleRuntimeTokenRenewal();
2200
- }
2201
-
2202
- // Recover the runtime token through the orchestrator. The runner proxies its own
2203
- // (possibly expired) token; the orchestrator re-mints it via the relay using its
2204
- // standing credential. Returns true on success.
2205
- private async remintViaOrchestrator(): Promise<boolean> {
2206
- const orchUrl = process.env.AGENT_RELAY_ORCHESTRATOR_URL;
2207
- if (!orchUrl || !this.currentToken) return false;
2208
- try {
2209
- const res = await fetch(`${orchUrl.replace(/\/+$/, "")}/api/runtime-tokens/runner-renew`, {
2210
- method: "POST",
2211
- headers: { "Content-Type": "application/json" },
2212
- body: JSON.stringify({ token: this.currentToken }),
2213
- signal: AbortSignal.timeout(10_000),
2214
- });
2215
- if (!res.ok) return false;
2216
- const renewed = await res.json() as { token?: string; record?: { jti: string; profileId?: string; expiresAt?: number } };
2217
- if (!renewed?.token || !renewed.record) return false;
2218
- this.applyRenewedToken(renewed.token, renewed.record, "runtime-token-reminted");
2219
- this.logRunnerDiagnostic(`[runner] runtime token re-minted via orchestrator (jti ${renewed.record.jti})`);
2220
- return true;
2221
- } catch (error) {
2222
- this.logRuntimeTokenRenewalFailure(error);
2223
- return false;
2224
- }
2225
- }
2226
-
2227
- private logRuntimeTokenRenewalFailure(error: unknown): void {
2228
- const key = httpErrorKey(error);
2229
- const now = Date.now();
2230
- if (
2231
- this.tokenRenewLastLog?.key === key &&
2232
- now - this.tokenRenewLastLog.at < HTTP_LIVENESS_LOG_INTERVAL_MS
2233
- ) return;
2234
- this.tokenRenewLastLog = { key, at: now };
2235
- this.logRunnerDiagnostic(`[runner] runtime token renewal failed: ${String(error)}`);
2236
- }
2237
-
2238
- private heartbeatRuntimeMeta(): Record<string, unknown> | undefined {
2239
- const processContext = this.latestProcessContext();
2240
- const probeMetrics = this.latestProbeMetrics();
2241
- const probeContext = probeMetrics ? contextStateFromProbeMetrics(probeMetrics) : undefined;
2242
- const context = processContext ?? probeContext;
2243
- const terminalSession = this.providerTerminalSession();
2244
- const terminalSocket = this.providerTerminalSocket();
2245
- const probeModel: ProbeModelInfo | undefined = probeMetrics?.model || probeMetrics?.effort
2246
- ? { model: probeMetrics.model, effort: probeMetrics.effort }
2247
- : undefined;
2248
- const meta: Record<string, unknown> = {
2249
- providerCapabilities: runtimeProviderCapabilities(
2250
- this.options,
2251
- context,
2252
- probeModel,
2253
- ),
2254
- ...(terminalSession ? { tmuxSession: terminalSession } : {}),
2255
- ...(terminalSocket ? { tmuxSocket: terminalSocket } : {}),
2256
- };
2257
- if (context) meta.context = context;
2258
- return meta;
2259
- }
2260
-
2261
- private providerTerminalSession(): string | undefined {
2262
- return typeof this.process?.meta?.tmuxSession === "string" ? this.process.meta.tmuxSession : undefined;
2263
- }
2264
-
2265
- private providerTerminalSocket(): string | undefined {
2266
- return typeof this.process?.meta?.tmuxSocket === "string" ? this.process.meta.tmuxSocket : undefined;
2267
- }
2268
-
2269
- private latestProcessContext(): ContextState | undefined {
2270
- const getContext = this.process?.meta?.getContext;
2271
- if (typeof getContext === "function") {
2272
- const context = getContext();
2273
- if (isContextState(context)) return context;
2274
- }
2275
- const context = this.process?.meta?.context;
2276
- return isContextState(context) ? context : undefined;
2277
- }
2278
-
2279
- private latestProbeMetrics() {
2280
- const stateDir = process.env.AGENT_RELAY_CONTEXT_STATE_DIR;
2281
- return readContextProbeState(this.agentId, stateDir);
2282
- }
2283
-
2284
- private matchesMessage(message: Message): boolean {
2285
- return runnerMessageMatches(message, {
2286
- agentId: this.agentId,
2287
- label: this.options.label,
2288
- provider: this.options.provider,
2289
- tags: this.options.tags,
2290
- capabilities: this.options.capabilities,
2291
- defaultTags: this.options.providerConfig.defaultTags,
2292
- defaultCapabilities: this.options.providerConfig.defaultCapabilities,
2293
- allowBroadTargets: this.options.agentProfile?.relay?.context !== false,
2294
- });
2295
- }
2296
-
2297
- private startClaimRenewer(): void {
2298
- if (this.claimRenewTimer) return;
2299
- this.claimRenewTimer = setInterval(() => {
2300
- void this.renewActiveTaskClaims();
2301
- }, CLAIM_RENEW_INTERVAL_MS);
2302
- }
2303
-
2304
- private async renewActiveTaskClaims(): Promise<void> {
2305
- for (const claim of this.activeTaskClaims.values()) {
2306
- const renewed = await this.http.renewMessageClaim(claim.messageId, this.agentId).catch(() => ({ ok: false, claimExpiresAt: undefined }));
2307
- if (renewed.ok) {
2308
- this.claims.startClaim("message", String(claim.messageId), renewed.claimExpiresAt);
2309
- this.claims.startClaim("task", String(claim.taskId), renewed.claimExpiresAt);
2310
- }
2311
- }
2312
- this.publishStatus();
2313
- }
2314
-
2315
- private async completeObservedTaskClaims(): Promise<void> {
2316
- const completed = [...this.activeTaskClaims.values()].filter((claim) => claim.observedProviderBusy);
2317
- for (const claim of completed) {
2318
- const ok = await this.updateTaskStatus(claim.taskId, {
2319
- status: "done",
2320
- agentId: this.agentId,
2321
- body: `Runner observed provider completion for message #${claim.messageId}`,
2322
- metadata: { messageId: claim.messageId, completedBy: "runner" },
2323
- })
2324
- .then(() => true)
2325
- .catch((error) => {
2326
- logger.error("task", `task ${claim.taskId} completion update failed: ${error}`);
2327
- return false;
2328
- });
2329
- if (!ok) continue;
2330
- this.activeTaskClaims.delete(claim.messageId);
2331
- this.claims.finishClaim("message", String(claim.messageId));
2332
- this.claims.finishClaim("task", String(claim.taskId));
2333
- }
2334
- if (completed.length > 0) this.publishStatus();
2335
- }
2336
-
2337
- private async updateTaskStatus(taskId: number, input: TaskStatusInput): Promise<void> {
2338
- await this.http.updateTaskStatus(taskId, input);
2339
- }
2340
-
2341
- private clearActiveClaim(message: Message): void {
2342
- const taskId = taskIdFromMessage(message);
2343
- this.claims.finishClaim("message", String(message.id));
2344
- if (!taskId) return;
2345
- this.activeTaskClaims.delete(message.id);
2346
- this.claims.finishClaim("task", String(taskId));
2347
- }
2348
- }
2349
-
2350
- export function runnerMessageMatches(message: Pick<Message, "to" | "resolvedToAgent">, target: {
2351
- agentId: string;
2352
- label?: string;
2353
- provider: string;
2354
- tags: string[];
2355
- capabilities: string[];
2356
- defaultTags: string[];
2357
- defaultCapabilities: string[];
2358
- allowBroadTargets?: boolean;
2359
- }): boolean {
2360
- if (message.resolvedToAgent === target.agentId) return true;
2361
- if (message.to === target.agentId) return true;
2362
- if (target.label && message.to === `label:${target.label}`) return true;
2363
- if (target.allowBroadTargets === false) return false;
2364
- if (message.to === "broadcast") return true;
2365
- if (message.to.startsWith("tag:")) {
2366
- const tag = message.to.slice("tag:".length);
2367
- return target.tags.includes(tag) || target.defaultTags.includes(tag) || tag === target.provider;
2368
- }
2369
- if (message.to.startsWith("cap:")) {
2370
- const cap = message.to.slice("cap:".length);
2371
- return target.capabilities.includes(cap) || target.defaultCapabilities.includes(cap);
2372
- }
2373
- return false;
2374
- }
2375
-
2376
- export function taskIdFromMessage(message: Pick<Message, "payload">): number | undefined {
2377
- const taskId = message.payload?.taskId;
2378
- return Number.isSafeInteger(taskId) ? taskId as number : undefined;
2379
- }
2380
-
2381
- // The server appends a self-claim/status instruction to task message bodies
2382
- // (taskMessageBody in src/db.ts: "Claim this task before working it, then update
2383
- // task status when finished."). For runner-managed agents the runner already owns
2384
- // claim + status, so that line is not only redundant but actively harmful: the
2385
- // agent tries to follow it, has no real claim command, and improvises a stray
2386
- // `agent-relay /send-claimable claim <id>` that leaves an orphan claimable message.
2387
- // Strip the trailing guidance from messages the runner has auto-claimed.
2388
- const RUNNER_CLAIMED_GUIDANCE_RE = /\s*Claim this task before working it, then update task status when finished\.\s*$/;
2389
-
2390
- export function stripRunnerClaimedGuidance(body: string): string {
2391
- return body.replace(RUNNER_CLAIMED_GUIDANCE_RE, "");
2392
- }
2393
-
2394
- function csvTags(raw: string | undefined): string[] {
2395
- return (raw || "").split(",").map((tag) => tag.trim()).filter(Boolean);
2396
- }
2397
-
2398
- function relayBusUrl(relayUrl: string): string {
2399
- const url = new URL(relayUrl);
2400
- url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
2401
- url.pathname = `${url.pathname.replace(/\/+$/, "")}/bus`;
2402
- url.search = "";
2403
- return url.toString();
2404
- }
2405
-
2406
- export function runnerAgentStatus(status: "idle" | "busy" | "offline" | "error"): "idle" | "busy" | "offline" {
2407
- return status === "error" ? "offline" : status;
2408
- }
2409
-
2410
- export function runnerBusErrorAction(code: string, stopped: boolean): "ignore" | "log" | "stop" {
2411
- if (code === "STALE_SESSION") return stopped ? "ignore" : "stop";
2412
- return "log";
2413
- }
2414
-
2415
- export function runnerShouldResolveProviderExit(status: SemanticStatus, exitCommandInProgress: boolean): boolean {
2416
- return !exitCommandInProgress && (status === "offline" || status === "error");
2417
- }
2418
-
2419
- export function runnerShouldRestartUnexpectedProviderExit(
2420
- status: SemanticStatus,
2421
- input: {
2422
- exitCommandInProgress: boolean;
2423
- stopped: boolean;
2424
- restartInProgress: boolean;
2425
- provider?: string;
2426
- headless: boolean;
2427
- hasTerminalSession: boolean;
2428
- },
2429
- ): boolean {
2430
- return status === "offline"
2431
- && !input.exitCommandInProgress
2432
- && !input.stopped
2433
- && !input.restartInProgress
2434
- && input.headless
2435
- && input.hasTerminalSession;
2436
- }
2437
-
2438
- export function latestClaudeResumeIdFromText(text: string): string | undefined {
2439
- let latest: string | undefined;
2440
- CLAUDE_RESUME_RE.lastIndex = 0;
2441
- for (let match = CLAUDE_RESUME_RE.exec(text); match; match = CLAUDE_RESUME_RE.exec(text)) {
2442
- latest = match[1];
2443
- }
2444
- return latest;
2445
- }
2446
-
2447
- function latestClaudeResumeIdFromLogFile(path: string): string | undefined {
2448
- let fd: number | undefined;
2449
- try {
2450
- const stat = statSync(path);
2451
- const length = Math.min(stat.size, LOG_TAIL_BYTES);
2452
- const offset = Math.max(0, stat.size - length);
2453
- const buffer = Buffer.alloc(length);
2454
- fd = openSync(path, "r");
2455
- readSync(fd, buffer, 0, length, offset);
2456
- return latestClaudeResumeIdFromText(buffer.toString("utf8"));
2457
- } catch {
2458
- return undefined;
2459
- } finally {
2460
- if (fd !== undefined) closeSync(fd);
2461
- }
2462
- }
2463
-
2464
- function commandTimeoutMs(params: Record<string, unknown>, fallback = 10_000): number {
2465
- const raw = params.timeoutMs;
2466
- if (typeof raw !== "number" || !Number.isSafeInteger(raw) || raw <= 0) return fallback;
2467
- return Math.min(raw, 60_000);
2468
- }
2469
-
2470
- export function shouldLogDeliveryFailure(error: unknown): boolean {
2471
- const message = errMessage(error);
2472
- return message !== "no Claude monitor connected";
2473
- }
2474
-
2475
- export function runtimeTokenRenewDelayMs(expiresAtSeconds: number, nowMs = Date.now()): number | undefined {
2476
- const expiresAtMs = expiresAtSeconds * 1000;
2477
- const ttlMs = expiresAtMs - nowMs;
2478
- if (!Number.isFinite(ttlMs) || ttlMs <= 0) return undefined;
2479
- const leadMs = Math.min(60 * 60 * 1000, Math.max(60_000, Math.floor(ttlMs * 0.2)));
2480
- const jitterMs = Math.floor(Math.random() * Math.min(30_000, Math.max(0, leadMs / 2)));
2481
- return Math.max(1_000, ttlMs - leadMs - jitterMs);
2482
- }
2483
-
2484
- export function runtimeTokenRenewTimerSchedule(delayMs: number): { delayMs: number; renew: boolean } | undefined {
2485
- if (!Number.isFinite(delayMs) || delayMs <= 0) return undefined;
2486
- if (delayMs > MAX_TIMER_DELAY_MS) return { delayMs: MAX_TIMER_DELAY_MS, renew: false };
2487
- return { delayMs, renew: true };
2488
- }
2489
-
2490
- function lifecycleCapabilities(): Record<string, true> {
2491
- return {
2492
- shutdownHard: true,
2493
- restartHard: true,
2494
- semanticStatus: true,
2495
- transportReconnect: true,
2496
- };
2497
- }
2498
-
2499
- interface ProbeModelInfo {
2500
- model?: string;
2501
- effort?: string;
2502
- }
2503
-
2504
- function runtimeProviderCapabilities(options: RunnerOptions, contextState?: ContextState, probeModel?: ProbeModelInfo): ProviderCapabilities {
2505
- const model = options.model ?? probeModel?.model;
2506
- const effort = options.effort ?? probeModel?.effort;
2507
- const modelSource = options.model ? "runtime" as const : probeModel?.model ? "provider" as const : "runtime" as const;
2508
- const shellMode = runtimeShellMode(options);
2509
- return {
2510
- lifecycle: {
2511
- managed: true,
2512
- shutdownHard: options.provider === "claude" ? options.headless : true,
2513
- restartHard: options.provider === "claude" ? options.headless : true,
2514
- semanticStatus: true,
2515
- reconnect: true,
2516
- },
2517
- model: {
2518
- provider: options.provider,
2519
- alias: model,
2520
- id: model,
2521
- effort,
2522
- source: modelSource,
2523
- confidence: options.model ? "reported" : probeModel?.model ? "reported" : "unknown",
2524
- lastUpdatedAt: options.startedAt,
2525
- },
2526
- session: {
2527
- approvalMode: options.approvalMode,
2528
- fileRead: true,
2529
- fileWrite: options.approvalMode !== "read-only",
2530
- shell: shellMode !== "none",
2531
- shellMode,
2532
- source: "runtime",
2533
- confidence: "reported",
2534
- lastUpdatedAt: options.startedAt,
2535
- },
2536
- ...runtimeProviderContextCapabilities(options, contextState),
2537
- ...runtimeProviderTerminalCapabilities(options),
2538
- liveSession: {
2539
- capture: true,
2540
- inject: Boolean(options.adapter.deliverInitialPrompt),
2541
- interrupt: Boolean(options.adapter.interrupt),
2542
- // Both providers mirror directly-typed prompts and stream reasoning/tool
2543
- // activity into chat (Claude via hooks + transcript tail, Codex via
2544
- // app-server item events).
2545
- promptEcho: true,
2546
- reasoning: true,
2547
- slashCommands: options.provider === "claude" || options.provider === "codex",
2548
- },
2549
- source: "runtime",
2550
- confidence: "reported",
2551
- lastUpdatedAt: options.startedAt,
2552
- };
2553
- }
2554
-
2555
- function runtimeShellMode(options: RunnerOptions): NonNullable<NonNullable<ProviderCapabilities["session"]>["shellMode"]> {
2556
- if (options.approvalMode === "read-only") {
2557
- return options.provider === "claude" ? "read-only-guarded" : "none";
2558
- }
2559
- return options.approvalMode === "open" ? "unrestricted" : "guarded";
2560
- }
2561
-
2562
- function runtimeProviderTerminalCapabilities(options: RunnerOptions): Pick<ProviderCapabilities, "terminal"> {
2563
- if (options.provider === "claude" && options.headless) {
2564
- return {
2565
- terminal: {
2566
- live: {
2567
- read: true,
2568
- write: true,
2569
- },
2570
- },
2571
- };
2572
- }
2573
- if (options.provider === "codex") {
2574
- return {
2575
- terminal: {
2576
- attach: {
2577
- create: true,
2578
- read: true,
2579
- write: true,
2580
- detach: true,
2581
- },
2582
- },
2583
- };
2584
- }
2585
- return {};
2586
- }
2587
-
2588
- function appliedAgentProfileMetadata(provider: string, profile: AgentProfile): Record<string, unknown> {
2589
- const projection = provider === "claude" || provider === "codex"
2590
- ? agentProfileProjectionReport({ provider, profile })
2591
- : undefined;
2592
- return {
2593
- name: profile.name,
2594
- base: profile.base,
2595
- provider: profile.provider ?? "any",
2596
- relay: profile.relay,
2597
- instructions: {
2598
- repoInstructions: profile.instructions.repoInstructions,
2599
- globalInstructions: profile.instructions.globalInstructions,
2600
- appendCount: profile.instructions.append.length,
2601
- hasSystem: Boolean(profile.instructions.system),
2602
- },
2603
- skills: profile.skills.filter((item) => item.enabled).length,
2604
- plugins: profile.plugins.filter((item) => item.enabled).length,
2605
- mcp: { mode: profile.mcp.mode },
2606
- hooks: { mode: profile.hooks.mode },
2607
- permissions: profile.permissions,
2608
- projection,
2609
- };
2610
- }
2611
-
2612
- function runtimeProviderContextCapabilities(options: RunnerOptions, contextState?: ContextState): Pick<ProviderCapabilities, "context"> {
2613
- const context: NonNullable<ProviderCapabilities["context"]> = { resume: "none" };
2614
- const supportsManagedContext = options.provider === "codex" || (options.provider === "claude" && options.headless);
2615
- if (contextState) {
2616
- context.stats = { source: contextState.source, confidence: contextState.confidence };
2617
- if (typeof contextState.tokensMax === "number" && Number.isFinite(contextState.tokensMax)) {
2618
- context.windowTokens = contextState.tokensMax;
2619
- }
2620
- }
2621
- if (supportsManagedContext) {
2622
- context.compact = true;
2623
- context.clear = true;
2624
- }
2625
- context.inject = true;
2626
- if (supportsManagedContext && options.adapter.compact && options.adapter.compactSupportsInstructions) context.resume = "native";
2627
- else if (supportsManagedContext && options.adapter.clearContext) context.resume = "clear-inject";
2628
- return Object.keys(context).length ? { context } : {};
2629
- }
2630
-
2631
- function providerStateFromActiveWork(activeWork: Array<{ kind: string; metadata?: Record<string, unknown> }>): Record<string, unknown> | null {
2632
- const providerTurn = activeWork.find((item) => item.kind === "provider-turn");
2633
- const state = providerTurn?.metadata?.providerState;
2634
- return state && typeof state === "object" && !Array.isArray(state) ? state as Record<string, unknown> : null;
2635
- }
2636
-
2637
- function isHttpAuthError(error: unknown): boolean {
2638
- const status = typeof error === "object" && error !== null ? (error as { status?: unknown }).status : undefined;
2639
- return status === 401 || status === 403;
2640
- }
2641
-
2642
- function isHttpStatusError(error: unknown, code: number): boolean {
2643
- const status = typeof error === "object" && error !== null ? (error as { status?: unknown }).status : undefined;
2644
- return status === code;
2645
- }
2646
-
2647
- function httpErrorKey(error: unknown): string {
2648
- const status = typeof error === "object" && error !== null ? (error as { status?: unknown }).status : undefined;
2649
- if (typeof status === "number") return `status:${status}`;
2650
- return String(error);
2651
- }
2652
-
2653
- function isContextState(value: unknown): value is ContextState {
2654
- if (!value || typeof value !== "object" || Array.isArray(value)) return false;
2655
- const state = value as Record<string, unknown>;
2656
- return typeof state.utilization === "number" &&
2657
- typeof state.lifecycleState === "string" &&
2658
- typeof state.source === "string" &&
2659
- typeof state.confidence === "string";
2660
- }
1
+ export { AgentRunner } from "./runner-core";
2
+ export {
3
+ latestClaudeResumeIdFromText,
4
+ runnerAgentStatus,
5
+ runnerBusErrorAction,
6
+ runnerMessageMatches,
7
+ runnerShouldResolveProviderExit,
8
+ runnerShouldRestartUnexpectedProviderExit,
9
+ runtimeTokenRenewDelayMs,
10
+ runtimeTokenRenewTimerSchedule,
11
+ shouldLogDeliveryFailure,
12
+ stripRunnerClaimedGuidance,
13
+ taskIdFromMessage,
14
+ } from "./runner-helpers";