agent-relay-runner 0.62.3 → 0.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/plugins/claude/.claude-plugin/plugin.json +1 -1
- package/src/busy-reconciler.ts +84 -0
- package/src/mcp-outbox.ts +86 -0
- package/src/rate-limit.ts +2 -2
- package/src/runner-core.ts +2224 -0
- package/src/runner-helpers.ts +317 -0
- package/src/runner-insights.ts +106 -0
- package/src/runner.ts +14 -2660
package/src/runner.ts
CHANGED
|
@@ -1,2660 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
import { Outbox, type OutboxRecord } from "./outbox";
|
|
16
|
-
import { extractLastAssistantTurn, extractLastAssistantTurnAfterEntry, countTranscriptEntries, extractFinalAssistantMessage, extractHookAssistantMessage, extractLatestTurnSteps, stepDedupKeys, transcriptLooksComplete } from "./adapters/claude-transcript";
|
|
17
|
-
import { computeContextRatio } from "./session-insights";
|
|
18
|
-
import { agentProfileProjectionReport } from "./profile-projection";
|
|
19
|
-
import { profileUsesHostProviderGlobals } from "./profile-home";
|
|
20
|
-
import { RELAY_MCP_TOKEN_ENV, relayMcpEndpoint } from "./relay-mcp";
|
|
21
|
-
import { RelayMcpProxy } from "./relay-mcp-proxy";
|
|
22
|
-
import { runtimeMetadata } from "./version";
|
|
23
|
-
import { logger, parseLogLevel } from "./logger";
|
|
24
|
-
import { ensureSessionScratch, reapSessionScratch, sweepStaleSessions, type SessionScratchLayout } from "./session-scratch";
|
|
25
|
-
import { boundContinuationArchiveSegment, deliverContinuationArchiveRecord } from "./continuation-archive";
|
|
26
|
-
|
|
27
|
-
// A destructive session transition. The runner runs end-of-session work (Insights
|
|
28
|
-
// capture, #183/#184) before the invasive operation and, during that window, presents a
|
|
29
|
-
// distinct non-addressable lifecycle state. Bus commands and provider hooks (Claude
|
|
30
|
-
// PreCompact / SessionEnd) both normalize to one of these.
|
|
31
|
-
type SessionDestroyReason = "compact" | "clear" | "restart" | "shutdown" | "kill" | "crash";
|
|
32
|
-
|
|
33
|
-
// Reasons after which the runner process won't survive to drain the durable outbox (and the
|
|
34
|
-
// per-agent outbox is never reopened once the agent is gone). For these, pre-destroy must
|
|
35
|
-
// block on delivery of the just-captured Insights datapoint, not just enqueue it (#183).
|
|
36
|
-
// `restart` (bus command) deliberately excluded: the runner stays alive and drains normally.
|
|
37
|
-
function reasonExitsRunner(reason: SessionDestroyReason): boolean {
|
|
38
|
-
return reason === "shutdown" || reason === "kill" || reason === "crash";
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
// `finalizing-<reason>` is the transient pre-destroy window; the others are the executing
|
|
42
|
-
// teardown states the dashboard already renders.
|
|
43
|
-
type LifecycleAction =
|
|
44
|
-
| "shutting-down" | "killing" | "restarting"
|
|
45
|
-
| `finalizing-${SessionDestroyReason}`;
|
|
46
|
-
|
|
47
|
-
// Pre-destroy work is best-effort and must never hang teardown. Capping it keeps a slow
|
|
48
|
-
// transcript read or a wedged provider from stalling a shutdown the operator asked for.
|
|
49
|
-
const PRE_DESTROY_TIMEOUT_MS = 4_000;
|
|
50
|
-
|
|
51
|
-
// Bounded window to deliver the durable outbox before an exit-bound teardown (#183). Kept
|
|
52
|
-
// short so a wedged/down server can't stall an operator-requested shutdown for long; a
|
|
53
|
-
// row that still can't land is logged, not silently dropped.
|
|
54
|
-
const OUTBOX_FLUSH_TIMEOUT_MS = 3_000;
|
|
55
|
-
|
|
56
|
-
// Map a lifecycle bus command to its destructive boundary reason, or undefined for
|
|
57
|
-
// non-destructive commands (interrupt, inject, reconnect, permission decisions).
|
|
58
|
-
function boundaryReasonForCommand(type: string): SessionDestroyReason | undefined {
|
|
59
|
-
switch (type) {
|
|
60
|
-
case "agent.compact": return "compact";
|
|
61
|
-
case "agent.clearContext": return "clear";
|
|
62
|
-
case "agent.restart": return "restart";
|
|
63
|
-
case "agent.shutdown": return "shutdown";
|
|
64
|
-
case "agent.kill": return "kill";
|
|
65
|
-
default: return undefined;
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
interface RunnerOptions {
|
|
70
|
-
provider: string;
|
|
71
|
-
model?: string;
|
|
72
|
-
effort?: string;
|
|
73
|
-
runnerId: string;
|
|
74
|
-
instanceId: string;
|
|
75
|
-
agentId?: string;
|
|
76
|
-
relayUrl: string;
|
|
77
|
-
token?: string;
|
|
78
|
-
tokenJti?: string;
|
|
79
|
-
tokenProfileId?: string;
|
|
80
|
-
tokenExpiresAt?: number;
|
|
81
|
-
rootTokenFallback?: boolean;
|
|
82
|
-
cwd: string;
|
|
83
|
-
headless: boolean;
|
|
84
|
-
approvalMode: string;
|
|
85
|
-
label?: string;
|
|
86
|
-
rig?: string;
|
|
87
|
-
profile?: string;
|
|
88
|
-
agentProfile?: AgentProfile;
|
|
89
|
-
prompt?: string;
|
|
90
|
-
systemPromptAppend?: string;
|
|
91
|
-
tags: string[];
|
|
92
|
-
capabilities: string[];
|
|
93
|
-
providerArgs: string[];
|
|
94
|
-
policyName?: string;
|
|
95
|
-
spawnRequestId?: string;
|
|
96
|
-
automationId?: string; automationRunId?: string;
|
|
97
|
-
workspace?: WorkspaceMetadata; lifecycle?: AgentLifecycle;
|
|
98
|
-
tmuxSession?: string;
|
|
99
|
-
startedAt: number;
|
|
100
|
-
providerConfig: ProviderConfig;
|
|
101
|
-
adapter: ProviderAdapter;
|
|
102
|
-
onProviderExit?: (code: number | null) => void;
|
|
103
|
-
exitProcessOnShutdown?: boolean;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
interface ActiveTaskClaim {
|
|
107
|
-
messageId: number;
|
|
108
|
-
taskId: number;
|
|
109
|
-
observedProviderBusy: boolean;
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
const CLAIM_RENEW_INTERVAL_MS = 5 * 60 * 1000;
|
|
113
|
-
const HTTP_LIVENESS_INTERVAL_MS = 20_000;
|
|
114
|
-
const HTTP_LIVENESS_LOG_INTERVAL_MS = 5 * 60 * 1000;
|
|
115
|
-
const TOKEN_RENEW_RETRY_MS = 60_000;
|
|
116
|
-
// Debounce reactive token recovery so a burst of 401-ing calls in the same window
|
|
117
|
-
// triggers a single re-mint attempt, not one per failing request.
|
|
118
|
-
const REACTIVE_TOKEN_RECOVERY_DEBOUNCE_MS = 10_000;
|
|
119
|
-
const UNEXPECTED_EXIT_WINDOW_MS = 2 * 60 * 1000;
|
|
120
|
-
const RAPID_EXIT_MS = 30 * 1000;
|
|
121
|
-
const MAX_RAPID_UNEXPECTED_EXITS = 3;
|
|
122
|
-
const MAX_TIMER_DELAY_MS = 2_147_483_647;
|
|
123
|
-
const LOG_TAIL_BYTES = 128 * 1024;
|
|
124
|
-
// A UserPromptSubmit echo matching a runner-injected prompt within this window is
|
|
125
|
-
// the same prompt arriving back from the provider — drop it to avoid a duplicate.
|
|
126
|
-
const PROMPT_ECHO_DEDUP_MS = 30_000;
|
|
127
|
-
// Busy reconciler: a conservative LAST-RESORT backstop for a turn that ended
|
|
128
|
-
// without the provider's Stop hook clearing busy (e.g. ESC straight into the web
|
|
129
|
-
// terminal). It must never fire during a live turn, so it requires a long,
|
|
130
|
-
// unbroken idle streak — an active turn shows its working spinner well within
|
|
131
|
-
// this window, which resets the streak. ~32s of uninterrupted idle = really done.
|
|
132
|
-
const BUSY_RECONCILE_POLL_MS = 4_000;
|
|
133
|
-
const BUSY_RECONCILE_IDLE_CONFIRM = 8;
|
|
134
|
-
// When the reconciler never observed the provider busy this turn (a turn faster
|
|
135
|
-
// than the 4s poll — common for short voice/autosend replies), it can't trust a
|
|
136
|
-
// quick idle the way it does after seeing the spinner. But refusing forever wedged
|
|
137
|
-
// fast turns in "busy" when the Stop hook's idle was lost (#199). So we still
|
|
138
|
-
// force-clear, just after a much longer unbroken-idle window — an active turn would
|
|
139
|
-
// have flashed its spinner into at least one of these probes and reset the streak.
|
|
140
|
-
const BUSY_RECONCILE_IDLE_CONFIRM_NO_BUSY = 15;
|
|
141
|
-
// After a dashboard interrupt, give the provider a moment to drop out of its turn,
|
|
142
|
-
// then reconcile immediately so the user sees "stopped" without waiting for the backstop.
|
|
143
|
-
const INTERRUPT_RECONCILE_DELAY_MS = 1_500;
|
|
144
|
-
// Relay-injected content (delivered messages, memory context) is wrapped with
|
|
145
|
-
// these markers; a UserPromptSubmit echo starting with one is a runner injection,
|
|
146
|
-
// not a human typing into the terminal, so it must not be mirrored as a prompt.
|
|
147
|
-
// `<task-notification>` is the harness wrapper for async monitor/task wake-ups
|
|
148
|
-
// (which embed a `[relay message #…]` deeper inside, defeating a plain prefix
|
|
149
|
-
// match) — it is emitted only by the harness, never typed by a human, so any
|
|
150
|
-
// echo starting with it is a system injection, not a user prompt (#289).
|
|
151
|
-
const RELAY_INJECTION_MARKERS = ["[relay message #", "[agent-relay", "<task-notification>"];
|
|
152
|
-
// #329: a cold isolated profile (first-run trust/onboarding/install gates) can take far longer
|
|
153
|
-
// than the warm-restart window to render an input-ready TUI, so the first prompt delivery gets a
|
|
154
|
-
// generous timeout. If it still misses, the prompt is re-attempted on the next provider ready
|
|
155
|
-
// signal (short timeout, since it just went idle), up to a bounded number of attempts.
|
|
156
|
-
const INITIAL_PROMPT_FIRST_READY_TIMEOUT_MS = 30_000;
|
|
157
|
-
const INITIAL_PROMPT_RETRY_READY_TIMEOUT_MS = 10_000;
|
|
158
|
-
const MAX_INITIAL_PROMPT_ATTEMPTS = 6;
|
|
159
|
-
// Reasoning tailer poll cadence (item 5). Coarse on purpose — reasoning is a
|
|
160
|
-
// discreet progress signal, not a token stream, so ~1.2s keeps it light.
|
|
161
|
-
const REASONING_POLL_MS = 1_200;
|
|
162
|
-
const CLAUDE_RESUME_RE = /\bclaude\s+--resume\s+([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b/gi;
|
|
163
|
-
|
|
164
|
-
interface RunnerTimelineEvent {
|
|
165
|
-
status: string;
|
|
166
|
-
id?: string;
|
|
167
|
-
timestamp: number;
|
|
168
|
-
title?: string;
|
|
169
|
-
body?: string;
|
|
170
|
-
icon?: string;
|
|
171
|
-
metadata?: Record<string, unknown>;
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
export class AgentRunner {
|
|
175
|
-
private readonly agentId: string;
|
|
176
|
-
private readonly claims = new ClaimTracker();
|
|
177
|
-
private readonly http: RelayHttpClient;
|
|
178
|
-
private readonly bus: RelayBusClient;
|
|
179
|
-
// Phase 2 (#196): the Stop hook reads reply obligations from this local snapshot, never
|
|
180
|
-
// from the server — so a slow server can no longer wedge a turn (the crux fix).
|
|
181
|
-
private readonly obligationCache: ReplyObligationCache;
|
|
182
|
-
// Phase 2 (#196): Runner→server append-log events (session turns, reasoning, prompts,
|
|
183
|
-
// insights, hook-fatal) go through this durable, disk-backed, timestamped queue instead of
|
|
184
|
-
// direct fire-and-forget HTTP — so nothing is lost across a server/Runner restart.
|
|
185
|
-
private readonly outbox: Outbox;
|
|
186
|
-
// #332: a fast-lane outbox for display-only session-mirror trace events. Routing them through
|
|
187
|
-
// their own FIFO queue stops one transient trace failure from head-of-line blocking real-message
|
|
188
|
-
// delivery (and vice versa); its backoff is capped low since a stale live-mirror frame is cheap.
|
|
189
|
-
private readonly sessionOutbox: Outbox;
|
|
190
|
-
private currentToken?: string;
|
|
191
|
-
private currentTokenJti?: string;
|
|
192
|
-
private currentTokenProfileId?: string;
|
|
193
|
-
private currentTokenExpiresAt?: number;
|
|
194
|
-
private control?: ControlServer;
|
|
195
|
-
// Stage 2 (#215): the local MCP endpoint the agent connects to, fronting the relay so the
|
|
196
|
-
// Runner owns reconnect/backoff + a durable buffer. Disabled via AGENT_RELAY_MCP_PROXY=0
|
|
197
|
-
// (then the agent connects to the relay directly, Stage-1 behaviour). The proxy secret is the
|
|
198
|
-
// bearer the agent presents to the localhost proxy — it decouples the agent from the rotating
|
|
199
|
-
// relay token (the proxy injects the live token relay-side).
|
|
200
|
-
private proxy?: RelayMcpProxy;
|
|
201
|
-
private mcpProxyEndpoint?: string;
|
|
202
|
-
private readonly mcpProxyEnabled: boolean;
|
|
203
|
-
private readonly mcpProxySecret: string;
|
|
204
|
-
private process?: ManagedProcess;
|
|
205
|
-
private stopped = false;
|
|
206
|
-
private exitCommandInProgress = false;
|
|
207
|
-
private restartInProgress = false;
|
|
208
|
-
// Set for the whole unexpected-exit restart, including the pre-restart backoff
|
|
209
|
-
// sleep, so a second exit during that window can't launch a parallel restart.
|
|
210
|
-
private restartPending = false;
|
|
211
|
-
private delivering = false;
|
|
212
|
-
private drainTimer?: Timer;
|
|
213
|
-
private claimRenewTimer?: Timer;
|
|
214
|
-
private httpLivenessTimer?: Timer;
|
|
215
|
-
private httpLivenessInFlight = false;
|
|
216
|
-
private httpLivenessAuthFailed = false;
|
|
217
|
-
private httpLivenessLastLog?: { key: string; at: number };
|
|
218
|
-
private tokenRenewTimer?: Timer;
|
|
219
|
-
private tokenRenewInFlight = false;
|
|
220
|
-
private tokenRenewLastLog?: { key: string; at: number };
|
|
221
|
-
private reactiveTokenRecoveryAt?: number;
|
|
222
|
-
private processStartedAt = 0;
|
|
223
|
-
private providerSessionId = crypto.randomUUID();
|
|
224
|
-
// Last transcript path seen this session — used by end-of-session Insights (#184)
|
|
225
|
-
// when the SessionEnd hook payload omits it.
|
|
226
|
-
private lastTranscriptPath?: string;
|
|
227
|
-
private lifecycleAction?: LifecycleAction;
|
|
228
|
-
// #183/#184 per-segment cursor: how many of the current session's normalized events
|
|
229
|
-
// have already been folded into an observation, and the key (transcript path / Codex
|
|
230
|
-
// session) that count belongs to. A boundary captures only events since the last one,
|
|
231
|
-
// so each datapoint is one work chunk between context resets; a key change or a shrink
|
|
232
|
-
// (transcript rotated, Codex buffer trimmed) resets the cursor.
|
|
233
|
-
private insightsObserved = 0;
|
|
234
|
-
private insightsCursorKey = "";
|
|
235
|
-
private archiveObservedChars = 0;
|
|
236
|
-
private archiveCursorKey = "";
|
|
237
|
-
// Memoized repo-name project id for insight observations (resolved once; involves a
|
|
238
|
-
// git toplevel lookup for direct agents). Aggregates by repo, not per-session worktree.
|
|
239
|
-
private insightProjectName?: string;
|
|
240
|
-
// Coalesces concurrent pre-session-destroy runs (e.g. the shutdown bus command and the
|
|
241
|
-
// SessionEnd hook both fire for the same teardown) so the cursor isn't raced.
|
|
242
|
-
private preDestroyPromise?: Promise<void>;
|
|
243
|
-
private readonly unexpectedExitTimes: number[] = [];
|
|
244
|
-
private readonly pendingMessages = new Map<number, Message>();
|
|
245
|
-
private readonly activeTaskClaims = new Map<number, ActiveTaskClaim>();
|
|
246
|
-
private pendingTimelineEvent?: RunnerTimelineEvent;
|
|
247
|
-
private pendingPromptMessageId?: number;
|
|
248
|
-
// #329: a spawn-time initial prompt whose first delivery timed out before the TUI was
|
|
249
|
-
// ready. Held (not dropped) for a ready-signal-driven re-attempt; cleared on success.
|
|
250
|
-
private pendingInitialPrompt?: string;
|
|
251
|
-
private deliveringInitialPrompt = false;
|
|
252
|
-
private initialPromptAttempts = 0;
|
|
253
|
-
// Session-mirror: a synthesized id grouping a turn's reasoning/tool steps and
|
|
254
|
-
// its final response. Set when a provider-turn starts, cleared when it ends.
|
|
255
|
-
private currentTurnId?: string;
|
|
256
|
-
private currentTurnStartedAt?: number;
|
|
257
|
-
// #435: high-water-mark for the pre-turn narrative flush. Counts how many
|
|
258
|
-
// valid JSONL entries were in the transcript at last pre-flush time so the
|
|
259
|
-
// Stop-hook capture (full mode) can skip what was already emitted.
|
|
260
|
-
private narrativeFlushEntryCount = 0;
|
|
261
|
-
// True while a turn that was already in flight is being compacted. Claude's PostCompact
|
|
262
|
-
// hook posts a single `idle`, but a mid-turn compaction RESUMES the turn afterward — so
|
|
263
|
-
// treating that idle as a turn end (flip idle, kill the reasoning tail) makes chat go dark
|
|
264
|
-
// until the next prompt. Set when `compacting` arrives with a turn running; cleared on the
|
|
265
|
-
// genuine end (a plain idle with no compaction timeline).
|
|
266
|
-
private compactionMidTurn = false;
|
|
267
|
-
// Prompt-echo dedup: a short, time-bounded queue of prompts the runner itself
|
|
268
|
-
// injected (chat box or initial prompt) that are still awaiting their matching
|
|
269
|
-
// UserPromptSubmit echo. A single slot dropped earlier entries when several prompts
|
|
270
|
-
// were injected before their echoes returned (rapid sends while the provider is busy
|
|
271
|
-
// and queues them) — the evicted ones then double-posted. Match consumes one entry.
|
|
272
|
-
private injectedPrompts: Array<{ text: string; at: number }> = [];
|
|
273
|
-
// Busy reconciler: consecutive idle probes observed while claims still say busy.
|
|
274
|
-
private busyReconcileIdleStreak = 0;
|
|
275
|
-
private busyReconcileTimer?: ReturnType<typeof setInterval>;
|
|
276
|
-
// The reconciler only trusts an "idle" reading once it has seen the provider
|
|
277
|
-
// actually busy this turn — so a flaky/always-idle probe can never false-clear.
|
|
278
|
-
private busyReconcileSawBusy = false;
|
|
279
|
-
// Verbose session-mirror diagnostics (turn lifecycle, reconciler probes, tail
|
|
280
|
-
// emits) → a dedicated clean log. Always on for key transitions; AGENT_RELAY_SESSION_DEBUG=1 adds the high-frequency probe/emit lines.
|
|
281
|
-
private readonly sessionDebugVerbose = process.env.AGENT_RELAY_SESSION_DEBUG === "1";
|
|
282
|
-
// Tracks whether the provider is in a legitimate blocked/approval state, so the
|
|
283
|
-
// busy reconciler doesn't mistake a permission prompt for a stuck-busy turn.
|
|
284
|
-
private providerBlocked = false;
|
|
285
|
-
private terminalFailure?: { reason: string; message: string; providerState?: Record<string, unknown> };
|
|
286
|
-
// #286: a usage/rate-limit hold. The turn truly ended (idle), but the agent is
|
|
287
|
-
// held until the limit resets, so this can't ride an active-work claim like the
|
|
288
|
-
// permission-blocked state does. publishStatus surfaces it as the agent's
|
|
289
|
-
// providerState; it clears when a new turn starts (the relay's resume message
|
|
290
|
-
// wakes one) so the blocked badge lifts on its own.
|
|
291
|
-
private rateLimitHold?: Record<string, unknown>;
|
|
292
|
-
// Reasoning tailer (item 5): streams the in-flight turn's reasoning/tool steps
|
|
293
|
-
// from the Claude transcript into chat as discreet session events.
|
|
294
|
-
private reasoningTail?: { timer: ReturnType<typeof setInterval>; seen: Set<string> };
|
|
295
|
-
private scratch?: SessionScratchLayout;
|
|
296
|
-
// #417: for non-claude providers, tracks how many times each spawner obligation has
|
|
297
|
-
// been re-injected so a non-responsive agent can't spin in a re-injection loop.
|
|
298
|
-
private readonly pendingObligationInjections = new Map<number, number>();
|
|
299
|
-
|
|
300
|
-
constructor(private readonly options: RunnerOptions) {
|
|
301
|
-
this.agentId = options.agentId ?? options.runnerId;
|
|
302
|
-
// Bind the process-global logger to this agent. AGENT_RELAY_SESSION_DEBUG=1 is
|
|
303
|
-
// kept as a back-compat alias for the verbose probe/emit lines, now expressed
|
|
304
|
-
// as log level "debug" (AGENT_RELAY_LOG_LEVEL still wins when both are set).
|
|
305
|
-
logger.configure({
|
|
306
|
-
agentId: this.agentId,
|
|
307
|
-
headless: options.headless,
|
|
308
|
-
...(this.sessionDebugVerbose && !parseLogLevel(process.env.AGENT_RELAY_LOG_LEVEL) ? { level: "debug" as const } : {}),
|
|
309
|
-
});
|
|
310
|
-
this.currentToken = options.token;
|
|
311
|
-
this.currentTokenJti = options.tokenJti;
|
|
312
|
-
this.currentTokenProfileId = options.tokenProfileId;
|
|
313
|
-
this.currentTokenExpiresAt = options.tokenExpiresAt;
|
|
314
|
-
this.mcpProxyEnabled = !["0", "false", "off"].includes((process.env.AGENT_RELAY_MCP_PROXY ?? "").trim().toLowerCase());
|
|
315
|
-
this.mcpProxySecret = crypto.randomUUID();
|
|
316
|
-
const runtime = runtimeMetadata(options.provider);
|
|
317
|
-
this.http = new RelayHttpClient({ baseUrl: options.relayUrl, token: this.currentToken });
|
|
318
|
-
this.obligationCache = new ReplyObligationCache({ fetch: () => this.http.listReplyObligations(this.agentId) });
|
|
319
|
-
// Co-locate the durable outbox with the runner's runtime state (survives reboot) when the
|
|
320
|
-
// orchestrator told us where that is; otherwise the Outbox falls back to a temp dir.
|
|
321
|
-
const outboxDir = process.env.AGENT_RELAY_RUNNER_OUTBOX_DIR
|
|
322
|
-
?? (process.env.AGENT_RELAY_RUNNER_INFO_FILE ? join(dirname(process.env.AGENT_RELAY_RUNNER_INFO_FILE), "outbox") : undefined);
|
|
323
|
-
this.outbox = new Outbox({ agentId: this.agentId, dir: outboxDir, send: (record) => this.deliverOutboxEvent(record) });
|
|
324
|
-
this.sessionOutbox = new Outbox({
|
|
325
|
-
agentId: `${this.agentId}-session`,
|
|
326
|
-
dir: outboxDir,
|
|
327
|
-
send: (record) => this.deliverOutboxEvent(record),
|
|
328
|
-
// Ephemeral trace frames: cap the per-row stall low and poison sooner than the main queue.
|
|
329
|
-
maxBackoffMs: 5_000,
|
|
330
|
-
maxAttempts: 6,
|
|
331
|
-
});
|
|
332
|
-
this.bus = new RelayBusClient({
|
|
333
|
-
url: relayBusUrl(options.relayUrl),
|
|
334
|
-
role: "provider",
|
|
335
|
-
componentId: options.runnerId,
|
|
336
|
-
agentId: this.agentId,
|
|
337
|
-
instanceId: options.instanceId,
|
|
338
|
-
token: this.currentToken,
|
|
339
|
-
machine: hostname(),
|
|
340
|
-
capabilities: [
|
|
341
|
-
...new Set([
|
|
342
|
-
...options.providerConfig.defaultCapabilities,
|
|
343
|
-
...options.capabilities,
|
|
344
|
-
...csvTags(process.env.AGENT_RELAY_CAPS),
|
|
345
|
-
"lifecycle.shutdown.hard",
|
|
346
|
-
"lifecycle.restart.hard",
|
|
347
|
-
"lifecycle.status.semantic",
|
|
348
|
-
"lifecycle.heartbeat",
|
|
349
|
-
"lifecycle.reconnect.transport",
|
|
350
|
-
"commands.lifecycle",
|
|
351
|
-
"capabilities.report",
|
|
352
|
-
]),
|
|
353
|
-
],
|
|
354
|
-
tags: [...new Set([options.provider, ...csvTags(process.env.AGENT_RELAY_TAGS), ...options.tags, ...options.providerConfig.defaultTags, ...(options.headless ? ["headless"] : [])])],
|
|
355
|
-
meta: {
|
|
356
|
-
...runtime,
|
|
357
|
-
version: runtime.package.version,
|
|
358
|
-
provider: options.provider,
|
|
359
|
-
model: options.model ?? null,
|
|
360
|
-
effort: options.effort ?? null,
|
|
361
|
-
profile: options.profile ?? null,
|
|
362
|
-
agentProfile: options.agentProfile ? appliedAgentProfileMetadata(options.provider, options.agentProfile) : null,
|
|
363
|
-
runnerId: options.runnerId,
|
|
364
|
-
startedAt: options.startedAt,
|
|
365
|
-
tmuxSession: this.providerTerminalSession() ?? options.tmuxSession ?? null,
|
|
366
|
-
tmuxSocket: this.providerTerminalSocket() ?? null,
|
|
367
|
-
policyName: options.policyName ?? null,
|
|
368
|
-
spawnRequestId: options.spawnRequestId ?? null,
|
|
369
|
-
automationId: options.automationId ?? null,
|
|
370
|
-
automationRunId: options.automationRunId ?? null,
|
|
371
|
-
workspace: options.workspace ?? null,
|
|
372
|
-
lifecycle: options.lifecycle ?? "persistent",
|
|
373
|
-
workspaceMode: options.workspace?.requestedMode ?? options.workspace?.mode ?? process.env.AGENT_RELAY_WORKSPACE_MODE ?? null,
|
|
374
|
-
runnerManaged: true,
|
|
375
|
-
cwd: options.cwd,
|
|
376
|
-
approvalMode: options.approvalMode,
|
|
377
|
-
label: options.label ?? null,
|
|
378
|
-
auth: {
|
|
379
|
-
profileId: this.currentTokenProfileId ?? null,
|
|
380
|
-
jti: this.currentTokenJti ?? null,
|
|
381
|
-
expiresAt: this.currentTokenExpiresAt ?? null,
|
|
382
|
-
rootTokenFallback: options.rootTokenFallback === true,
|
|
383
|
-
},
|
|
384
|
-
lifecycleCapabilities: lifecycleCapabilities(),
|
|
385
|
-
providerCapabilities: runtimeProviderCapabilities(options),
|
|
386
|
-
},
|
|
387
|
-
heartbeatMeta: () => this.heartbeatRuntimeMeta(),
|
|
388
|
-
heartbeatIntervalMs: 30_000,
|
|
389
|
-
initialStatus: "idle",
|
|
390
|
-
});
|
|
391
|
-
}
|
|
392
|
-
|
|
393
|
-
get id(): string {
|
|
394
|
-
return this.agentId;
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
async run(): Promise<void> {
|
|
398
|
-
this.control = startControlServer({
|
|
399
|
-
onStatus: (status) => this.setProviderStatus(status),
|
|
400
|
-
onTerminalAttachSpec: () => this.terminalAttachSpec(),
|
|
401
|
-
// Hot-path-safe: answered instantly from the local snapshot, never a server
|
|
402
|
-
// round-trip. The snapshot is kept warm by the background refresh below (#196).
|
|
403
|
-
onReplyObligations: () => Promise.resolve(this.obligationCache.get()),
|
|
404
|
-
onSessionTurn: (input) => this.publishSessionTurn(input),
|
|
405
|
-
onUserPrompt: (input) => this.handleUserPrompt(input),
|
|
406
|
-
onSessionBoundary: (input) => this.handleSessionBoundary(input),
|
|
407
|
-
onHookFatal: (report) => this.reportHookFatal(report),
|
|
408
|
-
});
|
|
409
|
-
this.startMcpProxy();
|
|
410
|
-
this.writeRunnerInfoFile();
|
|
411
|
-
this.options.adapter.onStatusChange((status) => {
|
|
412
|
-
if (this.restartInProgress || this.restartPending) return;
|
|
413
|
-
const semanticStatus = typeof status === "string" ? status : status.status;
|
|
414
|
-
if (this.shouldRestartUnexpectedProviderExit(semanticStatus)) {
|
|
415
|
-
void this.restartUnexpectedProviderExit(semanticStatus);
|
|
416
|
-
return;
|
|
417
|
-
}
|
|
418
|
-
this.setProviderStatus(status);
|
|
419
|
-
if (runnerShouldResolveProviderExit(semanticStatus, this.exitCommandInProgress)) this.options.onProviderExit?.(semanticStatus === "offline" ? 0 : 1);
|
|
420
|
-
});
|
|
421
|
-
this.options.adapter.onSessionEvent?.((event) => { void this.publishProviderSessionEvent(event); });
|
|
422
|
-
this.bus.on("message.new", (message) => {
|
|
423
|
-
// A delivered message may create a new reply obligation — warm the snapshot so the
|
|
424
|
-
// next turn-end sees it without a hot-path server read.
|
|
425
|
-
this.obligationCache.markDirty();
|
|
426
|
-
this.enqueueMessage(message as Message);
|
|
427
|
-
});
|
|
428
|
-
this.bus.on("command", (type, params, commandId, command) => {
|
|
429
|
-
void this.handleCommand(type, params, commandId, command);
|
|
430
|
-
});
|
|
431
|
-
this.bus.on("error", (code, message) => this.handleBusError(String(code), String(message)));
|
|
432
|
-
await this.bus.connect();
|
|
433
|
-
this.obligationCache.start();
|
|
434
|
-
this.outbox.start();
|
|
435
|
-
this.sessionOutbox.start();
|
|
436
|
-
this.ensureScratch();
|
|
437
|
-
void this.sweepStaleScratch();
|
|
438
|
-
this.process = await this.spawnProvider();
|
|
439
|
-
this.writeRunnerInfoFile();
|
|
440
|
-
this.processStartedAt = Date.now();
|
|
441
|
-
this.publishStatus();
|
|
442
|
-
await this.deliverInitialPrompt();
|
|
443
|
-
await this.bootstrapUnreadMessages();
|
|
444
|
-
this.startClaimRenewer();
|
|
445
|
-
this.startHttpLiveness();
|
|
446
|
-
this.scheduleRuntimeTokenRenewal();
|
|
447
|
-
this.scheduleDrain();
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
async stop(): Promise<void> {
|
|
451
|
-
const alreadyStopped = this.stopped;
|
|
452
|
-
this.stopped = true;
|
|
453
|
-
reapSessionScratch({
|
|
454
|
-
agentId: this.agentId,
|
|
455
|
-
cwd: this.options.cwd,
|
|
456
|
-
fallbackBaseDir: process.env.AGENT_RELAY_ORCHESTRATOR_BASE_DIR,
|
|
457
|
-
});
|
|
458
|
-
if (!alreadyStopped) await this.bus.statusAsync({ agentStatus: "offline", ready: false });
|
|
459
|
-
if (this.process && !alreadyStopped) {
|
|
460
|
-
await this.options.adapter.shutdown(this.process, {
|
|
461
|
-
graceful: true,
|
|
462
|
-
timeoutMs: this.options.providerConfig.headless.shutdownTimeoutMs,
|
|
463
|
-
});
|
|
464
|
-
}
|
|
465
|
-
if (this.drainTimer) clearTimeout(this.drainTimer);
|
|
466
|
-
this.drainTimer = undefined;
|
|
467
|
-
if (this.claimRenewTimer) clearInterval(this.claimRenewTimer);
|
|
468
|
-
this.claimRenewTimer = undefined;
|
|
469
|
-
if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
|
|
470
|
-
this.httpLivenessTimer = undefined;
|
|
471
|
-
if (this.tokenRenewTimer) clearTimeout(this.tokenRenewTimer);
|
|
472
|
-
this.tokenRenewTimer = undefined;
|
|
473
|
-
this.disarmBusyReconciler();
|
|
474
|
-
this.stopReasoningTail();
|
|
475
|
-
this.obligationCache.stop();
|
|
476
|
-
this.outbox.close();
|
|
477
|
-
this.sessionOutbox.close();
|
|
478
|
-
this.proxy?.stop();
|
|
479
|
-
this.control?.stop();
|
|
480
|
-
await this.bus.close();
|
|
481
|
-
}
|
|
482
|
-
|
|
483
|
-
// Start the local MCP proxy the agent connects to (Stage 2, #215). Forwards tool calls to the
|
|
484
|
-
// relay with the runner's LIVE token, buffers bufferable writes durably during a relay outage,
|
|
485
|
-
// and narrows the tool list to this agent's workspace context. Best-effort: if it can't bind,
|
|
486
|
-
// we fall back to a direct relay MCP connection (the agent env still works, no resilience).
|
|
487
|
-
private startMcpProxy(): void {
|
|
488
|
-
if (!this.mcpProxyEnabled) return;
|
|
489
|
-
try {
|
|
490
|
-
this.proxy = new RelayMcpProxy({
|
|
491
|
-
relayMcpEndpoint: relayMcpEndpoint(this.options.relayUrl),
|
|
492
|
-
getToken: () => this.currentToken,
|
|
493
|
-
authSecret: this.mcpProxySecret,
|
|
494
|
-
enqueueBuffered: (call) => {
|
|
495
|
-
this.outbox.enqueue({
|
|
496
|
-
kind: "mcp-tool-call",
|
|
497
|
-
payload: { tool: call.tool, arguments: call.arguments },
|
|
498
|
-
idempotencyKey: call.idempotencyKey,
|
|
499
|
-
});
|
|
500
|
-
},
|
|
501
|
-
initialContext: { isolatedWorktree: this.ownsIsolatedWorktree() },
|
|
502
|
-
});
|
|
503
|
-
this.mcpProxyEndpoint = this.proxy.start().url;
|
|
504
|
-
logger.info("mcp-proxy", `runner MCP proxy listening at ${this.mcpProxyEndpoint} (worktree=${this.ownsIsolatedWorktree()})`);
|
|
505
|
-
} catch (error) {
|
|
506
|
-
this.proxy = undefined;
|
|
507
|
-
this.mcpProxyEndpoint = undefined;
|
|
508
|
-
logger.warn("mcp-proxy", `failed to start MCP proxy; agent will connect to the relay directly: ${errMessage(error)}`);
|
|
509
|
-
}
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
private ownsIsolatedWorktree(): boolean {
|
|
513
|
-
const mode = this.options.workspace?.requestedMode ?? this.options.workspace?.mode ?? process.env.AGENT_RELAY_WORKSPACE_MODE;
|
|
514
|
-
return mode === "isolated";
|
|
515
|
-
}
|
|
516
|
-
|
|
517
|
-
private async spawnProvider(): Promise<ManagedProcess> {
|
|
518
|
-
this.providerSessionId = crypto.randomUUID();
|
|
519
|
-
this.lastTranscriptPath = undefined;
|
|
520
|
-
const includeProviderGlobals = profileUsesHostProviderGlobals(this.options);
|
|
521
|
-
const env = {
|
|
522
|
-
...process.env as Record<string, string>,
|
|
523
|
-
...(includeProviderGlobals ? this.options.providerConfig.env : {}),
|
|
524
|
-
AGENT_RELAY_RUNNER_PORT: String(this.control!.port),
|
|
525
|
-
AGENT_RELAY_RUNNER_ID: this.options.runnerId,
|
|
526
|
-
AGENT_RELAY_PROVIDER_SESSION_ID: this.providerSessionId,
|
|
527
|
-
AGENT_RELAY_ID: this.agentId,
|
|
528
|
-
// The repo-name project id the runner uses for insight observations. Exposed so
|
|
529
|
-
// agent-side signals (e.g. /introspect) tag the SAME project and aggregate together,
|
|
530
|
-
// instead of the agent's worktree cwd splitting one repo into many "projects".
|
|
531
|
-
AGENT_RELAY_PROJECT: this.insightProject(),
|
|
532
|
-
...(this.scratch ? { AGENT_RELAY_SCRATCH_DIR: this.scratch.tmpDir } : {}),
|
|
533
|
-
AGENT_RELAY_URL: this.options.relayUrl,
|
|
534
|
-
AGENT_RELAY_APPROVAL: this.options.approvalMode,
|
|
535
|
-
...(this.currentToken ? { AGENT_RELAY_TOKEN: this.currentToken } : {}),
|
|
536
|
-
// Dedicated, un-clobberable credential for the injected MCP endpoint. A rig's
|
|
537
|
-
// settings.json `env.AGENT_RELAY_TOKEN` would override the scoped token above at
|
|
538
|
-
// MCP-parse time → server-actor auth, no identity (#233). The MCP config references
|
|
539
|
-
// ${AGENT_RELAY_SESSION_TOKEN}, which rigs never set. See runner/src/relay-mcp.ts.
|
|
540
|
-
//
|
|
541
|
-
// Stage 2 (#215): when the proxy is active the agent connects to the LOCAL proxy, so this
|
|
542
|
-
// holds the per-session PROXY SECRET (not the relay token). The proxy injects the live
|
|
543
|
-
// relay token itself — the agent never holds it, and token rotation is invisible. With the
|
|
544
|
-
// proxy disabled this stays the scoped relay token (Stage-1 direct connection).
|
|
545
|
-
...(this.proxy
|
|
546
|
-
? { [RELAY_MCP_TOKEN_ENV]: this.mcpProxySecret }
|
|
547
|
-
: (this.currentToken ? { [RELAY_MCP_TOKEN_ENV]: this.currentToken } : {})),
|
|
548
|
-
...(this.currentTokenJti ? { AGENT_RELAY_TOKEN_JTI: this.currentTokenJti } : {}),
|
|
549
|
-
...(this.currentTokenProfileId ? { AGENT_RELAY_TOKEN_PROFILE: this.currentTokenProfileId } : {}),
|
|
550
|
-
...(this.currentTokenExpiresAt ? { AGENT_RELAY_TOKEN_EXPIRES_AT: String(this.currentTokenExpiresAt) } : {}),
|
|
551
|
-
};
|
|
552
|
-
const config: RunnerSpawnConfig = {
|
|
553
|
-
provider: this.options.provider,
|
|
554
|
-
model: this.options.model,
|
|
555
|
-
effort: this.options.effort,
|
|
556
|
-
runnerId: this.options.runnerId,
|
|
557
|
-
instanceId: this.options.instanceId,
|
|
558
|
-
agentId: this.agentId,
|
|
559
|
-
relayUrl: this.options.relayUrl,
|
|
560
|
-
cwd: this.options.cwd,
|
|
561
|
-
headless: this.options.headless,
|
|
562
|
-
approvalMode: this.options.approvalMode,
|
|
563
|
-
...(this.options.label ? { label: this.options.label } : {}),
|
|
564
|
-
...(this.options.rig ? { rig: this.options.rig } : {}),
|
|
565
|
-
...(this.options.profile ? { profile: this.options.profile } : {}),
|
|
566
|
-
...(this.options.agentProfile ? { agentProfile: this.options.agentProfile } : {}),
|
|
567
|
-
...(this.options.prompt ? { prompt: this.options.prompt } : {}),
|
|
568
|
-
...(this.options.systemPromptAppend ? { systemPromptAppend: this.options.systemPromptAppend } : {}),
|
|
569
|
-
...(this.options.tmuxSession ? { tmuxSession: this.options.tmuxSession } : {}),
|
|
570
|
-
providerArgs: this.options.providerArgs,
|
|
571
|
-
providerConfig: this.options.providerConfig,
|
|
572
|
-
env,
|
|
573
|
-
controlPort: this.control!.port,
|
|
574
|
-
// Stage 2 (#215): the MCP endpoint the agent's client should target — the runner-local
|
|
575
|
-
// proxy when active, undefined when disabled (adapters fall back to the direct relay URL).
|
|
576
|
-
...(this.mcpProxyEndpoint ? { relayMcpEndpoint: this.mcpProxyEndpoint } : {}),
|
|
577
|
-
monitor: {
|
|
578
|
-
deliver: (messages) => this.control!.deliverToMonitor(messages),
|
|
579
|
-
},
|
|
580
|
-
};
|
|
581
|
-
const launchSeededPrompt = this.options.adapter.seedsInitialPromptAtLaunch ? this.options.prompt?.trim() : ""; if (launchSeededPrompt) this.recordInjectedPrompt(launchSeededPrompt);
|
|
582
|
-
return this.options.adapter.spawn(config);
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
private async terminalAttachSpec(): Promise<TerminalAttachSpec> {
|
|
586
|
-
if (!this.process) throw new Error("provider process is unavailable");
|
|
587
|
-
if (!this.options.adapter.terminalAttachSpec) throw new Error("provider does not support terminal attach");
|
|
588
|
-
return this.options.adapter.terminalAttachSpec(this.process);
|
|
589
|
-
}
|
|
590
|
-
|
|
591
|
-
private writeRunnerInfoFile(): void {
|
|
592
|
-
const file = process.env.AGENT_RELAY_RUNNER_INFO_FILE;
|
|
593
|
-
if (!file || !this.control) return;
|
|
594
|
-
try {
|
|
595
|
-
mkdirSync(dirname(file), { recursive: true });
|
|
596
|
-
writeFileSync(file, JSON.stringify({
|
|
597
|
-
agentId: this.agentId,
|
|
598
|
-
runnerId: this.options.runnerId,
|
|
599
|
-
provider: this.options.provider,
|
|
600
|
-
controlUrl: this.control.url,
|
|
601
|
-
pid: process.pid,
|
|
602
|
-
...(this.providerTerminalSession() ? { tmuxSession: this.providerTerminalSession() } : {}),
|
|
603
|
-
...(this.providerTerminalSocket() ? { tmuxSocket: this.providerTerminalSocket() } : {}),
|
|
604
|
-
startedAt: this.options.startedAt,
|
|
605
|
-
}, null, 2) + "\n", { mode: 0o600 });
|
|
606
|
-
} catch (error) {
|
|
607
|
-
logger.error("runner", `failed to write runner info file: ${error}`);
|
|
608
|
-
}
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
private enqueueMessage(message: Message): void {
|
|
612
|
-
if (!this.matchesMessage(message)) return;
|
|
613
|
-
if (message.kind === "session") return;
|
|
614
|
-
this.pendingMessages.set(message.id, message);
|
|
615
|
-
this.scheduleDrain();
|
|
616
|
-
}
|
|
617
|
-
|
|
618
|
-
private async bootstrapUnreadMessages(): Promise<void> {
|
|
619
|
-
try {
|
|
620
|
-
const messages = await this.http.pollMessages({ for: this.agentId, unread: true, limit: 100 });
|
|
621
|
-
for (const message of messages) this.enqueueMessage(message);
|
|
622
|
-
} catch (error) {
|
|
623
|
-
logger.error("runner", `inbox bootstrap failed: ${error}`);
|
|
624
|
-
}
|
|
625
|
-
}
|
|
626
|
-
|
|
627
|
-
private async deliverInitialPrompt(): Promise<void> {
|
|
628
|
-
const prompt = this.options.prompt?.trim();
|
|
629
|
-
if (!prompt || this.options.adapter.seedsInitialPromptAtLaunch) return;
|
|
630
|
-
await this.attemptInitialPromptDelivery(prompt, INITIAL_PROMPT_FIRST_READY_TIMEOUT_MS);
|
|
631
|
-
}
|
|
632
|
-
|
|
633
|
-
// Deliver the spawn-time first prompt, surviving a cold-start TUI that isn't input-ready yet
|
|
634
|
-
// (#329). Timeout stashes it for ready-signal retry and surfaces the stall on the timeline.
|
|
635
|
-
private async attemptInitialPromptDelivery(prompt: string, readyTimeoutMs: number): Promise<void> {
|
|
636
|
-
if (this.deliveringInitialPrompt || this.stopped || !this.process || !this.options.adapter.deliverInitialPrompt) return;
|
|
637
|
-
this.deliveringInitialPrompt = true;
|
|
638
|
-
const attempt = (this.initialPromptAttempts += 1);
|
|
639
|
-
try {
|
|
640
|
-
this.recordInjectedPrompt(prompt); await this.options.adapter.deliverInitialPrompt(this.process, prompt, { readyTimeoutMs });
|
|
641
|
-
this.pendingInitialPrompt = undefined;
|
|
642
|
-
} catch (error) {
|
|
643
|
-
const giveUp = attempt >= MAX_INITIAL_PROMPT_ATTEMPTS;
|
|
644
|
-
this.pendingInitialPrompt = giveUp ? undefined : prompt;
|
|
645
|
-
logger.error("runner", `initial prompt attempt ${attempt} failed${giveUp ? " (giving up)" : "; will retry on next ready signal"}: ${errMessage(error)}`);
|
|
646
|
-
this.publishRunnerTimelineEvent({ status: "prompt_failed", timestamp: Date.now(), title: "Initial prompt not delivered", body: errMessage(error), metadata: { attempts: attempt, terminal: giveUp } });
|
|
647
|
-
} finally {
|
|
648
|
-
this.deliveringInitialPrompt = false;
|
|
649
|
-
}
|
|
650
|
-
}
|
|
651
|
-
|
|
652
|
-
private scheduleDrain(delayMs = 0): void {
|
|
653
|
-
if (this.stopped || this.drainTimer) return;
|
|
654
|
-
this.drainTimer = setTimeout(() => {
|
|
655
|
-
this.drainTimer = undefined;
|
|
656
|
-
void this.drainMessages();
|
|
657
|
-
}, delayMs);
|
|
658
|
-
}
|
|
659
|
-
|
|
660
|
-
private async drainMessages(): Promise<void> {
|
|
661
|
-
if (this.stopped || this.delivering || this.pendingMessages.size === 0) return;
|
|
662
|
-
if (!this.process) {
|
|
663
|
-
this.scheduleDrain(500);
|
|
664
|
-
return;
|
|
665
|
-
}
|
|
666
|
-
this.delivering = true;
|
|
667
|
-
const messages = [...this.pendingMessages.values()].sort((a, b) => a.id - b.id);
|
|
668
|
-
this.pendingMessages.clear();
|
|
669
|
-
const deliverable: Message[] = [];
|
|
670
|
-
const providerAlreadyBusy = this.claims.reasons().includes("provider-turn");
|
|
671
|
-
for (const message of messages) {
|
|
672
|
-
let toDeliver = message;
|
|
673
|
-
if (message.claimable) {
|
|
674
|
-
const claimed = await this.http.claimMessageResult(message.id, this.agentId).catch(() => ({ ok: false, claimExpiresAt: undefined }));
|
|
675
|
-
if (!claimed.ok) continue;
|
|
676
|
-
this.claims.startClaim("message", String(message.id), claimed.claimExpiresAt);
|
|
677
|
-
const taskId = taskIdFromMessage(message);
|
|
678
|
-
if (taskId) {
|
|
679
|
-
this.activeTaskClaims.set(message.id, { messageId: message.id, taskId, observedProviderBusy: providerAlreadyBusy });
|
|
680
|
-
this.claims.startClaim("task", String(taskId), claimed.claimExpiresAt);
|
|
681
|
-
await this.updateTaskStatus(taskId, {
|
|
682
|
-
status: "in_progress",
|
|
683
|
-
agentId: this.agentId,
|
|
684
|
-
metadata: { messageId: message.id, completedBy: "runner" },
|
|
685
|
-
}).catch((error) => logger.error("task", `task ${taskId} in_progress update failed: ${error}`));
|
|
686
|
-
// Runner owns claim + status here; drop the server's self-claim instruction
|
|
687
|
-
// so the agent doesn't improvise a stray claim send (see stripRunnerClaimedGuidance).
|
|
688
|
-
toDeliver = { ...message, body: stripRunnerClaimedGuidance(message.body) };
|
|
689
|
-
}
|
|
690
|
-
}
|
|
691
|
-
deliverable.push(toDeliver);
|
|
692
|
-
}
|
|
693
|
-
if (deliverable.length === 0) {
|
|
694
|
-
this.delivering = false;
|
|
695
|
-
this.publishStatus();
|
|
696
|
-
return;
|
|
697
|
-
}
|
|
698
|
-
this.publishStatus();
|
|
699
|
-
let failed = false;
|
|
700
|
-
try {
|
|
701
|
-
const prepared = await messagesWithCachedAttachments(deliverable, this.http, {
|
|
702
|
-
agentId: this.agentId,
|
|
703
|
-
onError: (message) => logger.error("runner", message),
|
|
704
|
-
});
|
|
705
|
-
await this.options.adapter.deliver(this.process, prepared);
|
|
706
|
-
for (const message of deliverable) {
|
|
707
|
-
await this.http.markRead(message.id, this.agentId).catch(() => {});
|
|
708
|
-
if (!taskIdFromMessage(message)) this.claims.finishClaim("message", String(message.id));
|
|
709
|
-
}
|
|
710
|
-
} catch (error) {
|
|
711
|
-
failed = true;
|
|
712
|
-
if (shouldLogDeliveryFailure(error)) logger.warn("delivery", `message delivery failed: ${error}`);
|
|
713
|
-
for (const message of deliverable) {
|
|
714
|
-
this.clearActiveClaim(message);
|
|
715
|
-
this.pendingMessages.set(message.id, message);
|
|
716
|
-
}
|
|
717
|
-
} finally {
|
|
718
|
-
this.delivering = false;
|
|
719
|
-
this.publishStatus();
|
|
720
|
-
if (this.pendingMessages.size > 0) this.scheduleDrain(failed ? 1000 : 0);
|
|
721
|
-
}
|
|
722
|
-
}
|
|
723
|
-
|
|
724
|
-
private async handleCommand(type: string, params: Record<string, unknown>, commandId: string, command?: Record<string, unknown>): Promise<void> {
|
|
725
|
-
const target = typeof command?.target === "string" ? command.target : this.agentId;
|
|
726
|
-
if (target !== this.agentId && target !== this.options.runnerId) return;
|
|
727
|
-
if (type !== "agent.shutdown" && type !== "agent.restart" && type !== "agent.reconnect" && type !== "agent.kill" && type !== "agent.compact" && type !== "agent.clearContext" && type !== "agent.injectContext" && type !== "agent.permissionDecision" && type !== "agent.interrupt" && type !== "prompt.inject") return;
|
|
728
|
-
|
|
729
|
-
const exitAfterCommand = type === "agent.shutdown" || type === "agent.kill";
|
|
730
|
-
if (exitAfterCommand) this.exitCommandInProgress = true;
|
|
731
|
-
this.claims.startClaim("command", commandId);
|
|
732
|
-
try {
|
|
733
|
-
await this.updateCommand(commandId, "accepted");
|
|
734
|
-
await this.updateCommand(commandId, "running");
|
|
735
|
-
// Pre-session-destroy seam (#183): for destructive transitions, run end-of-session
|
|
736
|
-
// work (Insights capture, #184) BEFORE the invasive operation, surfaced as a
|
|
737
|
-
// non-addressable "finalizing" state so the agent isn't mistaken for merely busy.
|
|
738
|
-
const destroyReason = boundaryReasonForCommand(type);
|
|
739
|
-
if (destroyReason) await this.runPreSessionDestroy(destroyReason);
|
|
740
|
-
// Move from the transient finalizing window to the executing teardown state (or drop
|
|
741
|
-
// it entirely for compact/clear, which complete promptly once capture is done).
|
|
742
|
-
if (exitAfterCommand) this.lifecycleAction = type === "agent.kill" ? "killing" : "shutting-down";
|
|
743
|
-
else if (type === "agent.restart") this.lifecycleAction = "restarting";
|
|
744
|
-
else this.lifecycleAction = undefined;
|
|
745
|
-
this.publishStatus();
|
|
746
|
-
let providerResult: Record<string, unknown> | void = undefined;
|
|
747
|
-
if (type === "agent.restart") await this.restartProvider();
|
|
748
|
-
else if (type === "agent.reconnect") this.publishStatus();
|
|
749
|
-
else if (type === "agent.compact") {
|
|
750
|
-
if (!this.options.adapter.compact || !this.process) throw new Error("provider does not support compact");
|
|
751
|
-
providerResult = await this.options.adapter.compact(this.process, {
|
|
752
|
-
instructions: typeof params.instructions === "string" ? params.instructions : undefined,
|
|
753
|
-
});
|
|
754
|
-
} else if (type === "agent.clearContext") {
|
|
755
|
-
if (!this.options.adapter.clearContext || !this.process) throw new Error("provider does not support clearContext");
|
|
756
|
-
providerResult = await this.options.adapter.clearContext(this.process);
|
|
757
|
-
} else if (type === "agent.interrupt") {
|
|
758
|
-
if (!this.options.adapter.interrupt || !this.process) throw new Error("provider does not support interrupt");
|
|
759
|
-
this.sessionLog("interrupt requested from dashboard");
|
|
760
|
-
providerResult = await this.options.adapter.interrupt(this.process);
|
|
761
|
-
this.scheduleInterruptReconcile();
|
|
762
|
-
} else if (type === "agent.injectContext") {
|
|
763
|
-
if (!this.process) throw new Error("provider process is unavailable");
|
|
764
|
-
providerResult = await this.injectContext(params);
|
|
765
|
-
} else if (type === "agent.permissionDecision") {
|
|
766
|
-
providerResult = await this.respondToPermissionDecision(params);
|
|
767
|
-
} else if (type === "prompt.inject") {
|
|
768
|
-
providerResult = await this.injectPrompt(params);
|
|
769
|
-
} else await this.shutdownProvider(type === "agent.kill", commandTimeoutMs(params));
|
|
770
|
-
await this.updateCommand(commandId, "succeeded", {
|
|
771
|
-
action: type,
|
|
772
|
-
agentId: this.agentId,
|
|
773
|
-
runnerId: this.options.runnerId,
|
|
774
|
-
policyName: this.options.policyName,
|
|
775
|
-
spawnRequestId: this.options.spawnRequestId,
|
|
776
|
-
reason: typeof params.reason === "string" ? params.reason : undefined,
|
|
777
|
-
...(providerResult ? { providerResult } : {}),
|
|
778
|
-
});
|
|
779
|
-
} catch (error) {
|
|
780
|
-
await this.updateCommand(commandId, "failed", undefined, errMessage(error)).catch(() => {});
|
|
781
|
-
} finally {
|
|
782
|
-
this.claims.finishClaim("command", commandId);
|
|
783
|
-
if (exitAfterCommand) {
|
|
784
|
-
if (params.preserveRegistration === true) {
|
|
785
|
-
await this.http.setStatus(this.agentId, "offline", this.options.instanceId).catch(() => {});
|
|
786
|
-
} else {
|
|
787
|
-
await this.http.deleteAgent(this.agentId).catch(() => {});
|
|
788
|
-
}
|
|
789
|
-
if (this.options.exitProcessOnShutdown !== false) {
|
|
790
|
-
setTimeout(() => void this.stop().catch((error) => {
|
|
791
|
-
logger.error("lifecycle", `stop after command failed: ${error}`);
|
|
792
|
-
}).finally(() => process.exit(0)), 10);
|
|
793
|
-
}
|
|
794
|
-
} else if (!this.stopped) {
|
|
795
|
-
this.lifecycleAction = undefined;
|
|
796
|
-
this.publishStatus();
|
|
797
|
-
}
|
|
798
|
-
}
|
|
799
|
-
}
|
|
800
|
-
|
|
801
|
-
private async injectContext(params: Record<string, unknown>): Promise<Record<string, unknown>> {
|
|
802
|
-
const content = typeof params.content === "string" ? params.content.trim() : "";
|
|
803
|
-
if (!content) throw new Error("content required");
|
|
804
|
-
const memoryIds = Array.isArray(params.memoryIds) ? params.memoryIds.filter((id): id is string => typeof id === "string") : [];
|
|
805
|
-
await this.options.adapter.deliver(this.process!, [{
|
|
806
|
-
id: 0,
|
|
807
|
-
from: "system",
|
|
808
|
-
to: this.agentId,
|
|
809
|
-
kind: "system",
|
|
810
|
-
subject: "Agent Relay memory context",
|
|
811
|
-
body: content,
|
|
812
|
-
payload: {
|
|
813
|
-
memoryInjection: true,
|
|
814
|
-
reason: typeof params.reason === "string" ? params.reason : undefined,
|
|
815
|
-
memoryIds,
|
|
816
|
-
},
|
|
817
|
-
readBy: [],
|
|
818
|
-
createdAt: Date.now(),
|
|
819
|
-
}]);
|
|
820
|
-
return { memoryIds, injectedMemoryCount: memoryIds.length };
|
|
821
|
-
}
|
|
822
|
-
|
|
823
|
-
private async respondToPermissionDecision(params: Record<string, unknown>): Promise<Record<string, unknown>> {
|
|
824
|
-
const approvalId = typeof params.approvalId === "string" ? params.approvalId : "";
|
|
825
|
-
const decision = typeof params.decision === "string" ? params.decision : "";
|
|
826
|
-
if (!approvalId) throw new Error("approvalId required");
|
|
827
|
-
if (decision !== "approve" && decision !== "approve-session" && decision !== "deny" && decision !== "abort" && decision !== "answer") {
|
|
828
|
-
throw new Error("decision must be approve, approve-session, deny, abort, or answer");
|
|
829
|
-
}
|
|
830
|
-
const answersRaw = params.answers;
|
|
831
|
-
const answers = answersRaw && typeof answersRaw === "object" && !Array.isArray(answersRaw)
|
|
832
|
-
? Object.fromEntries(
|
|
833
|
-
Object.entries(answersRaw as Record<string, unknown>).filter(([, v]) => typeof v === "string") as [string, string][],
|
|
834
|
-
)
|
|
835
|
-
: undefined;
|
|
836
|
-
if (decision === "answer" && (!answers || Object.keys(answers).length === 0)) {
|
|
837
|
-
throw new Error("answers required for answer decision");
|
|
838
|
-
}
|
|
839
|
-
const input: ProviderPermissionDecisionInput = {
|
|
840
|
-
approvalId,
|
|
841
|
-
decision: decision as ProviderPermissionDecision,
|
|
842
|
-
...(typeof params.reason === "string" ? { reason: params.reason } : {}),
|
|
843
|
-
...(answers ? { answers } : {}),
|
|
844
|
-
};
|
|
845
|
-
if (this.control?.resolvePermissionDecision(input)) {
|
|
846
|
-
return { approvalId, decision, provider: "claude" };
|
|
847
|
-
}
|
|
848
|
-
if (!this.process) throw new Error("provider process is unavailable");
|
|
849
|
-
if (!this.options.adapter.respondToPermissionDecision) throw new Error("provider does not support permission decisions");
|
|
850
|
-
const result = await this.options.adapter.respondToPermissionDecision(this.process, input);
|
|
851
|
-
return { approvalId, decision, ...(result ? { providerResult: result } : {}) };
|
|
852
|
-
}
|
|
853
|
-
|
|
854
|
-
private async injectPrompt(params: Record<string, unknown>): Promise<Record<string, unknown>> {
|
|
855
|
-
const body = typeof params.body === "string" ? params.body : "";
|
|
856
|
-
if (!body) throw new Error("body required");
|
|
857
|
-
if (!this.process) throw new Error("provider process is unavailable");
|
|
858
|
-
if (!this.options.adapter.deliverInitialPrompt) throw new Error("provider does not support prompt injection");
|
|
859
|
-
const messageId = typeof params.messageId === "number" ? params.messageId : undefined;
|
|
860
|
-
const attachments = Array.isArray(params.attachments)
|
|
861
|
-
? params.attachments.filter((item): item is Record<string, unknown> => item !== null && typeof item === "object" && !Array.isArray(item))
|
|
862
|
-
: [];
|
|
863
|
-
if (messageId) this.pendingPromptMessageId = messageId;
|
|
864
|
-
const prompt = await this.promptInjectionText(body, messageId, attachments);
|
|
865
|
-
// Mark so the matching UserPromptSubmit echo isn't double-posted: a chat-box
|
|
866
|
-
// prompt already created its own session message shown in the dashboard.
|
|
867
|
-
this.recordInjectedPrompt(prompt.trim());
|
|
868
|
-
await this.options.adapter.deliverInitialPrompt(this.process, prompt);
|
|
869
|
-
return { injected: true, messageId, attachmentCount: attachments.length };
|
|
870
|
-
}
|
|
871
|
-
|
|
872
|
-
private async promptInjectionText(body: string, messageId: number | undefined, attachments: Record<string, unknown>[]): Promise<string> {
|
|
873
|
-
if (!attachments.length) return body;
|
|
874
|
-
const messages = await messagesWithCachedAttachments([{
|
|
875
|
-
id: messageId ?? 0,
|
|
876
|
-
from: "user",
|
|
877
|
-
to: this.agentId,
|
|
878
|
-
kind: "session",
|
|
879
|
-
body,
|
|
880
|
-
payload: { attachments },
|
|
881
|
-
readBy: [],
|
|
882
|
-
createdAt: Date.now(),
|
|
883
|
-
}], this.http, {
|
|
884
|
-
agentId: this.agentId,
|
|
885
|
-
onError: (message) => logger.warn("attachments", message),
|
|
886
|
-
});
|
|
887
|
-
const attachmentText = providerAttachmentText(messages[0]!);
|
|
888
|
-
return attachmentText ? `${body}\n\n${attachmentText}` : body;
|
|
889
|
-
}
|
|
890
|
-
|
|
891
|
-
private async restartProvider(): Promise<void> {
|
|
892
|
-
this.restartInProgress = true;
|
|
893
|
-
try {
|
|
894
|
-
if (this.process) {
|
|
895
|
-
await this.options.adapter.shutdown(this.process, {
|
|
896
|
-
graceful: true,
|
|
897
|
-
timeoutMs: this.options.providerConfig.headless.shutdownTimeoutMs,
|
|
898
|
-
});
|
|
899
|
-
}
|
|
900
|
-
this.claims.clearTerminalStatus();
|
|
901
|
-
this.claims.clearWorkKind("provider-turn");
|
|
902
|
-
this.claims.clearWorkKind("subagent");
|
|
903
|
-
if (this.stopped) return;
|
|
904
|
-
this.process = await this.spawnProvider();
|
|
905
|
-
this.processStartedAt = Date.now();
|
|
906
|
-
} finally {
|
|
907
|
-
this.restartInProgress = false;
|
|
908
|
-
}
|
|
909
|
-
}
|
|
910
|
-
|
|
911
|
-
private shouldRestartUnexpectedProviderExit(status: SemanticStatus): boolean {
|
|
912
|
-
return runnerShouldRestartUnexpectedProviderExit(status, {
|
|
913
|
-
exitCommandInProgress: this.exitCommandInProgress,
|
|
914
|
-
stopped: this.stopped,
|
|
915
|
-
restartInProgress: this.restartInProgress,
|
|
916
|
-
provider: this.options.provider,
|
|
917
|
-
headless: this.options.headless,
|
|
918
|
-
hasTerminalSession: typeof this.process?.meta?.tmuxSession === "string"
|
|
919
|
-
|| (this.options.adapter.supportsUnexpectedExitRestart?.() ?? false),
|
|
920
|
-
});
|
|
921
|
-
}
|
|
922
|
-
|
|
923
|
-
private async restartUnexpectedProviderExit(status: SemanticStatus): Promise<void> {
|
|
924
|
-
if (this.restartPending) return;
|
|
925
|
-
this.restartPending = true;
|
|
926
|
-
try {
|
|
927
|
-
// Best-effort Insights capture for the segment that just ended in a crash (#183). This
|
|
928
|
-
// path has no controlled teardown, so without it crashed sessions silently drop their
|
|
929
|
-
// context-ratio datapoint. The process handle is still set (cleared later), so the
|
|
930
|
-
// Claude transcript is readable; the runner stays alive here (restart or offline), so the
|
|
931
|
-
// durable outbox drains normally — no flush needed.
|
|
932
|
-
await Promise.race([
|
|
933
|
-
this.captureContextRatio("crash"),
|
|
934
|
-
new Promise<void>((resolve) => setTimeout(resolve, PRE_DESTROY_TIMEOUT_MS)),
|
|
935
|
-
]).catch((error) => this.sessionLog(`insights: crash capture failed: ${errMessage(error)}`));
|
|
936
|
-
|
|
937
|
-
const now = Date.now();
|
|
938
|
-
const runtimeMs = this.processStartedAt ? now - this.processStartedAt : Number.POSITIVE_INFINITY;
|
|
939
|
-
const recent = this.unexpectedExitTimes.filter((time) => now - time <= UNEXPECTED_EXIT_WINDOW_MS);
|
|
940
|
-
recent.push(now);
|
|
941
|
-
this.unexpectedExitTimes.splice(0, this.unexpectedExitTimes.length, ...recent);
|
|
942
|
-
const diagnostics = this.providerExitDiagnostics(status, runtimeMs);
|
|
943
|
-
|
|
944
|
-
this.publishRunnerTimelineEvent({
|
|
945
|
-
status: "provider.exit_detected",
|
|
946
|
-
id: `provider-exit-${this.providerSessionId}-${now}`,
|
|
947
|
-
timestamp: now,
|
|
948
|
-
title: "Provider exited",
|
|
949
|
-
body: `${this.options.provider} reported ${status} after ${Math.round(runtimeMs / 1000)}s`,
|
|
950
|
-
icon: "ti-plug-off",
|
|
951
|
-
metadata: {
|
|
952
|
-
eventType: "provider.exit_detected",
|
|
953
|
-
...diagnostics,
|
|
954
|
-
},
|
|
955
|
-
});
|
|
956
|
-
|
|
957
|
-
if (this.shouldStopUnexpectedProviderExit(diagnostics)) {
|
|
958
|
-
const hasResumeId = typeof diagnostics.claudeResumeId === "string" && diagnostics.claudeResumeId.length > 0;
|
|
959
|
-
logger.warn("lifecycle", `${this.options.provider} exited; leaving agent offline for manual recovery`);
|
|
960
|
-
this.publishRunnerTimelineEvent({
|
|
961
|
-
status: "provider.restart_decision",
|
|
962
|
-
id: `provider-restart-decision-${this.providerSessionId}-${now}`,
|
|
963
|
-
timestamp: Date.now(),
|
|
964
|
-
title: "Provider restart skipped",
|
|
965
|
-
body: hasResumeId
|
|
966
|
-
? "Claude exited; runner will not auto-resume. Resume id captured for manual recovery."
|
|
967
|
-
: "Claude exited; runner will not restart automatically.",
|
|
968
|
-
icon: "ti-player-stop",
|
|
969
|
-
metadata: {
|
|
970
|
-
eventType: "provider.restart_decision",
|
|
971
|
-
decision: "stop-surface",
|
|
972
|
-
reason: hasResumeId ? "claude-exit-manual-resume-available" : "claude-exit-manual-intervention-required",
|
|
973
|
-
...diagnostics,
|
|
974
|
-
},
|
|
975
|
-
});
|
|
976
|
-
this.process = undefined;
|
|
977
|
-
this.setProviderStatus({
|
|
978
|
-
status,
|
|
979
|
-
reason: "provider-turn",
|
|
980
|
-
id: `provider-exit-${this.providerSessionId}`,
|
|
981
|
-
clear: ["provider-turn", "subagent"],
|
|
982
|
-
});
|
|
983
|
-
return;
|
|
984
|
-
}
|
|
985
|
-
|
|
986
|
-
if (runtimeMs < RAPID_EXIT_MS && recent.length > MAX_RAPID_UNEXPECTED_EXITS) {
|
|
987
|
-
logger.error("lifecycle", `provider session exited ${recent.length} times within ${Math.round(UNEXPECTED_EXIT_WINDOW_MS / 1000)}s; giving up`);
|
|
988
|
-
this.publishRunnerTimelineEvent({
|
|
989
|
-
status: "provider.restart_decision",
|
|
990
|
-
id: `provider-restart-decision-${this.providerSessionId}-${now}`,
|
|
991
|
-
timestamp: Date.now(),
|
|
992
|
-
title: "Provider restart skipped",
|
|
993
|
-
body: `rapid unexpected exits exceeded ${MAX_RAPID_UNEXPECTED_EXITS}`,
|
|
994
|
-
icon: "ti-alert-triangle",
|
|
995
|
-
metadata: {
|
|
996
|
-
eventType: "provider.restart_decision",
|
|
997
|
-
decision: "give-up",
|
|
998
|
-
reason: "rapid-unexpected-provider-exits",
|
|
999
|
-
rapidExitCount: recent.length,
|
|
1000
|
-
rapidExitWindowMs: UNEXPECTED_EXIT_WINDOW_MS,
|
|
1001
|
-
maxRapidUnexpectedExits: MAX_RAPID_UNEXPECTED_EXITS,
|
|
1002
|
-
...diagnostics,
|
|
1003
|
-
},
|
|
1004
|
-
});
|
|
1005
|
-
this.setProviderStatus(status);
|
|
1006
|
-
this.options.onProviderExit?.(0);
|
|
1007
|
-
return;
|
|
1008
|
-
}
|
|
1009
|
-
|
|
1010
|
-
const delayMs = Math.min(10_000, Math.max(500, 500 * recent.length));
|
|
1011
|
-
logger.warn("lifecycle", `provider session exited unexpectedly after ${Math.round(runtimeMs / 1000)}s; restarting in ${delayMs}ms`);
|
|
1012
|
-
this.publishRunnerTimelineEvent({
|
|
1013
|
-
status: "provider.restart_decision",
|
|
1014
|
-
id: `provider-restart-decision-${this.providerSessionId}-${now}`,
|
|
1015
|
-
timestamp: Date.now(),
|
|
1016
|
-
title: "Provider restart scheduled",
|
|
1017
|
-
body: `runner will start a fresh ${this.options.provider} provider in ${delayMs}ms`,
|
|
1018
|
-
icon: "ti-refresh",
|
|
1019
|
-
metadata: {
|
|
1020
|
-
eventType: "provider.restart_decision",
|
|
1021
|
-
decision: "restart-fresh",
|
|
1022
|
-
reason: "unexpected-headless-terminal-exit",
|
|
1023
|
-
delayMs,
|
|
1024
|
-
rapidExitCount: recent.length,
|
|
1025
|
-
rapidExitWindowMs: UNEXPECTED_EXIT_WINDOW_MS,
|
|
1026
|
-
...diagnostics,
|
|
1027
|
-
},
|
|
1028
|
-
});
|
|
1029
|
-
await Bun.sleep(delayMs);
|
|
1030
|
-
if (this.stopped || this.exitCommandInProgress) return;
|
|
1031
|
-
try {
|
|
1032
|
-
await this.restartProvider();
|
|
1033
|
-
this.publishStatus();
|
|
1034
|
-
this.scheduleDrain();
|
|
1035
|
-
} catch (error) {
|
|
1036
|
-
logger.error("lifecycle", `provider restart after unexpected exit failed: ${error}`);
|
|
1037
|
-
this.setProviderStatus("error");
|
|
1038
|
-
this.options.onProviderExit?.(1);
|
|
1039
|
-
}
|
|
1040
|
-
} finally {
|
|
1041
|
-
this.restartPending = false;
|
|
1042
|
-
}
|
|
1043
|
-
}
|
|
1044
|
-
|
|
1045
|
-
private shouldStopUnexpectedProviderExit(diagnostics: Record<string, unknown>): boolean {
|
|
1046
|
-
return this.options.provider === "claude" && diagnostics.exitCommandInProgress !== true;
|
|
1047
|
-
}
|
|
1048
|
-
|
|
1049
|
-
private async shutdownProvider(hard: boolean, timeoutMs = this.options.providerConfig.headless.shutdownTimeoutMs): Promise<void> {
|
|
1050
|
-
this.lifecycleAction = hard ? "killing" : "shutting-down";
|
|
1051
|
-
this.publishStatus();
|
|
1052
|
-
await this.bus.statusAsync({ agentStatus: "idle", ready: false, meta: { lifecycleAction: this.lifecycleAction, lifecycleActionAt: Date.now() } });
|
|
1053
|
-
if (this.process) {
|
|
1054
|
-
await this.options.adapter.shutdown(this.process, {
|
|
1055
|
-
graceful: !hard,
|
|
1056
|
-
timeoutMs,
|
|
1057
|
-
});
|
|
1058
|
-
}
|
|
1059
|
-
this.claims.setTerminalStatus("offline");
|
|
1060
|
-
this.publishStatus();
|
|
1061
|
-
this.stopped = true;
|
|
1062
|
-
}
|
|
1063
|
-
|
|
1064
|
-
private publishRunnerTimelineEvent(event: RunnerTimelineEvent): void {
|
|
1065
|
-
this.pendingTimelineEvent = {
|
|
1066
|
-
...event,
|
|
1067
|
-
metadata: {
|
|
1068
|
-
source: "runner",
|
|
1069
|
-
provider: this.options.provider,
|
|
1070
|
-
runnerId: this.options.runnerId,
|
|
1071
|
-
agentId: this.agentId,
|
|
1072
|
-
policyName: this.options.policyName ?? null,
|
|
1073
|
-
spawnRequestId: this.options.spawnRequestId ?? null,
|
|
1074
|
-
label: this.options.label ?? null,
|
|
1075
|
-
providerSessionId: this.providerSessionId,
|
|
1076
|
-
...(event.metadata ?? {}),
|
|
1077
|
-
},
|
|
1078
|
-
};
|
|
1079
|
-
this.publishStatus();
|
|
1080
|
-
}
|
|
1081
|
-
|
|
1082
|
-
private providerExitDiagnostics(status: SemanticStatus, runtimeMs: number): Record<string, unknown> {
|
|
1083
|
-
const tmuxSession = typeof this.process?.meta?.tmuxSession === "string" ? this.process.meta.tmuxSession : undefined;
|
|
1084
|
-
const tmuxSocket = typeof this.process?.meta?.tmuxSocket === "string" ? this.process.meta.tmuxSocket : undefined;
|
|
1085
|
-
const exitSource = tmuxSession ? "tmux-session-ended" : this.process?.process ? "process-exit" : "provider-status";
|
|
1086
|
-
const logFile = typeof process.env.AGENT_RELAY_LOG_FILE === "string" ? process.env.AGENT_RELAY_LOG_FILE : undefined;
|
|
1087
|
-
const claudeResumeId = this.options.provider === "claude" && logFile ? latestClaudeResumeIdFromLogFile(logFile) : undefined;
|
|
1088
|
-
return {
|
|
1089
|
-
status,
|
|
1090
|
-
runtimeMs: Number.isFinite(runtimeMs) ? runtimeMs : null,
|
|
1091
|
-
exitSource,
|
|
1092
|
-
exitCommandInProgress: this.exitCommandInProgress,
|
|
1093
|
-
stopped: this.stopped,
|
|
1094
|
-
restartInProgress: this.restartInProgress,
|
|
1095
|
-
restartPending: this.restartPending,
|
|
1096
|
-
headless: this.options.headless,
|
|
1097
|
-
hasTerminalSession: Boolean(tmuxSession),
|
|
1098
|
-
tmuxSession: tmuxSession ?? null,
|
|
1099
|
-
tmuxSocket: tmuxSocket ?? null,
|
|
1100
|
-
claudeResumeId: claudeResumeId ?? null,
|
|
1101
|
-
};
|
|
1102
|
-
}
|
|
1103
|
-
|
|
1104
|
-
private async updateCommand(commandId: string, status: string, result?: Record<string, unknown>, error?: string): Promise<void> {
|
|
1105
|
-
await this.bus.updateCommand(commandId, { status, ...(result ? { result } : {}), ...(error ? { error } : {}) });
|
|
1106
|
-
}
|
|
1107
|
-
|
|
1108
|
-
private handleBusError(code: string, message: string): void {
|
|
1109
|
-
const action = runnerBusErrorAction(code, this.stopped);
|
|
1110
|
-
if (action === "ignore") return;
|
|
1111
|
-
logger.error("bus", `bus error ${code}: ${message}`);
|
|
1112
|
-
if (action === "stop") {
|
|
1113
|
-
void this.stop().catch((error) => {
|
|
1114
|
-
logger.error("bus", `stop after bus error failed: ${error}`);
|
|
1115
|
-
}).finally(() => process.exit(0));
|
|
1116
|
-
}
|
|
1117
|
-
}
|
|
1118
|
-
|
|
1119
|
-
private setProviderStatus(update: ProviderStatusUpdate): void {
|
|
1120
|
-
const status = typeof update === "string" ? update : update.status;
|
|
1121
|
-
const providerSessionId = typeof update === "string" ? undefined : update.providerSessionId;
|
|
1122
|
-
if (providerSessionId && providerSessionId !== this.providerSessionId) return;
|
|
1123
|
-
const reason = typeof update === "string" ? "provider-turn" : update.reason ?? "provider-turn";
|
|
1124
|
-
const id = typeof update === "string" ? reason : update.id ?? reason;
|
|
1125
|
-
if (typeof update !== "string" && update.timeline) {
|
|
1126
|
-
this.pendingTimelineEvent = {
|
|
1127
|
-
status: update.timeline.status,
|
|
1128
|
-
...(update.timeline.id ? { id: update.timeline.id } : {}),
|
|
1129
|
-
timestamp: update.timeline.timestamp ?? Date.now(),
|
|
1130
|
-
...(update.timeline.title ? { title: update.timeline.title } : {}),
|
|
1131
|
-
...(update.timeline.body ? { body: update.timeline.body } : {}),
|
|
1132
|
-
...(update.timeline.icon ? { icon: update.timeline.icon } : {}),
|
|
1133
|
-
...(update.timeline.metadata ? { metadata: update.timeline.metadata } : {}),
|
|
1134
|
-
};
|
|
1135
|
-
}
|
|
1136
|
-
if (typeof update !== "string" && update.providerState) {
|
|
1137
|
-
const ps = update.providerState as { state?: unknown; reason?: unknown };
|
|
1138
|
-
this.providerBlocked = ps.state === "blocked";
|
|
1139
|
-
// A rate-limit hold persists across the idle the turn ends on; any other
|
|
1140
|
-
// providerState supersedes it (clears a stale hold).
|
|
1141
|
-
if (ps.state === "blocked" && ps.reason === "rate_limit") {
|
|
1142
|
-
const fresh = !this.rateLimitHold;
|
|
1143
|
-
this.rateLimitHold = update.providerState;
|
|
1144
|
-
if (fresh) this.publishRateLimitNotice(update.providerState);
|
|
1145
|
-
} else {
|
|
1146
|
-
this.rateLimitHold = undefined;
|
|
1147
|
-
}
|
|
1148
|
-
} else if (status === "idle") {
|
|
1149
|
-
this.providerBlocked = false;
|
|
1150
|
-
}
|
|
1151
|
-
// Forward progress (a real turn) lifts the hold so the blocked badge clears.
|
|
1152
|
-
if (status === "busy") this.rateLimitHold = undefined;
|
|
1153
|
-
if (typeof update !== "string" && status === "error") {
|
|
1154
|
-
const terminalReason = typeof update.metadata?.terminalFailureReason === "string"
|
|
1155
|
-
? update.metadata.terminalFailureReason
|
|
1156
|
-
: typeof update.providerState?.reason === "string"
|
|
1157
|
-
? update.providerState.reason
|
|
1158
|
-
: "provider-error";
|
|
1159
|
-
const terminalMessage = typeof update.metadata?.terminalFailureMessage === "string"
|
|
1160
|
-
? update.metadata.terminalFailureMessage
|
|
1161
|
-
: typeof update.providerState?.message === "string"
|
|
1162
|
-
? update.providerState.message
|
|
1163
|
-
: "Provider reported an unrecoverable error.";
|
|
1164
|
-
this.terminalFailure = {
|
|
1165
|
-
reason: terminalReason,
|
|
1166
|
-
message: terminalMessage,
|
|
1167
|
-
...(update.providerState ? { providerState: update.providerState } : {}),
|
|
1168
|
-
};
|
|
1169
|
-
} else if (status === "idle" || status === "busy") {
|
|
1170
|
-
this.terminalFailure = undefined;
|
|
1171
|
-
}
|
|
1172
|
-
// Compaction lifecycle (Claude PreCompact→`compacting` busy / PostCompact→`compacted`
|
|
1173
|
-
// idle). A `compacted` idle for a turn that predated compaction is a hook artifact — the
|
|
1174
|
-
// turn resumes — so it must NOT end the turn. Discriminate on whether a turn was running,
|
|
1175
|
-
// captured BEFORE the busy logic below mints a currentTurnId (it does for /compact at idle).
|
|
1176
|
-
const timelineStatus = typeof update !== "string" ? update.timeline?.status : undefined;
|
|
1177
|
-
if (timelineStatus === "compacting") {
|
|
1178
|
-
this.compactionMidTurn = this.currentTurnId !== undefined;
|
|
1179
|
-
} else if (timelineStatus === "compacted") {
|
|
1180
|
-
this.publishCompactionNotice();
|
|
1181
|
-
if (this.compactionMidTurn) {
|
|
1182
|
-
// Keep the turn (and its live reasoning tail) alive; the genuine Stop hook ends it.
|
|
1183
|
-
// pendingTimelineEvent (the marker) was set above and is consumed by publishStatus.
|
|
1184
|
-
this.sessionLog(`compaction completed mid-turn (turn ${this.currentTurnId ?? "?"}) — staying busy`);
|
|
1185
|
-
this.publishStatus();
|
|
1186
|
-
return;
|
|
1187
|
-
}
|
|
1188
|
-
}
|
|
1189
|
-
if (status === "busy" && reason === "provider-turn") {
|
|
1190
|
-
if (!this.currentTurnId) {
|
|
1191
|
-
this.currentTurnId = typeof update !== "string" && update.id ? update.id : crypto.randomUUID();
|
|
1192
|
-
this.currentTurnStartedAt = Date.now();
|
|
1193
|
-
this.compactionMidTurn = false;
|
|
1194
|
-
this.sessionLog(`turn started (turn ${this.currentTurnId})`);
|
|
1195
|
-
}
|
|
1196
|
-
this.armBusyReconciler();
|
|
1197
|
-
} else if (status === "idle" && reason === "provider-turn") {
|
|
1198
|
-
if (this.currentTurnId) this.sessionLog(`turn ended via provider idle (turn ${this.currentTurnId})`);
|
|
1199
|
-
this.currentTurnId = undefined;
|
|
1200
|
-
this.currentTurnStartedAt = undefined;
|
|
1201
|
-
this.compactionMidTurn = false;
|
|
1202
|
-
this.disarmBusyReconciler();
|
|
1203
|
-
this.stopReasoningTail();
|
|
1204
|
-
// #417: claude enforces obligations via the stop.sh hook; for every other provider
|
|
1205
|
-
// re-inject any pending spawner obligation so it surfaces the /reply command.
|
|
1206
|
-
if (this.options.provider !== "claude") void this.reInjectPendingObligation();
|
|
1207
|
-
}
|
|
1208
|
-
if (status === "busy") {
|
|
1209
|
-
this.claims.clearTerminalStatus();
|
|
1210
|
-
this.claims.startWork(reason, id, typeof update === "string" ? {} : {
|
|
1211
|
-
label: update.label,
|
|
1212
|
-
role: update.role,
|
|
1213
|
-
parentId: update.parentId,
|
|
1214
|
-
metadata: {
|
|
1215
|
-
...(update.metadata ?? {}),
|
|
1216
|
-
...(update.providerState ? { providerState: update.providerState } : {}),
|
|
1217
|
-
},
|
|
1218
|
-
});
|
|
1219
|
-
if (reason === "provider-turn") {
|
|
1220
|
-
for (const claim of this.activeTaskClaims.values()) claim.observedProviderBusy = true;
|
|
1221
|
-
}
|
|
1222
|
-
} else if (status === "idle") {
|
|
1223
|
-
this.claims.clearTerminalStatus();
|
|
1224
|
-
this.claims.finishWork(reason, id);
|
|
1225
|
-
if (reason === "provider-turn") void this.completeObservedTaskClaims();
|
|
1226
|
-
}
|
|
1227
|
-
else if (status === "offline" || status === "error") this.claims.setTerminalStatus(status);
|
|
1228
|
-
if (typeof update !== "string") {
|
|
1229
|
-
for (const kind of update.clear ?? []) this.claims.clearWorkKind(kind);
|
|
1230
|
-
}
|
|
1231
|
-
// #329: the provider just went idle (ready). If a first initial-prompt delivery timed out on
|
|
1232
|
-
// a cold-start TUI, re-attempt now — the ready wait should resolve almost immediately.
|
|
1233
|
-
if (status === "idle" && this.pendingInitialPrompt && !this.deliveringInitialPrompt) {
|
|
1234
|
-
void this.attemptInitialPromptDelivery(this.pendingInitialPrompt, INITIAL_PROMPT_RETRY_READY_TIMEOUT_MS);
|
|
1235
|
-
}
|
|
1236
|
-
this.publishStatus();
|
|
1237
|
-
}
|
|
1238
|
-
|
|
1239
|
-
// #417: when a non-claude provider finishes a turn with a pending spawner obligation
|
|
1240
|
-
// still unresolved, re-inject the obligation as a synthetic message so providerMessageText
|
|
1241
|
-
// surfaces the specific `agent-relay /reply <id>` command. Claude handles this via the
|
|
1242
|
-
// stop.sh hook; every other provider needs this in-process nudge.
|
|
1243
|
-
private async reInjectPendingObligation(): Promise<void> {
|
|
1244
|
-
if (!this.process) return;
|
|
1245
|
-
// Filter to spawner obligations only — user obligations go through the existing codex
|
|
1246
|
-
// prompt-capture path (publishProviderSessionEvent) and the session mirror (#418), never
|
|
1247
|
-
// an explicit /reply. Shared predicate so this rule stays in lockstep with the Claude
|
|
1248
|
-
// Stop-hook nag exemption (control-server.replyObligationStopDecision).
|
|
1249
|
-
const obligations = this.obligationCache.get().filter(obligationRequiresExplicitReply);
|
|
1250
|
-
if (!obligations.length) return;
|
|
1251
|
-
const MAX_REINJECTIONS = 3;
|
|
1252
|
-
for (const obligation of obligations) {
|
|
1253
|
-
const count = this.pendingObligationInjections.get(obligation.messageId) ?? 0;
|
|
1254
|
-
if (count >= MAX_REINJECTIONS) continue;
|
|
1255
|
-
this.pendingObligationInjections.set(obligation.messageId, count + 1);
|
|
1256
|
-
this.sessionLog(`re-injecting pending reply obligation #${obligation.messageId} from ${obligation.from} (attempt ${count + 1}/${MAX_REINJECTIONS})`);
|
|
1257
|
-
const replayMessage = {
|
|
1258
|
-
id: obligation.messageId,
|
|
1259
|
-
from: obligation.from,
|
|
1260
|
-
to: this.agentId,
|
|
1261
|
-
kind: obligation.kind,
|
|
1262
|
-
...(obligation.subject ? { subject: obligation.subject } : {}),
|
|
1263
|
-
body: obligation.bodyPreview,
|
|
1264
|
-
payload: {},
|
|
1265
|
-
readBy: [] as string[],
|
|
1266
|
-
createdAt: obligation.createdAt,
|
|
1267
|
-
};
|
|
1268
|
-
// Suppress the userMessage echo that providers (e.g. Codex) reflect back when
|
|
1269
|
-
// a turn is started with relay-formatted content — the echo body is exactly what
|
|
1270
|
-
// providerMessageText produces, which starts with "[relay message #" and is caught
|
|
1271
|
-
// by the RELAY_INJECTION_MARKERS prefix check, but record it explicitly as
|
|
1272
|
-
// defense-in-depth for adapters that might echo only the unformatted body text.
|
|
1273
|
-
this.recordInjectedPrompt(providerMessageText([replayMessage]));
|
|
1274
|
-
await this.options.adapter.deliver(this.process, [replayMessage]);
|
|
1275
|
-
// Kick off a cache refresh so a cleared obligation (agent replied) is detected
|
|
1276
|
-
// before the next idle fires — without this, the stale snapshot re-injects up to
|
|
1277
|
-
// MAX_REINJECTIONS times even after the agent has already /reply-ed.
|
|
1278
|
-
this.obligationCache.markDirty();
|
|
1279
|
-
// One obligation at a time — let the provider reply before nudging again.
|
|
1280
|
-
break;
|
|
1281
|
-
}
|
|
1282
|
-
}
|
|
1283
|
-
|
|
1284
|
-
// Session-mirror lane: capture the assistant turn from the Claude transcript and
|
|
1285
|
-
// post it as a "session" message so it shows in the dashboard chat with zero
|
|
1286
|
-
// agent tokens. Capture is UNCONDITIONAL — it no longer depends on a triggering
|
|
1287
|
-
// relay message existing, so turns started from the web terminal (which create
|
|
1288
|
-
// no relay message) are mirrored too. A reply obligation, when present, is still
|
|
1289
|
-
// used as replyTo so the Stop hook stops nagging the agent to /reply.
|
|
1290
|
-
//
|
|
1291
|
-
// isPreFlush: true (#435) — mid-turn call from a PreToolUse hook (AskUserQuestion /
|
|
1292
|
-
// ExitPlanMode / permission). Emits any un-mirrored narrative from the transcript
|
|
1293
|
-
// tail immediately, before the interactive form appears in chat, then updates the
|
|
1294
|
-
// entry-count cursor so the eventual Stop-hook call (full mode) skips it.
|
|
1295
|
-
private async publishSessionTurn(input: { transcriptPath: string; lastAssistantMessage?: unknown; isPreFlush?: boolean }): Promise<void> {
|
|
1296
|
-
if (input.transcriptPath) this.lastTranscriptPath = input.transcriptPath;
|
|
1297
|
-
|
|
1298
|
-
if (input.isPreFlush) {
|
|
1299
|
-
if (!input.transcriptPath) return;
|
|
1300
|
-
let jsonl: string | undefined;
|
|
1301
|
-
try { jsonl = await readFile(input.transcriptPath, "utf8"); } catch { return; }
|
|
1302
|
-
if (!jsonl) return;
|
|
1303
|
-
const body = extractLastAssistantTurnAfterEntry(jsonl, this.narrativeFlushEntryCount);
|
|
1304
|
-
this.narrativeFlushEntryCount = countTranscriptEntries(jsonl);
|
|
1305
|
-
if (!body) return;
|
|
1306
|
-
const turnId = this.currentTurnId;
|
|
1307
|
-
this.sessionLog(`pre-flush narrative for turn ${turnId ?? "?"} (${body.length} chars)`);
|
|
1308
|
-
await this.publishSessionEvent({
|
|
1309
|
-
from: this.agentId,
|
|
1310
|
-
to: "user",
|
|
1311
|
-
body,
|
|
1312
|
-
session: { type: "response", origin: "provider", ...(turnId ? { turnId } : {}) },
|
|
1313
|
-
});
|
|
1314
|
-
return;
|
|
1315
|
-
}
|
|
1316
|
-
|
|
1317
|
-
const turnId = this.currentTurnId;
|
|
1318
|
-
this.stopReasoningTail();
|
|
1319
|
-
// Optional correlation for threading + obligation clearing — never a capture gate.
|
|
1320
|
-
let replyToMessageId: number | undefined;
|
|
1321
|
-
const pendingPrompt = this.pendingPromptMessageId;
|
|
1322
|
-
if (pendingPrompt) {
|
|
1323
|
-
replyToMessageId = pendingPrompt;
|
|
1324
|
-
this.pendingPromptMessageId = undefined;
|
|
1325
|
-
} else {
|
|
1326
|
-
// Correlation-only (threading + obligation clearing) — the local snapshot is fresh
|
|
1327
|
-
// enough and never blocks the response-capture path (#196).
|
|
1328
|
-
const obligation = [...this.obligationCache.get()].reverse().find((o) => o.from === "user");
|
|
1329
|
-
replyToMessageId = obligation?.messageId;
|
|
1330
|
-
}
|
|
1331
|
-
|
|
1332
|
-
// Grab and reset the pre-flush cursor before extraction: the cursor may be
|
|
1333
|
-
// non-zero when a PreToolUse hook flushed narrative earlier this turn (#435).
|
|
1334
|
-
// In "full" mode, skip those already-emitted entries so they don't duplicate.
|
|
1335
|
-
const flushCount = this.narrativeFlushEntryCount;
|
|
1336
|
-
this.narrativeFlushEntryCount = 0;
|
|
1337
|
-
|
|
1338
|
-
// The Stop hook can fire before the final assistant entry is flushed to disk.
|
|
1339
|
-
// Claude writes thinking and text as separate entries (both with end_turn), so
|
|
1340
|
-
// the transcript can "look complete" while the text entry is still pending.
|
|
1341
|
-
// Retry until both the transcript has an end_turn AND extraction yields text.
|
|
1342
|
-
let body = "";
|
|
1343
|
-
let jsonl: string | undefined;
|
|
1344
|
-
try { jsonl = await readFile(input.transcriptPath, "utf8"); } catch { jsonl = undefined; }
|
|
1345
|
-
if (jsonl !== undefined) {
|
|
1346
|
-
for (let attempt = 0; attempt < 5; attempt++) {
|
|
1347
|
-
if (attempt > 0) {
|
|
1348
|
-
await new Promise((r) => setTimeout(r, 100));
|
|
1349
|
-
try { jsonl = await readFile(input.transcriptPath, "utf8"); } catch { break; }
|
|
1350
|
-
}
|
|
1351
|
-
if (!transcriptLooksComplete(jsonl)) continue;
|
|
1352
|
-
// Full mode: if a pre-flush already emitted entries 1..flushCount, only
|
|
1353
|
-
// collect text from entries after that mark to avoid double-emit (#435).
|
|
1354
|
-
const extract = this.options.providerConfig.chatCaptureMode === "full"
|
|
1355
|
-
? (j: string) => (flushCount > 0 ? extractLastAssistantTurnAfterEntry(j, flushCount) : extractLastAssistantTurn(j))
|
|
1356
|
-
: extractFinalAssistantMessage;
|
|
1357
|
-
body = extract(jsonl);
|
|
1358
|
-
if (body) break;
|
|
1359
|
-
}
|
|
1360
|
-
}
|
|
1361
|
-
// Fallback: last_assistant_message from the Stop hook payload, which bypasses
|
|
1362
|
-
// the transcript file race entirely.
|
|
1363
|
-
if (!body && input.lastAssistantMessage) {
|
|
1364
|
-
body = extractHookAssistantMessage(input.lastAssistantMessage);
|
|
1365
|
-
}
|
|
1366
|
-
// A pure tool-use turn with no closing text is fine to skip — its reasoning and
|
|
1367
|
-
// tool steps already carried the visibility into chat.
|
|
1368
|
-
if (!body) {
|
|
1369
|
-
this.sessionLog(`response capture: no closing text for turn ${turnId ?? "?"} (skipped)`);
|
|
1370
|
-
return;
|
|
1371
|
-
}
|
|
1372
|
-
|
|
1373
|
-
this.sessionLog(`response captured for turn ${turnId ?? "?"} (${body.length} chars${replyToMessageId ? `, replyTo #${replyToMessageId}` : ", no replyTo"})`);
|
|
1374
|
-
await this.publishSessionEvent({
|
|
1375
|
-
from: this.agentId,
|
|
1376
|
-
to: "user",
|
|
1377
|
-
body,
|
|
1378
|
-
...(replyToMessageId ? { replyTo: replyToMessageId } : {}),
|
|
1379
|
-
session: { type: "response", origin: "provider", ...(turnId ? { turnId } : {}) },
|
|
1380
|
-
});
|
|
1381
|
-
// The agent's reply may have cleared an obligation — refresh the snapshot so the next
|
|
1382
|
-
// turn-end doesn't re-prompt for a message already answered (#196).
|
|
1383
|
-
if (replyToMessageId) this.obligationCache.markDirty();
|
|
1384
|
-
}
|
|
1385
|
-
|
|
1386
|
-
// Post one session-mirror event (prompt echo, assistant response, reasoning or
|
|
1387
|
-
// tool step) as a `kind: "session"` relay message tagged with payload.session so
|
|
1388
|
-
// the dashboard can render the live provider session faithfully. Display-only:
|
|
1389
|
-
// session messages are never delivered back into a provider.
|
|
1390
|
-
private publishSessionEvent(input: {
|
|
1391
|
-
from: string;
|
|
1392
|
-
to: string;
|
|
1393
|
-
body: string;
|
|
1394
|
-
session: MessageSessionMeta;
|
|
1395
|
-
replyTo?: number;
|
|
1396
|
-
}): void {
|
|
1397
|
-
// Durable, ordered, timestamped (#196): the actual POST happens in deliverOutboxEvent,
|
|
1398
|
-
// retried until it lands. occurredAt is stamped now so a queued event reports when it
|
|
1399
|
-
// truly happened, not when the server finally accepted it. Routed through the fast-lane
|
|
1400
|
-
// sessionOutbox (#332) so a transient trace failure can't head-of-line block real messages.
|
|
1401
|
-
// A stepId-bearing step (Codex tool running→completed, streamed reasoning/response) uses a
|
|
1402
|
-
// STABLE idempotency key so the server upserts the row in place instead of appending a dup.
|
|
1403
|
-
const stepId = input.session.stepId;
|
|
1404
|
-
this.sessionOutbox.enqueue({
|
|
1405
|
-
kind: "session-message",
|
|
1406
|
-
...(stepId ? { idempotencyKey: `session-step:${input.from}:${input.session.turnId ?? ""}:${stepId}` } : {}),
|
|
1407
|
-
payload: {
|
|
1408
|
-
from: input.from,
|
|
1409
|
-
to: input.to,
|
|
1410
|
-
...(input.replyTo ? { replyTo: input.replyTo } : {}),
|
|
1411
|
-
kind: "session",
|
|
1412
|
-
body: input.body,
|
|
1413
|
-
payload: { session: { provider: this.options.provider, ...input.session } },
|
|
1414
|
-
} satisfies SendMessageInput,
|
|
1415
|
-
});
|
|
1416
|
-
}
|
|
1417
|
-
|
|
1418
|
-
// Map queued records to HTTP calls. Throw to retry, return to ack/delete.
|
|
1419
|
-
private async deliverOutboxEvent(record: OutboxRecord): Promise<void> {
|
|
1420
|
-
try {
|
|
1421
|
-
if (record.kind === "session-message") {
|
|
1422
|
-
await this.http.sendMessage({
|
|
1423
|
-
...(record.payload as SendMessageInput),
|
|
1424
|
-
occurredAt: record.occurredAt,
|
|
1425
|
-
idempotencyKey: record.idempotencyKey,
|
|
1426
|
-
});
|
|
1427
|
-
return;
|
|
1428
|
-
}
|
|
1429
|
-
if (record.kind === "insight") {
|
|
1430
|
-
await this.http.recordInsightObservation({
|
|
1431
|
-
...(record.payload as Parameters<RelayHttpClient["recordInsightObservation"]>[0]),
|
|
1432
|
-
occurredAt: record.occurredAt,
|
|
1433
|
-
});
|
|
1434
|
-
return;
|
|
1435
|
-
}
|
|
1436
|
-
if (record.kind === "continuation-archive") {
|
|
1437
|
-
await deliverContinuationArchiveRecord({ record, http: this.http, updatePayload: (seq, payload) => this.outbox.updatePayload(seq, payload), sessionLog: (message) => this.sessionLog(message) });
|
|
1438
|
-
return;
|
|
1439
|
-
}
|
|
1440
|
-
if (record.kind === "mcp-tool-call") {
|
|
1441
|
-
await this.deliverBufferedMcpCall(record);
|
|
1442
|
-
return;
|
|
1443
|
-
}
|
|
1444
|
-
logger.warn("outbox", `dropping event with unknown kind: ${record.kind}`);
|
|
1445
|
-
} catch (error) {
|
|
1446
|
-
// 409 = the server intentionally rejected it (e.g. Insights/feature toggled off). That
|
|
1447
|
-
// is a permanent "don't want this", not a transient failure — ack so it doesn't retry.
|
|
1448
|
-
if (isHttpStatusError(error, 409)) return;
|
|
1449
|
-
if (isHttpAuthError(error)) this.recoverRuntimeTokenAfterAuthFailure("outbox");
|
|
1450
|
-
throw error; // transient (or auth, post-recovery) → let the outbox retry with backoff
|
|
1451
|
-
}
|
|
1452
|
-
}
|
|
1453
|
-
|
|
1454
|
-
// Replay a buffered MCP tool call (Stage 2, #215) that the proxy queued while the relay was
|
|
1455
|
-
// unreachable. POST it to the relay MCP endpoint with the LIVE token — same path the live call
|
|
1456
|
-
// would have taken. Throw to retry (transient), return to ack (delivered or permanently
|
|
1457
|
-
// rejected). The proxy stamped an idempotencyKey into the arguments so a retry that already
|
|
1458
|
-
// landed server-side is deduped, not double-sent.
|
|
1459
|
-
private async deliverBufferedMcpCall(record: OutboxRecord): Promise<void> {
|
|
1460
|
-
const payload = record.payload as { tool: string; arguments: Record<string, unknown> };
|
|
1461
|
-
const headers: Record<string, string> = { "content-type": "application/json" };
|
|
1462
|
-
if (this.currentToken) headers.authorization = `Bearer ${this.currentToken}`;
|
|
1463
|
-
const response = await fetch(relayMcpEndpoint(this.options.relayUrl), {
|
|
1464
|
-
method: "POST",
|
|
1465
|
-
headers,
|
|
1466
|
-
body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: payload.tool, arguments: payload.arguments } }),
|
|
1467
|
-
});
|
|
1468
|
-
if (response.status === 401 || response.status === 403) {
|
|
1469
|
-
this.recoverRuntimeTokenAfterAuthFailure("mcp-outbox");
|
|
1470
|
-
throw new Error(`relay rejected buffered ${payload.tool} with ${response.status}`);
|
|
1471
|
-
}
|
|
1472
|
-
if (response.status >= 500) throw new Error(`relay ${response.status} on buffered ${payload.tool}`);
|
|
1473
|
-
if (!response.ok) {
|
|
1474
|
-
// A 4xx (e.g. target gone, validation) is a permanent rejection — retrying won't help.
|
|
1475
|
-
// Ack so it doesn't block the queue, but log loudly: a queued write did not land.
|
|
1476
|
-
const body = await response.text().catch(() => "");
|
|
1477
|
-
logger.warn("mcp-outbox", `buffered ${payload.tool} permanently rejected (${response.status}); dropping: ${body.slice(0, 200)}`);
|
|
1478
|
-
return;
|
|
1479
|
-
}
|
|
1480
|
-
// HTTP 200 but the JSON-RPC body may still carry a tool-level error. Those reflect the same
|
|
1481
|
-
// permanent-rejection semantics (bad target, validation) — ack and log, don't loop.
|
|
1482
|
-
const json = await response.json().catch(() => null) as { error?: { message?: string } } | null;
|
|
1483
|
-
if (json?.error) {
|
|
1484
|
-
logger.warn("mcp-outbox", `buffered ${payload.tool} returned a tool error; dropping: ${json.error.message ?? "(no detail)"}`);
|
|
1485
|
-
}
|
|
1486
|
-
}
|
|
1487
|
-
|
|
1488
|
-
// A hook reported an unhandled failure (#198 seam). Already logged FATAL by the control
|
|
1489
|
-
// server; here we additionally surface it durably to the server as a generic insight so
|
|
1490
|
-
// it shows up in observability rather than only in the per-agent log (#196).
|
|
1491
|
-
private reportHookFatal(report: { hook: string; error: string }): void {
|
|
1492
|
-
try {
|
|
1493
|
-
this.outbox.enqueue({
|
|
1494
|
-
kind: "insight",
|
|
1495
|
-
payload: {
|
|
1496
|
-
sessionId: this.providerSessionId,
|
|
1497
|
-
project: this.insightProject(),
|
|
1498
|
-
agentId: this.agentId,
|
|
1499
|
-
signal: "hook_fatal",
|
|
1500
|
-
value: { hook: report.hook, error: report.error },
|
|
1501
|
-
source: "server",
|
|
1502
|
-
},
|
|
1503
|
-
});
|
|
1504
|
-
} catch (error) {
|
|
1505
|
-
logger.error("outbox", `failed to queue hook-fatal report: ${errMessage(error)}`);
|
|
1506
|
-
}
|
|
1507
|
-
}
|
|
1508
|
-
|
|
1509
|
-
// A human typed a prompt directly into the provider (web terminal / TUI). Mirror
|
|
1510
|
-
// it into the dashboard chat so both surfaces stay in sync, and kick off reasoning
|
|
1511
|
-
// tailing for the turn. Skips prompts the runner itself injected (chat box, relay
|
|
1512
|
-
// deliveries) so those aren't double-posted.
|
|
1513
|
-
private async handleUserPrompt(input: { prompt: string; transcriptPath?: string }): Promise<void> {
|
|
1514
|
-
if (input.transcriptPath) this.lastTranscriptPath = input.transcriptPath;
|
|
1515
|
-
if (!this.currentTurnId) {
|
|
1516
|
-
this.currentTurnId = crypto.randomUUID();
|
|
1517
|
-
this.currentTurnStartedAt = Date.now();
|
|
1518
|
-
}
|
|
1519
|
-
const text = input.prompt.trim();
|
|
1520
|
-
if (text && !this.isRunnerInjectedPrompt(text)) {
|
|
1521
|
-
this.sessionLog(`prompt echoed from terminal (${text.length} chars)`);
|
|
1522
|
-
await this.publishSessionEvent({
|
|
1523
|
-
from: "user",
|
|
1524
|
-
to: this.agentId,
|
|
1525
|
-
body: text,
|
|
1526
|
-
session: { type: "prompt", origin: "terminal", turnId: this.currentTurnId },
|
|
1527
|
-
});
|
|
1528
|
-
} else if (text) {
|
|
1529
|
-
this.sessionDebug("user-prompt hook: skipped echo (runner-injected)");
|
|
1530
|
-
}
|
|
1531
|
-
if (input.transcriptPath) this.startReasoningTail(input.transcriptPath);
|
|
1532
|
-
}
|
|
1533
|
-
|
|
1534
|
-
// A provider lifecycle hook reported a session boundary (Claude PreCompact / SessionEnd
|
|
1535
|
-
// → control server). Normalize the raw provider reason to a SessionDestroyReason and run
|
|
1536
|
-
// the same pre-destroy seam the bus commands use. `clear`/`compact` continue the session;
|
|
1537
|
-
// anything else (logout, prompt_input_exit, other) is a real termination.
|
|
1538
|
-
private async handleSessionBoundary(input: { reason?: string; transcriptPath?: string }): Promise<void> {
|
|
1539
|
-
// Reason mapping is fail-safe-toward-termination: only the two known session-
|
|
1540
|
-
// CONTINUING reasons are special-cased; everything else (logout, prompt_input_exit,
|
|
1541
|
-
// other, AND any future reason) maps to "shutdown" → full pre-destroy capture.
|
|
1542
|
-
// ⚠ If Claude Code adds a new BENIGN/continuing boundary reason, add it here — until
|
|
1543
|
-
// then it will trigger a (harmless but wasteful) full context capture on a session
|
|
1544
|
-
// that isn't actually ending.
|
|
1545
|
-
const reason = input.reason === "compact" ? "compact"
|
|
1546
|
-
: input.reason === "clear" ? "clear"
|
|
1547
|
-
: "shutdown";
|
|
1548
|
-
await this.runPreSessionDestroy(reason, { transcriptPath: input.transcriptPath });
|
|
1549
|
-
// clear/compact CONTINUE the session — the finalizing window is transient. The bus-command
|
|
1550
|
-
// path (handleCommand) restores the addressable status in its finally; the hook path has no
|
|
1551
|
-
// such teardown, so without this the dashboard stays stuck on "wrapping up — messaging
|
|
1552
|
-
// paused" with the composer disabled forever. Only restore if we're still in the exact
|
|
1553
|
-
// finalizing state we published — a concurrent bus command (kill/restart/shutdown) may have
|
|
1554
|
-
// transitioned lifecycleAction since, and that must win. "shutdown" reasons end the session,
|
|
1555
|
-
// so their finalizing state is superseded by going offline; leave it be.
|
|
1556
|
-
if ((reason === "clear" || reason === "compact") && this.lifecycleAction === `finalizing-${reason}` && !this.stopped) {
|
|
1557
|
-
this.lifecycleAction = undefined;
|
|
1558
|
-
this.publishStatus();
|
|
1559
|
-
}
|
|
1560
|
-
}
|
|
1561
|
-
|
|
1562
|
-
// The pre-session-destroy seam (#183): the single place end-of-session work runs before
|
|
1563
|
-
// an invasive transition (compact/clear/restart/shutdown/kill). Best-effort and
|
|
1564
|
-
// time-boxed so it never hangs teardown; concurrent calls for the same teardown coalesce
|
|
1565
|
-
// (a shutdown bus command and the SessionEnd hook can both fire). During the window the
|
|
1566
|
-
// agent is published non-addressable so the operator sees "wrapping up", not "busy".
|
|
1567
|
-
private runPreSessionDestroy(reason: SessionDestroyReason, opts?: { transcriptPath?: string }): Promise<void> {
|
|
1568
|
-
if (this.preDestroyPromise) return this.preDestroyPromise;
|
|
1569
|
-
const run = (async () => {
|
|
1570
|
-
this.publishFinalizing(reason);
|
|
1571
|
-
try {
|
|
1572
|
-
await Promise.race([
|
|
1573
|
-
Promise.all([
|
|
1574
|
-
this.captureContextRatio(reason, opts),
|
|
1575
|
-
this.captureContinuationArchive(reason, opts),
|
|
1576
|
-
]).then(() => undefined),
|
|
1577
|
-
new Promise<void>((resolve) => setTimeout(resolve, PRE_DESTROY_TIMEOUT_MS)),
|
|
1578
|
-
]);
|
|
1579
|
-
} catch (error) {
|
|
1580
|
-
this.sessionLog(`pre-destroy capture failed: ${errMessage(error)}`);
|
|
1581
|
-
}
|
|
1582
|
-
// For exit-bound transitions the runner won't be alive afterward to drain the durable
|
|
1583
|
-
// outbox, so block (bounded) on delivering what capture just enqueued. This runs before
|
|
1584
|
-
// handleCommand's finally deletes the agent, so the runtime token is still valid here.
|
|
1585
|
-
if (reasonExitsRunner(reason)) {
|
|
1586
|
-
try {
|
|
1587
|
-
const delivered = await this.outbox.flush(OUTBOX_FLUSH_TIMEOUT_MS);
|
|
1588
|
-
if (!delivered) this.sessionLog(`insights: outbox flush incomplete before ${reason} (${this.outbox.pendingCount()} pending)`);
|
|
1589
|
-
} catch (error) {
|
|
1590
|
-
this.sessionLog(`insights: outbox flush failed: ${errMessage(error)}`);
|
|
1591
|
-
}
|
|
1592
|
-
}
|
|
1593
|
-
})();
|
|
1594
|
-
this.preDestroyPromise = run;
|
|
1595
|
-
void run.finally(() => { this.preDestroyPromise = undefined; });
|
|
1596
|
-
return run;
|
|
1597
|
-
}
|
|
1598
|
-
|
|
1599
|
-
// Publish the transient pre-destroy state: a non-offline status with ready:false (so the
|
|
1600
|
-
// agent drops out of isAgentOnline fan-out targeting without going "offline") plus a
|
|
1601
|
-
// finalizing-<reason> lifecycleAction the dashboard renders as "wrapping up" with the
|
|
1602
|
-
// composer disabled.
|
|
1603
|
-
private publishFinalizing(reason: SessionDestroyReason): void {
|
|
1604
|
-
this.lifecycleAction = `finalizing-${reason}`;
|
|
1605
|
-
void this.bus.statusAsync({ agentStatus: "busy", ready: false, meta: { lifecycleAction: this.lifecycleAction, lifecycleActionAt: Date.now() } });
|
|
1606
|
-
}
|
|
1607
|
-
|
|
1608
|
-
// Compute the #184 context-gathering ratio for the segment since the last boundary and
|
|
1609
|
-
// queue it (durable outbox, #196). Provider-agnostic: the adapter normalizes its session
|
|
1610
|
-
// into the shared SessionEvent stream; the math + classifier live in session-insights.ts.
|
|
1611
|
-
// Per-segment via a runner-side cursor, so each datapoint is one work chunk between
|
|
1612
|
-
// context resets. Mechanical, model-free → zero agent tokens, un-gameable.
|
|
1613
|
-
// Repo-name project id for insight observations, resolved once. Aggregating by the raw
|
|
1614
|
-
// cwd would split one repo across many "projects" (each isolated agent runs from a
|
|
1615
|
-
// unique per-session worktree dir). See resolveProjectName.
|
|
1616
|
-
private insightProject(): string {
|
|
1617
|
-
return (this.insightProjectName ??= resolveProjectName(this.options.cwd, this.options.workspace));
|
|
1618
|
-
}
|
|
1619
|
-
|
|
1620
|
-
private async captureContinuationArchive(reason: SessionDestroyReason, opts?: { transcriptPath?: string }): Promise<void> {
|
|
1621
|
-
const adapter = this.options.adapter;
|
|
1622
|
-
if (!adapter.collectSessionArchiveSegment || !this.process) return;
|
|
1623
|
-
const transcriptPath = opts?.transcriptPath ?? this.lastTranscriptPath;
|
|
1624
|
-
const archive = await adapter.collectSessionArchiveSegment(this.process, { transcriptPath });
|
|
1625
|
-
if (!archive) return;
|
|
1626
|
-
const key = transcriptPath ?? `session:${this.providerSessionId}`;
|
|
1627
|
-
if (key !== this.archiveCursorKey || archive.length < this.archiveObservedChars) {
|
|
1628
|
-
this.archiveCursorKey = key;
|
|
1629
|
-
this.archiveObservedChars = 0;
|
|
1630
|
-
}
|
|
1631
|
-
const segment = archive.slice(this.archiveObservedChars).trim();
|
|
1632
|
-
this.archiveObservedChars = archive.length;
|
|
1633
|
-
if (!segment) return;
|
|
1634
|
-
const bounded = boundContinuationArchiveSegment(segment);
|
|
1635
|
-
if (bounded.droppedBytes > 0) this.sessionLog(`continuation archive truncated at ${bounded.keptBytes} bytes; dropped ${bounded.droppedBytes} bytes (${reason})`);
|
|
1636
|
-
this.outbox.enqueue({
|
|
1637
|
-
kind: "continuation-archive",
|
|
1638
|
-
payload: {
|
|
1639
|
-
agentId: this.agentId,
|
|
1640
|
-
segment: bounded.segment,
|
|
1641
|
-
},
|
|
1642
|
-
});
|
|
1643
|
-
this.sessionLog(`continuation archive queued (${bounded.segment.length} chars, ${reason})`);
|
|
1644
|
-
}
|
|
1645
|
-
|
|
1646
|
-
private async captureContextRatio(reason: SessionDestroyReason, opts?: { transcriptPath?: string }): Promise<void> {
|
|
1647
|
-
const adapter = this.options.adapter;
|
|
1648
|
-
if (!adapter.collectSessionEvents || !this.process) return;
|
|
1649
|
-
const transcriptPath = opts?.transcriptPath ?? this.lastTranscriptPath;
|
|
1650
|
-
const events = await adapter.collectSessionEvents(this.process, { transcriptPath });
|
|
1651
|
-
if (!events) return;
|
|
1652
|
-
// Reset the cursor when the underlying log changed identity (transcript rotated on
|
|
1653
|
-
// resume) or shrank (Codex buffer trimmed) — otherwise the slice would be wrong.
|
|
1654
|
-
const key = transcriptPath ?? `session:${this.providerSessionId}`;
|
|
1655
|
-
if (key !== this.insightsCursorKey || events.length < this.insightsObserved) {
|
|
1656
|
-
this.insightsCursorKey = key;
|
|
1657
|
-
this.insightsObserved = 0;
|
|
1658
|
-
}
|
|
1659
|
-
const segment = events.slice(this.insightsObserved);
|
|
1660
|
-
this.insightsObserved = events.length;
|
|
1661
|
-
const analysis = computeContextRatio(segment);
|
|
1662
|
-
if (!analysis) return; // no tool calls this segment = nothing substantive to measure
|
|
1663
|
-
this.outbox.enqueue({
|
|
1664
|
-
kind: "insight",
|
|
1665
|
-
payload: {
|
|
1666
|
-
sessionId: this.providerSessionId,
|
|
1667
|
-
project: this.insightProject(),
|
|
1668
|
-
agentId: this.agentId,
|
|
1669
|
-
signal: "context_ratio",
|
|
1670
|
-
value: { ...analysis.metric, endReason: reason },
|
|
1671
|
-
outcome: { ...analysis.outcome },
|
|
1672
|
-
source: "server",
|
|
1673
|
-
},
|
|
1674
|
-
});
|
|
1675
|
-
this.sessionLog(`insights: context_ratio ${analysis.metric.ratio.toFixed(2)} (${analysis.metric.gatheringCalls}/${analysis.metric.totalToolCalls} gathering, ${reason}) queued`);
|
|
1676
|
-
}
|
|
1677
|
-
|
|
1678
|
-
// Route a provider-emitted session event (Codex app-server) into the chat mirror.
|
|
1679
|
-
// Mirrors the same semantics as the Claude lane: prompts are echoed with dedup,
|
|
1680
|
-
// and a response is only auto-captured when the agent won't separately reply to a
|
|
1681
|
-
// relay obligation (so relay-triggered turns aren't double-posted).
|
|
1682
|
-
private async publishProviderSessionEvent(event: ProviderSessionEvent): Promise<void> {
|
|
1683
|
-
const body = event.body.trim();
|
|
1684
|
-
if (!body) return;
|
|
1685
|
-
const turnId = event.turnId ?? this.currentTurnId;
|
|
1686
|
-
if (event.type === "prompt") {
|
|
1687
|
-
if (this.isRunnerInjectedPrompt(body)) return;
|
|
1688
|
-
await this.publishSessionEvent({
|
|
1689
|
-
from: "user",
|
|
1690
|
-
to: this.agentId,
|
|
1691
|
-
body,
|
|
1692
|
-
session: { type: "prompt", origin: event.origin ?? "terminal", ...(turnId ? { turnId } : {}) },
|
|
1693
|
-
});
|
|
1694
|
-
return;
|
|
1695
|
-
}
|
|
1696
|
-
if (event.type === "response") {
|
|
1697
|
-
// Dashboard prompt injection is already answered by this captured App Server
|
|
1698
|
-
// response. Other Relay inbox obligations still belong to the agent's explicit
|
|
1699
|
-
// reply path, so those keep suppressing auto-capture to avoid duplicates.
|
|
1700
|
-
let replyToMessageId: number | undefined;
|
|
1701
|
-
const pendingPrompt = this.pendingPromptMessageId;
|
|
1702
|
-
if (pendingPrompt) {
|
|
1703
|
-
replyToMessageId = pendingPrompt;
|
|
1704
|
-
this.pendingPromptMessageId = undefined;
|
|
1705
|
-
} else if (this.obligationCache.get().some((o) => o.from === "user" && this.obligationPredatesCurrentTurn(o))) {
|
|
1706
|
-
// The agent will answer the relay obligation itself — don't double-post (#196).
|
|
1707
|
-
return;
|
|
1708
|
-
}
|
|
1709
|
-
await this.publishSessionEvent({
|
|
1710
|
-
from: this.agentId,
|
|
1711
|
-
to: "user",
|
|
1712
|
-
body,
|
|
1713
|
-
...(replyToMessageId ? { replyTo: replyToMessageId } : {}),
|
|
1714
|
-
session: { type: "response", origin: event.origin ?? "provider", ...(turnId ? { turnId } : {}) },
|
|
1715
|
-
});
|
|
1716
|
-
if (replyToMessageId) this.obligationCache.markDirty();
|
|
1717
|
-
return;
|
|
1718
|
-
}
|
|
1719
|
-
if (this.options.providerConfig.reasoningCapture === false) return;
|
|
1720
|
-
await this.publishSessionEvent({
|
|
1721
|
-
from: this.agentId,
|
|
1722
|
-
to: "user",
|
|
1723
|
-
body,
|
|
1724
|
-
session: {
|
|
1725
|
-
type: event.type,
|
|
1726
|
-
origin: event.origin ?? "provider",
|
|
1727
|
-
...(turnId ? { turnId } : {}),
|
|
1728
|
-
...(event.label ? { label: event.label } : {}),
|
|
1729
|
-
...(event.status ? { status: event.status } : {}),
|
|
1730
|
-
...(event.streaming !== undefined ? { streaming: event.streaming } : {}),
|
|
1731
|
-
...(event.stepId ? { stepId: event.stepId } : {}),
|
|
1732
|
-
},
|
|
1733
|
-
});
|
|
1734
|
-
}
|
|
1735
|
-
|
|
1736
|
-
// Remember an injected prompt so its UserPromptSubmit echo can be suppressed. Prunes
|
|
1737
|
-
// expired entries first; a defensive length cap guards against echoes that never
|
|
1738
|
-
// arrive (e.g. the provider drops a queued prompt) so the queue can't grow unbounded.
|
|
1739
|
-
private recordInjectedPrompt(text: string): void {
|
|
1740
|
-
const now = Date.now();
|
|
1741
|
-
this.injectedPrompts = this.injectedPrompts.filter((p) => now - p.at < PROMPT_ECHO_DEDUP_MS);
|
|
1742
|
-
this.injectedPrompts.push({ text, at: now });
|
|
1743
|
-
if (this.injectedPrompts.length > 50) this.injectedPrompts.shift();
|
|
1744
|
-
}
|
|
1745
|
-
|
|
1746
|
-
private isRunnerInjectedPrompt(text: string): boolean {
|
|
1747
|
-
if (RELAY_INJECTION_MARKERS.some((marker) => text.startsWith(marker))) return true;
|
|
1748
|
-
const now = Date.now();
|
|
1749
|
-
this.injectedPrompts = this.injectedPrompts.filter((p) => now - p.at < PROMPT_ECHO_DEDUP_MS);
|
|
1750
|
-
const idx = this.injectedPrompts.findIndex((p) => p.text === text);
|
|
1751
|
-
if (idx !== -1) {
|
|
1752
|
-
this.injectedPrompts.splice(idx, 1); // consume one — identical repeats each match once
|
|
1753
|
-
return true;
|
|
1754
|
-
}
|
|
1755
|
-
return false;
|
|
1756
|
-
}
|
|
1757
|
-
|
|
1758
|
-
private obligationPredatesCurrentTurn(obligation: { createdAt: number }): boolean {
|
|
1759
|
-
return this.currentTurnStartedAt === undefined || obligation.createdAt <= this.currentTurnStartedAt;
|
|
1760
|
-
}
|
|
1761
|
-
|
|
1762
|
-
// --- Busy-state reconciler (item 2) -------------------------------------------------
|
|
1763
|
-
// A safety net for turns that end out of band (interrupted from the web terminal,
|
|
1764
|
-
// a hook that never fired) where the runner would otherwise stay stuck "busy".
|
|
1765
|
-
private armBusyReconciler(): void {
|
|
1766
|
-
if (this.busyReconcileTimer || !this.options.adapter.probeActivity) return;
|
|
1767
|
-
this.busyReconcileIdleStreak = 0;
|
|
1768
|
-
this.busyReconcileSawBusy = false;
|
|
1769
|
-
this.busyReconcileTimer = setInterval(() => { void this.runBusyReconcile(); }, BUSY_RECONCILE_POLL_MS);
|
|
1770
|
-
}
|
|
1771
|
-
|
|
1772
|
-
private disarmBusyReconciler(): void {
|
|
1773
|
-
if (this.busyReconcileTimer) clearInterval(this.busyReconcileTimer);
|
|
1774
|
-
this.busyReconcileTimer = undefined;
|
|
1775
|
-
this.busyReconcileIdleStreak = 0;
|
|
1776
|
-
this.busyReconcileSawBusy = false;
|
|
1777
|
-
}
|
|
1778
|
-
|
|
1779
|
-
private async runBusyReconcile(): Promise<void> {
|
|
1780
|
-
if (this.stopped || !this.process || !this.options.adapter.probeActivity) { this.disarmBusyReconciler(); return; }
|
|
1781
|
-
// Only act while the runner still believes a provider turn is in flight, and
|
|
1782
|
-
// never override a legitimate approval/blocked state.
|
|
1783
|
-
if (this.claims.currentStatus() !== "busy" || this.providerBlocked) { this.busyReconcileIdleStreak = 0; return; }
|
|
1784
|
-
if (!this.claims.activeWork().some((w) => w.kind === "provider-turn")) { this.disarmBusyReconciler(); return; }
|
|
1785
|
-
let activity: "busy" | "idle" | "unknown";
|
|
1786
|
-
try { activity = await this.options.adapter.probeActivity(this.process); } catch { return; }
|
|
1787
|
-
if (activity === "busy") this.busyReconcileSawBusy = true;
|
|
1788
|
-
// Reset the streak on anything that isn't a confident idle.
|
|
1789
|
-
if (activity !== "idle") {
|
|
1790
|
-
this.busyReconcileIdleStreak = 0;
|
|
1791
|
-
this.sessionDebug(`reconcile probe=${activity} sawBusy=${this.busyReconcileSawBusy} streak=${this.busyReconcileIdleStreak}`);
|
|
1792
|
-
return;
|
|
1793
|
-
}
|
|
1794
|
-
this.busyReconcileIdleStreak += 1;
|
|
1795
|
-
// Confirm faster once we've seen the spinner this turn; otherwise demand a much
|
|
1796
|
-
// longer all-idle window before trusting it (rescues fast turns without
|
|
1797
|
-
// false-clearing a live turn that simply hasn't flashed busy into a probe yet).
|
|
1798
|
-
const confirm = this.busyReconcileSawBusy ? BUSY_RECONCILE_IDLE_CONFIRM : BUSY_RECONCILE_IDLE_CONFIRM_NO_BUSY;
|
|
1799
|
-
this.sessionDebug(`reconcile probe=idle sawBusy=${this.busyReconcileSawBusy} streak=${this.busyReconcileIdleStreak}/${confirm}`);
|
|
1800
|
-
if (this.busyReconcileIdleStreak < confirm) return;
|
|
1801
|
-
this.disarmBusyReconciler();
|
|
1802
|
-
this.forceClearProviderTurn(this.busyReconcileSawBusy ? "backstop reconciler" : "backstop reconciler (no-busy-observed)");
|
|
1803
|
-
}
|
|
1804
|
-
|
|
1805
|
-
// Force-clear a stuck provider-turn claim directly. Unlike the idle status path
|
|
1806
|
-
// it does NOT depend on a matching claim id (the Stop hook keys busy as
|
|
1807
|
-
// provider-turn:provider-turn, but reconciliation has no specific id), and it
|
|
1808
|
-
// deliberately leaves the reasoning tail alone so a late clear can't truncate
|
|
1809
|
-
// a turn's activity stream.
|
|
1810
|
-
private forceClearProviderTurn(reason: string): void {
|
|
1811
|
-
if (!this.claims.activeWork().some((w) => w.kind === "provider-turn")) return;
|
|
1812
|
-
this.sessionLog(`force-clearing stuck provider-turn (${reason})`);
|
|
1813
|
-
this.claims.clearWorkKind("provider-turn");
|
|
1814
|
-
this.currentTurnId = undefined;
|
|
1815
|
-
this.compactionMidTurn = false;
|
|
1816
|
-
this.publishStatus();
|
|
1817
|
-
}
|
|
1818
|
-
|
|
1819
|
-
// After a dashboard interrupt, the provider should drop out of its turn; reconcile
|
|
1820
|
-
// promptly so the busy indicator clears even if the Stop hook doesn't fire.
|
|
1821
|
-
private scheduleInterruptReconcile(): void {
|
|
1822
|
-
setTimeout(() => {
|
|
1823
|
-
if (this.stopped || !this.process) return;
|
|
1824
|
-
void (async () => {
|
|
1825
|
-
if (this.claims.currentStatus() !== "busy" || this.providerBlocked) return;
|
|
1826
|
-
let activity: "busy" | "idle" | "unknown" = "unknown";
|
|
1827
|
-
try { if (this.options.adapter.probeActivity) activity = await this.options.adapter.probeActivity(this.process!); } catch { return; }
|
|
1828
|
-
this.sessionDebug(`post-interrupt reconcile probe=${activity}`);
|
|
1829
|
-
if (activity === "idle") this.forceClearProviderTurn("post-interrupt");
|
|
1830
|
-
})();
|
|
1831
|
-
}, INTERRUPT_RECONCILE_DELAY_MS);
|
|
1832
|
-
}
|
|
1833
|
-
|
|
1834
|
-
// --- Turn-step tailer (item 5) ------------------------------------------------------
|
|
1835
|
-
// Tail the in-flight turn's Claude transcript and surface new narration/reasoning/tool
|
|
1836
|
-
// steps as session events, in transcript order. `narration` (the agent's intermediate
|
|
1837
|
-
// text) is the primary visible content; reasoning visibility is a client-side toggle.
|
|
1838
|
-
// Coalesced and coarse; the final response still comes through publishSessionTurn.
|
|
1839
|
-
// `reasoningCapture: false` disables the whole live trace (server-side kill switch).
|
|
1840
|
-
private startReasoningTail(transcriptPath: string): void {
|
|
1841
|
-
if (this.options.providerConfig.reasoningCapture === false) return;
|
|
1842
|
-
this.stopReasoningTail();
|
|
1843
|
-
// Track emitted steps by content signature, not by index/count: the "latest
|
|
1844
|
-
// turn" window in the transcript can shrink/reset (a tool_result entry, a
|
|
1845
|
-
// mid-turn user line), and an index cursor would then either re-emit or stall
|
|
1846
|
-
// and drop the rest of the turn. A seen-set is idempotent under any reshuffle.
|
|
1847
|
-
// The signature is salted with each step's occurrence-within-window (stepDedupKeys)
|
|
1848
|
-
// so two identical steps in one turn — same tool, same input — both surface (#265).
|
|
1849
|
-
const seen = new Set<string>();
|
|
1850
|
-
const turnIdAtStart = this.currentTurnId;
|
|
1851
|
-
// On the first poll the new prompt usually hasn't landed in the transcript yet,
|
|
1852
|
-
// so extractLatestTurnSteps still returns the PRIOR (completed) turn. Seed those
|
|
1853
|
-
// signatures as already-seen so we don't replay last turn's reasoning/tools as
|
|
1854
|
-
// this turn's activity. Once our prompt lands the window resets at the new user
|
|
1855
|
-
// boundary and genuinely-new steps emit normally. Only seed when the transcript
|
|
1856
|
-
// is complete (last entry is an end_turn assistant) — otherwise we're already
|
|
1857
|
-
// inside the new turn and those steps are legitimately ours.
|
|
1858
|
-
let seeded = false;
|
|
1859
|
-
const poll = async (): Promise<void> => {
|
|
1860
|
-
let jsonl: string;
|
|
1861
|
-
try { jsonl = await readFile(transcriptPath, "utf8"); } catch { return; }
|
|
1862
|
-
let steps: ReturnType<typeof extractLatestTurnSteps>;
|
|
1863
|
-
try { steps = extractLatestTurnSteps(jsonl); } catch { return; }
|
|
1864
|
-
const keyed = stepDedupKeys(steps).map((sig, i) => ({ sig, step: steps[i]! }));
|
|
1865
|
-
if (!seeded) {
|
|
1866
|
-
seeded = true;
|
|
1867
|
-
if (transcriptLooksComplete(jsonl)) {
|
|
1868
|
-
for (const { sig } of keyed) seen.add(sig);
|
|
1869
|
-
}
|
|
1870
|
-
}
|
|
1871
|
-
const turnId = this.currentTurnId ?? turnIdAtStart;
|
|
1872
|
-
let emitted = 0;
|
|
1873
|
-
for (const { sig, step } of keyed) {
|
|
1874
|
-
if (seen.has(sig)) continue;
|
|
1875
|
-
seen.add(sig);
|
|
1876
|
-
emitted += 1;
|
|
1877
|
-
void this.publishSessionEvent({
|
|
1878
|
-
from: this.agentId,
|
|
1879
|
-
to: "user",
|
|
1880
|
-
body: step.text,
|
|
1881
|
-
session: { type: step.type, origin: "provider", ...(turnId ? { turnId } : {}), ...(step.label ? { label: step.label } : {}) },
|
|
1882
|
-
});
|
|
1883
|
-
}
|
|
1884
|
-
if (emitted) this.sessionDebug(`reasoning tail emitted ${emitted} step(s) (turn ${turnId ?? "?"}, ${seen.size} total)`);
|
|
1885
|
-
};
|
|
1886
|
-
this.reasoningTail = { seen, timer: setInterval(() => { void poll(); }, REASONING_POLL_MS) };
|
|
1887
|
-
this.sessionLog(`reasoning tail started (turn ${turnIdAtStart ?? "?"})`);
|
|
1888
|
-
void poll();
|
|
1889
|
-
}
|
|
1890
|
-
|
|
1891
|
-
private stopReasoningTail(): void {
|
|
1892
|
-
if (this.reasoningTail) clearInterval(this.reasoningTail.timer);
|
|
1893
|
-
this.reasoningTail = undefined;
|
|
1894
|
-
}
|
|
1895
|
-
|
|
1896
|
-
// Mirror a discreet, durable "context compacted" marker into chat via the existing
|
|
1897
|
-
// session-mirror lane (not a parallel channel). `notice` renders as an inline timeline
|
|
1898
|
-
// marker (never a bubble) and survives reload, unlike the ephemeral timeline-status one.
|
|
1899
|
-
private publishCompactionNotice(): void {
|
|
1900
|
-
this.publishSessionEvent({
|
|
1901
|
-
from: this.agentId,
|
|
1902
|
-
to: "user",
|
|
1903
|
-
body: "🗜 Context compacted",
|
|
1904
|
-
session: { type: "notice", origin: "provider", label: "compacted", ...(this.currentTurnId ? { turnId: this.currentTurnId } : {}) },
|
|
1905
|
-
});
|
|
1906
|
-
}
|
|
1907
|
-
|
|
1908
|
-
// #286: a discreet, durable chat marker when a usage/rate-limit hold begins, via
|
|
1909
|
-
// the same session-mirror lane as the compaction notice. Outbound session event
|
|
1910
|
-
// (NOT an inbound message) so it shows in the dashboard chat WITHOUT waking a turn
|
|
1911
|
-
// — waking a still-limited agent would just re-fail. The relay sends the waking
|
|
1912
|
-
// resume message later, once the window has reset.
|
|
1913
|
-
private publishRateLimitNotice(providerState: Record<string, unknown>): void {
|
|
1914
|
-
const label = typeof providerState.label === "string" && providerState.label ? providerState.label : "usage limit reached";
|
|
1915
|
-
this.publishSessionEvent({
|
|
1916
|
-
from: this.agentId,
|
|
1917
|
-
to: "user",
|
|
1918
|
-
body: `⏳ ${label} — holding; agent-relay will auto-resume at reset.`,
|
|
1919
|
-
session: { type: "notice", origin: "provider", label: "rate-limit", ...(this.currentTurnId ? { turnId: this.currentTurnId } : {}) },
|
|
1920
|
-
});
|
|
1921
|
-
}
|
|
1922
|
-
|
|
1923
|
-
private publishStatus(): void {
|
|
1924
|
-
this.claims.expire();
|
|
1925
|
-
const status = this.claims.currentStatus();
|
|
1926
|
-
const agentStatus = runnerAgentStatus(status);
|
|
1927
|
-
const activeWork = this.claims.activeWork();
|
|
1928
|
-
const activeSubagents = activeWork.filter((item) => item.kind === "subagent");
|
|
1929
|
-
const terminalFailure = this.terminalFailure;
|
|
1930
|
-
const providerState = terminalFailure?.providerState ?? this.rateLimitHold ?? providerStateFromActiveWork(activeWork);
|
|
1931
|
-
this.bus.setSemanticStatus(status === "offline" || status === "error" ? "idle" : status);
|
|
1932
|
-
const timelineEvent = this.pendingTimelineEvent;
|
|
1933
|
-
this.pendingTimelineEvent = undefined;
|
|
1934
|
-
this.bus.status({
|
|
1935
|
-
agentStatus,
|
|
1936
|
-
ready: agentStatus !== "offline" && !this.stopped,
|
|
1937
|
-
meta: {
|
|
1938
|
-
runnerId: this.options.runnerId,
|
|
1939
|
-
startedAt: this.options.startedAt,
|
|
1940
|
-
tmuxSession: this.providerTerminalSession() ?? this.options.tmuxSession ?? null,
|
|
1941
|
-
tmuxSocket: this.providerTerminalSocket() ?? null,
|
|
1942
|
-
policyName: this.options.policyName ?? null,
|
|
1943
|
-
spawnRequestId: this.options.spawnRequestId ?? null,
|
|
1944
|
-
automationId: this.options.automationId ?? null,
|
|
1945
|
-
automationRunId: this.options.automationRunId ?? null,
|
|
1946
|
-
workspace: this.options.workspace ?? null,
|
|
1947
|
-
lifecycle: this.options.lifecycle ?? "persistent", workspaceMode: this.options.workspace?.requestedMode ?? this.options.workspace?.mode ?? process.env.AGENT_RELAY_WORKSPACE_MODE ?? null,
|
|
1948
|
-
lifecycleAction: this.lifecycleAction ?? null,
|
|
1949
|
-
profile: this.options.profile ?? null,
|
|
1950
|
-
...(status === "error" ? { terminalStatus: "error" } : {}),
|
|
1951
|
-
...(terminalFailure ? {
|
|
1952
|
-
lastError: terminalFailure.message,
|
|
1953
|
-
terminalFailureReason: terminalFailure.reason,
|
|
1954
|
-
terminalFailureMessage: terminalFailure.message,
|
|
1955
|
-
} : {}),
|
|
1956
|
-
busyReasons: this.claims.reasons(),
|
|
1957
|
-
activeWork,
|
|
1958
|
-
activeSubagents,
|
|
1959
|
-
activeSubagentCount: activeSubagents.length,
|
|
1960
|
-
providerState,
|
|
1961
|
-
auth: {
|
|
1962
|
-
profileId: this.currentTokenProfileId ?? null,
|
|
1963
|
-
jti: this.currentTokenJti ?? null,
|
|
1964
|
-
expiresAt: this.currentTokenExpiresAt ?? null,
|
|
1965
|
-
rootTokenFallback: this.options.rootTokenFallback === true,
|
|
1966
|
-
},
|
|
1967
|
-
...(timelineEvent ? { timelineEvent } : {}),
|
|
1968
|
-
transport: this.bus.transportState,
|
|
1969
|
-
},
|
|
1970
|
-
});
|
|
1971
|
-
}
|
|
1972
|
-
|
|
1973
|
-
private startHttpLiveness(): void {
|
|
1974
|
-
if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
|
|
1975
|
-
void this.publishHttpLiveness();
|
|
1976
|
-
this.httpLivenessTimer = setInterval(() => {
|
|
1977
|
-
void this.publishHttpLiveness();
|
|
1978
|
-
}, HTTP_LIVENESS_INTERVAL_MS);
|
|
1979
|
-
}
|
|
1980
|
-
|
|
1981
|
-
private async publishHttpLiveness(): Promise<void> {
|
|
1982
|
-
if (this.stopped || this.httpLivenessInFlight || this.httpLivenessAuthFailed) return;
|
|
1983
|
-
this.httpLivenessInFlight = true;
|
|
1984
|
-
const status = this.claims.currentStatus();
|
|
1985
|
-
const agentStatus = runnerAgentStatus(status);
|
|
1986
|
-
try {
|
|
1987
|
-
await this.http.setStatus(this.agentId, agentStatus, this.options.instanceId);
|
|
1988
|
-
await this.http.setReady(this.agentId, agentStatus !== "offline", this.options.instanceId);
|
|
1989
|
-
await this.http.heartbeat(this.agentId, this.options.instanceId);
|
|
1990
|
-
await this.bootstrapUnreadMessages();
|
|
1991
|
-
} catch (error) {
|
|
1992
|
-
if (!this.stopped) this.handleHttpLivenessFailure(error);
|
|
1993
|
-
} finally {
|
|
1994
|
-
this.httpLivenessInFlight = false;
|
|
1995
|
-
}
|
|
1996
|
-
}
|
|
1997
|
-
|
|
1998
|
-
private handleHttpLivenessFailure(error: unknown): void {
|
|
1999
|
-
const authFailed = isHttpAuthError(error);
|
|
2000
|
-
this.logHttpLivenessFailure(error, authFailed);
|
|
2001
|
-
if (!authFailed) return;
|
|
2002
|
-
this.httpLivenessAuthFailed = true;
|
|
2003
|
-
if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
|
|
2004
|
-
this.httpLivenessTimer = undefined;
|
|
2005
|
-
// A 401/403 here is the only timely signal that the token died — stopping the
|
|
2006
|
-
// liveness timer means there is no second chance, so recover from THIS failure.
|
|
2007
|
-
this.recoverRuntimeTokenAfterAuthFailure("http-liveness");
|
|
2008
|
-
}
|
|
2009
|
-
|
|
2010
|
-
// A definitive relay auth failure (401/403) means the runtime token is dead right
|
|
2011
|
-
// now — expired, or (the common case) revoked when the relay marked this agent
|
|
2012
|
-
// stale across its own restart/reconnect. The proactive renew timer is keyed to
|
|
2013
|
-
// TTL and structurally cannot catch a revocation, so the auth failure itself must
|
|
2014
|
-
// drive recovery. renewRuntimeToken() prefers an orchestrator re-mint, which heals
|
|
2015
|
-
// an expired token AND a token revoked-as-"stale" (the restart strand, #484) —
|
|
2016
|
-
// but NOT a hard kill (admin revoke / rotation / agent removal), which stays dead
|
|
2017
|
-
// by design. Debounced so a burst of failing calls re-mints once.
|
|
2018
|
-
private recoverRuntimeTokenAfterAuthFailure(source: string): void {
|
|
2019
|
-
if (this.stopped || this.tokenRenewInFlight) return;
|
|
2020
|
-
if (!this.isRuntimeTokenRenewable() && !this.canRemintViaOrchestrator()) return;
|
|
2021
|
-
const now = Date.now();
|
|
2022
|
-
if (this.reactiveTokenRecoveryAt && now - this.reactiveTokenRecoveryAt < REACTIVE_TOKEN_RECOVERY_DEBOUNCE_MS) return;
|
|
2023
|
-
this.reactiveTokenRecoveryAt = now;
|
|
2024
|
-
this.logRunnerDiagnostic(`[runner] relay auth failure on ${source}; recovering runtime token`);
|
|
2025
|
-
void this.renewRuntimeToken();
|
|
2026
|
-
}
|
|
2027
|
-
|
|
2028
|
-
private logHttpLivenessFailure(error: unknown, authFailed: boolean): void {
|
|
2029
|
-
const key = httpErrorKey(error);
|
|
2030
|
-
const now = Date.now();
|
|
2031
|
-
if (
|
|
2032
|
-
this.httpLivenessLastLog?.key === key &&
|
|
2033
|
-
now - this.httpLivenessLastLog.at < HTTP_LIVENESS_LOG_INTERVAL_MS
|
|
2034
|
-
) return;
|
|
2035
|
-
this.httpLivenessLastLog = { key, at: now };
|
|
2036
|
-
const suffix = authFailed ? "auth failed; stopping HTTP liveness retries until restart" : String(error);
|
|
2037
|
-
this.logRunnerDiagnostic(`[runner] HTTP liveness update failed: ${suffix}`);
|
|
2038
|
-
}
|
|
2039
|
-
|
|
2040
|
-
// Runner operational diagnostics (HTTP liveness, token renewal failures). Routed
|
|
2041
|
-
// through the leveled logger at warn — see logger.ts. Kept as a thin wrapper so
|
|
2042
|
-
// the existing call sites and their `[runner]` framing stay put.
|
|
2043
|
-
private logRunnerDiagnostic(message: string): void {
|
|
2044
|
-
logger.warn("runner", message.replace(/^\[runner\]\s*/, ""));
|
|
2045
|
-
}
|
|
2046
|
-
|
|
2047
|
-
// Session-mirror diagnostics → the leveled logger (component "mirror"), written
|
|
2048
|
-
// to the dashboard-surfaced session-mirror-<agent>.log. Key transitions log at
|
|
2049
|
-
// info; the single place to look when chat/terminal sync misbehaves.
|
|
2050
|
-
private sessionLog(message: string): void {
|
|
2051
|
-
logger.info("mirror", message);
|
|
2052
|
-
}
|
|
2053
|
-
|
|
2054
|
-
// Verbose, high-frequency lines (per-probe, per-emit) — surfaced only at log
|
|
2055
|
-
// level "debug" (AGENT_RELAY_LOG_LEVEL=debug, or flip live via /log-level).
|
|
2056
|
-
private sessionDebug(message: string): void {
|
|
2057
|
-
logger.debug("mirror", message);
|
|
2058
|
-
}
|
|
2059
|
-
|
|
2060
|
-
private ensureScratch(): void {
|
|
2061
|
-
try {
|
|
2062
|
-
this.scratch = ensureSessionScratch({
|
|
2063
|
-
agentId: this.agentId,
|
|
2064
|
-
cwd: this.options.cwd,
|
|
2065
|
-
fallbackBaseDir: process.env.AGENT_RELAY_ORCHESTRATOR_BASE_DIR,
|
|
2066
|
-
});
|
|
2067
|
-
} catch (error) {
|
|
2068
|
-
this.logRunnerDiagnostic(`session scratch setup failed: ${errMessage(error)}`);
|
|
2069
|
-
}
|
|
2070
|
-
}
|
|
2071
|
-
|
|
2072
|
-
private async sweepStaleScratch(): Promise<void> {
|
|
2073
|
-
try {
|
|
2074
|
-
const agents = await this.http.listAgents().catch(() => [] as Awaited<ReturnType<RelayHttpClient["listAgents"]>>);
|
|
2075
|
-
const keep = new Set(agents.map((a) => a.id));
|
|
2076
|
-
keep.add(this.agentId);
|
|
2077
|
-
const removed = sweepStaleSessions({
|
|
2078
|
-
cwd: this.options.cwd,
|
|
2079
|
-
fallbackBaseDir: process.env.AGENT_RELAY_ORCHESTRATOR_BASE_DIR,
|
|
2080
|
-
keepAgentIds: keep,
|
|
2081
|
-
now: Date.now(),
|
|
2082
|
-
});
|
|
2083
|
-
if (removed.length) this.logRunnerDiagnostic(`swept ${removed.length} stale session scratch dir(s)`);
|
|
2084
|
-
} catch {}
|
|
2085
|
-
}
|
|
2086
|
-
|
|
2087
|
-
private scheduleRuntimeTokenRenewal(delayMs?: number): void {
|
|
2088
|
-
if (this.tokenRenewTimer) clearTimeout(this.tokenRenewTimer);
|
|
2089
|
-
this.tokenRenewTimer = undefined;
|
|
2090
|
-
if (this.stopped) return;
|
|
2091
|
-
const canSelfRenew = this.isRuntimeTokenRenewable();
|
|
2092
|
-
const canRemint = this.canRemintViaOrchestrator();
|
|
2093
|
-
// Keep the renewal clock ticking as long as the session can recover its token
|
|
2094
|
-
// by EITHER path. Without the re-mint fallback an expired token would stop the
|
|
2095
|
-
// timer forever (the old deadlock that stranded live agents off the bus).
|
|
2096
|
-
if (!canSelfRenew && !canRemint) return;
|
|
2097
|
-
let computedDelay = delayMs;
|
|
2098
|
-
if (computedDelay === undefined) {
|
|
2099
|
-
computedDelay = canSelfRenew
|
|
2100
|
-
? runtimeTokenRenewDelayMs(this.currentTokenExpiresAt!, Date.now())
|
|
2101
|
-
: TOKEN_RENEW_RETRY_MS; // expired but re-mintable → retry via orchestrator soon
|
|
2102
|
-
if (computedDelay === undefined) computedDelay = TOKEN_RENEW_RETRY_MS;
|
|
2103
|
-
}
|
|
2104
|
-
const schedule = runtimeTokenRenewTimerSchedule(computedDelay);
|
|
2105
|
-
if (!schedule) return;
|
|
2106
|
-
this.tokenRenewTimer = setTimeout(() => {
|
|
2107
|
-
this.tokenRenewTimer = undefined;
|
|
2108
|
-
if (!schedule.renew) {
|
|
2109
|
-
this.scheduleRuntimeTokenRenewal();
|
|
2110
|
-
return;
|
|
2111
|
-
}
|
|
2112
|
-
void this.renewRuntimeToken();
|
|
2113
|
-
}, schedule.delayMs);
|
|
2114
|
-
}
|
|
2115
|
-
|
|
2116
|
-
// Can the runner self-renew right now? Requires a non-expired runner-profile token
|
|
2117
|
-
// (the relay rejects renewal of an expired token).
|
|
2118
|
-
private isRuntimeTokenRenewable(): boolean {
|
|
2119
|
-
return Boolean(
|
|
2120
|
-
this.currentToken &&
|
|
2121
|
-
this.currentTokenExpiresAt &&
|
|
2122
|
-
this.currentTokenExpiresAt * 1000 > Date.now() &&
|
|
2123
|
-
(this.currentTokenProfileId === "provider-agent" || this.currentTokenProfileId === "provider-interactive"),
|
|
2124
|
-
);
|
|
2125
|
-
}
|
|
2126
|
-
|
|
2127
|
-
// Can the runner recover its token via the orchestrator? Works even when the token
|
|
2128
|
-
// is already expired — the orchestrator's standing credential is the authority.
|
|
2129
|
-
private canRemintViaOrchestrator(): boolean {
|
|
2130
|
-
return Boolean(
|
|
2131
|
-
process.env.AGENT_RELAY_ORCHESTRATOR_URL &&
|
|
2132
|
-
this.currentToken &&
|
|
2133
|
-
(this.currentTokenProfileId === "provider-agent" || this.currentTokenProfileId === "provider-interactive"),
|
|
2134
|
-
);
|
|
2135
|
-
}
|
|
2136
|
-
|
|
2137
|
-
private async renewRuntimeToken(): Promise<void> {
|
|
2138
|
-
if (this.stopped || this.tokenRenewInFlight || !this.currentToken) return;
|
|
2139
|
-
this.tokenRenewInFlight = true;
|
|
2140
|
-
try {
|
|
2141
|
-
// Preferred path: self-renew directly against the relay while the token is
|
|
2142
|
-
// still valid. Cheapest and needs no orchestrator round-trip.
|
|
2143
|
-
if (this.isRuntimeTokenRenewable()) {
|
|
2144
|
-
try {
|
|
2145
|
-
const renewed = await this.http.renewRuntimeToken();
|
|
2146
|
-
this.applyRenewedToken(renewed.token, renewed.record, "runtime-token-renewed");
|
|
2147
|
-
return;
|
|
2148
|
-
} catch (error) {
|
|
2149
|
-
this.logRuntimeTokenRenewalFailure(error);
|
|
2150
|
-
// Relay unreachable or token rejected — fall through to orchestrator re-mint.
|
|
2151
|
-
}
|
|
2152
|
-
}
|
|
2153
|
-
// Recovery path: token expired, or self-renew failed. Ask the orchestrator —
|
|
2154
|
-
// it holds a long-lived credential and can mint a fresh runner token, so a
|
|
2155
|
-
// live session heals instead of being stranded off the bus.
|
|
2156
|
-
if (this.canRemintViaOrchestrator() && await this.remintViaOrchestrator()) return;
|
|
2157
|
-
this.pendingTimelineEvent = {
|
|
2158
|
-
status: "runtime-token-renewal-failed",
|
|
2159
|
-
timestamp: Date.now(),
|
|
2160
|
-
};
|
|
2161
|
-
this.publishStatus();
|
|
2162
|
-
this.scheduleRuntimeTokenRenewal(TOKEN_RENEW_RETRY_MS);
|
|
2163
|
-
} finally {
|
|
2164
|
-
this.tokenRenewInFlight = false;
|
|
2165
|
-
}
|
|
2166
|
-
}
|
|
2167
|
-
|
|
2168
|
-
// Apply a freshly issued token across every live surface — runner state, the
|
|
2169
|
-
// RunnerOptions bag (re-injected into the provider on respawn), the HTTP client,
|
|
2170
|
-
// the bus client — then force a bus handshake with the new token and reschedule.
|
|
2171
|
-
private applyRenewedToken(
|
|
2172
|
-
token: string,
|
|
2173
|
-
record: { jti: string; profileId?: string; expiresAt?: number },
|
|
2174
|
-
status: "runtime-token-renewed" | "runtime-token-reminted",
|
|
2175
|
-
): void {
|
|
2176
|
-
this.currentToken = token;
|
|
2177
|
-
this.currentTokenJti = record.jti;
|
|
2178
|
-
this.currentTokenProfileId = record.profileId ?? this.currentTokenProfileId;
|
|
2179
|
-
this.currentTokenExpiresAt = record.expiresAt;
|
|
2180
|
-
this.options.token = token;
|
|
2181
|
-
this.options.tokenJti = record.jti;
|
|
2182
|
-
this.options.tokenProfileId = this.currentTokenProfileId;
|
|
2183
|
-
this.options.tokenExpiresAt = this.currentTokenExpiresAt;
|
|
2184
|
-
this.http.setToken(token);
|
|
2185
|
-
this.bus.setToken(token);
|
|
2186
|
-
// The proxy reads the token live via getToken(), so forwarding already uses the new one.
|
|
2187
|
-
// A re-mint can change scope (e.g. a profile change), so refresh the relay tool list and
|
|
2188
|
-
// emit tools/list_changed if the visible set changed (#215 — token-scope transition).
|
|
2189
|
-
void this.proxy?.refreshTools().catch(() => {});
|
|
2190
|
-
this.httpLivenessAuthFailed = false;
|
|
2191
|
-
this.reactiveTokenRecoveryAt = undefined;
|
|
2192
|
-
// An earlier auth failure may have stopped the liveness loop; restart it so the
|
|
2193
|
-
// agent reports live again on the fresh token. startHttpLiveness clears any
|
|
2194
|
-
// existing timer first, so this is safe on the normal (proactive) renew path too.
|
|
2195
|
-
this.startHttpLiveness();
|
|
2196
|
-
this.pendingTimelineEvent = { status, id: record.jti, timestamp: Date.now() };
|
|
2197
|
-
this.bus.reconnectTransport(status === "runtime-token-reminted" ? "runtime token re-minted" : "runtime token renewed");
|
|
2198
|
-
this.publishStatus();
|
|
2199
|
-
this.scheduleRuntimeTokenRenewal();
|
|
2200
|
-
}
|
|
2201
|
-
|
|
2202
|
-
// Recover the runtime token through the orchestrator. The runner proxies its own
|
|
2203
|
-
// (possibly expired) token; the orchestrator re-mints it via the relay using its
|
|
2204
|
-
// standing credential. Returns true on success.
|
|
2205
|
-
private async remintViaOrchestrator(): Promise<boolean> {
|
|
2206
|
-
const orchUrl = process.env.AGENT_RELAY_ORCHESTRATOR_URL;
|
|
2207
|
-
if (!orchUrl || !this.currentToken) return false;
|
|
2208
|
-
try {
|
|
2209
|
-
const res = await fetch(`${orchUrl.replace(/\/+$/, "")}/api/runtime-tokens/runner-renew`, {
|
|
2210
|
-
method: "POST",
|
|
2211
|
-
headers: { "Content-Type": "application/json" },
|
|
2212
|
-
body: JSON.stringify({ token: this.currentToken }),
|
|
2213
|
-
signal: AbortSignal.timeout(10_000),
|
|
2214
|
-
});
|
|
2215
|
-
if (!res.ok) return false;
|
|
2216
|
-
const renewed = await res.json() as { token?: string; record?: { jti: string; profileId?: string; expiresAt?: number } };
|
|
2217
|
-
if (!renewed?.token || !renewed.record) return false;
|
|
2218
|
-
this.applyRenewedToken(renewed.token, renewed.record, "runtime-token-reminted");
|
|
2219
|
-
this.logRunnerDiagnostic(`[runner] runtime token re-minted via orchestrator (jti ${renewed.record.jti})`);
|
|
2220
|
-
return true;
|
|
2221
|
-
} catch (error) {
|
|
2222
|
-
this.logRuntimeTokenRenewalFailure(error);
|
|
2223
|
-
return false;
|
|
2224
|
-
}
|
|
2225
|
-
}
|
|
2226
|
-
|
|
2227
|
-
private logRuntimeTokenRenewalFailure(error: unknown): void {
|
|
2228
|
-
const key = httpErrorKey(error);
|
|
2229
|
-
const now = Date.now();
|
|
2230
|
-
if (
|
|
2231
|
-
this.tokenRenewLastLog?.key === key &&
|
|
2232
|
-
now - this.tokenRenewLastLog.at < HTTP_LIVENESS_LOG_INTERVAL_MS
|
|
2233
|
-
) return;
|
|
2234
|
-
this.tokenRenewLastLog = { key, at: now };
|
|
2235
|
-
this.logRunnerDiagnostic(`[runner] runtime token renewal failed: ${String(error)}`);
|
|
2236
|
-
}
|
|
2237
|
-
|
|
2238
|
-
private heartbeatRuntimeMeta(): Record<string, unknown> | undefined {
|
|
2239
|
-
const processContext = this.latestProcessContext();
|
|
2240
|
-
const probeMetrics = this.latestProbeMetrics();
|
|
2241
|
-
const probeContext = probeMetrics ? contextStateFromProbeMetrics(probeMetrics) : undefined;
|
|
2242
|
-
const context = processContext ?? probeContext;
|
|
2243
|
-
const terminalSession = this.providerTerminalSession();
|
|
2244
|
-
const terminalSocket = this.providerTerminalSocket();
|
|
2245
|
-
const probeModel: ProbeModelInfo | undefined = probeMetrics?.model || probeMetrics?.effort
|
|
2246
|
-
? { model: probeMetrics.model, effort: probeMetrics.effort }
|
|
2247
|
-
: undefined;
|
|
2248
|
-
const meta: Record<string, unknown> = {
|
|
2249
|
-
providerCapabilities: runtimeProviderCapabilities(
|
|
2250
|
-
this.options,
|
|
2251
|
-
context,
|
|
2252
|
-
probeModel,
|
|
2253
|
-
),
|
|
2254
|
-
...(terminalSession ? { tmuxSession: terminalSession } : {}),
|
|
2255
|
-
...(terminalSocket ? { tmuxSocket: terminalSocket } : {}),
|
|
2256
|
-
};
|
|
2257
|
-
if (context) meta.context = context;
|
|
2258
|
-
return meta;
|
|
2259
|
-
}
|
|
2260
|
-
|
|
2261
|
-
private providerTerminalSession(): string | undefined {
|
|
2262
|
-
return typeof this.process?.meta?.tmuxSession === "string" ? this.process.meta.tmuxSession : undefined;
|
|
2263
|
-
}
|
|
2264
|
-
|
|
2265
|
-
private providerTerminalSocket(): string | undefined {
|
|
2266
|
-
return typeof this.process?.meta?.tmuxSocket === "string" ? this.process.meta.tmuxSocket : undefined;
|
|
2267
|
-
}
|
|
2268
|
-
|
|
2269
|
-
private latestProcessContext(): ContextState | undefined {
|
|
2270
|
-
const getContext = this.process?.meta?.getContext;
|
|
2271
|
-
if (typeof getContext === "function") {
|
|
2272
|
-
const context = getContext();
|
|
2273
|
-
if (isContextState(context)) return context;
|
|
2274
|
-
}
|
|
2275
|
-
const context = this.process?.meta?.context;
|
|
2276
|
-
return isContextState(context) ? context : undefined;
|
|
2277
|
-
}
|
|
2278
|
-
|
|
2279
|
-
private latestProbeMetrics() {
|
|
2280
|
-
const stateDir = process.env.AGENT_RELAY_CONTEXT_STATE_DIR;
|
|
2281
|
-
return readContextProbeState(this.agentId, stateDir);
|
|
2282
|
-
}
|
|
2283
|
-
|
|
2284
|
-
private matchesMessage(message: Message): boolean {
|
|
2285
|
-
return runnerMessageMatches(message, {
|
|
2286
|
-
agentId: this.agentId,
|
|
2287
|
-
label: this.options.label,
|
|
2288
|
-
provider: this.options.provider,
|
|
2289
|
-
tags: this.options.tags,
|
|
2290
|
-
capabilities: this.options.capabilities,
|
|
2291
|
-
defaultTags: this.options.providerConfig.defaultTags,
|
|
2292
|
-
defaultCapabilities: this.options.providerConfig.defaultCapabilities,
|
|
2293
|
-
allowBroadTargets: this.options.agentProfile?.relay?.context !== false,
|
|
2294
|
-
});
|
|
2295
|
-
}
|
|
2296
|
-
|
|
2297
|
-
private startClaimRenewer(): void {
|
|
2298
|
-
if (this.claimRenewTimer) return;
|
|
2299
|
-
this.claimRenewTimer = setInterval(() => {
|
|
2300
|
-
void this.renewActiveTaskClaims();
|
|
2301
|
-
}, CLAIM_RENEW_INTERVAL_MS);
|
|
2302
|
-
}
|
|
2303
|
-
|
|
2304
|
-
private async renewActiveTaskClaims(): Promise<void> {
|
|
2305
|
-
for (const claim of this.activeTaskClaims.values()) {
|
|
2306
|
-
const renewed = await this.http.renewMessageClaim(claim.messageId, this.agentId).catch(() => ({ ok: false, claimExpiresAt: undefined }));
|
|
2307
|
-
if (renewed.ok) {
|
|
2308
|
-
this.claims.startClaim("message", String(claim.messageId), renewed.claimExpiresAt);
|
|
2309
|
-
this.claims.startClaim("task", String(claim.taskId), renewed.claimExpiresAt);
|
|
2310
|
-
}
|
|
2311
|
-
}
|
|
2312
|
-
this.publishStatus();
|
|
2313
|
-
}
|
|
2314
|
-
|
|
2315
|
-
private async completeObservedTaskClaims(): Promise<void> {
|
|
2316
|
-
const completed = [...this.activeTaskClaims.values()].filter((claim) => claim.observedProviderBusy);
|
|
2317
|
-
for (const claim of completed) {
|
|
2318
|
-
const ok = await this.updateTaskStatus(claim.taskId, {
|
|
2319
|
-
status: "done",
|
|
2320
|
-
agentId: this.agentId,
|
|
2321
|
-
body: `Runner observed provider completion for message #${claim.messageId}`,
|
|
2322
|
-
metadata: { messageId: claim.messageId, completedBy: "runner" },
|
|
2323
|
-
})
|
|
2324
|
-
.then(() => true)
|
|
2325
|
-
.catch((error) => {
|
|
2326
|
-
logger.error("task", `task ${claim.taskId} completion update failed: ${error}`);
|
|
2327
|
-
return false;
|
|
2328
|
-
});
|
|
2329
|
-
if (!ok) continue;
|
|
2330
|
-
this.activeTaskClaims.delete(claim.messageId);
|
|
2331
|
-
this.claims.finishClaim("message", String(claim.messageId));
|
|
2332
|
-
this.claims.finishClaim("task", String(claim.taskId));
|
|
2333
|
-
}
|
|
2334
|
-
if (completed.length > 0) this.publishStatus();
|
|
2335
|
-
}
|
|
2336
|
-
|
|
2337
|
-
private async updateTaskStatus(taskId: number, input: TaskStatusInput): Promise<void> {
|
|
2338
|
-
await this.http.updateTaskStatus(taskId, input);
|
|
2339
|
-
}
|
|
2340
|
-
|
|
2341
|
-
private clearActiveClaim(message: Message): void {
|
|
2342
|
-
const taskId = taskIdFromMessage(message);
|
|
2343
|
-
this.claims.finishClaim("message", String(message.id));
|
|
2344
|
-
if (!taskId) return;
|
|
2345
|
-
this.activeTaskClaims.delete(message.id);
|
|
2346
|
-
this.claims.finishClaim("task", String(taskId));
|
|
2347
|
-
}
|
|
2348
|
-
}
|
|
2349
|
-
|
|
2350
|
-
export function runnerMessageMatches(message: Pick<Message, "to" | "resolvedToAgent">, target: {
|
|
2351
|
-
agentId: string;
|
|
2352
|
-
label?: string;
|
|
2353
|
-
provider: string;
|
|
2354
|
-
tags: string[];
|
|
2355
|
-
capabilities: string[];
|
|
2356
|
-
defaultTags: string[];
|
|
2357
|
-
defaultCapabilities: string[];
|
|
2358
|
-
allowBroadTargets?: boolean;
|
|
2359
|
-
}): boolean {
|
|
2360
|
-
if (message.resolvedToAgent === target.agentId) return true;
|
|
2361
|
-
if (message.to === target.agentId) return true;
|
|
2362
|
-
if (target.label && message.to === `label:${target.label}`) return true;
|
|
2363
|
-
if (target.allowBroadTargets === false) return false;
|
|
2364
|
-
if (message.to === "broadcast") return true;
|
|
2365
|
-
if (message.to.startsWith("tag:")) {
|
|
2366
|
-
const tag = message.to.slice("tag:".length);
|
|
2367
|
-
return target.tags.includes(tag) || target.defaultTags.includes(tag) || tag === target.provider;
|
|
2368
|
-
}
|
|
2369
|
-
if (message.to.startsWith("cap:")) {
|
|
2370
|
-
const cap = message.to.slice("cap:".length);
|
|
2371
|
-
return target.capabilities.includes(cap) || target.defaultCapabilities.includes(cap);
|
|
2372
|
-
}
|
|
2373
|
-
return false;
|
|
2374
|
-
}
|
|
2375
|
-
|
|
2376
|
-
export function taskIdFromMessage(message: Pick<Message, "payload">): number | undefined {
|
|
2377
|
-
const taskId = message.payload?.taskId;
|
|
2378
|
-
return Number.isSafeInteger(taskId) ? taskId as number : undefined;
|
|
2379
|
-
}
|
|
2380
|
-
|
|
2381
|
-
// The server appends a self-claim/status instruction to task message bodies
|
|
2382
|
-
// (taskMessageBody in src/db.ts: "Claim this task before working it, then update
|
|
2383
|
-
// task status when finished."). For runner-managed agents the runner already owns
|
|
2384
|
-
// claim + status, so that line is not only redundant but actively harmful: the
|
|
2385
|
-
// agent tries to follow it, has no real claim command, and improvises a stray
|
|
2386
|
-
// `agent-relay /send-claimable claim <id>` that leaves an orphan claimable message.
|
|
2387
|
-
// Strip the trailing guidance from messages the runner has auto-claimed.
|
|
2388
|
-
const RUNNER_CLAIMED_GUIDANCE_RE = /\s*Claim this task before working it, then update task status when finished\.\s*$/;
|
|
2389
|
-
|
|
2390
|
-
export function stripRunnerClaimedGuidance(body: string): string {
|
|
2391
|
-
return body.replace(RUNNER_CLAIMED_GUIDANCE_RE, "");
|
|
2392
|
-
}
|
|
2393
|
-
|
|
2394
|
-
function csvTags(raw: string | undefined): string[] {
|
|
2395
|
-
return (raw || "").split(",").map((tag) => tag.trim()).filter(Boolean);
|
|
2396
|
-
}
|
|
2397
|
-
|
|
2398
|
-
function relayBusUrl(relayUrl: string): string {
|
|
2399
|
-
const url = new URL(relayUrl);
|
|
2400
|
-
url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
|
|
2401
|
-
url.pathname = `${url.pathname.replace(/\/+$/, "")}/bus`;
|
|
2402
|
-
url.search = "";
|
|
2403
|
-
return url.toString();
|
|
2404
|
-
}
|
|
2405
|
-
|
|
2406
|
-
export function runnerAgentStatus(status: "idle" | "busy" | "offline" | "error"): "idle" | "busy" | "offline" {
|
|
2407
|
-
return status === "error" ? "offline" : status;
|
|
2408
|
-
}
|
|
2409
|
-
|
|
2410
|
-
export function runnerBusErrorAction(code: string, stopped: boolean): "ignore" | "log" | "stop" {
|
|
2411
|
-
if (code === "STALE_SESSION") return stopped ? "ignore" : "stop";
|
|
2412
|
-
return "log";
|
|
2413
|
-
}
|
|
2414
|
-
|
|
2415
|
-
export function runnerShouldResolveProviderExit(status: SemanticStatus, exitCommandInProgress: boolean): boolean {
|
|
2416
|
-
return !exitCommandInProgress && (status === "offline" || status === "error");
|
|
2417
|
-
}
|
|
2418
|
-
|
|
2419
|
-
export function runnerShouldRestartUnexpectedProviderExit(
|
|
2420
|
-
status: SemanticStatus,
|
|
2421
|
-
input: {
|
|
2422
|
-
exitCommandInProgress: boolean;
|
|
2423
|
-
stopped: boolean;
|
|
2424
|
-
restartInProgress: boolean;
|
|
2425
|
-
provider?: string;
|
|
2426
|
-
headless: boolean;
|
|
2427
|
-
hasTerminalSession: boolean;
|
|
2428
|
-
},
|
|
2429
|
-
): boolean {
|
|
2430
|
-
return status === "offline"
|
|
2431
|
-
&& !input.exitCommandInProgress
|
|
2432
|
-
&& !input.stopped
|
|
2433
|
-
&& !input.restartInProgress
|
|
2434
|
-
&& input.headless
|
|
2435
|
-
&& input.hasTerminalSession;
|
|
2436
|
-
}
|
|
2437
|
-
|
|
2438
|
-
export function latestClaudeResumeIdFromText(text: string): string | undefined {
|
|
2439
|
-
let latest: string | undefined;
|
|
2440
|
-
CLAUDE_RESUME_RE.lastIndex = 0;
|
|
2441
|
-
for (let match = CLAUDE_RESUME_RE.exec(text); match; match = CLAUDE_RESUME_RE.exec(text)) {
|
|
2442
|
-
latest = match[1];
|
|
2443
|
-
}
|
|
2444
|
-
return latest;
|
|
2445
|
-
}
|
|
2446
|
-
|
|
2447
|
-
function latestClaudeResumeIdFromLogFile(path: string): string | undefined {
|
|
2448
|
-
let fd: number | undefined;
|
|
2449
|
-
try {
|
|
2450
|
-
const stat = statSync(path);
|
|
2451
|
-
const length = Math.min(stat.size, LOG_TAIL_BYTES);
|
|
2452
|
-
const offset = Math.max(0, stat.size - length);
|
|
2453
|
-
const buffer = Buffer.alloc(length);
|
|
2454
|
-
fd = openSync(path, "r");
|
|
2455
|
-
readSync(fd, buffer, 0, length, offset);
|
|
2456
|
-
return latestClaudeResumeIdFromText(buffer.toString("utf8"));
|
|
2457
|
-
} catch {
|
|
2458
|
-
return undefined;
|
|
2459
|
-
} finally {
|
|
2460
|
-
if (fd !== undefined) closeSync(fd);
|
|
2461
|
-
}
|
|
2462
|
-
}
|
|
2463
|
-
|
|
2464
|
-
function commandTimeoutMs(params: Record<string, unknown>, fallback = 10_000): number {
|
|
2465
|
-
const raw = params.timeoutMs;
|
|
2466
|
-
if (typeof raw !== "number" || !Number.isSafeInteger(raw) || raw <= 0) return fallback;
|
|
2467
|
-
return Math.min(raw, 60_000);
|
|
2468
|
-
}
|
|
2469
|
-
|
|
2470
|
-
export function shouldLogDeliveryFailure(error: unknown): boolean {
|
|
2471
|
-
const message = errMessage(error);
|
|
2472
|
-
return message !== "no Claude monitor connected";
|
|
2473
|
-
}
|
|
2474
|
-
|
|
2475
|
-
export function runtimeTokenRenewDelayMs(expiresAtSeconds: number, nowMs = Date.now()): number | undefined {
|
|
2476
|
-
const expiresAtMs = expiresAtSeconds * 1000;
|
|
2477
|
-
const ttlMs = expiresAtMs - nowMs;
|
|
2478
|
-
if (!Number.isFinite(ttlMs) || ttlMs <= 0) return undefined;
|
|
2479
|
-
const leadMs = Math.min(60 * 60 * 1000, Math.max(60_000, Math.floor(ttlMs * 0.2)));
|
|
2480
|
-
const jitterMs = Math.floor(Math.random() * Math.min(30_000, Math.max(0, leadMs / 2)));
|
|
2481
|
-
return Math.max(1_000, ttlMs - leadMs - jitterMs);
|
|
2482
|
-
}
|
|
2483
|
-
|
|
2484
|
-
export function runtimeTokenRenewTimerSchedule(delayMs: number): { delayMs: number; renew: boolean } | undefined {
|
|
2485
|
-
if (!Number.isFinite(delayMs) || delayMs <= 0) return undefined;
|
|
2486
|
-
if (delayMs > MAX_TIMER_DELAY_MS) return { delayMs: MAX_TIMER_DELAY_MS, renew: false };
|
|
2487
|
-
return { delayMs, renew: true };
|
|
2488
|
-
}
|
|
2489
|
-
|
|
2490
|
-
function lifecycleCapabilities(): Record<string, true> {
|
|
2491
|
-
return {
|
|
2492
|
-
shutdownHard: true,
|
|
2493
|
-
restartHard: true,
|
|
2494
|
-
semanticStatus: true,
|
|
2495
|
-
transportReconnect: true,
|
|
2496
|
-
};
|
|
2497
|
-
}
|
|
2498
|
-
|
|
2499
|
-
interface ProbeModelInfo {
|
|
2500
|
-
model?: string;
|
|
2501
|
-
effort?: string;
|
|
2502
|
-
}
|
|
2503
|
-
|
|
2504
|
-
function runtimeProviderCapabilities(options: RunnerOptions, contextState?: ContextState, probeModel?: ProbeModelInfo): ProviderCapabilities {
|
|
2505
|
-
const model = options.model ?? probeModel?.model;
|
|
2506
|
-
const effort = options.effort ?? probeModel?.effort;
|
|
2507
|
-
const modelSource = options.model ? "runtime" as const : probeModel?.model ? "provider" as const : "runtime" as const;
|
|
2508
|
-
const shellMode = runtimeShellMode(options);
|
|
2509
|
-
return {
|
|
2510
|
-
lifecycle: {
|
|
2511
|
-
managed: true,
|
|
2512
|
-
shutdownHard: options.provider === "claude" ? options.headless : true,
|
|
2513
|
-
restartHard: options.provider === "claude" ? options.headless : true,
|
|
2514
|
-
semanticStatus: true,
|
|
2515
|
-
reconnect: true,
|
|
2516
|
-
},
|
|
2517
|
-
model: {
|
|
2518
|
-
provider: options.provider,
|
|
2519
|
-
alias: model,
|
|
2520
|
-
id: model,
|
|
2521
|
-
effort,
|
|
2522
|
-
source: modelSource,
|
|
2523
|
-
confidence: options.model ? "reported" : probeModel?.model ? "reported" : "unknown",
|
|
2524
|
-
lastUpdatedAt: options.startedAt,
|
|
2525
|
-
},
|
|
2526
|
-
session: {
|
|
2527
|
-
approvalMode: options.approvalMode,
|
|
2528
|
-
fileRead: true,
|
|
2529
|
-
fileWrite: options.approvalMode !== "read-only",
|
|
2530
|
-
shell: shellMode !== "none",
|
|
2531
|
-
shellMode,
|
|
2532
|
-
source: "runtime",
|
|
2533
|
-
confidence: "reported",
|
|
2534
|
-
lastUpdatedAt: options.startedAt,
|
|
2535
|
-
},
|
|
2536
|
-
...runtimeProviderContextCapabilities(options, contextState),
|
|
2537
|
-
...runtimeProviderTerminalCapabilities(options),
|
|
2538
|
-
liveSession: {
|
|
2539
|
-
capture: true,
|
|
2540
|
-
inject: Boolean(options.adapter.deliverInitialPrompt),
|
|
2541
|
-
interrupt: Boolean(options.adapter.interrupt),
|
|
2542
|
-
// Both providers mirror directly-typed prompts and stream reasoning/tool
|
|
2543
|
-
// activity into chat (Claude via hooks + transcript tail, Codex via
|
|
2544
|
-
// app-server item events).
|
|
2545
|
-
promptEcho: true,
|
|
2546
|
-
reasoning: true,
|
|
2547
|
-
slashCommands: options.provider === "claude" || options.provider === "codex",
|
|
2548
|
-
},
|
|
2549
|
-
source: "runtime",
|
|
2550
|
-
confidence: "reported",
|
|
2551
|
-
lastUpdatedAt: options.startedAt,
|
|
2552
|
-
};
|
|
2553
|
-
}
|
|
2554
|
-
|
|
2555
|
-
function runtimeShellMode(options: RunnerOptions): NonNullable<NonNullable<ProviderCapabilities["session"]>["shellMode"]> {
|
|
2556
|
-
if (options.approvalMode === "read-only") {
|
|
2557
|
-
return options.provider === "claude" ? "read-only-guarded" : "none";
|
|
2558
|
-
}
|
|
2559
|
-
return options.approvalMode === "open" ? "unrestricted" : "guarded";
|
|
2560
|
-
}
|
|
2561
|
-
|
|
2562
|
-
function runtimeProviderTerminalCapabilities(options: RunnerOptions): Pick<ProviderCapabilities, "terminal"> {
|
|
2563
|
-
if (options.provider === "claude" && options.headless) {
|
|
2564
|
-
return {
|
|
2565
|
-
terminal: {
|
|
2566
|
-
live: {
|
|
2567
|
-
read: true,
|
|
2568
|
-
write: true,
|
|
2569
|
-
},
|
|
2570
|
-
},
|
|
2571
|
-
};
|
|
2572
|
-
}
|
|
2573
|
-
if (options.provider === "codex") {
|
|
2574
|
-
return {
|
|
2575
|
-
terminal: {
|
|
2576
|
-
attach: {
|
|
2577
|
-
create: true,
|
|
2578
|
-
read: true,
|
|
2579
|
-
write: true,
|
|
2580
|
-
detach: true,
|
|
2581
|
-
},
|
|
2582
|
-
},
|
|
2583
|
-
};
|
|
2584
|
-
}
|
|
2585
|
-
return {};
|
|
2586
|
-
}
|
|
2587
|
-
|
|
2588
|
-
function appliedAgentProfileMetadata(provider: string, profile: AgentProfile): Record<string, unknown> {
|
|
2589
|
-
const projection = provider === "claude" || provider === "codex"
|
|
2590
|
-
? agentProfileProjectionReport({ provider, profile })
|
|
2591
|
-
: undefined;
|
|
2592
|
-
return {
|
|
2593
|
-
name: profile.name,
|
|
2594
|
-
base: profile.base,
|
|
2595
|
-
provider: profile.provider ?? "any",
|
|
2596
|
-
relay: profile.relay,
|
|
2597
|
-
instructions: {
|
|
2598
|
-
repoInstructions: profile.instructions.repoInstructions,
|
|
2599
|
-
globalInstructions: profile.instructions.globalInstructions,
|
|
2600
|
-
appendCount: profile.instructions.append.length,
|
|
2601
|
-
hasSystem: Boolean(profile.instructions.system),
|
|
2602
|
-
},
|
|
2603
|
-
skills: profile.skills.filter((item) => item.enabled).length,
|
|
2604
|
-
plugins: profile.plugins.filter((item) => item.enabled).length,
|
|
2605
|
-
mcp: { mode: profile.mcp.mode },
|
|
2606
|
-
hooks: { mode: profile.hooks.mode },
|
|
2607
|
-
permissions: profile.permissions,
|
|
2608
|
-
projection,
|
|
2609
|
-
};
|
|
2610
|
-
}
|
|
2611
|
-
|
|
2612
|
-
function runtimeProviderContextCapabilities(options: RunnerOptions, contextState?: ContextState): Pick<ProviderCapabilities, "context"> {
|
|
2613
|
-
const context: NonNullable<ProviderCapabilities["context"]> = { resume: "none" };
|
|
2614
|
-
const supportsManagedContext = options.provider === "codex" || (options.provider === "claude" && options.headless);
|
|
2615
|
-
if (contextState) {
|
|
2616
|
-
context.stats = { source: contextState.source, confidence: contextState.confidence };
|
|
2617
|
-
if (typeof contextState.tokensMax === "number" && Number.isFinite(contextState.tokensMax)) {
|
|
2618
|
-
context.windowTokens = contextState.tokensMax;
|
|
2619
|
-
}
|
|
2620
|
-
}
|
|
2621
|
-
if (supportsManagedContext) {
|
|
2622
|
-
context.compact = true;
|
|
2623
|
-
context.clear = true;
|
|
2624
|
-
}
|
|
2625
|
-
context.inject = true;
|
|
2626
|
-
if (supportsManagedContext && options.adapter.compact && options.adapter.compactSupportsInstructions) context.resume = "native";
|
|
2627
|
-
else if (supportsManagedContext && options.adapter.clearContext) context.resume = "clear-inject";
|
|
2628
|
-
return Object.keys(context).length ? { context } : {};
|
|
2629
|
-
}
|
|
2630
|
-
|
|
2631
|
-
function providerStateFromActiveWork(activeWork: Array<{ kind: string; metadata?: Record<string, unknown> }>): Record<string, unknown> | null {
|
|
2632
|
-
const providerTurn = activeWork.find((item) => item.kind === "provider-turn");
|
|
2633
|
-
const state = providerTurn?.metadata?.providerState;
|
|
2634
|
-
return state && typeof state === "object" && !Array.isArray(state) ? state as Record<string, unknown> : null;
|
|
2635
|
-
}
|
|
2636
|
-
|
|
2637
|
-
function isHttpAuthError(error: unknown): boolean {
|
|
2638
|
-
const status = typeof error === "object" && error !== null ? (error as { status?: unknown }).status : undefined;
|
|
2639
|
-
return status === 401 || status === 403;
|
|
2640
|
-
}
|
|
2641
|
-
|
|
2642
|
-
function isHttpStatusError(error: unknown, code: number): boolean {
|
|
2643
|
-
const status = typeof error === "object" && error !== null ? (error as { status?: unknown }).status : undefined;
|
|
2644
|
-
return status === code;
|
|
2645
|
-
}
|
|
2646
|
-
|
|
2647
|
-
function httpErrorKey(error: unknown): string {
|
|
2648
|
-
const status = typeof error === "object" && error !== null ? (error as { status?: unknown }).status : undefined;
|
|
2649
|
-
if (typeof status === "number") return `status:${status}`;
|
|
2650
|
-
return String(error);
|
|
2651
|
-
}
|
|
2652
|
-
|
|
2653
|
-
function isContextState(value: unknown): value is ContextState {
|
|
2654
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
2655
|
-
const state = value as Record<string, unknown>;
|
|
2656
|
-
return typeof state.utilization === "number" &&
|
|
2657
|
-
typeof state.lifecycleState === "string" &&
|
|
2658
|
-
typeof state.source === "string" &&
|
|
2659
|
-
typeof state.confidence === "string";
|
|
2660
|
-
}
|
|
1
|
+
export { AgentRunner } from "./runner-core";
|
|
2
|
+
export {
|
|
3
|
+
latestClaudeResumeIdFromText,
|
|
4
|
+
runnerAgentStatus,
|
|
5
|
+
runnerBusErrorAction,
|
|
6
|
+
runnerMessageMatches,
|
|
7
|
+
runnerShouldResolveProviderExit,
|
|
8
|
+
runnerShouldRestartUnexpectedProviderExit,
|
|
9
|
+
runtimeTokenRenewDelayMs,
|
|
10
|
+
runtimeTokenRenewTimerSchedule,
|
|
11
|
+
shouldLogDeliveryFailure,
|
|
12
|
+
stripRunnerClaimedGuidance,
|
|
13
|
+
taskIdFromMessage,
|
|
14
|
+
} from "./runner-helpers";
|