agent-relay-runner 0.91.1 → 0.91.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/plugins/claude/.claude-plugin/plugin.json +1 -1
- package/plugins/claude/hooks/stop-failure.sh +13 -1
- package/src/adapters/claude.ts +1 -1
- package/src/config.ts +5 -0
- package/src/native-self-resume.ts +202 -0
- package/src/rate-limit.ts +35 -3
- package/src/runner-core.ts +29 -30
- package/src/runner-helpers.ts +33 -0
package/package.json
CHANGED
|
@@ -10,15 +10,27 @@ error="$(relay_json_string_field error_type "$payload")"
|
|
|
10
10
|
[ -z "$error" ] && error="$(relay_json_string_field error "$payload")"
|
|
11
11
|
reset_time="$(relay_json_number_field reset_time "$payload")"
|
|
12
12
|
error_message="$(relay_json_string_field error_message "$payload")"
|
|
13
|
+
[ -z "$error_message" ] && error_message="$(relay_json_string_field error_details "$payload")"
|
|
14
|
+
[ -z "$error_message" ] && error_message="$(relay_json_string_field last_assistant_message "$payload")"
|
|
13
15
|
|
|
14
16
|
# #286: a usage/rate-limit stall is transient and time-bounded — hold the agent
|
|
15
17
|
# (providerState blocked) and auto-resume at reset, instead of silently going idle.
|
|
16
18
|
# The disambiguator for billing_error is reset_time: a subscription rolling-window
|
|
17
19
|
# limit carries one (and is auto-resumable); true credit exhaustion does not (fatal).
|
|
18
20
|
case "$error" in
|
|
19
|
-
rate_limit|overloaded)
|
|
21
|
+
rate_limit|overloaded|server_error|api_error)
|
|
20
22
|
relay_post_rate_limit "$error" "$reset_time" "$error_message"
|
|
21
23
|
;;
|
|
24
|
+
unknown)
|
|
25
|
+
case "$error_message" in
|
|
26
|
+
*"529"*|*"Overloaded"*|*"overloaded"*|*"temporar"*|*"server-side"*|*"server error"*)
|
|
27
|
+
relay_post_rate_limit "$error" "$reset_time" "$error_message"
|
|
28
|
+
;;
|
|
29
|
+
*)
|
|
30
|
+
relay_post_status_clearing_subagents idle
|
|
31
|
+
;;
|
|
32
|
+
esac
|
|
33
|
+
;;
|
|
22
34
|
billing_error)
|
|
23
35
|
case "$reset_time" in
|
|
24
36
|
''|*[!0-9]*) relay_post_status_clearing_subagents error ;;
|
package/src/adapters/claude.ts
CHANGED
|
@@ -559,7 +559,7 @@ export function claudeRateLimitStatus(text: string, sessionName?: string): Provi
|
|
|
559
559
|
status: "idle",
|
|
560
560
|
clear: ["subagent"],
|
|
561
561
|
providerState: buildRateLimitProviderState({
|
|
562
|
-
errorType: "session_limit",
|
|
562
|
+
errorType: parsed.errorType ?? "session_limit", kind: parsed.kind ?? "usage_limit",
|
|
563
563
|
...(parsed.resetAt ? { resetAt: parsed.resetAt } : {}),
|
|
564
564
|
message: parsed.message,
|
|
565
565
|
source: sessionName ? `claude-pane:${sessionName}` : "claude-pane",
|
package/src/config.ts
CHANGED
|
@@ -99,6 +99,11 @@ export function registrationTimeoutMsFromEnv(): number {
|
|
|
99
99
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : 60_000;
|
|
100
100
|
}
|
|
101
101
|
|
|
102
|
+
export function nativeSelfResumeTimeoutMsFromEnv(): number {
|
|
103
|
+
const parsed = Number(process.env.AGENT_RELAY_NATIVE_SELF_RESUME_TIMEOUT_MS);
|
|
104
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : 120_000;
|
|
105
|
+
}
|
|
106
|
+
|
|
102
107
|
export function agentProfileNameFromEnv(): string | undefined {
|
|
103
108
|
return process.env.AGENT_RELAY_AGENT_PROFILE;
|
|
104
109
|
}
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import { errMessage } from "agent-relay-sdk";
|
|
2
|
+
import type { ProviderStatusEvent, SemanticStatus } from "./adapter";
|
|
3
|
+
import { nativeSelfResumeTimeoutMsFromEnv } from "./config";
|
|
4
|
+
|
|
5
|
+
interface RunnerTimelineEvent {
|
|
6
|
+
status: string;
|
|
7
|
+
id?: string;
|
|
8
|
+
timestamp: number;
|
|
9
|
+
title?: string;
|
|
10
|
+
body?: string;
|
|
11
|
+
icon?: string;
|
|
12
|
+
metadata?: Record<string, unknown>;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface NativeSelfResumeHandoff {
|
|
16
|
+
commandId: string;
|
|
17
|
+
generation?: number;
|
|
18
|
+
compactedAt?: number;
|
|
19
|
+
timer: ReturnType<typeof setTimeout>;
|
|
20
|
+
resolve: (result: Record<string, unknown>) => void;
|
|
21
|
+
reject: (error: Error) => void;
|
|
22
|
+
promise: Promise<Record<string, unknown>>;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function isNativeSelfResumeCompact(type: string, params: Record<string, unknown>): boolean {
|
|
26
|
+
if (type !== "agent.compact") return false;
|
|
27
|
+
const selfResume = params.selfResume;
|
|
28
|
+
return Boolean(selfResume && typeof selfResume === "object" && !Array.isArray(selfResume) && (selfResume as Record<string, unknown>).mode === "native");
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export class NativeSelfResumeTracker {
|
|
32
|
+
private current?: NativeSelfResumeHandoff;
|
|
33
|
+
|
|
34
|
+
get active(): NativeSelfResumeHandoff | undefined {
|
|
35
|
+
return this.current;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
begin(commandId: string, params: Record<string, unknown>): NativeSelfResumeHandoff {
|
|
39
|
+
this.finish(this.current);
|
|
40
|
+
let resolve!: (result: Record<string, unknown>) => void;
|
|
41
|
+
let reject!: (error: Error) => void;
|
|
42
|
+
const promise = new Promise<Record<string, unknown>>((res, rej) => {
|
|
43
|
+
resolve = res;
|
|
44
|
+
reject = rej;
|
|
45
|
+
});
|
|
46
|
+
const selfResume = params.selfResume && typeof params.selfResume === "object" && !Array.isArray(params.selfResume)
|
|
47
|
+
? params.selfResume as Record<string, unknown>
|
|
48
|
+
: undefined;
|
|
49
|
+
const generation = typeof selfResume?.generation === "number" ? selfResume.generation : undefined;
|
|
50
|
+
const timeoutMs = nativeSelfResumeTimeoutMsFromEnv();
|
|
51
|
+
const handoff: NativeSelfResumeHandoff = {
|
|
52
|
+
commandId,
|
|
53
|
+
generation,
|
|
54
|
+
promise,
|
|
55
|
+
resolve,
|
|
56
|
+
reject,
|
|
57
|
+
timer: setTimeout(() => {
|
|
58
|
+
if (this.current !== handoff) return;
|
|
59
|
+
this.current = undefined;
|
|
60
|
+
reject(new Error(`native self-resume timed out after ${Math.round(timeoutMs / 1000)}s`));
|
|
61
|
+
}, timeoutMs),
|
|
62
|
+
};
|
|
63
|
+
this.current = handoff;
|
|
64
|
+
return handoff;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
finish(handoff?: NativeSelfResumeHandoff): void {
|
|
68
|
+
if (!handoff) return;
|
|
69
|
+
clearTimeout(handoff.timer);
|
|
70
|
+
if (this.current === handoff) this.current = undefined;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
resolve(handoff: NativeSelfResumeHandoff, result: Record<string, unknown>): void {
|
|
74
|
+
this.finish(handoff);
|
|
75
|
+
handoff.resolve(result);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
reject(handoff: NativeSelfResumeHandoff, error: Error): void {
|
|
79
|
+
this.finish(handoff);
|
|
80
|
+
handoff.reject(error);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
noteCompacted(timestamp: number = Date.now()): void {
|
|
84
|
+
if (!this.current) return;
|
|
85
|
+
this.current.compactedAt = timestamp;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
interface NativeSelfResumeRecoveryOptions {
|
|
90
|
+
handoff: NativeSelfResumeHandoff;
|
|
91
|
+
tracker: NativeSelfResumeTracker;
|
|
92
|
+
providerSessionId: string;
|
|
93
|
+
status: SemanticStatus;
|
|
94
|
+
diagnostics: Record<string, unknown>;
|
|
95
|
+
now: number;
|
|
96
|
+
publishTimeline(event: RunnerTimelineEvent): void;
|
|
97
|
+
setTerminalProviderExit(marker: Record<string, unknown>): void;
|
|
98
|
+
setProviderStatus(update: ProviderStatusEvent): void;
|
|
99
|
+
restartProvider(resumeId?: string): Promise<void>;
|
|
100
|
+
publishStatus(): void;
|
|
101
|
+
scheduleDrain(): void;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export async function recoverNativeSelfResumeExit(opts: NativeSelfResumeRecoveryOptions): Promise<void> {
|
|
105
|
+
const resumeId = typeof opts.diagnostics.claudeResumeId === "string" && opts.diagnostics.claudeResumeId.length > 0
|
|
106
|
+
? opts.diagnostics.claudeResumeId
|
|
107
|
+
: undefined;
|
|
108
|
+
if (opts.handoff.compactedAt && resumeId) {
|
|
109
|
+
await recoverWithClaudeResume(opts, resumeId);
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
await recoverWithFreshLaunch(opts, resumeId);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
async function recoverWithClaudeResume(opts: NativeSelfResumeRecoveryOptions, resumeId: string): Promise<void> {
|
|
116
|
+
opts.publishTimeline({
|
|
117
|
+
status: "provider.restart_decision",
|
|
118
|
+
id: `provider-self-resume-relaunch-${opts.providerSessionId}-${opts.now}`,
|
|
119
|
+
timestamp: Date.now(),
|
|
120
|
+
title: "Self-resume relaunching",
|
|
121
|
+
body: `Claude exited during native self-resume; runner is relaunching with claude --resume ${resumeId}.`,
|
|
122
|
+
icon: "ti-refresh",
|
|
123
|
+
metadata: selfResumeMetadata(opts.handoff, opts.diagnostics, {
|
|
124
|
+
decision: "resume-native-self-resume",
|
|
125
|
+
reason: "self-resume-exit-after-compact",
|
|
126
|
+
claudeResumeId: resumeId,
|
|
127
|
+
compactedAt: opts.handoff.compactedAt,
|
|
128
|
+
}),
|
|
129
|
+
});
|
|
130
|
+
try {
|
|
131
|
+
await opts.restartProvider(resumeId);
|
|
132
|
+
opts.publishStatus();
|
|
133
|
+
opts.scheduleDrain();
|
|
134
|
+
opts.tracker.resolve(opts.handoff, { recovered: true, via: "claude-resume", claudeResumeId: resumeId });
|
|
135
|
+
} catch (error) {
|
|
136
|
+
const message = errMessage(error);
|
|
137
|
+
opts.setTerminalProviderExit(selfResumeMetadata(opts.handoff, opts.diagnostics, {
|
|
138
|
+
reason: "self-resume-relaunch-failed",
|
|
139
|
+
claudeResumeId: resumeId,
|
|
140
|
+
compactedAt: opts.handoff.compactedAt,
|
|
141
|
+
lastError: message,
|
|
142
|
+
}));
|
|
143
|
+
opts.setProviderStatus({
|
|
144
|
+
status: "offline",
|
|
145
|
+
reason: "provider-turn",
|
|
146
|
+
id: `provider-self-resume-relaunch-failed-${opts.providerSessionId}`,
|
|
147
|
+
clear: ["provider-turn", "subagent", "background-script"],
|
|
148
|
+
});
|
|
149
|
+
opts.tracker.reject(opts.handoff, new Error(`self-resume relaunch failed: ${message}`));
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
async function recoverWithFreshLaunch(opts: NativeSelfResumeRecoveryOptions, resumeId?: string): Promise<void> {
|
|
154
|
+
opts.publishTimeline({
|
|
155
|
+
status: "provider.restart_decision",
|
|
156
|
+
id: `provider-self-resume-fresh-launch-${opts.providerSessionId}-${opts.now}`,
|
|
157
|
+
timestamp: Date.now(),
|
|
158
|
+
title: "Self-resume fresh-launching",
|
|
159
|
+
body: "Claude exited during native self-resume before a compacted marker; runner is fresh-launching so Relay can inject the persisted continuation.",
|
|
160
|
+
icon: "ti-refresh",
|
|
161
|
+
metadata: selfResumeMetadata(opts.handoff, opts.diagnostics, {
|
|
162
|
+
decision: "fresh-launch-native-self-resume",
|
|
163
|
+
reason: "self-resume-exit-before-compact",
|
|
164
|
+
...(resumeId ? { claudeResumeId: resumeId } : {}),
|
|
165
|
+
}),
|
|
166
|
+
});
|
|
167
|
+
try {
|
|
168
|
+
await opts.restartProvider();
|
|
169
|
+
opts.publishStatus();
|
|
170
|
+
opts.scheduleDrain();
|
|
171
|
+
opts.tracker.resolve(opts.handoff, {
|
|
172
|
+
recovered: true,
|
|
173
|
+
via: "fresh-launch",
|
|
174
|
+
...(resumeId ? { claudeResumeId: resumeId } : {}),
|
|
175
|
+
});
|
|
176
|
+
} catch (error) {
|
|
177
|
+
const message = errMessage(error);
|
|
178
|
+
opts.setTerminalProviderExit(selfResumeMetadata(opts.handoff, opts.diagnostics, {
|
|
179
|
+
reason: "self-resume-fresh-launch-failed",
|
|
180
|
+
...(resumeId ? { claudeResumeId: resumeId } : {}),
|
|
181
|
+
lastError: message,
|
|
182
|
+
}));
|
|
183
|
+
opts.setProviderStatus({
|
|
184
|
+
status: "offline",
|
|
185
|
+
reason: "provider-turn",
|
|
186
|
+
id: `provider-self-resume-fresh-launch-failed-${opts.providerSessionId}`,
|
|
187
|
+
clear: ["provider-turn", "subagent", "background-script"],
|
|
188
|
+
});
|
|
189
|
+
opts.tracker.reject(opts.handoff, new Error(`self-resume fresh launch failed: ${message}`));
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function selfResumeMetadata(handoff: NativeSelfResumeHandoff, diagnostics: Record<string, unknown>, extra: Record<string, unknown>): Record<string, unknown> {
|
|
194
|
+
return {
|
|
195
|
+
eventType: "provider.restart_decision",
|
|
196
|
+
selfResume: true,
|
|
197
|
+
selfResumeCommandId: handoff.commandId,
|
|
198
|
+
selfResumeGeneration: handoff.generation ?? null,
|
|
199
|
+
...extra,
|
|
200
|
+
...diagnostics,
|
|
201
|
+
};
|
|
202
|
+
}
|
package/src/rate-limit.ts
CHANGED
|
@@ -17,6 +17,7 @@ const MAX_RESET_AHEAD_MS = 7 * 24 * 60 * 60 * 1000;
|
|
|
17
17
|
|
|
18
18
|
interface RateLimitHoldInput {
|
|
19
19
|
errorType?: string;
|
|
20
|
+
kind?: "usage_limit" | "transient_overload";
|
|
20
21
|
/** Unix ms the limit window resets, when known. */
|
|
21
22
|
resetAt?: number;
|
|
22
23
|
message?: string;
|
|
@@ -27,12 +28,18 @@ interface RateLimitHoldInput {
|
|
|
27
28
|
export function buildRateLimitProviderState(input: RateLimitHoldInput): Record<string, unknown> {
|
|
28
29
|
const now = Date.now();
|
|
29
30
|
const resetAt = typeof input.resetAt === "number" && Number.isFinite(input.resetAt) ? input.resetAt : undefined;
|
|
30
|
-
const
|
|
31
|
+
const kind = input.kind ?? rateLimitKindForError(input.errorType);
|
|
32
|
+
const label = kind === "transient_overload"
|
|
33
|
+
? "provider overloaded"
|
|
34
|
+
: resetAt ? `usage limit · resets ${formatResetClock(resetAt)}` : "usage limit reached";
|
|
31
35
|
return {
|
|
32
36
|
state: "blocked",
|
|
33
37
|
reason: RATE_LIMIT_BLOCK_REASON,
|
|
34
38
|
label,
|
|
35
|
-
recommendedAction:
|
|
39
|
+
recommendedAction: kind === "transient_overload"
|
|
40
|
+
? "Holding briefly for provider overload; agent-relay will retry with bounded backoff."
|
|
41
|
+
: "Holding until the usage window resets; agent-relay auto-resumes the agent.",
|
|
42
|
+
kind,
|
|
36
43
|
source: input.source ?? "claude",
|
|
37
44
|
errorType: input.errorType ?? RATE_LIMIT_BLOCK_REASON,
|
|
38
45
|
enteredAt: now,
|
|
@@ -42,6 +49,10 @@ export function buildRateLimitProviderState(input: RateLimitHoldInput): Record<s
|
|
|
42
49
|
};
|
|
43
50
|
}
|
|
44
51
|
|
|
52
|
+
function rateLimitKindForError(errorType: string | undefined): "usage_limit" | "transient_overload" {
|
|
53
|
+
return /^(overloaded|server_error|api_error|unknown|529|5\d\d)$/i.test(errorType ?? "") ? "transient_overload" : "usage_limit";
|
|
54
|
+
}
|
|
55
|
+
|
|
45
56
|
/** Host-local HH:MM for the chat notice / badge label (matches how the CLI shows "resets 3:45pm"). */
|
|
46
57
|
function formatResetClock(resetAtMs: number): string {
|
|
47
58
|
try {
|
|
@@ -64,6 +75,9 @@ interface PaneRateLimit {
|
|
|
64
75
|
message: string;
|
|
65
76
|
/** Unix ms reset, when parseable from the pane (else undefined → resume falls back to poll). */
|
|
66
77
|
resetAt?: number;
|
|
78
|
+
/** Usage limits wait for reset; transient provider overloads retry quickly. */
|
|
79
|
+
kind?: "usage_limit" | "transient_overload";
|
|
80
|
+
errorType?: string;
|
|
67
81
|
}
|
|
68
82
|
|
|
69
83
|
/**
|
|
@@ -75,6 +89,8 @@ interface PaneRateLimit {
|
|
|
75
89
|
*/
|
|
76
90
|
export function parseClaudeRateLimitPane(text: string, nowMs: number = Date.now()): PaneRateLimit | null {
|
|
77
91
|
if (!text) return null;
|
|
92
|
+
const overload = parseClaudeTransientOverloadPane(text);
|
|
93
|
+
if (overload) return overload;
|
|
78
94
|
const limit = LIMIT_PHRASE_RE.exec(text);
|
|
79
95
|
if (!limit) return null;
|
|
80
96
|
const message = limitLineAt(text, limit.index);
|
|
@@ -95,6 +111,21 @@ export function parseClaudeRateLimitPane(text: string, nowMs: number = Date.now(
|
|
|
95
111
|
return null;
|
|
96
112
|
}
|
|
97
113
|
|
|
114
|
+
const TRANSIENT_API_ERROR_RE = /\bAPI Error:\s*((?:5\d\d)|529)\b[^\n]*(?:overload|server-side|temporar|try again|status\.claude\.com)?[^\n]*/i;
|
|
115
|
+
const TRANSIENT_5XX_RE = /\b(?:5(?:00|02|03|04|20|29)|529)\b[^\n]*(?:overload|server-side|temporar|try again|status\.claude\.com)\b[^\n]*/i;
|
|
116
|
+
|
|
117
|
+
function parseClaudeTransientOverloadPane(text: string): PaneRateLimit | null {
|
|
118
|
+
const match = TRANSIENT_API_ERROR_RE.exec(text) ?? TRANSIENT_5XX_RE.exec(text);
|
|
119
|
+
if (!match) return null;
|
|
120
|
+
const message = limitLineAt(text, match.index);
|
|
121
|
+
const code = match[1] ?? /\b(5\d\d|529)\b/.exec(match[0])?.[1];
|
|
122
|
+
return {
|
|
123
|
+
message: message || match[0].trim(),
|
|
124
|
+
kind: "transient_overload",
|
|
125
|
+
errorType: code === "529" ? "overloaded" : "server_error",
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
98
129
|
export interface ClaudeRateLimitPaneGateState {
|
|
99
130
|
consecutiveDetections: number;
|
|
100
131
|
reported: boolean;
|
|
@@ -116,7 +147,8 @@ export function gatedClaudeRateLimitStatus(
|
|
|
116
147
|
status: "idle",
|
|
117
148
|
clear: ["subagent"],
|
|
118
149
|
providerState: buildRateLimitProviderState({
|
|
119
|
-
errorType: "session_limit",
|
|
150
|
+
errorType: parsed.errorType ?? "session_limit",
|
|
151
|
+
kind: parsed.kind ?? "usage_limit",
|
|
120
152
|
...(parsed.resetAt ? { resetAt: parsed.resetAt } : {}),
|
|
121
153
|
message: parsed.message,
|
|
122
154
|
source: options.sessionName ? `claude-pane:${options.sessionName}` : "claude-pane",
|
package/src/runner-core.ts
CHANGED
|
@@ -30,6 +30,7 @@ import { runnerLaunchInjectionSessionEvents, type RunnerRelayInjectionEvent } fr
|
|
|
30
30
|
import { providerTerminalSession, providerTerminalSocket } from "./process-meta";
|
|
31
31
|
import { capsFromEnv, logLevelFromEnv, mcpProxyEnabledFromEnv, orchestratorBaseDirFromEnv, orchestratorUrlFromEnv, registrationTimeoutMsFromEnv, runnerInfoFileFromEnv, runnerLogFileFromEnv, runnerOutboxDirWithInfoFallback, sessionDebugEnabled, tagsFromEnv, workspaceModeFromEnv } from "./config";
|
|
32
32
|
import { boundaryReasonForCommand, PRE_DESTROY_TIMEOUT_MS, reasonExitsRunner, type LifecycleAction, type SessionDestroyReason } from "./session-destroy";
|
|
33
|
+
import { isNativeSelfResumeCompact, NativeSelfResumeTracker, recoverNativeSelfResumeExit } from "./native-self-resume";
|
|
33
34
|
import {
|
|
34
35
|
appliedAgentProfileMetadata,
|
|
35
36
|
commandTimeoutMs,
|
|
@@ -38,8 +39,8 @@ import {
|
|
|
38
39
|
isContextState,
|
|
39
40
|
isHttpAuthError,
|
|
40
41
|
isHttpStatusError,
|
|
41
|
-
latestClaudeResumeIdFromLogFile,
|
|
42
42
|
lifecycleCapabilities,
|
|
43
|
+
providerExitDiagnostics,
|
|
43
44
|
providerStateFromActiveWork,
|
|
44
45
|
registerWithinDeadline,
|
|
45
46
|
relayBusUrl,
|
|
@@ -198,6 +199,7 @@ export class AgentRunner {
|
|
|
198
199
|
// terminal `agent.exited` event with a #636 diagnosis instead of the death going silent. Cleared
|
|
199
200
|
// on any genuine restart back to busy/idle.
|
|
200
201
|
private terminalProviderExit?: Record<string, unknown>;
|
|
202
|
+
private nativeSelfResume = new NativeSelfResumeTracker();
|
|
201
203
|
private exitCommandInProgress = false;
|
|
202
204
|
private restartInProgress = false;
|
|
203
205
|
// Set for the whole unexpected-exit restart, including the pre-restart backoff
|
|
@@ -521,7 +523,7 @@ export class AgentRunner {
|
|
|
521
523
|
return mode === "isolated";
|
|
522
524
|
}
|
|
523
525
|
|
|
524
|
-
private async spawnProvider(): Promise<ManagedProcess> {
|
|
526
|
+
private async spawnProvider(opts: { resumeId?: string; suppressPrompt?: boolean } = {}): Promise<ManagedProcess> {
|
|
525
527
|
this.providerSessionId = crypto.randomUUID();
|
|
526
528
|
this.lastTranscriptPath = undefined;
|
|
527
529
|
const includeProviderGlobals = profileUsesHostProviderGlobals(this.options);
|
|
@@ -571,10 +573,10 @@ export class AgentRunner {
|
|
|
571
573
|
...(this.options.rig ? { rig: this.options.rig } : {}),
|
|
572
574
|
...(this.options.profile ? { profile: this.options.profile } : {}),
|
|
573
575
|
...(this.options.agentProfile ? { agentProfile: this.options.agentProfile } : {}),
|
|
574
|
-
...(this.options.prompt ? { prompt: this.options.prompt } : {}),
|
|
576
|
+
...(this.options.prompt && !opts.resumeId && !opts.suppressPrompt ? { prompt: this.options.prompt } : {}),
|
|
575
577
|
...(this.options.systemPromptAppend ? { systemPromptAppend: this.options.systemPromptAppend } : {}),
|
|
576
578
|
...(this.options.tmuxSession ? { tmuxSession: this.options.tmuxSession } : {}),
|
|
577
|
-
providerArgs: this.options.providerArgs,
|
|
579
|
+
providerArgs: opts.resumeId ? [...this.options.providerArgs, "--resume", opts.resumeId] : this.options.providerArgs,
|
|
578
580
|
providerConfig: this.options.providerConfig,
|
|
579
581
|
env,
|
|
580
582
|
controlPort: this.control!.port,
|
|
@@ -588,7 +590,7 @@ export class AgentRunner {
|
|
|
588
590
|
deliver: (messages) => this.control!.deliverToMonitor(messages),
|
|
589
591
|
},
|
|
590
592
|
};
|
|
591
|
-
const launchSeededPrompt = this.options.adapter.seedsInitialPromptAtLaunch ? this.options.prompt?.trim() : ""; if (launchSeededPrompt) this.recordInjectedPrompt(launchSeededPrompt);
|
|
593
|
+
const launchSeededPrompt = this.options.adapter.seedsInitialPromptAtLaunch && !opts.resumeId && !opts.suppressPrompt ? this.options.prompt?.trim() : ""; if (launchSeededPrompt) this.recordInjectedPrompt(launchSeededPrompt);
|
|
592
594
|
const managedProcess = await this.options.adapter.spawn(config);
|
|
593
595
|
const injectionEvents = runnerLaunchInjectionSessionEvents({ carried: this.options.relayInjectionEvents, provider: this.options.provider, config, providerConfig: this.options.providerConfig, agentId: this.agentId, providerSessionId: this.providerSessionId });
|
|
594
596
|
for (const event of injectionEvents) {
|
|
@@ -744,6 +746,9 @@ export class AgentRunner {
|
|
|
744
746
|
|
|
745
747
|
const exitAfterCommand = type === "agent.shutdown" || type === "agent.kill";
|
|
746
748
|
if (exitAfterCommand) this.exitCommandInProgress = true;
|
|
749
|
+
const nativeSelfResume = isNativeSelfResumeCompact(type, params)
|
|
750
|
+
? this.nativeSelfResume.begin(commandId, params)
|
|
751
|
+
: undefined;
|
|
747
752
|
this.claims.startClaim("command", commandId);
|
|
748
753
|
try {
|
|
749
754
|
await this.updateCommand(commandId, "accepted");
|
|
@@ -767,6 +772,10 @@ export class AgentRunner {
|
|
|
767
772
|
providerResult = await this.options.adapter.compact(this.process, {
|
|
768
773
|
instructions: typeof params.instructions === "string" ? params.instructions : undefined,
|
|
769
774
|
});
|
|
775
|
+
if (nativeSelfResume) {
|
|
776
|
+
const handoff = await nativeSelfResume.promise;
|
|
777
|
+
providerResult = { ...(providerResult ?? {}), nativeSelfResume: handoff };
|
|
778
|
+
}
|
|
770
779
|
} else if (type === "agent.clearContext") {
|
|
771
780
|
if (!this.options.adapter.clearContext || !this.process) throw new Error("provider does not support clearContext");
|
|
772
781
|
providerResult = await this.options.adapter.clearContext(this.process);
|
|
@@ -793,6 +802,7 @@ export class AgentRunner {
|
|
|
793
802
|
...(providerResult ? { providerResult } : {}),
|
|
794
803
|
});
|
|
795
804
|
} catch (error) {
|
|
805
|
+
if (nativeSelfResume) this.nativeSelfResume.finish(nativeSelfResume);
|
|
796
806
|
await this.updateCommand(commandId, "failed", undefined, errMessage(error)).catch(() => {});
|
|
797
807
|
} finally {
|
|
798
808
|
this.claims.finishClaim("command", commandId);
|
|
@@ -808,6 +818,7 @@ export class AgentRunner {
|
|
|
808
818
|
}).finally(() => process.exit(0)), 10);
|
|
809
819
|
}
|
|
810
820
|
} else if (!this.stopped) {
|
|
821
|
+
if (nativeSelfResume) this.nativeSelfResume.finish(nativeSelfResume);
|
|
811
822
|
this.lifecycleAction = undefined;
|
|
812
823
|
this.publishStatus();
|
|
813
824
|
}
|
|
@@ -904,7 +915,7 @@ export class AgentRunner {
|
|
|
904
915
|
return attachmentText ? `${body}\n\n${attachmentText}` : body;
|
|
905
916
|
}
|
|
906
917
|
|
|
907
|
-
private async restartProvider(): Promise<void> {
|
|
918
|
+
private async restartProvider(opts: { resumeId?: string; suppressPrompt?: boolean } = {}): Promise<void> {
|
|
908
919
|
this.restartInProgress = true;
|
|
909
920
|
try {
|
|
910
921
|
if (this.process) {
|
|
@@ -918,7 +929,7 @@ export class AgentRunner {
|
|
|
918
929
|
this.claims.clearWorkKind("subagent");
|
|
919
930
|
this.claims.clearWorkKind("background-script");
|
|
920
931
|
if (this.stopped) return;
|
|
921
|
-
this.process = await this.spawnProvider();
|
|
932
|
+
this.process = await this.spawnProvider(opts);
|
|
922
933
|
this.processStartedAt = Date.now();
|
|
923
934
|
} finally {
|
|
924
935
|
this.restartInProgress = false;
|
|
@@ -956,7 +967,7 @@ export class AgentRunner {
|
|
|
956
967
|
const recent = this.unexpectedExitTimes.filter((time) => now - time <= UNEXPECTED_EXIT_WINDOW_MS);
|
|
957
968
|
recent.push(now);
|
|
958
969
|
this.unexpectedExitTimes.splice(0, this.unexpectedExitTimes.length, ...recent);
|
|
959
|
-
const diagnostics =
|
|
970
|
+
const diagnostics = providerExitDiagnostics({ status, runtimeMs, processMeta: this.process?.meta, hasProcess: Boolean(this.process?.process), exitCommandInProgress: this.exitCommandInProgress, stopped: this.stopped, restartInProgress: this.restartInProgress, restartPending: this.restartPending, headless: this.options.headless, provider: this.options.provider, logFile: runnerLogFileFromEnv() });
|
|
960
971
|
|
|
961
972
|
this.publishRunnerTimelineEvent({
|
|
962
973
|
status: "provider.exit_detected",
|
|
@@ -971,6 +982,12 @@ export class AgentRunner {
|
|
|
971
982
|
},
|
|
972
983
|
});
|
|
973
984
|
|
|
985
|
+
const nativeSelfResume = this.nativeSelfResume.active;
|
|
986
|
+
if (nativeSelfResume && this.options.provider === "claude") {
|
|
987
|
+
await recoverNativeSelfResumeExit({ handoff: nativeSelfResume, tracker: this.nativeSelfResume, providerSessionId: this.providerSessionId, status, diagnostics, now, publishTimeline: (event) => this.publishRunnerTimelineEvent(event), setTerminalProviderExit: (marker) => { this.terminalProviderExit = marker; }, setProviderStatus: (update) => this.setProviderStatus(update), restartProvider: (resumeId) => this.restartProvider(resumeId ? { resumeId } : { suppressPrompt: true }), publishStatus: () => this.publishStatus(), scheduleDrain: () => this.scheduleDrain() });
|
|
988
|
+
return;
|
|
989
|
+
}
|
|
990
|
+
|
|
974
991
|
if (this.shouldStopUnexpectedProviderExit(diagnostics)) {
|
|
975
992
|
const hasResumeId = typeof diagnostics.claudeResumeId === "string" && diagnostics.claudeResumeId.length > 0;
|
|
976
993
|
logger.warn("lifecycle", `${this.options.provider} exited; leaving agent offline for manual recovery`);
|
|
@@ -1104,28 +1121,6 @@ export class AgentRunner {
|
|
|
1104
1121
|
this.publishStatus();
|
|
1105
1122
|
}
|
|
1106
1123
|
|
|
1107
|
-
private providerExitDiagnostics(status: SemanticStatus, runtimeMs: number): Record<string, unknown> {
|
|
1108
|
-
const tmuxSession = typeof this.process?.meta?.tmuxSession === "string" ? this.process.meta.tmuxSession : undefined;
|
|
1109
|
-
const tmuxSocket = typeof this.process?.meta?.tmuxSocket === "string" ? this.process.meta.tmuxSocket : undefined;
|
|
1110
|
-
const exitSource = tmuxSession ? "tmux-session-ended" : this.process?.process ? "process-exit" : "provider-status";
|
|
1111
|
-
const logFile = runnerLogFileFromEnv();
|
|
1112
|
-
const claudeResumeId = this.options.provider === "claude" && logFile ? latestClaudeResumeIdFromLogFile(logFile) : undefined;
|
|
1113
|
-
return {
|
|
1114
|
-
status,
|
|
1115
|
-
runtimeMs: Number.isFinite(runtimeMs) ? runtimeMs : null,
|
|
1116
|
-
exitSource,
|
|
1117
|
-
exitCommandInProgress: this.exitCommandInProgress,
|
|
1118
|
-
stopped: this.stopped,
|
|
1119
|
-
restartInProgress: this.restartInProgress,
|
|
1120
|
-
restartPending: this.restartPending,
|
|
1121
|
-
headless: this.options.headless,
|
|
1122
|
-
hasTerminalSession: Boolean(tmuxSession),
|
|
1123
|
-
tmuxSession: tmuxSession ?? null,
|
|
1124
|
-
tmuxSocket: tmuxSocket ?? null,
|
|
1125
|
-
claudeResumeId: claudeResumeId ?? null,
|
|
1126
|
-
};
|
|
1127
|
-
}
|
|
1128
|
-
|
|
1129
1124
|
private async updateCommand(commandId: string, status: string, result?: Record<string, unknown>, error?: string): Promise<void> {
|
|
1130
1125
|
await this.bus.updateCommand(commandId, { status, ...(result ? { result } : {}), ...(error ? { error } : {}) });
|
|
1131
1126
|
}
|
|
@@ -1222,6 +1217,7 @@ export class AgentRunner {
|
|
|
1222
1217
|
if (timelineStatus === "compacting") {
|
|
1223
1218
|
this.compactionMidTurn = this.currentTurnId !== undefined;
|
|
1224
1219
|
} else if (timelineStatus === "compacted") {
|
|
1220
|
+
this.nativeSelfResume.noteCompacted(typeof update !== "string" && typeof update.timeline?.timestamp === "number" ? update.timeline.timestamp : Date.now());
|
|
1225
1221
|
this.publishCompactionNotice();
|
|
1226
1222
|
if (this.compactionMidTurn) {
|
|
1227
1223
|
// Keep the turn (and its live reasoning tail) alive; the genuine Stop hook ends it.
|
|
@@ -1304,6 +1300,9 @@ export class AgentRunner {
|
|
|
1304
1300
|
this.rateLimitHold = hold;
|
|
1305
1301
|
if (fresh) this.publishRateLimitNotice(hold);
|
|
1306
1302
|
}
|
|
1303
|
+
if (status === "idle" && this.nativeSelfResume.active) {
|
|
1304
|
+
this.nativeSelfResume.resolve(this.nativeSelfResume.active, { resumed: true, via: "provider-idle" });
|
|
1305
|
+
}
|
|
1307
1306
|
this.publishStatus();
|
|
1308
1307
|
}
|
|
1309
1308
|
|
package/src/runner-helpers.ts
CHANGED
|
@@ -157,6 +157,39 @@ export function latestClaudeResumeIdFromLogFile(path: string): string | undefine
|
|
|
157
157
|
}
|
|
158
158
|
}
|
|
159
159
|
|
|
160
|
+
export function providerExitDiagnostics(input: {
|
|
161
|
+
status: SemanticStatus;
|
|
162
|
+
runtimeMs: number;
|
|
163
|
+
processMeta?: Record<string, unknown>;
|
|
164
|
+
hasProcess: boolean;
|
|
165
|
+
exitCommandInProgress: boolean;
|
|
166
|
+
stopped: boolean;
|
|
167
|
+
restartInProgress: boolean;
|
|
168
|
+
restartPending: boolean;
|
|
169
|
+
headless: boolean;
|
|
170
|
+
provider: string;
|
|
171
|
+
logFile?: string;
|
|
172
|
+
}): Record<string, unknown> {
|
|
173
|
+
const tmuxSession = typeof input.processMeta?.tmuxSession === "string" ? input.processMeta.tmuxSession : undefined;
|
|
174
|
+
const tmuxSocket = typeof input.processMeta?.tmuxSocket === "string" ? input.processMeta.tmuxSocket : undefined;
|
|
175
|
+
const exitSource = tmuxSession ? "tmux-session-ended" : input.hasProcess ? "process-exit" : "provider-status";
|
|
176
|
+
const claudeResumeId = input.provider === "claude" && input.logFile ? latestClaudeResumeIdFromLogFile(input.logFile) : undefined;
|
|
177
|
+
return {
|
|
178
|
+
status: input.status,
|
|
179
|
+
runtimeMs: Number.isFinite(input.runtimeMs) ? input.runtimeMs : null,
|
|
180
|
+
exitSource,
|
|
181
|
+
exitCommandInProgress: input.exitCommandInProgress,
|
|
182
|
+
stopped: input.stopped,
|
|
183
|
+
restartInProgress: input.restartInProgress,
|
|
184
|
+
restartPending: input.restartPending,
|
|
185
|
+
headless: input.headless,
|
|
186
|
+
hasTerminalSession: Boolean(tmuxSession),
|
|
187
|
+
tmuxSession: tmuxSession ?? null,
|
|
188
|
+
tmuxSocket: tmuxSocket ?? null,
|
|
189
|
+
claudeResumeId: claudeResumeId ?? null,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
160
193
|
export function commandTimeoutMs(params: Record<string, unknown>, fallback = 10_000): number {
|
|
161
194
|
const raw = params.timeoutMs;
|
|
162
195
|
if (typeof raw !== "number" || !Number.isSafeInteger(raw) || raw <= 0) return fallback;
|