@team-agent/installer 0.5.62 → 0.5.63
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -1
- package/Cargo.toml +1 -1
- package/crates/team-agent/src/coordinator/steps/abnormal.rs +11 -863
- package/crates/team-agent/src/coordinator/tests/api_error_recovery.rs +139 -366
- package/crates/team-agent/src/coordinator/tests/tick_core.rs +0 -7
- package/crates/team-agent/src/coordinator/tick.rs +0 -13
- package/crates/team-agent/src/lifecycle/restart/agent.rs +0 -63
- package/crates/team-agent/src/lifecycle/restart.rs +0 -4
- package/crates/team-agent/src/state/repository.rs +0 -13
- package/package.json +4 -4
|
@@ -1,306 +1,134 @@
|
|
|
1
1
|
use super::*;
|
|
2
2
|
|
|
3
|
-
use
|
|
4
|
-
atomic::{AtomicU32, Ordering},
|
|
5
|
-
Arc,
|
|
6
|
-
};
|
|
3
|
+
use crate::transport::test_support::OfflineTransport;
|
|
7
4
|
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
// bounded, visible worker recovery or loudly explains why not. Recovery must not
|
|
12
|
-
// run inside the abnormal detector's pre-save window, must not treat a live-pid
|
|
13
|
-
// Noop as success, and must never synthesize a hidden provider turn.
|
|
5
|
+
// Removal contract for the retired abnormal-exit automatic recovery path.
|
|
6
|
+
// Abnormal exits remain visible and keep a copyable manual start-agent hint,
|
|
7
|
+
// but coordinator ticks must never schedule or execute lifecycle work.
|
|
14
8
|
|
|
15
9
|
#[test]
|
|
16
|
-
fn
|
|
17
|
-
let case = RecoveryCase::new("
|
|
18
|
-
case.
|
|
19
|
-
let
|
|
10
|
+
fn abnormal_exit_is_reported_without_auto_recovery_intent_or_action() {
|
|
11
|
+
let case = RecoveryCase::new("reported-no-auto");
|
|
12
|
+
case.seed_agent("baseline-429");
|
|
13
|
+
let transport = OfflineTransport::new().with_session_present(true);
|
|
14
|
+
let coord = case.coord(transport.clone());
|
|
20
15
|
coord.tick().expect("baseline tick");
|
|
21
16
|
|
|
22
|
-
case.append_api_error("
|
|
23
|
-
coord.tick().expect("fresh
|
|
17
|
+
case.append_api_error("fresh-429");
|
|
18
|
+
coord.tick().expect("fresh abnormal tick");
|
|
24
19
|
|
|
25
20
|
let events = case.events();
|
|
26
21
|
let abnormal = find_event(&events, "worker.abnormal_exit").expect("abnormal event");
|
|
27
22
|
assert_eq!(abnormal["agent_id"], serde_json::json!("fe-admin"));
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
assert_eq!(scheduled["agent_id"], serde_json::json!("fe-admin"));
|
|
31
|
-
assert_eq!(scheduled["apiErrorStatus"], serde_json::json!(429));
|
|
32
|
-
assert_eq!(scheduled["error"], serde_json::json!("rate_limit"));
|
|
33
|
-
assert_eq!(scheduled["attempt"], serde_json::json!(0));
|
|
23
|
+
assert_eq!(abnormal["signature"], serde_json::json!("api_error"));
|
|
24
|
+
assert_eq!(abnormal["apiErrorStatus"], serde_json::json!(429));
|
|
34
25
|
assert!(
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
26
|
+
events.iter().all(|event| {
|
|
27
|
+
let name = event
|
|
28
|
+
.get("event")
|
|
29
|
+
.and_then(serde_json::Value::as_str)
|
|
30
|
+
.unwrap_or_default();
|
|
31
|
+
!name.starts_with("worker.abnormal_exit.recovery_")
|
|
32
|
+
&& !name.starts_with("worker.abnormal_exit.backpressure_")
|
|
33
|
+
}),
|
|
34
|
+
"abnormal detection must not emit automatic recovery/backpressure events; events={events:?}"
|
|
40
35
|
);
|
|
41
36
|
|
|
42
37
|
let state = case.state();
|
|
43
|
-
let intent = state
|
|
44
|
-
.pointer("/coordinator/abnormal_api_error_recovery/agents/fe-admin")
|
|
45
|
-
.expect("R1: state must persist recovery intent");
|
|
46
|
-
assert_eq!(intent["status"], serde_json::json!("scheduled"));
|
|
47
|
-
assert_eq!(intent["attempts"], serde_json::json!(0));
|
|
48
38
|
assert!(
|
|
49
|
-
|
|
50
|
-
.
|
|
51
|
-
.
|
|
52
|
-
|
|
53
|
-
|
|
39
|
+
state
|
|
40
|
+
.pointer("/coordinator/abnormal_api_error_recovery")
|
|
41
|
+
.is_none(),
|
|
42
|
+
"abnormal detection must not persist automatic recovery intent; state={state}"
|
|
43
|
+
);
|
|
44
|
+
assert!(
|
|
45
|
+
transport.spawn_records().is_empty(),
|
|
46
|
+
"abnormal detection must not respawn a worker; spawns={:?}",
|
|
47
|
+
transport.spawn_records()
|
|
54
48
|
);
|
|
55
49
|
|
|
56
50
|
let content = case.latest_leader_notification();
|
|
57
51
|
assert!(
|
|
58
|
-
content.contains("
|
|
52
|
+
content.contains("No automatic restart was performed.")
|
|
53
|
+
&& content.contains("To recover manually, run: team-agent start-agent fe-admin")
|
|
59
54
|
&& content.contains("--workspace")
|
|
60
55
|
&& content.contains("--team research")
|
|
61
56
|
&& content.contains("--force")
|
|
62
57
|
&& content.contains("--json"),
|
|
63
|
-
"
|
|
64
|
-
);
|
|
65
|
-
assert!(
|
|
66
|
-
!content.contains("No automatic restart was performed."),
|
|
67
|
-
"R1: old unconditional no-auto-restart text must be replaced; content={content}"
|
|
58
|
+
"abnormal notification must remain visible and keep the copyable manual recovery command; content={content}"
|
|
68
59
|
);
|
|
69
60
|
}
|
|
70
61
|
|
|
71
62
|
#[test]
|
|
72
|
-
fn
|
|
73
|
-
let case = RecoveryCase::new("
|
|
74
|
-
case.
|
|
75
|
-
let
|
|
63
|
+
fn first_turn_death_cannot_enter_auto_respawn_loop_and_manual_start_still_recovers() {
|
|
64
|
+
let case = RecoveryCase::new("first-turn-death-no-loop");
|
|
65
|
+
case.seed_agent("baseline-429");
|
|
66
|
+
let transport = OfflineTransport::new().with_session_present(true);
|
|
67
|
+
let coord = case.coord(transport.clone());
|
|
76
68
|
coord.tick().expect("baseline tick");
|
|
77
69
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
"R2: non-retryable model errors must not schedule auto recovery; state={state}"
|
|
87
|
-
);
|
|
88
|
-
let content = case.latest_leader_notification();
|
|
89
|
-
assert!(
|
|
90
|
-
content.contains("non_retryable_api_error")
|
|
91
|
-
&& content.contains("model_not_found")
|
|
92
|
-
&& content.contains("team-agent start-agent fe-admin")
|
|
93
|
-
&& content.contains("--force"),
|
|
94
|
-
"R2: notification must be guide-only with model/config context and manual command; content={content}"
|
|
95
|
-
);
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
#[test]
|
|
99
|
-
fn r3_live_pid_noop_is_never_counted_as_recovery_success() {
|
|
100
|
-
let abnormal = source("src/coordinator/steps/abnormal.rs");
|
|
101
|
-
let lifecycle = source("src/lifecycle/restart/agent.rs");
|
|
102
|
-
let recovery_mentions_noop_block = abnormal.contains("noop_not_recovery")
|
|
103
|
-
&& abnormal.contains("worker.abnormal_exit.recovery_blocked");
|
|
104
|
-
let lifecycle_has_stop_then_start_mode =
|
|
105
|
-
abnormal.contains("stop_then_start") || lifecycle.contains("stop_before_start");
|
|
106
|
-
assert!(
|
|
107
|
-
recovery_mentions_noop_block || lifecycle_has_stop_then_start_mode,
|
|
108
|
-
"R3: api_error recovery must either stop-before-start live panes or record recovery_blocked(reason=noop_not_recovery); abnormal.rs/lifecycle.rs missing both guards"
|
|
109
|
-
);
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
#[test]
|
|
113
|
-
fn r4_short_window_429_cohort_enters_backpressure_with_one_canary() {
|
|
114
|
-
let case = RecoveryCase::new("r4-backpressure");
|
|
115
|
-
let agents = [
|
|
116
|
-
("architect", 429, "rate_limit", "baseline-a"),
|
|
117
|
-
("fe-admin", 429, "rate_limit", "baseline-b"),
|
|
118
|
-
("fe-staff", 429, "rate_limit", "baseline-c"),
|
|
119
|
-
("fe-shop", 429, "rate_limit", "baseline-d"),
|
|
120
|
-
];
|
|
121
|
-
case.seed_agents(&agents);
|
|
122
|
-
let coord = case.coord();
|
|
123
|
-
coord.tick().expect("baseline tick");
|
|
124
|
-
for (agent, _, _, _) in agents {
|
|
125
|
-
case.append_api_error(agent, 429, "rate_limit", &format!("fresh-{agent}"));
|
|
126
|
-
}
|
|
127
|
-
coord.tick().expect("fresh cohort tick");
|
|
70
|
+
// A-29 shape: the replacement worker dies before completing its first
|
|
71
|
+
// turn, with a fresh explicit provider error. The following tick reports
|
|
72
|
+
// the death; a persisted due row models the next 6.5-second loop turn.
|
|
73
|
+
case.mark_first_turn_dead();
|
|
74
|
+
case.append_api_error("fresh-first-turn-death");
|
|
75
|
+
coord.tick().expect("fresh first-turn death tick");
|
|
76
|
+
case.seed_legacy_due_recovery();
|
|
77
|
+
coord.tick().expect("tick with legacy due intent");
|
|
128
78
|
|
|
129
79
|
let events = case.events();
|
|
130
|
-
let
|
|
131
|
-
|
|
132
|
-
assert_eq!(
|
|
133
|
-
assert_eq!(backpressure["provider"], serde_json::json!("claude_code"));
|
|
134
|
-
assert_eq!(backpressure["apiErrorStatus"], serde_json::json!(429));
|
|
80
|
+
let abnormal = find_event(&events, "worker.abnormal_exit").expect("abnormal event");
|
|
81
|
+
assert_eq!(abnormal["dead_process"], serde_json::json!(true));
|
|
82
|
+
assert_eq!(abnormal["fresh_error"], serde_json::json!(true));
|
|
135
83
|
assert!(
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
84
|
+
events.iter().all(|event| {
|
|
85
|
+
let name = event
|
|
86
|
+
.get("event")
|
|
87
|
+
.and_then(serde_json::Value::as_str)
|
|
88
|
+
.unwrap_or_default();
|
|
89
|
+
!name.starts_with("worker.abnormal_exit.recovery_")
|
|
90
|
+
&& !name.starts_with("worker.abnormal_exit.backpressure_")
|
|
91
|
+
}),
|
|
92
|
+
"first-turn death must report only; automatic recovery events make the A-29 loop possible; events={events:?}"
|
|
141
93
|
);
|
|
142
|
-
|
|
143
|
-
let scheduled_count = events
|
|
144
|
-
.iter()
|
|
145
|
-
.filter(|event| {
|
|
146
|
-
event.get("event").and_then(serde_json::Value::as_str)
|
|
147
|
-
== Some("worker.abnormal_exit.recovery_scheduled")
|
|
148
|
-
})
|
|
149
|
-
.count();
|
|
150
94
|
assert!(
|
|
151
|
-
|
|
152
|
-
"
|
|
95
|
+
transport.spawn_records().is_empty(),
|
|
96
|
+
"first-turn death and legacy due intents must never respawn; spawns={:?}",
|
|
97
|
+
transport.spawn_records()
|
|
153
98
|
);
|
|
154
|
-
let state = case.state();
|
|
155
|
-
let bp = state
|
|
156
|
-
.pointer("/coordinator/abnormal_api_error_recovery/backpressure")
|
|
157
|
-
.and_then(serde_json::Value::as_object)
|
|
158
|
-
.expect("R4: backpressure state must be persisted");
|
|
159
|
-
assert_eq!(
|
|
160
|
-
bp.len(),
|
|
161
|
-
1,
|
|
162
|
-
"R4: cohort must share one backpressure record; state={state}"
|
|
163
|
-
);
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
#[test]
|
|
167
|
-
fn r5_retry_budget_exhausts_and_stays_loud() {
|
|
168
|
-
let abnormal = source("src/coordinator/steps/abnormal.rs");
|
|
169
|
-
for needle in [
|
|
170
|
-
"max_attempts",
|
|
171
|
-
"worker.abnormal_exit.recovery_exhausted",
|
|
172
|
-
"manual_command",
|
|
173
|
-
"attempts",
|
|
174
|
-
] {
|
|
175
|
-
assert!(
|
|
176
|
-
abnormal.contains(needle),
|
|
177
|
-
"R5: retry budget contract requires `{needle}` in abnormal api_error recovery state/events"
|
|
178
|
-
);
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
#[test]
|
|
183
|
-
fn r6_abnormal_detector_does_not_mutate_lifecycle_inside_pre_save_window() {
|
|
184
|
-
let body = detect_abnormal_body();
|
|
185
|
-
for forbidden in [
|
|
186
|
-
"start_agent_at_paths",
|
|
187
|
-
"start_agent(",
|
|
188
|
-
"stop_agent_at_paths",
|
|
189
|
-
"reset_agent",
|
|
190
|
-
] {
|
|
191
|
-
assert!(
|
|
192
|
-
!body.contains(forbidden),
|
|
193
|
-
"R6: detect_abnormal_exits is pre-save detection only and must not call `{forbidden}`"
|
|
194
|
-
);
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
#[test]
|
|
199
|
-
fn r6_recovery_fields_survive_the_next_tick_after_atomic_save() {
|
|
200
|
-
let case = RecoveryCase::new("r6-save-boundary");
|
|
201
|
-
case.seed_agents(&[("fe-admin", 429, "rate_limit", "baseline-429")]);
|
|
202
|
-
case.seed_due_recovery("fe-admin");
|
|
203
|
-
let coord = case.coord();
|
|
204
|
-
coord.tick().expect("recovery tick");
|
|
205
|
-
|
|
206
|
-
let events = case.events();
|
|
207
99
|
assert!(
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
100
|
+
transport.calls().iter().all(|call| !matches!(
|
|
101
|
+
*call,
|
|
102
|
+
"spawn_first" | "spawn_into" | "kill_pane" | "kill_window"
|
|
103
|
+
)),
|
|
104
|
+
"automatic loop prevention must be structural: no lifecycle transport action; calls={:?}",
|
|
105
|
+
transport.calls()
|
|
211
106
|
);
|
|
212
|
-
let state = case.state();
|
|
213
|
-
let recovery = state
|
|
214
|
-
.pointer("/coordinator/abnormal_api_error_recovery/agents/fe-admin")
|
|
215
|
-
.expect("R6: recovery entry must survive tick");
|
|
216
|
-
assert!(
|
|
217
|
-
recovery.get("last_attempt_at").is_some()
|
|
218
|
-
|| recovery.get("last_error").is_some()
|
|
219
|
-
|| recovery.get("blocked_reason").is_some(),
|
|
220
|
-
"R6: lifecycle-written recovery result fields must survive next tick save; recovery={recovery}"
|
|
221
|
-
);
|
|
222
|
-
}
|
|
223
107
|
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
let
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
let calls = std::sync::Arc::clone(&transport.calls);
|
|
233
|
-
let coord = Coordinator::for_test(
|
|
234
|
-
WorkspacePath::new(case.root.clone()),
|
|
235
|
-
Box::new(registry),
|
|
236
|
-
Box::new(transport),
|
|
237
|
-
None,
|
|
108
|
+
case.seed_healthy_coordinator();
|
|
109
|
+
let manual_transport = OfflineTransport::new().with_session_present(true);
|
|
110
|
+
let outcome = crate::lifecycle::start_agent_with_transport(
|
|
111
|
+
&case.root,
|
|
112
|
+
&crate::model::ids::AgentId::new("fe-admin"),
|
|
113
|
+
false,
|
|
114
|
+
false,
|
|
115
|
+
false,
|
|
238
116
|
None,
|
|
117
|
+
&manual_transport,
|
|
239
118
|
);
|
|
240
119
|
|
|
241
|
-
coord.tick().expect("recovery tick");
|
|
242
|
-
|
|
243
|
-
assert_eq!(
|
|
244
|
-
counters.adapter_calls.load(Ordering::SeqCst),
|
|
245
|
-
0,
|
|
246
|
-
"R7: recovery must not call provider adapter/client APIs"
|
|
247
|
-
);
|
|
248
120
|
assert!(
|
|
249
|
-
!
|
|
250
|
-
"
|
|
251
|
-
calls.lock().unwrap()
|
|
121
|
+
matches!(outcome, Ok(crate::lifecycle::StartAgentOutcome::Running { .. })),
|
|
122
|
+
"manual start-agent path must remain functional after an abnormal exit; outcome={outcome:?}"
|
|
252
123
|
);
|
|
253
124
|
assert_eq!(
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
"
|
|
125
|
+
manual_transport.spawn_records().len(),
|
|
126
|
+
1,
|
|
127
|
+
"manual start-agent must spawn exactly one replacement worker; spawns={:?}",
|
|
128
|
+
manual_transport.spawn_records()
|
|
257
129
|
);
|
|
258
130
|
}
|
|
259
131
|
|
|
260
|
-
#[test]
|
|
261
|
-
fn r8_terminal_recovery_intents_are_idempotent_and_clear_due_time() {
|
|
262
|
-
for status in ["succeeded", "blocked", "exhausted"] {
|
|
263
|
-
let case = RecoveryCase::new(&format!("r8-terminal-{status}"));
|
|
264
|
-
case.seed_agents(&[("fe-admin", 429, "rate_limit", "baseline-429")]);
|
|
265
|
-
case.seed_due_recovery_with_status("fe-admin", status);
|
|
266
|
-
|
|
267
|
-
run_due_recoveries(&case);
|
|
268
|
-
|
|
269
|
-
let events = case.events();
|
|
270
|
-
assert!(
|
|
271
|
-
find_event(&events, "worker.abnormal_exit.recovery_started").is_none(),
|
|
272
|
-
"R8: terminal status `{status}` with stale next_retry_at must be skipped before lifecycle dispatch; events={events:?}"
|
|
273
|
-
);
|
|
274
|
-
let state = case.state();
|
|
275
|
-
let recovery = state
|
|
276
|
-
.pointer("/coordinator/abnormal_api_error_recovery/agents/fe-admin")
|
|
277
|
-
.expect("R8: terminal recovery entry remains auditable");
|
|
278
|
-
assert_eq!(recovery["status"], serde_json::json!(status));
|
|
279
|
-
assert!(
|
|
280
|
-
recovery.get("next_retry_at").is_none()
|
|
281
|
-
|| recovery.get("next_retry_at") == Some(&serde_json::Value::Null),
|
|
282
|
-
"R8: terminal status `{status}` must clear stale next_retry_at so future ticks cannot reuse it; recovery={recovery}"
|
|
283
|
-
);
|
|
284
|
-
}
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
#[test]
|
|
288
|
-
fn r8_non_terminal_due_intents_still_dispatch() {
|
|
289
|
-
for status in ["scheduled", "running"] {
|
|
290
|
-
let case = RecoveryCase::new(&format!("r8-nonterminal-{status}"));
|
|
291
|
-
case.seed_agents(&[("fe-admin", 429, "rate_limit", "baseline-429")]);
|
|
292
|
-
case.seed_due_recovery_with_status("fe-admin", status);
|
|
293
|
-
|
|
294
|
-
run_due_recoveries(&case);
|
|
295
|
-
|
|
296
|
-
let events = case.events();
|
|
297
|
-
assert!(
|
|
298
|
-
find_event(&events, "worker.abnormal_exit.recovery_started").is_some(),
|
|
299
|
-
"R8: non-terminal status `{status}` with due next_retry_at must still dispatch recovery; events={events:?}"
|
|
300
|
-
);
|
|
301
|
-
}
|
|
302
|
-
}
|
|
303
|
-
|
|
304
132
|
struct RecoveryCase {
|
|
305
133
|
root: std::path::PathBuf,
|
|
306
134
|
}
|
|
@@ -312,7 +140,7 @@ impl RecoveryCase {
|
|
|
312
140
|
.unwrap_or_else(std::env::temp_dir);
|
|
313
141
|
std::fs::create_dir_all(&base).unwrap();
|
|
314
142
|
let root = base.join(format!(
|
|
315
|
-
"team-agent-
|
|
143
|
+
"team-agent-remove-auto-recovery-{tag}-{}-{}",
|
|
316
144
|
std::process::id(),
|
|
317
145
|
std::time::SystemTime::now()
|
|
318
146
|
.duration_since(std::time::UNIX_EPOCH)
|
|
@@ -323,37 +151,19 @@ impl RecoveryCase {
|
|
|
323
151
|
Self { root }
|
|
324
152
|
}
|
|
325
153
|
|
|
326
|
-
fn coord(&self) -> Coordinator {
|
|
154
|
+
fn coord(&self, transport: OfflineTransport) -> Coordinator {
|
|
327
155
|
Coordinator::for_test(
|
|
328
156
|
WorkspacePath::new(self.root.clone()),
|
|
329
157
|
Box::new(MockRegistry::new(&[], &[])),
|
|
330
|
-
Box::new(
|
|
158
|
+
Box::new(transport),
|
|
331
159
|
None,
|
|
332
160
|
None,
|
|
333
161
|
)
|
|
334
162
|
}
|
|
335
163
|
|
|
336
|
-
fn
|
|
337
|
-
let
|
|
338
|
-
|
|
339
|
-
let rollout = self.rollout(agent);
|
|
340
|
-
std::fs::write(&rollout, claude_api_error_line(uuid, *status, error)).unwrap();
|
|
341
|
-
agent_map.insert(
|
|
342
|
-
(*agent).to_string(),
|
|
343
|
-
serde_json::json!({
|
|
344
|
-
"agent_id": agent,
|
|
345
|
-
"id": agent,
|
|
346
|
-
"provider": "claude_code",
|
|
347
|
-
"model": "sonnet",
|
|
348
|
-
"status": "running",
|
|
349
|
-
"window": agent,
|
|
350
|
-
"rollout_path": rollout,
|
|
351
|
-
"provider_process_alive": true,
|
|
352
|
-
"spawn_epoch": 1,
|
|
353
|
-
"spawned_at": "2026-07-12T10:28:00Z"
|
|
354
|
-
}),
|
|
355
|
-
);
|
|
356
|
-
}
|
|
164
|
+
fn seed_agent(&self, uuid: &str) {
|
|
165
|
+
let rollout = self.rollout();
|
|
166
|
+
std::fs::write(&rollout, claude_api_error_line(uuid)).unwrap();
|
|
357
167
|
let state = serde_json::json!({
|
|
358
168
|
"active_team_key": "research",
|
|
359
169
|
"team_key": "research",
|
|
@@ -365,44 +175,73 @@ impl RecoveryCase {
|
|
|
365
175
|
"pane_id": "%leader",
|
|
366
176
|
"provider": "codex"
|
|
367
177
|
},
|
|
368
|
-
"agents":
|
|
178
|
+
"agents": {
|
|
179
|
+
"fe-admin": {
|
|
180
|
+
"agent_id": "fe-admin",
|
|
181
|
+
"id": "fe-admin",
|
|
182
|
+
"provider": "claude_code",
|
|
183
|
+
"model": "sonnet",
|
|
184
|
+
"status": "running",
|
|
185
|
+
"window": "fe-admin",
|
|
186
|
+
"rollout_path": rollout,
|
|
187
|
+
"session_id": "session-supermarket",
|
|
188
|
+
"provider_process_alive": true,
|
|
189
|
+
"spawn_epoch": 1,
|
|
190
|
+
"spawned_at": "2026-07-12T10:28:00Z"
|
|
191
|
+
}
|
|
192
|
+
},
|
|
369
193
|
"tasks": []
|
|
370
194
|
});
|
|
371
195
|
crate::state::persist::save_runtime_state(&self.root, &state).unwrap();
|
|
372
196
|
}
|
|
373
197
|
|
|
374
|
-
fn
|
|
375
|
-
self.seed_due_recovery_with_status(agent, "scheduled");
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
fn seed_due_recovery_with_status(&self, agent: &str, status: &str) {
|
|
198
|
+
fn seed_legacy_due_recovery(&self) {
|
|
379
199
|
let mut state = self.state();
|
|
380
|
-
state["coordinator"]["abnormal_api_error_recovery"]["agents"][
|
|
381
|
-
"error_key":
|
|
200
|
+
state["coordinator"]["abnormal_api_error_recovery"]["agents"]["fe-admin"] = serde_json::json!({
|
|
201
|
+
"error_key": "worker.abnormal_exit.error:fe-admin:rollout:api_error:seed",
|
|
382
202
|
"cohort_key": "research:claude_code:api_error:429:rate_limit",
|
|
383
|
-
"status":
|
|
203
|
+
"status": "scheduled",
|
|
384
204
|
"attempts": 0,
|
|
385
205
|
"max_attempts": 2,
|
|
386
206
|
"next_retry_at": "2000-01-01T00:00:00Z",
|
|
387
|
-
"
|
|
388
|
-
|
|
389
|
-
|
|
207
|
+
"manual_command": format!(
|
|
208
|
+
"team-agent start-agent fe-admin --workspace '{}' --team research --force --json",
|
|
209
|
+
self.root.display()
|
|
210
|
+
)
|
|
390
211
|
});
|
|
391
212
|
crate::state::persist::save_runtime_state(&self.root, &state).unwrap();
|
|
392
213
|
}
|
|
393
214
|
|
|
394
|
-
fn
|
|
215
|
+
fn mark_first_turn_dead(&self) {
|
|
216
|
+
let mut state = self.state();
|
|
217
|
+
let agent = &mut state["agents"]["fe-admin"];
|
|
218
|
+
agent["provider_process_alive"] = serde_json::json!(false);
|
|
219
|
+
agent["process_liveness"] = serde_json::json!("dead");
|
|
220
|
+
agent["last_output_at"] = serde_json::Value::Null;
|
|
221
|
+
agent["first_send_at"] = serde_json::Value::Null;
|
|
222
|
+
crate::state::persist::save_runtime_state(&self.root, &state).unwrap();
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
fn seed_healthy_coordinator(&self) {
|
|
226
|
+
let workspace = WorkspacePath::new(self.root.clone());
|
|
227
|
+
let _ = crate::message_store::MessageStore::open(&self.root).unwrap();
|
|
228
|
+
let pid = Pid::new(std::process::id());
|
|
229
|
+
write_coordinator_metadata(&workspace, pid, MetadataSource::Boot).unwrap();
|
|
230
|
+
std::fs::write(coordinator_pid_path(&workspace), pid.to_string()).unwrap();
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
fn append_api_error(&self, uuid: &str) {
|
|
395
234
|
use std::io::Write as _;
|
|
396
235
|
let mut file = std::fs::OpenOptions::new()
|
|
397
236
|
.append(true)
|
|
398
|
-
.open(self.rollout(
|
|
237
|
+
.open(self.rollout())
|
|
399
238
|
.unwrap();
|
|
400
|
-
file.write_all(claude_api_error_line(uuid
|
|
239
|
+
file.write_all(claude_api_error_line(uuid).as_bytes())
|
|
401
240
|
.unwrap();
|
|
402
241
|
}
|
|
403
242
|
|
|
404
|
-
fn rollout(&self
|
|
405
|
-
self.root.join(
|
|
243
|
+
fn rollout(&self) -> std::path::PathBuf {
|
|
244
|
+
self.root.join("rollout-fe-admin.jsonl")
|
|
406
245
|
}
|
|
407
246
|
|
|
408
247
|
fn state(&self) -> serde_json::Value {
|
|
@@ -423,17 +262,6 @@ impl RecoveryCase {
|
|
|
423
262
|
)
|
|
424
263
|
.unwrap_or_default()
|
|
425
264
|
}
|
|
426
|
-
|
|
427
|
-
fn non_leader_message_count(&self) -> i64 {
|
|
428
|
-
let db = crate::model::paths::runtime_dir(&self.root).join("team.db");
|
|
429
|
-
let conn = crate::db::schema::open_db(&db).unwrap();
|
|
430
|
-
conn.query_row(
|
|
431
|
-
"select count(*) from messages where recipient != 'leader'",
|
|
432
|
-
[],
|
|
433
|
-
|row| row.get::<_, i64>(0),
|
|
434
|
-
)
|
|
435
|
-
.unwrap()
|
|
436
|
-
}
|
|
437
265
|
}
|
|
438
266
|
|
|
439
267
|
impl Drop for RecoveryCase {
|
|
@@ -444,37 +272,7 @@ impl Drop for RecoveryCase {
|
|
|
444
272
|
}
|
|
445
273
|
}
|
|
446
274
|
|
|
447
|
-
|
|
448
|
-
struct CounterState {
|
|
449
|
-
adapter_calls: Arc<AtomicU32>,
|
|
450
|
-
}
|
|
451
|
-
|
|
452
|
-
struct CountingRegistry {
|
|
453
|
-
counters: CounterState,
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
impl CountingRegistry {
|
|
457
|
-
fn new() -> Self {
|
|
458
|
-
Self {
|
|
459
|
-
counters: CounterState {
|
|
460
|
-
adapter_calls: Arc::new(AtomicU32::new(0)),
|
|
461
|
-
},
|
|
462
|
-
}
|
|
463
|
-
}
|
|
464
|
-
|
|
465
|
-
fn counters(&self) -> CounterState {
|
|
466
|
-
self.counters.clone()
|
|
467
|
-
}
|
|
468
|
-
}
|
|
469
|
-
|
|
470
|
-
impl ProviderRegistry for CountingRegistry {
|
|
471
|
-
fn adapter_for(&self, provider: Provider) -> Box<dyn ProviderAdapter> {
|
|
472
|
-
self.counters.adapter_calls.fetch_add(1, Ordering::SeqCst);
|
|
473
|
-
crate::provider::get_adapter(provider)
|
|
474
|
-
}
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
fn claude_api_error_line(uuid: &str, status: i64, error: &str) -> String {
|
|
275
|
+
fn claude_api_error_line(uuid: &str) -> String {
|
|
478
276
|
format!(
|
|
479
277
|
"{}\n",
|
|
480
278
|
serde_json::json!({
|
|
@@ -483,11 +281,11 @@ fn claude_api_error_line(uuid: &str, status: i64, error: &str) -> String {
|
|
|
483
281
|
"uuid": uuid,
|
|
484
282
|
"requestId": format!("req-{uuid}"),
|
|
485
283
|
"message": {"role": "assistant", "content": [
|
|
486
|
-
{"type": "text", "text":
|
|
284
|
+
{"type": "text", "text": "API Error: 429 rate_limit"}
|
|
487
285
|
]},
|
|
488
|
-
"error":
|
|
286
|
+
"error": "rate_limit",
|
|
489
287
|
"isApiErrorMessage": true,
|
|
490
|
-
"apiErrorStatus":
|
|
288
|
+
"apiErrorStatus": 429,
|
|
491
289
|
"sessionId": "session-supermarket",
|
|
492
290
|
"version": "2.1.181"
|
|
493
291
|
})
|
|
@@ -500,28 +298,3 @@ fn find_event<'a>(events: &'a [serde_json::Value], name: &str) -> Option<&'a ser
|
|
|
500
298
|
.rev()
|
|
501
299
|
.find(|event| event.get("event").and_then(serde_json::Value::as_str) == Some(name))
|
|
502
300
|
}
|
|
503
|
-
|
|
504
|
-
fn source(rel: &str) -> String {
|
|
505
|
-
std::fs::read_to_string(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(rel))
|
|
506
|
-
.expect("read source")
|
|
507
|
-
}
|
|
508
|
-
|
|
509
|
-
fn detect_abnormal_body() -> String {
|
|
510
|
-
let source = source("src/coordinator/steps/abnormal.rs");
|
|
511
|
-
let start = source
|
|
512
|
-
.find("pub(crate) fn detect_abnormal_exits")
|
|
513
|
-
.expect("detect_abnormal_exits exists");
|
|
514
|
-
let end = source[start..]
|
|
515
|
-
.find("#[derive")
|
|
516
|
-
.map(|offset| start + offset)
|
|
517
|
-
.expect("derive after detect_abnormal_exits");
|
|
518
|
-
source[start..end].to_string()
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
fn run_due_recoveries(case: &RecoveryCase) {
|
|
522
|
-
crate::coordinator::steps::abnormal::attempt_due_recoveries(
|
|
523
|
-
&case.root,
|
|
524
|
-
&crate::event_log::EventLog::new(&case.root),
|
|
525
|
-
&MockTransport::new(true),
|
|
526
|
-
);
|
|
527
|
-
}
|
|
@@ -106,13 +106,6 @@ fn tick_side_effect_order_is_the_fixed_sequence() {
|
|
|
106
106
|
"detect_drift",
|
|
107
107
|
"detect_api_errors",
|
|
108
108
|
"atomic_save",
|
|
109
|
-
// 0.5.36 (`.team/artifacts/supermarket-api-error-recovery-locate.md` §7.3):
|
|
110
|
-
// post-save recovery step reloads fresh state, consumes due
|
|
111
|
-
// recovery intents written pre-save, and drives the lifecycle
|
|
112
|
-
// helper. Sits between atomic_save and collect_results by design
|
|
113
|
-
// so nested lifecycle saves cannot be clobbered by tick's final
|
|
114
|
-
// in-memory save.
|
|
115
|
-
"attempt_api_error_recoveries",
|
|
116
109
|
"collect_results",
|
|
117
110
|
"prune_dedupe_log",
|
|
118
111
|
];
|
|
@@ -470,19 +470,6 @@ impl Coordinator {
|
|
|
470
470
|
})?;
|
|
471
471
|
}
|
|
472
472
|
|
|
473
|
-
// 0.5.36 (`.team/artifacts/supermarket-api-error-recovery-locate.md` §7.3):
|
|
474
|
-
// post-save recovery step. Reloads fresh state, consumes due
|
|
475
|
-
// recovery intents that were WRITTEN during the pre-save detector
|
|
476
|
-
// step, and drives the lifecycle helper. Runs OUTSIDE the pre-save
|
|
477
|
-
// window so nested lifecycle saves cannot be clobbered by this
|
|
478
|
-
// tick's final in-memory save (R6 boundary guard).
|
|
479
|
-
self.record_step(TickStepGroup::Abnormal, "attempt_api_error_recoveries");
|
|
480
|
-
crate::coordinator::steps::abnormal::attempt_due_recoveries(
|
|
481
|
-
self.workspace.as_path(),
|
|
482
|
-
&event_log,
|
|
483
|
-
self.transport.as_ref(),
|
|
484
|
-
);
|
|
485
|
-
|
|
486
473
|
self.record_step(TickStepGroup::Delivery, "collect_results");
|
|
487
474
|
collections.results =
|
|
488
475
|
collect_results(crate::messaging::collect_results_and_notify_watchers(
|