@team-agent/installer 0.5.62 → 0.5.63

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,306 +1,134 @@
1
1
  use super::*;
2
2
 
3
- use std::sync::{
4
- atomic::{AtomicU32, Ordering},
5
- Arc,
6
- };
3
+ use crate::transport::test_support::OfflineTransport;
7
4
 
8
- // 0.5.36 supermarket api_error recovery contracts.
9
- //
10
- // User-facing promise: a fresh retryable provider api_error either schedules a
11
- // bounded, visible worker recovery or loudly explains why not. Recovery must not
12
- // run inside the abnormal detector's pre-save window, must not treat a live-pid
13
- // Noop as success, and must never synthesize a hidden provider turn.
5
+ // Removal contract for the retired abnormal-exit automatic recovery path.
6
+ // Abnormal exits remain visible and keep a copyable manual start-agent hint,
7
+ // but coordinator ticks must never schedule or execute lifecycle work.
14
8
 
15
9
  #[test]
16
- fn r1_retryable_429_schedules_recovery_and_copyable_command() {
17
- let case = RecoveryCase::new("r1-retryable");
18
- case.seed_agents(&[("fe-admin", 429, "rate_limit", "baseline-429")]);
19
- let coord = case.coord();
10
+ fn abnormal_exit_is_reported_without_auto_recovery_intent_or_action() {
11
+ let case = RecoveryCase::new("reported-no-auto");
12
+ case.seed_agent("baseline-429");
13
+ let transport = OfflineTransport::new().with_session_present(true);
14
+ let coord = case.coord(transport.clone());
20
15
  coord.tick().expect("baseline tick");
21
16
 
22
- case.append_api_error("fe-admin", 429, "rate_limit", "fresh-429");
23
- coord.tick().expect("fresh retryable tick");
17
+ case.append_api_error("fresh-429");
18
+ coord.tick().expect("fresh abnormal tick");
24
19
 
25
20
  let events = case.events();
26
21
  let abnormal = find_event(&events, "worker.abnormal_exit").expect("abnormal event");
27
22
  assert_eq!(abnormal["agent_id"], serde_json::json!("fe-admin"));
28
- let scheduled = find_event(&events, "worker.abnormal_exit.recovery_scheduled")
29
- .expect("R1: retryable api_error/429 must emit recovery_scheduled");
30
- assert_eq!(scheduled["agent_id"], serde_json::json!("fe-admin"));
31
- assert_eq!(scheduled["apiErrorStatus"], serde_json::json!(429));
32
- assert_eq!(scheduled["error"], serde_json::json!("rate_limit"));
33
- assert_eq!(scheduled["attempt"], serde_json::json!(0));
23
+ assert_eq!(abnormal["signature"], serde_json::json!("api_error"));
24
+ assert_eq!(abnormal["apiErrorStatus"], serde_json::json!(429));
34
25
  assert!(
35
- scheduled
36
- .get("due_at")
37
- .and_then(serde_json::Value::as_str)
38
- .is_some(),
39
- "R1: recovery_scheduled must include a concrete due_at; event={scheduled}"
26
+ events.iter().all(|event| {
27
+ let name = event
28
+ .get("event")
29
+ .and_then(serde_json::Value::as_str)
30
+ .unwrap_or_default();
31
+ !name.starts_with("worker.abnormal_exit.recovery_")
32
+ && !name.starts_with("worker.abnormal_exit.backpressure_")
33
+ }),
34
+ "abnormal detection must not emit automatic recovery/backpressure events; events={events:?}"
40
35
  );
41
36
 
42
37
  let state = case.state();
43
- let intent = state
44
- .pointer("/coordinator/abnormal_api_error_recovery/agents/fe-admin")
45
- .expect("R1: state must persist recovery intent");
46
- assert_eq!(intent["status"], serde_json::json!("scheduled"));
47
- assert_eq!(intent["attempts"], serde_json::json!(0));
48
38
  assert!(
49
- intent
50
- .get("next_retry_at")
51
- .and_then(serde_json::Value::as_str)
52
- .is_some(),
53
- "R1: intent must carry next_retry_at; intent={intent}"
39
+ state
40
+ .pointer("/coordinator/abnormal_api_error_recovery")
41
+ .is_none(),
42
+ "abnormal detection must not persist automatic recovery intent; state={state}"
43
+ );
44
+ assert!(
45
+ transport.spawn_records().is_empty(),
46
+ "abnormal detection must not respawn a worker; spawns={:?}",
47
+ transport.spawn_records()
54
48
  );
55
49
 
56
50
  let content = case.latest_leader_notification();
57
51
  assert!(
58
- content.contains("team-agent start-agent fe-admin")
52
+ content.contains("No automatic restart was performed.")
53
+ && content.contains("To recover manually, run: team-agent start-agent fe-admin")
59
54
  && content.contains("--workspace")
60
55
  && content.contains("--team research")
61
56
  && content.contains("--force")
62
57
  && content.contains("--json"),
63
- "R1: leader notification must include one copyable forced start-agent command; content={content}"
64
- );
65
- assert!(
66
- !content.contains("No automatic restart was performed."),
67
- "R1: old unconditional no-auto-restart text must be replaced; content={content}"
58
+ "abnormal notification must remain visible and keep the copyable manual recovery command; content={content}"
68
59
  );
69
60
  }
70
61
 
71
62
  #[test]
72
- fn r2_non_retryable_api_error_guides_only_without_intent() {
73
- let case = RecoveryCase::new("r2-nonretry");
74
- case.seed_agents(&[("fe-admin", 404, "model_not_found", "baseline-404")]);
75
- let coord = case.coord();
63
+ fn first_turn_death_cannot_enter_auto_respawn_loop_and_manual_start_still_recovers() {
64
+ let case = RecoveryCase::new("first-turn-death-no-loop");
65
+ case.seed_agent("baseline-429");
66
+ let transport = OfflineTransport::new().with_session_present(true);
67
+ let coord = case.coord(transport.clone());
76
68
  coord.tick().expect("baseline tick");
77
69
 
78
- case.append_api_error("fe-admin", 404, "model_not_found", "fresh-404");
79
- coord.tick().expect("fresh non-retryable tick");
80
-
81
- let state = case.state();
82
- assert!(
83
- state
84
- .pointer("/coordinator/abnormal_api_error_recovery/agents/fe-admin")
85
- .is_none(),
86
- "R2: non-retryable model errors must not schedule auto recovery; state={state}"
87
- );
88
- let content = case.latest_leader_notification();
89
- assert!(
90
- content.contains("non_retryable_api_error")
91
- && content.contains("model_not_found")
92
- && content.contains("team-agent start-agent fe-admin")
93
- && content.contains("--force"),
94
- "R2: notification must be guide-only with model/config context and manual command; content={content}"
95
- );
96
- }
97
-
98
- #[test]
99
- fn r3_live_pid_noop_is_never_counted_as_recovery_success() {
100
- let abnormal = source("src/coordinator/steps/abnormal.rs");
101
- let lifecycle = source("src/lifecycle/restart/agent.rs");
102
- let recovery_mentions_noop_block = abnormal.contains("noop_not_recovery")
103
- && abnormal.contains("worker.abnormal_exit.recovery_blocked");
104
- let lifecycle_has_stop_then_start_mode =
105
- abnormal.contains("stop_then_start") || lifecycle.contains("stop_before_start");
106
- assert!(
107
- recovery_mentions_noop_block || lifecycle_has_stop_then_start_mode,
108
- "R3: api_error recovery must either stop-before-start live panes or record recovery_blocked(reason=noop_not_recovery); abnormal.rs/lifecycle.rs missing both guards"
109
- );
110
- }
111
-
112
- #[test]
113
- fn r4_short_window_429_cohort_enters_backpressure_with_one_canary() {
114
- let case = RecoveryCase::new("r4-backpressure");
115
- let agents = [
116
- ("architect", 429, "rate_limit", "baseline-a"),
117
- ("fe-admin", 429, "rate_limit", "baseline-b"),
118
- ("fe-staff", 429, "rate_limit", "baseline-c"),
119
- ("fe-shop", 429, "rate_limit", "baseline-d"),
120
- ];
121
- case.seed_agents(&agents);
122
- let coord = case.coord();
123
- coord.tick().expect("baseline tick");
124
- for (agent, _, _, _) in agents {
125
- case.append_api_error(agent, 429, "rate_limit", &format!("fresh-{agent}"));
126
- }
127
- coord.tick().expect("fresh cohort tick");
70
+ // A-29 shape: the replacement worker dies before completing its first
71
+ // turn, with a fresh explicit provider error. The following tick reports
72
+ // the death; a persisted due row models the next 6.5-second loop turn.
73
+ case.mark_first_turn_dead();
74
+ case.append_api_error("fresh-first-turn-death");
75
+ coord.tick().expect("fresh first-turn death tick");
76
+ case.seed_legacy_due_recovery();
77
+ coord.tick().expect("tick with legacy due intent");
128
78
 
129
79
  let events = case.events();
130
- let backpressure = find_event(&events, "worker.abnormal_exit.backpressure_started")
131
- .expect("R4: four same-team/provider 429s inside 120s must start backpressure");
132
- assert_eq!(backpressure["team_id"], serde_json::json!("research"));
133
- assert_eq!(backpressure["provider"], serde_json::json!("claude_code"));
134
- assert_eq!(backpressure["apiErrorStatus"], serde_json::json!(429));
80
+ let abnormal = find_event(&events, "worker.abnormal_exit").expect("abnormal event");
81
+ assert_eq!(abnormal["dead_process"], serde_json::json!(true));
82
+ assert_eq!(abnormal["fresh_error"], serde_json::json!(true));
135
83
  assert!(
136
- backpressure
137
- .get("cooldown_until")
138
- .and_then(serde_json::Value::as_str)
139
- .is_some(),
140
- "R4: backpressure event must include shared cooldown_until; event={backpressure}"
84
+ events.iter().all(|event| {
85
+ let name = event
86
+ .get("event")
87
+ .and_then(serde_json::Value::as_str)
88
+ .unwrap_or_default();
89
+ !name.starts_with("worker.abnormal_exit.recovery_")
90
+ && !name.starts_with("worker.abnormal_exit.backpressure_")
91
+ }),
92
+ "first-turn death must report only; automatic recovery events make the A-29 loop possible; events={events:?}"
141
93
  );
142
-
143
- let scheduled_count = events
144
- .iter()
145
- .filter(|event| {
146
- event.get("event").and_then(serde_json::Value::as_str)
147
- == Some("worker.abnormal_exit.recovery_scheduled")
148
- })
149
- .count();
150
94
  assert!(
151
- scheduled_count <= 1,
152
- "R4: at most one canary recovery may be scheduled while cohort is backpressured; scheduled_count={scheduled_count}"
95
+ transport.spawn_records().is_empty(),
96
+ "first-turn death and legacy due intents must never respawn; spawns={:?}",
97
+ transport.spawn_records()
153
98
  );
154
- let state = case.state();
155
- let bp = state
156
- .pointer("/coordinator/abnormal_api_error_recovery/backpressure")
157
- .and_then(serde_json::Value::as_object)
158
- .expect("R4: backpressure state must be persisted");
159
- assert_eq!(
160
- bp.len(),
161
- 1,
162
- "R4: cohort must share one backpressure record; state={state}"
163
- );
164
- }
165
-
166
- #[test]
167
- fn r5_retry_budget_exhausts_and_stays_loud() {
168
- let abnormal = source("src/coordinator/steps/abnormal.rs");
169
- for needle in [
170
- "max_attempts",
171
- "worker.abnormal_exit.recovery_exhausted",
172
- "manual_command",
173
- "attempts",
174
- ] {
175
- assert!(
176
- abnormal.contains(needle),
177
- "R5: retry budget contract requires `{needle}` in abnormal api_error recovery state/events"
178
- );
179
- }
180
- }
181
-
182
- #[test]
183
- fn r6_abnormal_detector_does_not_mutate_lifecycle_inside_pre_save_window() {
184
- let body = detect_abnormal_body();
185
- for forbidden in [
186
- "start_agent_at_paths",
187
- "start_agent(",
188
- "stop_agent_at_paths",
189
- "reset_agent",
190
- ] {
191
- assert!(
192
- !body.contains(forbidden),
193
- "R6: detect_abnormal_exits is pre-save detection only and must not call `{forbidden}`"
194
- );
195
- }
196
- }
197
-
198
- #[test]
199
- fn r6_recovery_fields_survive_the_next_tick_after_atomic_save() {
200
- let case = RecoveryCase::new("r6-save-boundary");
201
- case.seed_agents(&[("fe-admin", 429, "rate_limit", "baseline-429")]);
202
- case.seed_due_recovery("fe-admin");
203
- let coord = case.coord();
204
- coord.tick().expect("recovery tick");
205
-
206
- let events = case.events();
207
99
  assert!(
208
- find_event(&events, "worker.abnormal_exit.recovery_started").is_some()
209
- || find_event(&events, "worker.abnormal_exit.recovery_blocked").is_some(),
210
- "R6: due recovery must run after atomic_save and emit started or blocked; events={events:?}"
100
+ transport.calls().iter().all(|call| !matches!(
101
+ *call,
102
+ "spawn_first" | "spawn_into" | "kill_pane" | "kill_window"
103
+ )),
104
+ "automatic loop prevention must be structural: no lifecycle transport action; calls={:?}",
105
+ transport.calls()
211
106
  );
212
- let state = case.state();
213
- let recovery = state
214
- .pointer("/coordinator/abnormal_api_error_recovery/agents/fe-admin")
215
- .expect("R6: recovery entry must survive tick");
216
- assert!(
217
- recovery.get("last_attempt_at").is_some()
218
- || recovery.get("last_error").is_some()
219
- || recovery.get("blocked_reason").is_some(),
220
- "R6: lifecycle-written recovery result fields must survive next tick save; recovery={recovery}"
221
- );
222
- }
223
107
 
224
- #[test]
225
- fn r7_recovery_never_synthesizes_hidden_provider_turn_or_sdk_call() {
226
- let case = RecoveryCase::new("r7-no-hidden-turn");
227
- case.seed_agents(&[("fe-admin", 429, "rate_limit", "baseline-429")]);
228
- case.seed_due_recovery("fe-admin");
229
- let registry = CountingRegistry::new();
230
- let counters = registry.counters();
231
- let transport = MockTransport::new(true);
232
- let calls = std::sync::Arc::clone(&transport.calls);
233
- let coord = Coordinator::for_test(
234
- WorkspacePath::new(case.root.clone()),
235
- Box::new(registry),
236
- Box::new(transport),
237
- None,
108
+ case.seed_healthy_coordinator();
109
+ let manual_transport = OfflineTransport::new().with_session_present(true);
110
+ let outcome = crate::lifecycle::start_agent_with_transport(
111
+ &case.root,
112
+ &crate::model::ids::AgentId::new("fe-admin"),
113
+ false,
114
+ false,
115
+ false,
238
116
  None,
117
+ &manual_transport,
239
118
  );
240
119
 
241
- coord.tick().expect("recovery tick");
242
-
243
- assert_eq!(
244
- counters.adapter_calls.load(Ordering::SeqCst),
245
- 0,
246
- "R7: recovery must not call provider adapter/client APIs"
247
- );
248
120
  assert!(
249
- !calls.lock().unwrap().iter().any(|call| *call == "inject"),
250
- "R7: recovery must not inject a synthetic worker turn; transport calls={:?}",
251
- calls.lock().unwrap()
121
+ matches!(outcome, Ok(crate::lifecycle::StartAgentOutcome::Running { .. })),
122
+ "manual start-agent path must remain functional after an abnormal exit; outcome={outcome:?}"
252
123
  );
253
124
  assert_eq!(
254
- case.non_leader_message_count(),
255
- 0,
256
- "R7: recovery may notify leader, but must not create hidden worker prompt messages"
125
+ manual_transport.spawn_records().len(),
126
+ 1,
127
+ "manual start-agent must spawn exactly one replacement worker; spawns={:?}",
128
+ manual_transport.spawn_records()
257
129
  );
258
130
  }
259
131
 
260
- #[test]
261
- fn r8_terminal_recovery_intents_are_idempotent_and_clear_due_time() {
262
- for status in ["succeeded", "blocked", "exhausted"] {
263
- let case = RecoveryCase::new(&format!("r8-terminal-{status}"));
264
- case.seed_agents(&[("fe-admin", 429, "rate_limit", "baseline-429")]);
265
- case.seed_due_recovery_with_status("fe-admin", status);
266
-
267
- run_due_recoveries(&case);
268
-
269
- let events = case.events();
270
- assert!(
271
- find_event(&events, "worker.abnormal_exit.recovery_started").is_none(),
272
- "R8: terminal status `{status}` with stale next_retry_at must be skipped before lifecycle dispatch; events={events:?}"
273
- );
274
- let state = case.state();
275
- let recovery = state
276
- .pointer("/coordinator/abnormal_api_error_recovery/agents/fe-admin")
277
- .expect("R8: terminal recovery entry remains auditable");
278
- assert_eq!(recovery["status"], serde_json::json!(status));
279
- assert!(
280
- recovery.get("next_retry_at").is_none()
281
- || recovery.get("next_retry_at") == Some(&serde_json::Value::Null),
282
- "R8: terminal status `{status}` must clear stale next_retry_at so future ticks cannot reuse it; recovery={recovery}"
283
- );
284
- }
285
- }
286
-
287
- #[test]
288
- fn r8_non_terminal_due_intents_still_dispatch() {
289
- for status in ["scheduled", "running"] {
290
- let case = RecoveryCase::new(&format!("r8-nonterminal-{status}"));
291
- case.seed_agents(&[("fe-admin", 429, "rate_limit", "baseline-429")]);
292
- case.seed_due_recovery_with_status("fe-admin", status);
293
-
294
- run_due_recoveries(&case);
295
-
296
- let events = case.events();
297
- assert!(
298
- find_event(&events, "worker.abnormal_exit.recovery_started").is_some(),
299
- "R8: non-terminal status `{status}` with due next_retry_at must still dispatch recovery; events={events:?}"
300
- );
301
- }
302
- }
303
-
304
132
  struct RecoveryCase {
305
133
  root: std::path::PathBuf,
306
134
  }
@@ -312,7 +140,7 @@ impl RecoveryCase {
312
140
  .unwrap_or_else(std::env::temp_dir);
313
141
  std::fs::create_dir_all(&base).unwrap();
314
142
  let root = base.join(format!(
315
- "team-agent-0536-{tag}-{}-{}",
143
+ "team-agent-remove-auto-recovery-{tag}-{}-{}",
316
144
  std::process::id(),
317
145
  std::time::SystemTime::now()
318
146
  .duration_since(std::time::UNIX_EPOCH)
@@ -323,37 +151,19 @@ impl RecoveryCase {
323
151
  Self { root }
324
152
  }
325
153
 
326
- fn coord(&self) -> Coordinator {
154
+ fn coord(&self, transport: OfflineTransport) -> Coordinator {
327
155
  Coordinator::for_test(
328
156
  WorkspacePath::new(self.root.clone()),
329
157
  Box::new(MockRegistry::new(&[], &[])),
330
- Box::new(MockTransport::new(true)),
158
+ Box::new(transport),
331
159
  None,
332
160
  None,
333
161
  )
334
162
  }
335
163
 
336
- fn seed_agents(&self, agents: &[(&str, i64, &str, &str)]) {
337
- let mut agent_map = serde_json::Map::new();
338
- for (agent, status, error, uuid) in agents {
339
- let rollout = self.rollout(agent);
340
- std::fs::write(&rollout, claude_api_error_line(uuid, *status, error)).unwrap();
341
- agent_map.insert(
342
- (*agent).to_string(),
343
- serde_json::json!({
344
- "agent_id": agent,
345
- "id": agent,
346
- "provider": "claude_code",
347
- "model": "sonnet",
348
- "status": "running",
349
- "window": agent,
350
- "rollout_path": rollout,
351
- "provider_process_alive": true,
352
- "spawn_epoch": 1,
353
- "spawned_at": "2026-07-12T10:28:00Z"
354
- }),
355
- );
356
- }
164
+ fn seed_agent(&self, uuid: &str) {
165
+ let rollout = self.rollout();
166
+ std::fs::write(&rollout, claude_api_error_line(uuid)).unwrap();
357
167
  let state = serde_json::json!({
358
168
  "active_team_key": "research",
359
169
  "team_key": "research",
@@ -365,44 +175,73 @@ impl RecoveryCase {
365
175
  "pane_id": "%leader",
366
176
  "provider": "codex"
367
177
  },
368
- "agents": serde_json::Value::Object(agent_map),
178
+ "agents": {
179
+ "fe-admin": {
180
+ "agent_id": "fe-admin",
181
+ "id": "fe-admin",
182
+ "provider": "claude_code",
183
+ "model": "sonnet",
184
+ "status": "running",
185
+ "window": "fe-admin",
186
+ "rollout_path": rollout,
187
+ "session_id": "session-supermarket",
188
+ "provider_process_alive": true,
189
+ "spawn_epoch": 1,
190
+ "spawned_at": "2026-07-12T10:28:00Z"
191
+ }
192
+ },
369
193
  "tasks": []
370
194
  });
371
195
  crate::state::persist::save_runtime_state(&self.root, &state).unwrap();
372
196
  }
373
197
 
374
- fn seed_due_recovery(&self, agent: &str) {
375
- self.seed_due_recovery_with_status(agent, "scheduled");
376
- }
377
-
378
- fn seed_due_recovery_with_status(&self, agent: &str, status: &str) {
198
+ fn seed_legacy_due_recovery(&self) {
379
199
  let mut state = self.state();
380
- state["coordinator"]["abnormal_api_error_recovery"]["agents"][agent] = serde_json::json!({
381
- "error_key": format!("worker.abnormal_exit.error:{agent}:rollout:api_error:seed"),
200
+ state["coordinator"]["abnormal_api_error_recovery"]["agents"]["fe-admin"] = serde_json::json!({
201
+ "error_key": "worker.abnormal_exit.error:fe-admin:rollout:api_error:seed",
382
202
  "cohort_key": "research:claude_code:api_error:429:rate_limit",
383
- "status": status,
203
+ "status": "scheduled",
384
204
  "attempts": 0,
385
205
  "max_attempts": 2,
386
206
  "next_retry_at": "2000-01-01T00:00:00Z",
387
- "last_attempt_at": null,
388
- "last_error": null,
389
- "manual_command": format!("team-agent start-agent {agent} --workspace '{}' --team research --force --json", self.root.display())
207
+ "manual_command": format!(
208
+ "team-agent start-agent fe-admin --workspace '{}' --team research --force --json",
209
+ self.root.display()
210
+ )
390
211
  });
391
212
  crate::state::persist::save_runtime_state(&self.root, &state).unwrap();
392
213
  }
393
214
 
394
- fn append_api_error(&self, agent: &str, status: i64, error: &str, uuid: &str) {
215
+ fn mark_first_turn_dead(&self) {
216
+ let mut state = self.state();
217
+ let agent = &mut state["agents"]["fe-admin"];
218
+ agent["provider_process_alive"] = serde_json::json!(false);
219
+ agent["process_liveness"] = serde_json::json!("dead");
220
+ agent["last_output_at"] = serde_json::Value::Null;
221
+ agent["first_send_at"] = serde_json::Value::Null;
222
+ crate::state::persist::save_runtime_state(&self.root, &state).unwrap();
223
+ }
224
+
225
+ fn seed_healthy_coordinator(&self) {
226
+ let workspace = WorkspacePath::new(self.root.clone());
227
+ let _ = crate::message_store::MessageStore::open(&self.root).unwrap();
228
+ let pid = Pid::new(std::process::id());
229
+ write_coordinator_metadata(&workspace, pid, MetadataSource::Boot).unwrap();
230
+ std::fs::write(coordinator_pid_path(&workspace), pid.to_string()).unwrap();
231
+ }
232
+
233
+ fn append_api_error(&self, uuid: &str) {
395
234
  use std::io::Write as _;
396
235
  let mut file = std::fs::OpenOptions::new()
397
236
  .append(true)
398
- .open(self.rollout(agent))
237
+ .open(self.rollout())
399
238
  .unwrap();
400
- file.write_all(claude_api_error_line(uuid, status, error).as_bytes())
239
+ file.write_all(claude_api_error_line(uuid).as_bytes())
401
240
  .unwrap();
402
241
  }
403
242
 
404
- fn rollout(&self, agent: &str) -> std::path::PathBuf {
405
- self.root.join(format!("rollout-{agent}.jsonl"))
243
+ fn rollout(&self) -> std::path::PathBuf {
244
+ self.root.join("rollout-fe-admin.jsonl")
406
245
  }
407
246
 
408
247
  fn state(&self) -> serde_json::Value {
@@ -423,17 +262,6 @@ impl RecoveryCase {
423
262
  )
424
263
  .unwrap_or_default()
425
264
  }
426
-
427
- fn non_leader_message_count(&self) -> i64 {
428
- let db = crate::model::paths::runtime_dir(&self.root).join("team.db");
429
- let conn = crate::db::schema::open_db(&db).unwrap();
430
- conn.query_row(
431
- "select count(*) from messages where recipient != 'leader'",
432
- [],
433
- |row| row.get::<_, i64>(0),
434
- )
435
- .unwrap()
436
- }
437
265
  }
438
266
 
439
267
  impl Drop for RecoveryCase {
@@ -444,37 +272,7 @@ impl Drop for RecoveryCase {
444
272
  }
445
273
  }
446
274
 
447
- #[derive(Clone)]
448
- struct CounterState {
449
- adapter_calls: Arc<AtomicU32>,
450
- }
451
-
452
- struct CountingRegistry {
453
- counters: CounterState,
454
- }
455
-
456
- impl CountingRegistry {
457
- fn new() -> Self {
458
- Self {
459
- counters: CounterState {
460
- adapter_calls: Arc::new(AtomicU32::new(0)),
461
- },
462
- }
463
- }
464
-
465
- fn counters(&self) -> CounterState {
466
- self.counters.clone()
467
- }
468
- }
469
-
470
- impl ProviderRegistry for CountingRegistry {
471
- fn adapter_for(&self, provider: Provider) -> Box<dyn ProviderAdapter> {
472
- self.counters.adapter_calls.fetch_add(1, Ordering::SeqCst);
473
- crate::provider::get_adapter(provider)
474
- }
475
- }
476
-
477
- fn claude_api_error_line(uuid: &str, status: i64, error: &str) -> String {
275
+ fn claude_api_error_line(uuid: &str) -> String {
478
276
  format!(
479
277
  "{}\n",
480
278
  serde_json::json!({
@@ -483,11 +281,11 @@ fn claude_api_error_line(uuid: &str, status: i64, error: &str) -> String {
483
281
  "uuid": uuid,
484
282
  "requestId": format!("req-{uuid}"),
485
283
  "message": {"role": "assistant", "content": [
486
- {"type": "text", "text": format!("API Error: {status} {error}")}
284
+ {"type": "text", "text": "API Error: 429 rate_limit"}
487
285
  ]},
488
- "error": error,
286
+ "error": "rate_limit",
489
287
  "isApiErrorMessage": true,
490
- "apiErrorStatus": status,
288
+ "apiErrorStatus": 429,
491
289
  "sessionId": "session-supermarket",
492
290
  "version": "2.1.181"
493
291
  })
@@ -500,28 +298,3 @@ fn find_event<'a>(events: &'a [serde_json::Value], name: &str) -> Option<&'a ser
500
298
  .rev()
501
299
  .find(|event| event.get("event").and_then(serde_json::Value::as_str) == Some(name))
502
300
  }
503
-
504
- fn source(rel: &str) -> String {
505
- std::fs::read_to_string(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(rel))
506
- .expect("read source")
507
- }
508
-
509
- fn detect_abnormal_body() -> String {
510
- let source = source("src/coordinator/steps/abnormal.rs");
511
- let start = source
512
- .find("pub(crate) fn detect_abnormal_exits")
513
- .expect("detect_abnormal_exits exists");
514
- let end = source[start..]
515
- .find("#[derive")
516
- .map(|offset| start + offset)
517
- .expect("derive after detect_abnormal_exits");
518
- source[start..end].to_string()
519
- }
520
-
521
- fn run_due_recoveries(case: &RecoveryCase) {
522
- crate::coordinator::steps::abnormal::attempt_due_recoveries(
523
- &case.root,
524
- &crate::event_log::EventLog::new(&case.root),
525
- &MockTransport::new(true),
526
- );
527
- }
@@ -106,13 +106,6 @@ fn tick_side_effect_order_is_the_fixed_sequence() {
106
106
  "detect_drift",
107
107
  "detect_api_errors",
108
108
  "atomic_save",
109
- // 0.5.36 (`.team/artifacts/supermarket-api-error-recovery-locate.md` §7.3):
110
- // post-save recovery step reloads fresh state, consumes due
111
- // recovery intents written pre-save, and drives the lifecycle
112
- // helper. Sits between atomic_save and collect_results by design
113
- // so nested lifecycle saves cannot be clobbered by tick's final
114
- // in-memory save.
115
- "attempt_api_error_recoveries",
116
109
  "collect_results",
117
110
  "prune_dedupe_log",
118
111
  ];
@@ -470,19 +470,6 @@ impl Coordinator {
470
470
  })?;
471
471
  }
472
472
 
473
- // 0.5.36 (`.team/artifacts/supermarket-api-error-recovery-locate.md` §7.3):
474
- // post-save recovery step. Reloads fresh state, consumes due
475
- // recovery intents that were WRITTEN during the pre-save detector
476
- // step, and drives the lifecycle helper. Runs OUTSIDE the pre-save
477
- // window so nested lifecycle saves cannot be clobbered by this
478
- // tick's final in-memory save (R6 boundary guard).
479
- self.record_step(TickStepGroup::Abnormal, "attempt_api_error_recoveries");
480
- crate::coordinator::steps::abnormal::attempt_due_recoveries(
481
- self.workspace.as_path(),
482
- &event_log,
483
- self.transport.as_ref(),
484
- );
485
-
486
473
  self.record_step(TickStepGroup::Delivery, "collect_results");
487
474
  collections.results =
488
475
  collect_results(crate::messaging::collect_results_and_notify_watchers(