@team-agent/installer 0.5.35 → 0.5.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -1
- package/Cargo.toml +1 -1
- package/crates/team-agent/src/coordinator/steps/abnormal.rs +810 -2
- package/crates/team-agent/src/coordinator/tests/api_error_recovery.rs +480 -0
- package/crates/team-agent/src/coordinator/tests/mod.rs +1 -0
- package/crates/team-agent/src/coordinator/tests/tick_core.rs +7 -0
- package/crates/team-agent/src/coordinator/tick.rs +13 -0
- package/crates/team-agent/src/lifecycle/restart/agent.rs +65 -0
- package/crates/team-agent/src/lifecycle/restart.rs +4 -0
- package/crates/team-agent/src/state/repository.rs +15 -0
- package/package.json +4 -4
|
@@ -0,0 +1,480 @@
|
|
|
1
|
+
use super::*;
|
|
2
|
+
|
|
3
|
+
use std::sync::{
|
|
4
|
+
atomic::{AtomicU32, Ordering},
|
|
5
|
+
Arc,
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
// 0.5.36 supermarket api_error recovery contracts.
|
|
9
|
+
//
|
|
10
|
+
// User-facing promise: a fresh retryable provider api_error either schedules a
|
|
11
|
+
// bounded, visible worker recovery or loudly explains why not. Recovery must not
|
|
12
|
+
// run inside the abnormal detector's pre-save window, must not treat a live-pid
|
|
13
|
+
// Noop as success, and must never synthesize a hidden provider turn.
|
|
14
|
+
|
|
15
|
+
#[test]
|
|
16
|
+
fn r1_retryable_429_schedules_recovery_and_copyable_command() {
|
|
17
|
+
let case = RecoveryCase::new("r1-retryable");
|
|
18
|
+
case.seed_agents(&[("fe-admin", 429, "rate_limit", "baseline-429")]);
|
|
19
|
+
let coord = case.coord();
|
|
20
|
+
coord.tick().expect("baseline tick");
|
|
21
|
+
|
|
22
|
+
case.append_api_error("fe-admin", 429, "rate_limit", "fresh-429");
|
|
23
|
+
coord.tick().expect("fresh retryable tick");
|
|
24
|
+
|
|
25
|
+
let events = case.events();
|
|
26
|
+
let abnormal = find_event(&events, "worker.abnormal_exit").expect("abnormal event");
|
|
27
|
+
assert_eq!(abnormal["agent_id"], serde_json::json!("fe-admin"));
|
|
28
|
+
let scheduled = find_event(&events, "worker.abnormal_exit.recovery_scheduled")
|
|
29
|
+
.expect("R1: retryable api_error/429 must emit recovery_scheduled");
|
|
30
|
+
assert_eq!(scheduled["agent_id"], serde_json::json!("fe-admin"));
|
|
31
|
+
assert_eq!(scheduled["apiErrorStatus"], serde_json::json!(429));
|
|
32
|
+
assert_eq!(scheduled["error"], serde_json::json!("rate_limit"));
|
|
33
|
+
assert_eq!(scheduled["attempt"], serde_json::json!(0));
|
|
34
|
+
assert!(
|
|
35
|
+
scheduled
|
|
36
|
+
.get("due_at")
|
|
37
|
+
.and_then(serde_json::Value::as_str)
|
|
38
|
+
.is_some(),
|
|
39
|
+
"R1: recovery_scheduled must include a concrete due_at; event={scheduled}"
|
|
40
|
+
);
|
|
41
|
+
|
|
42
|
+
let state = case.state();
|
|
43
|
+
let intent = state
|
|
44
|
+
.pointer("/coordinator/abnormal_api_error_recovery/agents/fe-admin")
|
|
45
|
+
.expect("R1: state must persist recovery intent");
|
|
46
|
+
assert_eq!(intent["status"], serde_json::json!("scheduled"));
|
|
47
|
+
assert_eq!(intent["attempts"], serde_json::json!(0));
|
|
48
|
+
assert!(
|
|
49
|
+
intent
|
|
50
|
+
.get("next_retry_at")
|
|
51
|
+
.and_then(serde_json::Value::as_str)
|
|
52
|
+
.is_some(),
|
|
53
|
+
"R1: intent must carry next_retry_at; intent={intent}"
|
|
54
|
+
);
|
|
55
|
+
|
|
56
|
+
let content = case.latest_leader_notification();
|
|
57
|
+
assert!(
|
|
58
|
+
content.contains("team-agent start-agent fe-admin")
|
|
59
|
+
&& content.contains("--workspace")
|
|
60
|
+
&& content.contains("--team research")
|
|
61
|
+
&& content.contains("--force")
|
|
62
|
+
&& content.contains("--json"),
|
|
63
|
+
"R1: leader notification must include one copyable forced start-agent command; content={content}"
|
|
64
|
+
);
|
|
65
|
+
assert!(
|
|
66
|
+
!content.contains("No automatic restart was performed."),
|
|
67
|
+
"R1: old unconditional no-auto-restart text must be replaced; content={content}"
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
#[test]
|
|
72
|
+
fn r2_non_retryable_api_error_guides_only_without_intent() {
|
|
73
|
+
let case = RecoveryCase::new("r2-nonretry");
|
|
74
|
+
case.seed_agents(&[("fe-admin", 404, "model_not_found", "baseline-404")]);
|
|
75
|
+
let coord = case.coord();
|
|
76
|
+
coord.tick().expect("baseline tick");
|
|
77
|
+
|
|
78
|
+
case.append_api_error("fe-admin", 404, "model_not_found", "fresh-404");
|
|
79
|
+
coord.tick().expect("fresh non-retryable tick");
|
|
80
|
+
|
|
81
|
+
let state = case.state();
|
|
82
|
+
assert!(
|
|
83
|
+
state
|
|
84
|
+
.pointer("/coordinator/abnormal_api_error_recovery/agents/fe-admin")
|
|
85
|
+
.is_none(),
|
|
86
|
+
"R2: non-retryable model errors must not schedule auto recovery; state={state}"
|
|
87
|
+
);
|
|
88
|
+
let content = case.latest_leader_notification();
|
|
89
|
+
assert!(
|
|
90
|
+
content.contains("non_retryable_api_error")
|
|
91
|
+
&& content.contains("model_not_found")
|
|
92
|
+
&& content.contains("team-agent start-agent fe-admin")
|
|
93
|
+
&& content.contains("--force"),
|
|
94
|
+
"R2: notification must be guide-only with model/config context and manual command; content={content}"
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
#[test]
|
|
99
|
+
fn r3_live_pid_noop_is_never_counted_as_recovery_success() {
|
|
100
|
+
let abnormal = source("src/coordinator/steps/abnormal.rs");
|
|
101
|
+
let lifecycle = source("src/lifecycle/restart/agent.rs");
|
|
102
|
+
let recovery_mentions_noop_block = abnormal.contains("noop_not_recovery")
|
|
103
|
+
&& abnormal.contains("worker.abnormal_exit.recovery_blocked");
|
|
104
|
+
let lifecycle_has_stop_then_start_mode =
|
|
105
|
+
abnormal.contains("stop_then_start") || lifecycle.contains("stop_before_start");
|
|
106
|
+
assert!(
|
|
107
|
+
recovery_mentions_noop_block || lifecycle_has_stop_then_start_mode,
|
|
108
|
+
"R3: api_error recovery must either stop-before-start live panes or record recovery_blocked(reason=noop_not_recovery); abnormal.rs/lifecycle.rs missing both guards"
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
#[test]
|
|
113
|
+
fn r4_short_window_429_cohort_enters_backpressure_with_one_canary() {
|
|
114
|
+
let case = RecoveryCase::new("r4-backpressure");
|
|
115
|
+
let agents = [
|
|
116
|
+
("architect", 429, "rate_limit", "baseline-a"),
|
|
117
|
+
("fe-admin", 429, "rate_limit", "baseline-b"),
|
|
118
|
+
("fe-staff", 429, "rate_limit", "baseline-c"),
|
|
119
|
+
("fe-shop", 429, "rate_limit", "baseline-d"),
|
|
120
|
+
];
|
|
121
|
+
case.seed_agents(&agents);
|
|
122
|
+
let coord = case.coord();
|
|
123
|
+
coord.tick().expect("baseline tick");
|
|
124
|
+
for (agent, _, _, _) in agents {
|
|
125
|
+
case.append_api_error(agent, 429, "rate_limit", &format!("fresh-{agent}"));
|
|
126
|
+
}
|
|
127
|
+
coord.tick().expect("fresh cohort tick");
|
|
128
|
+
|
|
129
|
+
let events = case.events();
|
|
130
|
+
let backpressure = find_event(&events, "worker.abnormal_exit.backpressure_started")
|
|
131
|
+
.expect("R4: four same-team/provider 429s inside 120s must start backpressure");
|
|
132
|
+
assert_eq!(backpressure["team_id"], serde_json::json!("research"));
|
|
133
|
+
assert_eq!(backpressure["provider"], serde_json::json!("claude_code"));
|
|
134
|
+
assert_eq!(backpressure["apiErrorStatus"], serde_json::json!(429));
|
|
135
|
+
assert!(
|
|
136
|
+
backpressure
|
|
137
|
+
.get("cooldown_until")
|
|
138
|
+
.and_then(serde_json::Value::as_str)
|
|
139
|
+
.is_some(),
|
|
140
|
+
"R4: backpressure event must include shared cooldown_until; event={backpressure}"
|
|
141
|
+
);
|
|
142
|
+
|
|
143
|
+
let scheduled_count = events
|
|
144
|
+
.iter()
|
|
145
|
+
.filter(|event| {
|
|
146
|
+
event.get("event").and_then(serde_json::Value::as_str)
|
|
147
|
+
== Some("worker.abnormal_exit.recovery_scheduled")
|
|
148
|
+
})
|
|
149
|
+
.count();
|
|
150
|
+
assert!(
|
|
151
|
+
scheduled_count <= 1,
|
|
152
|
+
"R4: at most one canary recovery may be scheduled while cohort is backpressured; scheduled_count={scheduled_count}"
|
|
153
|
+
);
|
|
154
|
+
let state = case.state();
|
|
155
|
+
let bp = state
|
|
156
|
+
.pointer("/coordinator/abnormal_api_error_recovery/backpressure")
|
|
157
|
+
.and_then(serde_json::Value::as_object)
|
|
158
|
+
.expect("R4: backpressure state must be persisted");
|
|
159
|
+
assert_eq!(
|
|
160
|
+
bp.len(),
|
|
161
|
+
1,
|
|
162
|
+
"R4: cohort must share one backpressure record; state={state}"
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
#[test]
|
|
167
|
+
fn r5_retry_budget_exhausts_and_stays_loud() {
|
|
168
|
+
let abnormal = source("src/coordinator/steps/abnormal.rs");
|
|
169
|
+
for needle in [
|
|
170
|
+
"max_attempts",
|
|
171
|
+
"worker.abnormal_exit.recovery_exhausted",
|
|
172
|
+
"manual_command",
|
|
173
|
+
"attempts",
|
|
174
|
+
] {
|
|
175
|
+
assert!(
|
|
176
|
+
abnormal.contains(needle),
|
|
177
|
+
"R5: retry budget contract requires `{needle}` in abnormal api_error recovery state/events"
|
|
178
|
+
);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
#[test]
|
|
183
|
+
fn r6_abnormal_detector_does_not_mutate_lifecycle_inside_pre_save_window() {
|
|
184
|
+
let body = detect_abnormal_body();
|
|
185
|
+
for forbidden in [
|
|
186
|
+
"start_agent_at_paths",
|
|
187
|
+
"start_agent(",
|
|
188
|
+
"stop_agent_at_paths",
|
|
189
|
+
"reset_agent",
|
|
190
|
+
] {
|
|
191
|
+
assert!(
|
|
192
|
+
!body.contains(forbidden),
|
|
193
|
+
"R6: detect_abnormal_exits is pre-save detection only and must not call `{forbidden}`"
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
#[test]
|
|
199
|
+
fn r6_recovery_fields_survive_the_next_tick_after_atomic_save() {
|
|
200
|
+
let case = RecoveryCase::new("r6-save-boundary");
|
|
201
|
+
case.seed_agents(&[("fe-admin", 429, "rate_limit", "baseline-429")]);
|
|
202
|
+
case.seed_due_recovery("fe-admin");
|
|
203
|
+
let coord = case.coord();
|
|
204
|
+
coord.tick().expect("recovery tick");
|
|
205
|
+
|
|
206
|
+
let events = case.events();
|
|
207
|
+
assert!(
|
|
208
|
+
find_event(&events, "worker.abnormal_exit.recovery_started").is_some()
|
|
209
|
+
|| find_event(&events, "worker.abnormal_exit.recovery_blocked").is_some(),
|
|
210
|
+
"R6: due recovery must run after atomic_save and emit started or blocked; events={events:?}"
|
|
211
|
+
);
|
|
212
|
+
let state = case.state();
|
|
213
|
+
let recovery = state
|
|
214
|
+
.pointer("/coordinator/abnormal_api_error_recovery/agents/fe-admin")
|
|
215
|
+
.expect("R6: recovery entry must survive tick");
|
|
216
|
+
assert!(
|
|
217
|
+
recovery.get("last_attempt_at").is_some()
|
|
218
|
+
|| recovery.get("last_error").is_some()
|
|
219
|
+
|| recovery.get("blocked_reason").is_some(),
|
|
220
|
+
"R6: lifecycle-written recovery result fields must survive next tick save; recovery={recovery}"
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
#[test]
|
|
225
|
+
fn r7_recovery_never_synthesizes_hidden_provider_turn_or_sdk_call() {
|
|
226
|
+
let case = RecoveryCase::new("r7-no-hidden-turn");
|
|
227
|
+
case.seed_agents(&[("fe-admin", 429, "rate_limit", "baseline-429")]);
|
|
228
|
+
case.seed_due_recovery("fe-admin");
|
|
229
|
+
let registry = CountingRegistry::new();
|
|
230
|
+
let counters = registry.counters();
|
|
231
|
+
let transport = MockTransport::new(true);
|
|
232
|
+
let calls = std::sync::Arc::clone(&transport.calls);
|
|
233
|
+
let coord = Coordinator::for_test(
|
|
234
|
+
WorkspacePath::new(case.root.clone()),
|
|
235
|
+
Box::new(registry),
|
|
236
|
+
Box::new(transport),
|
|
237
|
+
None,
|
|
238
|
+
None,
|
|
239
|
+
);
|
|
240
|
+
|
|
241
|
+
coord.tick().expect("recovery tick");
|
|
242
|
+
|
|
243
|
+
assert_eq!(
|
|
244
|
+
counters.adapter_calls.load(Ordering::SeqCst),
|
|
245
|
+
0,
|
|
246
|
+
"R7: recovery must not call provider adapter/client APIs"
|
|
247
|
+
);
|
|
248
|
+
assert!(
|
|
249
|
+
!calls.lock().unwrap().iter().any(|call| *call == "inject"),
|
|
250
|
+
"R7: recovery must not inject a synthetic worker turn; transport calls={:?}",
|
|
251
|
+
calls.lock().unwrap()
|
|
252
|
+
);
|
|
253
|
+
assert_eq!(
|
|
254
|
+
case.non_leader_message_count(),
|
|
255
|
+
0,
|
|
256
|
+
"R7: recovery may notify leader, but must not create hidden worker prompt messages"
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
struct RecoveryCase {
|
|
261
|
+
root: std::path::PathBuf,
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
impl RecoveryCase {
|
|
265
|
+
fn new(tag: &str) -> Self {
|
|
266
|
+
let base = std::env::var_os("TEAM_AGENT_TEST_TMP")
|
|
267
|
+
.map(std::path::PathBuf::from)
|
|
268
|
+
.unwrap_or_else(std::env::temp_dir);
|
|
269
|
+
std::fs::create_dir_all(&base).unwrap();
|
|
270
|
+
let root = base.join(format!(
|
|
271
|
+
"team-agent-0536-{tag}-{}-{}",
|
|
272
|
+
std::process::id(),
|
|
273
|
+
std::time::SystemTime::now()
|
|
274
|
+
.duration_since(std::time::UNIX_EPOCH)
|
|
275
|
+
.unwrap()
|
|
276
|
+
.as_nanos()
|
|
277
|
+
));
|
|
278
|
+
std::fs::create_dir_all(&root).unwrap();
|
|
279
|
+
Self { root }
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
fn coord(&self) -> Coordinator {
|
|
283
|
+
Coordinator::for_test(
|
|
284
|
+
WorkspacePath::new(self.root.clone()),
|
|
285
|
+
Box::new(MockRegistry::new(&[], &[])),
|
|
286
|
+
Box::new(MockTransport::new(true)),
|
|
287
|
+
None,
|
|
288
|
+
None,
|
|
289
|
+
)
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
fn seed_agents(&self, agents: &[(&str, i64, &str, &str)]) {
|
|
293
|
+
let mut agent_map = serde_json::Map::new();
|
|
294
|
+
for (agent, status, error, uuid) in agents {
|
|
295
|
+
let rollout = self.rollout(agent);
|
|
296
|
+
std::fs::write(&rollout, claude_api_error_line(uuid, *status, error)).unwrap();
|
|
297
|
+
agent_map.insert(
|
|
298
|
+
(*agent).to_string(),
|
|
299
|
+
serde_json::json!({
|
|
300
|
+
"agent_id": agent,
|
|
301
|
+
"id": agent,
|
|
302
|
+
"provider": "claude_code",
|
|
303
|
+
"model": "sonnet",
|
|
304
|
+
"status": "running",
|
|
305
|
+
"window": agent,
|
|
306
|
+
"rollout_path": rollout,
|
|
307
|
+
"provider_process_alive": true,
|
|
308
|
+
"spawn_epoch": 1,
|
|
309
|
+
"spawned_at": "2026-07-12T10:28:00Z"
|
|
310
|
+
}),
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
let state = serde_json::json!({
|
|
314
|
+
"active_team_key": "research",
|
|
315
|
+
"team_key": "research",
|
|
316
|
+
"workspace": self.root,
|
|
317
|
+
"session_name": "team-research",
|
|
318
|
+
"leader_receiver": {
|
|
319
|
+
"mode": "direct_tmux",
|
|
320
|
+
"status": "attached",
|
|
321
|
+
"pane_id": "%leader",
|
|
322
|
+
"provider": "codex"
|
|
323
|
+
},
|
|
324
|
+
"agents": serde_json::Value::Object(agent_map),
|
|
325
|
+
"tasks": []
|
|
326
|
+
});
|
|
327
|
+
crate::state::persist::save_runtime_state(&self.root, &state).unwrap();
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
fn seed_due_recovery(&self, agent: &str) {
|
|
331
|
+
let mut state = self.state();
|
|
332
|
+
state["coordinator"]["abnormal_api_error_recovery"]["agents"][agent] = serde_json::json!({
|
|
333
|
+
"error_key": format!("worker.abnormal_exit.error:{agent}:rollout:api_error:seed"),
|
|
334
|
+
"cohort_key": "research:claude_code:api_error:429:rate_limit",
|
|
335
|
+
"status": "scheduled",
|
|
336
|
+
"attempts": 0,
|
|
337
|
+
"max_attempts": 2,
|
|
338
|
+
"next_retry_at": "2000-01-01T00:00:00Z",
|
|
339
|
+
"last_attempt_at": null,
|
|
340
|
+
"last_error": null,
|
|
341
|
+
"manual_command": format!("team-agent start-agent {agent} --workspace '{}' --team research --force --json", self.root.display())
|
|
342
|
+
});
|
|
343
|
+
crate::state::persist::save_runtime_state(&self.root, &state).unwrap();
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
fn append_api_error(&self, agent: &str, status: i64, error: &str, uuid: &str) {
|
|
347
|
+
use std::io::Write as _;
|
|
348
|
+
let mut file = std::fs::OpenOptions::new()
|
|
349
|
+
.append(true)
|
|
350
|
+
.open(self.rollout(agent))
|
|
351
|
+
.unwrap();
|
|
352
|
+
file.write_all(claude_api_error_line(uuid, status, error).as_bytes())
|
|
353
|
+
.unwrap();
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
fn rollout(&self, agent: &str) -> std::path::PathBuf {
|
|
357
|
+
self.root.join(format!("rollout-{agent}.jsonl"))
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
fn state(&self) -> serde_json::Value {
|
|
361
|
+
crate::state::persist::load_runtime_state(&self.root).unwrap()
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
fn events(&self) -> Vec<serde_json::Value> {
|
|
365
|
+
read_event_log_dir(&self.root)
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
fn latest_leader_notification(&self) -> String {
|
|
369
|
+
let db = crate::model::paths::runtime_dir(&self.root).join("team.db");
|
|
370
|
+
let conn = crate::db::schema::open_db(&db).unwrap();
|
|
371
|
+
conn.query_row(
|
|
372
|
+
"select content from messages where recipient = 'leader' order by created_at desc limit 1",
|
|
373
|
+
[],
|
|
374
|
+
|row| row.get::<_, String>(0),
|
|
375
|
+
)
|
|
376
|
+
.unwrap_or_default()
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
fn non_leader_message_count(&self) -> i64 {
|
|
380
|
+
let db = crate::model::paths::runtime_dir(&self.root).join("team.db");
|
|
381
|
+
let conn = crate::db::schema::open_db(&db).unwrap();
|
|
382
|
+
conn.query_row(
|
|
383
|
+
"select count(*) from messages where recipient != 'leader'",
|
|
384
|
+
[],
|
|
385
|
+
|row| row.get::<_, i64>(0),
|
|
386
|
+
)
|
|
387
|
+
.unwrap()
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
impl Drop for RecoveryCase {
|
|
392
|
+
fn drop(&mut self) {
|
|
393
|
+
if std::env::var("TEAM_AGENT_KEEP_TEST_TMP").as_deref() != Ok("1") {
|
|
394
|
+
let _ = std::fs::remove_dir_all(&self.root);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
#[derive(Clone)]
|
|
400
|
+
struct CounterState {
|
|
401
|
+
adapter_calls: Arc<AtomicU32>,
|
|
402
|
+
error_list_calls: Arc<AtomicU32>,
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
struct CountingRegistry {
|
|
406
|
+
counters: CounterState,
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
impl CountingRegistry {
|
|
410
|
+
fn new() -> Self {
|
|
411
|
+
Self {
|
|
412
|
+
counters: CounterState {
|
|
413
|
+
adapter_calls: Arc::new(AtomicU32::new(0)),
|
|
414
|
+
error_list_calls: Arc::new(AtomicU32::new(0)),
|
|
415
|
+
},
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
fn counters(&self) -> CounterState {
|
|
420
|
+
self.counters.clone()
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
impl ProviderRegistry for CountingRegistry {
|
|
425
|
+
fn adapter_for(&self, provider: Provider) -> Box<dyn ProviderAdapter> {
|
|
426
|
+
self.counters.adapter_calls.fetch_add(1, Ordering::SeqCst);
|
|
427
|
+
crate::provider::get_adapter(provider)
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
fn error_lists(&self, _provider: Provider) -> ErrorLists {
|
|
431
|
+
self.counters
|
|
432
|
+
.error_list_calls
|
|
433
|
+
.fetch_add(1, Ordering::SeqCst);
|
|
434
|
+
ErrorLists::default()
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
fn claude_api_error_line(uuid: &str, status: i64, error: &str) -> String {
|
|
439
|
+
format!(
|
|
440
|
+
"{}\n",
|
|
441
|
+
serde_json::json!({
|
|
442
|
+
"type": "assistant",
|
|
443
|
+
"parentUuid": format!("parent-{uuid}"),
|
|
444
|
+
"uuid": uuid,
|
|
445
|
+
"requestId": format!("req-{uuid}"),
|
|
446
|
+
"message": {"role": "assistant", "content": [
|
|
447
|
+
{"type": "text", "text": format!("API Error: {status} {error}")}
|
|
448
|
+
]},
|
|
449
|
+
"error": error,
|
|
450
|
+
"isApiErrorMessage": true,
|
|
451
|
+
"apiErrorStatus": status,
|
|
452
|
+
"sessionId": "session-supermarket",
|
|
453
|
+
"version": "2.1.181"
|
|
454
|
+
})
|
|
455
|
+
)
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
fn find_event<'a>(events: &'a [serde_json::Value], name: &str) -> Option<&'a serde_json::Value> {
|
|
459
|
+
events
|
|
460
|
+
.iter()
|
|
461
|
+
.rev()
|
|
462
|
+
.find(|event| event.get("event").and_then(serde_json::Value::as_str) == Some(name))
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
fn source(rel: &str) -> String {
|
|
466
|
+
std::fs::read_to_string(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(rel))
|
|
467
|
+
.expect("read source")
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
fn detect_abnormal_body() -> String {
|
|
471
|
+
let source = source("src/coordinator/steps/abnormal.rs");
|
|
472
|
+
let start = source
|
|
473
|
+
.find("pub(crate) fn detect_abnormal_exits")
|
|
474
|
+
.expect("detect_abnormal_exits exists");
|
|
475
|
+
let end = source[start..]
|
|
476
|
+
.find("#[derive")
|
|
477
|
+
.map(|offset| start + offset)
|
|
478
|
+
.expect("derive after detect_abnormal_exits");
|
|
479
|
+
source[start..end].to_string()
|
|
480
|
+
}
|
|
@@ -91,6 +91,13 @@ fn tick_side_effect_order_is_the_fixed_sequence() {
|
|
|
91
91
|
"detect_drift",
|
|
92
92
|
"detect_api_errors",
|
|
93
93
|
"atomic_save",
|
|
94
|
+
// 0.5.36 (`.team/artifacts/supermarket-api-error-recovery-locate.md` §7.3):
|
|
95
|
+
// post-save recovery step reloads fresh state, consumes due
|
|
96
|
+
// recovery intents written pre-save, and drives the lifecycle
|
|
97
|
+
// helper. Sits between atomic_save and collect_results by design
|
|
98
|
+
// so nested lifecycle saves cannot be clobbered by tick's final
|
|
99
|
+
// in-memory save.
|
|
100
|
+
"attempt_api_error_recoveries",
|
|
94
101
|
"collect_results",
|
|
95
102
|
"prune_dedupe_log",
|
|
96
103
|
];
|
|
@@ -437,6 +437,19 @@ impl Coordinator {
|
|
|
437
437
|
));
|
|
438
438
|
}
|
|
439
439
|
|
|
440
|
+
// 0.5.36 (`.team/artifacts/supermarket-api-error-recovery-locate.md` §7.3):
|
|
441
|
+
// post-save recovery step. Reloads fresh state, consumes due
|
|
442
|
+
// recovery intents that were WRITTEN during the pre-save detector
|
|
443
|
+
// step, and drives the lifecycle helper. Runs OUTSIDE the pre-save
|
|
444
|
+
// window so nested lifecycle saves cannot be clobbered by this
|
|
445
|
+
// tick's final in-memory save (R6 boundary guard).
|
|
446
|
+
self.record_step("attempt_api_error_recoveries");
|
|
447
|
+
crate::coordinator::steps::abnormal::attempt_due_recoveries(
|
|
448
|
+
self.workspace.as_path(),
|
|
449
|
+
&event_log,
|
|
450
|
+
self.transport.as_ref(),
|
|
451
|
+
);
|
|
452
|
+
|
|
440
453
|
self.record_step("collect_results");
|
|
441
454
|
collections.results =
|
|
442
455
|
collect_results(crate::messaging::collect_results_and_notify_watchers(
|
|
@@ -795,6 +795,71 @@ pub fn stop_agent_with_transport(
|
|
|
795
795
|
)
|
|
796
796
|
}
|
|
797
797
|
|
|
798
|
+
/// 0.5.36 (`.team/artifacts/supermarket-api-error-recovery-locate.md` §7.3/§7.4):
|
|
799
|
+
/// api_error recovery entry point. Runs post-atomic_save from coordinator
|
|
800
|
+
/// tick to replace the stuck provider process on a retryable outage. It
|
|
801
|
+
/// resolves lifecycle paths, force-stops the live pane if the worker is
|
|
802
|
+
/// still alive (so `start_agent_at_paths(force=true)` cannot be a Noop —
|
|
803
|
+
/// see R3 contract), then starts the agent with `allow_fresh=false` to
|
|
804
|
+
/// preserve session context. Returns a small, typed outcome so the
|
|
805
|
+
/// caller only needs the delta for event emission; the underlying
|
|
806
|
+
/// lifecycle helpers own their state saves.
|
|
807
|
+
pub(crate) fn start_agent_at_paths_for_recovery(
|
|
808
|
+
workspace: &Path,
|
|
809
|
+
agent_id: &AgentId,
|
|
810
|
+
team: Option<&str>,
|
|
811
|
+
transport: &dyn crate::transport::Transport,
|
|
812
|
+
) -> Result<
|
|
813
|
+
crate::coordinator::steps::abnormal::RecoveryLifecycleOutcome,
|
|
814
|
+
crate::coordinator::steps::abnormal::RecoveryError,
|
|
815
|
+
> {
|
|
816
|
+
use crate::coordinator::steps::abnormal::{RecoveryError, RecoveryLifecycleOutcome};
|
|
817
|
+
let paths = match lifecycle_paths(workspace, team) {
|
|
818
|
+
Ok(paths) => paths,
|
|
819
|
+
Err(_) if team.is_none() => LifecyclePaths {
|
|
820
|
+
run_workspace: workspace.to_path_buf(),
|
|
821
|
+
spec_workspace: workspace.to_path_buf(),
|
|
822
|
+
},
|
|
823
|
+
Err(error) => return Err(RecoveryError::Lifecycle(error.to_string())),
|
|
824
|
+
};
|
|
825
|
+
// Best-effort stop: if the live pane still exists we tear it down so the
|
|
826
|
+
// subsequent start creates a fresh provider process instead of a Noop.
|
|
827
|
+
// Stop errors are absorbed into the start attempt result — the important
|
|
828
|
+
// invariant is that a successful start returns Running, never Noop.
|
|
829
|
+
let _ = stop_agent_with_transport(
|
|
830
|
+
&paths.run_workspace,
|
|
831
|
+
agent_id,
|
|
832
|
+
team,
|
|
833
|
+
transport,
|
|
834
|
+
);
|
|
835
|
+
let start_result = start_agent_with_transport(
|
|
836
|
+
&paths.run_workspace,
|
|
837
|
+
agent_id,
|
|
838
|
+
/* force */ true,
|
|
839
|
+
/* open_display */ false,
|
|
840
|
+
/* allow_fresh */ false,
|
|
841
|
+
team,
|
|
842
|
+
transport,
|
|
843
|
+
);
|
|
844
|
+
match start_result {
|
|
845
|
+
Ok(StartAgentOutcome::Running {
|
|
846
|
+
start_mode,
|
|
847
|
+
target,
|
|
848
|
+
env,
|
|
849
|
+
..
|
|
850
|
+
}) => Ok(RecoveryLifecycleOutcome {
|
|
851
|
+
start_mode: format!("{:?}", start_mode),
|
|
852
|
+
target,
|
|
853
|
+
coordinator_started: env.coordinator_started,
|
|
854
|
+
}),
|
|
855
|
+
Ok(StartAgentOutcome::Noop { .. }) => Err(RecoveryError::NoopBlocked),
|
|
856
|
+
Ok(StartAgentOutcome::Paused { .. }) => Err(RecoveryError::Lifecycle(
|
|
857
|
+
"agent is paused; recovery skipped".to_string(),
|
|
858
|
+
)),
|
|
859
|
+
Err(error) => Err(RecoveryError::Lifecycle(error.to_string())),
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
|
|
798
863
|
pub(super) fn stop_agent_at_paths(
|
|
799
864
|
workspace: &Path,
|
|
800
865
|
spec_workspace: &Path,
|
|
@@ -33,6 +33,10 @@ mod selection;
|
|
|
33
33
|
mod team_state;
|
|
34
34
|
|
|
35
35
|
pub(crate) use agent::start_agent_at_paths;
|
|
36
|
+
// 0.5.36 supermarket api_error recovery: coordinator post-save step calls
|
|
37
|
+
// this to replace a stuck provider process without going through the
|
|
38
|
+
// public start-agent CLI path.
|
|
39
|
+
pub(crate) use agent::start_agent_at_paths_for_recovery;
|
|
36
40
|
pub use agent::{
|
|
37
41
|
reset_agent, reset_agent_with_transport, start_agent, start_agent_with_transport, stop_agent,
|
|
38
42
|
stop_agent_with_transport,
|
|
@@ -168,6 +168,14 @@ pub enum StateWriteIntent<'a> {
|
|
|
168
168
|
CoordinatorConptyShim {
|
|
169
169
|
team_key: Option<&'a str>,
|
|
170
170
|
},
|
|
171
|
+
/// 0.5.36 (`.team/artifacts/supermarket-api-error-recovery-locate.md` §7.3):
|
|
172
|
+
/// post-save recovery step writes back the recovery intent outcome
|
|
173
|
+
/// (attempts / status / last_error / blocked_reason). Distinct intent
|
|
174
|
+
/// so future S1b migration can route it deliberately.
|
|
175
|
+
CoordinatorApiErrorRecovery {
|
|
176
|
+
team_key: Option<&'a str>,
|
|
177
|
+
agent_id: &'a str,
|
|
178
|
+
},
|
|
171
179
|
McpAssignTask {
|
|
172
180
|
team_key: Option<&'a str>,
|
|
173
181
|
task_id: &'a str,
|
|
@@ -311,6 +319,13 @@ fn route_direct(
|
|
|
311
319
|
// CoordinatorConptyShim -> root save
|
|
312
320
|
// (coordinator/conpty_shim.rs:426/674).
|
|
313
321
|
StateWriteIntent::CoordinatorConptyShim { .. } => helper_write_root(workspace, state),
|
|
322
|
+
// 0.5.36 CoordinatorApiErrorRecovery -> root save. The recovery
|
|
323
|
+
// intent lives under `coordinator.abnormal_api_error_recovery`, a
|
|
324
|
+
// root-scoped bookkeeping namespace that must survive across team
|
|
325
|
+
// boundaries; use the same helper family as CoordinatorConptyShim.
|
|
326
|
+
StateWriteIntent::CoordinatorApiErrorRecovery { .. } => {
|
|
327
|
+
helper_write_root(workspace, state)
|
|
328
|
+
}
|
|
314
329
|
// McpAssignTask uses the reapply variant today; direct save routes
|
|
315
330
|
// to the root helper for parity.
|
|
316
331
|
StateWriteIntent::McpAssignTask { .. } => helper_write_root(workspace, state),
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@team-agent/installer",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.36",
|
|
4
4
|
"description": "npx installer for Team Agent",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"codex",
|
|
@@ -20,9 +20,9 @@
|
|
|
20
20
|
"team-agent-installer": "npm/install.mjs"
|
|
21
21
|
},
|
|
22
22
|
"optionalDependencies": {
|
|
23
|
-
"@team-agent/cli-darwin-arm64": "0.5.
|
|
24
|
-
"@team-agent/cli-darwin-x64": "0.5.
|
|
25
|
-
"@team-agent/cli-linux-x64": "0.5.
|
|
23
|
+
"@team-agent/cli-darwin-arm64": "0.5.36",
|
|
24
|
+
"@team-agent/cli-darwin-x64": "0.5.36",
|
|
25
|
+
"@team-agent/cli-linux-x64": "0.5.36"
|
|
26
26
|
},
|
|
27
27
|
"scripts": {
|
|
28
28
|
"postinstall": "node npm/bincheck.mjs",
|