@team-agent/installer 0.3.13 → 0.3.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -1
- package/Cargo.toml +1 -1
- package/crates/team-agent/src/cli/adapters.rs +1 -0
- package/crates/team-agent/src/cli/emit.rs +92 -11
- package/crates/team-agent/src/cli/mod.rs +228 -85
- package/crates/team-agent/src/cli/send.rs +252 -0
- package/crates/team-agent/src/cli/tests/run_delegation.rs +15 -3
- package/crates/team-agent/src/cli/tests/shutdown_kill_plan.rs +159 -2
- package/crates/team-agent/src/cli/types.rs +30 -1
- package/crates/team-agent/src/compiler/tests.rs +2 -2
- package/crates/team-agent/src/compiler.rs +1 -1
- package/crates/team-agent/src/fake_worker.rs +32 -145
- package/crates/team-agent/src/leader/start.rs +279 -4
- package/crates/team-agent/src/lifecycle/display.rs +3 -3
- package/crates/team-agent/src/lifecycle/launch.rs +692 -92
- package/crates/team-agent/src/lifecycle/restart/agent.rs +137 -17
- package/crates/team-agent/src/lifecycle/restart/common.rs +88 -42
- package/crates/team-agent/src/lifecycle/restart/rebuild.rs +197 -33
- package/crates/team-agent/src/lifecycle/restart/selection.rs +9 -6
- package/crates/team-agent/src/lifecycle/tests/core.rs +195 -50
- package/crates/team-agent/src/lifecycle/tests/launch_spawn.rs +934 -72
- package/crates/team-agent/src/lifecycle/types.rs +5 -0
- package/crates/team-agent/src/lifecycle/worker_command_context.rs +8 -0
- package/crates/team-agent/src/mcp_server/wire.rs +153 -3
- package/crates/team-agent/src/messaging/leader_receiver.rs +276 -43
- package/crates/team-agent/src/messaging/mod.rs +6 -3
- package/crates/team-agent/src/messaging/results.rs +240 -158
- package/crates/team-agent/src/messaging/send.rs +3 -2
- package/crates/team-agent/src/messaging/tests/e23.rs +35 -0
- package/crates/team-agent/src/messaging/tests/mod.rs +6 -2
- package/crates/team-agent/src/messaging/tests/runtime.rs +9 -9
- package/crates/team-agent/src/os_probe.rs +11 -0
- package/crates/team-agent/src/state/persist.rs +6 -0
- package/crates/team-agent/src/tmux_backend.rs +90 -0
- package/crates/team-agent/src/transport/test_support.rs +46 -1
- package/crates/team-agent/src/transport.rs +31 -0
- package/package.json +4 -4
- package/skills/team-agent/references/recovery-runbook.md +277 -0
|
@@ -1898,26 +1898,26 @@ fn r8_attach_requeue_exhausted_to_notify_failed_golden_attach_event() {
|
|
|
1898
1898
|
);
|
|
1899
1899
|
}
|
|
1900
1900
|
|
|
1901
|
-
// E15
|
|
1901
|
+
// E15/E23 双投守卫:report_result 的 pane fallback 必须被 `if !outcome.ok` 守为
|
|
1902
1902
|
// deliver 失败时的真兜底,绝不在 deliver 成功后无条件再投一次(=用户看到两条同内容回复)。
|
|
1903
1903
|
#[test]
|
|
1904
1904
|
fn e15_direct_inject_is_gated_by_deliver_failure_not_unconditional() {
|
|
1905
1905
|
let src = include_str!("../results.rs");
|
|
1906
|
-
//
|
|
1906
|
+
// E23:旧 private direct inject 已合并到 shared deliver_to_leader_fallback_pane primitive。
|
|
1907
1907
|
let gate = "if !outcome.ok {";
|
|
1908
|
-
let
|
|
1908
|
+
let fallback_call = "deliver_to_leader_fallback_pane(";
|
|
1909
1909
|
let gate_pos = src.find(gate);
|
|
1910
|
-
let inject_pos = src.find(inject_call);
|
|
1911
1910
|
assert!(
|
|
1912
1911
|
gate_pos.is_some(),
|
|
1913
|
-
"E15:
|
|
1912
|
+
"E15/E23: pane fallback must be gated by `if !outcome.ok` (deliver-fail fallback)"
|
|
1914
1913
|
);
|
|
1914
|
+
let fallback_after_gate = src[gate_pos.unwrap()..].find(fallback_call);
|
|
1915
1915
|
assert!(
|
|
1916
|
-
|
|
1917
|
-
"
|
|
1916
|
+
fallback_after_gate.is_some(),
|
|
1917
|
+
"E15/E23: report_result fallback must call the shared fallback pane primitive after the failure gate"
|
|
1918
1918
|
);
|
|
1919
1919
|
assert!(
|
|
1920
|
-
|
|
1921
|
-
"
|
|
1920
|
+
!src.contains("inject_leader_notification_direct"),
|
|
1921
|
+
"E23: private direct inject must stay deleted; all callers use deliver_to_leader_fallback_pane"
|
|
1922
1922
|
);
|
|
1923
1923
|
}
|
|
@@ -35,6 +35,17 @@ pub(crate) fn probe_timeout() -> Option<ProbeTimeout> {
|
|
|
35
35
|
PROBE_TIMEOUT.with(|timeout| timeout.borrow().clone())
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
#[cfg(test)]
|
|
39
|
+
pub(crate) fn set_probe_timeout_for_test(probe: &'static str, pid: Option<u32>, timeout_ms: u64) {
|
|
40
|
+
PROBE_TIMEOUT.with(|current| {
|
|
41
|
+
*current.borrow_mut() = Some(ProbeTimeout {
|
|
42
|
+
probe,
|
|
43
|
+
pid,
|
|
44
|
+
timeout_ms,
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
|
|
38
49
|
pub(crate) fn bounded_command_output_with_probe(
|
|
39
50
|
command: &mut Command,
|
|
40
51
|
probe: &'static str,
|
|
@@ -419,7 +419,13 @@ fn backfill_capture_fields(incoming_agent: &mut Value, latest_agent: &Value) {
|
|
|
419
419
|
let Some(incoming_row) = incoming_agent.as_object_mut() else {
|
|
420
420
|
return;
|
|
421
421
|
};
|
|
422
|
+
let incoming_session = incoming_row.get("session_id").and_then(Value::as_str);
|
|
423
|
+
let latest_session = latest_agent.get("session_id").and_then(Value::as_str);
|
|
424
|
+
let session_changed = incoming_session.is_some() && incoming_session != latest_session;
|
|
422
425
|
for field in CAPTURE_FIELDS {
|
|
426
|
+
if session_changed && field != "session_id" {
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
423
429
|
if incoming_row.get(field).is_none_or(Value::is_null) {
|
|
424
430
|
if let Some(value) = latest_agent.get(field).filter(|value| !value.is_null()) {
|
|
425
431
|
incoming_row.insert(field.to_string(), value.clone());
|
|
@@ -180,6 +180,14 @@ enum TmuxSocketEndpoint {
|
|
|
180
180
|
Path(String),
|
|
181
181
|
}
|
|
182
182
|
|
|
183
|
+
impl TmuxSocketEndpoint {
|
|
184
|
+
fn as_endpoint(&self) -> &str {
|
|
185
|
+
match self {
|
|
186
|
+
TmuxSocketEndpoint::Name(socket) | TmuxSocketEndpoint::Path(socket) => socket,
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
183
191
|
impl TmuxBackend {
|
|
184
192
|
/// Backend bound to the real `tmux` subprocess on the SHARED default socket (no `-L`).
|
|
185
193
|
/// Non-team callers + existing argv/unit tests stay unaffected.
|
|
@@ -514,6 +522,55 @@ impl TmuxBackend {
|
|
|
514
522
|
})
|
|
515
523
|
}
|
|
516
524
|
|
|
525
|
+
fn spawn_split(
|
|
526
|
+
&self,
|
|
527
|
+
session: &SessionName,
|
|
528
|
+
window: &WindowName,
|
|
529
|
+
argv: &[String],
|
|
530
|
+
cwd: &Path,
|
|
531
|
+
env: &BTreeMap<String, String>,
|
|
532
|
+
env_unset: &[String],
|
|
533
|
+
) -> Result<SpawnResult, TransportError> {
|
|
534
|
+
let command = shell_command(argv, cwd, env, env_unset);
|
|
535
|
+
let target = format!("{}:{}", session.as_str(), window.as_str());
|
|
536
|
+
let split_argv = vec![
|
|
537
|
+
"tmux".to_string(),
|
|
538
|
+
"split-window".to_string(),
|
|
539
|
+
"-t".to_string(),
|
|
540
|
+
target.clone(),
|
|
541
|
+
"-h".to_string(),
|
|
542
|
+
"-P".to_string(),
|
|
543
|
+
"-F".to_string(),
|
|
544
|
+
"#{pane_id}".to_string(),
|
|
545
|
+
"sh".to_string(),
|
|
546
|
+
"-lc".to_string(),
|
|
547
|
+
command,
|
|
548
|
+
];
|
|
549
|
+
let output = self.run_spawn(&split_argv)?;
|
|
550
|
+
let pane = output.stdout.trim();
|
|
551
|
+
if pane.is_empty() {
|
|
552
|
+
return Err(TransportError::Subprocess {
|
|
553
|
+
argv: split_argv,
|
|
554
|
+
code: output.code,
|
|
555
|
+
stderr: format!("tmux split-window returned no pane id for {target}"),
|
|
556
|
+
});
|
|
557
|
+
}
|
|
558
|
+
let layout_argv = vec![
|
|
559
|
+
"tmux".to_string(),
|
|
560
|
+
"select-layout".to_string(),
|
|
561
|
+
"-t".to_string(),
|
|
562
|
+
target,
|
|
563
|
+
"tiled".to_string(),
|
|
564
|
+
];
|
|
565
|
+
self.run_ok(&layout_argv)?;
|
|
566
|
+
Ok(SpawnResult {
|
|
567
|
+
pane_id: PaneId::new(pane),
|
|
568
|
+
session: session.clone(),
|
|
569
|
+
window: window.clone(),
|
|
570
|
+
child_pid: None,
|
|
571
|
+
})
|
|
572
|
+
}
|
|
573
|
+
|
|
517
574
|
fn run_ok(&self, argv: &[String]) -> Result<(), TransportError> {
|
|
518
575
|
let argv = self.tmux_argv(argv);
|
|
519
576
|
let output = self.runner.run(&argv)?;
|
|
@@ -722,6 +779,12 @@ impl Transport for TmuxBackend {
|
|
|
722
779
|
true
|
|
723
780
|
}
|
|
724
781
|
|
|
782
|
+
fn tmux_endpoint(&self) -> Option<String> {
|
|
783
|
+
self.socket
|
|
784
|
+
.as_ref()
|
|
785
|
+
.map(|endpoint| endpoint.as_endpoint().to_string())
|
|
786
|
+
}
|
|
787
|
+
|
|
725
788
|
fn spawn_first(
|
|
726
789
|
&self,
|
|
727
790
|
session: &SessionName,
|
|
@@ -768,6 +831,18 @@ impl Transport for TmuxBackend {
|
|
|
768
831
|
self.spawn(session, window, argv, cwd, env, &[], false)
|
|
769
832
|
}
|
|
770
833
|
|
|
834
|
+
fn spawn_split_with_env_unset(
|
|
835
|
+
&self,
|
|
836
|
+
session: &SessionName,
|
|
837
|
+
window: &WindowName,
|
|
838
|
+
argv: &[String],
|
|
839
|
+
cwd: &Path,
|
|
840
|
+
env: &BTreeMap<String, String>,
|
|
841
|
+
env_unset: &[String],
|
|
842
|
+
) -> Result<SpawnResult, TransportError> {
|
|
843
|
+
self.spawn_split(session, window, argv, cwd, env, env_unset)
|
|
844
|
+
}
|
|
845
|
+
|
|
771
846
|
fn inject(
|
|
772
847
|
&self,
|
|
773
848
|
target: &Target,
|
|
@@ -1041,6 +1116,11 @@ impl Transport for TmuxBackend {
|
|
|
1041
1116
|
Ok(SetEnvOutcome::Applied)
|
|
1042
1117
|
}
|
|
1043
1118
|
|
|
1119
|
+
fn kill_server(&self) -> Result<(), TransportError> {
|
|
1120
|
+
TmuxBackend::kill_server(self);
|
|
1121
|
+
Ok(())
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1044
1124
|
fn kill_session(&self, session: &SessionName) -> Result<(), TransportError> {
|
|
1045
1125
|
let argv = self.tmux_argv(&[
|
|
1046
1126
|
"tmux".to_string(),
|
|
@@ -1061,6 +1141,16 @@ impl Transport for TmuxBackend {
|
|
|
1061
1141
|
self.run_ok(&argv)
|
|
1062
1142
|
}
|
|
1063
1143
|
|
|
1144
|
+
fn kill_pane(&self, pane: &PaneId) -> Result<(), TransportError> {
|
|
1145
|
+
let argv = self.tmux_argv(&[
|
|
1146
|
+
"tmux".to_string(),
|
|
1147
|
+
"kill-pane".to_string(),
|
|
1148
|
+
"-t".to_string(),
|
|
1149
|
+
pane.as_str().to_string(),
|
|
1150
|
+
]);
|
|
1151
|
+
self.run_ok(&argv)
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1064
1154
|
fn attach_session(&self, session: &SessionName) -> Result<AttachOutcome, TransportError> {
|
|
1065
1155
|
let argv = [
|
|
1066
1156
|
"tmux".to_string(),
|
|
@@ -15,6 +15,8 @@ use super::{
|
|
|
15
15
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
16
16
|
pub struct SpawnRecord {
|
|
17
17
|
pub kind: String,
|
|
18
|
+
pub session: SessionName,
|
|
19
|
+
pub window: WindowName,
|
|
18
20
|
pub argv: Vec<String>,
|
|
19
21
|
}
|
|
20
22
|
|
|
@@ -33,6 +35,7 @@ struct OfflineState {
|
|
|
33
35
|
spawns: Vec<SpawnRecord>,
|
|
34
36
|
inject_targets: Vec<Target>,
|
|
35
37
|
inject_payloads: Vec<String>,
|
|
38
|
+
tmux_endpoint: Option<String>,
|
|
36
39
|
}
|
|
37
40
|
|
|
38
41
|
impl Default for OfflineState {
|
|
@@ -51,6 +54,7 @@ impl Default for OfflineState {
|
|
|
51
54
|
spawns: Vec::new(),
|
|
52
55
|
inject_targets: Vec::new(),
|
|
53
56
|
inject_payloads: Vec::new(),
|
|
57
|
+
tmux_endpoint: None,
|
|
54
58
|
}
|
|
55
59
|
}
|
|
56
60
|
}
|
|
@@ -116,6 +120,11 @@ impl OfflineTransport {
|
|
|
116
120
|
self
|
|
117
121
|
}
|
|
118
122
|
|
|
123
|
+
pub fn with_tmux_endpoint(self, endpoint: impl Into<String>) -> Self {
|
|
124
|
+
self.with_state(|state| state.tmux_endpoint = Some(endpoint.into()));
|
|
125
|
+
self
|
|
126
|
+
}
|
|
127
|
+
|
|
119
128
|
pub fn calls(&self) -> Vec<&'static str> {
|
|
120
129
|
self.with_state(|state| state.calls.clone())
|
|
121
130
|
}
|
|
@@ -130,6 +139,16 @@ impl OfflineTransport {
|
|
|
130
139
|
})
|
|
131
140
|
}
|
|
132
141
|
|
|
142
|
+
pub fn spawn_window_records(&self) -> Vec<(String, String)> {
|
|
143
|
+
self.with_state(|state| {
|
|
144
|
+
state
|
|
145
|
+
.spawns
|
|
146
|
+
.iter()
|
|
147
|
+
.map(|record| (record.kind.clone(), record.window.as_str().to_string()))
|
|
148
|
+
.collect()
|
|
149
|
+
})
|
|
150
|
+
}
|
|
151
|
+
|
|
133
152
|
pub fn inject_targets(&self) -> Vec<Target> {
|
|
134
153
|
self.with_state(|state| state.inject_targets.clone())
|
|
135
154
|
}
|
|
@@ -161,7 +180,12 @@ impl OfflineTransport {
|
|
|
161
180
|
) -> Result<SpawnResult, TransportError> {
|
|
162
181
|
let pane_index = self.with_state(|state| {
|
|
163
182
|
state.calls.push(kind);
|
|
164
|
-
state.spawns.push(SpawnRecord {
|
|
183
|
+
state.spawns.push(SpawnRecord {
|
|
184
|
+
kind: kind.to_string(),
|
|
185
|
+
session: session.clone(),
|
|
186
|
+
window: window.clone(),
|
|
187
|
+
argv: argv.to_vec(),
|
|
188
|
+
});
|
|
165
189
|
if let Some(error) = state.spawn_failures.get(window.as_str()) {
|
|
166
190
|
return Err(TransportError::Spawn {
|
|
167
191
|
backend: BackendKind::Tmux,
|
|
@@ -201,6 +225,10 @@ impl Transport for OfflineTransport {
|
|
|
201
225
|
BackendKind::Tmux
|
|
202
226
|
}
|
|
203
227
|
|
|
228
|
+
fn tmux_endpoint(&self) -> Option<String> {
|
|
229
|
+
self.with_state(|state| state.tmux_endpoint.clone())
|
|
230
|
+
}
|
|
231
|
+
|
|
204
232
|
fn spawn_first(
|
|
205
233
|
&self,
|
|
206
234
|
session: &SessionName,
|
|
@@ -223,6 +251,18 @@ impl Transport for OfflineTransport {
|
|
|
223
251
|
self.spawn_result("spawn_into", session, window, argv)
|
|
224
252
|
}
|
|
225
253
|
|
|
254
|
+
fn spawn_split_with_env_unset(
|
|
255
|
+
&self,
|
|
256
|
+
session: &SessionName,
|
|
257
|
+
window: &WindowName,
|
|
258
|
+
argv: &[String],
|
|
259
|
+
_cwd: &Path,
|
|
260
|
+
_env: &BTreeMap<String, String>,
|
|
261
|
+
_env_unset: &[String],
|
|
262
|
+
) -> Result<SpawnResult, TransportError> {
|
|
263
|
+
self.spawn_result("spawn_split", session, window, argv)
|
|
264
|
+
}
|
|
265
|
+
|
|
226
266
|
fn inject(
|
|
227
267
|
&self,
|
|
228
268
|
target: &Target,
|
|
@@ -330,6 +370,11 @@ impl Transport for OfflineTransport {
|
|
|
330
370
|
Ok(())
|
|
331
371
|
}
|
|
332
372
|
|
|
373
|
+
fn kill_pane(&self, _pane: &PaneId) -> Result<(), TransportError> {
|
|
374
|
+
self.record("kill_pane");
|
|
375
|
+
Ok(())
|
|
376
|
+
}
|
|
377
|
+
|
|
333
378
|
fn attach_session(
|
|
334
379
|
&self,
|
|
335
380
|
_session: &SessionName,
|
|
@@ -405,6 +405,13 @@ pub trait Transport: Send + Sync {
|
|
|
405
405
|
false
|
|
406
406
|
}
|
|
407
407
|
|
|
408
|
+
/// Physical tmux endpoint used by this transport when known. For tmux this is
|
|
409
|
+
/// either a full `-S` socket path or a `-L` socket name; non-tmux/test
|
|
410
|
+
/// transports can leave it unknown.
|
|
411
|
+
fn tmux_endpoint(&self) -> Option<String> {
|
|
412
|
+
None
|
|
413
|
+
}
|
|
414
|
+
|
|
408
415
|
// —— SPAWN(ST):所有后端天然满足;cwd/env 是 spawn 参数,无独立动词(§gap-setenv)——
|
|
409
416
|
|
|
410
417
|
/// tmux=`new-session -d` / wezterm=`spawn --new-window` / conpty=`openpty`+spawn。
|
|
@@ -459,6 +466,22 @@ pub trait Transport: Send + Sync {
|
|
|
459
466
|
self.spawn_into(session, window, argv, cwd, env)
|
|
460
467
|
}
|
|
461
468
|
|
|
469
|
+
/// Spawn a worker as an additional pane in an existing session/window.
|
|
470
|
+
/// Backends without a split-pane primitive may conservatively fall back to
|
|
471
|
+
/// `spawn_into`; tmux overrides this with `split-window`.
|
|
472
|
+
fn spawn_split_with_env_unset(
|
|
473
|
+
&self,
|
|
474
|
+
session: &SessionName,
|
|
475
|
+
window: &WindowName,
|
|
476
|
+
argv: &[String],
|
|
477
|
+
cwd: &Path,
|
|
478
|
+
env: &BTreeMap<String, String>,
|
|
479
|
+
env_unset: &[String],
|
|
480
|
+
) -> Result<SpawnResult, TransportError> {
|
|
481
|
+
let _ = env_unset;
|
|
482
|
+
self.spawn_into(session, window, argv, cwd, env)
|
|
483
|
+
}
|
|
484
|
+
|
|
462
485
|
// —— INJECT / CAPTURE / QUERY(RIE):按稳定 Target 寻址 ——
|
|
463
486
|
|
|
464
487
|
/// 归并 set/load-buffer + paste-buffer + send submit;空文本走纯 send-keys
|
|
@@ -519,10 +542,18 @@ pub trait Transport: Send + Sync {
|
|
|
519
542
|
|
|
520
543
|
// —— LIFECYCLE(SL)——
|
|
521
544
|
|
|
545
|
+
fn kill_server(&self) -> Result<(), TransportError> {
|
|
546
|
+
Ok(())
|
|
547
|
+
}
|
|
548
|
+
|
|
522
549
|
fn kill_session(&self, session: &SessionName) -> Result<(), TransportError>;
|
|
523
550
|
|
|
524
551
|
fn kill_window(&self, target: &Target) -> Result<(), TransportError>;
|
|
525
552
|
|
|
553
|
+
fn kill_pane(&self, pane: &PaneId) -> Result<(), TransportError> {
|
|
554
|
+
self.kill_window(&Target::Pane(pane.clone()))
|
|
555
|
+
}
|
|
556
|
+
|
|
526
557
|
/// 交互前台 attach(leader 用)。tmux=`attach-session`;wezterm=GUI 即 attach
|
|
527
558
|
/// (`GuiAttachIsImplicit`);conpty=typed 不支持(`Unsupported`)。
|
|
528
559
|
fn attach_session(
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@team-agent/installer",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.15",
|
|
4
4
|
"description": "npx installer for Team Agent",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"codex",
|
|
@@ -20,9 +20,9 @@
|
|
|
20
20
|
"team-agent-installer": "npm/install.mjs"
|
|
21
21
|
},
|
|
22
22
|
"optionalDependencies": {
|
|
23
|
-
"@team-agent/cli-darwin-arm64": "0.3.
|
|
24
|
-
"@team-agent/cli-darwin-x64": "0.3.
|
|
25
|
-
"@team-agent/cli-linux-x64": "0.3.
|
|
23
|
+
"@team-agent/cli-darwin-arm64": "0.3.15",
|
|
24
|
+
"@team-agent/cli-darwin-x64": "0.3.15",
|
|
25
|
+
"@team-agent/cli-linux-x64": "0.3.15"
|
|
26
26
|
},
|
|
27
27
|
"scripts": {
|
|
28
28
|
"postinstall": "node npm/bincheck.mjs",
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
# Team Agent Recovery Reference
|
|
2
|
+
|
|
3
|
+
This reference is for the Rust Team Agent runtime. This reference is a live document.
|
|
4
|
+
|
|
5
|
+
When a Team Agent framework problem appears, check this reference first. After the problem is resolved, update this reference before closing the work, whether or not the final fix used an existing entry here. New recovery advice may be added only after RS validation on a real Team Agent environment; unverified Python-only procedures stay out of this external reference.
|
|
6
|
+
|
|
7
|
+
## Validation Gate
|
|
8
|
+
|
|
9
|
+
Every prescription below must carry an RS verification record. A command copied from Python 0.2.11 or from an incident note is not valid until it has been verified against the RS runtime. If RS behavior differs, document the RS behavior, not the Python behavior.
|
|
10
|
+
|
|
11
|
+
Use placeholders only:
|
|
12
|
+
|
|
13
|
+
- `$WORKSPACE` for the team workspace
|
|
14
|
+
- `$TEAM` for the active team key when a command needs one
|
|
15
|
+
- `$AGENT_ID` for a worker id
|
|
16
|
+
- `$TASK_ID` for a task id
|
|
17
|
+
|
|
18
|
+
Do not write recipes that depend on a user's private absolute paths, private specification fields, or unverified local state.
|
|
19
|
+
|
|
20
|
+
## Recovery Levels
|
|
21
|
+
|
|
22
|
+
| Level | Who may run it | Scope | Misuse consequence |
|
|
23
|
+
| --- | --- | --- | --- |
|
|
24
|
+
| L1 | Any agent, including a worker | Read-only diagnosis, file report handoff, status/log/result inspection, current-payload E23 fallback only | Low risk, but must not mutate bindings or lifecycle state |
|
|
25
|
+
| L2 | Leader only | Resume-oriented lifecycle repair such as restart of an existing team or worker | A worker running L2 must be treated as `worker.privilege_escalation_attempt` |
|
|
26
|
+
| L3 | User/operator only | Claiming/taking over leader ownership, state surgery, coordinator respawn, fresh/reset | E19/#232 class risk: wrong workspace or pane can steal ownership or destroy context |
|
|
27
|
+
|
|
28
|
+
Spine for every incident:
|
|
29
|
+
|
|
30
|
+
1. Diagnose with L1 read-only commands.
|
|
31
|
+
2. Prefer resume or repair that preserves context.
|
|
32
|
+
3. Use L3 surgery only when the user/operator explicitly approves and the coordinator/write owner is controlled.
|
|
33
|
+
4. Use fresh/reset only as the last resort.
|
|
34
|
+
|
|
35
|
+
Fresh/reset red line:
|
|
36
|
+
|
|
37
|
+
> WARNING: context will be lost. Run this only after diagnostics, resume, and repair have all failed and the user has explicitly typed approval. Do not make it the default. Each use must emit `recovery.context_reset` with source and approval timestamp.
|
|
38
|
+
|
|
39
|
+
## Symptom Index
|
|
40
|
+
|
|
41
|
+
| Symptom | Root cause family | Prescription |
|
|
42
|
+
| --- | --- | --- |
|
|
43
|
+
| Status is confusing, messages/results may be pending, or a worker says it cannot report | Need a safe read-only picture first | §1 L1 Read-Only Triage |
|
|
44
|
+
| Leader pane exists but messages say `leader_not_attached`, `rebind_required`, `team_owner_mismatch`, or leader 被抢 | Leader ownership or receiver binding drift | §2 L3 Claim Or Rebind Leader |
|
|
45
|
+
| A live team must be recovered without losing worker context | Resume/restart path needed | §3 L2 Restart With Resume First |
|
|
46
|
+
| Shutdown/restart cleanup status is ambiguous | Need residue-based truth, not a single `ok` field | §4 L1/L2 Cleanup Verification |
|
|
47
|
+
| MCP tool call fails with `Transport closed` while the worker still has the payload | Provider-owned MCP stdio child died | §5 Superseded/Pending E23 Fallback Boundary |
|
|
48
|
+
|
|
49
|
+
## 1. L1 Read-Only Triage
|
|
50
|
+
|
|
51
|
+
[contextual: preserved] [level: L1] Any agent may run this. It reads state only and does not mutate the team.
|
|
52
|
+
|
|
53
|
+
Diagnosis:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
team-agent status --workspace "$WORKSPACE" --json --detail
|
|
57
|
+
team-agent collect --workspace "$WORKSPACE" --json
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
What to look for:
|
|
61
|
+
|
|
62
|
+
- `leader_receiver.status`
|
|
63
|
+
- `coordinator.status`
|
|
64
|
+
- agent `status`
|
|
65
|
+
- uncollected results
|
|
66
|
+
- messages stuck in `accepted`, `failed`, or `rebind_required`
|
|
67
|
+
|
|
68
|
+
Resume/repair:
|
|
69
|
+
|
|
70
|
+
- If the answer identifies an L2/L3 condition, do not run the elevated command from a worker pane. Send the evidence to the leader/user.
|
|
71
|
+
- If MCP reporting is broken, write a handoff artifact under the team's artifacts directory and tell the leader the relative path.
|
|
72
|
+
|
|
73
|
+
Last resort:
|
|
74
|
+
|
|
75
|
+
- None. This entry is read-only. Do not turn triage into fresh/reset.
|
|
76
|
+
|
|
77
|
+
RS verification record:
|
|
78
|
+
|
|
79
|
+
- `#249 Python 0.2.11 -> Rust 0.3.2 upgrade compatibility`: Rust `status`, `collect`, and `read` succeeded against an upgraded live team; restart resumed and post-restart MCP delivery worked. Evidence ledger: `dogfood-execution-ledger-T2.md`, case `#249`.
|
|
80
|
+
- Mac mini LOOP/G1-G4 evidence also captured status and command outputs after restart/shutdown flows; see `g1g4-macmini-confirm-report.md`.
|
|
81
|
+
|
|
82
|
+
Python 0.2.11 compatibility note:
|
|
83
|
+
|
|
84
|
+
- Python teams used equivalent `status`/`collect` verbs. Python-specific wrapper paths are deprecated and are intentionally not listed here.
|
|
85
|
+
|
|
86
|
+
## 2. L3 Claim Or Rebind Leader
|
|
87
|
+
|
|
88
|
+
[contextual: preserved] [level: L3] User/operator only. A worker must not run this. Misuse can steal the leader binding or recreate E19/#232 style ownership damage.
|
|
89
|
+
|
|
90
|
+
Use this when L1 triage shows the leader pane is live but leader-bound delivery is blocked by ownership or receiver drift.
|
|
91
|
+
|
|
92
|
+
Diagnosis:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
team-agent status --workspace "$WORKSPACE" --json --detail
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Confirm:
|
|
99
|
+
|
|
100
|
+
- The current pane is the intended leader/operator pane.
|
|
101
|
+
- The workspace is the intended team workspace.
|
|
102
|
+
- The target team is the intended `$TEAM` when multiple teams share a workspace.
|
|
103
|
+
|
|
104
|
+
Resume/repair:
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
team-agent claim-leader --workspace "$WORKSPACE" --team "$TEAM" --json
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
If the previous owner is live and the user explicitly approves stealing ownership, add the product's confirmation flag:
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
team-agent claim-leader --workspace "$WORKSPACE" --team "$TEAM" --confirm --json
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Validate:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
team-agent status --workspace "$WORKSPACE" --json --detail
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
The receiver should be attached and new leader-bound delivery should no longer report `leader_not_attached`.
|
|
123
|
+
|
|
124
|
+
Extreme surgery:
|
|
125
|
+
|
|
126
|
+
- Do not edit binding fields by hand unless a user/operator has approved the exact state repair and the coordinator/write owner is stopped or otherwise controlled.
|
|
127
|
+
|
|
128
|
+
Last resort:
|
|
129
|
+
|
|
130
|
+
- Fresh/reset is not a leader claim repair. Do not use it here.
|
|
131
|
+
|
|
132
|
+
RS verification record:
|
|
133
|
+
|
|
134
|
+
- Dogfood T2 R2: old live team restart/claim/takeover passed from the persistent leader pane. Evidence ledger: `dogfood-execution-ledger-T2.md`, case `R2`.
|
|
135
|
+
- RS #235 bug3 Mac mini run: `claim-leader` and `takeover` claimed live panes and persisted `leader_receiver.status="attached"` with `claimed_via="claim-leader"`. Evidence root id: `7b146ce-bug3-20260606T023011Z`.
|
|
136
|
+
- G1-G4 report: help path for `claim-leader` exited 0 and did not trigger action validation. Evidence: `g1g4-macmini-confirm-report.md`, CR-063.
|
|
137
|
+
|
|
138
|
+
Python 0.2.11 compatibility note:
|
|
139
|
+
|
|
140
|
+
- Python had analogous claim/takeover verbs. Use RS commands above for RS teams.
|
|
141
|
+
|
|
142
|
+
## 3. L2 Restart With Resume First
|
|
143
|
+
|
|
144
|
+
[contextual: partial] [level: L2] Leader only. A worker must not run this. The goal is to preserve session context; resume is always before fresh.
|
|
145
|
+
|
|
146
|
+
Use this when L1 triage shows a team or worker needs lifecycle repair but the state still carries enough runtime/session context to resume.
|
|
147
|
+
|
|
148
|
+
Diagnosis:
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
team-agent status --workspace "$WORKSPACE" --json --detail
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Resume/repair:
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
team-agent restart "$WORKSPACE" --team "$TEAM" --json
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
For a single worker where the leader has explicitly selected that worker:
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
team-agent restart-agent "$AGENT_ID" --workspace "$WORKSPACE" --team "$TEAM" --json
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Validate:
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
team-agent status --workspace "$WORKSPACE" --json --detail
|
|
170
|
+
team-agent collect --workspace "$WORKSPACE" --json
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
The expected result is `status="restarted"` or an equivalent running projection without duplicate-session errors. Do not treat a partial failure on one worker as permission to reset the whole team.
|
|
174
|
+
|
|
175
|
+
Extreme surgery:
|
|
176
|
+
|
|
177
|
+
- If restart reports a resume-integrity refusal, stop and escalate. Do not silently rerun with fresh flags.
|
|
178
|
+
|
|
179
|
+
Last resort:
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
# Last resort only; see red line at top of this reference.
|
|
183
|
+
team-agent restart "$WORKSPACE" --team "$TEAM" --allow-fresh --json
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
WARNING: context will be lost. Run this only after diagnostics, resume, and repair have all failed and the user has explicitly typed approval. Do not make it the default. Each use must emit `recovery.context_reset` with source and approval timestamp.
|
|
187
|
+
|
|
188
|
+
RS verification record:
|
|
189
|
+
|
|
190
|
+
- Dogfood T2 R2: restart returned restarted and status after restart showed a running team. Evidence ledger: `dogfood-execution-ledger-T2.md`.
|
|
191
|
+
- Dogfood #249: Rust restart resumed session `019e9ffd-c99c-7b60-bff3-9612ab35bcf5`, post-restart MCP delivery worked, and cleanup residue was 0. Evidence ledger: `dogfood-execution-ledger-T2.md`, case `#249`.
|
|
192
|
+
- G1-G4 Mac mini reconfirmation: restart recovered the same fake team in CR-007, after acknowledge-idle context in CR-021, and in coordinator recovery context in CR-052. Evidence: `g1g4-macmini-confirm-report.md`.
|
|
193
|
+
|
|
194
|
+
Python 0.2.11 compatibility note:
|
|
195
|
+
|
|
196
|
+
- Python restart procedures are deprecated compatibility background only. For RS teams, use the RS commands above.
|
|
197
|
+
|
|
198
|
+
## 4. L1/L2 Cleanup Verification
|
|
199
|
+
|
|
200
|
+
[contextual: partial] [level: L1 for verification, L2 for shutdown command] A worker may inspect cleanup evidence. Only the leader should run lifecycle cleanup.
|
|
201
|
+
|
|
202
|
+
Diagnosis:
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
team-agent status --workspace "$WORKSPACE" --json --detail
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Resume/repair:
|
|
209
|
+
|
|
210
|
+
- If a leader intentionally stops or shuts down a test team, verify by residue, not by a single top-level `ok` field.
|
|
211
|
+
- For cleanup verification, inspect whether the team session is gone and whether residual process/session lists are empty in the command output or status evidence.
|
|
212
|
+
|
|
213
|
+
Leader command:
|
|
214
|
+
|
|
215
|
+
```bash
|
|
216
|
+
team-agent shutdown --workspace "$WORKSPACE" --keep-logs --json
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
Validate:
|
|
220
|
+
|
|
221
|
+
- `session_killed=true` is decisive for a session that was expected to be killed.
|
|
222
|
+
- `residuals.sessions=[]` and `residuals.processes=[]` are decisive for cleanup.
|
|
223
|
+
- A missing coordinator after a successful cleanup can be expected; do not classify it alone as a product failure.
|
|
224
|
+
|
|
225
|
+
Extreme surgery:
|
|
226
|
+
|
|
227
|
+
- Do not manually kill shared tmux servers from a worker pane.
|
|
228
|
+
|
|
229
|
+
Last resort:
|
|
230
|
+
|
|
231
|
+
- None. Cleanup verification is not a context reset path.
|
|
232
|
+
|
|
233
|
+
RS verification record:
|
|
234
|
+
|
|
235
|
+
- Dogfood T2 R3: quick-start with true Codex worker, MCP server live, shutdown returned `ok=true`, `session_killed=true`, endpoint `has-session` rc=1, and post MCP/node/codex/process residue lines were all 0. Evidence ledger: `dogfood-execution-ledger-T2.md`.
|
|
236
|
+
- G1-G4 CR-005: stop returned `ok=true`, `status="stopped"`, `session_killed=true`; following shutdown reported missing without treating that alone as failure. Evidence: `g1g4-macmini-confirm-report.md`.
|
|
237
|
+
|
|
238
|
+
Python 0.2.11 compatibility note:
|
|
239
|
+
|
|
240
|
+
- Python cleanup notes may mention direct tmux commands. They are not listed here as RS recovery prescriptions.
|
|
241
|
+
|
|
242
|
+
## 5. Superseded/Pending E23 Fallback Boundary
|
|
243
|
+
|
|
244
|
+
[contextual: preserved] [level: L1] E23 fallback is the only narrow worker-scope exception to the normal tool boundary. It is for a current payload already in hand after a leader-bound MCP delivery failure.
|
|
245
|
+
|
|
246
|
+
Current status:
|
|
247
|
+
|
|
248
|
+
- Product implementation exists in the E23 branch for `fallback-send-leader` and `fallback-report-result`.
|
|
249
|
+
- Local RS verification has covered CLI help and focused E23 tests.
|
|
250
|
+
- This reference does not yet promote the fallback CLI to a general external prescription because the new documentation gate requires current-release RS validation in the Mac mini environment.
|
|
251
|
+
|
|
252
|
+
Until that validation is added:
|
|
253
|
+
|
|
254
|
+
- Workers should write a file handoff artifact for the current payload and ask for leader/user recovery.
|
|
255
|
+
- Leaders should prefer restart/resume of the affected worker after the current payload is preserved.
|
|
256
|
+
|
|
257
|
+
Do not use E23 fallback as a general message path. It does not permit arbitrary worker execution of L2/L3 recovery operations.
|
|
258
|
+
|
|
259
|
+
RS verification record:
|
|
260
|
+
|
|
261
|
+
- Local only at the time of this reference update: E23 focused tests and CLI help on the merged E23 documentation branch. Mac mini current-release validation is still required before adding exact fallback commands here.
|
|
262
|
+
|
|
263
|
+
Python 0.2.11 compatibility note:
|
|
264
|
+
|
|
265
|
+
- Python MCP stdio EOF had no durable reconnect. File handoff was the practical fallback. RS E23 adds product support, but this entry remains gated until real-machine validation is complete.
|
|
266
|
+
|
|
267
|
+
## Held Out Of This Reference
|
|
268
|
+
|
|
269
|
+
[contextual: unknown] Python-only procedures from older incident notes are intentionally not included as external RS prescriptions:
|
|
270
|
+
|
|
271
|
+
- legacy roster block cloning
|
|
272
|
+
- direct live runtime-state injection
|
|
273
|
+
- raw coordinator respawn commands
|
|
274
|
+
- direct tmux-server surgery
|
|
275
|
+
- hard-coded runtime paths
|
|
276
|
+
|
|
277
|
+
These may be useful forensic background for maintainers, but each must pass the RS validation gate before becoming a user-facing recipe.
|