@team-agent/installer 0.3.14 → 0.3.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -1
- package/Cargo.toml +1 -1
- package/crates/team-agent/src/cli/adapters.rs +1 -0
- package/crates/team-agent/src/cli/emit.rs +92 -11
- package/crates/team-agent/src/cli/leader.rs +2 -0
- package/crates/team-agent/src/cli/mod.rs +255 -37
- package/crates/team-agent/src/cli/send.rs +252 -0
- package/crates/team-agent/src/cli/status.rs +5 -1
- package/crates/team-agent/src/cli/status_port.rs +24 -0
- package/crates/team-agent/src/cli/tests/base.rs +21 -3
- package/crates/team-agent/src/cli/tests/divergence.rs +6 -6
- package/crates/team-agent/src/cli/tests/run_delegation.rs +15 -3
- package/crates/team-agent/src/cli/tests/shutdown_kill_plan.rs +285 -2
- package/crates/team-agent/src/cli/tests/status_send.rs +25 -0
- package/crates/team-agent/src/cli/types.rs +32 -1
- package/crates/team-agent/src/compiler/tests.rs +2 -2
- package/crates/team-agent/src/compiler.rs +1 -1
- package/crates/team-agent/src/fake_worker.rs +32 -145
- package/crates/team-agent/src/leader/start.rs +272 -8
- package/crates/team-agent/src/leader/tests/basics.rs +1 -0
- package/crates/team-agent/src/leader/tests/identity.rs +121 -25
- package/crates/team-agent/src/leader/types.rs +9 -0
- package/crates/team-agent/src/lifecycle/display.rs +3 -3
- package/crates/team-agent/src/lifecycle/launch.rs +631 -70
- package/crates/team-agent/src/lifecycle/restart/agent.rs +137 -17
- package/crates/team-agent/src/lifecycle/restart/common.rs +38 -2
- package/crates/team-agent/src/lifecycle/restart/rebuild.rs +79 -4
- package/crates/team-agent/src/lifecycle/tests/core.rs +14 -5
- package/crates/team-agent/src/lifecycle/tests/launch_spawn.rs +731 -8
- package/crates/team-agent/src/lifecycle/types.rs +5 -0
- package/crates/team-agent/src/lifecycle/worker_command_context.rs +8 -0
- package/crates/team-agent/src/mcp_server/wire.rs +153 -3
- package/crates/team-agent/src/messaging/leader_receiver.rs +276 -43
- package/crates/team-agent/src/messaging/mod.rs +6 -3
- package/crates/team-agent/src/messaging/results.rs +240 -158
- package/crates/team-agent/src/messaging/send.rs +3 -2
- package/crates/team-agent/src/messaging/tests/e23.rs +35 -0
- package/crates/team-agent/src/messaging/tests/mod.rs +6 -2
- package/crates/team-agent/src/messaging/tests/runtime.rs +9 -9
- package/crates/team-agent/src/os_probe.rs +11 -0
- package/crates/team-agent/src/state/persist.rs +6 -0
- package/crates/team-agent/src/tmux_backend.rs +85 -0
- package/crates/team-agent/src/transport/test_support.rs +46 -1
- package/crates/team-agent/src/transport.rs +27 -0
- package/package.json +4 -4
- package/skills/team-agent/references/recovery-runbook.md +277 -0
package/Cargo.lock
CHANGED
package/Cargo.toml
CHANGED
|
@@ -107,6 +107,7 @@ pub fn cmd_quick_start(args: &QuickStartArgs) -> Result<CmdResult, CliError> {
|
|
|
107
107
|
args.team_id.as_deref(),
|
|
108
108
|
args.yes,
|
|
109
109
|
args.fresh,
|
|
110
|
+
!args.no_display,
|
|
110
111
|
)?;
|
|
111
112
|
let readiness = value.get("readiness").and_then(Value::as_object);
|
|
112
113
|
let all_resumable_have_session = readiness
|
|
@@ -77,6 +77,12 @@ fn dispatch(command: &str, args: &[String], cwd: &Path) -> Result<ExitCode, CliE
|
|
|
77
77
|
"quick-start" => cmd_quick_start(&quick_start_args(args, cwd)?).map(emit_result),
|
|
78
78
|
"compile" => cmd_compile(&compile_args(args, cwd)?).map(emit_result),
|
|
79
79
|
"send" => cmd_send(&send_args(args, cwd)?).map(emit_result),
|
|
80
|
+
"fallback-send-leader" => {
|
|
81
|
+
cmd_fallback_send_leader(&fallback_send_leader_args(args, cwd)?).map(emit_result)
|
|
82
|
+
}
|
|
83
|
+
"fallback-report-result" => {
|
|
84
|
+
cmd_fallback_report_result(&fallback_report_result_args(args, cwd)?).map(emit_result)
|
|
85
|
+
}
|
|
80
86
|
"allow-peer-talk" => {
|
|
81
87
|
cmd_allow_peer_talk(&allow_peer_talk_args(args, cwd)?).map(emit_result)
|
|
82
88
|
}
|
|
@@ -136,6 +142,8 @@ const DISPATCH_COMMANDS: &[&str] = &[
|
|
|
136
142
|
"quick-start",
|
|
137
143
|
"compile",
|
|
138
144
|
"send",
|
|
145
|
+
"fallback-send-leader",
|
|
146
|
+
"fallback-report-result",
|
|
139
147
|
"allow-peer-talk",
|
|
140
148
|
"status",
|
|
141
149
|
"stop",
|
|
@@ -202,10 +210,12 @@ fn command_help(command: Option<&str>) -> String {
|
|
|
202
210
|
)
|
|
203
211
|
}
|
|
204
212
|
Some("init") => "usage: team-agent init [--workspace WORKSPACE] [--force] [--json]".to_string(),
|
|
205
|
-
Some("quick-start") => "usage: team-agent quick-start [TEAMDIR] [--workspace WORKSPACE] [--name NAME] [--team-id TEAM|--team TEAM] [--yes] [--fresh] [--json]\n\ndefaults: display_backend=
|
|
213
|
+
Some("quick-start") => "usage: team-agent quick-start [TEAMDIR] [--workspace WORKSPACE] [--name NAME] [--team-id TEAM|--team TEAM] [--yes] [--fresh] [--no-display] [--json]\n\ndefaults: display_backend=adaptive; set display_backend: none in TEAM.md or pass --no-display to use one worker window per agent.".to_string(),
|
|
206
214
|
Some("start") => "usage: team-agent start [TEAMDIR] [--yes] [--fresh] [--json]".to_string(),
|
|
207
215
|
Some("compile") => "usage: team-agent compile --team TEAM [--out FILE] [--json]".to_string(),
|
|
208
216
|
Some("send") => "usage: team-agent send TARGET MESSAGE... [--workspace WORKSPACE] [--team TEAM] [--targets AGENTS] [--task TASK] [--sender SENDER] [--watch-result] [--requires-ack|--no-ack] [--no-wait] [--timeout SECONDS] [--confirm-human] [--message-id ID] [--json]".to_string(),
|
|
217
|
+
Some("fallback-send-leader") => "usage: team-agent fallback-send-leader --workspace WORKSPACE --team TEAM --sender AGENT --message-id ID --content TEXT --primary-error ERROR [--task TASK] [--json]\n\nEmergency one-shot fallback only after a leader-bound MCP send transport failure; restart-agent after use.".to_string(),
|
|
218
|
+
Some("fallback-report-result") => "usage: team-agent fallback-report-result --workspace WORKSPACE --team TEAM --agent-id AGENT --task-id TASK --result-json JSON --primary-error ERROR [--json]\n\nEmergency one-shot fallback only after a report_result MCP transport failure; persists the result DB row, then restart-agent after use.".to_string(),
|
|
209
219
|
Some("allow-peer-talk") => "usage: team-agent allow-peer-talk A B [--workspace WORKSPACE] [--json]".to_string(),
|
|
210
220
|
Some("status") => "usage: team-agent status [AGENT] [--workspace WORKSPACE] [--team TEAM] [--summary|--json] [--detail]".to_string(),
|
|
211
221
|
Some("stop") => "usage: team-agent stop [--workspace WORKSPACE] [--team TEAM] [--keep-logs] [--json]".to_string(),
|
|
@@ -347,7 +357,9 @@ fn install_skill_args(args: &[String]) -> Result<InstallSkillArgs, CliError> {
|
|
|
347
357
|
|
|
348
358
|
/// 取 `--flag <value>` 的值(用于 install-skill 的 --source/--dest,parse_args 不覆盖的旗标)。
|
|
349
359
|
fn flag_value(args: &[String], flag: &str) -> Option<String> {
|
|
350
|
-
args.iter()
|
|
360
|
+
args.iter()
|
|
361
|
+
.position(|a| a == flag)
|
|
362
|
+
.and_then(|i| args.get(i + 1).cloned())
|
|
351
363
|
}
|
|
352
364
|
|
|
353
365
|
/// `team-agent install-skill`(RED-1 单源):repo `skills/team-agent` → `~/.codex|.claude|.copilot`。
|
|
@@ -355,7 +367,9 @@ fn flag_value(args: &[String], flag: &str) -> Option<String> {
|
|
|
355
367
|
fn cmd_install_skill(args: &InstallSkillArgs) -> Result<CmdResult, CliError> {
|
|
356
368
|
// 卸载分支(单源:走同一 SkillTarget 表的 dest_dir;all → SINGLE_TARGETS 全集)。
|
|
357
369
|
if args.uninstall {
|
|
358
|
-
let home = std::env::var_os("HOME")
|
|
370
|
+
let home = std::env::var_os("HOME")
|
|
371
|
+
.map(PathBuf::from)
|
|
372
|
+
.unwrap_or_else(|| PathBuf::from("."));
|
|
359
373
|
let targets: Vec<crate::packaging::SkillTarget> = match args.target {
|
|
360
374
|
crate::packaging::SkillTarget::All => {
|
|
361
375
|
crate::packaging::SkillTarget::SINGLE_TARGETS.to_vec()
|
|
@@ -367,7 +381,8 @@ fn cmd_install_skill(args: &InstallSkillArgs) -> Result<CmdResult, CliError> {
|
|
|
367
381
|
if let Some(dest) = t.dest_dir(&home) {
|
|
368
382
|
let existed = dest.0.exists();
|
|
369
383
|
if existed && !args.dry_run {
|
|
370
|
-
std::fs::remove_dir_all(&dest.0)
|
|
384
|
+
std::fs::remove_dir_all(&dest.0)
|
|
385
|
+
.map_err(|e| CliError::Runtime(e.to_string()))?;
|
|
371
386
|
}
|
|
372
387
|
removed.push(serde_json::json!({
|
|
373
388
|
"target": t,
|
|
@@ -386,13 +401,14 @@ fn cmd_install_skill(args: &InstallSkillArgs) -> Result<CmdResult, CliError> {
|
|
|
386
401
|
.source
|
|
387
402
|
.clone()
|
|
388
403
|
.ok_or_else(|| CliError::Usage("missing --source <skill dir>".to_string()))?;
|
|
389
|
-
let outcomes =
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
404
|
+
let outcomes =
|
|
405
|
+
crate::packaging::install::install_skill(&crate::packaging::SkillInstallOptions {
|
|
406
|
+
target: args.target,
|
|
407
|
+
dest: args.dest.clone(),
|
|
408
|
+
dry_run: args.dry_run,
|
|
409
|
+
source,
|
|
410
|
+
})
|
|
411
|
+
.map_err(|e| CliError::Runtime(e.to_string()))?;
|
|
396
412
|
let installed: Vec<serde_json::Value> = outcomes
|
|
397
413
|
.iter()
|
|
398
414
|
.map(|o| {
|
|
@@ -643,6 +659,11 @@ struct ParsedArgs {
|
|
|
643
659
|
pane: Option<String>,
|
|
644
660
|
provider: Option<String>,
|
|
645
661
|
message_id: Option<String>,
|
|
662
|
+
content: Option<String>,
|
|
663
|
+
primary_error: Option<String>,
|
|
664
|
+
agent_id: Option<String>,
|
|
665
|
+
task_id: Option<String>,
|
|
666
|
+
result_json: Option<String>,
|
|
646
667
|
}
|
|
647
668
|
|
|
648
669
|
fn parse_args(args: &[String]) -> ParsedArgs {
|
|
@@ -664,7 +685,9 @@ fn parse_args(args: &[String]) -> ParsedArgs {
|
|
|
664
685
|
"--team-id" => parsed.team_id = next_arg(args, &mut i),
|
|
665
686
|
"--targets" | "--target" | "--to" => parsed.targets = next_arg(args, &mut i),
|
|
666
687
|
"--task" => parsed.task = next_arg(args, &mut i),
|
|
688
|
+
"--task-id" => parsed.task_id = next_arg(args, &mut i),
|
|
667
689
|
"--sender" => parsed.sender = next_arg(args, &mut i),
|
|
690
|
+
"--agent-id" => parsed.agent_id = next_arg(args, &mut i),
|
|
668
691
|
"--watch-result" => parsed.watch_result = true,
|
|
669
692
|
"--requires-ack" => parsed.requires_ack = true,
|
|
670
693
|
"--no-ack" => parsed.no_ack = true,
|
|
@@ -719,6 +742,9 @@ fn parse_args(args: &[String]) -> ParsedArgs {
|
|
|
719
742
|
"--pane" => parsed.pane = next_arg(args, &mut i),
|
|
720
743
|
"--provider" => parsed.provider = next_arg(args, &mut i),
|
|
721
744
|
"--message-id" => parsed.message_id = next_arg(args, &mut i),
|
|
745
|
+
"--content" => parsed.content = next_arg(args, &mut i),
|
|
746
|
+
"--primary-error" => parsed.primary_error = next_arg(args, &mut i),
|
|
747
|
+
"--result-json" => parsed.result_json = next_arg(args, &mut i),
|
|
722
748
|
"-h" | "--help" => {}
|
|
723
749
|
other if other.starts_with("--team=") => {
|
|
724
750
|
parsed.team = Some(other.trim_start_matches("--team=").to_string());
|
|
@@ -792,6 +818,7 @@ fn quick_start_args(args: &[String], cwd: &Path) -> Result<QuickStartArgs, CliEr
|
|
|
792
818
|
team_id: parsed.team_id.or(parsed.team),
|
|
793
819
|
yes: parsed.yes,
|
|
794
820
|
fresh: parsed.fresh,
|
|
821
|
+
no_display: parsed.no_display,
|
|
795
822
|
json: parsed.json,
|
|
796
823
|
})
|
|
797
824
|
}
|
|
@@ -862,6 +889,60 @@ fn send_args(args: &[String], cwd: &Path) -> Result<SendArgs, CliError> {
|
|
|
862
889
|
})
|
|
863
890
|
}
|
|
864
891
|
|
|
892
|
+
fn fallback_send_leader_args(
|
|
893
|
+
args: &[String],
|
|
894
|
+
cwd: &Path,
|
|
895
|
+
) -> Result<FallbackSendLeaderArgs, CliError> {
|
|
896
|
+
let parsed = parse_args(args);
|
|
897
|
+
Ok(FallbackSendLeaderArgs {
|
|
898
|
+
workspace: workspace(&parsed, cwd),
|
|
899
|
+
team: parsed.team,
|
|
900
|
+
sender: parsed
|
|
901
|
+
.sender
|
|
902
|
+
.or(parsed.agent_id)
|
|
903
|
+
.ok_or_else(|| CliError::Usage("missing --sender <agent>".to_string()))?,
|
|
904
|
+
task: parsed.task.or(parsed.task_id),
|
|
905
|
+
message_id: parsed
|
|
906
|
+
.message_id
|
|
907
|
+
.ok_or_else(|| CliError::Usage("missing --message-id <id>".to_string()))?,
|
|
908
|
+
content: parsed
|
|
909
|
+
.content
|
|
910
|
+
.or_else(|| (!parsed.positionals.is_empty()).then(|| parsed.positionals.join(" ")))
|
|
911
|
+
.ok_or_else(|| CliError::Usage("missing --content <text>".to_string()))?,
|
|
912
|
+
primary_error: parsed
|
|
913
|
+
.primary_error
|
|
914
|
+
.ok_or_else(|| CliError::Usage("missing --primary-error <error>".to_string()))?,
|
|
915
|
+
json: parsed.json,
|
|
916
|
+
})
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
fn fallback_report_result_args(
|
|
920
|
+
args: &[String],
|
|
921
|
+
cwd: &Path,
|
|
922
|
+
) -> Result<FallbackReportResultArgs, CliError> {
|
|
923
|
+
let parsed = parse_args(args);
|
|
924
|
+
Ok(FallbackReportResultArgs {
|
|
925
|
+
workspace: workspace(&parsed, cwd),
|
|
926
|
+
team: parsed.team,
|
|
927
|
+
agent_id: parsed
|
|
928
|
+
.agent_id
|
|
929
|
+
.or(parsed.sender)
|
|
930
|
+
.ok_or_else(|| CliError::Usage("missing --agent-id <agent>".to_string()))?,
|
|
931
|
+
task_id: parsed
|
|
932
|
+
.task_id
|
|
933
|
+
.or(parsed.task)
|
|
934
|
+
.ok_or_else(|| CliError::Usage("missing --task-id <task>".to_string()))?,
|
|
935
|
+
result_json: parsed
|
|
936
|
+
.result_json
|
|
937
|
+
.or_else(|| parsed.result)
|
|
938
|
+
.ok_or_else(|| CliError::Usage("missing --result-json <json>".to_string()))?,
|
|
939
|
+
primary_error: parsed
|
|
940
|
+
.primary_error
|
|
941
|
+
.ok_or_else(|| CliError::Usage("missing --primary-error <error>".to_string()))?,
|
|
942
|
+
json: parsed.json,
|
|
943
|
+
})
|
|
944
|
+
}
|
|
945
|
+
|
|
865
946
|
fn allow_peer_talk_args(args: &[String], cwd: &Path) -> Result<AllowPeerTalkArgs, CliError> {
|
|
866
947
|
let parsed = parse_args(args);
|
|
867
948
|
Ok(AllowPeerTalkArgs {
|
|
@@ -29,6 +29,8 @@ pub fn leader_launcher_args(values: &[String]) -> Result<LeaderLauncherArgs, Cli
|
|
|
29
29
|
out.attach_existing = true;
|
|
30
30
|
} else if token == "--confirm" {
|
|
31
31
|
out.confirm_attach = true;
|
|
32
|
+
} else if token == "--external-leader" {
|
|
33
|
+
out.external_leader = true;
|
|
32
34
|
} else if token == "--attach-session" {
|
|
33
35
|
let Some(value) = values.get(idx + 1) else {
|
|
34
36
|
return Err(CliError::Runtime(
|
|
@@ -115,9 +115,16 @@ pub mod lifecycle_port {
|
|
|
115
115
|
team_id: Option<&str>,
|
|
116
116
|
yes: bool,
|
|
117
117
|
fresh: bool,
|
|
118
|
+
open_display: bool,
|
|
118
119
|
) -> Result<Value, CliError> {
|
|
119
|
-
match crate::lifecycle::
|
|
120
|
-
workspace,
|
|
120
|
+
match crate::lifecycle::quick_start_in_workspace_with_display(
|
|
121
|
+
workspace,
|
|
122
|
+
agents_dir,
|
|
123
|
+
name,
|
|
124
|
+
yes,
|
|
125
|
+
fresh,
|
|
126
|
+
team_id,
|
|
127
|
+
open_display,
|
|
121
128
|
) {
|
|
122
129
|
Ok(report) => Ok(quick_start_value(report)),
|
|
123
130
|
Err(e) => Ok(error_value(e)),
|
|
@@ -141,6 +148,7 @@ pub mod lifecycle_port {
|
|
|
141
148
|
attach.attach_existing,
|
|
142
149
|
attach.confirm_attach,
|
|
143
150
|
attach_session.as_ref(),
|
|
151
|
+
attach.external_leader,
|
|
144
152
|
)
|
|
145
153
|
.map_err(|e| CliError::Runtime(e.to_string()))?;
|
|
146
154
|
let outcome = crate::leader::start::execute_leader_plan(&plan, cwd)
|
|
@@ -150,10 +158,21 @@ pub mod lifecycle_port {
|
|
|
150
158
|
crate::leader::LeaderLaunchStatus::Detached => true,
|
|
151
159
|
crate::leader::LeaderLaunchStatus::NotStarted => false,
|
|
152
160
|
};
|
|
161
|
+
let leader_attach_command = if plan.is_external_leader {
|
|
162
|
+
None
|
|
163
|
+
} else {
|
|
164
|
+
plan.session_name.as_ref().and_then(|session| {
|
|
165
|
+
crate::tmux_backend::attach_command_for_workspace(cwd, session, "leader")
|
|
166
|
+
})
|
|
167
|
+
};
|
|
153
168
|
Ok(json!({
|
|
154
169
|
"ok": ok,
|
|
155
170
|
"provider": provider,
|
|
156
171
|
"mode": plan.mode,
|
|
172
|
+
"leader_topology": if plan.is_external_leader { "external" } else { "managed" },
|
|
173
|
+
"is_external_leader": plan.is_external_leader,
|
|
174
|
+
"leader_window": plan.leader_window.as_ref().map(|window| window.as_str().to_string()),
|
|
175
|
+
"leader_attach_command": leader_attach_command,
|
|
157
176
|
"status": outcome.status,
|
|
158
177
|
"exit_code": outcome.exit_code,
|
|
159
178
|
"reason": outcome.reason,
|
|
@@ -169,7 +188,7 @@ pub mod lifecycle_port {
|
|
|
169
188
|
.map_err(|e| CliError::Runtime(e.to_string()))?;
|
|
170
189
|
let state = shutdown_state_for_team(&run_ws, team)?;
|
|
171
190
|
let transport = if let Some(endpoint) = legacy_worker_tmux_endpoint(&state) {
|
|
172
|
-
|
|
191
|
+
shutdown_transport_for_endpoint(endpoint)
|
|
173
192
|
} else {
|
|
174
193
|
shutdown_workspace_transport(&run_ws)
|
|
175
194
|
};
|
|
@@ -275,9 +294,39 @@ pub mod lifecycle_port {
|
|
|
275
294
|
// no independent ps/tmux re-derivation (N39).
|
|
276
295
|
let pane_targets = transport.list_targets().unwrap_or_default();
|
|
277
296
|
let sessions = socket_session_names_from_targets(&pane_targets);
|
|
297
|
+
if !state_uses_external_leader(state) {
|
|
298
|
+
return managed_leader_socket_cleanup(transport, state, &sessions, event_log);
|
|
299
|
+
}
|
|
278
300
|
let anchor_sessions = anchor_sessions_from_state(state, &pane_targets, event_log);
|
|
279
301
|
match sessions_to_kill(&sessions, &anchor_sessions) {
|
|
280
302
|
KillDecision::KillServerExclusive => {
|
|
303
|
+
if state
|
|
304
|
+
.get("tmux_socket_source")
|
|
305
|
+
.and_then(Value::as_str)
|
|
306
|
+
== Some("leader_env")
|
|
307
|
+
{
|
|
308
|
+
let _ = event_log.write(
|
|
309
|
+
"shutdown.kill_server_skipped_shared_socket",
|
|
310
|
+
json!({
|
|
311
|
+
"reason": "leader_env_tmux_socket",
|
|
312
|
+
"spared_sessions": [],
|
|
313
|
+
"killed_sessions": sessions.iter().map(|s| s.as_str()).collect::<Vec<_>>(),
|
|
314
|
+
}),
|
|
315
|
+
);
|
|
316
|
+
let mut error = None;
|
|
317
|
+
for session in &sessions {
|
|
318
|
+
if let Err(err) = transport.kill_session(session) {
|
|
319
|
+
if !tmux_absent_error(&err.to_string()) {
|
|
320
|
+
error.get_or_insert_with(|| err.to_string());
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
return ShutdownSocketCleanup {
|
|
325
|
+
killed_sessions: sessions,
|
|
326
|
+
spared_sessions: Vec::new(),
|
|
327
|
+
error,
|
|
328
|
+
};
|
|
329
|
+
}
|
|
281
330
|
let error = transport.kill_server().err().map(|error| error.to_string());
|
|
282
331
|
ShutdownSocketCleanup {
|
|
283
332
|
killed_sessions: sessions,
|
|
@@ -313,6 +362,66 @@ pub mod lifecycle_port {
|
|
|
313
362
|
}
|
|
314
363
|
}
|
|
315
364
|
|
|
365
|
+
fn state_uses_external_leader(state: &Value) -> bool {
|
|
366
|
+
state
|
|
367
|
+
.get("is_external_leader")
|
|
368
|
+
.and_then(Value::as_bool)
|
|
369
|
+
.unwrap_or(true)
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
fn managed_leader_socket_cleanup(
|
|
373
|
+
transport: &dyn crate::transport::Transport,
|
|
374
|
+
state: &Value,
|
|
375
|
+
sessions: &[crate::transport::SessionName],
|
|
376
|
+
event_log: &crate::event_log::EventLog,
|
|
377
|
+
) -> ShutdownSocketCleanup {
|
|
378
|
+
let target = state
|
|
379
|
+
.get("session_name")
|
|
380
|
+
.and_then(Value::as_str)
|
|
381
|
+
.filter(|session| !session.is_empty())
|
|
382
|
+
.map(crate::transport::SessionName::new);
|
|
383
|
+
let mut to_kill = Vec::new();
|
|
384
|
+
if let Some(target) = target {
|
|
385
|
+
if sessions.is_empty()
|
|
386
|
+
|| sessions
|
|
387
|
+
.iter()
|
|
388
|
+
.any(|session| session.as_str() == target.as_str())
|
|
389
|
+
{
|
|
390
|
+
to_kill.push(target);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
let spared = sessions
|
|
394
|
+
.iter()
|
|
395
|
+
.filter(|session| {
|
|
396
|
+
!to_kill
|
|
397
|
+
.iter()
|
|
398
|
+
.any(|target| target.as_str() == session.as_str())
|
|
399
|
+
})
|
|
400
|
+
.cloned()
|
|
401
|
+
.collect::<Vec<_>>();
|
|
402
|
+
let _ = event_log.write(
|
|
403
|
+
"shutdown.kill_server_skipped_managed_leader",
|
|
404
|
+
json!({
|
|
405
|
+
"reason": "managed_leader_topology",
|
|
406
|
+
"spared_sessions": spared.iter().map(|s| s.as_str()).collect::<Vec<_>>(),
|
|
407
|
+
"killed_sessions": to_kill.iter().map(|s| s.as_str()).collect::<Vec<_>>(),
|
|
408
|
+
}),
|
|
409
|
+
);
|
|
410
|
+
let mut error = None;
|
|
411
|
+
for session in &to_kill {
|
|
412
|
+
if let Err(err) = transport.kill_session(session) {
|
|
413
|
+
if !tmux_absent_error(&err.to_string()) {
|
|
414
|
+
error.get_or_insert_with(|| err.to_string());
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
ShutdownSocketCleanup {
|
|
419
|
+
killed_sessions: to_kill,
|
|
420
|
+
spared_sessions: spared,
|
|
421
|
+
error,
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
|
|
316
425
|
pub fn shutdown_with_transport(
|
|
317
426
|
workspace: &Path,
|
|
318
427
|
keep_logs: bool,
|
|
@@ -449,8 +558,12 @@ pub mod lifecycle_port {
|
|
|
449
558
|
);
|
|
450
559
|
deadline.check("stop_coordinator")?;
|
|
451
560
|
let mut coordinator_timeout = false;
|
|
561
|
+
let mut coordinator_post_stop = CoordinatorStopObservation::NotNeeded;
|
|
562
|
+
let mut coordinator_pid_for_report = None;
|
|
452
563
|
let stopped = if team.is_none() {
|
|
453
564
|
let wp = crate::coordinator::WorkspacePath::new(run_workspace.clone());
|
|
565
|
+
let coordinator_pid_before_stop = crate::coordinator::coordinator_health(&wp).pid;
|
|
566
|
+
coordinator_pid_for_report = coordinator_pid_before_stop.map(|pid| pid.get());
|
|
454
567
|
match stop_coordinator_bounded(wp, std::time::Duration::from_millis(900)) {
|
|
455
568
|
Some(Ok(report)) => Some(report),
|
|
456
569
|
Some(Err(error)) => {
|
|
@@ -459,16 +572,31 @@ pub mod lifecycle_port {
|
|
|
459
572
|
}
|
|
460
573
|
None => {
|
|
461
574
|
coordinator_timeout = true;
|
|
575
|
+
let wp = crate::coordinator::WorkspacePath::new(run_workspace.clone());
|
|
576
|
+
coordinator_post_stop =
|
|
577
|
+
coordinator_post_stop_observation(&wp, coordinator_pid_before_stop);
|
|
462
578
|
None
|
|
463
579
|
}
|
|
464
580
|
}
|
|
465
581
|
} else {
|
|
466
582
|
None
|
|
467
583
|
};
|
|
584
|
+
if let Some(stopped) = stopped.as_ref().filter(|stopped| !stopped.ok) {
|
|
585
|
+
let wp = crate::coordinator::WorkspacePath::new(run_workspace.clone());
|
|
586
|
+
coordinator_post_stop = coordinator_post_stop_observation(&wp, stopped.pid);
|
|
587
|
+
}
|
|
468
588
|
let probe_timeout = crate::os_probe::probe_timeout();
|
|
469
|
-
|
|
470
|
-
//
|
|
471
|
-
|
|
589
|
+
let probe_timeout_kind = probe_timeout.as_ref().map(|timeout| timeout.probe);
|
|
590
|
+
// swallow batch 1: a failed ps probe degrades cleanup truthfully — the
|
|
591
|
+
// empty table must never read as a clean "no residual processes". A slow
|
|
592
|
+
// per-process cwd probe is diagnostic only once session/process residuals
|
|
593
|
+
// are otherwise clean.
|
|
594
|
+
let cleanup_truth_degraded = probe_degraded || probe_timeout_kind == Some("ps_table");
|
|
595
|
+
let diagnostic_probe_degraded = probe_timeout_kind == Some("lsof_cwd");
|
|
596
|
+
let other_probe_timeout_degraded =
|
|
597
|
+
probe_timeout.is_some() && !cleanup_truth_degraded && !diagnostic_probe_degraded;
|
|
598
|
+
let verification_degraded =
|
|
599
|
+
cleanup_truth_degraded || diagnostic_probe_degraded || other_probe_timeout_degraded;
|
|
472
600
|
let session_killed = !killed_sessions.is_empty()
|
|
473
601
|
&& kill_error.is_none()
|
|
474
602
|
&& session_residuals.is_empty()
|
|
@@ -494,33 +622,19 @@ pub mod lifecycle_port {
|
|
|
494
622
|
let coordinator_pid = stopped
|
|
495
623
|
.as_ref()
|
|
496
624
|
.and_then(|stopped| stopped.pid.map(|p| p.get()));
|
|
497
|
-
let
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
"partial"
|
|
511
|
-
} else if kill_error.is_some() {
|
|
512
|
-
"failed"
|
|
513
|
-
} else {
|
|
514
|
-
"partial"
|
|
515
|
-
};
|
|
516
|
-
let phase = if coordinator_timeout {
|
|
517
|
-
Some("stop_coordinator")
|
|
518
|
-
} else if verification_degraded {
|
|
519
|
-
Some("os_probe")
|
|
520
|
-
} else {
|
|
521
|
-
None
|
|
522
|
-
};
|
|
523
|
-
let probe_timeout_kind = probe_timeout.as_ref().map(|timeout| timeout.probe);
|
|
625
|
+
let coordinator_pid = coordinator_pid.or(coordinator_pid_for_report);
|
|
626
|
+
let outcome = classify_shutdown_outcome(ShutdownOutcomeInput {
|
|
627
|
+
kill_error: kill_error.is_some(),
|
|
628
|
+
session_residuals: !session_residuals.is_empty(),
|
|
629
|
+
process_residuals: !process_residuals.is_empty(),
|
|
630
|
+
cleanup_truth_degraded,
|
|
631
|
+
coordinator_timeout,
|
|
632
|
+
coordinator_stop_ok: stopped.as_ref().map(|stopped| stopped.ok),
|
|
633
|
+
coordinator_post_stop,
|
|
634
|
+
});
|
|
635
|
+
let ok = outcome.ok;
|
|
636
|
+
let status = outcome.status;
|
|
637
|
+
let phase = outcome.phase;
|
|
524
638
|
let probe_timeout_value = probe_timeout.as_ref().map(|timeout| {
|
|
525
639
|
json!({
|
|
526
640
|
"probe": timeout.probe,
|
|
@@ -577,14 +691,110 @@ pub mod lifecycle_port {
|
|
|
577
691
|
/// worker thread — on a timely result it joins immediately; on timeout it gives the
|
|
578
692
|
/// thread one short grace join window instead of dropping it detached (repeated
|
|
579
693
|
/// shutdowns no longer accumulate leaked threads racing the same workspace).
|
|
694
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
695
|
+
pub(crate) enum CoordinatorStopObservation {
|
|
696
|
+
NotNeeded,
|
|
697
|
+
Gone,
|
|
698
|
+
Running,
|
|
699
|
+
Unknown,
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
pub(crate) struct ShutdownOutcome {
|
|
703
|
+
pub(crate) ok: bool,
|
|
704
|
+
pub(crate) status: &'static str,
|
|
705
|
+
pub(crate) phase: Option<&'static str>,
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
pub(crate) struct ShutdownOutcomeInput {
|
|
709
|
+
pub(crate) kill_error: bool,
|
|
710
|
+
pub(crate) session_residuals: bool,
|
|
711
|
+
pub(crate) process_residuals: bool,
|
|
712
|
+
pub(crate) cleanup_truth_degraded: bool,
|
|
713
|
+
pub(crate) coordinator_timeout: bool,
|
|
714
|
+
pub(crate) coordinator_stop_ok: Option<bool>,
|
|
715
|
+
pub(crate) coordinator_post_stop: CoordinatorStopObservation,
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
pub(crate) fn classify_shutdown_outcome(input: ShutdownOutcomeInput) -> ShutdownOutcome {
|
|
719
|
+
let coordinator_clean = match input.coordinator_post_stop {
|
|
720
|
+
CoordinatorStopObservation::Gone => true,
|
|
721
|
+
CoordinatorStopObservation::Running | CoordinatorStopObservation::Unknown => false,
|
|
722
|
+
CoordinatorStopObservation::NotNeeded => {
|
|
723
|
+
!input.coordinator_timeout && input.coordinator_stop_ok.unwrap_or(true)
|
|
724
|
+
}
|
|
725
|
+
};
|
|
726
|
+
let ok = coordinator_clean
|
|
727
|
+
&& !input.kill_error
|
|
728
|
+
&& !input.session_residuals
|
|
729
|
+
&& !input.process_residuals
|
|
730
|
+
&& !input.cleanup_truth_degraded;
|
|
731
|
+
if ok {
|
|
732
|
+
return ShutdownOutcome {
|
|
733
|
+
ok,
|
|
734
|
+
status: "ok",
|
|
735
|
+
phase: None,
|
|
736
|
+
};
|
|
737
|
+
}
|
|
738
|
+
let (status, phase) = if input.coordinator_timeout && !coordinator_clean {
|
|
739
|
+
("timeout", Some("stop_coordinator"))
|
|
740
|
+
} else if input.cleanup_truth_degraded {
|
|
741
|
+
("partial", Some("os_probe"))
|
|
742
|
+
} else if input.kill_error {
|
|
743
|
+
("failed", None)
|
|
744
|
+
} else {
|
|
745
|
+
("partial", None)
|
|
746
|
+
};
|
|
747
|
+
ShutdownOutcome { ok, status, phase }
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
fn coordinator_post_stop_observation(
|
|
751
|
+
workspace: &crate::coordinator::WorkspacePath,
|
|
752
|
+
pid: Option<crate::coordinator::Pid>,
|
|
753
|
+
) -> CoordinatorStopObservation {
|
|
754
|
+
if let Some(pid) = pid {
|
|
755
|
+
match crate::coordinator::pid_is_running(pid) {
|
|
756
|
+
Ok(true) => return CoordinatorStopObservation::Running,
|
|
757
|
+
Ok(false) => return CoordinatorStopObservation::Gone,
|
|
758
|
+
Err(_) => {}
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
let health = crate::coordinator::coordinator_health(workspace);
|
|
762
|
+
match health.status {
|
|
763
|
+
crate::coordinator::CoordinatorHealthStatus::Running => {
|
|
764
|
+
CoordinatorStopObservation::Running
|
|
765
|
+
}
|
|
766
|
+
crate::coordinator::CoordinatorHealthStatus::Missing
|
|
767
|
+
| crate::coordinator::CoordinatorHealthStatus::InvalidPid
|
|
768
|
+
| crate::coordinator::CoordinatorHealthStatus::Stale => {
|
|
769
|
+
CoordinatorStopObservation::Gone
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
|
|
580
774
|
fn stop_coordinator_bounded(
|
|
581
775
|
workspace: crate::coordinator::WorkspacePath,
|
|
582
776
|
timeout: std::time::Duration,
|
|
583
777
|
) -> Option<Result<crate::coordinator::types::StopReport, String>> {
|
|
778
|
+
stop_coordinator_bounded_with(workspace, timeout, |workspace| {
|
|
779
|
+
crate::coordinator::stop_coordinator(workspace).map_err(|error| error.to_string())
|
|
780
|
+
})
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
pub(crate) fn stop_coordinator_bounded_with<F>(
|
|
784
|
+
workspace: crate::coordinator::WorkspacePath,
|
|
785
|
+
timeout: std::time::Duration,
|
|
786
|
+
stop: F,
|
|
787
|
+
) -> Option<Result<crate::coordinator::types::StopReport, String>>
|
|
788
|
+
where
|
|
789
|
+
F: FnOnce(
|
|
790
|
+
&crate::coordinator::WorkspacePath,
|
|
791
|
+
) -> Result<crate::coordinator::types::StopReport, String>
|
|
792
|
+
+ Send
|
|
793
|
+
+ 'static,
|
|
794
|
+
{
|
|
584
795
|
let (tx, rx) = std::sync::mpsc::channel();
|
|
585
796
|
let handle = std::thread::spawn(move || {
|
|
586
|
-
let result =
|
|
587
|
-
crate::coordinator::stop_coordinator(&workspace).map_err(|error| error.to_string());
|
|
797
|
+
let result = stop(&workspace);
|
|
588
798
|
let _ = tx.send(result);
|
|
589
799
|
});
|
|
590
800
|
let outcome = rx.recv_timeout(timeout).ok();
|
|
@@ -599,8 +809,7 @@ pub mod lifecycle_port {
|
|
|
599
809
|
match rx.recv_timeout(std::time::Duration::from_millis(250)) {
|
|
600
810
|
Ok(late) => {
|
|
601
811
|
let _ = handle.join();
|
|
602
|
-
|
|
603
|
-
None
|
|
812
|
+
Some(late)
|
|
604
813
|
}
|
|
605
814
|
Err(_) => {
|
|
606
815
|
if handle.is_finished() {
|
|
@@ -652,10 +861,19 @@ pub mod lifecycle_port {
|
|
|
652
861
|
crate::tmux_backend::TmuxBackend::for_workspace(workspace)
|
|
653
862
|
}
|
|
654
863
|
|
|
864
|
+
fn shutdown_transport_for_endpoint(endpoint: &str) -> crate::tmux_backend::TmuxBackend {
|
|
865
|
+
if Path::new(endpoint).is_absolute() {
|
|
866
|
+
crate::tmux_backend::TmuxBackend::for_tmux_endpoint(endpoint)
|
|
867
|
+
} else {
|
|
868
|
+
crate::tmux_backend::TmuxBackend::for_socket_name(endpoint)
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
|
|
655
872
|
fn legacy_worker_tmux_endpoint(state: &Value) -> Option<&str> {
|
|
656
873
|
state
|
|
657
874
|
.get("tmux_endpoint")
|
|
658
875
|
.and_then(Value::as_str)
|
|
876
|
+
.or_else(|| state.get("tmux_socket").and_then(Value::as_str))
|
|
659
877
|
.filter(|endpoint| !endpoint.is_empty())
|
|
660
878
|
}
|
|
661
879
|
|