@team-agent/installer 0.3.12 → 0.3.14

package/crates/team-agent/src/lifecycle/restart/rebuild.rs

@@ -116,10 +116,9 @@ pub fn restart_with_transport_with_session_convergence_deadline(
     // 显式存在性门(下移后):selected.spec_path 经读序 B 已定位 runtime/legacy spec。
     // 缺(空目录 restart 等)→ 报真实期望路径,不误导去用户目录找。
     if !selected.spec_path.as_ref().is_some_and(|p| p.exists()) {
-        let expected = selected
-            .spec_path
-            .clone()
-            .unwrap_or_else(|| crate::model::paths::runtime_spec_path(&selected.run_workspace, &selected.team_key));
+        let expected = selected.spec_path.clone().unwrap_or_else(|| {
+            crate::model::paths::runtime_spec_path(&selected.run_workspace, &selected.team_key)
+        });
         return Err(LifecycleError::TeamSelect(format!(
             "missing spec for restart: {} (run `team-agent quick-start <teamdir>` first, or restore the team's role docs)",
             expected.display()
@@ -130,9 +129,11 @@ pub fn restart_with_transport_with_session_convergence_deadline(
     // E5 task#3 / RC-A6a + E4(leader 裁定:每次 restart 都从角色定义重建 runtime spec,覆盖):
     // 角色定义=第一真相源。角色齐 → compile_team 重建 + 保留运行期 override(session_name)+
     // 写 runtime spec。角色缺(TEAM.md/agents 不在)→ 显式拒(列缺哪些),旧 spec 原地保留不删不用。
-    let spec = rebuild_runtime_spec_from_roles(&selected.run_workspace, &selected.team_key, &state)?;
+    let spec =
+        rebuild_runtime_spec_from_roles(&selected.run_workspace, &selected.team_key, &state)?;
     // 重建后 spec_workspace 恒为 runtime spec 的父目录(.team/runtime/<team_key>/)。
-    let runtime_spec = crate::model::paths::runtime_spec_path(&selected.run_workspace, &selected.team_key);
+    let runtime_spec =
+        crate::model::paths::runtime_spec_path(&selected.run_workspace, &selected.team_key);
     let spec_workspace = runtime_spec.parent().ok_or_else(|| {
         LifecycleError::TeamSelect("active team spec workspace not found".to_string())
     })?;
@@ -173,13 +174,23 @@ pub fn restart_with_transport_with_session_convergence_deadline(
         crate::state::projection::save_team_scoped_state(&selected.run_workspace, &state)
             .map_err(|e| LifecycleError::StatePersist(e.to_string()))?;
     }
-    let forced_fresh_missing = if convergence.converged {
+    let mut forced_fresh_missing = if convergence.converged {
         std::collections::BTreeSet::new()
     } else {
         convergence.missing.iter().cloned().collect()
     };
     let forced_fresh_convergence = (!convergence.converged).then_some(convergence.clone());
-    let plan = classify_restart_plan_with_resume_validation(Some(&selected.run_workspace), &state, allow_fresh)?;
+    let plan = classify_restart_plan_with_resume_validation(
+        Some(&selected.run_workspace),
+        &state,
+        allow_fresh,
+    )?;
+    for decision in &plan.decisions {
+        if matches!(decision.decision, ResumeDecision::FreshStart) && decision.session_id.is_some()
+        {
+            forced_fresh_missing.insert(decision.agent_id.as_str().to_string());
+        }
+    }
     write_restart_resume_decision_events(
         &selected.run_workspace,
         &state,
@@ -212,35 +223,65 @@ pub fn restart_with_transport_with_session_convergence_deadline(
         crate::state::projection::save_team_scoped_state(&selected.run_workspace, &state)
             .map_err(|e| LifecycleError::StatePersist(e.to_string()))?;
     }
-    let mut last_spawned: Option<AgentId> = None;
+    let mut successful_agents: Vec<RestartedAgent> = Vec::new();
+    let mut failed_agents: Vec<RestartFailedAgent> = Vec::new();
+    let mut fatal_resume_failure = false;
+    // B5 restart isolation loop: per-agent spawn failures must be recorded and
+    // isolated here. G1 resume-integrity failures set a fatal flag and skip later
+    // spawns; do not reintroduce `?`, `break`, or `return` inside this loop.
+    // BEGIN_B5_RESTART_ISOLATION_LOOP
     for decision in &plan.decisions {
-        let agent = state
+        if fatal_resume_failure {
+            continue;
+        }
+        let Some(agent) = state
             .get("agents")
             .and_then(|v| v.get(decision.agent_id.as_str()))
-            .ok_or_else(|| {
-                LifecycleError::RequirementUnmet(format!(
-                    "agent {} not found for restart",
-                    decision.agent_id
-                ))
-            })?
-            .clone();
+            .cloned()
+        else {
+            let error = format!("agent {} not found for restart", decision.agent_id);
+            mark_agent_restart_failed(&mut state, decision, &error);
+            let _ = write_restart_agent_failed_event(
+                &selected.run_workspace,
+                decision,
+                "spawn",
+                &error,
+            );
+            failed_agents.push(restart_failed_agent(decision, "spawn", error));
+            continue;
+        };
         let session_id = if matches!(decision.restart_mode, StartMode::Resumed) {
             decision.session_id.as_ref()
         } else {
             None
         };
-        let session_live = session_live_or_default(transport, &session_name, false);
+        let mut session_live = session_live_or_default(transport, &session_name, false);
         if !session_live {
-            if let Some(previous) = &last_spawned {
-                return Err(LifecycleError::Transport(format!(
+            if let Some(previous) = successful_agents.pop() {
+                let error = format!(
                     "session_disappeared_after_spawn: provider_resume_exited for {}; session {} disappeared before spawning {}",
-                    previous,
+                    previous.agent_id,
                     session_name.as_str(),
                     decision.agent_id
-                )));
+                );
+                mark_agent_restart_failed(&mut state, &previous, &error);
+                let _ = write_restart_agent_failed_event(
+                    &selected.run_workspace,
+                    &previous,
+                    "resume",
+                    &error,
+                );
+                failed_agents.push(restart_failed_agent(&previous, "resume", error));
+                if is_resume_integrity_failure(&previous, "resume", "") {
+                    fatal_resume_failure = true;
+                }
+                session_live = false;
             }
         }
-        let spawn = spawn_agent_window(
+        if fatal_resume_failure {
+            continue;
+        }
+        let spawn = match spawn_agent_window(
             &selected.run_workspace,
             &session_name,
             &decision.agent_id,
@@ -250,10 +291,42 @@ pub fn restart_with_transport_with_session_convergence_deadline(
             transport,
             Some(&safety),
             Some(spec_workspace),
-        )?;
-        verify_spawned_agent_live(&decision.agent_id, &spawn, transport)?;
-        mark_agent_respawned(&mut state, &decision.agent_id, &spawn, transport, &safety)?;
-        last_spawned = Some(decision.agent_id.clone());
+        ) {
+            Ok(spawn) => spawn,
+            Err(error) => {
+                let error = error.to_string();
+                mark_agent_restart_failed(&mut state, decision, &error);
+                let phase = restart_failure_phase(decision, "spawn", &error);
+                let _ = write_restart_agent_failed_event(
+                    &selected.run_workspace,
+                    decision,
+                    phase,
+                    &error,
+                );
+                failed_agents.push(restart_failed_agent(decision, phase, error));
+                if phase == "resume" {
+                    fatal_resume_failure = true;
+                }
+                continue;
+            }
+        };
+        if let Err(error) = verify_spawned_agent_live(&decision.agent_id, &spawn, transport)
+            .and_then(|_| {
+                mark_agent_respawned(&mut state, &decision.agent_id, &spawn, transport, &safety)
+            })
+        {
+            let error = error.to_string();
+            mark_agent_restart_failed(&mut state, decision, &error);
+            let phase = restart_failure_phase(decision, "readiness", &error);
+            let _ =
+                write_restart_agent_failed_event(&selected.run_workspace, decision, phase, &error);
+            failed_agents.push(restart_failed_agent(decision, phase, error));
+            if phase == "resume" {
+                fatal_resume_failure = true;
+            }
+            continue;
+        }
+        successful_agents.push(decision.clone());
         if let Some(agent) = state
             .get_mut("agents")
             .and_then(serde_json::Value::as_object_mut)
@@ -263,14 +336,47 @@ pub fn restart_with_transport_with_session_convergence_deadline(
             persist_effective_approval_policy_for_restart(agent, &safety);
         }
     }
+    // END_B5_RESTART_ISOLATION_LOOP
     crate::state::projection::save_team_scoped_state(&selected.run_workspace, &state)
         .map_err(|e| LifecycleError::StatePersist(e.to_string()))?;
+    if fatal_resume_failure {
+        let attach_commands = Vec::new();
+        let next_actions = restart_failure_next_actions(&failed_agents);
+        write_restart_completed_event(
+            &selected.run_workspace,
+            &successful_agents,
+            &failed_agents,
+            "fail",
+        )?;
+        return Ok(RestartReport::Failed {
+            session_name,
+            failed_agents,
+            next_actions,
+            attach_commands,
+        });
+    }
+    if successful_agents.is_empty() && !failed_agents.is_empty() {
+        let attach_commands = Vec::new();
+        let next_actions = restart_failure_next_actions(&failed_agents);
+        write_restart_completed_event(
+            &selected.run_workspace,
+            &successful_agents,
+            &failed_agents,
+            "fail",
+        )?;
+        return Ok(RestartReport::Failed {
+            session_name,
+            failed_agents,
+            next_actions,
+            attach_commands,
+        });
+    }
     let coordinator_started = start_coordinator_for_workspace(&selected.run_workspace)?;
     wait_restart_readiness_or_timeout(
         &selected.run_workspace,
         &state,
         &session_name,
-        &plan.decisions,
+        &successful_agents,
         transport,
         restart_readiness_deadline(readiness_deadline_ms),
         restart_readiness_poll_interval(),
@@ -278,14 +384,37 @@ pub fn restart_with_transport_with_session_convergence_deadline(
     let attach_commands = crate::tmux_backend::attach_commands_for_windows(
         &selected.run_workspace,
         &session_name,
-        plan.decisions
+        successful_agents
             .iter()
             .map(|decision| decision.agent_id.as_str()),
     );
-    let next_actions = attach_commands.clone();
+    let mut next_actions = attach_commands.clone();
+    if !failed_agents.is_empty() {
+        next_actions.extend(restart_failure_next_actions(&failed_agents));
+        write_restart_completed_event(
+            &selected.run_workspace,
+            &successful_agents,
+            &failed_agents,
+            "partial",
+        )?;
+        return Ok(RestartReport::Partial {
+            session_name,
+            agents: successful_agents,
+            failed_agents,
+            coordinator_started,
+            next_actions,
+            attach_commands,
+        });
+    }
+    write_restart_completed_event(
+        &selected.run_workspace,
+        &successful_agents,
+        &failed_agents,
+        "ok",
+    )?;
     Ok(RestartReport::Restarted {
         session_name,
-        agents: plan.decisions,
+        agents: successful_agents,
         coordinator_started,
         next_actions,
         attach_commands,
@@ -478,9 +607,9 @@ fn parse_duration_value_seconds_ms(value: &str) -> Option<u64> {
 }
 
 fn restart_readiness_deadline(requested_ms: Option<u64>) -> std::time::Duration {
-    requested_ms.map(std::time::Duration::from_millis).unwrap_or_else(|| {
-        env_duration_ms(&["TEAM_AGENT_RESTART_READINESS_DEADLINE_MS"], 30_000)
-    })
+    requested_ms
+        .map(std::time::Duration::from_millis)
+        .unwrap_or_else(|| env_duration_ms(&["TEAM_AGENT_RESTART_READINESS_DEADLINE_MS"], 30_000))
 }
 
 fn restart_readiness_poll_interval() -> std::time::Duration {
@@ -518,11 +647,14 @@ fn wait_restart_readiness_or_timeout(
         let elapsed = started.elapsed();
         if elapsed >= deadline {
             write_restart_readiness_timeout_event(workspace, readiness, deadline, elapsed)?;
-            return Err(LifecycleError::RequirementUnmet(restart_readiness_timeout_message(
-                workspace, readiness, deadline,
-            )));
+            return Err(LifecycleError::RequirementUnmet(
+                restart_readiness_timeout_message(workspace, readiness, deadline),
+            ));
         }
-        std::thread::sleep(std::cmp::min(poll_interval, deadline.saturating_sub(elapsed)));
+        std::thread::sleep(std::cmp::min(
+            poll_interval,
+            deadline.saturating_sub(elapsed),
+        ));
     }
 }
 
@@ -538,7 +670,11 @@ fn restart_readiness(
     let coordinator_workspace = crate::coordinator::WorkspacePath::new(workspace.to_path_buf());
     let coordinator_alive =
         crate::coordinator::coordinator_health(&coordinator_workspace).ok && session_created;
-    RestartReadiness { session_created, worker_pane_addressable, coordinator_alive }
+    RestartReadiness {
+        session_created,
+        worker_pane_addressable,
+        coordinator_alive,
+    }
 }
 
 fn restart_worker_panes_addressable(
@@ -650,7 +786,11 @@ fn restart_readiness_missing_summary(readiness: RestartReadiness) -> String {
 }
 
 fn yes_no(value: bool) -> &'static str {
-    if value { "yes" } else { "no" }
+    if value {
+        "yes"
+    } else {
+        "no"
+    }
 }
 
 fn verify_spawned_agent_live(
@@ -749,6 +889,161 @@ fn mark_agent_respawned(
     Ok(())
 }
 
+fn restart_failed_agent(
+    decision: &RestartedAgent,
+    phase: impl Into<String>,
+    error: String,
+) -> RestartFailedAgent {
+    RestartFailedAgent {
+        agent_id: decision.agent_id.clone(),
+        restart_mode: decision.restart_mode,
+        decision: decision.decision,
+        session_id: decision.session_id.clone(),
+        phase: phase.into(),
+        error,
+    }
+}
+
+fn restart_failure_phase(
+    decision: &RestartedAgent,
+    phase: &'static str,
+    error: &str,
+) -> &'static str {
+    if is_resume_integrity_failure(decision, phase, error) {
+        "resume"
+    } else {
+        phase
+    }
+}
+
+fn is_resume_integrity_failure(decision: &RestartedAgent, phase: &str, error: &str) -> bool {
+    if !matches!(decision.restart_mode, StartMode::Resumed) {
+        return false;
+    }
+    if phase == "resume" || phase == "readiness" {
+        return true;
+    }
+    error.contains("session_disappeared_after_spawn")
+        || error.contains("provider_resume_exited")
+        || error.contains("resume_not_ready")
+        || error.contains("resume_atomicity")
+        || error.contains("no live pane")
+        || error.contains("no-pane")
+}
+
+fn mark_agent_restart_failed(
+    state: &mut serde_json::Value,
+    decision: &RestartedAgent,
+    error: &str,
+) {
+    let Some(agent) = state
+        .get_mut("agents")
+        .and_then(serde_json::Value::as_object_mut)
+        .and_then(|agents| agents.get_mut(decision.agent_id.as_str()))
+        .and_then(serde_json::Value::as_object_mut)
+    else {
+        return;
+    };
+    agent.insert("status".to_string(), serde_json::json!("failed"));
+    agent.insert("restart_error".to_string(), serde_json::json!(error));
+    agent.insert(
+        "restart_failed_at".to_string(),
+        serde_json::json!(chrono::Utc::now().to_rfc3339()),
+    );
+    agent.remove("pane_id");
+    agent.remove("pane_pid");
+}
+
+fn write_restart_agent_failed_event(
+    workspace: &Path,
+    decision: &RestartedAgent,
+    phase: &str,
+    error: &str,
+) -> Result<(), LifecycleError> {
+    crate::event_log::EventLog::new(workspace)
+        .write(
+            "restart.agent_failed",
+            serde_json::json!({
+                "agent_id": decision.agent_id.as_str(),
+                "restart_mode": start_mode_wire(decision.restart_mode),
+                "decision": resume_decision_wire(decision.decision),
+                "session_id": decision.session_id.as_ref().map(|session| session.as_str()),
+                "phase": phase,
+                "error": error,
+                "action": format!(
+                    "inspect worker {} output, then restart that worker with `team-agent restart-agent {}` or rerun `team-agent restart --allow-fresh`",
+                    decision.agent_id,
+                    decision.agent_id
+                ),
+                "log": format!(
+                    ".team/logs/coordinator.log and .team/runtime/state.json agent={}",
+                    decision.agent_id
+                ),
+            }),
+        )
+        .map(|_| ())
+        .map_err(|e| LifecycleError::StatePersist(e.to_string()))
+}
+
+fn write_restart_completed_event(
+    workspace: &Path,
+    successful_agents: &[RestartedAgent],
+    failed_agents: &[RestartFailedAgent],
+    rc: &str,
+) -> Result<(), LifecycleError> {
+    crate::event_log::EventLog::new(workspace)
+        .write(
+            "restart.completed",
+            serde_json::json!({
+                "rc": rc,
+                "status": rc,
+                "successful_agents": successful_agents
+                    .iter()
+                    .map(|agent| agent.agent_id.as_str())
+                    .collect::<Vec<_>>(),
+                "failed_agents": failed_agents
+                    .iter()
+                    .map(|failure| serde_json::json!({
+                        "agent_id": failure.agent_id.as_str(),
+                        "phase": failure.phase,
+                        "error": failure.error,
+                    }))
+                    .collect::<Vec<_>>(),
+            }),
+        )
+        .map(|_| ())
+        .map_err(|e| LifecycleError::StatePersist(e.to_string()))
+}
+
+fn restart_failure_next_actions(failed_agents: &[RestartFailedAgent]) -> Vec<String> {
+    failed_agents
+        .iter()
+        .map(|failure| {
+            format!(
+                "inspect worker {} output, then restart that worker with `team-agent restart-agent {}` or rerun `team-agent restart --allow-fresh`",
+                failure.agent_id, failure.agent_id
+            )
+        })
+        .collect()
+}
+
+fn start_mode_wire(mode: StartMode) -> &'static str {
+    match mode {
+        StartMode::Resumed => "resumed",
+        StartMode::Fresh => "fresh",
+        StartMode::FreshAfterMissingRollout => "fresh_after_missing_rollout",
+        StartMode::Noop => "noop",
+    }
+}
+
+fn resume_decision_wire(decision: ResumeDecision) -> &'static str {
+    match decision {
+        ResumeDecision::Resume => "resume",
+        ResumeDecision::FreshStart => "fresh_start",
+        ResumeDecision::Refuse => "refuse",
+    }
+}
+
 fn write_restart_resume_decision_events(
     workspace: &Path,
     state: &serde_json::Value,
@@ -768,11 +1063,7 @@ fn write_restart_resume_decision_events(
             .filter(|s| !s.is_empty())
             .map(|s| s.to_string());
         let session_id = decision.session_id.as_ref().map(|s| s.as_str().to_string());
-        let decision_wire = match decision.decision {
-            ResumeDecision::Resume => "resume",
-            ResumeDecision::FreshStart => "fresh_start",
-            ResumeDecision::Refuse => "refuse",
-        };
+        let decision_wire = resume_decision_wire(decision.decision);
         write_restart_resume_decision_event(
             workspace,
             decision.agent_id.as_str(),
@@ -956,13 +1247,16 @@ fn rebuild_runtime_spec_from_roles(
     let agents_dir = team_dir.join("agents");
     let has_role_doc = std::fs::read_dir(&agents_dir)
         .map(|entries| {
-            entries.flatten().any(|e| {
-                e.path().extension().and_then(|x| x.to_str()) == Some("md")
-            })
+            entries
+                .flatten()
+                .any(|e| e.path().extension().and_then(|x| x.to_str()) == Some("md"))
         })
         .unwrap_or(false);
     if !has_role_doc {
-        missing.push(format!("{}/*.md (at least one role doc)", agents_dir.display()));
+        missing.push(format!(
+            "{}/*.md (at least one role doc)",
+            agents_dir.display()
+        ));
     }
     if !missing.is_empty() {
         // N38 三行式:error / action / log。旧 runtime spec 原地保留(不删不用)。

package/crates/team-agent/src/lifecycle/restart/selection.rs

@@ -1,5 +1,5 @@
-use super::*;
 use super::common::*;
+use super::*;
 
 /// bug-085 四象限 `start_mode` 决策(`start.py:179-188` + `_resume_rollout_missing` `start.py:66-69`),
 /// **从 start_agent 的整条 lock+spawn 路径里分离出的纯函数**(gate gap:porter 需要单元级 RED
@@ -19,7 +19,8 @@ pub fn decide_start_mode(
     match session_id {
         None => StartMode::Fresh,
         Some(_) => {
-            let missing_resume_backing = resumable_provider_requires_backing(provider) && !rollout_exists;
+            let missing_resume_backing = !provider_wire_supports_resume(provider)
+                || (resumable_provider_requires_backing(provider) && !rollout_exists);
             match (missing_resume_backing, allow_fresh) {
                 (true, true) => StartMode::FreshAfterMissingRollout,
                 (true, false) => StartMode::Noop,
@@ -188,8 +189,10 @@ pub(crate) fn classify_restart_plan_with_resume_validation(
         // (leader 从未发消息 → first_send_at=null → interacted=false)会被它静默 fresh 丢上下文。
         let provider = agent_provider(agent);
         let provider_wire = provider_wire(provider);
-        let resume_backing_exists = match (workspace, session_id.as_ref()) {
-            (Some(workspace), Some(session)) => resume_backing_exists_for_agent(
+        let provider_can_resume = provider_supports_resume(provider);
+        let resume_backing_exists = match (workspace, session_id.as_ref(), provider_can_resume) {
+            (_, Some(_), false) => false,
+            (Some(workspace), Some(session), true) => resume_backing_exists_for_agent(
                 workspace,
                 &agent_id,
                 agent,
@@ -197,14 +200,14 @@ pub(crate) fn classify_restart_plan_with_resume_validation(
                 session,
                 agent_rollout_path(agent).as_ref(),
             ),
-            (None, Some(_)) if resumable_provider_requires_backing(provider_wire) => {
+            (None, Some(_), true) if resumable_provider_requires_backing(provider_wire) => {
                 agent_rollout_path(agent)
                     .as_ref()
                     .is_some_and(|path| path.as_path().exists())
             }
             _ => true,
         };
-        let decision = if session_id.is_some() && resume_backing_exists {
+        let decision = if session_id.is_some() && provider_can_resume && resume_backing_exists {
             ResumeDecision::Resume
         } else if session_id.is_some() && allow_fresh {
             ResumeDecision::FreshStart