@team-agent/installer 0.3.39 → 0.4.0

package/crates/team-agent/src/lifecycle/restart/rebuild.rs

@@ -72,6 +72,9 @@ pub fn restart_with_transport_with_readiness_deadline(
                 .map(|agent_id| UnresumableWorker {
                     agent_id,
                     reason: "session_capture_incomplete".to_string(),
+                    refusal_reason: Some(
+                        crate::provider::session::ResumeRefusalReason::SessionCaptureIncomplete,
+                    ),
                     session_id: None,
                     first_send_at: None,
                 })
@@ -190,6 +193,7 @@ pub fn restart_with_transport_with_session_convergence_deadline(
         &plan.decisions,
         &forced_fresh_missing,
         forced_fresh_convergence.as_ref(),
+        &plan.unresumable,
     )?;
     if !plan.corrupt_entries.is_empty() {
         return Ok(RestartReport::RefusedInvalidFirstSendAt {
@@ -205,6 +209,29 @@ pub fn restart_with_transport_with_session_convergence_deadline(
             error: "restart requires resumable workers before live spawn; rerun with --allow-fresh to start fresh".to_string(),
         });
     }
+    // unit-3 (Stage 1) session-identity preflight: before any kill, reject
+    // the case where `state.session_name` actually holds a leader launcher
+    // session name (`team-agent-leader-*`). Proceeding would tear down the
+    // leader pane (E49 / 0.3.39 leader mis-kill). Nothing is created or
+    // killed — the caller gets a structured refusal that distinguishes this
+    // dirty-state from a normal resume/atomicity refusal.
+    match crate::lifecycle::restart::preflight::check_session_preflight(&state) {
+        crate::lifecycle::restart::preflight::SessionPreflight::Ok => {}
+        crate::lifecycle::restart::preflight::SessionPreflight::WorkerSessionIsLeaderSession {
+            session_name,
+            reason,
+        } => {
+            return Ok(RestartReport::RefusedDirtyTopology {
+                session_name: session_name.clone(),
+                reason: reason.clone(),
+                error: format!(
+                    "restart refused: state.session_name `{session_name}` is a leader \
+                     launcher session ({reason}); aborting before any tmux kill. Repair \
+                     state.session_name to the worker session and re-run."
+                ),
+            });
+        }
+    }
     let session_name = state_session_name(&state);
     if session_live_or_default(transport, &session_name, false) {
         // 0.3.28 Step 5 (warn-only): per architecture, restart should REFUSE
@@ -310,6 +337,10 @@ pub fn restart_with_transport_with_session_convergence_deadline(
             Some(&safety),
             layout_placement.as_ref(),
             None,
+            // Issue 2 (Round 3b gate review §6): thread the resolved
+            // selected.team_key so the worker MCP env carries the right
+            // owner_team_id even when top-level active_team_key is stale.
+            Some(selected.team_key.as_str()),
         ) {
             Ok(spawn) => spawn,
             Err(error) => {
@@ -396,6 +427,175 @@ pub fn restart_with_transport_with_session_convergence_deadline(
             attach_commands,
         });
     }
+    // RM-039-SESS-001 step 2 (architect verdict 2026-06-22): post-respawn
+    // resume backing validation.
+    //
+    // Pre-kill preflight already proves backing exists, but killing the
+    // provider process can cause the provider to rotate / clean up its
+    // transient backing file (the Claude case in the evidence: the
+    // sessions/<pid>.json file is the process-tracking record and goes away
+    // when the worker dies). Resumed agents preserve the OLD
+    // capture tuple (mark_agent_respawned only clears it on Fresh/
+    // FreshAfterMissingRollout), so without this re-probe the runtime
+    // reports `restart.completed rc:"ok"` while L2 provider backing truth
+    // is missing — a false-green restart.
+    //
+    // Strategy:
+    //   1. For every Resumed worker in `successful_agents`, re-run the
+    //      same `resume_backing_probe_for_agent` used at preflight.
+    //   2. Emit `restart.resume_postflight` carrying agent_id, session_id,
+    //      exists, checked_paths.
+    //   3. If exists=false: clear stale capture fields (rollout_path,
+    //      captured_at, captured_via, attribution_confidence) but
+    //      preserve session_id; set `_pending_session_id` to the same
+    //      session_id so convergence searches for the resumed session
+    //      explicitly. Then run one bounded convergence pass.
+    //   4. Re-probe. If still missing and !allow_fresh, demote the agent
+    //      from successful to failed with phase `resume_postflight` and
+    //      `session_backing_store_missing_after_restart`. The outer
+    //      branches below will then return Failed/Partial instead of OK.
+    {
+        let mut postflight_failed: Vec<crate::lifecycle::types::RestartFailedAgent> = Vec::new();
+        let mut survivors: Vec<crate::lifecycle::types::RestartedAgent> =
+            Vec::with_capacity(successful_agents.len());
+        let convergence_deadline = session_convergence_deadline(session_converge_deadline_ms);
+        let convergence_poll = session_convergence_poll_interval();
+        for decision in successful_agents.drain(..) {
+            if !matches!(decision.restart_mode, crate::lifecycle::types::StartMode::Resumed) {
+                survivors.push(decision);
+                continue;
+            }
+            let Some(agent) = state
+                .get("agents")
+                .and_then(|v| v.get(decision.agent_id.as_str()))
+                .cloned()
+            else {
+                survivors.push(decision);
+                continue;
+            };
+            let provider = agent_provider(&agent);
+            let session_id = match decision.session_id.as_ref() {
+                Some(sid) => sid.clone(),
+                None => {
+                    // No session id on a Resumed decision shouldn't happen,
+                    // but if it does there is nothing to revalidate.
+                    survivors.push(decision);
+                    continue;
+                }
+            };
+            let probe = resume_backing_probe_for_agent(
+                &selected.run_workspace,
+                &decision.agent_id,
+                &agent,
+                provider,
+                &session_id,
+                agent_rollout_path(&agent).as_ref(),
+            );
+            write_restart_resume_postflight_event(
+                &selected.run_workspace,
+                &decision.agent_id,
+                Some(&session_id),
+                probe.exists,
+                &probe.checked_paths,
+                /* recaptured = */ false,
+            )?;
+            if probe.exists {
+                survivors.push(decision);
+                continue;
+            }
+            // Backing went missing between preflight and post-spawn.
+            // Clear stale capture fields, keep session_id, mark
+            // _pending_session_id, run a bounded convergence pass.
+            if let Some(agents) = state
+                .pointer_mut("/agents")
+                .and_then(serde_json::Value::as_object_mut)
+            {
+                if let Some(agent_obj) = agents
+                    .get_mut(decision.agent_id.as_str())
+                    .and_then(serde_json::Value::as_object_mut)
+                {
+                    agent_obj.remove("rollout_path");
+                    agent_obj.remove("captured_at");
+                    agent_obj.remove("captured_via");
+                    agent_obj.remove("attribution_confidence");
+                    agent_obj.insert(
+                        "_pending_session_id".to_string(),
+                        serde_json::json!(session_id.as_str()),
+                    );
+                }
+            }
+            // Persist state before convergence so the helper sees the
+            // cleared tuple.
+            save_restart_state(&selected.run_workspace, &mut state, &selected.team_key)?;
+            let _converge = converge_missing_provider_sessions(
+                &mut state,
+                convergence_deadline,
+                convergence_poll,
+                &selected.run_workspace,
+                allow_fresh,
+            )?;
+            save_restart_state(&selected.run_workspace, &mut state, &selected.team_key)?;
+            // Re-probe with the freshest agent state.
+            let agent_after = state
+                .get("agents")
+                .and_then(|v| v.get(decision.agent_id.as_str()))
+                .cloned()
+                .unwrap_or(serde_json::Value::Null);
+            let probe_after = resume_backing_probe_for_agent(
+                &selected.run_workspace,
+                &decision.agent_id,
+                &agent_after,
+                provider,
+                &session_id,
+                agent_rollout_path(&agent_after).as_ref(),
+            );
+            write_restart_resume_postflight_event(
+                &selected.run_workspace,
+                &decision.agent_id,
+                Some(&session_id),
+                probe_after.exists,
+                &probe_after.checked_paths,
+                /* recaptured = */ true,
+            )?;
+            if probe_after.exists {
+                survivors.push(decision);
+                continue;
+            }
+            // Still missing. Demote the agent.
+            let phase = "resume_postflight";
+            let error = "session_backing_store_missing_after_restart".to_string();
+            mark_agent_restart_failed(&mut state, &decision, &error);
+            let _ = write_restart_agent_failed_event(
+                &selected.run_workspace,
+                &decision,
+                phase,
+                &error,
+            );
+            postflight_failed.push(restart_failed_agent(&decision, phase, error));
+        }
+        successful_agents = survivors;
+        failed_agents.extend(postflight_failed);
+        if !failed_agents.is_empty() {
+            save_restart_state(&selected.run_workspace, &mut state, &selected.team_key)?;
+        }
+    }
+    if successful_agents.is_empty() && !failed_agents.is_empty() {
+        // Postflight demoted every survivor — emit Failed before coordinator start.
+        let attach_commands = Vec::new();
+        let next_actions = restart_failure_next_actions(&failed_agents);
+        write_restart_completed_event(
+            &selected.run_workspace,
+            &successful_agents,
+            &failed_agents,
+            "fail",
+        )?;
+        return Ok(RestartReport::Failed {
+            session_name,
+            failed_agents,
+            next_actions,
+            attach_commands,
+        });
+    }
     let coordinator_started = start_coordinator_for_workspace(&selected.run_workspace)?;
     wait_restart_readiness_or_timeout(
         &selected.run_workspace,
@@ -939,6 +1139,18 @@ fn mark_agent_respawned(
         "spawn_cwd".to_string(),
         serde_json::json!(spawn.spawn_cwd.to_string_lossy().to_string()),
     );
+    // Issue 2 (Round 3b gate review §6): persist the resolved owner_team_id
+    // back into the agent row so future restarts read it directly from the
+    // agent row (cascade priority 2) instead of relying on top-level
+    // active_team_key (priority 3).
+    if let Some(ref team_id) = spawn.owner_team_id {
+        if !team_id.is_empty() {
+            agent.insert(
+                "owner_team_id".to_string(),
+                serde_json::json!(team_id),
+            );
+        }
+    }
     if matches!(
         restart_mode,
         StartMode::Fresh | StartMode::FreshAfterMissingRollout
@@ -1099,6 +1311,39 @@ fn write_restart_agent_failed_event(
         .map_err(|e| LifecycleError::StatePersist(e.to_string()))
 }
 
+/// RM-039-SESS-001 step 2 (architect verdict 2026-06-22): emit
+/// `restart.resume_postflight` for each Resumed worker we re-probed after
+/// spawn. `recaptured=true` means this event is the second probe after
+/// one bounded `converge_missing_provider_sessions` pass; `false` means
+/// it's the first probe right after spawn. `exists` reflects whether the
+/// provider backing file actually exists on disk; `checked_paths` lists
+/// every path the runtime probed.
+fn write_restart_resume_postflight_event(
+    workspace: &Path,
+    agent_id: &AgentId,
+    session_id: Option<&SessionId>,
+    exists: bool,
+    checked_paths: &[std::path::PathBuf],
+    recaptured: bool,
+) -> Result<(), LifecycleError> {
+    crate::event_log::EventLog::new(workspace)
+        .write(
+            "restart.resume_postflight",
+            serde_json::json!({
+                "agent_id": agent_id.as_str(),
+                "session_id": session_id.map(SessionId::as_str),
+                "exists": exists,
+                "checked_paths": checked_paths
+                    .iter()
+                    .map(|p| p.to_string_lossy().into_owned())
+                    .collect::<Vec<_>>(),
+                "recaptured": recaptured,
+            }),
+        )
+        .map(|_| ())
+        .map_err(|e| LifecycleError::StatePersist(e.to_string()))
+}
+
 fn write_restart_completed_event(
     workspace: &Path,
     successful_agents: &[RestartedAgent],
@@ -1165,7 +1410,13 @@ fn write_restart_resume_decision_events(
     decisions: &[RestartedAgent],
     forced_fresh_missing: &std::collections::BTreeSet<String>,
     forced_fresh_convergence: Option<&crate::session_capture::SessionConvergence>,
+    unresumable: &[crate::lifecycle::types::UnresumableWorker],
 ) -> Result<(), LifecycleError> {
+    // Layer 2 (leader directive 2026-06-22): every restart.resume_decision
+    // event for a Refuse decision must carry the structured refusal_reason
+    // wire string so operators can see WHY without crawling state.json.
+    let refusal_index: std::collections::BTreeMap<&str, &crate::lifecycle::types::UnresumableWorker> =
+        unresumable.iter().map(|u| (u.agent_id.as_str(), u)).collect();
     for decision in decisions {
         let agent = state
             .get("agents")
@@ -1187,6 +1438,7 @@ fn write_restart_resume_decision_events(
             decision_wire,
             forced_fresh_missing.contains(decision.agent_id.as_str()),
             forced_fresh_convergence,
+            refusal_index.get(decision.agent_id.as_str()).copied(),
         )?;
     }
     Ok(())
@@ -1201,6 +1453,7 @@ fn write_restart_resume_decision_event(
     decision: &str,
     forced_fresh: bool,
     forced_fresh_convergence: Option<&crate::session_capture::SessionConvergence>,
+    unresumable: Option<&crate::lifecycle::types::UnresumableWorker>,
 ) -> Result<(), LifecycleError> {
     use std::io::Write as _;
 
@@ -1219,6 +1472,60 @@ fn write_restart_resume_decision_event(
         "first_send_at": first_send_at,
         "session_id": session_id,
     });
+    // Layer 2 (leader directive 2026-06-22): when this is a Refuse decision
+    // and we have a structured ResumeRefusalReason, emit the wire string +
+    // optional recovery hint so the event-log audit is actionable. Falls
+    // back to the legacy free-form `reason` string when no structured
+    // refusal_reason is set.
+    if decision == "refuse" {
+        if let Some(u) = unresumable {
+            if let Some(obj) = event.as_object_mut() {
+                obj.insert("refusal_reason".to_string(), serde_json::json!(u.reason));
+                if let Some(structured) = &u.refusal_reason {
+                    obj.insert(
+                        "refusal_reason_wire".to_string(),
+                        serde_json::json!(structured.wire()),
+                    );
+                    if let crate::provider::session::ResumeRefusalReason::SessionBackingStoreMissing {
+                        checked_paths,
+                        recovery_hint,
+                    } = structured
+                    {
+                        if !checked_paths.is_empty() {
+                            obj.insert(
+                                "checked_paths".to_string(),
+                                serde_json::json!(checked_paths
+                                    .iter()
+                                    .map(|p| p.to_string_lossy().into_owned())
+                                    .collect::<Vec<_>>()),
+                            );
+                        }
+                        if let Some(hint) = recovery_hint {
+                            let mut h = serde_json::Map::new();
+                            h.insert("provider".to_string(), serde_json::json!(hint.provider));
+                            if let Some(name) = &hint.provider_session_name_hint {
+                                h.insert("name".to_string(), serde_json::json!(name));
+                            }
+                            if let Some(cwd) = &hint.spawn_cwd {
+                                h.insert(
+                                    "spawn_cwd".to_string(),
+                                    serde_json::json!(cwd.to_string_lossy()),
+                                );
+                            }
+                            h.insert(
+                                "picker_hint".to_string(),
+                                serde_json::json!(hint.picker_hint()),
+                            );
+                            obj.insert(
+                                "recovery_hint".to_string(),
+                                serde_json::Value::Object(h),
+                            );
+                        }
+                    }
+                }
+            }
+        }
+    }
     if forced_fresh {
         if let Some(event) = event.as_object_mut() {
             event.insert("forced_fresh".to_string(), serde_json::json!(true));
@@ -1595,6 +1902,7 @@ mod tests {
             profile_launch: crate::provider::ProviderProfileLaunch::default(),
             layout_placement: None,
             spawn_cwd: std::path::PathBuf::from("/tmp/team-epoch"),
+            owner_team_id: None,
         };
         let before = chrono::Utc::now();
 

package/crates/team-agent/src/lifecycle/restart/selection.rs

@@ -190,23 +190,37 @@ pub(crate) fn classify_restart_plan_with_resume_validation(
         let provider = agent_provider(agent);
         let provider_wire = provider_wire(provider);
         let provider_can_resume = provider_supports_resume(provider);
-        let resume_backing_exists = match (workspace, session_id.as_ref(), provider_can_resume) {
-            (_, Some(_), false) => false,
-            (Some(workspace), Some(session), true) => resume_backing_exists_for_agent(
-                workspace,
-                &agent_id,
-                agent,
-                provider,
-                session,
-                agent_rollout_path(agent).as_ref(),
-            ),
-            (None, Some(_), true) if resumable_provider_requires_backing(provider_wire) => {
-                agent_rollout_path(agent)
-                    .as_ref()
-                    .is_some_and(|path| path.as_path().exists())
-            }
-            _ => true,
-        };
+        // Layer 2 self-healing (leader follow-up 2026-06-22): use the
+        // structured probe so we can carry the list of paths the runtime
+        // actually checked into the refusal — operators need to see WHICH
+        // places we looked, not just "missing".
+        let (resume_backing_exists, backing_checked_paths) =
+            match (workspace, session_id.as_ref(), provider_can_resume) {
+                (_, Some(_), false) => (false, Vec::new()),
+                (Some(workspace), Some(session), true) => {
+                    let probe = resume_backing_probe_for_agent(
+                        workspace,
+                        &agent_id,
+                        agent,
+                        provider,
+                        session,
+                        agent_rollout_path(agent).as_ref(),
+                    );
+                    (probe.exists, probe.checked_paths)
+                }
+                (None, Some(_), true) if resumable_provider_requires_backing(provider_wire) => {
+                    let path_opt = agent_rollout_path(agent);
+                    let exists = path_opt
+                        .as_ref()
+                        .is_some_and(|path| path.as_path().exists());
+                    let checked = path_opt
+                        .as_ref()
+                        .map(|p| vec![p.as_path().to_path_buf()])
+                        .unwrap_or_default();
+                    (exists, checked)
+                }
+                _ => (true, Vec::new()),
+            };
         let decision = if session_id.is_some() && provider_can_resume && resume_backing_exists {
             ResumeDecision::Resume
         } else if session_id.is_some() && allow_fresh {
@@ -219,13 +233,64 @@ pub(crate) fn classify_restart_plan_with_resume_validation(
             ResumeDecision::Refuse
         };
         if matches!(decision, ResumeDecision::Refuse) {
+            // unit-5: surface structured ResumeRefusalReason alongside the
+            // legacy free-form string. The string wire is preserved exactly
+            // (round-tripped through ResumeRefusalReason::wire) so the
+            // CLI/JSON contract does not change.
+            let (reason_str, structured) = if session_id.is_some() {
+                if !provider_can_resume {
+                    (
+                        "session_unresumable".to_string(),
+                        crate::provider::session::ResumeRefusalReason::ProviderResumeUnsupported {
+                            provider: provider_wire.to_string(),
+                        },
+                    )
+                } else if !resume_backing_exists {
+                    // Today the legacy wire collapses backing-missing under
+                    // the catch-all `session_unresumable` — keep that wire,
+                    // but record the structured reason so the new shape is
+                    // available to callers that want it.
+                    //
+                    // Layer 2 self-healing (architect probe 2026-06-22): attach
+                    // a recovery hint pointing at the agent_id (used as
+                    // launch-time `--name`) and spawn_cwd. Operator-facing
+                    // diagnostic only — no auto-resume off the hint.
+                    let recovery_hint = Some(
+                        crate::provider::session::RecoveryHint {
+                            provider_session_name_hint: Some(agent_id.as_str().to_string()),
+                            spawn_cwd: agent
+                                .get("spawn_cwd")
+                                .and_then(|v| v.as_str())
+                                .filter(|s| !s.is_empty())
+                                .map(std::path::PathBuf::from),
+                            provider: provider_wire.to_string(),
+                        },
+                    );
+                    (
+                        "session_unresumable".to_string(),
+                        crate::provider::session::ResumeRefusalReason::SessionBackingStoreMissing {
+                            checked_paths: backing_checked_paths.clone(),
+                            recovery_hint,
+                        },
+                    )
+                } else {
+                    (
+                        "session_unresumable".to_string(),
+                        crate::provider::session::ResumeRefusalReason::Other {
+                            legacy_reason: "session_unresumable".to_string(),
+                        },
+                    )
+                }
+            } else {
+                (
+                    "no_persisted_session_id".to_string(),
+                    crate::provider::session::ResumeRefusalReason::NoSessionId,
+                )
+            };
             unresumable.push(UnresumableWorker {
                 agent_id: agent_id.clone(),
-                reason: if session_id.is_some() {
-                    "session_unresumable".to_string()
-                } else {
-                    "no_persisted_session_id".to_string()
-                },
+                reason: reason_str,
+                refusal_reason: Some(structured),
                 session_id: session_id.clone(),
                 first_send_at: first_send_at_raw.as_str().map(|s| s.to_string()),
             });

package/crates/team-agent/src/lifecycle/restart.rs

@@ -26,6 +26,7 @@ struct LifecyclePathRefs<'a> {
 mod agent;
 mod common;
 mod orchestrator;
+pub mod preflight;
 mod rebuild;
 mod remove;
 mod selection;
@@ -46,6 +47,11 @@ pub use rebuild::{
 pub(crate) use rebuild::restart_with_transport_with_session_convergence_deadline;
 pub use remove::{remove_agent, remove_agent_with_transport};
 pub use selection::{classify_first_send_at, classify_restart_plan, decide_start_mode, python_type_name};
+// Layer 2 (leader follow-up 2026-06-22): test-visible workspace-aware
+// classification so lifecycle/tests/restart.rs can exercise the
+// SessionBackingStoreMissing + checked_paths + RecoveryHint path
+// end-to-end without spinning up a full restart.
+pub(crate) use selection::classify_restart_plan_with_resume_validation;
 pub(crate) use team_state::write_team_state;
 
 pub(crate) fn lifecycle_run_workspace(workspace: &Path) -> Result<std::path::PathBuf, LifecycleError> {

package/crates/team-agent/src/lifecycle/tests/launch_spawn.rs

@@ -2544,12 +2544,17 @@ fn quick_start_state_seeds_spec_path_workspace_leader_display_backend() {
             "tasks",
             "display_backend",
             "is_external_leader",
+            // 0.4.0 refactor: `team_key` added as top-level topology marker
+            // alongside active_team_key (refactor-modular-architecture). The
+            // owner / leader_receiver / owner_epoch invariants below still
+            // hold — those live only under teams[<active_team_key>].
+            "team_key",
             "active_team_key",
             "teams",
         ],
         "state.json top-level key order must match golden launch/core.py:62-71 \
          plus R1 topology marker and Bug 1/2 team-in-team suffix \
-         (is_external_leader, active_team_key, teams). \
+         (is_external_leader, team_key, active_team_key, teams). \
          Bug 2 owner team-scope (N1/N12/N18/N29) deliberately keeps owner / \
          leader_receiver / owner_epoch OFF the top level — they live ONLY under \
          teams[<active_team_key>] so per-team isolation has a single source of \