@team-agent/installer 0.5.50 → 0.5.51

package/crates/team-agent/src/cli/tests/run_delegation.rs

@@ -320,6 +320,7 @@ fn cli_add_agent_duplicate_id_surfaces_real_error() {
         workspace: team,
         team: None,
         role_file: dup_role.to_string_lossy().to_string(),
+        force: false,
         no_display: false,
         json: true,
     };

package/crates/team-agent/src/cli/types.rs

@@ -497,6 +497,7 @@ pub struct AddAgentArgs {
     pub workspace: PathBuf,
     pub team: Option<String>,
     pub role_file: String,
+    pub force: bool,
     pub no_display: bool,
     pub json: bool,
 }

package/crates/team-agent/src/coordinator/conpty_shim.rs

@@ -41,7 +41,7 @@ use std::time::Duration;
 
 use conpty_transport::{pipe_name_for, NamedPipeClient};
 
-use crate::state::persist::{runtime_state_path, save_runtime_state};
+use crate::state::persist::runtime_state_path;
 use crate::state::StateError;
 
 /// Maximum shim connect+hello attempts before giving up. Each attempt
@@ -397,15 +397,11 @@ fn finalize(
     client: NamedPipeClient,
     workspace: &Path,
 ) -> Result<ShimHandle, ShimError> {
-    let state_path = runtime_state_path(workspace);
-    let mut state = if state_path.exists() {
-        match std::fs::read_to_string(&state_path) {
-            Ok(text) => serde_json::from_str::<Value>(&text).unwrap_or_else(|_| json!({})),
-            Err(_) => json!({}),
-        }
-    } else {
-        json!({})
-    };
+    let mut state = crate::state::repository::StateRepository::new(workspace)
+        .load_workspace_if_exists_without_migrations()
+        .ok()
+        .flatten()
+        .unwrap_or_else(|| json!({}));
     // CR C-1: token NOT stored. Only pid/pipe_name/pipe_ready.
     let obj = state
         .as_object_mut()
@@ -431,7 +427,14 @@ fn finalize(
             "pipe_ready": true,
         }),
     );
-    save_runtime_state(workspace, &state).map_err(|e| ShimError::StatePersist { source: e })?;
+    crate::state::repository::StateRepository::new(workspace)
+        .save(
+            crate::state::repository::StateWriteIntent::CoordinatorConptyShim {
+                team_key: state.get("active_team_key").and_then(Value::as_str),
+            },
+            &state,
+        )
+        .map_err(|e| ShimError::StatePersist { source: e })?;
     // 0.5.x Windows portability Batch 7 F6: state merge now preserves
     // the `transport.shim` block through downstream saves (see
     // `state::persist::preserve_transport_shim`), so the Batch 6
@@ -450,12 +453,9 @@ fn finalize(
 /// Windows worker). Callers use `platform::process::terminate_pid`
 /// on the returned pid.
 pub fn recorded_shim_pid(workspace: &Path) -> Option<u32> {
-    let state_path = runtime_state_path(workspace);
-    if !state_path.exists() {
-        return None;
-    }
-    let text = std::fs::read_to_string(&state_path).ok()?;
-    let state: Value = serde_json::from_str(&text).ok()?;
+    let state = crate::state::repository::StateRepository::new(workspace)
+        .load_workspace_if_exists_without_migrations()
+        .ok()??;
     state
         .get("transport")?
         .get("shim")?
@@ -466,12 +466,9 @@ pub fn recorded_shim_pid(workspace: &Path) -> Option<u32> {
 
 /// Read `state.transport.shim.pipe_name` for reconnect routing.
 pub fn recorded_shim_pipe_name(workspace: &Path) -> Option<String> {
-    let state_path = runtime_state_path(workspace);
-    if !state_path.exists() {
-        return None;
-    }
-    let text = std::fs::read_to_string(&state_path).ok()?;
-    let state: Value = serde_json::from_str(&text).ok()?;
+    let state = crate::state::repository::StateRepository::new(workspace)
+        .load_workspace_if_exists_without_migrations()
+        .ok()??;
     state
         .get("transport")?
         .get("shim")?
@@ -647,19 +644,26 @@ pub fn mark_transport_unavailable(workspace: &Path, reason: &str) -> Result<(),
     // Clear the shim block from state so subsequent factory
     // `conpty_pipe_ready` checks return false and the operator sees
     // an honest "no live shim" via status --detail.
-    let state_path = runtime_state_path(workspace);
-    if !state_path.exists() {
-        return Ok(());
-    }
-    let text = std::fs::read_to_string(&state_path).map_err(StateError::from)?;
-    let mut state: Value = serde_json::from_str(&text).unwrap_or_else(|_| serde_json::json!({}));
+    let mut state = match crate::state::repository::StateRepository::new(workspace)
+        .load_workspace_if_exists_without_migrations()
+    {
+        Ok(Some(state)) => state,
+        Ok(None) => return Ok(()),
+        Err(StateError::Json(_)) => serde_json::json!({}),
+        Err(error) => return Err(error),
+    };
     if let Some(transport) = state.get_mut("transport").and_then(|t| t.as_object_mut()) {
         if let Some(shim) = transport.get_mut("shim").and_then(|s| s.as_object_mut()) {
             shim.insert("pipe_ready".to_string(), serde_json::json!(false));
             shim.insert("unavailable_reason".to_string(), serde_json::json!(reason));
         }
     }
-    save_runtime_state(workspace, &state)
+    crate::state::repository::StateRepository::new(workspace).save(
+        crate::state::repository::StateWriteIntent::CoordinatorConptyShim {
+            team_key: state.get("active_team_key").and_then(Value::as_str),
+        },
+        &state,
+    )
 }
 
 #[cfg(test)]

package/crates/team-agent/src/coordinator/steps/abnormal.rs

@@ -32,6 +32,10 @@ pub(crate) fn detect_abnormal_exits(
     let team = crate::state::projection::team_state_key(&snapshot);
     let session_name = snapshot.get("session_name").and_then(Value::as_str);
     for agent in abnormal_watch_agents(&snapshot) {
+        // Pane/process liveness is independent of transcript content. A
+        // frozen rollout is common after pane death, so probe before the
+        // metadata dedupe gate; only the expensive tail scan stays deduped.
+        let liveness = agent_process_liveness(&agent, session_name, targets, transport);
         let rollout_path = resolve_agent_rollout_path(workspace, &agent.rollout_path);
         let metadata = match std::fs::metadata(&rollout_path) {
             Ok(metadata) => metadata,
@@ -67,6 +71,7 @@ pub(crate) fn detect_abnormal_exits(
             abnormal_watch_stored_metadata(&snapshot, &agent.agent_id),
         ) {
             if stored == (size, mtime) {
+                refresh_abnormal_watch_liveness(state, &agent.agent_id, &liveness);
                 continue;
             }
         }
@@ -97,7 +102,6 @@ pub(crate) fn detect_abnormal_exits(
                 continue;
             }
         };
-        let liveness = agent_process_liveness(&agent, session_name, targets, transport);
         let fact = crate::provider::latest_explicit_error_fact(agent.provider, &text);
         let error_observation_key = fact
             .as_ref()
@@ -218,6 +222,7 @@ pub(crate) fn detect_abnormal_exits(
         } else {
             "refused"
         };
+        let provider_process_dead = provider_process_dead_fact(&liveness);
         event_log.write(
             "worker.abnormal_exit",
             serde_json::json!({
@@ -227,7 +232,7 @@ pub(crate) fn detect_abnormal_exits(
                 "path": agent.rollout_path_display.as_str(),
                 "dead_process": liveness.state == ProcessLiveness::Dead,
                 "process_dead": liveness.state == ProcessLiveness::Dead,
-                "provider_process_dead": liveness.state == ProcessLiveness::Dead,
+                "provider_process_dead": provider_process_dead,
                 "latest_error": true,
                 "latest_explicit_error": true,
                 "error_recency": error_recency.as_str(),
@@ -235,7 +240,7 @@ pub(crate) fn detect_abnormal_exits(
                 "dead_process_and_latest_error": liveness.state == ProcessLiveness::Dead,
                 "dead_process_and_latest_explicit_error": liveness.state == ProcessLiveness::Dead,
                 "process_dead_and_latest_explicit_error": liveness.state == ProcessLiveness::Dead,
-                "provider_process_dead_and_latest_explicit_error": liveness.state == ProcessLiveness::Dead,
+                "provider_process_dead_and_latest_explicit_error": provider_process_dead,
                 "signature": fact.signature.as_str(),
                 "turn_id": fact.turn_id.as_ref().map(|id| id.as_str()),
                 "apiErrorStatus": fact.api_error_status,
@@ -256,6 +261,50 @@ pub(crate) fn detect_abnormal_exits(
     Ok(())
 }
 
+fn provider_process_dead_fact(liveness: &ProcessCheck) -> bool {
+    liveness.state == ProcessLiveness::Dead
+        && !liveness.detail.starts_with("pane_dead:")
+        && !liveness.detail.starts_with("window_missing:")
+}
+
+fn worker_provider_exited_fact(liveness: &ProcessCheck) -> bool {
+    provider_process_dead_fact(liveness) && liveness.detail.starts_with("worker_provider_exited:")
+}
+
+fn refresh_abnormal_watch_liveness(state: &mut Value, agent_id: &str, liveness: &ProcessCheck) {
+    let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch")
+        .and_then(|watch| watch.get_mut(agent_id))
+        .and_then(Value::as_object_mut)
+    else {
+        return;
+    };
+    let dead_process = liveness.state == ProcessLiveness::Dead;
+    let provider_process_dead = provider_process_dead_fact(liveness);
+    let latest_explicit_error = watch
+        .get("latest_explicit_error")
+        .and_then(Value::as_bool)
+        .unwrap_or(false);
+    // Persist fact transitions, not the tick's observation time; otherwise an
+    // unchanged transcript makes every steady tick rewrite state.json.
+    let patch = serde_json::json!({
+        "last_liveness": process_liveness_wire(liveness.state),
+        "last_liveness_detail": liveness.detail.as_str(),
+        "dead_process": dead_process,
+        "process_dead": dead_process,
+        "provider_process_dead": provider_process_dead,
+        "worker_provider_exited": worker_provider_exited_fact(liveness),
+        "provider_process_dead_and_latest_explicit_error": provider_process_dead && latest_explicit_error,
+    });
+    if let Some(patch) = patch.as_object() {
+        if patch
+            .iter()
+            .any(|(key, value)| watch.get(key) != Some(value))
+        {
+            watch.extend(patch.clone());
+        }
+    }
+}
+
 #[derive(Debug, Clone)]
 struct AbnormalWatchAgent {
     agent_id: String,
@@ -816,8 +865,8 @@ fn abnormal_watch_payload(
     // marker (detail prefix `worker_provider_exited:`). status_port's
     // RuntimeFreshness collector reads this field to downgrade the
     // corresponding agent row.
-    let worker_provider_exited =
-        dead_process && liveness.detail.starts_with("worker_provider_exited:");
+    let provider_process_dead = provider_process_dead_fact(&liveness);
+    let worker_provider_exited = worker_provider_exited_fact(&liveness);
     serde_json::json!({
         "path": agent.rollout_path_display.as_str(),
         "provider": provider_wire(agent.provider),
@@ -829,7 +878,7 @@ fn abnormal_watch_payload(
         "last_liveness_detail": liveness.detail,
         "dead_process": dead_process,
         "process_dead": dead_process,
-        "provider_process_dead": dead_process,
+        "provider_process_dead": provider_process_dead,
         "worker_provider_exited": worker_provider_exited,
         "latest_error": latest_explicit_error,
         "latest_explicit_error": latest_explicit_error,
@@ -839,11 +888,10 @@ fn abnormal_watch_payload(
         "dead_process_and_latest_error": dead_process && latest_explicit_error,
         "dead_process_and_latest_explicit_error": dead_process && latest_explicit_error,
         "process_dead_and_latest_explicit_error": dead_process && latest_explicit_error,
-        "provider_process_dead_and_latest_explicit_error": dead_process && latest_explicit_error,
+        "provider_process_dead_and_latest_explicit_error": provider_process_dead && latest_explicit_error,
         "suppressed_reason": suppressed_reason,
         "notification": notify,
         "last_error": error,
-        "last_checked_at": chrono::Utc::now().to_rfc3339(),
     })
 }
 
@@ -1060,6 +1108,7 @@ fn write_abnormal_check(
     mtime_ns: Option<u64>,
 ) -> Result<(), TickError> {
     let dead_process = liveness.state == ProcessLiveness::Dead;
+    let provider_process_dead = provider_process_dead_fact(liveness);
     let latest_explicit_error = fact.is_some();
     event_log.write(
         "worker.abnormal_exit.check",
@@ -1073,7 +1122,7 @@ fn write_abnormal_check(
             "mtime_ns": mtime_ns,
             "dead_process": dead_process,
             "process_dead": dead_process,
-            "provider_process_dead": dead_process,
+            "provider_process_dead": provider_process_dead,
             "latest_error": latest_explicit_error,
             "latest_explicit_error": latest_explicit_error,
             "error_recency": error_recency.as_str(),
@@ -1081,7 +1130,7 @@ fn write_abnormal_check(
             "dead_process_and_latest_error": dead_process && latest_explicit_error,
             "dead_process_and_latest_explicit_error": dead_process && latest_explicit_error,
             "process_dead_and_latest_explicit_error": dead_process && latest_explicit_error,
-            "provider_process_dead_and_latest_explicit_error": dead_process && latest_explicit_error,
+            "provider_process_dead_and_latest_explicit_error": provider_process_dead && latest_explicit_error,
             "notification": matches!(decision, AbnormalExitDecision::Notify),
             "suppressed_reason": match decision {
                 AbnormalExitDecision::Suppress(reason) => Some(reason),
@@ -1107,6 +1156,7 @@ fn write_abnormal_suppressed(
     liveness: &ProcessCheck,
     reason: &str,
 ) -> Result<(), TickError> {
+    let provider_process_dead = provider_process_dead_fact(liveness);
     event_log.write(
         "abnormal_exit.single_signal_suppressed",
         serde_json::json!({
@@ -1118,7 +1168,7 @@ fn write_abnormal_suppressed(
             "notification": false,
             "dead_process": liveness.state == ProcessLiveness::Dead,
             "process_dead": liveness.state == ProcessLiveness::Dead,
-            "provider_process_dead": liveness.state == ProcessLiveness::Dead,
+            "provider_process_dead": provider_process_dead,
             "latest_error": false,
             "latest_explicit_error": false,
             "error_recency": ErrorRecency::None.as_str(),
@@ -1845,7 +1895,13 @@ pub(crate) fn attempt_due_recoveries(
         return;
     };
     if clear_stale_terminal_next_retry_at(&mut state) {
-        let _ = crate::state::persist::save_runtime_state(workspace, &state);
+        let _ = crate::state::repository::StateRepository::new(workspace).save(
+            crate::state::repository::StateWriteIntent::CoordinatorApiErrorRecovery {
+                team_key: state.get("active_team_key").and_then(Value::as_str),
+                agent_id: None,
+            },
+            &state,
+        );
     }
     let due_agents = collect_due_recovery_agents(&state);
     for agent_id in due_agents {
@@ -2098,7 +2154,13 @@ fn write_recovery_intent_result(workspace: &Path, agent_id: &str, update: Recove
             }
         }
     }
-    let _ = crate::state::persist::save_runtime_state(workspace, &state);
+    let _ = crate::state::repository::StateRepository::new(workspace).save(
+        crate::state::repository::StateWriteIntent::CoordinatorApiErrorRecovery {
+            team_key: state.get("active_team_key").and_then(Value::as_str),
+            agent_id: Some(agent_id),
+        },
+        &state,
+    );
 }
 
 #[cfg(test)]
@@ -2510,6 +2572,66 @@ mod tests {
         assert_eq!(suppressed["fresh_error"], serde_json::json!(false));
     }
 
+    #[test]
+    fn abnormal_unchanged_transcript_keeps_provider_death_out_of_pane_projection() {
+        let dir = temp_abnormal_dir("unchanged-provider-dead");
+        let rollout = dir.join("rollout-w1.jsonl");
+        std::fs::write(
+            &rollout,
+            "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"completed\"}}}\n",
+        )
+        .unwrap();
+        seed_abnormal_state(&dir, &rollout, "alive", 1);
+        let coordinator = abnormal_test_coordinator(&dir);
+        coordinator.tick().unwrap();
+
+        let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
+        state["agents"]["w1"]["process_liveness"] = serde_json::json!("dead");
+        crate::state::persist::save_runtime_state(&dir, &state).unwrap();
+        coordinator.tick().unwrap();
+
+        let state = crate::state::persist::load_runtime_state(&dir).unwrap();
+        let agent = &state["agents"]["w1"];
+        assert!(
+            agent.get("provider_process_dead").is_none(),
+            "provider death belongs to abnormal watch, not the pane-dead seat projection: {agent}"
+        );
+        assert!(
+            agent.get("stale_reason").is_none(),
+            "provider death must not be mislabeled pane_dead: {agent}"
+        );
+        let watch = &state["coordinator"]["abnormal_exit_watch"]["w1"];
+        assert_eq!(watch["provider_process_dead"], serde_json::json!(true));
+        assert_eq!(watch["worker_provider_exited"], serde_json::json!(false));
+        assert_eq!(
+            watch["last_liveness_detail"],
+            serde_json::json!("explicit:dead")
+        );
+    }
+
+    #[test]
+    fn abnormal_pane_absence_is_not_provider_process_death() {
+        let agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(1), None);
+        for detail in ["pane_dead:%1", "window_missing:w1"] {
+            let payload = abnormal_watch_payload(
+                &agent,
+                Some(1),
+                Some(2),
+                process_check(ProcessLiveness::Dead, detail.to_string()),
+                None,
+                ErrorRecency::None,
+                None,
+                None,
+            );
+            assert_eq!(
+                payload["provider_process_dead"],
+                serde_json::json!(false),
+                "pane absence and provider death are distinct facts: {payload}"
+            );
+            assert_eq!(payload["worker_provider_exited"], serde_json::json!(false));
+        }
+    }
+
     #[test]
     fn abnormal_recency_treats_cohort_change_as_stale() {
         let agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(2), None);

package/crates/team-agent/src/coordinator/tick.rs

@@ -752,6 +752,43 @@ impl Coordinator {
                 }
             };
             if !windows.iter().any(|known| known == &window) {
+                // Missing is proof only inside a non-empty same-session
+                // inventory (a live peer exists). An empty snapshot is an
+                // unavailable transport, not proof every worker died.
+                if windows.is_empty() {
+                    continue;
+                }
+                if let Some(agent_obj) = agent.as_object_mut() {
+                    agent_obj.insert("status".to_string(), serde_json::json!("stopped"));
+                    agent_obj.insert("worker_state".to_string(), serde_json::json!("DEAD"));
+                    agent_obj.insert("stale".to_string(), serde_json::json!(true));
+                    agent_obj.insert("stale_reason".to_string(), serde_json::json!("pane_dead"));
+                }
+                let conn = crate::db::schema::open_db(store.db_path()).map_err(|error| {
+                    TickError::MessageStore(crate::message_store::MessageStoreError::Db(error))
+                })?;
+                conn.execute(
+                    "insert into agent_health(owner_team_id, agent_id, status, updated_at) \
+                     values (?1, ?2, 'DEAD', ?3) \
+                     on conflict(owner_team_id, agent_id) do update set \
+                     status='DEAD', updated_at=excluded.updated_at",
+                    rusqlite::params![
+                        team_key.as_ref().map(|key| key.as_str()),
+                        agent_id,
+                        chrono::Utc::now().to_rfc3339()
+                    ],
+                )
+                .map_err(|error| {
+                    TickError::MessageStore(crate::message_store::MessageStoreError::Sqlite(error))
+                })?;
+                let _ = event_log.write(
+                    "coordinator.agent_pane_dead",
+                    serde_json::json!({
+                        "agent_id": agent_id,
+                        "target": format!("{target:?}"),
+                        "stale_reason": "pane_dead",
+                    }),
+                );
                 continue;
             }
             // Warm-idle suppression still gates pane fallback ONLY. When

package/crates/team-agent/src/db/agent_health_capture.rs

@@ -13,13 +13,14 @@ use std::path::Path;
 
 use crate::model::ids::AgentId;
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct CapturedHealth {
     pub owner_team_id: Option<String>,
     pub status: Option<String>,
     pub last_output_at: Option<String>,
     pub context_usage_pct: Option<i64>,
     pub current_task_id: Option<String>,
+    pub updated_at: String,
 }
 
 /// golden agents.py:185 `copy.deepcopy(store.agent_health().get(agent_id))` — read the row BEFORE
@@ -27,6 +28,7 @@ pub struct CapturedHealth {
 /// absent.
 pub fn select_agent_health(
     workspace: &Path,
+    owner_team_id: &str,
     agent_id: &AgentId,
 ) -> Result<Option<CapturedHealth>, crate::db::DbError> {
     let store = crate::message_store::MessageStore::open(workspace)
@@ -34,9 +36,9 @@ pub fn select_agent_health(
     let conn = crate::db::schema::open_db(store.db_path())?;
     let row = conn
         .query_row(
-            "select owner_team_id, status, last_output_at, context_usage_pct, current_task_id \
-             from agent_health where agent_id = ?1",
-            [agent_id.as_str()],
+            "select owner_team_id, status, last_output_at, context_usage_pct, current_task_id, updated_at \
+             from agent_health where owner_team_id = ?1 and agent_id = ?2",
+            rusqlite::params![owner_team_id, agent_id.as_str()],
             |r| {
                 Ok(CapturedHealth {
                     owner_team_id: r.get::<_, Option<String>>(0)?,
@@ -44,6 +46,7 @@ pub fn select_agent_health(
                     last_output_at: r.get::<_, Option<String>>(2)?,
                     context_usage_pct: r.get::<_, Option<i64>>(3)?,
                     current_task_id: r.get::<_, Option<String>>(4)?,
+                    updated_at: r.get::<_, String>(5)?,
                 })
             },
         )
@@ -55,42 +58,44 @@ pub fn select_agent_health(
 /// or delete the row when there was nothing to restore.
 pub fn restore_agent_health(
     workspace: &Path,
+    owner_team_id: &str,
     agent_id: &AgentId,
     row: &Option<CapturedHealth>,
 ) -> Result<(), crate::db::DbError> {
     let Some(row) = row else {
-        return delete_agent_health(workspace, agent_id);
+        return delete_agent_health(workspace, owner_team_id, agent_id);
     };
     let store = crate::message_store::MessageStore::open(workspace)
         .map_err(|e| crate::db::DbError::Schema(e.to_string()))?;
     let conn = crate::db::schema::open_db(store.db_path())?;
     let status = row.status.clone().unwrap_or_else(|| "IDLE".to_string());
-    let now = chrono::Utc::now()
-        .format("%Y-%m-%dT%H:%M:%S%.6f+00:00")
-        .to_string();
     conn.execute(
         "insert into agent_health (owner_team_id, agent_id, status, last_output_at, context_usage_pct, current_task_id, updated_at) \
          values (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
         rusqlite::params![
-            row.owner_team_id,
+            owner_team_id,
             agent_id.as_str(),
             status,
             row.last_output_at,
             row.context_usage_pct,
             row.current_task_id,
-            now,
+            row.updated_at,
         ],
     )?;
     Ok(())
 }
 
-fn delete_agent_health(workspace: &Path, agent_id: &AgentId) -> Result<(), crate::db::DbError> {
+fn delete_agent_health(
+    workspace: &Path,
+    owner_team_id: &str,
+    agent_id: &AgentId,
+) -> Result<(), crate::db::DbError> {
     let store = crate::message_store::MessageStore::open(workspace)
         .map_err(|e| crate::db::DbError::Schema(e.to_string()))?;
     let conn = crate::db::schema::open_db(store.db_path())?;
     conn.execute(
-        "delete from agent_health where agent_id = ?1",
-        [agent_id.as_str()],
+        "delete from agent_health where owner_team_id = ?1 and agent_id = ?2",
+        rusqlite::params![owner_team_id, agent_id.as_str()],
     )?;
     Ok(())
 }