@team-agent/installer 0.5.42 → 0.5.44

package/crates/team-agent/src/coordinator/mod.rs

@@ -81,13 +81,13 @@ pub mod types;
 // ── Re-export the full module-root surface (RE-EXPORT INVARIANT) ──────────────
 // 这些 `pub use`/`pub(crate) use` 把每个曾在 module 根可见的 item 重新 surface 到
 // `crate::coordinator::X`,使外部 crate 路径 + 测试 `use super::*` 解析保持不变。
-pub use types::*;
-pub use tick::*;
 pub use backoff::*;
-pub use orphan::*;
 pub use health::*;
+pub use orphan::*;
 pub use runtime_detectors::*;
 pub use runtime_observation::*;
+pub use tick::*;
+pub use types::*;
 
 #[cfg(test)]
 mod tests;

package/crates/team-agent/src/coordinator/orphan.rs

@@ -82,7 +82,10 @@ pub fn process_abnormal_records(
             .unwrap_or("unknown")
             .to_string();
         let signature_lower = signature.to_lowercase();
-        let turn_id = record.get("turn_id").and_then(Value::as_str).map(TurnId::new);
+        let turn_id = record
+            .get("turn_id")
+            .and_then(Value::as_str)
+            .map(TurnId::new);
         let bucket = match &turn_id {
             Some(id) => id.as_str().to_string(),
             None => stable_fingerprint(record),
@@ -95,13 +98,14 @@ pub fn process_abnormal_records(
         if lists.whitelist.iter().any(|needle| raw.contains(needle)) {
             continue;
         }
-        let decision = if lists.blacklist.iter().any(|needle| {
-            raw.contains(needle) || signature_lower.contains(&needle.to_lowercase())
-        }) {
-            AbnormalDecision::NotifyBlacklist
-        } else {
-            AbnormalDecision::NotifyDefault
-        };
+        let decision =
+            if lists.blacklist.iter().any(|needle| {
+                raw.contains(needle) || signature_lower.contains(&needle.to_lowercase())
+            }) {
+                AbnormalDecision::NotifyBlacklist
+            } else {
+                AbnormalDecision::NotifyDefault
+            };
         let kind = record.get("kind").and_then(Value::as_str);
         notifications.push(AbnormalNotification {
             signature,
@@ -141,7 +145,13 @@ pub fn detect_whole_team_gone(
         return whole_team_report(true, WholeTeamGoneClass::CleanShutdown, false, false, false);
     }
     if snapshot.restart_in_progress {
-        return whole_team_report(true, WholeTeamGoneClass::RestartInProgress, false, false, false);
+        return whole_team_report(
+            true,
+            WholeTeamGoneClass::RestartInProgress,
+            false,
+            false,
+            false,
+        );
     }
     let marker_written = marker_store.set_marker(
         "whole_team_gone",
@@ -158,7 +168,9 @@ pub fn detect_whole_team_gone(
 
 fn stable_fingerprint(value: &Value) -> String {
     let mut hasher = std::collections::hash_map::DefaultHasher::new();
-    serde_json::to_string(value).unwrap_or_else(|_| value.to_string()).hash(&mut hasher);
+    serde_json::to_string(value)
+        .unwrap_or_else(|_| value.to_string())
+        .hash(&mut hasher);
     format!("{:016x}", hasher.finish())
 }
 

package/crates/team-agent/src/coordinator/steps/abnormal.rs

@@ -178,11 +178,8 @@ pub(crate) fn detect_abnormal_exits(
         // intent (retryable only), then compose the policy-aware tail. This
         // is INTENT WRITE only — actual lifecycle work happens post-save
         // via `attempt_due_recoveries` (§7.3, R6 guard).
-        let manual_command = recovery_manual_command(
-            agent.agent_id.as_str(),
-            workspace,
-            team.as_str(),
-        );
+        let manual_command =
+            recovery_manual_command(agent.agent_id.as_str(), workspace, team.as_str());
         let error_observation_key_string = error_observation_key.clone().unwrap_or_default();
         let (recovery_class, recovery_schedule, recovery_cohort_key, recovery_backpressure_until) =
             process_api_error_recovery_intent(
@@ -202,14 +199,8 @@ pub(crate) fn detect_abnormal_exits(
             &fact,
             &manual_command,
         );
-        let content = format_abnormal_exit_message(
-            &team,
-            &agent,
-            &fact,
-            &liveness,
-            size,
-            &recovery_tail,
-        );
+        let content =
+            format_abnormal_exit_message(&team, &agent, &fact, &liveness, size, &recovery_tail);
         let outcome = crate::messaging::send_to_leader_receiver(
             workspace,
             state,
@@ -722,8 +713,8 @@ fn command_process_check_with_marker(
         return check;
     }
     process_check(
-        ProcessLiveness::Dead,
-        format!("provider_not_foreground:{command}"),
+        ProcessLiveness::Unverifiable,
+        format!("provider_evidence_unverifiable:{command}"),
     )
 }
 
@@ -742,8 +733,8 @@ fn pane_command_process_check_with_marker(
         return check;
     }
     process_check(
-        ProcessLiveness::Dead,
-        format!("provider_not_foreground:{command}"),
+        ProcessLiveness::Unverifiable,
+        format!("provider_evidence_unverifiable:{command}"),
     )
 }
 
@@ -1471,7 +1462,10 @@ fn record_backpressure_event(
     if reset_window {
         obj.insert("window_started_at".to_string(), serde_json::json!(now_str));
         obj.insert("count".to_string(), serde_json::json!(0u64));
-        obj.insert("agents".to_string(), serde_json::json!(Vec::<String>::new()));
+        obj.insert(
+            "agents".to_string(),
+            serde_json::json!(Vec::<String>::new()),
+        );
     }
     obj.insert("last_seen_at".to_string(), serde_json::json!(now_str));
     let count = obj
@@ -1504,7 +1498,10 @@ fn record_backpressure_event(
             .map(|dt| dt.with_timezone(&chrono::Utc));
     } else if (count as usize) >= BACKPRESSURE_THRESHOLD {
         let until = now + cooldown;
-        obj.insert("cooldown_until".to_string(), serde_json::json!(until.to_rfc3339()));
+        obj.insert(
+            "cooldown_until".to_string(),
+            serde_json::json!(until.to_rfc3339()),
+        );
         obj.insert("status".to_string(), serde_json::json!("active"));
         cooldown_until = Some(until);
         just_activated = true;
@@ -1721,9 +1718,9 @@ fn process_api_error_recovery_intent(
     // cohort already holds a scheduled intent, defer the new arrival to
     // the cohort cooldown (or a synthetic short cooldown if none yet).
     if canary_active {
-        let synthetic = bp_decision.cooldown_until.unwrap_or_else(|| {
-            now + chrono::Duration::seconds(BACKPRESSURE_COOLDOWN_SECS)
-        });
+        let synthetic = bp_decision
+            .cooldown_until
+            .unwrap_or_else(|| now + chrono::Duration::seconds(BACKPRESSURE_COOLDOWN_SECS));
         bp_decision.cooldown_until = Some(synthetic);
     }
     if bp_decision.just_activated {
@@ -1816,7 +1813,12 @@ fn process_api_error_recovery_intent(
             }),
         )?;
     }
-    Ok((class, schedule, bp_decision.cohort_key, bp_decision.cooldown_until))
+    Ok((
+        class,
+        schedule,
+        bp_decision.cohort_key,
+        bp_decision.cooldown_until,
+    ))
 }
 
 // ─────────────────────────────────────────────────────────────────────────
@@ -1837,16 +1839,22 @@ pub(crate) fn attempt_due_recoveries(
     event_log: &EventLog,
     transport: &dyn crate::transport::Transport,
 ) {
-    let Ok(state) = crate::state::persist::load_runtime_state(workspace) else {
+    // 0.5.43 debt-sweep (§5 D-j): single fresh post-save load. Do NOT
+    // re-use the tick's pre-save Value — that would reopen the 0.5.36
+    // R6 lifecycle-save-clobbered-by-tick window (highest risk item in
+    // this slice per locate). We own our load here, mutate in memory
+    // (`clear_stale_terminal_next_retry_at` is now a pure
+    // `(&mut Value) -> bool` scrub), persist through the existing
+    // allowlisted root save when the scrub actually changed a row, and
+    // then collect due agents from the SAME post-scrub Value so
+    // terminal intents newly stripped of `next_retry_at` cannot double-
+    // fire below.
+    let Ok(mut state) = crate::state::persist::load_runtime_state(workspace) else {
         return;
     };
-    // 0.5.37 (`architect res_b8a6d40f3765`, R8): clear stale `next_retry_at`
-    // on terminal intents (succeeded/blocked/exhausted) BEFORE dispatching.
-    // Without this, a terminal record whose past `next_retry_at` survives
-    // a coordinator restart could get re-dispatched on the next tick,
-    // producing a spurious `recovery_started` event for an intent whose
-    // lifecycle already completed.
-    clear_stale_terminal_next_retry_at(workspace);
+    if clear_stale_terminal_next_retry_at(&mut state) {
+        let _ = crate::state::persist::save_runtime_state(workspace, &state);
+    }
     let due_agents = collect_due_recovery_agents(&state);
     for agent_id in due_agents {
         run_single_recovery(workspace, event_log, transport, &agent_id);
@@ -1863,10 +1871,7 @@ fn collect_due_recovery_agents(state: &Value) -> Vec<String> {
     };
     let mut due = Vec::new();
     for (agent_id, intent) in agents {
-        let status = intent
-            .get("status")
-            .and_then(Value::as_str)
-            .unwrap_or("");
+        let status = intent.get("status").and_then(Value::as_str).unwrap_or("");
         // 0.5.37 R8: dispatch only for non-terminal states. `succeeded`,
         // `blocked`, `exhausted`, `backpressured` are terminal / awaiting
         // manual action — a lifecycle dispatch here would be double-fire
@@ -1890,17 +1895,15 @@ fn collect_due_recovery_agents(state: &Value) -> Vec<String> {
     due
 }
 
-/// 0.5.37 (`architect res_b8a6d40f3765`, R8): terminal recovery states
-/// (`succeeded`, `blocked`, `exhausted`) must not carry a stale
-/// `next_retry_at`. Load state, scrub the key on any terminal entry, and
-/// only persist when at least one row changed so we don't dirty the tick's
-/// atomic_save contract for idle workspaces.
-fn clear_stale_terminal_next_retry_at(workspace: &Path) {
-    let Ok(mut state) = crate::state::persist::load_runtime_state(workspace) else {
-        return;
-    };
+/// 0.5.37 R8 + 0.5.43 debt-sweep §5 D-j: pure scrub helper. Terminal
+/// recovery intents (`succeeded`/`blocked`/`exhausted`) must not carry
+/// a stale `next_retry_at`. Returns true when at least one row was
+/// stripped, so the caller can save through its own load/write cycle
+/// (no double-load, no double-save). No I/O in this function — the
+/// only responsibility is mutation on a caller-owned `Value`.
+fn clear_stale_terminal_next_retry_at(state: &mut Value) -> bool {
     let mut mutated = false;
-    if let Some(agents) = recovery_intent_agents(&mut state) {
+    if let Some(agents) = recovery_intent_agents(state) {
         for (_agent_id, entry) in agents.iter_mut() {
             let Some(obj) = entry.as_object_mut() else {
                 continue;
@@ -1913,19 +1916,14 @@ fn clear_stale_terminal_next_retry_at(workspace: &Path) {
             if !matches!(status.as_str(), "succeeded" | "blocked" | "exhausted") {
                 continue;
             }
-            let had_stale = obj
-                .get("next_retry_at")
-                .filter(|v| !v.is_null())
-                .is_some();
+            let had_stale = obj.get("next_retry_at").filter(|v| !v.is_null()).is_some();
             if had_stale {
                 obj.remove("next_retry_at");
                 mutated = true;
             }
         }
     }
-    if mutated {
-        let _ = crate::state::persist::save_runtime_state(workspace, &state);
-    }
+    mutated
 }
 
 fn run_single_recovery(
@@ -2094,10 +2092,7 @@ fn write_recovery_intent_result(workspace: &Path, agent_id: &str, update: Recove
             );
             match update.blocked_reason {
                 Some(reason) => {
-                    obj.insert(
-                        "blocked_reason".to_string(),
-                        serde_json::json!(reason),
-                    );
+                    obj.insert("blocked_reason".to_string(), serde_json::json!(reason));
                 }
                 None => {
                     obj.remove("blocked_reason");

package/crates/team-agent/src/coordinator/tests/abnormal.rs

@@ -22,9 +22,16 @@ fn abnormal_default_notify_for_structured_fault() {
     )
     .unwrap();
     assert_eq!(out.notifications.len(), 1);
-    assert_eq!(out.notifications[0].decision, AbnormalDecision::NotifyDefault);
+    assert_eq!(
+        out.notifications[0].decision,
+        AbnormalDecision::NotifyDefault
+    );
     // §84:绝不触碰 provider adapter client。
-    assert_eq!(reg.adapter_calls.get(), 0, "MUST NOT call adapter_for (provider client)");
+    assert_eq!(
+        reg.adapter_calls.get(),
+        0,
+        "MUST NOT call adapter_for (provider client)"
+    );
 }
 
 #[test]
@@ -40,7 +47,10 @@ fn abnormal_blacklist_hit_yields_notify_blacklist() {
     )
     .unwrap();
     assert_eq!(out.notifications.len(), 1);
-    assert_eq!(out.notifications[0].decision, AbnormalDecision::NotifyBlacklist);
+    assert_eq!(
+        out.notifications[0].decision,
+        AbnormalDecision::NotifyBlacklist
+    );
     assert_eq!(reg.adapter_calls.get(), 0);
 }
 
@@ -56,7 +66,11 @@ fn abnormal_whitelist_beats_blacklist_and_skips() {
         &AbnormalNotificationState::default(),
     )
     .unwrap();
-    assert_eq!(out.notifications.len(), 0, "whitelist wins → skip (probe confirmed)");
+    assert_eq!(
+        out.notifications.len(),
+        0,
+        "whitelist wins → skip (probe confirmed)"
+    );
     assert_eq!(reg.adapter_calls.get(), 0);
 }
 
@@ -91,7 +105,11 @@ fn abnormal_same_turn_id_folds_to_one_notification() {
         &AbnormalNotificationState::default(),
     )
     .unwrap();
-    assert_eq!(out.notifications.len(), 1, "retry-loop in SAME turn folds (probe: 1)");
+    assert_eq!(
+        out.notifications.len(),
+        1,
+        "retry-loop in SAME turn folds (probe: 1)"
+    );
     // seen 状态回存,key = "sig_a\0t1"。
     assert!(out.notification_state.seen.contains("sig_a\u{0}t1"));
 }
@@ -112,7 +130,11 @@ fn abnormal_missing_turn_id_distinct_faults_each_notify() {
         &AbnormalNotificationState::default(),
     )
     .unwrap();
-    assert_eq!(out.notifications.len(), 2, "distinct raw → 2 notifies (probe)");
+    assert_eq!(
+        out.notifications.len(),
+        2,
+        "distinct raw → 2 notifies (probe)"
+    );
 }
 
 #[test]
@@ -130,7 +152,11 @@ fn abnormal_missing_turn_id_identical_faults_fold() {
         &AbnormalNotificationState::default(),
     )
     .unwrap();
-    assert_eq!(out.notifications.len(), 1, "identical raw folds even w/o turn_id (probe)");
+    assert_eq!(
+        out.notifications.len(),
+        1,
+        "identical raw folds even w/o turn_id (probe)"
+    );
 }
 
 #[test]
@@ -140,14 +166,12 @@ fn abnormal_preseeded_seen_suppresses_duplicate_across_calls() {
     let mut state = AbnormalNotificationState::default();
     state.seen.insert("sig_a\u{0}t1".to_string());
     let records = vec![serde_json::json!({"signature": "sig_a", "turn_id": "t1", "raw": "boom"})];
-    let out = process_abnormal_records(
-        &records,
-        &reg,
-        Provider::Codex,
-        &state,
-    )
-    .unwrap();
-    assert_eq!(out.notifications.len(), 0, "pre-seeded seen suppresses (probe: 0)");
+    let out = process_abnormal_records(&records, &reg, Provider::Codex, &state).unwrap();
+    assert_eq!(
+        out.notifications.len(),
+        0,
+        "pre-seeded seen suppresses (probe: 0)"
+    );
 }
 
 #[test]
@@ -387,7 +411,10 @@ fn whole_team_gone_clean_shutdown_is_silent() {
     assert!(!r.notify);
     assert!(!r.escalate_user_on_next_leader_command);
     assert!(!r.marker_written);
-    assert!(ms.markers.is_empty(), "clean shutdown writes NO durable marker");
+    assert!(
+        ms.markers.is_empty(),
+        "clean shutdown writes NO durable marker"
+    );
 }
 
 #[test]
@@ -425,9 +452,15 @@ fn whole_team_gone_unexpected_writes_marker_and_defers_escalation() {
     assert!(r.whole_team_gone);
     assert_eq!(r.classification, WholeTeamGoneClass::UnexpectedExit);
     assert!(r.notify);
-    assert!(r.escalate_user_on_next_leader_command, "defer to next leader command");
+    assert!(
+        r.escalate_user_on_next_leader_command,
+        "defer to next leader command"
+    );
     assert!(r.marker_written);
-    assert!(ms.markers.contains_key("whole_team_gone"), "durable marker named whole_team_gone");
+    assert!(
+        ms.markers.contains_key("whole_team_gone"),
+        "durable marker named whole_team_gone"
+    );
 }
 
 #[test]
@@ -446,5 +479,8 @@ fn whole_team_gone_unexpected_marker_failure_reflected() {
     let r = detect_whole_team_gone(&snap, &mut ms);
     assert_eq!(r.classification, WholeTeamGoneClass::UnexpectedExit);
     assert!(r.notify);
-    assert!(!r.marker_written, "marker set failed → marker_written false");
+    assert!(
+        !r.marker_written,
+        "marker set failed → marker_written false"
+    );
 }