@team-agent/installer 0.4.10 → 0.5.0

package/crates/team-agent/src/coordinator/tick.rs

@@ -107,10 +107,7 @@ pub enum TickError {
 /// returns `None` and the tick falls back to the raw state — preserving
 /// behavior for legacy single-team workspaces and tests that don't seed
 /// `teams.<key>`. Sibling teams under `state.teams.*` are NOT touched.
-fn coordinator_team_scoped_state(
-    workspace: &std::path::Path,
-    raw_state: &Value,
-) -> Option<Value> {
+fn coordinator_team_scoped_state(workspace: &std::path::Path, raw_state: &Value) -> Option<Value> {
     let active = raw_state
         .get("active_team_key")
         .and_then(Value::as_str)
@@ -334,7 +331,13 @@ impl Coordinator {
                 BTreeMap::new()
             }
         };
-        if let Err(error) = self.detect_abnormal_exits(&mut state, &event_log, &pane_snapshot) {
+        if let Err(error) = crate::coordinator::steps::abnormal::detect_abnormal_exits(
+            self.workspace.as_path(),
+            self.transport.as_ref(),
+            &mut state,
+            &event_log,
+            &pane_snapshot,
+        ) {
             let _ = event_log.write(
                 "coordinator.tick.detect_abnormal_failed",
                 serde_json::json!({"error": error.to_string()}),
@@ -481,9 +484,7 @@ impl Coordinator {
         // can replay durable session truth even if state was lost. Architect
         // §4 fix #5.
         for agent_id in &report.assigned {
-            let agent = state
-                .get("agents")
-                .and_then(|a| a.get(agent_id.as_str()));
+            let agent = state.get("agents").and_then(|a| a.get(agent_id.as_str()));
             if let Some(agent) = agent {
                 event_log.write(
                     "session.captured",
@@ -515,11 +516,13 @@ impl Coordinator {
             let agent = state
                 .get("agents")
                 .and_then(|a| a.get(missing.agent_id.as_str()));
-            let pane_id = agent
-                .and_then(|a| a.get("pane_id"))
+            let pane_id = agent.and_then(|a| a.get("pane_id")).and_then(Value::as_str);
+            let pane_pid = agent
+                .and_then(|a| a.get("pane_pid"))
+                .and_then(Value::as_u64);
+            let provider = agent
+                .and_then(|a| a.get("provider"))
                 .and_then(Value::as_str);
-            let pane_pid = agent.and_then(|a| a.get("pane_pid")).and_then(Value::as_u64);
-            let provider = agent.and_then(|a| a.get("provider")).and_then(Value::as_str);
             let session_id_in_argv = agent
                 .and_then(|a| a.get("session_id_in_argv"))
                 .and_then(Value::as_str);
@@ -802,7 +805,8 @@ impl Coordinator {
                         .map(str::to_string),
                     last_output_at,
                     rollout_path,
-                    process_liveness: explicit_process_liveness(agent),
+                    process_liveness:
+                        crate::coordinator::steps::abnormal::explicit_process_liveness(agent),
                 },
             );
         }
@@ -845,182 +849,6 @@ impl Coordinator {
         })
     }
 
-    /// #236 `worker.abnormal_exit` watcher.
-    ///
-    /// Notify only when both signals are true: the provider process is dead AND the
-    /// latest transcript/rollout JSONL record is an explicit provider error. Dead-only
-    /// and error-only observations are written as check/suppressed audit events with
-    /// `notification=false`; they never call the N32 leader funnel. This path is
-    /// intentionally separate from the generic transcript-only abnormal fact track.
-    fn detect_abnormal_exits(
-        &self,
-        state: &mut Value,
-        event_log: &EventLog,
-        targets: &[crate::transport::PaneInfo],
-    ) -> Result<(), TickError> {
-        let snapshot = state.clone();
-        let team = crate::state::projection::team_state_key(&snapshot);
-        let session_name = snapshot.get("session_name").and_then(Value::as_str);
-        for agent in abnormal_watch_agents(&snapshot) {
-            let rollout_path =
-                resolve_agent_rollout_path(self.workspace.as_path(), &agent.rollout_path);
-            let metadata = match std::fs::metadata(&rollout_path) {
-                Ok(metadata) => metadata,
-                Err(error) => {
-                    upsert_abnormal_watch(
-                        state,
-                        &agent.agent_id,
-                        abnormal_watch_payload(
-                            &agent,
-                            None,
-                            None,
-                            "unverifiable",
-                            None,
-                            Some(error.to_string()),
-                        ),
-                    );
-                    continue;
-                }
-            };
-            let size = metadata.len();
-            let mtime_ns = metadata_mtime_ns(&metadata);
-            // P1 (C-P1-2/3): (size, mtime_ns) pair gate — an unchanged transcript is not
-            // read at all (live sample: 332MB whole-file read per agent per 2s tick).
-            // ANY field change (including a size shrink / truncate) falls through to the
-            // re-read below.
-            if let (Some(mtime), Some(stored)) = (
-                mtime_ns,
-                abnormal_watch_stored_metadata(&snapshot, &agent.agent_id),
-            ) {
-                if stored == (size, mtime) {
-                    continue;
-                }
-            }
-            // P1 (C-P1-1): bounded tail read — the abnormal decision only consumes the
-            // LATEST transcript record; window matches Python `_TAIL_BYTES` (131072,
-            // idle_takeover_wiring.py:13), never less.
-            let text = match read_tail_text(&rollout_path, ABNORMAL_TAIL_BYTES) {
-                Ok(text) => text,
-                Err(error) => {
-                    upsert_abnormal_watch(
-                        state,
-                        &agent.agent_id,
-                        abnormal_watch_payload(
-                            &agent,
-                            Some(size),
-                            mtime_ns,
-                            "unverifiable",
-                            None,
-                            Some(error.to_string()),
-                        ),
-                    );
-                    continue;
-                }
-            };
-            let liveness =
-                agent_process_liveness(&agent, session_name, targets, self.transport.as_ref());
-            let fact = crate::provider::latest_explicit_error_fact(agent.provider, &text);
-            let decision = abnormal_exit_decision(liveness.state, fact.as_ref());
-            let check_key = abnormal_check_key(&agent, &liveness, fact.as_ref(), size);
-            upsert_abnormal_watch(
-                state,
-                &agent.agent_id,
-                abnormal_watch_payload(
-                    &agent,
-                    Some(size),
-                    mtime_ns,
-                    process_liveness_wire(liveness.state),
-                    fact.as_ref().map(|f| f.signature.as_str()),
-                    None,
-                ),
-            );
-            if abnormal_last_check_key(state, &agent.agent_id).as_deref()
-                != Some(check_key.as_str())
-            {
-                write_abnormal_check(
-                    event_log,
-                    &team,
-                    &agent,
-                    &liveness,
-                    fact.as_ref(),
-                    decision,
-                    size,
-                    mtime_ns,
-                )?;
-                mark_abnormal_checked(state, &agent.agent_id, &check_key);
-            }
-            let fact = match (decision, fact) {
-                (AbnormalExitDecision::Notify, Some(fact)) => fact,
-                (AbnormalExitDecision::Suppress(reason), _) => {
-                    let suppress_key = abnormal_suppression_key(&agent, &liveness, reason, size);
-                    if abnormal_last_suppressed_key(state, &agent.agent_id).as_deref()
-                        != Some(suppress_key.as_str())
-                    {
-                        write_abnormal_suppressed(event_log, &team, &agent, &liveness, reason)?;
-                        mark_abnormal_suppressed(state, &agent.agent_id, &suppress_key);
-                    }
-                    continue;
-                }
-                (AbnormalExitDecision::NoSignal, _) => continue,
-                (AbnormalExitDecision::Notify, None) => continue,
-            };
-            let dedupe_key = abnormal_dedupe_key(&agent, &fact, size);
-            if abnormal_last_notified_key(state, &agent.agent_id).as_deref()
-                == Some(dedupe_key.as_str())
-            {
-                continue;
-            }
-            let content = format_abnormal_exit_message(&team, &agent, &fact, &liveness, size);
-            let outcome = crate::messaging::send_to_leader_receiver(
-                self.workspace.as_path(),
-                state,
-                "leader",
-                &content,
-                None,
-                &agent.agent_id,
-                false,
-                Some(&dedupe_key),
-                event_log,
-            )?;
-            let notification_status = if outcome.ok {
-                "queued"
-            } else if matches!(outcome.status, crate::messaging::DeliveryStatus::Blocked) {
-                "rebind_required"
-            } else {
-                "refused"
-            };
-            event_log.write(
-                "worker.abnormal_exit",
-                serde_json::json!({
-                    "team_id": team.as_str(),
-                    "agent_id": agent.agent_id.as_str(),
-                    "provider": provider_wire(agent.provider),
-                    "path": agent.rollout_path_display.as_str(),
-                    "dead_process": true,
-                    "process_dead": true,
-                    "provider_process_dead": true,
-                    "latest_error": true,
-                    "latest_explicit_error": true,
-                    "dead_process_and_latest_error": true,
-                    "dead_process_and_latest_explicit_error": true,
-                    "process_dead_and_latest_explicit_error": true,
-                    "provider_process_dead_and_latest_explicit_error": true,
-                    "signature": fact.signature.as_str(),
-                    "turn_id": fact.turn_id.as_ref().map(|id| id.as_str()),
-                    "size": size,
-                    "mtime_ns": mtime_ns,
-                    "process_liveness": process_liveness_wire(liveness.state),
-                    "pid_status": liveness.detail.as_str(),
-                    "notification_message_id": outcome.message_id,
-                    "notification_status": notification_status,
-                    "notification_channel": outcome.channel,
-                }),
-            )?;
-            mark_abnormal_notified(state, &agent.agent_id, &dedupe_key);
-        }
-        Ok(())
-    }
-
     fn handle_startup_prompts(
         &self,
         state: &mut Value,
@@ -1656,743 +1484,6 @@ fn turn_state_wire(state: TurnState) -> &'static str {
     }
 }
 
-use crate::provider::wire::provider_wire;
-
-#[derive(Debug, Clone)]
-struct AbnormalWatchAgent {
-    agent_id: String,
-    provider: crate::model::enums::Provider,
-    rollout_path: PathBuf,
-    rollout_path_display: String,
-    status: Option<String>,
-    process_liveness: Option<ProcessLiveness>,
-    window: Option<String>,
-    pane_id: Option<String>,
-    pid: Option<Pid>,
-    current_command: Option<String>,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-struct ProcessCheck {
-    state: ProcessLiveness,
-    detail: String,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum AbnormalExitDecision {
-    Notify,
-    Suppress(&'static str),
-    NoSignal,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-struct AbnormalExitGate {
-    provider_process_dead: bool,
-    latest_explicit_error: bool,
-}
-
-impl AbnormalExitGate {
-    fn new(process_liveness: ProcessLiveness, latest_explicit_error: bool) -> Self {
-        Self {
-            provider_process_dead: process_liveness == ProcessLiveness::Dead,
-            latest_explicit_error,
-        }
-    }
-
-    fn should_notify_worker_abnormal_exit(self) -> bool {
-        should_notify_worker_abnormal_exit(self.provider_process_dead, self.latest_explicit_error)
-    }
-
-    fn suppressed_reason(self) -> Option<&'static str> {
-        match (self.provider_process_dead, self.latest_explicit_error) {
-            (true, false) => Some("dead_only"),
-            (false, true) => Some("error_only"),
-            _ => None,
-        }
-    }
-}
-
-fn abnormal_exit_decision(
-    process_liveness: ProcessLiveness,
-    latest_explicit_error: Option<&crate::provider::FaultFact>,
-) -> AbnormalExitDecision {
-    let gate = AbnormalExitGate::new(process_liveness, latest_explicit_error.is_some());
-    if gate.should_notify_worker_abnormal_exit() {
-        return AbnormalExitDecision::Notify;
-    }
-    match gate.suppressed_reason() {
-        Some(reason) => AbnormalExitDecision::Suppress(reason),
-        None => AbnormalExitDecision::NoSignal,
-    }
-}
-
-fn should_notify_worker_abnormal_exit(
-    provider_process_dead: bool,
-    latest_explicit_error: bool,
-) -> bool {
-    provider_process_dead && latest_explicit_error
-}
-
-fn resolve_agent_rollout_path(workspace: &Path, path: &Path) -> PathBuf {
-    if path.is_absolute() {
-        path.to_path_buf()
-    } else {
-        workspace.join(path)
-    }
-}
-
-fn abnormal_watch_agents(state: &Value) -> Vec<AbnormalWatchAgent> {
-    let Some(agents) = state.get("agents").and_then(Value::as_object) else {
-        return Vec::new();
-    };
-    agents
-        .iter()
-        .filter_map(|(agent_id, agent)| {
-            if matches!(agent.get("status").and_then(Value::as_str), Some("paused")) {
-                return None;
-            }
-            let provider = agent
-                .get("provider")
-                .and_then(Value::as_str)
-                .and_then(parse_provider)?;
-            let rollout_path_display = ["rollout_path", "transcript_path", "session_log_path"]
-                .into_iter()
-                .find_map(|key| agent.get(key).and_then(Value::as_str))
-                .filter(|path| !path.is_empty())?
-                .to_string();
-            Some(AbnormalWatchAgent {
-                agent_id: agent_id.clone(),
-                provider,
-                rollout_path: PathBuf::from(&rollout_path_display),
-                rollout_path_display,
-                status: agent
-                    .get("status")
-                    .and_then(Value::as_str)
-                    .map(str::to_string),
-                process_liveness: explicit_process_liveness(agent),
-                window: agent
-                    .get("window")
-                    .and_then(Value::as_str)
-                    .map(str::to_string),
-                pane_id: agent
-                    .get("pane_id")
-                    .and_then(Value::as_str)
-                    .map(str::to_string),
-                pid: agent_pid(agent),
-                current_command: agent
-                    .get("pane_current_command")
-                    .or_else(|| agent.get("current_command"))
-                    .and_then(Value::as_str)
-                    .map(str::to_string),
-            })
-        })
-        .collect()
-}
-
-fn agent_pid(agent: &Value) -> Option<Pid> {
-    ["provider_pid", "process_id", "pid", "child_pid", "pane_pid"]
-        .into_iter()
-        .find_map(|key| json_u32(agent.get(key)).map(Pid::new))
-}
-
-fn explicit_process_liveness(agent: &Value) -> Option<ProcessLiveness> {
-    if let Some(process) = agent
-        .get("provider_process")
-        .or_else(|| agent.get("process"))
-    {
-        if let Some(liveness) = explicit_process_liveness(process) {
-            return Some(liveness);
-        }
-    }
-    for key in [
-        "provider_process_liveness",
-        "process_liveness",
-        "pane_liveness",
-    ] {
-        match agent.get(key).and_then(Value::as_str) {
-            Some("dead") => return Some(ProcessLiveness::Dead),
-            Some("alive" | "live") => return Some(ProcessLiveness::Alive),
-            Some("unverifiable" | "unknown") => return Some(ProcessLiveness::Unverifiable),
-            _ => {}
-        }
-    }
-    for key in [
-        "provider_process_alive",
-        "process_alive",
-        "provider_alive",
-        "alive",
-    ] {
-        if let Some(alive) = agent.get(key).and_then(Value::as_bool) {
-            return Some(if alive {
-                ProcessLiveness::Alive
-            } else {
-                ProcessLiveness::Dead
-            });
-        }
-    }
-    for key in [
-        "provider_process_dead",
-        "process_dead",
-        "provider_dead",
-        "dead",
-    ] {
-        if let Some(dead) = agent.get(key).and_then(Value::as_bool) {
-            return Some(if dead {
-                ProcessLiveness::Dead
-            } else {
-                ProcessLiveness::Alive
-            });
-        }
-    }
-    for key in ["status", "state", "liveness"] {
-        match agent.get(key).and_then(Value::as_str) {
-            Some("dead" | "exited" | "terminated" | "crashed" | "missing") => {
-                return Some(ProcessLiveness::Dead);
-            }
-            Some("alive" | "live" | "running") => return Some(ProcessLiveness::Alive),
-            Some("unverifiable" | "unknown") => return Some(ProcessLiveness::Unverifiable),
-            _ => {}
-        }
-    }
-    None
-}
-
-fn json_u32(value: Option<&Value>) -> Option<u32> {
-    value
-        .and_then(|v| {
-            v.as_u64()
-                .or_else(|| v.as_i64().and_then(|n| u64::try_from(n).ok()))
-        })
-        .and_then(|n| u32::try_from(n).ok())
-}
-
-fn agent_process_liveness(
-    agent: &AbnormalWatchAgent,
-    session_name: Option<&str>,
-    targets: &[crate::transport::PaneInfo],
-    transport: &dyn crate::transport::Transport,
-) -> ProcessCheck {
-    if let Some(pid) = agent.pid {
-        return pid_process_check("pid", pid);
-    }
-    if let Some(liveness) = agent.process_liveness {
-        return process_check(
-            liveness,
-            format!("explicit:{}", process_liveness_wire(liveness)),
-        );
-    }
-    if agent.status.as_deref().is_some_and(|status| {
-        matches!(
-            status,
-            "stopped" | "missing" | "error" | "dead" | "exited" | "terminated" | "crashed"
-        )
-    }) {
-        return process_check(
-            ProcessLiveness::Dead,
-            format!("status:{}", agent.status.as_deref().unwrap_or("unknown")),
-        );
-    }
-    if let Some(command) = agent.current_command.as_deref() {
-        return command_process_check(agent.provider, command);
-    }
-    if let Some(target) = matching_agent_target(agent, session_name, targets) {
-        if let Some(command) = target.current_command.as_deref() {
-            return pane_command_process_check(agent.provider, target, command);
-        }
-        if let Some(pid) = target.pane_pid.map(Pid::new) {
-            return pid_process_check("pane_pid", pid);
-        }
-        return process_check(
-            ProcessLiveness::Unverifiable,
-            "pane_present_pid_unknown".to_string(),
-        );
-    }
-    if let Some(pane_id) = agent.pane_id.as_deref() {
-        let pane = crate::transport::PaneId::new(pane_id);
-        return match transport.liveness(&pane) {
-            Ok(crate::transport::PaneLiveness::Dead) => {
-                process_check(ProcessLiveness::Dead, format!("pane_dead:{pane_id}"))
-            }
-            Ok(crate::transport::PaneLiveness::Live) => process_check(
-                ProcessLiveness::Unverifiable,
-                format!("pane_live_pid_unknown:{pane_id}"),
-            ),
-            Ok(crate::transport::PaneLiveness::Unknown) => process_check(
-                ProcessLiveness::Unverifiable,
-                format!("pane_unknown:{pane_id}"),
-            ),
-            Err(error) => process_check(
-                ProcessLiveness::Unverifiable,
-                format!("pane_unverifiable:{pane_id}:{error}"),
-            ),
-        };
-    }
-    let (Some(session), Some(window)) = (session_name, agent.window.as_deref()) else {
-        return process_check(
-            ProcessLiveness::Unverifiable,
-            "missing_session_or_window".to_string(),
-        );
-    };
-    let session = crate::transport::SessionName::new(session);
-    match transport.list_windows(&session) {
-        Ok(windows) if windows.iter().any(|known| known.as_str() == window) => process_check(
-            ProcessLiveness::Unverifiable,
-            "window_present_pid_unknown".to_string(),
-        ),
-        Ok(_) => process_check(ProcessLiveness::Dead, format!("window_missing:{window}")),
-        Err(error) => process_check(
-            ProcessLiveness::Unverifiable,
-            format!("window_unverifiable:{window}:{error}"),
-        ),
-    }
-}
-
-fn matching_agent_target<'a>(
-    agent: &AbnormalWatchAgent,
-    session_name: Option<&str>,
-    targets: &'a [crate::transport::PaneInfo],
-) -> Option<&'a crate::transport::PaneInfo> {
-    if let Some(pane_id) = agent.pane_id.as_deref() {
-        if let Some(target) = targets
-            .iter()
-            .find(|target| target.pane_id.as_str() == pane_id)
-        {
-            return Some(target);
-        }
-    }
-    let (Some(session), Some(window)) = (session_name, agent.window.as_deref()) else {
-        return None;
-    };
-    targets.iter().find(|target| {
-        target.session.as_str() == session
-            && target
-                .window_name
-                .as_ref()
-                .is_some_and(|known| known.as_str() == window)
-    })
-}
-
-fn pid_process_check(label: &str, pid: Pid) -> ProcessCheck {
-    match pid_is_running(pid) {
-        Ok(true) => process_check(ProcessLiveness::Alive, format!("{label}_running:{pid}")),
-        Ok(false) => process_check(ProcessLiveness::Dead, format!("{label}_not_running:{pid}")),
-        Err(error) => process_check(
-            ProcessLiveness::Unverifiable,
-            format!("{label}_unverifiable:{pid}:{error}"),
-        ),
-    }
-}
-
-fn command_process_check(provider: crate::model::enums::Provider, command: &str) -> ProcessCheck {
-    if crate::leader::command_matches_provider(provider, command) {
-        process_check(ProcessLiveness::Alive, format!("current_command:{command}"))
-    } else {
-        process_check(
-            ProcessLiveness::Dead,
-            format!("provider_not_foreground:{command}"),
-        )
-    }
-}
-
-fn pane_command_process_check(
-    provider: crate::model::enums::Provider,
-    pane: &crate::transport::PaneInfo,
-    command: &str,
-) -> ProcessCheck {
-    if crate::leader::attribute_pane_provider(pane)
-        .is_some_and(|candidate| crate::leader::provider_matches(candidate, provider))
-    {
-        process_check(ProcessLiveness::Alive, format!("current_command:{command}"))
-    } else {
-        process_check(
-            ProcessLiveness::Dead,
-            format!("provider_not_foreground:{command}"),
-        )
-    }
-}
-
-fn process_check(state: ProcessLiveness, detail: String) -> ProcessCheck {
-    ProcessCheck { state, detail }
-}
-
-fn process_liveness_wire(state: ProcessLiveness) -> &'static str {
-    match state {
-        ProcessLiveness::Alive => "alive",
-        ProcessLiveness::Dead => "dead",
-        ProcessLiveness::Unverifiable => "unverifiable",
-    }
-}
-
-fn metadata_mtime_ns(metadata: &std::fs::Metadata) -> Option<u64> {
-    let duration = metadata
-        .modified()
-        .ok()?
-        .duration_since(std::time::UNIX_EPOCH)
-        .ok()?;
-    Some(
-        duration
-            .as_secs()
-            .saturating_mul(1_000_000_000)
-            .saturating_add(u64::from(duration.subsec_nanos())),
-    )
-}
-
-fn abnormal_watch_payload(
-    agent: &AbnormalWatchAgent,
-    size: Option<u64>,
-    mtime_ns: Option<u64>,
-    liveness: &str,
-    signature: Option<&str>,
-    error: Option<String>,
-) -> Value {
-    let dead_process = liveness == "dead";
-    let latest_explicit_error = signature.is_some();
-    let notify = dead_process && latest_explicit_error;
-    let suppressed_reason = match (dead_process, latest_explicit_error) {
-        (true, false) => Some("dead_only"),
-        (false, true) => Some("error_only"),
-        _ => None,
-    };
-    serde_json::json!({
-        "path": agent.rollout_path_display.as_str(),
-        "provider": provider_wire(agent.provider),
-        "mtime_ns": mtime_ns,
-        "size": size,
-        "last_offset": size,
-        "last_signature": signature,
-        "last_liveness": liveness,
-        "dead_process": dead_process,
-        "process_dead": dead_process,
-        "provider_process_dead": dead_process,
-        "latest_error": latest_explicit_error,
-        "latest_explicit_error": latest_explicit_error,
-        "dead_process_and_latest_error": notify,
-        "dead_process_and_latest_explicit_error": notify,
-        "process_dead_and_latest_explicit_error": notify,
-        "provider_process_dead_and_latest_explicit_error": notify,
-        "suppressed_reason": suppressed_reason,
-        "notification": notify,
-        "last_error": error,
-        "last_checked_at": chrono::Utc::now().to_rfc3339(),
-    })
-}
-
-fn upsert_abnormal_watch(state: &mut Value, agent_id: &str, mut payload: Value) {
-    let preserved = [
-        "last_notified_key",
-        "last_notified_at",
-        "last_suppressed_key",
-        "last_suppressed_at",
-        "last_check_key",
-        "last_check_at",
-    ]
-    .into_iter()
-    .filter_map(|key| abnormal_watch_field(state, agent_id, key).map(|value| (key, value)))
-    .collect::<Vec<_>>();
-    if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
-        if let Some(payload_obj) = payload.as_object_mut() {
-            for (key, value) in preserved {
-                payload_obj.insert(key.to_string(), value);
-            }
-        }
-        watch.insert(agent_id.to_string(), payload);
-    }
-}
-
-fn coordinator_child_object<'a>(
-    state: &'a mut Value,
-    key: &str,
-) -> Option<&'a mut serde_json::Map<String, Value>> {
-    if !state.is_object() {
-        *state = serde_json::json!({});
-    }
-    let state_obj = state.as_object_mut()?;
-    let coordinator = state_obj
-        .entry("coordinator".to_string())
-        .or_insert_with(|| serde_json::json!({}));
-    if !coordinator.is_object() {
-        *coordinator = serde_json::json!({});
-    }
-    let coord_obj = coordinator.as_object_mut()?;
-    let child = coord_obj
-        .entry(key.to_string())
-        .or_insert_with(|| serde_json::json!({}));
-    if !child.is_object() {
-        *child = serde_json::json!({});
-    }
-    child.as_object_mut()
-}
-
-fn abnormal_last_notified_key(state: &Value, agent_id: &str) -> Option<String> {
-    abnormal_watch_str(state, agent_id, "last_notified_key")
-}
-
-fn abnormal_last_suppressed_key(state: &Value, agent_id: &str) -> Option<String> {
-    abnormal_watch_str(state, agent_id, "last_suppressed_key")
-}
-
-fn abnormal_last_check_key(state: &Value, agent_id: &str) -> Option<String> {
-    abnormal_watch_str(state, agent_id, "last_check_key")
-}
-
-/// P1: Python `_TAIL_BYTES` parity (idle_takeover_wiring.py:13) — RS must not read less.
-const ABNORMAL_TAIL_BYTES: u64 = 131_072;
-
-/// P1: bounded tail read; a partial first line is harmless (the consumer only parses
-/// the latest complete JSONL record) and lossy UTF-8 keeps a mid-codepoint seek safe.
-fn read_tail_text(path: &Path, max_bytes: u64) -> std::io::Result<String> {
-    use std::io::{Read, Seek, SeekFrom};
-    let mut file = std::fs::File::open(path)?;
-    let len = file.metadata()?.len();
-    if len > max_bytes {
-        file.seek(SeekFrom::Start(len - max_bytes))?;
-    }
-    let mut bytes = Vec::new();
-    file.read_to_end(&mut bytes)?;
-    Ok(String::from_utf8_lossy(&bytes).into_owned())
-}
-
-/// P1: the previous tick's `(size, mtime_ns)` pair from the abnormal watch payload.
-fn abnormal_watch_stored_metadata(state: &Value, agent_id: &str) -> Option<(u64, u64)> {
-    let watch = state
-        .get("coordinator")?
-        .get("abnormal_exit_watch")?
-        .get(agent_id)?;
-    Some((
-        watch.get("size")?.as_u64()?,
-        watch.get("mtime_ns")?.as_u64()?,
-    ))
-}
-
-fn abnormal_watch_str(state: &Value, agent_id: &str, field: &str) -> Option<String> {
-    state
-        .get("coordinator")
-        .and_then(|v| v.get("abnormal_exit_watch"))
-        .and_then(|v| v.get(agent_id))
-        .and_then(|v| v.get(field))
-        .and_then(Value::as_str)
-        .map(str::to_string)
-}
-
-fn abnormal_watch_field(state: &Value, agent_id: &str, field: &str) -> Option<Value> {
-    state
-        .get("coordinator")
-        .and_then(|v| v.get("abnormal_exit_watch"))
-        .and_then(|v| v.get(agent_id))
-        .and_then(|v| v.get(field))
-        .cloned()
-}
-
-fn mark_abnormal_notified(state: &mut Value, agent_id: &str, key: &str) {
-    if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
-        let entry = watch
-            .entry(agent_id.to_string())
-            .or_insert_with(|| serde_json::json!({}));
-        if !entry.is_object() {
-            *entry = serde_json::json!({});
-        }
-        if let Some(obj) = entry.as_object_mut() {
-            obj.insert("last_notified_key".to_string(), serde_json::json!(key));
-            obj.insert(
-                "last_notified_at".to_string(),
-                serde_json::json!(chrono::Utc::now().to_rfc3339()),
-            );
-        }
-    }
-}
-
-fn mark_abnormal_suppressed(state: &mut Value, agent_id: &str, key: &str) {
-    if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
-        let entry = watch
-            .entry(agent_id.to_string())
-            .or_insert_with(|| serde_json::json!({}));
-        if !entry.is_object() {
-            *entry = serde_json::json!({});
-        }
-        if let Some(obj) = entry.as_object_mut() {
-            obj.insert("last_suppressed_key".to_string(), serde_json::json!(key));
-            obj.insert(
-                "last_suppressed_at".to_string(),
-                serde_json::json!(chrono::Utc::now().to_rfc3339()),
-            );
-        }
-    }
-}
-
-fn mark_abnormal_checked(state: &mut Value, agent_id: &str, key: &str) {
-    if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
-        let entry = watch
-            .entry(agent_id.to_string())
-            .or_insert_with(|| serde_json::json!({}));
-        if !entry.is_object() {
-            *entry = serde_json::json!({});
-        }
-        if let Some(obj) = entry.as_object_mut() {
-            obj.insert("last_check_key".to_string(), serde_json::json!(key));
-            obj.insert(
-                "last_check_at".to_string(),
-                serde_json::json!(chrono::Utc::now().to_rfc3339()),
-            );
-        }
-    }
-}
-
-fn write_abnormal_check(
-    event_log: &EventLog,
-    team: &str,
-    agent: &AbnormalWatchAgent,
-    liveness: &ProcessCheck,
-    fact: Option<&crate::provider::FaultFact>,
-    decision: AbnormalExitDecision,
-    size: u64,
-    mtime_ns: Option<u64>,
-) -> Result<(), TickError> {
-    let dead_process = liveness.state == ProcessLiveness::Dead;
-    let latest_explicit_error = fact.is_some();
-    event_log.write(
-        "worker.abnormal_exit.check",
-        serde_json::json!({
-            "team_id": team,
-            "agent_id": agent.agent_id.as_str(),
-            "provider": provider_wire(agent.provider),
-            "path": agent.rollout_path_display.as_str(),
-            "size": size,
-            "last_offset": size,
-            "mtime_ns": mtime_ns,
-            "dead_process": dead_process,
-            "process_dead": dead_process,
-            "provider_process_dead": dead_process,
-            "latest_error": latest_explicit_error,
-            "latest_explicit_error": latest_explicit_error,
-            "dead_process_and_latest_error": dead_process && latest_explicit_error,
-            "dead_process_and_latest_explicit_error": dead_process && latest_explicit_error,
-            "process_dead_and_latest_explicit_error": dead_process && latest_explicit_error,
-            "provider_process_dead_and_latest_explicit_error": dead_process && latest_explicit_error,
-            "notification": matches!(decision, AbnormalExitDecision::Notify),
-            "suppressed_reason": match decision {
-                AbnormalExitDecision::Suppress(reason) => Some(reason),
-                AbnormalExitDecision::Notify | AbnormalExitDecision::NoSignal => None,
-            },
-            "signature": fact.map(|fact| fact.signature.as_str()),
-            "turn_id": fact.and_then(|fact| fact.turn_id.as_ref().map(|id| id.as_str())),
-            "process_liveness": process_liveness_wire(liveness.state),
-            "pid_status": liveness.detail.as_str(),
-        }),
-    )?;
-    Ok(())
-}
-
-fn write_abnormal_suppressed(
-    event_log: &EventLog,
-    team: &str,
-    agent: &AbnormalWatchAgent,
-    liveness: &ProcessCheck,
-    reason: &str,
-) -> Result<(), TickError> {
-    event_log.write(
-        "abnormal_exit.single_signal_suppressed",
-        serde_json::json!({
-            "team_id": team,
-            "agent_id": agent.agent_id.as_str(),
-            "provider": provider_wire(agent.provider),
-            "path": agent.rollout_path_display.as_str(),
-            "reason": reason,
-            "notification": false,
-            "dead_process": liveness.state == ProcessLiveness::Dead,
-            "process_dead": liveness.state == ProcessLiveness::Dead,
-            "provider_process_dead": liveness.state == ProcessLiveness::Dead,
-            "latest_error": reason == "error_only",
-            "latest_explicit_error": reason == "error_only",
-            "dead_process_and_latest_error": false,
-            "dead_process_and_latest_explicit_error": false,
-            "process_dead_and_latest_explicit_error": false,
-            "provider_process_dead_and_latest_explicit_error": false,
-            "process_liveness": process_liveness_wire(liveness.state),
-            "pid_status": liveness.detail.as_str(),
-        }),
-    )?;
-    Ok(())
-}
-
-fn abnormal_dedupe_key(
-    agent: &AbnormalWatchAgent,
-    fact: &crate::provider::FaultFact,
-    size: u64,
-) -> String {
-    let bucket = fact
-        .turn_id
-        .as_ref()
-        .map(|id| id.as_str().to_string())
-        .unwrap_or_else(|| size.to_string());
-    format!(
-        "worker.abnormal_exit:{}:{}:{}:{}",
-        agent.agent_id,
-        agent.rollout_path_display,
-        fact.signature.as_str(),
-        bucket
-    )
-}
-
-fn abnormal_suppression_key(
-    agent: &AbnormalWatchAgent,
-    liveness: &ProcessCheck,
-    reason: &str,
-    size: u64,
-) -> String {
-    format!(
-        "abnormal_exit.single_signal_suppressed:{}:{}:{}:{}:{}",
-        agent.agent_id,
-        agent.rollout_path_display,
-        reason,
-        process_liveness_wire(liveness.state),
-        size
-    )
-}
-
-fn abnormal_check_key(
-    agent: &AbnormalWatchAgent,
-    liveness: &ProcessCheck,
-    fact: Option<&crate::provider::FaultFact>,
-    size: u64,
-) -> String {
-    format!(
-        "worker.abnormal_exit.check:{}:{}:{}:{}:{}",
-        agent.agent_id,
-        agent.rollout_path_display,
-        process_liveness_wire(liveness.state),
-        fact.map(|fact| fact.signature.as_str()).unwrap_or("-"),
-        size
-    )
-}
-
-fn format_abnormal_exit_message(
-    team: &str,
-    agent: &AbnormalWatchAgent,
-    fact: &crate::provider::FaultFact,
-    liveness: &ProcessCheck,
-    size: u64,
-) -> String {
-    let turn_id = fact.turn_id.as_ref().map(|id| id.as_str()).unwrap_or("-");
-    format!(
-        "Team Agent detected a provider abnormal exit.\n\n\
-event: worker.abnormal_exit\n\
-team: {team}\n\
-node: {node}\n\
-provider: {provider}\n\
-signature: {signature}\n\
-turn_id: {turn_id}\n\
-transcript: {path}\n\
-last_offset: {size}\n\
-pid_status: {pid_status}\n\n\
-No automatic restart was performed.",
-        node = agent.agent_id.as_str(),
-        provider = provider_wire(agent.provider),
-        signature = fact.signature.as_str(),
-        path = agent.rollout_path_display.as_str(),
-        pid_status = liveness.detail.as_str(),
-    )
-}
-
 fn monotonic_seconds() -> f64 {
     match std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) {
         Ok(duration) => duration.as_secs_f64(),
@@ -2768,7 +1859,7 @@ fn jsonl_activity_for_agent(agent: &Value) -> Option<crate::messaging::AgentActi
     // still forces re-read because size is part of the cache key.
     let metadata = std::fs::metadata(&rollout_path).ok()?;
     let size = metadata.len();
-    let mtime_ns = metadata_mtime_ns(&metadata)?;
+    let mtime_ns = crate::coordinator::steps::abnormal::metadata_mtime_ns(&metadata)?;
     if let Ok(cache) = jsonl_activity_cache().lock() {
         if let Some(entry) = cache.get(&rollout_path) {
             if entry.size == size && entry.mtime_ns == mtime_ns {
@@ -2781,8 +1872,13 @@ fn jsonl_activity_for_agent(agent: &Value) -> Option<crate::messaging::AgentActi
     // latest lifecycle records to determine idle/working state; the
     // abnormal-exit path uses the same 128KiB tail and is sufficient for
     // claude / codex / copilot lifecycle markers.
-    let log_text = read_tail_text(&rollout_path, JSONL_ACTIVITY_TAIL_BYTES).ok()?;
-    let process = explicit_process_liveness(agent).unwrap_or(ProcessLiveness::Unverifiable);
+    let log_text = crate::coordinator::steps::abnormal::read_tail_text(
+        &rollout_path,
+        JSONL_ACTIVITY_TAIL_BYTES,
+    )
+    .ok()?;
+    let process = crate::coordinator::steps::abnormal::explicit_process_liveness(agent)
+        .unwrap_or(ProcessLiveness::Unverifiable);
     let activity = crate::provider::classify(provider, &log_text, process, 0.0)
         .ok()
         .and_then(|result| {
@@ -3106,8 +2202,7 @@ fn write_activity(
     // 0.4.x Phase 1: resolve the 5-state worker runtime state alongside the
     // legacy `activity` write. CR R3: activity field is preserved as the
     // deprecated surface; worker_state is the new canonical surface.
-    let worker_state =
-        resolve_worker_runtime_state_with_fg_pgrp(agent, Some(activity));
+    let worker_state = resolve_worker_runtime_state_with_fg_pgrp(agent, Some(activity));
     let Some(agent_obj) = agent.as_object_mut() else {
         return previous_last_output;
     };
@@ -3177,7 +2272,10 @@ pub(crate) fn resolve_worker_runtime_state_with_fg_pgrp(
     }
 
     // §4.3 Busy via fg-pgrp probe (when pane_pid is available).
-    let pane_pid = agent.get("pane_pid").and_then(Value::as_u64).map(|p| p as u32);
+    let pane_pid = agent
+        .get("pane_pid")
+        .and_then(Value::as_u64)
+        .map(|p| p as u32);
     if let Some(pid) = pane_pid {
         match crate::os_probe::pane_foreground_and_root_pgrp(pid) {
             Ok(Some((tpgid, pgid))) => {
@@ -3345,6 +2443,7 @@ fn notify_session_missing(
 #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
 mod u1_tests {
     use super::*;
+    use std::io::Write as _;
 
     struct CaptureFailureRegistry;