@team-agent/installer 0.3.9 → 0.3.11

package/crates/team-agent/src/messaging/delivery.rs

@@ -17,7 +17,7 @@ use crate::transport::{
 use super::helpers::{message_exists, MessageStatusShadow};
 use super::{
     DeliveryOutcome, DeliveryRefusal, DeliveryStage, DeliveryStatus, MessagingError,
-    PaneWidthQuery, TrustRetryPayload,
+    PaneWidthQuery, TrustRetryPayload, SEND_RETRY_MAX_ATTEMPTS,
 };
 use crate::state::projection::OwnerTeamResolution;
 
@@ -286,7 +286,6 @@ pub fn deliver_pending_message(
             "submit_unverified:{}",
             submit_verification_wire(inject_report.submit_verification)
         );
-        store.mark(message_id, "submitted_unverified", Some(&reason))?;
         event_log.write(
             "send.unverified",
             serde_json::json!({
@@ -296,6 +295,29 @@ pub fn deliver_pending_message(
                 "attempts": inject_report.attempts,
             }),
         )?;
+        if inject_report.attempts >= u32::from(SEND_RETRY_MAX_ATTEMPTS) {
+            store.mark(message_id, "failed", Some("send_unverified_exhausted"))?;
+            emit_send_failed_exhausted(
+                workspace,
+                state,
+                event_log,
+                message_id,
+                &message.recipient,
+                inject_report.attempts,
+                &reason,
+            )?;
+            return Ok(DeliveryOutcome {
+                ok: false,
+                status: DeliveryStatus::Failed,
+                message_status: MessageStatusShadow("failed".to_string()),
+                message_id: Some(message_id.to_string()),
+                verification: Some(reason),
+                stage: Some(DeliveryStage::Submit),
+                reason: None,
+                channel: None,
+            });
+        }
+        store.mark(message_id, "submitted_unverified", Some(&reason))?;
         return Ok(DeliveryOutcome {
             ok: false,
             status: DeliveryStatus::Failed,
@@ -538,6 +560,65 @@ fn leader_receiver_field_in_state<'a>(
         .filter(|value| !value.is_empty())
 }
 
+fn emit_send_failed_exhausted(
+    workspace: &Path,
+    state: &serde_json::Value,
+    event_log: &EventLog,
+    message_id: &str,
+    recipient: &str,
+    attempts: u32,
+    verification: &str,
+) -> Result<(), MessagingError> {
+    event_log.write(
+        "send.failed",
+        serde_json::json!({
+            "message_id": message_id,
+            "recipient": recipient,
+            "attempts": attempts,
+            "max_attempts": SEND_RETRY_MAX_ATTEMPTS,
+            "reason": "send_unverified_exhausted",
+            "verification": verification,
+        }),
+    )?;
+    let content = format!(
+        "send.failed\nerror: send to {recipient} remained unverified after {attempts}/{SEND_RETRY_MAX_ATTEMPTS} attempts\naction: inspect the target pane and retry the send\nlog: .team/logs/events.jsonl"
+    );
+    match crate::messaging::send_to_leader_receiver(
+        workspace,
+        state,
+        "leader",
+        &content,
+        None,
+        "coordinator",
+        false,
+        Some(&format!("send.failed:{message_id}")),
+        event_log,
+    ) {
+        Ok(outcome) => {
+            event_log.write(
+                "send.failed_notification",
+                serde_json::json!({
+                    "message_id": message_id,
+                    "recipient": recipient,
+                    "leader_notification_status": super::helpers::status_wire(outcome.status),
+                    "leader_message_id": outcome.message_id,
+                }),
+            )?;
+        }
+        Err(error) => {
+            event_log.write(
+                "send.failed_notification_failed",
+                serde_json::json!({
+                    "message_id": message_id,
+                    "recipient": recipient,
+                    "error": error.to_string(),
+                }),
+            )?;
+        }
+    }
+    Ok(())
+}
+
 fn active_team_entry(state: &serde_json::Value) -> Option<&serde_json::Value> {
     let team = state.get("active_team_key").and_then(serde_json::Value::as_str)?;
     state

package/crates/team-agent/src/messaging/helpers.rs

@@ -29,6 +29,7 @@ pub(crate) fn status_wire(status: DeliveryStatus) -> &'static str {
         DeliveryStatus::Queued => "queued",
         DeliveryStatus::Blocked => "blocked",
         DeliveryStatus::Refused => "refused",
+        DeliveryStatus::Degraded => "degraded",
         DeliveryStatus::RetryScheduled => "retry_scheduled",
         DeliveryStatus::TrustAutoAnswerExhausted => "trust_auto_answer_exhausted",
         DeliveryStatus::AlreadyDelivered => "already_delivered",
@@ -40,7 +41,10 @@ pub(crate) fn status_wire(status: DeliveryStatus) -> &'static str {
     }
 }
 
-pub(crate) fn message_exists(store: &MessageStore, message_id: &str) -> Result<bool, MessagingError> {
+pub(crate) fn message_exists(
+    store: &MessageStore,
+    message_id: &str,
+) -> Result<bool, MessagingError> {
     let conn = crate::db::schema::open_db(store.db_path())?;
     let found: Option<String> = conn
         .query_row(
@@ -65,7 +69,10 @@ pub(crate) fn next_run_id() -> String {
     id.chars().filter(|c| *c != '_').take(12).collect()
 }
 
-pub(crate) fn required_str<'a>(value: &'a serde_json::Value, key: &str) -> Result<&'a str, MessagingError> {
+pub(crate) fn required_str<'a>(
+    value: &'a serde_json::Value,
+    key: &str,
+) -> Result<&'a str, MessagingError> {
     value
         .get(key)
         .and_then(|v| v.as_str())
@@ -85,7 +92,9 @@ pub(crate) fn validate_result_envelope(envelope: &serde_json::Value) -> Result<(
     }
     for key in ["changes", "tests", "risks", "artifacts", "next_actions"] {
         if !envelope.get(key).is_some_and(serde_json::Value::is_array) {
-            return Err(MessagingError::Validation(format!("missing required array field: {key}")));
+            return Err(MessagingError::Validation(format!(
+                "missing required array field: {key}"
+            )));
         }
     }
     Ok(())
@@ -122,10 +131,12 @@ pub(crate) fn non_provider_command(command: &str) -> Option<&str> {
 pub(crate) fn latest_prompt_signal(scrollback: &str) -> Option<AgentActivity> {
     let lower = scrollback.to_ascii_lowercase();
     let idle_pos = latest_idle_prompt_pos(scrollback);
-    let working_pos = ["working", "thinking", "⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
-        .iter()
-        .filter_map(|needle| lower.rfind(needle))
-        .max();
+    let working_pos = [
+        "working", "thinking", "⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏",
+    ]
+    .iter()
+    .filter_map(|needle| lower.rfind(needle))
+    .max();
     match (idle_pos, working_pos) {
         (Some(i), Some(w)) if i > w => Some(idle_activity()),
         (Some(_), None) => Some(idle_activity()),
@@ -168,10 +179,19 @@ pub fn fail_leader_delivery(
     error: Option<&str>,
 ) -> Result<DeliveryOutcome, MessagingError> {
     let store = MessageStore::open(workspace)?;
-    let sender = payload.get("sender").and_then(serde_json::Value::as_str).unwrap_or("system");
-    let content = payload.get("content").and_then(serde_json::Value::as_str).unwrap_or("");
+    let sender = payload
+        .get("sender")
+        .and_then(serde_json::Value::as_str)
+        .unwrap_or("system");
+    let content = payload
+        .get("content")
+        .and_then(serde_json::Value::as_str)
+        .unwrap_or("");
     let task_id = payload.get("task_id").and_then(serde_json::Value::as_str);
-    let message_id = match payload.get("message_id").and_then(serde_json::Value::as_str) {
+    let message_id = match payload
+        .get("message_id")
+        .and_then(serde_json::Value::as_str)
+    {
         Some(existing) => existing.to_string(),
         None => store.create_message(task_id, sender, "leader", content, None, false, None)?,
     };

package/crates/team-agent/src/messaging/send.rs

@@ -2,9 +2,10 @@
 
 use std::path::Path;
 
+use crate::coordinator::{CoordinatorHealthStatus, WorkspacePath};
 use crate::event_log::EventLog;
-use crate::model::ids::{TaskId, TeamKey};
 use crate::model::enums::PaneLiveness;
+use crate::model::ids::{TaskId, TeamKey};
 use crate::transport::{PaneId, Transport};
 
 use super::helpers::{status_wire, MessageStatusShadow};
@@ -118,7 +119,15 @@ pub fn send_message(
         MessageTarget::Single(target) => target,
         MessageTarget::Broadcast => {
             let recipients = broadcast_recipients(&state, &opts.sender, opts.team.as_ref());
-            return fanout_send(workspace, &state, &recipients, content, opts, &event_log, "*");
+            return fanout_send(
+                workspace,
+                &state,
+                &recipients,
+                content,
+                opts,
+                &event_log,
+                "*",
+            );
         }
         MessageTarget::Fanout(recipients) if recipients.is_empty() => {
             // swallow batch 3 ②: a failed send carries its reason (Python send error
@@ -135,7 +144,9 @@ pub fn send_message(
             });
         }
         MessageTarget::Fanout(recipients) => {
-            return fanout_send(workspace, &state, recipients, content, opts, &event_log, "fanout");
+            return fanout_send(
+                workspace, &state, recipients, content, opts, &event_log, "fanout",
+            );
         }
     };
     // send.py:259-261 — a non-leader target that is NOT a known team agent is refused
@@ -170,6 +181,10 @@ pub fn send_message(
             }
         }
     }
+    if let Some(outcome) = coordinator_unavailable_outcome(workspace, recipient, opts, &event_log)?
+    {
+        return Ok(outcome);
+    }
     let store = crate::message_store::MessageStore::open(workspace)?;
     let task_id = opts.task_id.as_ref().map(|t| t.as_str());
     let owner_team_id = opts.team.as_ref().map(|t| t.as_str());
@@ -220,9 +235,9 @@ fn task_exists(state: &serde_json::Value, task_id: &TaskId) -> bool {
         .get("tasks")
         .and_then(serde_json::Value::as_array)
         .is_some_and(|tasks| {
-            tasks
-                .iter()
-                .any(|task| task.get("id").and_then(serde_json::Value::as_str) == Some(task_id.as_str()))
+            tasks.iter().any(|task| {
+                task.get("id").and_then(serde_json::Value::as_str) == Some(task_id.as_str())
+            })
         })
 }
 
@@ -259,6 +274,46 @@ fn refused_outcome_with_verification(
     }
 }
 
+fn coordinator_unavailable_outcome(
+    workspace: &Path,
+    recipient: &str,
+    opts: &SendOptions,
+    event_log: &EventLog,
+) -> Result<Option<DeliveryOutcome>, MessagingError> {
+    let coordinator_workspace = WorkspacePath::new(workspace.to_path_buf());
+    let health = crate::coordinator::coordinator_health(&coordinator_workspace);
+    if health.ok || matches!(health.status, CoordinatorHealthStatus::Missing) {
+        return Ok(None);
+    }
+    let warning = format!(
+        "coordinator is not running; message was not queued for {recipient}. Run `team-agent diagnose` or restart the team before sending again."
+    );
+    event_log.write(
+        "send.coordinator_unavailable",
+        serde_json::json!({
+            "recipient": recipient,
+            "sender": opts.sender,
+            "coordinator_status": health.status,
+            "coordinator_pid": health.pid.map(|pid| pid.get()),
+            "message_queued": false,
+            "warning": warning,
+            "coordinator_log": crate::coordinator::coordinator_log_path(&coordinator_workspace)
+                .display()
+                .to_string(),
+        }),
+    )?;
+    Ok(Some(DeliveryOutcome {
+        ok: false,
+        status: DeliveryStatus::Degraded,
+        message_status: MessageStatusShadow("degraded".to_string()),
+        message_id: None,
+        verification: Some(warning),
+        stage: None,
+        reason: Some(DeliveryRefusal::CoordinatorUnavailable),
+        channel: Some("coordinator_unavailable".to_string()),
+    }))
+}
+
 fn rebind_required_outcome(message_id: Option<String>) -> DeliveryOutcome {
     rebind_required_outcome_with_verification(
         message_id,
@@ -291,7 +346,10 @@ fn sender_is_leader(state: &serde_json::Value, sender: &str) -> bool {
     sender == leader_id || sender == "leader" || sender == "Leader"
 }
 
-fn backfill_leader_binding_for_delivery_view(state: &mut serde_json::Value, raw_state: &serde_json::Value) {
+fn backfill_leader_binding_for_delivery_view(
+    state: &mut serde_json::Value,
+    raw_state: &serde_json::Value,
+) {
     let Some(obj) = state.as_object_mut() else {
         return;
     };
@@ -329,7 +387,9 @@ fn send_owner_gate_refusal(
         None,
     )
     .map_err(|e| MessagingError::Routing(e.to_string()))?;
-    if let Some(refusal) = crate::state::owner_gate::check_team_owner(state, &caller, false, &LiveLiveness) {
+    if let Some(refusal) =
+        crate::state::owner_gate::check_team_owner(state, &caller, false, &LiveLiveness)
+    {
         if caller.pane_id.is_empty() {
             return Ok(Some(refused_outcome(DeliveryRefusal::NoCallerPane)));
         }
@@ -365,7 +425,8 @@ fn explicit_claim_applied(workspace: &Path, _team_key: &str, _caller_pane: &str)
         .iter()
         .rev()
         .any(|event| {
-            event.get("event").and_then(serde_json::Value::as_str) == Some("leader_receiver.rebind_applied")
+            event.get("event").and_then(serde_json::Value::as_str)
+                == Some("leader_receiver.rebind_applied")
         })
 }
 
@@ -514,11 +575,7 @@ fn broadcast_recipients(
                 .and_then(|t| t.get("agents"))
                 .and_then(serde_json::Value::as_object)
         })
-        .or_else(|| {
-            state
-                .get("agents")
-                .and_then(serde_json::Value::as_object)
-        });
+        .or_else(|| state.get("agents").and_then(serde_json::Value::as_object));
     if let Some(agents) = agents_obj {
         for (agent_id, _) in agents {
             if agent_id == sender {

package/crates/team-agent/src/messaging/tests/basic.rs

@@ -1,6 +1,5 @@
 use super::*;
 
-
 // ════════════════════════════════════════════════════════════════════════
 // GROUP A — serde byte-locks (audit/event wire values; changing one byte
 // breaks downstream recognizers/event consumers). delivery.py / send.py /
@@ -15,6 +14,7 @@ fn delivery_status_serde_snake_case_byte_locked() {
         (DeliveryStatus::Queued, "\"queued\""),
         (DeliveryStatus::Blocked, "\"blocked\""),
         (DeliveryStatus::Refused, "\"refused\""),
+        (DeliveryStatus::Degraded, "\"degraded\""),
         (DeliveryStatus::RetryScheduled, "\"retry_scheduled\""),
         (
             DeliveryStatus::TrustAutoAnswerExhausted,
@@ -22,7 +22,10 @@ fn delivery_status_serde_snake_case_byte_locked() {
         ),
         (DeliveryStatus::AlreadyDelivered, "\"already_delivered\""),
         (DeliveryStatus::FallbackLog, "\"fallback_log\""),
-        (DeliveryStatus::BroadcastDelivered, "\"broadcast_delivered\""),
+        (
+            DeliveryStatus::BroadcastDelivered,
+            "\"broadcast_delivered\"",
+        ),
         (DeliveryStatus::BroadcastPartial, "\"broadcast_partial\""),
         (DeliveryStatus::FanoutDelivered, "\"fanout_delivered\""),
         (DeliveryStatus::FanoutPartial, "\"fanout_partial\""),
@@ -40,16 +43,32 @@ fn delivery_refusal_serde_snake_case_byte_locked() {
             DeliveryRefusal::HumanConfirmationRequired,
             "\"human_confirmation_required\"",
         ),
-        (DeliveryRefusal::MissingPermissions, "\"missing_permissions\""),
+        (
+            DeliveryRefusal::MissingPermissions,
+            "\"missing_permissions\"",
+        ),
         (DeliveryRefusal::RecipientBusy, "\"recipient_busy\""),
         (DeliveryRefusal::UnknownRecipient, "\"unknown_recipient\""),
-        (DeliveryRefusal::TmuxTargetMissing, "\"tmux_target_missing\""),
+        (
+            DeliveryRefusal::TmuxTargetMissing,
+            "\"tmux_target_missing\"",
+        ),
         (
             DeliveryRefusal::MessageAlreadyClaimed,
             "\"message_already_claimed\"",
         ),
-        (DeliveryRefusal::LeaderNotAttached, "\"leader_not_attached\""),
-        (DeliveryRefusal::TeamOwnerMismatch, "\"team_owner_mismatch\""),
+        (
+            DeliveryRefusal::LeaderNotAttached,
+            "\"leader_not_attached\"",
+        ),
+        (
+            DeliveryRefusal::CoordinatorUnavailable,
+            "\"coordinator_unavailable\"",
+        ),
+        (
+            DeliveryRefusal::TeamOwnerMismatch,
+            "\"team_owner_mismatch\"",
+        ),
         (DeliveryRefusal::Ambiguous, "\"ambiguous\""),
         (
             DeliveryRefusal::RecipientPaneInNonInputMode,
@@ -354,4 +373,3 @@ fn result_id_from_text_strips_trailing_whitespace() {
         Some("abc".to_string())
     );
 }
-