@team-agent/installer 0.5.56 → 0.5.57

package/Cargo.lock

@@ -575,7 +575,7 @@ dependencies = [
 
 [[package]]
 name = "team-agent"
-version = "0.5.56"
+version = "0.5.57"
 dependencies = [
  "anyhow",
  "chrono",

package/Cargo.toml

@@ -9,7 +9,7 @@ members = ["crates/team-agent", "crates/win-conpty-phase0", "crates/conpty-trans
 
 [workspace.package]
 edition = "2021"
-version = "0.5.56"
+version = "0.5.57"
 license = "AGPL-3.0"
 rust-version = "1.95"
 

package/crates/team-agent/src/db/message_store.rs

@@ -52,8 +52,14 @@ pub enum MessageStoreError {
     Sqlite(#[from] rusqlite::Error),
     #[error("io: {0}")]
     Io(#[from] std::io::Error),
+    #[error("event log: {0}")]
+    EventLog(#[from] crate::event_log::EventLogError),
     #[error("delivery receipt missing for message: {0}")]
     DeliveryReceiptMissing(String),
+    #[error("invalid message transition from {from} to {to}")]
+    InvalidTransition { from: String, to: String },
+    #[error("submitted recovery refused: {0}")]
+    RecoveryRefused(&'static str),
 }
 
 /// Outcome of [`MessageStore::claim_leader_notification_delivery`]
@@ -88,19 +94,96 @@ pub enum MessageRowStatus {
     StoredOnly,
     QueuedUntilLeaderAttach,
     QueuedCoordinatorUnavailable,
+    QueuedPaneMissing,
+    TargetResolved,
+    SubmittedAwaitingReceipt,
+    SubmittedUnverified,
+    Delivered,
+    Acknowledged,
+    Consumed,
+    Failed,
+    BlockedLeaderUnbound,
+    BlockedWorkerPaneMissing,
 }
 
 impl MessageRowStatus {
+    pub const ALL: &[Self] = &[
+        Self::Accepted,
+        Self::StoredOnly,
+        Self::QueuedUntilLeaderAttach,
+        Self::QueuedCoordinatorUnavailable,
+        Self::QueuedPaneMissing,
+        Self::TargetResolved,
+        Self::SubmittedAwaitingReceipt,
+        Self::SubmittedUnverified,
+        Self::Delivered,
+        Self::Acknowledged,
+        Self::Consumed,
+        Self::Failed,
+        Self::BlockedLeaderUnbound,
+        Self::BlockedWorkerPaneMissing,
+    ];
+
     pub const fn as_str(self) -> &'static str {
         match self {
             Self::Accepted => "accepted",
             Self::StoredOnly => "stored_only",
             Self::QueuedUntilLeaderAttach => "queued_until_leader_attach",
             Self::QueuedCoordinatorUnavailable => "queued_coordinator_unavailable",
+            Self::QueuedPaneMissing => "queued_pane_missing",
+            Self::TargetResolved => "target_resolved",
+            Self::SubmittedAwaitingReceipt => "submitted_pending_acceptance",
+            Self::SubmittedUnverified => "submitted_unverified",
+            Self::Delivered => "delivered",
+            Self::Acknowledged => "acknowledged",
+            Self::Consumed => "consumed",
+            Self::Failed => "failed",
+            Self::BlockedLeaderUnbound => "blocked_leader_unbound",
+            Self::BlockedWorkerPaneMissing => "queued_pane_missing",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct BlockedLeaderRequeueCounts {
+    pub blocked_leader_unbound: usize,
+    pub queued_until_leader_attach: usize,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct RecoveryBudget {
+    pub attempt_budget: u32,
+    pub max_age_seconds: u64,
+    pub owner_epoch: i64,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct PaneAvailable {
+    pub agent_id: String,
+    pub pane_id: String,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum RedisplayKind {
+    FrameworkReplay,
+    ProviderRerender,
+}
+
+impl RedisplayKind {
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::FrameworkReplay => "framework_replay",
+            Self::ProviderRerender => "provider_rerender",
         }
     }
 }
 
+impl BlockedLeaderRequeueCounts {
+    pub const fn total(self) -> usize {
+        self.blocked_leader_unbound + self.queued_until_leader_attach
+    }
+}
+
 /// Fully resolved durable-message insert. Grammar, scope and transport data do
 /// not belong here; callers must resolve those before crossing this boundary.
 pub struct PersistMessageInput<'a> {
@@ -294,7 +377,7 @@ impl MessageStore {
     ) -> Result<(), MessageStoreError> {
         let conn = crate::db::schema::open_db(&self.path)?;
         let now = now_ts();
-        conn.execute(
+        let changed = conn.execute(
             "update messages
              set status = case
                  when status = 'acknowledged'
@@ -310,9 +393,55 @@ impl MessageStore {
              end,
              acknowledged_at = case when ?2 = 'acknowledged' then ?3 else acknowledged_at end,
              error = case when ?2 = 'delivered' then null else coalesce(?4, error) end
-             where message_id = ?1",
+             where message_id = ?1
+               and status not in ('acknowledged', 'consumed', 'submitted_pending_acceptance')
+               and (status != 'delivered' or ?2 in ('delivered', 'acknowledged'))
+               and (?2 != 'acknowledged' or status = 'delivered')",
             params![message_id, status, now, error],
         )?;
+        if changed == 0 {
+            let prior: Option<String> = conn
+                .query_row(
+                    "select status from messages where message_id = ?1",
+                    params![message_id],
+                    |row| row.get(0),
+                )
+                .optional()?;
+            let workspace = self
+                .path
+                .parent()
+                .and_then(Path::parent)
+                .and_then(Path::parent)
+                .unwrap_or(Path::new("."));
+            if status == MessageRowStatus::Acknowledged.as_str()
+                && prior.as_deref() != Some(MessageRowStatus::Delivered.as_str())
+            {
+                crate::event_log::EventLog::new(workspace).write(
+                    "message.mark_refused",
+                    serde_json::json!({
+                        "message_id": message_id,
+                        "prior_status": prior,
+                        "requested_status": status,
+                        "reason": "invalid_acknowledgement_transition",
+                    }),
+                )?;
+                return Err(MessageStoreError::InvalidTransition {
+                    from: prior.unwrap_or_else(|| "missing".to_string()),
+                    to: status.to_string(),
+                });
+            }
+            let event_log = crate::event_log::EventLog::new(workspace);
+            event_log.write(
+                "message.mark_refused",
+                serde_json::json!({
+                    "message_id": message_id,
+                    "prior_status": prior,
+                    "requested_status": status,
+                    "reason": "protected_prior_state",
+                }),
+            )?;
+            return Ok(());
+        }
         Ok(())
     }
 
@@ -397,6 +526,155 @@ impl MessageStore {
         Ok(rows == 1)
     }
 
+    /// Read inbox rows through [`Self::inbox`]; recovery transitions below are
+    /// separate repository owners and are not claim eligibility.
+    pub fn requeue_blocked_leader_messages(
+        &self,
+        owner_team_id: &str,
+    ) -> Result<BlockedLeaderRequeueCounts, MessageStoreError> {
+        let mut conn = crate::db::schema::open_db(&self.path)?;
+        let tx = conn.transaction()?;
+        let now = now_ts();
+        let blocked_leader_unbound = tx.execute(
+            "update messages
+             set status = 'accepted', error = null, updated_at = ?1
+             where recipient = 'leader'
+               and owner_team_id = ?2
+               and status = ?3
+               and error = 'leader_not_attached'",
+            params![now, owner_team_id, MessageRowStatus::Failed.as_str()],
+        )?;
+        let queued_until_leader_attach = tx.execute(
+            "update messages
+             set status = 'accepted', error = null, updated_at = ?1
+             where recipient = 'leader'
+               and owner_team_id = ?2
+               and status = ?3",
+            params![
+                now,
+                owner_team_id,
+                MessageRowStatus::QueuedUntilLeaderAttach.as_str()
+            ],
+        )?;
+        tx.commit()?;
+        Ok(BlockedLeaderRequeueCounts {
+            blocked_leader_unbound,
+            queued_until_leader_attach,
+        })
+    }
+
+    pub fn recover_worker_pane_available(
+        &self,
+        agent_id: &str,
+        owner_team_id: Option<&str>,
+    ) -> Result<Vec<String>, MessageStoreError> {
+        let mut conn = crate::db::schema::open_db(&self.path)?;
+        let tx = conn.transaction()?;
+        let ids = {
+            let mut stmt = tx.prepare(
+                "select message_id from messages
+                 where recipient = ?1
+                   and status = ?2
+                   and error = 'tmux_target_missing'
+                   and (
+                       (?3 is null and owner_team_id is null)
+                       or owner_team_id = ?3
+                   )
+                 order by created_at, message_id",
+            )?;
+            let rows = stmt
+                .query_map(
+                    params![
+                        agent_id,
+                        MessageRowStatus::QueuedPaneMissing.as_str(),
+                        owner_team_id
+                    ],
+                    |row| row.get::<_, String>(0),
+                )?
+                .collect::<Result<Vec<_>, _>>()?;
+            rows
+        };
+        let now = now_ts();
+        for message_id in &ids {
+            tx.execute(
+                "update messages
+                 set status = ?1, error = null, updated_at = ?2
+                 where message_id = ?3",
+                params![MessageRowStatus::Accepted.as_str(), now, message_id],
+            )?;
+        }
+        tx.commit()?;
+        Ok(ids)
+    }
+
+    pub fn retry_submitted_explicit(
+        &self,
+        message_id: &str,
+        incident_id: &str,
+        operator_reason: &str,
+        budget: RecoveryBudget,
+    ) -> Result<bool, MessageStoreError> {
+        if incident_id.trim().is_empty() || operator_reason.trim().is_empty() {
+            return Err(MessageStoreError::RecoveryRefused(
+                "incident_id_and_operator_reason_required",
+            ));
+        }
+        if budget.attempt_budget == 0 || budget.max_age_seconds == 0 || budget.owner_epoch <= 0 {
+            return Err(MessageStoreError::RecoveryRefused(
+                "invalid_recovery_budget",
+            ));
+        }
+        let conn = crate::db::schema::open_db(&self.path)?;
+        let row: Option<(String, i64, String)> = conn
+            .query_row(
+                "select status, delivery_attempts, updated_at
+                 from messages where message_id = ?1",
+                params![message_id],
+                |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
+            )
+            .optional()?;
+        let Some((status, attempts, updated_at)) = row else {
+            return Ok(false);
+        };
+        if status != MessageRowStatus::SubmittedAwaitingReceipt.as_str()
+            && status != MessageRowStatus::SubmittedUnverified.as_str()
+        {
+            return Err(MessageStoreError::RecoveryRefused(
+                "message_not_submitted_recoverable",
+            ));
+        }
+        if attempts >= i64::from(budget.attempt_budget) {
+            return Err(MessageStoreError::RecoveryRefused(
+                "attempt_budget_exhausted",
+            ));
+        }
+        let updated_at = chrono::DateTime::parse_from_rfc3339(&updated_at)
+            .map_err(|_| MessageStoreError::RecoveryRefused("invalid_updated_at"))?;
+        let age = chrono::Utc::now()
+            .signed_duration_since(updated_at.with_timezone(&chrono::Utc))
+            .num_seconds();
+        if age < 0 || u64::try_from(age).unwrap_or(u64::MAX) > budget.max_age_seconds {
+            return Err(MessageStoreError::RecoveryRefused("message_too_old"));
+        }
+        let changed = conn.execute(
+            "update messages
+             set status = ?1, error = ?2, updated_at = ?3
+             where message_id = ?4
+               and status in (?5, ?6)
+               and delivery_attempts < ?7",
+            params![
+                MessageRowStatus::Accepted.as_str(),
+                format!("explicit_recovery:{incident_id}:{operator_reason}"),
+                now_ts(),
+                message_id,
+                MessageRowStatus::SubmittedAwaitingReceipt.as_str(),
+                MessageRowStatus::SubmittedUnverified.as_str(),
+                budget.attempt_budget
+            ],
+        )?;
+        Ok(changed == 1)
+    }
+
     /// Read inbox rows for an agent. This projection intentionally has no owner-team
     /// filter when the caller does not provide one: legacy/CLI inbox must surface
     /// NULL-owner messages stored for the agent.
@@ -863,6 +1141,7 @@ mod tests {
         let mid = s
             .create_message(Some("t"), "s", "r", "c", None, true, None)
             .unwrap();
+        s.mark(&mid, "delivered", None).unwrap();
         s.mark(&mid, "acknowledged", None).unwrap();
         assert_eq!(status_of(&read(&s), &mid), "acknowledged");
         assert!(col_str(&read(&s), &mid, "acknowledged_at").is_some());
@@ -882,15 +1161,15 @@ mod tests {
     }
 
     #[test]
-    fn mark_acknowledged_then_failed_overwrites() {
-        // 'failed' is NOT in the guarded delivery set → it overwrites acknowledged.
+    fn mark_acknowledged_then_failed_stays_terminal() {
         let s = store();
         let mid = s
             .create_message(Some("t"), "s", "r", "c", None, true, None)
             .unwrap();
+        s.mark(&mid, "delivered", None).unwrap();
         s.mark(&mid, "acknowledged", None).unwrap();
         s.mark(&mid, "failed", Some("x")).unwrap();
-        assert_eq!(status_of(&read(&s), &mid), "failed");
+        assert_eq!(status_of(&read(&s), &mid), "acknowledged");
     }
 
     #[test]

package/crates/team-agent/src/leader/incident.rs

@@ -0,0 +1,5 @@
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum LeaderIncident {
+    LeaderAttached,
+    LeaderClaimed,
+}

package/crates/team-agent/src/leader/mod.rs

@@ -76,6 +76,7 @@ use crate::state::StateError;
 
 // ── submodules(by responsibility) ──────────────────────────────────────────
 mod helpers;
+pub mod incident;
 pub mod inject;
 pub mod lease;
 pub mod owner_bind;
@@ -87,6 +88,7 @@ pub mod takeover;
 pub mod types;
 
 // ── RE-EXPORT INVARIANT:每个先前 root-visible 项原路径不变 ────────────────────
+pub use incident::*;
 pub use inject::*;
 pub use lease::*;
 pub use owner_bind::*;

package/crates/team-agent/src/messaging/delivery.rs

@@ -848,35 +848,7 @@ pub fn requeue_worker_target_missing_messages(
     recipient: &str,
     owner_team_id: Option<&str>,
 ) -> Result<Vec<String>, MessagingError> {
-    let conn = crate::db::schema::open_db(store.db_path())?;
-    let now = chrono::Utc::now().to_rfc3339();
-    let mut stmt = conn.prepare(
-        "select message_id from messages
-         where recipient = ?1
-           and status = 'queued_pane_missing'
-           and error = 'tmux_target_missing'
-           and (
-               (?2 is null and owner_team_id is null)
-               or owner_team_id = ?2
-           )
-         order by created_at, message_id",
-    )?;
-    let ids = stmt
-        .query_map(params![recipient, owner_team_id], |row| {
-            row.get::<_, String>(0)
-        })?
-        .collect::<Result<Vec<_>, _>>()?;
-    drop(stmt);
-    for message_id in &ids {
-        conn.execute(
-            "update messages
-             set status = 'accepted',
-                 error = null,
-                 updated_at = ?2
-             where message_id = ?1",
-            params![message_id, now.as_str()],
-        )?;
-    }
+    let ids = store.recover_worker_pane_available(recipient, owner_team_id)?;
     if !ids.is_empty() {
         event_log.write(
             "worker_receiver.blocked_messages_requeued",
@@ -2446,10 +2418,8 @@ fn save_scoped_state(
     state: &serde_json::Value,
     owner_team_id: Option<&str>,
 ) -> Result<(), MessagingError> {
-    let scoped_owner_team_id = owner_team_id
-        .filter(|team| !team.is_empty())
-        .filter(|_| {
-            state
+    let scoped_owner_team_id = owner_team_id.filter(|team| !team.is_empty()).filter(|_| {
+        state
             .get("teams")
             .and_then(serde_json::Value::as_object)
             .is_some_and(|teams| {
@@ -2461,7 +2431,7 @@ fn save_scoped_state(
                     })
                     .is_some_and(|team| teams.contains_key(&team))
             })
-        });
+    });
     crate::state::repository::StateRepository::new(workspace).save(
         crate::state::repository::StateWriteIntent::MessagingDeliveryState {
             owner_team_id: scoped_owner_team_id,
@@ -2480,10 +2450,8 @@ fn save_scoped_state_reapplying_after_conflict<F>(
 where
     F: FnOnce(&mut serde_json::Value),
 {
-    let scoped_owner_team_id = owner_team_id
-        .filter(|team| !team.is_empty())
-        .filter(|_| {
-            state
+    let scoped_owner_team_id = owner_team_id.filter(|team| !team.is_empty()).filter(|_| {
+        state
             .get("teams")
             .and_then(serde_json::Value::as_object)
             .is_some_and(|teams| {
@@ -2495,7 +2463,7 @@ where
                     })
                     .is_some_and(|team| teams.contains_key(&team))
             })
-        });
+    });
     crate::state::repository::StateRepository::new(workspace).save_reapplying(
         crate::state::repository::StateWriteIntent::MessagingDeliveryState {
             owner_team_id: scoped_owner_team_id,

package/crates/team-agent/src/messaging/tests/main_preserved.rs

@@ -23,6 +23,7 @@ fn stuck_cancel_snapshot_delivered_message_ids_uses_golden_status_set() {
     let m_ack = store
         .create_message(None, "leader", "w1", "ack-me", None, true, Some("teamX"))
         .unwrap();
+    store.mark(&m_ack, "delivered", None).unwrap();
     store.mark(&m_ack, "acknowledged", None).unwrap();
     let m_vis = store
         .create_message(None, "leader", "w1", "vis", None, true, Some("teamX"))

package/crates/team-agent/src/messaging/watchers.rs

@@ -9,6 +9,12 @@ use crate::message_store::{MessageStore, NotificationClaimParams};
 use crate::model::ids::{TaskId, TeamKey};
 use crate::transport::PaneId;
 
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct RecoveryIncident {
+    pub incident_id: String,
+    pub occurred_at: String,
+}
+
 use super::{MessagingError, WatcherNotice, RESULT_DELIVERY_MAX_ATTEMPTS};
 
 /// `notify_result_watchers` (`result_delivery.py:38`):匹配 + 去重 + (有界) 投递 result 给 leader
@@ -460,13 +466,31 @@ pub fn requeue_after_claim_leader(
         )?;
     }
     let requeued_blocked =
-        requeue_blocked_leader_messages(&conn, event_log, owner_team_id, claimed_pane_id)?;
+        requeue_blocked_leader_messages(store, event_log, owner_team_id, claimed_pane_id)?;
     if !out.is_empty() || requeued_blocked > 0 {
         let _ = retry_result_deliveries(workspace, event_log)?;
     }
     Ok(out)
 }
 
+pub fn recover_watchers_for_incident(
+    workspace: &Path,
+    store: &MessageStore,
+    event_log: &EventLog,
+    owner_team_id: &TeamKey,
+    claimed_pane_id: &PaneId,
+    incident: &RecoveryIncident,
+) -> Result<Vec<WatcherNotice>, MessagingError> {
+    requeue_after_claim_leader(
+        workspace,
+        store,
+        event_log,
+        owner_team_id,
+        claimed_pane_id,
+        Some(&incident.occurred_at),
+    )
+}
+
 /// 0.5.5 gate054 round-2: attach-leader (and claim-leader) requeue for leader messages
 /// that were refused with `rebind_required` while no leader pane was attached.
 ///
@@ -477,7 +501,7 @@ pub fn requeue_after_claim_leader(
 /// already crossed the transport boundary and is claim-immutable. Any future
 /// retry belongs to a separate typed recovery arm, not attach/claim convergence.
 pub(crate) fn requeue_blocked_leader_messages(
-    conn: &rusqlite::Connection,
+    store: &MessageStore,
     event_log: &EventLog,
     owner_team_id: &TeamKey,
     claimed_pane_id: &PaneId,
@@ -492,19 +516,8 @@ pub(crate) fn requeue_blocked_leader_messages(
     // could not have churned while the leader was unattached.
     // `submitted_pending_acceptance` remains parked and claim-immutable; a
     // future typed recovery arm must own any deliberate retry.
-    let requeued = conn.execute(
-        "update messages
-         set status = 'accepted',
-             error = null,
-             updated_at = ?2
-         where recipient = 'leader'
-           and owner_team_id = ?1
-           and (
-             (status = 'failed' and error = 'leader_not_attached')
-             or status = 'queued_until_leader_attach'
-           )",
-        params![owner_team_id.as_str(), chrono::Utc::now().to_rfc3339()],
-    )?;
+    let counts = store.requeue_blocked_leader_messages(owner_team_id.as_str())?;
+    let requeued = counts.total();
     if requeued > 0 {
         event_log.write(
             "leader_receiver.blocked_messages_requeued",
@@ -512,6 +525,10 @@ pub(crate) fn requeue_blocked_leader_messages(
                 "team_id": owner_team_id.as_str(),
                 "claimed_pane_id": claimed_pane_id.as_str(),
                 "count": requeued,
+                "by_prior_state": {
+                    "blocked_leader_unbound": counts.blocked_leader_unbound,
+                    "queued_until_leader_attach": counts.queued_until_leader_attach,
+                },
             }),
         )?;
     }
@@ -573,7 +590,7 @@ pub fn requeue_delivery_exhausted_watchers(
         )?;
     }
     drop(stmt);
-    let _ = requeue_blocked_leader_messages(&conn, event_log, owner_team_id, claimed_pane_id)?;
+    let _ = requeue_blocked_leader_messages(store, event_log, owner_team_id, claimed_pane_id)?;
     Ok(out)
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@team-agent/installer",
-  "version": "0.5.56",
+  "version": "0.5.57",
   "description": "npx installer for Team Agent",
   "keywords": [
     "codex",
@@ -20,9 +20,9 @@
     "team-agent-installer": "npm/install.mjs"
   },
   "optionalDependencies": {
-    "@team-agent/cli-darwin-arm64": "0.5.56",
-    "@team-agent/cli-darwin-x64": "0.5.56",
-    "@team-agent/cli-linux-x64": "0.5.56"
+    "@team-agent/cli-darwin-arm64": "0.5.57",
+    "@team-agent/cli-darwin-x64": "0.5.57",
+    "@team-agent/cli-linux-x64": "0.5.57"
   },
   "scripts": {
     "postinstall": "node npm/bincheck.mjs",