@team-agent/installer 0.5.55 → 0.5.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -1
- package/Cargo.toml +1 -1
- package/crates/team-agent/src/db/message_store.rs +284 -5
- package/crates/team-agent/src/leader/incident.rs +5 -0
- package/crates/team-agent/src/leader/mod.rs +2 -0
- package/crates/team-agent/src/messaging/delivery.rs +7 -39
- package/crates/team-agent/src/messaging/tests/main_preserved.rs +1 -0
- package/crates/team-agent/src/messaging/tests/runtime.rs +11 -4
- package/crates/team-agent/src/messaging/watchers.rs +39 -25
- package/package.json +4 -4
package/Cargo.lock
CHANGED
package/Cargo.toml
CHANGED
|
@@ -52,8 +52,14 @@ pub enum MessageStoreError {
|
|
|
52
52
|
Sqlite(#[from] rusqlite::Error),
|
|
53
53
|
#[error("io: {0}")]
|
|
54
54
|
Io(#[from] std::io::Error),
|
|
55
|
+
#[error("event log: {0}")]
|
|
56
|
+
EventLog(#[from] crate::event_log::EventLogError),
|
|
55
57
|
#[error("delivery receipt missing for message: {0}")]
|
|
56
58
|
DeliveryReceiptMissing(String),
|
|
59
|
+
#[error("invalid message transition from {from} to {to}")]
|
|
60
|
+
InvalidTransition { from: String, to: String },
|
|
61
|
+
#[error("submitted recovery refused: {0}")]
|
|
62
|
+
RecoveryRefused(&'static str),
|
|
57
63
|
}
|
|
58
64
|
|
|
59
65
|
/// Outcome of [`MessageStore::claim_leader_notification_delivery`]
|
|
@@ -88,19 +94,96 @@ pub enum MessageRowStatus {
|
|
|
88
94
|
StoredOnly,
|
|
89
95
|
QueuedUntilLeaderAttach,
|
|
90
96
|
QueuedCoordinatorUnavailable,
|
|
97
|
+
QueuedPaneMissing,
|
|
98
|
+
TargetResolved,
|
|
99
|
+
SubmittedAwaitingReceipt,
|
|
100
|
+
SubmittedUnverified,
|
|
101
|
+
Delivered,
|
|
102
|
+
Acknowledged,
|
|
103
|
+
Consumed,
|
|
104
|
+
Failed,
|
|
105
|
+
BlockedLeaderUnbound,
|
|
106
|
+
BlockedWorkerPaneMissing,
|
|
91
107
|
}
|
|
92
108
|
|
|
93
109
|
impl MessageRowStatus {
|
|
110
|
+
pub const ALL: &[Self] = &[
|
|
111
|
+
Self::Accepted,
|
|
112
|
+
Self::StoredOnly,
|
|
113
|
+
Self::QueuedUntilLeaderAttach,
|
|
114
|
+
Self::QueuedCoordinatorUnavailable,
|
|
115
|
+
Self::QueuedPaneMissing,
|
|
116
|
+
Self::TargetResolved,
|
|
117
|
+
Self::SubmittedAwaitingReceipt,
|
|
118
|
+
Self::SubmittedUnverified,
|
|
119
|
+
Self::Delivered,
|
|
120
|
+
Self::Acknowledged,
|
|
121
|
+
Self::Consumed,
|
|
122
|
+
Self::Failed,
|
|
123
|
+
Self::BlockedLeaderUnbound,
|
|
124
|
+
Self::BlockedWorkerPaneMissing,
|
|
125
|
+
];
|
|
126
|
+
|
|
94
127
|
pub const fn as_str(self) -> &'static str {
|
|
95
128
|
match self {
|
|
96
129
|
Self::Accepted => "accepted",
|
|
97
130
|
Self::StoredOnly => "stored_only",
|
|
98
131
|
Self::QueuedUntilLeaderAttach => "queued_until_leader_attach",
|
|
99
132
|
Self::QueuedCoordinatorUnavailable => "queued_coordinator_unavailable",
|
|
133
|
+
Self::QueuedPaneMissing => "queued_pane_missing",
|
|
134
|
+
Self::TargetResolved => "target_resolved",
|
|
135
|
+
Self::SubmittedAwaitingReceipt => "submitted_pending_acceptance",
|
|
136
|
+
Self::SubmittedUnverified => "submitted_unverified",
|
|
137
|
+
Self::Delivered => "delivered",
|
|
138
|
+
Self::Acknowledged => "acknowledged",
|
|
139
|
+
Self::Consumed => "consumed",
|
|
140
|
+
Self::Failed => "failed",
|
|
141
|
+
Self::BlockedLeaderUnbound => "blocked_leader_unbound",
|
|
142
|
+
Self::BlockedWorkerPaneMissing => "queued_pane_missing",
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
|
148
|
+
pub struct BlockedLeaderRequeueCounts {
|
|
149
|
+
pub blocked_leader_unbound: usize,
|
|
150
|
+
pub queued_until_leader_attach: usize,
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
154
|
+
pub struct RecoveryBudget {
|
|
155
|
+
pub attempt_budget: u32,
|
|
156
|
+
pub max_age_seconds: u64,
|
|
157
|
+
pub owner_epoch: i64,
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
161
|
+
pub struct PaneAvailable {
|
|
162
|
+
pub agent_id: String,
|
|
163
|
+
pub pane_id: String,
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
167
|
+
pub enum RedisplayKind {
|
|
168
|
+
FrameworkReplay,
|
|
169
|
+
ProviderRerender,
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
impl RedisplayKind {
|
|
173
|
+
pub const fn as_str(self) -> &'static str {
|
|
174
|
+
match self {
|
|
175
|
+
Self::FrameworkReplay => "framework_replay",
|
|
176
|
+
Self::ProviderRerender => "provider_rerender",
|
|
100
177
|
}
|
|
101
178
|
}
|
|
102
179
|
}
|
|
103
180
|
|
|
181
|
+
impl BlockedLeaderRequeueCounts {
|
|
182
|
+
pub const fn total(self) -> usize {
|
|
183
|
+
self.blocked_leader_unbound + self.queued_until_leader_attach
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
104
187
|
/// Fully resolved durable-message insert. Grammar, scope and transport data do
|
|
105
188
|
/// not belong here; callers must resolve those before crossing this boundary.
|
|
106
189
|
pub struct PersistMessageInput<'a> {
|
|
@@ -294,7 +377,7 @@ impl MessageStore {
|
|
|
294
377
|
) -> Result<(), MessageStoreError> {
|
|
295
378
|
let conn = crate::db::schema::open_db(&self.path)?;
|
|
296
379
|
let now = now_ts();
|
|
297
|
-
conn.execute(
|
|
380
|
+
let changed = conn.execute(
|
|
298
381
|
"update messages
|
|
299
382
|
set status = case
|
|
300
383
|
when status = 'acknowledged'
|
|
@@ -310,9 +393,55 @@ impl MessageStore {
|
|
|
310
393
|
end,
|
|
311
394
|
acknowledged_at = case when ?2 = 'acknowledged' then ?3 else acknowledged_at end,
|
|
312
395
|
error = case when ?2 = 'delivered' then null else coalesce(?4, error) end
|
|
313
|
-
where message_id = ?1
|
|
396
|
+
where message_id = ?1
|
|
397
|
+
and status not in ('acknowledged', 'consumed', 'submitted_pending_acceptance')
|
|
398
|
+
and (status != 'delivered' or ?2 in ('delivered', 'acknowledged'))
|
|
399
|
+
and (?2 != 'acknowledged' or status = 'delivered')",
|
|
314
400
|
params![message_id, status, now, error],
|
|
315
401
|
)?;
|
|
402
|
+
if changed == 0 {
|
|
403
|
+
let prior: Option<String> = conn
|
|
404
|
+
.query_row(
|
|
405
|
+
"select status from messages where message_id = ?1",
|
|
406
|
+
params![message_id],
|
|
407
|
+
|row| row.get(0),
|
|
408
|
+
)
|
|
409
|
+
.optional()?;
|
|
410
|
+
let workspace = self
|
|
411
|
+
.path
|
|
412
|
+
.parent()
|
|
413
|
+
.and_then(Path::parent)
|
|
414
|
+
.and_then(Path::parent)
|
|
415
|
+
.unwrap_or(Path::new("."));
|
|
416
|
+
if status == MessageRowStatus::Acknowledged.as_str()
|
|
417
|
+
&& prior.as_deref() != Some(MessageRowStatus::Delivered.as_str())
|
|
418
|
+
{
|
|
419
|
+
crate::event_log::EventLog::new(workspace).write(
|
|
420
|
+
"message.mark_refused",
|
|
421
|
+
serde_json::json!({
|
|
422
|
+
"message_id": message_id,
|
|
423
|
+
"prior_status": prior,
|
|
424
|
+
"requested_status": status,
|
|
425
|
+
"reason": "invalid_acknowledgement_transition",
|
|
426
|
+
}),
|
|
427
|
+
)?;
|
|
428
|
+
return Err(MessageStoreError::InvalidTransition {
|
|
429
|
+
from: prior.unwrap_or_else(|| "missing".to_string()),
|
|
430
|
+
to: status.to_string(),
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
let event_log = crate::event_log::EventLog::new(workspace);
|
|
434
|
+
event_log.write(
|
|
435
|
+
"message.mark_refused",
|
|
436
|
+
serde_json::json!({
|
|
437
|
+
"message_id": message_id,
|
|
438
|
+
"prior_status": prior,
|
|
439
|
+
"requested_status": status,
|
|
440
|
+
"reason": "protected_prior_state",
|
|
441
|
+
}),
|
|
442
|
+
)?;
|
|
443
|
+
return Ok(());
|
|
444
|
+
}
|
|
316
445
|
Ok(())
|
|
317
446
|
}
|
|
318
447
|
|
|
@@ -397,6 +526,155 @@ impl MessageStore {
|
|
|
397
526
|
Ok(rows == 1)
|
|
398
527
|
}
|
|
399
528
|
|
|
529
|
+
/// Read inbox rows through [`Self::inbox`]; recovery transitions below are
|
|
530
|
+
/// separate repository owners and are not claim eligibility.
|
|
531
|
+
pub fn requeue_blocked_leader_messages(
|
|
532
|
+
&self,
|
|
533
|
+
owner_team_id: &str,
|
|
534
|
+
) -> Result<BlockedLeaderRequeueCounts, MessageStoreError> {
|
|
535
|
+
let mut conn = crate::db::schema::open_db(&self.path)?;
|
|
536
|
+
let tx = conn.transaction()?;
|
|
537
|
+
let now = now_ts();
|
|
538
|
+
let blocked_leader_unbound = tx.execute(
|
|
539
|
+
"update messages
|
|
540
|
+
set status = 'accepted', error = null, updated_at = ?1
|
|
541
|
+
where recipient = 'leader'
|
|
542
|
+
and owner_team_id = ?2
|
|
543
|
+
and status = ?3
|
|
544
|
+
and error = 'leader_not_attached'",
|
|
545
|
+
params![now, owner_team_id, MessageRowStatus::Failed.as_str()],
|
|
546
|
+
)?;
|
|
547
|
+
let queued_until_leader_attach = tx.execute(
|
|
548
|
+
"update messages
|
|
549
|
+
set status = 'accepted', error = null, updated_at = ?1
|
|
550
|
+
where recipient = 'leader'
|
|
551
|
+
and owner_team_id = ?2
|
|
552
|
+
and status = ?3",
|
|
553
|
+
params![
|
|
554
|
+
now,
|
|
555
|
+
owner_team_id,
|
|
556
|
+
MessageRowStatus::QueuedUntilLeaderAttach.as_str()
|
|
557
|
+
],
|
|
558
|
+
)?;
|
|
559
|
+
tx.commit()?;
|
|
560
|
+
Ok(BlockedLeaderRequeueCounts {
|
|
561
|
+
blocked_leader_unbound,
|
|
562
|
+
queued_until_leader_attach,
|
|
563
|
+
})
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
pub fn recover_worker_pane_available(
|
|
567
|
+
&self,
|
|
568
|
+
agent_id: &str,
|
|
569
|
+
owner_team_id: Option<&str>,
|
|
570
|
+
) -> Result<Vec<String>, MessageStoreError> {
|
|
571
|
+
let mut conn = crate::db::schema::open_db(&self.path)?;
|
|
572
|
+
let tx = conn.transaction()?;
|
|
573
|
+
let ids = {
|
|
574
|
+
let mut stmt = tx.prepare(
|
|
575
|
+
"select message_id from messages
|
|
576
|
+
where recipient = ?1
|
|
577
|
+
and status = ?2
|
|
578
|
+
and error = 'tmux_target_missing'
|
|
579
|
+
and (
|
|
580
|
+
(?3 is null and owner_team_id is null)
|
|
581
|
+
or owner_team_id = ?3
|
|
582
|
+
)
|
|
583
|
+
order by created_at, message_id",
|
|
584
|
+
)?;
|
|
585
|
+
let rows = stmt
|
|
586
|
+
.query_map(
|
|
587
|
+
params![
|
|
588
|
+
agent_id,
|
|
589
|
+
MessageRowStatus::QueuedPaneMissing.as_str(),
|
|
590
|
+
owner_team_id
|
|
591
|
+
],
|
|
592
|
+
|row| row.get::<_, String>(0),
|
|
593
|
+
)?
|
|
594
|
+
.collect::<Result<Vec<_>, _>>()?;
|
|
595
|
+
rows
|
|
596
|
+
};
|
|
597
|
+
let now = now_ts();
|
|
598
|
+
for message_id in &ids {
|
|
599
|
+
tx.execute(
|
|
600
|
+
"update messages
|
|
601
|
+
set status = ?1, error = null, updated_at = ?2
|
|
602
|
+
where message_id = ?3",
|
|
603
|
+
params![MessageRowStatus::Accepted.as_str(), now, message_id],
|
|
604
|
+
)?;
|
|
605
|
+
}
|
|
606
|
+
tx.commit()?;
|
|
607
|
+
Ok(ids)
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
pub fn retry_submitted_explicit(
|
|
611
|
+
&self,
|
|
612
|
+
message_id: &str,
|
|
613
|
+
incident_id: &str,
|
|
614
|
+
operator_reason: &str,
|
|
615
|
+
budget: RecoveryBudget,
|
|
616
|
+
) -> Result<bool, MessageStoreError> {
|
|
617
|
+
if incident_id.trim().is_empty() || operator_reason.trim().is_empty() {
|
|
618
|
+
return Err(MessageStoreError::RecoveryRefused(
|
|
619
|
+
"incident_id_and_operator_reason_required",
|
|
620
|
+
));
|
|
621
|
+
}
|
|
622
|
+
if budget.attempt_budget == 0 || budget.max_age_seconds == 0 || budget.owner_epoch <= 0 {
|
|
623
|
+
return Err(MessageStoreError::RecoveryRefused(
|
|
624
|
+
"invalid_recovery_budget",
|
|
625
|
+
));
|
|
626
|
+
}
|
|
627
|
+
let conn = crate::db::schema::open_db(&self.path)?;
|
|
628
|
+
let row: Option<(String, i64, String)> = conn
|
|
629
|
+
.query_row(
|
|
630
|
+
"select status, delivery_attempts, updated_at
|
|
631
|
+
from messages where message_id = ?1",
|
|
632
|
+
params![message_id],
|
|
633
|
+
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
|
|
634
|
+
)
|
|
635
|
+
.optional()?;
|
|
636
|
+
let Some((status, attempts, updated_at)) = row else {
|
|
637
|
+
return Ok(false);
|
|
638
|
+
};
|
|
639
|
+
if status != MessageRowStatus::SubmittedAwaitingReceipt.as_str()
|
|
640
|
+
&& status != MessageRowStatus::SubmittedUnverified.as_str()
|
|
641
|
+
{
|
|
642
|
+
return Err(MessageStoreError::RecoveryRefused(
|
|
643
|
+
"message_not_submitted_recoverable",
|
|
644
|
+
));
|
|
645
|
+
}
|
|
646
|
+
if attempts >= i64::from(budget.attempt_budget) {
|
|
647
|
+
return Err(MessageStoreError::RecoveryRefused(
|
|
648
|
+
"attempt_budget_exhausted",
|
|
649
|
+
));
|
|
650
|
+
}
|
|
651
|
+
let updated_at = chrono::DateTime::parse_from_rfc3339(&updated_at)
|
|
652
|
+
.map_err(|_| MessageStoreError::RecoveryRefused("invalid_updated_at"))?;
|
|
653
|
+
let age = chrono::Utc::now()
|
|
654
|
+
.signed_duration_since(updated_at.with_timezone(&chrono::Utc))
|
|
655
|
+
.num_seconds();
|
|
656
|
+
if age < 0 || u64::try_from(age).unwrap_or(u64::MAX) > budget.max_age_seconds {
|
|
657
|
+
return Err(MessageStoreError::RecoveryRefused("message_too_old"));
|
|
658
|
+
}
|
|
659
|
+
let changed = conn.execute(
|
|
660
|
+
"update messages
|
|
661
|
+
set status = ?1, error = ?2, updated_at = ?3
|
|
662
|
+
where message_id = ?4
|
|
663
|
+
and status in (?5, ?6)
|
|
664
|
+
and delivery_attempts < ?7",
|
|
665
|
+
params![
|
|
666
|
+
MessageRowStatus::Accepted.as_str(),
|
|
667
|
+
format!("explicit_recovery:{incident_id}:{operator_reason}"),
|
|
668
|
+
now_ts(),
|
|
669
|
+
message_id,
|
|
670
|
+
MessageRowStatus::SubmittedAwaitingReceipt.as_str(),
|
|
671
|
+
MessageRowStatus::SubmittedUnverified.as_str(),
|
|
672
|
+
budget.attempt_budget
|
|
673
|
+
],
|
|
674
|
+
)?;
|
|
675
|
+
Ok(changed == 1)
|
|
676
|
+
}
|
|
677
|
+
|
|
400
678
|
/// Read inbox rows for an agent. This projection intentionally has no owner-team
|
|
401
679
|
/// filter when the caller does not provide one: legacy/CLI inbox must surface
|
|
402
680
|
/// NULL-owner messages stored for the agent.
|
|
@@ -863,6 +1141,7 @@ mod tests {
|
|
|
863
1141
|
let mid = s
|
|
864
1142
|
.create_message(Some("t"), "s", "r", "c", None, true, None)
|
|
865
1143
|
.unwrap();
|
|
1144
|
+
s.mark(&mid, "delivered", None).unwrap();
|
|
866
1145
|
s.mark(&mid, "acknowledged", None).unwrap();
|
|
867
1146
|
assert_eq!(status_of(&read(&s), &mid), "acknowledged");
|
|
868
1147
|
assert!(col_str(&read(&s), &mid, "acknowledged_at").is_some());
|
|
@@ -882,15 +1161,15 @@ mod tests {
|
|
|
882
1161
|
}
|
|
883
1162
|
|
|
884
1163
|
#[test]
|
|
885
|
-
fn
|
|
886
|
-
// 'failed' is NOT in the guarded delivery set → it overwrites acknowledged.
|
|
1164
|
+
fn mark_acknowledged_then_failed_stays_terminal() {
|
|
887
1165
|
let s = store();
|
|
888
1166
|
let mid = s
|
|
889
1167
|
.create_message(Some("t"), "s", "r", "c", None, true, None)
|
|
890
1168
|
.unwrap();
|
|
1169
|
+
s.mark(&mid, "delivered", None).unwrap();
|
|
891
1170
|
s.mark(&mid, "acknowledged", None).unwrap();
|
|
892
1171
|
s.mark(&mid, "failed", Some("x")).unwrap();
|
|
893
|
-
assert_eq!(status_of(&read(&s), &mid), "
|
|
1172
|
+
assert_eq!(status_of(&read(&s), &mid), "acknowledged");
|
|
894
1173
|
}
|
|
895
1174
|
|
|
896
1175
|
#[test]
|
|
@@ -76,6 +76,7 @@ use crate::state::StateError;
|
|
|
76
76
|
|
|
77
77
|
// ── submodules(by responsibility) ──────────────────────────────────────────
|
|
78
78
|
mod helpers;
|
|
79
|
+
pub mod incident;
|
|
79
80
|
pub mod inject;
|
|
80
81
|
pub mod lease;
|
|
81
82
|
pub mod owner_bind;
|
|
@@ -87,6 +88,7 @@ pub mod takeover;
|
|
|
87
88
|
pub mod types;
|
|
88
89
|
|
|
89
90
|
// ── RE-EXPORT INVARIANT:每个先前 root-visible 项原路径不变 ────────────────────
|
|
91
|
+
pub use incident::*;
|
|
90
92
|
pub use inject::*;
|
|
91
93
|
pub use lease::*;
|
|
92
94
|
pub use owner_bind::*;
|
|
@@ -848,35 +848,7 @@ pub fn requeue_worker_target_missing_messages(
|
|
|
848
848
|
recipient: &str,
|
|
849
849
|
owner_team_id: Option<&str>,
|
|
850
850
|
) -> Result<Vec<String>, MessagingError> {
|
|
851
|
-
let
|
|
852
|
-
let now = chrono::Utc::now().to_rfc3339();
|
|
853
|
-
let mut stmt = conn.prepare(
|
|
854
|
-
"select message_id from messages
|
|
855
|
-
where recipient = ?1
|
|
856
|
-
and status = 'queued_pane_missing'
|
|
857
|
-
and error = 'tmux_target_missing'
|
|
858
|
-
and (
|
|
859
|
-
(?2 is null and owner_team_id is null)
|
|
860
|
-
or owner_team_id = ?2
|
|
861
|
-
)
|
|
862
|
-
order by created_at, message_id",
|
|
863
|
-
)?;
|
|
864
|
-
let ids = stmt
|
|
865
|
-
.query_map(params![recipient, owner_team_id], |row| {
|
|
866
|
-
row.get::<_, String>(0)
|
|
867
|
-
})?
|
|
868
|
-
.collect::<Result<Vec<_>, _>>()?;
|
|
869
|
-
drop(stmt);
|
|
870
|
-
for message_id in &ids {
|
|
871
|
-
conn.execute(
|
|
872
|
-
"update messages
|
|
873
|
-
set status = 'accepted',
|
|
874
|
-
error = null,
|
|
875
|
-
updated_at = ?2
|
|
876
|
-
where message_id = ?1",
|
|
877
|
-
params![message_id, now.as_str()],
|
|
878
|
-
)?;
|
|
879
|
-
}
|
|
851
|
+
let ids = store.recover_worker_pane_available(recipient, owner_team_id)?;
|
|
880
852
|
if !ids.is_empty() {
|
|
881
853
|
event_log.write(
|
|
882
854
|
"worker_receiver.blocked_messages_requeued",
|
|
@@ -2446,10 +2418,8 @@ fn save_scoped_state(
|
|
|
2446
2418
|
state: &serde_json::Value,
|
|
2447
2419
|
owner_team_id: Option<&str>,
|
|
2448
2420
|
) -> Result<(), MessagingError> {
|
|
2449
|
-
let scoped_owner_team_id = owner_team_id
|
|
2450
|
-
|
|
2451
|
-
.filter(|_| {
|
|
2452
|
-
state
|
|
2421
|
+
let scoped_owner_team_id = owner_team_id.filter(|team| !team.is_empty()).filter(|_| {
|
|
2422
|
+
state
|
|
2453
2423
|
.get("teams")
|
|
2454
2424
|
.and_then(serde_json::Value::as_object)
|
|
2455
2425
|
.is_some_and(|teams| {
|
|
@@ -2461,7 +2431,7 @@ fn save_scoped_state(
|
|
|
2461
2431
|
})
|
|
2462
2432
|
.is_some_and(|team| teams.contains_key(&team))
|
|
2463
2433
|
})
|
|
2464
|
-
|
|
2434
|
+
});
|
|
2465
2435
|
crate::state::repository::StateRepository::new(workspace).save(
|
|
2466
2436
|
crate::state::repository::StateWriteIntent::MessagingDeliveryState {
|
|
2467
2437
|
owner_team_id: scoped_owner_team_id,
|
|
@@ -2480,10 +2450,8 @@ fn save_scoped_state_reapplying_after_conflict<F>(
|
|
|
2480
2450
|
where
|
|
2481
2451
|
F: FnOnce(&mut serde_json::Value),
|
|
2482
2452
|
{
|
|
2483
|
-
let scoped_owner_team_id = owner_team_id
|
|
2484
|
-
|
|
2485
|
-
.filter(|_| {
|
|
2486
|
-
state
|
|
2453
|
+
let scoped_owner_team_id = owner_team_id.filter(|team| !team.is_empty()).filter(|_| {
|
|
2454
|
+
state
|
|
2487
2455
|
.get("teams")
|
|
2488
2456
|
.and_then(serde_json::Value::as_object)
|
|
2489
2457
|
.is_some_and(|teams| {
|
|
@@ -2495,7 +2463,7 @@ where
|
|
|
2495
2463
|
})
|
|
2496
2464
|
.is_some_and(|team| teams.contains_key(&team))
|
|
2497
2465
|
})
|
|
2498
|
-
|
|
2466
|
+
});
|
|
2499
2467
|
crate::state::repository::StateRepository::new(workspace).save_reapplying(
|
|
2500
2468
|
crate::state::repository::StateWriteIntent::MessagingDeliveryState {
|
|
2501
2469
|
owner_team_id: scoped_owner_team_id,
|
|
@@ -23,6 +23,7 @@ fn stuck_cancel_snapshot_delivered_message_ids_uses_golden_status_set() {
|
|
|
23
23
|
let m_ack = store
|
|
24
24
|
.create_message(None, "leader", "w1", "ack-me", None, true, Some("teamX"))
|
|
25
25
|
.unwrap();
|
|
26
|
+
store.mark(&m_ack, "delivered", None).unwrap();
|
|
26
27
|
store.mark(&m_ack, "acknowledged", None).unwrap();
|
|
27
28
|
let m_vis = store
|
|
28
29
|
.create_message(None, "leader", "w1", "vis", None, true, Some("teamX"))
|
|
@@ -771,7 +771,7 @@ fn requeue_after_claim_leader_skips_already_notified() {
|
|
|
771
771
|
// watchers, one already notified (notified_message_id set), one un-notified.
|
|
772
772
|
// requeue must return ONLY the un-notified watcher; the notified one is NOT
|
|
773
773
|
// requeued and its notified_message_id SURVIVES (clearing it would cause a
|
|
774
|
-
// second injection). Probed golden: requeued == [w_un] (result_id
|
|
774
|
+
// second injection). Probed golden: requeued == [w_un] (result_id present,
|
|
775
775
|
// prior_state "pending"); notified watcher keeps notified_message_id.
|
|
776
776
|
let ws = tmp_ws("requeue");
|
|
777
777
|
let store = store_for(&ws);
|
|
@@ -786,7 +786,7 @@ fn requeue_after_claim_leader_skips_already_notified() {
|
|
|
786
786
|
"t1",
|
|
787
787
|
"alice",
|
|
788
788
|
"pending",
|
|
789
|
-
|
|
789
|
+
Some("res_retryable"),
|
|
790
790
|
None,
|
|
791
791
|
);
|
|
792
792
|
let w_notified = seed_watcher(
|
|
@@ -3209,7 +3209,7 @@ fn gate054_rebind_replay_requeues_failed_leader_message_on_attach() {
|
|
|
3209
3209
|
}
|
|
3210
3210
|
|
|
3211
3211
|
#[test]
|
|
3212
|
-
fn
|
|
3212
|
+
fn claim_preserves_pending_acceptance_for_all_recipients() {
|
|
3213
3213
|
let ws = tmp_ws("claim-pending-acceptance");
|
|
3214
3214
|
let store = store_for(&ws);
|
|
3215
3215
|
let log = EventLog::new(&ws);
|
|
@@ -3263,7 +3263,14 @@ fn claim_requeues_pending_acceptance_for_leader_only() {
|
|
|
3263
3263
|
)
|
|
3264
3264
|
.unwrap()
|
|
3265
3265
|
};
|
|
3266
|
-
assert_eq!(
|
|
3266
|
+
assert_eq!(
|
|
3267
|
+
row(&leader_message),
|
|
3268
|
+
(
|
|
3269
|
+
"submitted_pending_acceptance".to_string(),
|
|
3270
|
+
None,
|
|
3271
|
+
None
|
|
3272
|
+
)
|
|
3273
|
+
);
|
|
3267
3274
|
assert_eq!(
|
|
3268
3275
|
row(&worker_message),
|
|
3269
3276
|
(
|
|
@@ -9,6 +9,12 @@ use crate::message_store::{MessageStore, NotificationClaimParams};
|
|
|
9
9
|
use crate::model::ids::{TaskId, TeamKey};
|
|
10
10
|
use crate::transport::PaneId;
|
|
11
11
|
|
|
12
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
13
|
+
pub struct RecoveryIncident {
|
|
14
|
+
pub incident_id: String,
|
|
15
|
+
pub occurred_at: String,
|
|
16
|
+
}
|
|
17
|
+
|
|
12
18
|
use super::{MessagingError, WatcherNotice, RESULT_DELIVERY_MAX_ATTEMPTS};
|
|
13
19
|
|
|
14
20
|
/// `notify_result_watchers` (`result_delivery.py:38`):匹配 + 去重 + (有界) 投递 result 给 leader
|
|
@@ -415,7 +421,7 @@ pub fn requeue_after_claim_leader(
|
|
|
415
421
|
let conn = crate::db::schema::open_db(store.db_path())?;
|
|
416
422
|
let mut stmt = conn.prepare(
|
|
417
423
|
"select watcher_id, result_id, status, coalesce(completed_at, created_at) from result_watchers
|
|
418
|
-
where owner_team_id = ?1 and notified_message_id is null
|
|
424
|
+
where owner_team_id = ?1 and result_id is not null and notified_message_id is null
|
|
419
425
|
order by created_at, watcher_id",
|
|
420
426
|
)?;
|
|
421
427
|
let rows = stmt.query_map(params![owner_team_id.as_str()], |row| {
|
|
@@ -460,13 +466,31 @@ pub fn requeue_after_claim_leader(
|
|
|
460
466
|
)?;
|
|
461
467
|
}
|
|
462
468
|
let requeued_blocked =
|
|
463
|
-
requeue_blocked_leader_messages(
|
|
469
|
+
requeue_blocked_leader_messages(store, event_log, owner_team_id, claimed_pane_id)?;
|
|
464
470
|
if !out.is_empty() || requeued_blocked > 0 {
|
|
465
471
|
let _ = retry_result_deliveries(workspace, event_log)?;
|
|
466
472
|
}
|
|
467
473
|
Ok(out)
|
|
468
474
|
}
|
|
469
475
|
|
|
476
|
+
pub fn recover_watchers_for_incident(
|
|
477
|
+
workspace: &Path,
|
|
478
|
+
store: &MessageStore,
|
|
479
|
+
event_log: &EventLog,
|
|
480
|
+
owner_team_id: &TeamKey,
|
|
481
|
+
claimed_pane_id: &PaneId,
|
|
482
|
+
incident: &RecoveryIncident,
|
|
483
|
+
) -> Result<Vec<WatcherNotice>, MessagingError> {
|
|
484
|
+
requeue_after_claim_leader(
|
|
485
|
+
workspace,
|
|
486
|
+
store,
|
|
487
|
+
event_log,
|
|
488
|
+
owner_team_id,
|
|
489
|
+
claimed_pane_id,
|
|
490
|
+
Some(&incident.occurred_at),
|
|
491
|
+
)
|
|
492
|
+
}
|
|
493
|
+
|
|
470
494
|
/// 0.5.5 gate054 round-2: attach-leader (and claim-leader) requeue for leader messages
|
|
471
495
|
/// that were refused with `rebind_required` while no leader pane was attached.
|
|
472
496
|
///
|
|
@@ -474,11 +498,10 @@ pub fn requeue_after_claim_leader(
|
|
|
474
498
|
/// to `accepted` so `deliver_pending_messages` replays it through the SAME
|
|
475
499
|
/// pipeline. The `leader_notification_log` PK prevents a second notification
|
|
476
500
|
/// row for watcher-backed messages. A `submitted_pending_acceptance` row has
|
|
477
|
-
/// already crossed the transport boundary
|
|
478
|
-
///
|
|
479
|
-
/// can repeat the physical submit if a same-pane claim races the receipt window.
|
|
501
|
+
/// already crossed the transport boundary and is claim-immutable. Any future
|
|
502
|
+
/// retry belongs to a separate typed recovery arm, not attach/claim convergence.
|
|
480
503
|
pub(crate) fn requeue_blocked_leader_messages(
|
|
481
|
-
|
|
504
|
+
store: &MessageStore,
|
|
482
505
|
event_log: &EventLog,
|
|
483
506
|
owner_team_id: &TeamKey,
|
|
484
507
|
claimed_pane_id: &PaneId,
|
|
@@ -490,24 +513,11 @@ pub(crate) fn requeue_blocked_leader_messages(
|
|
|
490
513
|
// `deliver_pending_messages` picks them up as `accepted` and injects
|
|
491
514
|
// exactly once. status `queued_until_leader_attach` is deliberately NOT
|
|
492
515
|
// in the `claim_for_delivery` eligible set (see message_store.rs) so it
|
|
493
|
-
// could not have churned while the leader was unattached.
|
|
494
|
-
// `submitted_pending_acceptance`
|
|
495
|
-
//
|
|
496
|
-
|
|
497
|
-
let requeued =
|
|
498
|
-
"update messages
|
|
499
|
-
set status = 'accepted',
|
|
500
|
-
error = null,
|
|
501
|
-
updated_at = ?2
|
|
502
|
-
where recipient = 'leader'
|
|
503
|
-
and owner_team_id = ?1
|
|
504
|
-
and (
|
|
505
|
-
(status = 'failed' and error = 'leader_not_attached')
|
|
506
|
-
or status = 'queued_until_leader_attach'
|
|
507
|
-
or status = 'submitted_pending_acceptance'
|
|
508
|
-
)",
|
|
509
|
-
params![owner_team_id.as_str(), chrono::Utc::now().to_rfc3339()],
|
|
510
|
-
)?;
|
|
516
|
+
// could not have churned while the leader was unattached.
|
|
517
|
+
// `submitted_pending_acceptance` remains parked and claim-immutable; a
|
|
518
|
+
// future typed recovery arm must own any deliberate retry.
|
|
519
|
+
let counts = store.requeue_blocked_leader_messages(owner_team_id.as_str())?;
|
|
520
|
+
let requeued = counts.total();
|
|
511
521
|
if requeued > 0 {
|
|
512
522
|
event_log.write(
|
|
513
523
|
"leader_receiver.blocked_messages_requeued",
|
|
@@ -515,6 +525,10 @@ pub(crate) fn requeue_blocked_leader_messages(
|
|
|
515
525
|
"team_id": owner_team_id.as_str(),
|
|
516
526
|
"claimed_pane_id": claimed_pane_id.as_str(),
|
|
517
527
|
"count": requeued,
|
|
528
|
+
"by_prior_state": {
|
|
529
|
+
"blocked_leader_unbound": counts.blocked_leader_unbound,
|
|
530
|
+
"queued_until_leader_attach": counts.queued_until_leader_attach,
|
|
531
|
+
},
|
|
518
532
|
}),
|
|
519
533
|
)?;
|
|
520
534
|
}
|
|
@@ -576,7 +590,7 @@ pub fn requeue_delivery_exhausted_watchers(
|
|
|
576
590
|
)?;
|
|
577
591
|
}
|
|
578
592
|
drop(stmt);
|
|
579
|
-
let _ = requeue_blocked_leader_messages(
|
|
593
|
+
let _ = requeue_blocked_leader_messages(store, event_log, owner_team_id, claimed_pane_id)?;
|
|
580
594
|
Ok(out)
|
|
581
595
|
}
|
|
582
596
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@team-agent/installer",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.57",
|
|
4
4
|
"description": "npx installer for Team Agent",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"codex",
|
|
@@ -20,9 +20,9 @@
|
|
|
20
20
|
"team-agent-installer": "npm/install.mjs"
|
|
21
21
|
},
|
|
22
22
|
"optionalDependencies": {
|
|
23
|
-
"@team-agent/cli-darwin-arm64": "0.5.
|
|
24
|
-
"@team-agent/cli-darwin-x64": "0.5.
|
|
25
|
-
"@team-agent/cli-linux-x64": "0.5.
|
|
23
|
+
"@team-agent/cli-darwin-arm64": "0.5.57",
|
|
24
|
+
"@team-agent/cli-darwin-x64": "0.5.57",
|
|
25
|
+
"@team-agent/cli-linux-x64": "0.5.57"
|
|
26
26
|
},
|
|
27
27
|
"scripts": {
|
|
28
28
|
"postinstall": "node npm/bincheck.mjs",
|