@team-agent/installer 0.3.21 → 0.3.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -1
- package/Cargo.toml +1 -1
- package/crates/team-agent/src/cli/emit.rs +9 -1
- package/crates/team-agent/src/cli/mod.rs +158 -2
- package/crates/team-agent/src/cli/status.rs +61 -0
- package/crates/team-agent/src/cli/status_port.rs +2 -2
- package/crates/team-agent/src/cli/tests/base.rs +26 -0
- package/crates/team-agent/src/cli/tests/shutdown_kill_plan.rs +233 -9
- package/crates/team-agent/src/coordinator/tests/energy.rs +548 -0
- package/crates/team-agent/src/coordinator/tests/mod.rs +1 -0
- package/crates/team-agent/src/coordinator/tick.rs +452 -17
- package/crates/team-agent/src/lifecycle/display.rs +23 -302
- package/crates/team-agent/src/lifecycle/launch.rs +38 -0
- package/crates/team-agent/src/lifecycle/restart/agent.rs +27 -6
- package/crates/team-agent/src/lifecycle/restart/common.rs +76 -0
- package/crates/team-agent/src/lifecycle/restart/rebuild.rs +295 -13
- package/crates/team-agent/src/lifecycle/tests/launch_spawn.rs +30 -0
- package/crates/team-agent/src/lifecycle/tests/main_preserved.rs +89 -5
- package/crates/team-agent/src/messaging/delivery.rs +324 -95
- package/crates/team-agent/src/messaging/scheduler.rs +95 -73
- package/crates/team-agent/src/messaging/send.rs +10 -0
- package/crates/team-agent/src/messaging/tests/runtime.rs +460 -0
- package/crates/team-agent/src/messaging/tests/spine.rs +51 -2
- package/crates/team-agent/src/session_capture.rs +261 -1
- package/crates/team-agent/src/tmux_backend/tests.rs +97 -1
- package/crates/team-agent/src/tmux_backend.rs +138 -2
- package/crates/team-agent/src/transport/test_support.rs +25 -0
- package/crates/team-agent/src/transport.rs +11 -0
- package/package.json +4 -4
|
@@ -39,84 +39,106 @@ pub fn fire_due_scheduled_events(
|
|
|
39
39
|
};
|
|
40
40
|
let mut fired = Vec::new();
|
|
41
41
|
for (id, kind, target, payload_json) in due_events {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
payload
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
.get("
|
|
54
|
-
|
|
55
|
-
.unwrap_or(
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
.get("attempt")
|
|
88
|
-
.and_then(|v| v.as_u64())
|
|
89
|
-
.and_then(|v| u8::try_from(v).ok())
|
|
90
|
-
.unwrap_or(1),
|
|
91
|
-
max_attempts: payload
|
|
92
|
-
.get("max_attempts")
|
|
93
|
-
.and_then(|v| v.as_u64())
|
|
94
|
-
.and_then(|v| u8::try_from(v).ok())
|
|
95
|
-
.unwrap_or(TRUST_RETRY_MAX_ATTEMPTS),
|
|
96
|
-
first_target: PaneId::new(
|
|
97
|
-
payload
|
|
98
|
-
.get("first_target")
|
|
42
|
+
// U1 #5: per-event isolation — one poison event (bad payload / fire error) must not
|
|
43
|
+
// halt the whole pass. The fire body is confined to a closure; on Err we emit
|
|
44
|
+
// `scheduler.event_failed`, mark the row terminal 'failed' (no re-fire), and continue.
|
|
45
|
+
let fire_one = || -> Result<serde_json::Value, MessagingError> {
|
|
46
|
+
let scheduled_kind = parse_scheduled_kind(&kind)?;
|
|
47
|
+
let result = match scheduled_kind {
|
|
48
|
+
ScheduledKind::Send => {
|
|
49
|
+
let payload: serde_json::Value = serde_json::from_str(&payload_json)?;
|
|
50
|
+
let outcome = deliver_stored_message(
|
|
51
|
+
workspace,
|
|
52
|
+
Some(&target),
|
|
53
|
+
payload.get("content").and_then(|v| v.as_str()).unwrap_or(""),
|
|
54
|
+
None,
|
|
55
|
+
payload.get("sender").and_then(|v| v.as_str()).unwrap_or("leader"),
|
|
56
|
+
payload
|
|
57
|
+
.get("requires_ack")
|
|
58
|
+
.and_then(|v| v.as_bool())
|
|
59
|
+
.unwrap_or(false),
|
|
60
|
+
payload
|
|
61
|
+
.get("wait_visible")
|
|
62
|
+
.and_then(|v| v.as_bool())
|
|
63
|
+
.unwrap_or(false),
|
|
64
|
+
payload.get("timeout").and_then(|v| v.as_f64()).unwrap_or(30.0),
|
|
65
|
+
None,
|
|
66
|
+
)?;
|
|
67
|
+
serde_json::json!({
|
|
68
|
+
"ok": outcome.ok,
|
|
69
|
+
"status": status_wire(outcome.status),
|
|
70
|
+
"message_id": outcome.message_id,
|
|
71
|
+
})
|
|
72
|
+
}
|
|
73
|
+
ScheduledKind::HealthPing => {
|
|
74
|
+
event_log.write(
|
|
75
|
+
"scheduled.health_ping",
|
|
76
|
+
serde_json::json!({"event_id": id, "target": target}),
|
|
77
|
+
)?;
|
|
78
|
+
serde_json::json!({"ok": true, "status": "logged"})
|
|
79
|
+
}
|
|
80
|
+
ScheduledKind::TrustRetry => {
|
|
81
|
+
let payload: serde_json::Value = serde_json::from_str(&payload_json)?;
|
|
82
|
+
let outcome = handle_trust_retry_needed(
|
|
83
|
+
store,
|
|
84
|
+
&TrustRetryPayload {
|
|
85
|
+
message_id: payload
|
|
86
|
+
.get("message_id")
|
|
99
87
|
.and_then(|v| v.as_str())
|
|
100
|
-
.
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
88
|
+
.unwrap_or_default()
|
|
89
|
+
.to_string(),
|
|
90
|
+
attempt: payload
|
|
91
|
+
.get("attempt")
|
|
92
|
+
.and_then(|v| v.as_u64())
|
|
93
|
+
.and_then(|v| u8::try_from(v).ok())
|
|
94
|
+
.unwrap_or(1),
|
|
95
|
+
max_attempts: payload
|
|
96
|
+
.get("max_attempts")
|
|
97
|
+
.and_then(|v| v.as_u64())
|
|
98
|
+
.and_then(|v| u8::try_from(v).ok())
|
|
99
|
+
.unwrap_or(TRUST_RETRY_MAX_ATTEMPTS),
|
|
100
|
+
first_target: PaneId::new(
|
|
101
|
+
payload
|
|
102
|
+
.get("first_target")
|
|
103
|
+
.and_then(|v| v.as_str())
|
|
104
|
+
.unwrap_or(""),
|
|
105
|
+
),
|
|
106
|
+
},
|
|
107
|
+
event_log,
|
|
108
|
+
)?;
|
|
109
|
+
serde_json::json!({"ok": outcome.ok, "status": status_wire(outcome.status)})
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
Ok(result)
|
|
107
113
|
};
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
114
|
+
|
|
115
|
+
let (status, result_json) = match fire_one() {
|
|
116
|
+
Ok(result) => {
|
|
117
|
+
let ok = result.get("ok").and_then(serde_json::Value::as_bool).unwrap_or(false);
|
|
118
|
+
(if ok { "done" } else { "failed" }, result.to_string())
|
|
119
|
+
}
|
|
120
|
+
Err(error) => {
|
|
121
|
+
// Poison event: loud + terminal, never halts the pass or re-fires.
|
|
122
|
+
let _ = event_log.write(
|
|
123
|
+
"scheduler.event_failed",
|
|
124
|
+
serde_json::json!({
|
|
125
|
+
"event_id": id,
|
|
126
|
+
"kind": kind,
|
|
127
|
+
"error": error.to_string(),
|
|
128
|
+
}),
|
|
129
|
+
);
|
|
130
|
+
(
|
|
131
|
+
"failed",
|
|
132
|
+
serde_json::json!({"ok": false, "status": "failed", "error": error.to_string()})
|
|
133
|
+
.to_string(),
|
|
134
|
+
)
|
|
135
|
+
}
|
|
116
136
|
};
|
|
137
|
+
// The row-status write stays `?`: a DB write failure is a real store fault, not a
|
|
138
|
+
// per-event poison, and should surface (consistent with the rest of the fn).
|
|
117
139
|
conn.execute(
|
|
118
140
|
"update scheduled_events set status = ?2, fired_at = ?3, result_json = ?4 where id = ?1",
|
|
119
|
-
params![id, status, chrono::Utc::now().to_rfc3339(),
|
|
141
|
+
params![id, status, chrono::Utc::now().to_rfc3339(), result_json],
|
|
120
142
|
)?;
|
|
121
143
|
fired.push(id);
|
|
122
144
|
}
|
|
@@ -351,6 +351,9 @@ fn backfill_leader_binding_for_delivery_view(
|
|
|
351
351
|
state: &mut serde_json::Value,
|
|
352
352
|
raw_state: &serde_json::Value,
|
|
353
353
|
) {
|
|
354
|
+
if !can_backfill_top_level_leader_binding(raw_state) {
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
354
357
|
let Some(obj) = state.as_object_mut() else {
|
|
355
358
|
return;
|
|
356
359
|
};
|
|
@@ -366,6 +369,13 @@ fn backfill_leader_binding_for_delivery_view(
|
|
|
366
369
|
}
|
|
367
370
|
}
|
|
368
371
|
|
|
372
|
+
fn can_backfill_top_level_leader_binding(raw_state: &serde_json::Value) -> bool {
|
|
373
|
+
raw_state
|
|
374
|
+
.get("teams")
|
|
375
|
+
.and_then(serde_json::Value::as_object)
|
|
376
|
+
.is_none_or(|teams| teams.len() <= 1)
|
|
377
|
+
}
|
|
378
|
+
|
|
369
379
|
fn send_owner_gate_refusal(
|
|
370
380
|
workspace: &Path,
|
|
371
381
|
state: &serde_json::Value,
|
|
@@ -1,4 +1,21 @@
|
|
|
1
1
|
use super::*;
|
|
2
|
+
use crate::transport::test_support::OfflineTransport;
|
|
3
|
+
|
|
4
|
+
fn pane_info(pane_id: &str, session: &str, window: &str) -> PaneInfo {
|
|
5
|
+
PaneInfo {
|
|
6
|
+
pane_id: PaneId::new(pane_id),
|
|
7
|
+
session: SessionName::new(session),
|
|
8
|
+
window_index: None,
|
|
9
|
+
window_name: Some(WindowName::new(window)),
|
|
10
|
+
pane_index: None,
|
|
11
|
+
tty: None,
|
|
12
|
+
current_command: None,
|
|
13
|
+
current_path: None,
|
|
14
|
+
active: true,
|
|
15
|
+
pane_pid: None,
|
|
16
|
+
leader_env: BTreeMap::new(),
|
|
17
|
+
}
|
|
18
|
+
}
|
|
2
19
|
|
|
3
20
|
// ════════════════════════════════════════════════════════════════════════
|
|
4
21
|
// GROUP E — _fail_leader_delivery: bug-52 fallback-log semantics. ok=True but
|
|
@@ -1336,6 +1353,202 @@ impl Transport for UnverifiedInjectTransport {
|
|
|
1336
1353
|
}
|
|
1337
1354
|
}
|
|
1338
1355
|
|
|
1356
|
+
struct FailingInjectTransport {
|
|
1357
|
+
attempts: std::sync::Arc<std::sync::atomic::AtomicUsize>,
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
impl Transport for FailingInjectTransport {
|
|
1361
|
+
fn kind(&self) -> BackendKind {
|
|
1362
|
+
BackendKind::Tmux
|
|
1363
|
+
}
|
|
1364
|
+
fn spawn_first(
|
|
1365
|
+
&self,
|
|
1366
|
+
_s: &SessionName,
|
|
1367
|
+
_w: &WindowName,
|
|
1368
|
+
_a: &[String],
|
|
1369
|
+
_c: &Path,
|
|
1370
|
+
_e: &BTreeMap<String, String>,
|
|
1371
|
+
) -> Result<SpawnResult, TransportError> {
|
|
1372
|
+
unimplemented!("not reached in delivery")
|
|
1373
|
+
}
|
|
1374
|
+
fn spawn_into(
|
|
1375
|
+
&self,
|
|
1376
|
+
_s: &SessionName,
|
|
1377
|
+
_w: &WindowName,
|
|
1378
|
+
_a: &[String],
|
|
1379
|
+
_c: &Path,
|
|
1380
|
+
_e: &BTreeMap<String, String>,
|
|
1381
|
+
) -> Result<SpawnResult, TransportError> {
|
|
1382
|
+
unimplemented!("not reached in delivery")
|
|
1383
|
+
}
|
|
1384
|
+
fn inject(
|
|
1385
|
+
&self,
|
|
1386
|
+
t: &Target,
|
|
1387
|
+
_p: &InjectPayload,
|
|
1388
|
+
_s: Key,
|
|
1389
|
+
_b: bool,
|
|
1390
|
+
) -> Result<InjectReport, TransportError> {
|
|
1391
|
+
if matches!(t, Target::Pane(pane) if pane.as_str() == "%1") {
|
|
1392
|
+
self.attempts
|
|
1393
|
+
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
|
1394
|
+
}
|
|
1395
|
+
Err(TransportError::Inject {
|
|
1396
|
+
stage: crate::transport::InjectStage::PasteBuffer,
|
|
1397
|
+
source: std::io::Error::other("paste-buffer failed"),
|
|
1398
|
+
})
|
|
1399
|
+
}
|
|
1400
|
+
fn send_keys(&self, _t: &Target, _k: &[Key]) -> Result<(), TransportError> {
|
|
1401
|
+
Ok(())
|
|
1402
|
+
}
|
|
1403
|
+
fn capture(&self, _t: &Target, range: CaptureRange) -> Result<CapturedText, TransportError> {
|
|
1404
|
+
Ok(CapturedText {
|
|
1405
|
+
text: String::new(),
|
|
1406
|
+
range,
|
|
1407
|
+
})
|
|
1408
|
+
}
|
|
1409
|
+
fn query(&self, _t: &Target, _f: PaneField) -> Result<Option<String>, TransportError> {
|
|
1410
|
+
Ok(None)
|
|
1411
|
+
}
|
|
1412
|
+
fn liveness(&self, _p: &PaneId) -> Result<PaneLiveness, TransportError> {
|
|
1413
|
+
Ok(PaneLiveness::Unknown)
|
|
1414
|
+
}
|
|
1415
|
+
fn list_targets(&self) -> Result<Vec<PaneInfo>, TransportError> {
|
|
1416
|
+
Ok(Vec::new())
|
|
1417
|
+
}
|
|
1418
|
+
fn has_session(&self, _s: &SessionName) -> Result<bool, TransportError> {
|
|
1419
|
+
Ok(true)
|
|
1420
|
+
}
|
|
1421
|
+
fn list_windows(&self, _s: &SessionName) -> Result<Vec<WindowName>, TransportError> {
|
|
1422
|
+
Ok(Vec::new())
|
|
1423
|
+
}
|
|
1424
|
+
fn set_session_env(
|
|
1425
|
+
&self,
|
|
1426
|
+
_s: &SessionName,
|
|
1427
|
+
_k: &str,
|
|
1428
|
+
_v: &str,
|
|
1429
|
+
) -> Result<SetEnvOutcome, TransportError> {
|
|
1430
|
+
Ok(SetEnvOutcome::Applied)
|
|
1431
|
+
}
|
|
1432
|
+
fn kill_session(&self, _s: &SessionName) -> Result<(), TransportError> {
|
|
1433
|
+
Ok(())
|
|
1434
|
+
}
|
|
1435
|
+
fn kill_window(&self, _t: &Target) -> Result<(), TransportError> {
|
|
1436
|
+
Ok(())
|
|
1437
|
+
}
|
|
1438
|
+
fn attach_session(&self, _s: &SessionName) -> Result<AttachOutcome, TransportError> {
|
|
1439
|
+
Ok(AttachOutcome::Attached)
|
|
1440
|
+
}
|
|
1441
|
+
}
|
|
1442
|
+
|
|
1443
|
+
// U1 #7 step-2 (RED→GREEN) — pane-readback gate: a token payload whose token was NOT
|
|
1444
|
+
// read back as visible in the pane (`InjectVerification::CaptureMissingToken`, surfaced by
|
|
1445
|
+
// the step-1 tmux_backend readback) must NOT be counted delivered, even when the SUBMIT
|
|
1446
|
+
// itself verified. Today the delivery gate only consulted submit_verification → a silent
|
|
1447
|
+
// paste-drop reported delivered (false green). This transport returns a verified submit
|
|
1448
|
+
// (KeySentAfterVisibleToken) but a missing-token readback.
|
|
1449
|
+
struct ReadbackMissingTransport;
|
|
1450
|
+
impl Transport for ReadbackMissingTransport {
|
|
1451
|
+
fn kind(&self) -> BackendKind {
|
|
1452
|
+
BackendKind::Tmux
|
|
1453
|
+
}
|
|
1454
|
+
fn spawn_first(&self, _: &SessionName, _: &WindowName, _: &[String], _: &Path, _: &BTreeMap<String, String>) -> Result<SpawnResult, TransportError> {
|
|
1455
|
+
unimplemented!("not reached in delivery")
|
|
1456
|
+
}
|
|
1457
|
+
fn spawn_into(&self, _: &SessionName, _: &WindowName, _: &[String], _: &Path, _: &BTreeMap<String, String>) -> Result<SpawnResult, TransportError> {
|
|
1458
|
+
unimplemented!("not reached in delivery")
|
|
1459
|
+
}
|
|
1460
|
+
fn inject(&self, _: &Target, _: &InjectPayload, _: Key, _: bool) -> Result<InjectReport, TransportError> {
|
|
1461
|
+
Ok(InjectReport {
|
|
1462
|
+
stage_reached: crate::transport::InjectStage::Submit,
|
|
1463
|
+
// submit verified, but the token never appeared in the pane → readback missing.
|
|
1464
|
+
inject_verification: crate::transport::InjectVerification::CaptureMissingToken,
|
|
1465
|
+
submit_verification: crate::transport::SubmitVerification::KeySentAfterVisibleToken {
|
|
1466
|
+
key: Key::Enter,
|
|
1467
|
+
},
|
|
1468
|
+
turn_verification: crate::transport::TurnVerification::NotYetObserved,
|
|
1469
|
+
attempts: 1,
|
|
1470
|
+
})
|
|
1471
|
+
}
|
|
1472
|
+
fn send_keys(&self, _: &Target, _: &[Key]) -> Result<(), TransportError> {
|
|
1473
|
+
Ok(())
|
|
1474
|
+
}
|
|
1475
|
+
fn capture(&self, _: &Target, range: CaptureRange) -> Result<CapturedText, TransportError> {
|
|
1476
|
+
Ok(CapturedText { text: String::new(), range })
|
|
1477
|
+
}
|
|
1478
|
+
fn query(&self, _: &Target, _: PaneField) -> Result<Option<String>, TransportError> {
|
|
1479
|
+
Ok(None)
|
|
1480
|
+
}
|
|
1481
|
+
fn liveness(&self, _: &PaneId) -> Result<PaneLiveness, TransportError> {
|
|
1482
|
+
Ok(PaneLiveness::Unknown)
|
|
1483
|
+
}
|
|
1484
|
+
fn list_targets(&self) -> Result<Vec<PaneInfo>, TransportError> {
|
|
1485
|
+
Ok(Vec::new())
|
|
1486
|
+
}
|
|
1487
|
+
fn has_session(&self, _: &SessionName) -> Result<bool, TransportError> {
|
|
1488
|
+
Ok(true)
|
|
1489
|
+
}
|
|
1490
|
+
fn list_windows(&self, _: &SessionName) -> Result<Vec<WindowName>, TransportError> {
|
|
1491
|
+
Ok(Vec::new())
|
|
1492
|
+
}
|
|
1493
|
+
fn set_session_env(&self, _: &SessionName, _: &str, _: &str) -> Result<SetEnvOutcome, TransportError> {
|
|
1494
|
+
Ok(SetEnvOutcome::Applied)
|
|
1495
|
+
}
|
|
1496
|
+
fn kill_session(&self, _: &SessionName) -> Result<(), TransportError> {
|
|
1497
|
+
Ok(())
|
|
1498
|
+
}
|
|
1499
|
+
fn kill_window(&self, _: &Target) -> Result<(), TransportError> {
|
|
1500
|
+
Ok(())
|
|
1501
|
+
}
|
|
1502
|
+
fn attach_session(&self, _: &SessionName) -> Result<AttachOutcome, TransportError> {
|
|
1503
|
+
Ok(AttachOutcome::Attached)
|
|
1504
|
+
}
|
|
1505
|
+
}
|
|
1506
|
+
|
|
1507
|
+
#[test]
|
|
1508
|
+
fn deliver_pending_readback_missing_token_is_not_delivered() {
|
|
1509
|
+
let ws = tmp_ws("readbackmiss");
|
|
1510
|
+
let store = store_for(&ws);
|
|
1511
|
+
let log = EventLog::new(&ws);
|
|
1512
|
+
let state = serde_json::json!({
|
|
1513
|
+
"session_name": "team-readbackmiss",
|
|
1514
|
+
"leader_receiver": {"pane_id": "%leader"},
|
|
1515
|
+
"agents": {"w1": {"provider": "fake", "pane_id": "%1"}}
|
|
1516
|
+
});
|
|
1517
|
+
crate::state::persist::save_runtime_state(&ws, &state).unwrap();
|
|
1518
|
+
let message_id = store
|
|
1519
|
+
.create_message(None, "leader", "w1", "ping", None, false, None)
|
|
1520
|
+
.unwrap();
|
|
1521
|
+
|
|
1522
|
+
let out = deliver_pending_message(
|
|
1523
|
+
&ws,
|
|
1524
|
+
&store,
|
|
1525
|
+
&ReadbackMissingTransport,
|
|
1526
|
+
&message_id,
|
|
1527
|
+
&log,
|
|
1528
|
+
&state,
|
|
1529
|
+
)
|
|
1530
|
+
.unwrap();
|
|
1531
|
+
|
|
1532
|
+
assert!(
|
|
1533
|
+
!out.ok,
|
|
1534
|
+
"U1 #7: a submit-verified inject whose token was NOT read back (CaptureMissingToken) \
|
|
1535
|
+
must NOT be delivered — readback gate catches the silent paste drop"
|
|
1536
|
+
);
|
|
1537
|
+
assert_ne!(
|
|
1538
|
+
out.message_status.0, "delivered",
|
|
1539
|
+
"U1 #7: readback-missing must not mark the message delivered (false green)"
|
|
1540
|
+
);
|
|
1541
|
+
let events = log.tail(0).unwrap();
|
|
1542
|
+
assert!(
|
|
1543
|
+
events.iter().any(|event| event
|
|
1544
|
+
.get("reason")
|
|
1545
|
+
.and_then(serde_json::Value::as_str)
|
|
1546
|
+
.map(|r| r.contains("pane_readback_unverified"))
|
|
1547
|
+
.unwrap_or(false)),
|
|
1548
|
+
"U1 #7: the failure reason must name pane_readback_unverified (loud, not silent); got {events:?}"
|
|
1549
|
+
);
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1339
1552
|
#[test]
|
|
1340
1553
|
fn deliver_pending_exhausted_unverified_send_emits_failed_event() {
|
|
1341
1554
|
let ws = tmp_ws("sendfailed");
|
|
@@ -1379,6 +1592,253 @@ fn deliver_pending_exhausted_unverified_send_emits_failed_event() {
|
|
|
1379
1592
|
);
|
|
1380
1593
|
}
|
|
1381
1594
|
|
|
1595
|
+
#[test]
|
|
1596
|
+
fn slice1_inject_failures_are_bounded_and_stop_retrying_same_message() {
|
|
1597
|
+
let ws = tmp_ws("injectstorm");
|
|
1598
|
+
let store = store_for(&ws);
|
|
1599
|
+
let log = EventLog::new(&ws);
|
|
1600
|
+
let state = serde_json::json!({
|
|
1601
|
+
"session_name": "team-injectstorm",
|
|
1602
|
+
"leader_receiver": {"pane_id": "%leader"},
|
|
1603
|
+
"agents": {"w1": {"provider": "fake", "pane_id": "%1"}}
|
|
1604
|
+
});
|
|
1605
|
+
crate::state::persist::save_runtime_state(&ws, &state).unwrap();
|
|
1606
|
+
let message_id = store
|
|
1607
|
+
.create_message(None, "leader", "w1", "ping", None, false, None)
|
|
1608
|
+
.unwrap();
|
|
1609
|
+
let attempts = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
|
1610
|
+
let transport = FailingInjectTransport {
|
|
1611
|
+
attempts: std::sync::Arc::clone(&attempts),
|
|
1612
|
+
};
|
|
1613
|
+
|
|
1614
|
+
for _ in 0..(usize::from(SEND_RETRY_MAX_ATTEMPTS) + 2) {
|
|
1615
|
+
let _ = deliver_pending_messages(&ws, &state, &transport, &log).unwrap();
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1618
|
+
assert_eq!(
|
|
1619
|
+
attempts.load(std::sync::atomic::Ordering::Relaxed),
|
|
1620
|
+
usize::from(SEND_RETRY_MAX_ATTEMPTS),
|
|
1621
|
+
"same message/target must stop retrying after the bounded inject-failure threshold"
|
|
1622
|
+
);
|
|
1623
|
+
let conn = crate::db::schema::open_db(store.db_path()).unwrap();
|
|
1624
|
+
let (status, delivery_attempts, error): (String, i64, Option<String>) = conn
|
|
1625
|
+
.query_row(
|
|
1626
|
+
"select status, delivery_attempts, error from messages where message_id = ?1",
|
|
1627
|
+
[&message_id],
|
|
1628
|
+
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
|
|
1629
|
+
)
|
|
1630
|
+
.unwrap();
|
|
1631
|
+
assert_eq!(status, "failed");
|
|
1632
|
+
assert_eq!(delivery_attempts, i64::from(SEND_RETRY_MAX_ATTEMPTS));
|
|
1633
|
+
assert_eq!(error.as_deref(), Some("send_inject_exhausted"));
|
|
1634
|
+
let events = log.tail(0).unwrap();
|
|
1635
|
+
let inject_failures = events
|
|
1636
|
+
.iter()
|
|
1637
|
+
.filter(|event| {
|
|
1638
|
+
event.get("event").and_then(serde_json::Value::as_str) == Some("send.inject_failed")
|
|
1639
|
+
})
|
|
1640
|
+
.count();
|
|
1641
|
+
assert_eq!(
|
|
1642
|
+
inject_failures,
|
|
1643
|
+
usize::from(SEND_RETRY_MAX_ATTEMPTS),
|
|
1644
|
+
"failed physical injects should be audited only until the bounded threshold; events={events:?}"
|
|
1645
|
+
);
|
|
1646
|
+
assert!(
|
|
1647
|
+
events.iter().any(|event| {
|
|
1648
|
+
event.get("event").and_then(serde_json::Value::as_str) == Some("send.failed")
|
|
1649
|
+
&& event.get("reason").and_then(serde_json::Value::as_str)
|
|
1650
|
+
== Some("send_inject_exhausted")
|
|
1651
|
+
}),
|
|
1652
|
+
"terminal inject exhaustion must emit send.failed; events={events:?}"
|
|
1653
|
+
);
|
|
1654
|
+
}
|
|
1655
|
+
|
|
1656
|
+
#[test]
|
|
1657
|
+
fn u1_projection_refused_emits_delivery_event() {
|
|
1658
|
+
let ws = tmp_ws("u1projref");
|
|
1659
|
+
let store = store_for(&ws);
|
|
1660
|
+
let log = EventLog::new(&ws);
|
|
1661
|
+
let state = serde_json::json!({
|
|
1662
|
+
"session_name": "team-a",
|
|
1663
|
+
"active_team_key": "team-a",
|
|
1664
|
+
"teams": {
|
|
1665
|
+
"team-a": {
|
|
1666
|
+
"session_name": "team-a",
|
|
1667
|
+
"agents": {"w1": {"provider": "fake", "window": "w1"}}
|
|
1668
|
+
}
|
|
1669
|
+
},
|
|
1670
|
+
"agents": {"w1": {"provider": "fake", "window": "w1"}}
|
|
1671
|
+
});
|
|
1672
|
+
crate::state::persist::save_runtime_state(&ws, &state).unwrap();
|
|
1673
|
+
let message_id = store
|
|
1674
|
+
.create_message(
|
|
1675
|
+
None,
|
|
1676
|
+
"leader",
|
|
1677
|
+
"w1",
|
|
1678
|
+
"wrong team",
|
|
1679
|
+
None,
|
|
1680
|
+
false,
|
|
1681
|
+
Some("missing-team"),
|
|
1682
|
+
)
|
|
1683
|
+
.unwrap();
|
|
1684
|
+
|
|
1685
|
+
let delivered = deliver_pending_messages(&ws, &state, &OfflineTransport::new(), &log)
|
|
1686
|
+
.expect("projection refusal must not abort the delivery batch");
|
|
1687
|
+
|
|
1688
|
+
assert!(
|
|
1689
|
+
delivered.is_empty(),
|
|
1690
|
+
"the refused owner-team row must not be delivered"
|
|
1691
|
+
);
|
|
1692
|
+
let events = log.tail(0).unwrap();
|
|
1693
|
+
assert!(
|
|
1694
|
+
events.iter().any(|event| {
|
|
1695
|
+
event.get("event").and_then(serde_json::Value::as_str)
|
|
1696
|
+
== Some("delivery.projection_refused")
|
|
1697
|
+
&& event.get("message_id").and_then(serde_json::Value::as_str)
|
|
1698
|
+
== Some(message_id.as_str())
|
|
1699
|
+
&& event
|
|
1700
|
+
.get("owner_team_id")
|
|
1701
|
+
.and_then(serde_json::Value::as_str)
|
|
1702
|
+
== Some("missing-team")
|
|
1703
|
+
}),
|
|
1704
|
+
"OwnerTeamProjection::Refused must leave a delivery.projection_refused audit event; events={events:?}"
|
|
1705
|
+
);
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
#[test]
|
|
1709
|
+
fn u1_deliver_item_error_does_not_halt_remaining_batch() {
|
|
1710
|
+
let ws = tmp_ws("u1itemblock");
|
|
1711
|
+
let store = store_for(&ws);
|
|
1712
|
+
let log = EventLog::new(&ws);
|
|
1713
|
+
let state = serde_json::json!({
|
|
1714
|
+
"session_name": "team-u1itemblock",
|
|
1715
|
+
"agents": {
|
|
1716
|
+
"w1": {"provider": "fake", "window": "w1"},
|
|
1717
|
+
"w2": {"provider": "fake", "window": "w2"}
|
|
1718
|
+
}
|
|
1719
|
+
});
|
|
1720
|
+
crate::state::persist::save_runtime_state(&ws, &state).unwrap();
|
|
1721
|
+
let blocked = store
|
|
1722
|
+
.create_message(None, "leader", "w1", "poison", None, false, None)
|
|
1723
|
+
.unwrap();
|
|
1724
|
+
let delivered_id = store
|
|
1725
|
+
.create_message(None, "peer", "w2", "still deliver", None, false, None)
|
|
1726
|
+
.unwrap();
|
|
1727
|
+
|
|
1728
|
+
let state_path = crate::state::persist::runtime_state_path(&ws);
|
|
1729
|
+
std::fs::remove_file(&state_path).unwrap();
|
|
1730
|
+
std::fs::create_dir(&state_path).unwrap();
|
|
1731
|
+
let transport = OfflineTransport::new();
|
|
1732
|
+
|
|
1733
|
+
let delivered = deliver_pending_messages(&ws, &state, &transport, &log)
|
|
1734
|
+
.expect("one item error must not halt the whole pending batch");
|
|
1735
|
+
|
|
1736
|
+
assert_eq!(
|
|
1737
|
+
delivered,
|
|
1738
|
+
vec![delivered_id.clone()],
|
|
1739
|
+
"a poison item must be isolated while later pending rows still deliver"
|
|
1740
|
+
);
|
|
1741
|
+
let events = log.tail(0).unwrap();
|
|
1742
|
+
assert!(
|
|
1743
|
+
events.iter().any(|event| {
|
|
1744
|
+
event.get("event").and_then(serde_json::Value::as_str) == Some("delivery.item_blocked")
|
|
1745
|
+
&& event.get("message_id").and_then(serde_json::Value::as_str)
|
|
1746
|
+
== Some(blocked.as_str())
|
|
1747
|
+
}),
|
|
1748
|
+
"the isolated poison item must emit delivery.item_blocked; events={events:?}"
|
|
1749
|
+
);
|
|
1750
|
+
}
|
|
1751
|
+
|
|
1752
|
+
#[test]
|
|
1753
|
+
fn u1_resolve_inject_target_uses_live_pane_when_cached_pane_drifted() {
|
|
1754
|
+
let ws = tmp_ws("u1drift");
|
|
1755
|
+
let store = store_for(&ws);
|
|
1756
|
+
let log = EventLog::new(&ws);
|
|
1757
|
+
let state = serde_json::json!({
|
|
1758
|
+
"session_name": "team-u1drift",
|
|
1759
|
+
"agents": {
|
|
1760
|
+
"w1": {
|
|
1761
|
+
"provider": "fake",
|
|
1762
|
+
"pane_id": "%stale",
|
|
1763
|
+
"window": "w1"
|
|
1764
|
+
}
|
|
1765
|
+
}
|
|
1766
|
+
});
|
|
1767
|
+
crate::state::persist::save_runtime_state(&ws, &state).unwrap();
|
|
1768
|
+
let _ = store
|
|
1769
|
+
.create_message(None, "peer", "w1", "drift-safe", None, false, None)
|
|
1770
|
+
.unwrap();
|
|
1771
|
+
let transport =
|
|
1772
|
+
OfflineTransport::new().with_targets(vec![pane_info("%live", "team-u1drift", "w1")]);
|
|
1773
|
+
|
|
1774
|
+
let delivered = deliver_pending_messages(&ws, &state, &transport, &log)
|
|
1775
|
+
.expect("live target re-resolution must not fail");
|
|
1776
|
+
|
|
1777
|
+
assert_eq!(delivered.len(), 1);
|
|
1778
|
+
assert_eq!(
|
|
1779
|
+
transport.inject_targets(),
|
|
1780
|
+
vec![Target::Pane(PaneId::new("%live"))],
|
|
1781
|
+
"delivery must validate the cached pane against live targets and re-resolve by session/window when it drifted"
|
|
1782
|
+
);
|
|
1783
|
+
}
|
|
1784
|
+
|
|
1785
|
+
#[test]
|
|
1786
|
+
fn u1_multi_team_send_does_not_backfill_top_level_leader_binding() {
|
|
1787
|
+
let ws = tmp_ws("u1backfill");
|
|
1788
|
+
crate::state::persist::save_runtime_state(
|
|
1789
|
+
&ws,
|
|
1790
|
+
&serde_json::json!({
|
|
1791
|
+
"session_name": "team-a",
|
|
1792
|
+
"active_team_key": "team-a",
|
|
1793
|
+
"leader_receiver": {"status": "unbound", "pane_id": "__team_agent_unbound__"},
|
|
1794
|
+
"team_owner": {"pane_id": "__team_agent_unbound__"},
|
|
1795
|
+
"teams": {
|
|
1796
|
+
"team-a": {
|
|
1797
|
+
"session_name": "team-a",
|
|
1798
|
+
"leader_receiver": {"status": "unbound", "pane_id": "__team_agent_unbound__"},
|
|
1799
|
+
"agents": {}
|
|
1800
|
+
},
|
|
1801
|
+
"team-b": {
|
|
1802
|
+
"session_name": "team-b",
|
|
1803
|
+
"agents": {}
|
|
1804
|
+
}
|
|
1805
|
+
},
|
|
1806
|
+
"agents": {}
|
|
1807
|
+
}),
|
|
1808
|
+
)
|
|
1809
|
+
.unwrap();
|
|
1810
|
+
let opts = SendOptions {
|
|
1811
|
+
team: Some(TeamKey::new("team-b")),
|
|
1812
|
+
requires_ack: false,
|
|
1813
|
+
..SendOptions::default()
|
|
1814
|
+
};
|
|
1815
|
+
|
|
1816
|
+
let out = send_message(
|
|
1817
|
+
&ws,
|
|
1818
|
+
&MessageTarget::Single("leader".to_string()),
|
|
1819
|
+
"hello team-b",
|
|
1820
|
+
&opts,
|
|
1821
|
+
)
|
|
1822
|
+
.unwrap();
|
|
1823
|
+
|
|
1824
|
+
assert_eq!(
|
|
1825
|
+
out.status,
|
|
1826
|
+
DeliveryStatus::Queued,
|
|
1827
|
+
"team-b leader send must not inherit team-a/top-level unbound leader binding"
|
|
1828
|
+
);
|
|
1829
|
+
let message_id = out.message_id.expect("queued leader message id");
|
|
1830
|
+
let store = store_for(&ws);
|
|
1831
|
+
let conn = crate::db::schema::open_db(store.db_path()).unwrap();
|
|
1832
|
+
let owner_team_id: Option<String> = conn
|
|
1833
|
+
.query_row(
|
|
1834
|
+
"select owner_team_id from messages where message_id = ?1",
|
|
1835
|
+
[&message_id],
|
|
1836
|
+
|row| row.get(0),
|
|
1837
|
+
)
|
|
1838
|
+
.unwrap();
|
|
1839
|
+
assert_eq!(owner_team_id.as_deref(), Some("team-b"));
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1382
1842
|
// ════════════════════════════════════════════════════════════════════════
|
|
1383
1843
|
// GROUP V — retry_result_deliveries: re-route notify_failed watchers with
|
|
1384
1844
|
// dedupe_reason rebind_retry. result_delivery.py:19-35.
|