@team-agent/installer 0.5.78 → 0.5.81

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/Cargo.lock +1 -1
  2. package/Cargo.toml +1 -1
  3. package/crates/team-agent/src/cli/models.rs +31 -0
  4. package/crates/team-agent/src/cli/status_port/brief.rs +40 -16
  5. package/crates/team-agent/src/cli/status_port/runtime.rs +3 -1
  6. package/crates/team-agent/src/cli/status_port/tests.rs +37 -1
  7. package/crates/team-agent/src/cli/tests/status_brief.rs +199 -0
  8. package/crates/team-agent/src/coordinator/steps/abnormal.rs +122 -2
  9. package/crates/team-agent/src/coordinator/tests/mod.rs +1 -0
  10. package/crates/team-agent/src/coordinator/tests/s3_bounded.rs +239 -0
  11. package/crates/team-agent/src/coordinator/tick.rs +21 -13
  12. package/crates/team-agent/src/leader/start.rs +46 -74
  13. package/crates/team-agent/src/lifecycle/launch/pi_mcp.rs +21 -5
  14. package/crates/team-agent/src/lifecycle/tests/fixtures/pi_list_models_spaced_ids.stdout.txt +9 -0
  15. package/crates/team-agent/src/lifecycle/tests/pi_catalog_red.rs +71 -0
  16. package/crates/team-agent/src/mcp_server/lifecycle_tools/state_status.rs +2 -11
  17. package/crates/team-agent/src/mcp_server/tools.rs +1 -3
  18. package/crates/team-agent/src/messaging/delivery.rs +154 -72
  19. package/crates/team-agent/src/messaging/results.rs +1 -2
  20. package/crates/team-agent/src/messaging/tests/leader_inject_acceptance.rs +127 -0
  21. package/crates/team-agent/src/provider/session_scan/common/tests.rs +204 -0
  22. package/crates/team-agent/src/provider/session_scan/common.rs +24 -2
  23. package/crates/team-agent/src/state/abnormal_watch/tests/mod.rs +248 -0
  24. package/crates/team-agent/src/state/abnormal_watch.rs +144 -0
  25. package/crates/team-agent/src/state/mod.rs +1 -0
  26. package/crates/team-agent/src/state/persist.rs +47 -5
  27. package/crates/team-agent/src/state/projection.rs +31 -1
  28. package/crates/team-agent/src/state/repository/tests.rs +81 -0
  29. package/crates/team-agent/src/state/repository.rs +169 -1
  30. package/crates/team-agent/src/tmux_backend/tests.rs +28 -0
  31. package/crates/team-agent/src/tmux_backend.rs +31 -9
  32. package/crates/team-agent/src/transport.rs +6 -0
  33. package/package.json +4 -4
package/Cargo.lock CHANGED
@@ -581,7 +581,7 @@ dependencies = [
581
581
 
582
582
  [[package]]
583
583
  name = "team-agent"
584
- version = "0.5.78"
584
+ version = "0.5.81"
585
585
  dependencies = [
586
586
  "anyhow",
587
587
  "chrono",
package/Cargo.toml CHANGED
@@ -9,7 +9,7 @@ members = ["crates/team-agent", "crates/win-conpty-phase0", "crates/conpty-trans
9
9
 
10
10
  [workspace.package]
11
11
  edition = "2021"
12
- version = "0.5.78"
12
+ version = "0.5.81"
13
13
  license = "AGPL-3.0"
14
14
  rust-version = "1.95"
15
15
 
@@ -248,6 +248,37 @@ mod tests {
248
248
  None
249
249
  );
250
250
  }
251
+
252
+ #[cfg(unix)]
253
+ #[test]
254
+ fn search_survives_unrelated_spaced_model_ids() {
255
+ let catalog = include_str!("../lifecycle/tests/fixtures/pi_list_models_spaced_ids.stdout.txt");
256
+ let path = fixture(&format!("printf '%s' '{catalog}'"));
257
+ let result = cmd_models_with(
258
+ &ModelsArgs {
259
+ provider: "pi".into(),
260
+ search: Some("gpt-5.6-luna".into()),
261
+ json: true,
262
+ },
263
+ &path,
264
+ Duration::from_secs(10),
265
+ MAX_CATALOG_BYTES,
266
+ )
267
+ .unwrap();
268
+ cleanup_fixture(&path);
269
+ assert_eq!(result.exit, ExitCode::Ok);
270
+ let CmdOutput::Human(text) = result.output else {
271
+ panic!("expected JSON projection")
272
+ };
273
+ let value: Value = serde_json::from_str(&text).unwrap();
274
+ assert_eq!(value["ok"], true);
275
+ assert_eq!(value["auth"], "ok");
276
+ let models = value["models"].as_array().unwrap();
277
+ assert_eq!(models.len(), 2);
278
+ assert_eq!(models[0]["role_model"], "openai-codex/gpt-5.6-luna");
279
+ assert_eq!(models[1]["role_model"], "team-agent/gpt-5.6-luna");
280
+ }
281
+
251
282
  #[cfg(unix)]
252
283
  fn fixture(body: &str) -> std::path::PathBuf {
253
284
  use std::os::unix::fs::PermissionsExt;
@@ -294,24 +294,39 @@ fn display_optional(value: &Value) -> String {
294
294
  }
295
295
 
296
296
  fn registered_nodes(state: &Value, selected: Option<&str>) -> Vec<RegisteredNode> {
297
- let agents = state
298
- .get("agents")
299
- .and_then(Value::as_object)
300
- .or_else(|| {
301
- let key = state.get("active_team_key").and_then(Value::as_str)?;
302
- state
303
- .get("teams")
304
- .and_then(|teams| teams.get(key))
305
- .and_then(|team| team.get("agents"))
306
- .and_then(Value::as_object)
307
- });
297
+ // A projected state can still carry stale top-level agents. Once the
298
+ // selector identified a canonical team entry, that entry is authoritative;
299
+ // an empty/missing nested roster must not fall back to the stale projection.
300
+ let canonical_team = state
301
+ .get("active_team_key")
302
+ .and_then(Value::as_str)
303
+ .and_then(|key| state.get("teams").and_then(|teams| teams.get(key)));
304
+ let (agents, lifecycle, defaults) = match canonical_team {
305
+ Some(team) => (
306
+ team.get("agents").and_then(Value::as_object),
307
+ team.get("agent_lifecycle").and_then(Value::as_object),
308
+ team,
309
+ ),
310
+ None => (
311
+ state.get("agents").and_then(Value::as_object),
312
+ state.get("agent_lifecycle").and_then(Value::as_object),
313
+ state,
314
+ ),
315
+ };
308
316
  let Some(agents) = agents else {
309
317
  return Vec::new();
310
318
  };
311
319
  agents
312
320
  .iter()
313
321
  .filter(|(name, _)| selected.is_none_or(|selected| selected == name.as_str()))
314
- .map(|(name, value)| registered_node(name, value, state))
322
+ .filter(|(name, _)| {
323
+ !lifecycle
324
+ .and_then(|entries| entries.get(*name))
325
+ .and_then(|entry| entry.get("state"))
326
+ .and_then(Value::as_str)
327
+ .is_some_and(|state| state == "retired")
328
+ })
329
+ .map(|(name, value)| registered_node(name, value, defaults))
315
330
  .collect()
316
331
  }
317
332
 
@@ -803,7 +818,7 @@ fn parse_probe(bytes: &[u8], requested_endpoint: &str) -> Result<ProbeResult, ()
803
818
  .and_then(Value::as_array)
804
819
  .ok_or(())?
805
820
  .iter()
806
- .filter_map(parse_probe_node)
821
+ .filter_map(|node| parse_probe_node(node, report_socket))
807
822
  .collect();
808
823
  Ok(ProbeResult { nodes })
809
824
  }
@@ -818,9 +833,12 @@ fn probe_has_error(value: &Value) -> bool {
818
833
  }
819
834
  }
820
835
 
821
- fn parse_probe_node(value: &Value) -> Option<ProbeNode> {
836
+ fn parse_probe_node(value: &Value, report_socket: &str) -> Option<ProbeNode> {
822
837
  Some(ProbeNode {
823
- socket: non_empty(value, "socket")?,
838
+ // The accepted producer binds the socket once at the report envelope
839
+ // and omits the redundant per-node field. An explicit node socket is
840
+ // still preserved for strict mismatch rejection below.
841
+ socket: non_empty(value, "socket").unwrap_or_else(|| report_socket.to_string()),
824
842
  session: non_empty(value, "session")?,
825
843
  window: non_empty(value, "window_name")?,
826
844
  pane: non_empty(value, "pane_id")?,
@@ -926,7 +944,13 @@ fn match_result<'a>(
926
944
  fn matches_registered(node: &RegisteredNode, observed: &ProbeNode) -> bool {
927
945
  node.endpoint.as_deref() == Some(observed.socket.as_str())
928
946
  && node.session.as_deref() == Some(observed.session.as_str())
929
- && node.window.as_deref() == Some(observed.window.as_str())
947
+ // Older registrations may not persist a window. The endpoint,
948
+ // session, and pane tuple remains the narrow compatibility key; a
949
+ // present window is still an exact constraint.
950
+ && node
951
+ .window
952
+ .as_deref()
953
+ .is_none_or(|window| window == observed.window)
930
954
  && node.pane.as_deref() == Some(observed.pane.as_str())
931
955
  }
932
956
 
@@ -58,7 +58,9 @@ pub(crate) fn compute_runtime_freshness(
58
58
  for (agent_id, entry) in watch {
59
59
  let exited =
60
60
  entry.get("worker_provider_exited").and_then(Value::as_bool) == Some(true);
61
- if exited {
61
+ if exited
62
+ && crate::state::abnormal_watch::matches_observation(state, agent_id, entry)
63
+ {
62
64
  provider_exited_agents.insert(agent_id.clone());
63
65
  }
64
66
  }
@@ -28,7 +28,12 @@ fn health_fixture() -> crate::coordinator::HealthReport {
28
28
 
29
29
  #[test]
30
30
  fn runtime_freshness_provider_exit_requires_typed_positive_fact() {
31
- let state = serde_json::json!({
31
+ let mut state = serde_json::json!({
32
+ "active_team_key": "status-team",
33
+ "agents": {
34
+ "pane_only": {"provider": "codex", "rollout_path": "/fixture/pane", "spawn_epoch": 1},
35
+ "typed_exit": {"provider": "codex", "rollout_path": "/fixture/typed", "spawn_epoch": 1}
36
+ },
32
37
  "coordinator": {
33
38
  "abnormal_exit_watch": {
34
39
  "pane_only": {
@@ -43,6 +48,10 @@ fn runtime_freshness_provider_exit_requires_typed_positive_fact() {
43
48
  }
44
49
  });
45
50
 
51
+ for id in ["pane_only", "typed_exit"] {
52
+ state["coordinator"]["abnormal_exit_watch"][id]["watch_identity"] =
53
+ crate::state::abnormal_watch::observation_identity(&state, id).unwrap();
54
+ }
46
55
  let freshness = compute_runtime_freshness(
47
56
  Path::new("/nonexistent/status-port-typed-provider-exit-test"),
48
57
  &state,
@@ -52,3 +61,30 @@ fn runtime_freshness_provider_exit_requires_typed_positive_fact() {
52
61
  assert!(!freshness.provider_exited_agents.contains("pane_only"));
53
62
  assert!(freshness.provider_exited_agents.contains("typed_exit"));
54
63
  }
64
+
65
+ #[test]
66
+ fn runtime_freshness_rejects_foreign_legacy_and_unknown_watch_identity() {
67
+ let mut state = json!({
68
+ "active_team_key": "a",
69
+ "agents": {"w": {"provider": "codex", "rollout_path": "/fixture/w", "spawn_epoch": 1}},
70
+ "coordinator": {"abnormal_exit_watch": {"w": {"worker_provider_exited": true}}}
71
+ });
72
+ let path = Path::new("/nonexistent/status-watch-188");
73
+ assert!(compute_runtime_freshness(path, &state, &health_fixture())
74
+ .provider_exited_agents
75
+ .is_empty());
76
+ let identity = crate::state::abnormal_watch::observation_identity(&state, "w").unwrap();
77
+ state["coordinator"]["abnormal_exit_watch"]["w"]["watch_identity"] = identity;
78
+ for (pointer, value) in [
79
+ ("/active_team_key", json!("b")),
80
+ ("/agents/w/provider", json!("claude")),
81
+ ("/agents/w/spawn_epoch", json!(2)),
82
+ ("/agents/w/rollout_path", json!(null)),
83
+ ] {
84
+ let mut changed = state.clone();
85
+ *changed.pointer_mut(pointer).unwrap() = value;
86
+ assert!(compute_runtime_freshness(path, &changed, &health_fixture())
87
+ .provider_exited_agents
88
+ .is_empty());
89
+ }
90
+ }
@@ -685,6 +685,205 @@ fn cmd_status_multi_endpoint_uses_total_budget() {
685
685
  let _ = std::fs::remove_dir_all(&ws);
686
686
  }
687
687
 
688
+ #[cfg(unix)]
689
+ #[test]
690
+ #[serial(status_brief)]
691
+ fn cmd_status_uses_canonical_roster_and_filters_same_team_retired_tombstones() {
692
+ let ws = brief_workspace("canonical-roster");
693
+ let mut stale = serde_json::Map::new();
694
+ for index in 0..220 {
695
+ stale.insert(
696
+ format!("stale-{index}"),
697
+ production_agent(&format!("stale-{index}"), &format!("%{index}")),
698
+ );
699
+ }
700
+ let mut stopped = production_agent("stopped", "%9");
701
+ stopped["status"] = json!("stopped");
702
+ let state = json!({
703
+ "session_name": "team-demo",
704
+ "tmux_endpoint": "/tmp/ta-status-brief.sock",
705
+ "tmux_socket": "/tmp/ta-status-brief.sock",
706
+ "active_team_key": "demo",
707
+ "agents": stale,
708
+ "teams": {
709
+ "demo": {
710
+ "status": "alive",
711
+ "session_name": "team-demo",
712
+ "tmux_endpoint": "/tmp/ta-status-brief.sock",
713
+ "agents": {
714
+ "live": production_agent("live", "%7"),
715
+ "retired": production_agent("retired", "%8"),
716
+ "stopped": stopped
717
+ },
718
+ "agent_lifecycle": {
719
+ "retired": {"state": "retired"}
720
+ }
721
+ },
722
+ "sibling": {
723
+ "status": "alive",
724
+ "agent_lifecycle": {"live": {"state": "retired"}}
725
+ }
726
+ }
727
+ });
728
+ write_state(&ws, &state);
729
+ let bin = write_nodeprobe(
730
+ &ws.join("probe"),
731
+ "printf '%s\\n' '{\"schema_version\":1,\"socket\":\"/tmp/ta-status-brief.sock\",\"nodes\":[]}'\n",
732
+ );
733
+ let nodes = status_port::with_test_nodeprobe(bin, || {
734
+ json_nodes(cmd_status(&status_args(&ws, true, None, Some("demo"))).expect("status"))
735
+ });
736
+ let mut names = nodes
737
+ .iter()
738
+ .filter_map(|node| node["name"].as_str())
739
+ .collect::<Vec<_>>();
740
+ names.sort_unstable();
741
+ assert_eq!(names, vec!["live", "stopped"]);
742
+ assert_eq!(
743
+ nodes
744
+ .iter()
745
+ .find(|node| node["name"] == "stopped")
746
+ .and_then(|node| node["runtime_status"].as_str()),
747
+ Some("stopped")
748
+ );
749
+ let err = cmd_status(&status_args(&ws, true, Some("retired"), Some("demo"))).unwrap_err();
750
+ assert!(err.to_string().contains("unknown agent id: retired"), "{err}");
751
+
752
+ let empty = brief_workspace("canonical-empty");
753
+ write_state(
754
+ &empty,
755
+ &json!({
756
+ "active_team_key": "demo",
757
+ "agents": {"stale": production_agent("stale", "%1")},
758
+ "teams": {
759
+ "demo": {
760
+ "status": "alive",
761
+ "agents": {},
762
+ "agent_lifecycle": {}
763
+ }
764
+ }
765
+ }),
766
+ );
767
+ assert!(json_nodes(cmd_status(&status_args(&empty, true, None, None)).expect("status")).is_empty());
768
+ let _ = std::fs::remove_dir_all(&ws);
769
+ let _ = std::fs::remove_dir_all(&empty);
770
+ }
771
+
772
+ #[cfg(unix)]
773
+ #[test]
774
+ #[serial(status_brief)]
775
+ fn cmd_status_allows_only_missing_window_compatibility_match() {
776
+ let ws = brief_workspace("missing-window");
777
+ let mut agent = production_agent("worker", "%7");
778
+ agent.as_object_mut().unwrap().remove("window");
779
+ agent.as_object_mut().unwrap().remove("layout_window");
780
+ agent
781
+ .get_mut("display")
782
+ .and_then(Value::as_object_mut)
783
+ .unwrap()
784
+ .remove("window");
785
+ write_state(
786
+ &ws,
787
+ &production_state(json!({"worker": agent})),
788
+ );
789
+ let report = producer_report(
790
+ "/tmp/ta-status-brief.sock",
791
+ "observed-window",
792
+ "%7",
793
+ "pi_activity_channel",
794
+ );
795
+ let bin = write_nodeprobe(&ws.join("probe"), &format!("cat <<'EOF'\n{report}\nEOF\n"));
796
+ let nodes = status_port::with_test_nodeprobe(bin, || {
797
+ json_nodes(cmd_status(&status_args(&ws, true, None, None)).expect("status"))
798
+ });
799
+ assert_eq!(nodes[0]["runtime_status"], "running");
800
+ assert_eq!(nodes[0]["activity"], "idle");
801
+ assert_eq!(
802
+ nodes[0]["tmux_command"],
803
+ "tmux -S '/tmp/ta-status-brief.sock' attach -t 'team-demo:observed-window.%7'"
804
+ );
805
+ let _ = std::fs::remove_dir_all(&ws);
806
+
807
+ let conflict = brief_workspace("present-window-conflict");
808
+ write_state(
809
+ &conflict,
810
+ &production_state(json!({"worker": production_agent("worker", "%7")})),
811
+ );
812
+ let report = producer_report(
813
+ "/tmp/ta-status-brief.sock",
814
+ "different-window",
815
+ "%7",
816
+ "pi_activity_channel",
817
+ );
818
+ let bin = write_nodeprobe(&conflict.join("probe"), &format!("cat <<'EOF'\n{report}\nEOF\n"));
819
+ let nodes = status_port::with_test_nodeprobe(bin, || {
820
+ json_nodes(cmd_status(&status_args(&conflict, true, None, None)).expect("status"))
821
+ });
822
+ assert_eq!(nodes[0]["runtime_status"], "unknown");
823
+ assert!(nodes[0]["tmux_command"].is_null());
824
+ let _ = std::fs::remove_dir_all(&conflict);
825
+ }
826
+
827
+ #[cfg(unix)]
828
+ #[test]
829
+ #[serial(status_brief)]
830
+ fn cmd_status_accepts_envelope_bound_node_socket_but_rejects_explicit_mismatch() {
831
+ let ws = brief_workspace("envelope-node-socket");
832
+ write_state(
833
+ &ws,
834
+ &production_state(json!({"worker": production_agent("worker", "%7")})),
835
+ );
836
+ let mut report: Value =
837
+ serde_json::from_str(&producer_report(
838
+ "/tmp/ta-status-brief.sock",
839
+ "worker",
840
+ "%7",
841
+ "pi_activity_channel",
842
+ ))
843
+ .unwrap();
844
+ report["nodes"][0].as_object_mut().unwrap().remove("socket");
845
+ let report = serde_json::to_string(&report).unwrap();
846
+ let bin = write_nodeprobe(&ws.join("probe"), &format!("cat <<'EOF'\n{report}\nEOF\n"));
847
+ let nodes = status_port::with_test_nodeprobe(bin, || {
848
+ json_nodes(cmd_status(&status_args(&ws, true, None, None)).expect("status"))
849
+ });
850
+ assert_eq!(nodes[0]["runtime_status"], "running");
851
+ assert_eq!(nodes[0]["activity"], "idle");
852
+ assert_eq!(nodes[0]["health"], "normal");
853
+ assert_eq!(nodes[0]["session_name"], "pi-session");
854
+ assert_eq!(
855
+ nodes[0]["tmux_command"],
856
+ "tmux -S '/tmp/ta-status-brief.sock' attach -t 'team-demo:worker.%7'"
857
+ );
858
+ let _ = std::fs::remove_dir_all(&ws);
859
+
860
+ let mismatch = brief_workspace("explicit-node-socket-mismatch");
861
+ write_state(
862
+ &mismatch,
863
+ &production_state(json!({"worker": production_agent("worker", "%7")})),
864
+ );
865
+ let mut report: Value =
866
+ serde_json::from_str(&producer_report(
867
+ "/tmp/ta-status-brief.sock",
868
+ "worker",
869
+ "%7",
870
+ "pi_activity_channel",
871
+ ))
872
+ .unwrap();
873
+ report["nodes"][0]["socket"] = json!("/tmp/other.sock");
874
+ let report = serde_json::to_string(&report).unwrap();
875
+ let bin = write_nodeprobe(
876
+ &mismatch.join("probe"),
877
+ &format!("cat <<'EOF'\n{report}\nEOF\n"),
878
+ );
879
+ let nodes = status_port::with_test_nodeprobe(bin, || {
880
+ json_nodes(cmd_status(&status_args(&mismatch, true, None, None)).expect("status"))
881
+ });
882
+ assert_eq!(nodes[0]["runtime_status"], "unknown");
883
+ assert!(nodes[0]["tmux_command"].is_null());
884
+ let _ = std::fs::remove_dir_all(&mismatch);
885
+ }
886
+
688
887
  #[cfg(unix)]
689
888
  fn pid_alive(pid: u32) -> bool {
690
889
  unsafe { libc::kill(pid as i32, 0) == 0 }
@@ -73,6 +73,8 @@ pub(crate) fn detect_abnormal_exits(
73
73
  let team = crate::state::projection::team_state_key(&snapshot);
74
74
  let session_name = snapshot.get("session_name").and_then(Value::as_str);
75
75
  for agent in abnormal_watch_agents(&snapshot) {
76
+ // Reject untrusted history without skipping a fresh provider/path observation.
77
+ crate::state::abnormal_watch::begin_observation(state, &agent.agent_id);
76
78
  // Pane/process liveness is independent of transcript content. A
77
79
  // frozen rollout is common after pane death, so probe before the
78
80
  // metadata dedupe gate; only the expensive tail scan stays deduped.
@@ -109,7 +111,7 @@ pub(crate) fn detect_abnormal_exits(
109
111
  // re-read below.
110
112
  if let (Some(mtime), Some(stored)) = (
111
113
  mtime_ns,
112
- abnormal_watch_stored_metadata(&snapshot, &agent.agent_id),
114
+ abnormal_watch_stored_metadata(state, &agent.agent_id),
113
115
  ) {
114
116
  if stored == (size, mtime) {
115
117
  refresh_abnormal_watch_liveness(state, &agent.agent_id, &liveness);
@@ -149,7 +151,7 @@ pub(crate) fn detect_abnormal_exits(
149
151
  .map(|fact| abnormal_error_observation_key(&agent, fact));
150
152
  let error_observation_cohort = fact.as_ref().map(|_| abnormal_error_cohort_key(&agent));
151
153
  let error_recency = abnormal_error_recency(
152
- &snapshot,
154
+ state,
153
155
  &agent,
154
156
  error_observation_key.as_deref(),
155
157
  error_observation_cohort.as_deref(),
@@ -954,6 +956,9 @@ fn abnormal_watch_payload(
954
956
  }
955
957
 
956
958
  fn upsert_abnormal_watch(state: &mut Value, agent_id: &str, mut payload: Value) {
959
+ if let Some(identity) = crate::state::abnormal_watch::observation_identity(state, agent_id) {
960
+ payload["watch_identity"] = identity;
961
+ }
957
962
  let preserved = [
958
963
  "last_notified_key",
959
964
  "last_notified_at",
@@ -1413,6 +1418,121 @@ mod tests {
1413
1418
  crate::provider::get_adapter(provider)
1414
1419
  }
1415
1420
  }
1421
+ #[test]
1422
+ fn resource188_foreign_metadata_and_dedupe_cannot_skip_new_baseline() {
1423
+ let dir = temp_abnormal_dir("188-foreign-watch");
1424
+ let rollout = dir.join("rollout-w1.jsonl");
1425
+ let error = "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"old\",\"status\":\"failed\"}}}\n";
1426
+ std::fs::write(&rollout, error).unwrap();
1427
+ seed_abnormal_state(&dir, &rollout, "alive", 1);
1428
+ let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
1429
+ let metadata = std::fs::metadata(&rollout).unwrap();
1430
+ let mut foreign = crate::state::abnormal_watch::observation_identity(&state, "w1").unwrap();
1431
+ foreign["team_key"] = serde_json::json!("another-team");
1432
+ state["coordinator"] = serde_json::json!({"abnormal_exit_watch": {"w1": {
1433
+ "watch_identity": foreign,
1434
+ "size": metadata.len(), "mtime_ns": metadata_mtime_ns(&metadata),
1435
+ "last_notified_key": "foreign", "last_check_key": "foreign",
1436
+ "last_error_observation_key": "foreign", "last_error_observation_cohort": "foreign"
1437
+ }}});
1438
+ let log = EventLog::new(&dir);
1439
+ let transport = crate::transport::test_support::OfflineTransport::new();
1440
+ detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1441
+ let baseline = state["coordinator"]["abnormal_exit_watch"]["w1"].clone();
1442
+ assert_eq!(baseline["error_recency"], "stale");
1443
+ assert!(baseline.get("last_notified_key").is_none());
1444
+ assert!(crate::state::abnormal_watch::matches_observation(
1445
+ &state, "w1", &baseline
1446
+ ));
1447
+ let events = log.tail(0).unwrap();
1448
+ detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1449
+ assert_eq!(
1450
+ log.tail(0).unwrap(),
1451
+ events,
1452
+ "unchanged baseline must not re-alarm"
1453
+ );
1454
+ assert!(find_event(&events, "worker.abnormal_exit").is_none());
1455
+ }
1456
+
1457
+ #[test]
1458
+ fn resource188_legacy_missing_cohort_still_observes_fresh_errors_and_dedupes() {
1459
+ let dir = temp_abnormal_dir("188-legacy-cohort");
1460
+ let rollout = dir.join("rollout-w1.jsonl");
1461
+ let error = "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"old\",\"status\":\"failed\"}}}\n";
1462
+ std::fs::write(&rollout, error).unwrap();
1463
+ seed_abnormal_state(&dir, &rollout, "alive", 1);
1464
+ let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
1465
+ state["agents"]["w1"]
1466
+ .as_object_mut()
1467
+ .unwrap()
1468
+ .remove("spawn_epoch");
1469
+ let log = EventLog::new(&dir);
1470
+ let transport = crate::transport::test_support::OfflineTransport::new();
1471
+ detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1472
+ assert_eq!(
1473
+ state["coordinator"]["abnormal_exit_watch"]["w1"]["error_recency"],
1474
+ "stale"
1475
+ );
1476
+ assert!(find_event(&log.tail(0).unwrap(), "worker.abnormal_exit").is_none());
1477
+ std::fs::OpenOptions::new()
1478
+ .append(true)
1479
+ .open(&rollout)
1480
+ .unwrap()
1481
+ .write_all(error.replace("old", "new-error").as_bytes())
1482
+ .unwrap();
1483
+ detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1484
+ let events = log.tail(0).unwrap();
1485
+ assert_eq!(
1486
+ find_event(&events, "worker.abnormal_exit").unwrap()["error_recency"],
1487
+ "fresh"
1488
+ );
1489
+ let notified =
1490
+ state["coordinator"]["abnormal_exit_watch"]["w1"]["last_notified_key"].clone();
1491
+ assert!(notified.is_string());
1492
+ detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1493
+ assert_eq!(log.tail(0).unwrap(), events);
1494
+ assert_eq!(
1495
+ state["coordinator"]["abnormal_exit_watch"]["w1"]["last_notified_key"],
1496
+ notified
1497
+ );
1498
+ state["agents"]["w1"]["spawn_epoch"] = serde_json::json!(2);
1499
+ detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1500
+ let watch = &state["coordinator"]["abnormal_exit_watch"]["w1"];
1501
+ assert_eq!(
1502
+ watch["error_recency"], "stale",
1503
+ "a known generation must rebaseline unchanged bytes"
1504
+ );
1505
+ assert!(watch.get("last_notified_key").is_none());
1506
+ assert_eq!(
1507
+ log.tail(0)
1508
+ .unwrap()
1509
+ .iter()
1510
+ .filter(|event| event["event"] == "worker.abnormal_exit")
1511
+ .count(),
1512
+ 1
1513
+ );
1514
+ }
1515
+
1516
+ #[test]
1517
+ fn resource188_unknown_team_still_records_fresh_liveness_without_reusing_cache() {
1518
+ let dir = temp_abnormal_dir("188-unknown-team");
1519
+ let rollout = dir.join("rollout-w1.jsonl");
1520
+ std::fs::write(&rollout, "").unwrap();
1521
+ seed_abnormal_state(&dir, &rollout, "dead", 1);
1522
+ let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
1523
+ state.as_object_mut().unwrap().remove("active_team_key");
1524
+ let log = EventLog::new(&dir);
1525
+ let transport = crate::transport::test_support::OfflineTransport::new();
1526
+ detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1527
+ assert_eq!(
1528
+ state["coordinator"]["abnormal_exit_watch"]["w1"]["last_liveness"],
1529
+ "dead"
1530
+ );
1531
+ assert!(state["coordinator"]["abnormal_exit_watch"]["w1"]
1532
+ .get("watch_identity")
1533
+ .is_none());
1534
+ }
1535
+
1416
1536
  #[test]
1417
1537
  fn abnormal_stale_error_baselines_then_fresh_alive_error_notifies() {
1418
1538
  let dir = temp_abnormal_dir("alive-fresh");
@@ -282,6 +282,7 @@ fn read_event_log_dir(dir: &std::path::Path) -> Vec<serde_json::Value> {
282
282
  }
283
283
 
284
284
  mod a0_lostupdate;
285
+ mod s3_bounded;
285
286
  mod abnormal;
286
287
  mod api_error_recovery;
287
288
  mod basics;