@team-agent/installer 0.5.78 → 0.5.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -1
- package/Cargo.toml +1 -1
- package/crates/team-agent/src/cli/models.rs +31 -0
- package/crates/team-agent/src/cli/status_port/brief.rs +40 -16
- package/crates/team-agent/src/cli/status_port/runtime.rs +3 -1
- package/crates/team-agent/src/cli/status_port/tests.rs +37 -1
- package/crates/team-agent/src/cli/tests/status_brief.rs +199 -0
- package/crates/team-agent/src/coordinator/steps/abnormal.rs +122 -2
- package/crates/team-agent/src/coordinator/tests/mod.rs +1 -0
- package/crates/team-agent/src/coordinator/tests/s3_bounded.rs +239 -0
- package/crates/team-agent/src/coordinator/tick.rs +21 -13
- package/crates/team-agent/src/leader/start.rs +46 -74
- package/crates/team-agent/src/lifecycle/launch/pi_mcp.rs +21 -5
- package/crates/team-agent/src/lifecycle/tests/fixtures/pi_list_models_spaced_ids.stdout.txt +9 -0
- package/crates/team-agent/src/lifecycle/tests/pi_catalog_red.rs +71 -0
- package/crates/team-agent/src/mcp_server/lifecycle_tools/state_status.rs +2 -11
- package/crates/team-agent/src/mcp_server/tools.rs +1 -3
- package/crates/team-agent/src/messaging/delivery.rs +154 -72
- package/crates/team-agent/src/messaging/results.rs +1 -2
- package/crates/team-agent/src/messaging/tests/leader_inject_acceptance.rs +127 -0
- package/crates/team-agent/src/provider/session_scan/common/tests.rs +204 -0
- package/crates/team-agent/src/provider/session_scan/common.rs +24 -2
- package/crates/team-agent/src/state/abnormal_watch/tests/mod.rs +248 -0
- package/crates/team-agent/src/state/abnormal_watch.rs +144 -0
- package/crates/team-agent/src/state/mod.rs +1 -0
- package/crates/team-agent/src/state/persist.rs +47 -5
- package/crates/team-agent/src/state/projection.rs +31 -1
- package/crates/team-agent/src/state/repository/tests.rs +81 -0
- package/crates/team-agent/src/state/repository.rs +169 -1
- package/crates/team-agent/src/tmux_backend/tests.rs +28 -0
- package/crates/team-agent/src/tmux_backend.rs +31 -9
- package/crates/team-agent/src/transport.rs +6 -0
- package/package.json +4 -4
package/Cargo.lock
CHANGED
package/Cargo.toml
CHANGED
|
@@ -248,6 +248,37 @@ mod tests {
|
|
|
248
248
|
None
|
|
249
249
|
);
|
|
250
250
|
}
|
|
251
|
+
|
|
252
|
+
#[cfg(unix)]
|
|
253
|
+
#[test]
|
|
254
|
+
fn search_survives_unrelated_spaced_model_ids() {
|
|
255
|
+
let catalog = include_str!("../lifecycle/tests/fixtures/pi_list_models_spaced_ids.stdout.txt");
|
|
256
|
+
let path = fixture(&format!("printf '%s' '{catalog}'"));
|
|
257
|
+
let result = cmd_models_with(
|
|
258
|
+
&ModelsArgs {
|
|
259
|
+
provider: "pi".into(),
|
|
260
|
+
search: Some("gpt-5.6-luna".into()),
|
|
261
|
+
json: true,
|
|
262
|
+
},
|
|
263
|
+
&path,
|
|
264
|
+
Duration::from_secs(10),
|
|
265
|
+
MAX_CATALOG_BYTES,
|
|
266
|
+
)
|
|
267
|
+
.unwrap();
|
|
268
|
+
cleanup_fixture(&path);
|
|
269
|
+
assert_eq!(result.exit, ExitCode::Ok);
|
|
270
|
+
let CmdOutput::Human(text) = result.output else {
|
|
271
|
+
panic!("expected JSON projection")
|
|
272
|
+
};
|
|
273
|
+
let value: Value = serde_json::from_str(&text).unwrap();
|
|
274
|
+
assert_eq!(value["ok"], true);
|
|
275
|
+
assert_eq!(value["auth"], "ok");
|
|
276
|
+
let models = value["models"].as_array().unwrap();
|
|
277
|
+
assert_eq!(models.len(), 2);
|
|
278
|
+
assert_eq!(models[0]["role_model"], "openai-codex/gpt-5.6-luna");
|
|
279
|
+
assert_eq!(models[1]["role_model"], "team-agent/gpt-5.6-luna");
|
|
280
|
+
}
|
|
281
|
+
|
|
251
282
|
#[cfg(unix)]
|
|
252
283
|
fn fixture(body: &str) -> std::path::PathBuf {
|
|
253
284
|
use std::os::unix::fs::PermissionsExt;
|
|
@@ -294,24 +294,39 @@ fn display_optional(value: &Value) -> String {
|
|
|
294
294
|
}
|
|
295
295
|
|
|
296
296
|
fn registered_nodes(state: &Value, selected: Option<&str>) -> Vec<RegisteredNode> {
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
297
|
+
// A projected state can still carry stale top-level agents. Once the
|
|
298
|
+
// selector identified a canonical team entry, that entry is authoritative;
|
|
299
|
+
// an empty/missing nested roster must not fall back to the stale projection.
|
|
300
|
+
let canonical_team = state
|
|
301
|
+
.get("active_team_key")
|
|
302
|
+
.and_then(Value::as_str)
|
|
303
|
+
.and_then(|key| state.get("teams").and_then(|teams| teams.get(key)));
|
|
304
|
+
let (agents, lifecycle, defaults) = match canonical_team {
|
|
305
|
+
Some(team) => (
|
|
306
|
+
team.get("agents").and_then(Value::as_object),
|
|
307
|
+
team.get("agent_lifecycle").and_then(Value::as_object),
|
|
308
|
+
team,
|
|
309
|
+
),
|
|
310
|
+
None => (
|
|
311
|
+
state.get("agents").and_then(Value::as_object),
|
|
312
|
+
state.get("agent_lifecycle").and_then(Value::as_object),
|
|
313
|
+
state,
|
|
314
|
+
),
|
|
315
|
+
};
|
|
308
316
|
let Some(agents) = agents else {
|
|
309
317
|
return Vec::new();
|
|
310
318
|
};
|
|
311
319
|
agents
|
|
312
320
|
.iter()
|
|
313
321
|
.filter(|(name, _)| selected.is_none_or(|selected| selected == name.as_str()))
|
|
314
|
-
.
|
|
322
|
+
.filter(|(name, _)| {
|
|
323
|
+
!lifecycle
|
|
324
|
+
.and_then(|entries| entries.get(*name))
|
|
325
|
+
.and_then(|entry| entry.get("state"))
|
|
326
|
+
.and_then(Value::as_str)
|
|
327
|
+
.is_some_and(|state| state == "retired")
|
|
328
|
+
})
|
|
329
|
+
.map(|(name, value)| registered_node(name, value, defaults))
|
|
315
330
|
.collect()
|
|
316
331
|
}
|
|
317
332
|
|
|
@@ -803,7 +818,7 @@ fn parse_probe(bytes: &[u8], requested_endpoint: &str) -> Result<ProbeResult, ()
|
|
|
803
818
|
.and_then(Value::as_array)
|
|
804
819
|
.ok_or(())?
|
|
805
820
|
.iter()
|
|
806
|
-
.filter_map(parse_probe_node)
|
|
821
|
+
.filter_map(|node| parse_probe_node(node, report_socket))
|
|
807
822
|
.collect();
|
|
808
823
|
Ok(ProbeResult { nodes })
|
|
809
824
|
}
|
|
@@ -818,9 +833,12 @@ fn probe_has_error(value: &Value) -> bool {
|
|
|
818
833
|
}
|
|
819
834
|
}
|
|
820
835
|
|
|
821
|
-
fn parse_probe_node(value: &Value) -> Option<ProbeNode> {
|
|
836
|
+
fn parse_probe_node(value: &Value, report_socket: &str) -> Option<ProbeNode> {
|
|
822
837
|
Some(ProbeNode {
|
|
823
|
-
socket
|
|
838
|
+
// The accepted producer binds the socket once at the report envelope
|
|
839
|
+
// and omits the redundant per-node field. An explicit node socket is
|
|
840
|
+
// still preserved for strict mismatch rejection below.
|
|
841
|
+
socket: non_empty(value, "socket").unwrap_or_else(|| report_socket.to_string()),
|
|
824
842
|
session: non_empty(value, "session")?,
|
|
825
843
|
window: non_empty(value, "window_name")?,
|
|
826
844
|
pane: non_empty(value, "pane_id")?,
|
|
@@ -926,7 +944,13 @@ fn match_result<'a>(
|
|
|
926
944
|
fn matches_registered(node: &RegisteredNode, observed: &ProbeNode) -> bool {
|
|
927
945
|
node.endpoint.as_deref() == Some(observed.socket.as_str())
|
|
928
946
|
&& node.session.as_deref() == Some(observed.session.as_str())
|
|
929
|
-
|
|
947
|
+
// Older registrations may not persist a window. The endpoint,
|
|
948
|
+
// session, and pane tuple remains the narrow compatibility key; a
|
|
949
|
+
// present window is still an exact constraint.
|
|
950
|
+
&& node
|
|
951
|
+
.window
|
|
952
|
+
.as_deref()
|
|
953
|
+
.is_none_or(|window| window == observed.window)
|
|
930
954
|
&& node.pane.as_deref() == Some(observed.pane.as_str())
|
|
931
955
|
}
|
|
932
956
|
|
|
@@ -58,7 +58,9 @@ pub(crate) fn compute_runtime_freshness(
|
|
|
58
58
|
for (agent_id, entry) in watch {
|
|
59
59
|
let exited =
|
|
60
60
|
entry.get("worker_provider_exited").and_then(Value::as_bool) == Some(true);
|
|
61
|
-
if exited
|
|
61
|
+
if exited
|
|
62
|
+
&& crate::state::abnormal_watch::matches_observation(state, agent_id, entry)
|
|
63
|
+
{
|
|
62
64
|
provider_exited_agents.insert(agent_id.clone());
|
|
63
65
|
}
|
|
64
66
|
}
|
|
@@ -28,7 +28,12 @@ fn health_fixture() -> crate::coordinator::HealthReport {
|
|
|
28
28
|
|
|
29
29
|
#[test]
|
|
30
30
|
fn runtime_freshness_provider_exit_requires_typed_positive_fact() {
|
|
31
|
-
let state = serde_json::json!({
|
|
31
|
+
let mut state = serde_json::json!({
|
|
32
|
+
"active_team_key": "status-team",
|
|
33
|
+
"agents": {
|
|
34
|
+
"pane_only": {"provider": "codex", "rollout_path": "/fixture/pane", "spawn_epoch": 1},
|
|
35
|
+
"typed_exit": {"provider": "codex", "rollout_path": "/fixture/typed", "spawn_epoch": 1}
|
|
36
|
+
},
|
|
32
37
|
"coordinator": {
|
|
33
38
|
"abnormal_exit_watch": {
|
|
34
39
|
"pane_only": {
|
|
@@ -43,6 +48,10 @@ fn runtime_freshness_provider_exit_requires_typed_positive_fact() {
|
|
|
43
48
|
}
|
|
44
49
|
});
|
|
45
50
|
|
|
51
|
+
for id in ["pane_only", "typed_exit"] {
|
|
52
|
+
state["coordinator"]["abnormal_exit_watch"][id]["watch_identity"] =
|
|
53
|
+
crate::state::abnormal_watch::observation_identity(&state, id).unwrap();
|
|
54
|
+
}
|
|
46
55
|
let freshness = compute_runtime_freshness(
|
|
47
56
|
Path::new("/nonexistent/status-port-typed-provider-exit-test"),
|
|
48
57
|
&state,
|
|
@@ -52,3 +61,30 @@ fn runtime_freshness_provider_exit_requires_typed_positive_fact() {
|
|
|
52
61
|
assert!(!freshness.provider_exited_agents.contains("pane_only"));
|
|
53
62
|
assert!(freshness.provider_exited_agents.contains("typed_exit"));
|
|
54
63
|
}
|
|
64
|
+
|
|
65
|
+
#[test]
|
|
66
|
+
fn runtime_freshness_rejects_foreign_legacy_and_unknown_watch_identity() {
|
|
67
|
+
let mut state = json!({
|
|
68
|
+
"active_team_key": "a",
|
|
69
|
+
"agents": {"w": {"provider": "codex", "rollout_path": "/fixture/w", "spawn_epoch": 1}},
|
|
70
|
+
"coordinator": {"abnormal_exit_watch": {"w": {"worker_provider_exited": true}}}
|
|
71
|
+
});
|
|
72
|
+
let path = Path::new("/nonexistent/status-watch-188");
|
|
73
|
+
assert!(compute_runtime_freshness(path, &state, &health_fixture())
|
|
74
|
+
.provider_exited_agents
|
|
75
|
+
.is_empty());
|
|
76
|
+
let identity = crate::state::abnormal_watch::observation_identity(&state, "w").unwrap();
|
|
77
|
+
state["coordinator"]["abnormal_exit_watch"]["w"]["watch_identity"] = identity;
|
|
78
|
+
for (pointer, value) in [
|
|
79
|
+
("/active_team_key", json!("b")),
|
|
80
|
+
("/agents/w/provider", json!("claude")),
|
|
81
|
+
("/agents/w/spawn_epoch", json!(2)),
|
|
82
|
+
("/agents/w/rollout_path", json!(null)),
|
|
83
|
+
] {
|
|
84
|
+
let mut changed = state.clone();
|
|
85
|
+
*changed.pointer_mut(pointer).unwrap() = value;
|
|
86
|
+
assert!(compute_runtime_freshness(path, &changed, &health_fixture())
|
|
87
|
+
.provider_exited_agents
|
|
88
|
+
.is_empty());
|
|
89
|
+
}
|
|
90
|
+
}
|
|
@@ -685,6 +685,205 @@ fn cmd_status_multi_endpoint_uses_total_budget() {
|
|
|
685
685
|
let _ = std::fs::remove_dir_all(&ws);
|
|
686
686
|
}
|
|
687
687
|
|
|
688
|
+
#[cfg(unix)]
|
|
689
|
+
#[test]
|
|
690
|
+
#[serial(status_brief)]
|
|
691
|
+
fn cmd_status_uses_canonical_roster_and_filters_same_team_retired_tombstones() {
|
|
692
|
+
let ws = brief_workspace("canonical-roster");
|
|
693
|
+
let mut stale = serde_json::Map::new();
|
|
694
|
+
for index in 0..220 {
|
|
695
|
+
stale.insert(
|
|
696
|
+
format!("stale-{index}"),
|
|
697
|
+
production_agent(&format!("stale-{index}"), &format!("%{index}")),
|
|
698
|
+
);
|
|
699
|
+
}
|
|
700
|
+
let mut stopped = production_agent("stopped", "%9");
|
|
701
|
+
stopped["status"] = json!("stopped");
|
|
702
|
+
let state = json!({
|
|
703
|
+
"session_name": "team-demo",
|
|
704
|
+
"tmux_endpoint": "/tmp/ta-status-brief.sock",
|
|
705
|
+
"tmux_socket": "/tmp/ta-status-brief.sock",
|
|
706
|
+
"active_team_key": "demo",
|
|
707
|
+
"agents": stale,
|
|
708
|
+
"teams": {
|
|
709
|
+
"demo": {
|
|
710
|
+
"status": "alive",
|
|
711
|
+
"session_name": "team-demo",
|
|
712
|
+
"tmux_endpoint": "/tmp/ta-status-brief.sock",
|
|
713
|
+
"agents": {
|
|
714
|
+
"live": production_agent("live", "%7"),
|
|
715
|
+
"retired": production_agent("retired", "%8"),
|
|
716
|
+
"stopped": stopped
|
|
717
|
+
},
|
|
718
|
+
"agent_lifecycle": {
|
|
719
|
+
"retired": {"state": "retired"}
|
|
720
|
+
}
|
|
721
|
+
},
|
|
722
|
+
"sibling": {
|
|
723
|
+
"status": "alive",
|
|
724
|
+
"agent_lifecycle": {"live": {"state": "retired"}}
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
});
|
|
728
|
+
write_state(&ws, &state);
|
|
729
|
+
let bin = write_nodeprobe(
|
|
730
|
+
&ws.join("probe"),
|
|
731
|
+
"printf '%s\\n' '{\"schema_version\":1,\"socket\":\"/tmp/ta-status-brief.sock\",\"nodes\":[]}'\n",
|
|
732
|
+
);
|
|
733
|
+
let nodes = status_port::with_test_nodeprobe(bin, || {
|
|
734
|
+
json_nodes(cmd_status(&status_args(&ws, true, None, Some("demo"))).expect("status"))
|
|
735
|
+
});
|
|
736
|
+
let mut names = nodes
|
|
737
|
+
.iter()
|
|
738
|
+
.filter_map(|node| node["name"].as_str())
|
|
739
|
+
.collect::<Vec<_>>();
|
|
740
|
+
names.sort_unstable();
|
|
741
|
+
assert_eq!(names, vec!["live", "stopped"]);
|
|
742
|
+
assert_eq!(
|
|
743
|
+
nodes
|
|
744
|
+
.iter()
|
|
745
|
+
.find(|node| node["name"] == "stopped")
|
|
746
|
+
.and_then(|node| node["runtime_status"].as_str()),
|
|
747
|
+
Some("stopped")
|
|
748
|
+
);
|
|
749
|
+
let err = cmd_status(&status_args(&ws, true, Some("retired"), Some("demo"))).unwrap_err();
|
|
750
|
+
assert!(err.to_string().contains("unknown agent id: retired"), "{err}");
|
|
751
|
+
|
|
752
|
+
let empty = brief_workspace("canonical-empty");
|
|
753
|
+
write_state(
|
|
754
|
+
&empty,
|
|
755
|
+
&json!({
|
|
756
|
+
"active_team_key": "demo",
|
|
757
|
+
"agents": {"stale": production_agent("stale", "%1")},
|
|
758
|
+
"teams": {
|
|
759
|
+
"demo": {
|
|
760
|
+
"status": "alive",
|
|
761
|
+
"agents": {},
|
|
762
|
+
"agent_lifecycle": {}
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
}),
|
|
766
|
+
);
|
|
767
|
+
assert!(json_nodes(cmd_status(&status_args(&empty, true, None, None)).expect("status")).is_empty());
|
|
768
|
+
let _ = std::fs::remove_dir_all(&ws);
|
|
769
|
+
let _ = std::fs::remove_dir_all(&empty);
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
#[cfg(unix)]
|
|
773
|
+
#[test]
|
|
774
|
+
#[serial(status_brief)]
|
|
775
|
+
fn cmd_status_allows_only_missing_window_compatibility_match() {
|
|
776
|
+
let ws = brief_workspace("missing-window");
|
|
777
|
+
let mut agent = production_agent("worker", "%7");
|
|
778
|
+
agent.as_object_mut().unwrap().remove("window");
|
|
779
|
+
agent.as_object_mut().unwrap().remove("layout_window");
|
|
780
|
+
agent
|
|
781
|
+
.get_mut("display")
|
|
782
|
+
.and_then(Value::as_object_mut)
|
|
783
|
+
.unwrap()
|
|
784
|
+
.remove("window");
|
|
785
|
+
write_state(
|
|
786
|
+
&ws,
|
|
787
|
+
&production_state(json!({"worker": agent})),
|
|
788
|
+
);
|
|
789
|
+
let report = producer_report(
|
|
790
|
+
"/tmp/ta-status-brief.sock",
|
|
791
|
+
"observed-window",
|
|
792
|
+
"%7",
|
|
793
|
+
"pi_activity_channel",
|
|
794
|
+
);
|
|
795
|
+
let bin = write_nodeprobe(&ws.join("probe"), &format!("cat <<'EOF'\n{report}\nEOF\n"));
|
|
796
|
+
let nodes = status_port::with_test_nodeprobe(bin, || {
|
|
797
|
+
json_nodes(cmd_status(&status_args(&ws, true, None, None)).expect("status"))
|
|
798
|
+
});
|
|
799
|
+
assert_eq!(nodes[0]["runtime_status"], "running");
|
|
800
|
+
assert_eq!(nodes[0]["activity"], "idle");
|
|
801
|
+
assert_eq!(
|
|
802
|
+
nodes[0]["tmux_command"],
|
|
803
|
+
"tmux -S '/tmp/ta-status-brief.sock' attach -t 'team-demo:observed-window.%7'"
|
|
804
|
+
);
|
|
805
|
+
let _ = std::fs::remove_dir_all(&ws);
|
|
806
|
+
|
|
807
|
+
let conflict = brief_workspace("present-window-conflict");
|
|
808
|
+
write_state(
|
|
809
|
+
&conflict,
|
|
810
|
+
&production_state(json!({"worker": production_agent("worker", "%7")})),
|
|
811
|
+
);
|
|
812
|
+
let report = producer_report(
|
|
813
|
+
"/tmp/ta-status-brief.sock",
|
|
814
|
+
"different-window",
|
|
815
|
+
"%7",
|
|
816
|
+
"pi_activity_channel",
|
|
817
|
+
);
|
|
818
|
+
let bin = write_nodeprobe(&conflict.join("probe"), &format!("cat <<'EOF'\n{report}\nEOF\n"));
|
|
819
|
+
let nodes = status_port::with_test_nodeprobe(bin, || {
|
|
820
|
+
json_nodes(cmd_status(&status_args(&conflict, true, None, None)).expect("status"))
|
|
821
|
+
});
|
|
822
|
+
assert_eq!(nodes[0]["runtime_status"], "unknown");
|
|
823
|
+
assert!(nodes[0]["tmux_command"].is_null());
|
|
824
|
+
let _ = std::fs::remove_dir_all(&conflict);
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
#[cfg(unix)]
|
|
828
|
+
#[test]
|
|
829
|
+
#[serial(status_brief)]
|
|
830
|
+
fn cmd_status_accepts_envelope_bound_node_socket_but_rejects_explicit_mismatch() {
|
|
831
|
+
let ws = brief_workspace("envelope-node-socket");
|
|
832
|
+
write_state(
|
|
833
|
+
&ws,
|
|
834
|
+
&production_state(json!({"worker": production_agent("worker", "%7")})),
|
|
835
|
+
);
|
|
836
|
+
let mut report: Value =
|
|
837
|
+
serde_json::from_str(&producer_report(
|
|
838
|
+
"/tmp/ta-status-brief.sock",
|
|
839
|
+
"worker",
|
|
840
|
+
"%7",
|
|
841
|
+
"pi_activity_channel",
|
|
842
|
+
))
|
|
843
|
+
.unwrap();
|
|
844
|
+
report["nodes"][0].as_object_mut().unwrap().remove("socket");
|
|
845
|
+
let report = serde_json::to_string(&report).unwrap();
|
|
846
|
+
let bin = write_nodeprobe(&ws.join("probe"), &format!("cat <<'EOF'\n{report}\nEOF\n"));
|
|
847
|
+
let nodes = status_port::with_test_nodeprobe(bin, || {
|
|
848
|
+
json_nodes(cmd_status(&status_args(&ws, true, None, None)).expect("status"))
|
|
849
|
+
});
|
|
850
|
+
assert_eq!(nodes[0]["runtime_status"], "running");
|
|
851
|
+
assert_eq!(nodes[0]["activity"], "idle");
|
|
852
|
+
assert_eq!(nodes[0]["health"], "normal");
|
|
853
|
+
assert_eq!(nodes[0]["session_name"], "pi-session");
|
|
854
|
+
assert_eq!(
|
|
855
|
+
nodes[0]["tmux_command"],
|
|
856
|
+
"tmux -S '/tmp/ta-status-brief.sock' attach -t 'team-demo:worker.%7'"
|
|
857
|
+
);
|
|
858
|
+
let _ = std::fs::remove_dir_all(&ws);
|
|
859
|
+
|
|
860
|
+
let mismatch = brief_workspace("explicit-node-socket-mismatch");
|
|
861
|
+
write_state(
|
|
862
|
+
&mismatch,
|
|
863
|
+
&production_state(json!({"worker": production_agent("worker", "%7")})),
|
|
864
|
+
);
|
|
865
|
+
let mut report: Value =
|
|
866
|
+
serde_json::from_str(&producer_report(
|
|
867
|
+
"/tmp/ta-status-brief.sock",
|
|
868
|
+
"worker",
|
|
869
|
+
"%7",
|
|
870
|
+
"pi_activity_channel",
|
|
871
|
+
))
|
|
872
|
+
.unwrap();
|
|
873
|
+
report["nodes"][0]["socket"] = json!("/tmp/other.sock");
|
|
874
|
+
let report = serde_json::to_string(&report).unwrap();
|
|
875
|
+
let bin = write_nodeprobe(
|
|
876
|
+
&mismatch.join("probe"),
|
|
877
|
+
&format!("cat <<'EOF'\n{report}\nEOF\n"),
|
|
878
|
+
);
|
|
879
|
+
let nodes = status_port::with_test_nodeprobe(bin, || {
|
|
880
|
+
json_nodes(cmd_status(&status_args(&mismatch, true, None, None)).expect("status"))
|
|
881
|
+
});
|
|
882
|
+
assert_eq!(nodes[0]["runtime_status"], "unknown");
|
|
883
|
+
assert!(nodes[0]["tmux_command"].is_null());
|
|
884
|
+
let _ = std::fs::remove_dir_all(&mismatch);
|
|
885
|
+
}
|
|
886
|
+
|
|
688
887
|
#[cfg(unix)]
|
|
689
888
|
fn pid_alive(pid: u32) -> bool {
|
|
690
889
|
unsafe { libc::kill(pid as i32, 0) == 0 }
|
|
@@ -73,6 +73,8 @@ pub(crate) fn detect_abnormal_exits(
|
|
|
73
73
|
let team = crate::state::projection::team_state_key(&snapshot);
|
|
74
74
|
let session_name = snapshot.get("session_name").and_then(Value::as_str);
|
|
75
75
|
for agent in abnormal_watch_agents(&snapshot) {
|
|
76
|
+
// Reject untrusted history without skipping a fresh provider/path observation.
|
|
77
|
+
crate::state::abnormal_watch::begin_observation(state, &agent.agent_id);
|
|
76
78
|
// Pane/process liveness is independent of transcript content. A
|
|
77
79
|
// frozen rollout is common after pane death, so probe before the
|
|
78
80
|
// metadata dedupe gate; only the expensive tail scan stays deduped.
|
|
@@ -109,7 +111,7 @@ pub(crate) fn detect_abnormal_exits(
|
|
|
109
111
|
// re-read below.
|
|
110
112
|
if let (Some(mtime), Some(stored)) = (
|
|
111
113
|
mtime_ns,
|
|
112
|
-
abnormal_watch_stored_metadata(
|
|
114
|
+
abnormal_watch_stored_metadata(state, &agent.agent_id),
|
|
113
115
|
) {
|
|
114
116
|
if stored == (size, mtime) {
|
|
115
117
|
refresh_abnormal_watch_liveness(state, &agent.agent_id, &liveness);
|
|
@@ -149,7 +151,7 @@ pub(crate) fn detect_abnormal_exits(
|
|
|
149
151
|
.map(|fact| abnormal_error_observation_key(&agent, fact));
|
|
150
152
|
let error_observation_cohort = fact.as_ref().map(|_| abnormal_error_cohort_key(&agent));
|
|
151
153
|
let error_recency = abnormal_error_recency(
|
|
152
|
-
|
|
154
|
+
state,
|
|
153
155
|
&agent,
|
|
154
156
|
error_observation_key.as_deref(),
|
|
155
157
|
error_observation_cohort.as_deref(),
|
|
@@ -954,6 +956,9 @@ fn abnormal_watch_payload(
|
|
|
954
956
|
}
|
|
955
957
|
|
|
956
958
|
fn upsert_abnormal_watch(state: &mut Value, agent_id: &str, mut payload: Value) {
|
|
959
|
+
if let Some(identity) = crate::state::abnormal_watch::observation_identity(state, agent_id) {
|
|
960
|
+
payload["watch_identity"] = identity;
|
|
961
|
+
}
|
|
957
962
|
let preserved = [
|
|
958
963
|
"last_notified_key",
|
|
959
964
|
"last_notified_at",
|
|
@@ -1413,6 +1418,121 @@ mod tests {
|
|
|
1413
1418
|
crate::provider::get_adapter(provider)
|
|
1414
1419
|
}
|
|
1415
1420
|
}
|
|
1421
|
+
#[test]
|
|
1422
|
+
fn resource188_foreign_metadata_and_dedupe_cannot_skip_new_baseline() {
|
|
1423
|
+
let dir = temp_abnormal_dir("188-foreign-watch");
|
|
1424
|
+
let rollout = dir.join("rollout-w1.jsonl");
|
|
1425
|
+
let error = "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"old\",\"status\":\"failed\"}}}\n";
|
|
1426
|
+
std::fs::write(&rollout, error).unwrap();
|
|
1427
|
+
seed_abnormal_state(&dir, &rollout, "alive", 1);
|
|
1428
|
+
let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
|
|
1429
|
+
let metadata = std::fs::metadata(&rollout).unwrap();
|
|
1430
|
+
let mut foreign = crate::state::abnormal_watch::observation_identity(&state, "w1").unwrap();
|
|
1431
|
+
foreign["team_key"] = serde_json::json!("another-team");
|
|
1432
|
+
state["coordinator"] = serde_json::json!({"abnormal_exit_watch": {"w1": {
|
|
1433
|
+
"watch_identity": foreign,
|
|
1434
|
+
"size": metadata.len(), "mtime_ns": metadata_mtime_ns(&metadata),
|
|
1435
|
+
"last_notified_key": "foreign", "last_check_key": "foreign",
|
|
1436
|
+
"last_error_observation_key": "foreign", "last_error_observation_cohort": "foreign"
|
|
1437
|
+
}}});
|
|
1438
|
+
let log = EventLog::new(&dir);
|
|
1439
|
+
let transport = crate::transport::test_support::OfflineTransport::new();
|
|
1440
|
+
detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
|
|
1441
|
+
let baseline = state["coordinator"]["abnormal_exit_watch"]["w1"].clone();
|
|
1442
|
+
assert_eq!(baseline["error_recency"], "stale");
|
|
1443
|
+
assert!(baseline.get("last_notified_key").is_none());
|
|
1444
|
+
assert!(crate::state::abnormal_watch::matches_observation(
|
|
1445
|
+
&state, "w1", &baseline
|
|
1446
|
+
));
|
|
1447
|
+
let events = log.tail(0).unwrap();
|
|
1448
|
+
detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
|
|
1449
|
+
assert_eq!(
|
|
1450
|
+
log.tail(0).unwrap(),
|
|
1451
|
+
events,
|
|
1452
|
+
"unchanged baseline must not re-alarm"
|
|
1453
|
+
);
|
|
1454
|
+
assert!(find_event(&events, "worker.abnormal_exit").is_none());
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
#[test]
|
|
1458
|
+
fn resource188_legacy_missing_cohort_still_observes_fresh_errors_and_dedupes() {
|
|
1459
|
+
let dir = temp_abnormal_dir("188-legacy-cohort");
|
|
1460
|
+
let rollout = dir.join("rollout-w1.jsonl");
|
|
1461
|
+
let error = "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"old\",\"status\":\"failed\"}}}\n";
|
|
1462
|
+
std::fs::write(&rollout, error).unwrap();
|
|
1463
|
+
seed_abnormal_state(&dir, &rollout, "alive", 1);
|
|
1464
|
+
let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
|
|
1465
|
+
state["agents"]["w1"]
|
|
1466
|
+
.as_object_mut()
|
|
1467
|
+
.unwrap()
|
|
1468
|
+
.remove("spawn_epoch");
|
|
1469
|
+
let log = EventLog::new(&dir);
|
|
1470
|
+
let transport = crate::transport::test_support::OfflineTransport::new();
|
|
1471
|
+
detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
|
|
1472
|
+
assert_eq!(
|
|
1473
|
+
state["coordinator"]["abnormal_exit_watch"]["w1"]["error_recency"],
|
|
1474
|
+
"stale"
|
|
1475
|
+
);
|
|
1476
|
+
assert!(find_event(&log.tail(0).unwrap(), "worker.abnormal_exit").is_none());
|
|
1477
|
+
std::fs::OpenOptions::new()
|
|
1478
|
+
.append(true)
|
|
1479
|
+
.open(&rollout)
|
|
1480
|
+
.unwrap()
|
|
1481
|
+
.write_all(error.replace("old", "new-error").as_bytes())
|
|
1482
|
+
.unwrap();
|
|
1483
|
+
detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
|
|
1484
|
+
let events = log.tail(0).unwrap();
|
|
1485
|
+
assert_eq!(
|
|
1486
|
+
find_event(&events, "worker.abnormal_exit").unwrap()["error_recency"],
|
|
1487
|
+
"fresh"
|
|
1488
|
+
);
|
|
1489
|
+
let notified =
|
|
1490
|
+
state["coordinator"]["abnormal_exit_watch"]["w1"]["last_notified_key"].clone();
|
|
1491
|
+
assert!(notified.is_string());
|
|
1492
|
+
detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
|
|
1493
|
+
assert_eq!(log.tail(0).unwrap(), events);
|
|
1494
|
+
assert_eq!(
|
|
1495
|
+
state["coordinator"]["abnormal_exit_watch"]["w1"]["last_notified_key"],
|
|
1496
|
+
notified
|
|
1497
|
+
);
|
|
1498
|
+
state["agents"]["w1"]["spawn_epoch"] = serde_json::json!(2);
|
|
1499
|
+
detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
|
|
1500
|
+
let watch = &state["coordinator"]["abnormal_exit_watch"]["w1"];
|
|
1501
|
+
assert_eq!(
|
|
1502
|
+
watch["error_recency"], "stale",
|
|
1503
|
+
"a known generation must rebaseline unchanged bytes"
|
|
1504
|
+
);
|
|
1505
|
+
assert!(watch.get("last_notified_key").is_none());
|
|
1506
|
+
assert_eq!(
|
|
1507
|
+
log.tail(0)
|
|
1508
|
+
.unwrap()
|
|
1509
|
+
.iter()
|
|
1510
|
+
.filter(|event| event["event"] == "worker.abnormal_exit")
|
|
1511
|
+
.count(),
|
|
1512
|
+
1
|
|
1513
|
+
);
|
|
1514
|
+
}
|
|
1515
|
+
|
|
1516
|
+
#[test]
|
|
1517
|
+
fn resource188_unknown_team_still_records_fresh_liveness_without_reusing_cache() {
|
|
1518
|
+
let dir = temp_abnormal_dir("188-unknown-team");
|
|
1519
|
+
let rollout = dir.join("rollout-w1.jsonl");
|
|
1520
|
+
std::fs::write(&rollout, "").unwrap();
|
|
1521
|
+
seed_abnormal_state(&dir, &rollout, "dead", 1);
|
|
1522
|
+
let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
|
|
1523
|
+
state.as_object_mut().unwrap().remove("active_team_key");
|
|
1524
|
+
let log = EventLog::new(&dir);
|
|
1525
|
+
let transport = crate::transport::test_support::OfflineTransport::new();
|
|
1526
|
+
detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
|
|
1527
|
+
assert_eq!(
|
|
1528
|
+
state["coordinator"]["abnormal_exit_watch"]["w1"]["last_liveness"],
|
|
1529
|
+
"dead"
|
|
1530
|
+
);
|
|
1531
|
+
assert!(state["coordinator"]["abnormal_exit_watch"]["w1"]
|
|
1532
|
+
.get("watch_identity")
|
|
1533
|
+
.is_none());
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1416
1536
|
#[test]
|
|
1417
1537
|
fn abnormal_stale_error_baselines_then_fresh_alive_error_notifies() {
|
|
1418
1538
|
let dir = temp_abnormal_dir("alive-fresh");
|