@team-agent/installer 0.5.78 → 0.5.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -1
- package/Cargo.toml +1 -1
- package/crates/team-agent/src/cli/models.rs +31 -0
- package/crates/team-agent/src/cli/status_port/brief.rs +40 -16
- package/crates/team-agent/src/cli/status_port/runtime.rs +3 -1
- package/crates/team-agent/src/cli/status_port/tests.rs +37 -1
- package/crates/team-agent/src/cli/tests/status_brief.rs +199 -0
- package/crates/team-agent/src/coordinator/steps/abnormal.rs +122 -2
- package/crates/team-agent/src/coordinator/tests/mod.rs +1 -0
- package/crates/team-agent/src/coordinator/tests/s3_bounded.rs +239 -0
- package/crates/team-agent/src/coordinator/tick.rs +21 -13
- package/crates/team-agent/src/leader/start.rs +46 -74
- package/crates/team-agent/src/lifecycle/launch/pi_mcp.rs +21 -5
- package/crates/team-agent/src/lifecycle/tests/fixtures/pi_list_models_spaced_ids.stdout.txt +9 -0
- package/crates/team-agent/src/lifecycle/tests/pi_catalog_red.rs +71 -0
- package/crates/team-agent/src/mcp_server/lifecycle_tools/state_status.rs +2 -11
- package/crates/team-agent/src/mcp_server/tools.rs +1 -3
- package/crates/team-agent/src/messaging/delivery.rs +154 -72
- package/crates/team-agent/src/messaging/results.rs +1 -2
- package/crates/team-agent/src/messaging/tests/leader_inject_acceptance.rs +127 -0
- package/crates/team-agent/src/provider/session_scan/common/tests.rs +204 -0
- package/crates/team-agent/src/provider/session_scan/common.rs +24 -2
- package/crates/team-agent/src/state/abnormal_watch/tests/mod.rs +248 -0
- package/crates/team-agent/src/state/abnormal_watch.rs +144 -0
- package/crates/team-agent/src/state/mod.rs +1 -0
- package/crates/team-agent/src/state/persist.rs +47 -5
- package/crates/team-agent/src/state/projection.rs +31 -1
- package/crates/team-agent/src/state/repository/tests.rs +81 -0
- package/crates/team-agent/src/state/repository.rs +169 -1
- package/crates/team-agent/src/tmux_backend/tests.rs +28 -0
- package/crates/team-agent/src/tmux_backend.rs +31 -9
- package/crates/team-agent/src/transport.rs +6 -0
- package/package.json +4 -4
|
@@ -441,9 +441,11 @@ fn collect_candidate_files(
|
|
|
441
441
|
allowed_team_root: Option<&Path>,
|
|
442
442
|
out: &mut Vec<SessionCandidate>,
|
|
443
443
|
) -> Result<(), ProviderError> {
|
|
444
|
-
if depth > 4 {
|
|
444
|
+
if depth > 4 || !may_traverse_session_directory(dir, allowed_team_root) {
|
|
445
445
|
return Ok(());
|
|
446
446
|
}
|
|
447
|
+
#[cfg(test)]
|
|
448
|
+
tests::record_directory_read(dir);
|
|
447
449
|
let entries = match std::fs::read_dir(dir) {
|
|
448
450
|
Ok(entries) => entries,
|
|
449
451
|
Err(e) if depth == 0 => return Err(ProviderError::Io(format!("{}: {e}", dir.display()))),
|
|
@@ -453,8 +455,23 @@ fn collect_candidate_files(
|
|
|
453
455
|
let Ok(entry) = entry else {
|
|
454
456
|
continue;
|
|
455
457
|
};
|
|
458
|
+
#[cfg(test)]
|
|
459
|
+
tests::record_entry();
|
|
456
460
|
let path = entry.path();
|
|
457
|
-
if path
|
|
461
|
+
if !may_traverse_session_directory(&path, allowed_team_root) {
|
|
462
|
+
continue;
|
|
463
|
+
}
|
|
464
|
+
// read_dir already supplies the type for ordinary entries. Only links
|
|
465
|
+
// (or an unavailable type) need the old, following metadata lookup.
|
|
466
|
+
let is_dir = match entry.file_type() {
|
|
467
|
+
Ok(kind) if !kind.is_symlink() => kind.is_dir(),
|
|
468
|
+
_ => {
|
|
469
|
+
#[cfg(test)]
|
|
470
|
+
tests::record_following_metadata();
|
|
471
|
+
path.is_dir()
|
|
472
|
+
}
|
|
473
|
+
};
|
|
474
|
+
if is_dir {
|
|
458
475
|
collect_candidate_files(
|
|
459
476
|
&path,
|
|
460
477
|
agent_id,
|
|
@@ -473,6 +490,11 @@ fn collect_candidate_files(
|
|
|
473
490
|
Ok(())
|
|
474
491
|
}
|
|
475
492
|
|
|
493
|
+
fn may_traverse_session_directory(path: &Path, allowed_team_root: Option<&Path>) -> bool {
|
|
494
|
+
!path_is_under_team_runtime(path)
|
|
495
|
+
|| allowed_team_root.is_some_and(|root| path.starts_with(root) || root.starts_with(path))
|
|
496
|
+
}
|
|
497
|
+
|
|
476
498
|
fn looks_like_session_file(path: &Path, agent_id: &str, allowed_team_root: Option<&Path>) -> bool {
|
|
477
499
|
if path_is_under_team_runtime(path)
|
|
478
500
|
&& !allowed_team_root.is_some_and(|root| path.starts_with(root))
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
use super::*;
|
|
2
|
+
use crate::state::persist::{load_runtime_state, runtime_state_path, save_runtime_state};
|
|
3
|
+
|
|
4
|
+
fn workspace() -> std::path::PathBuf {
|
|
5
|
+
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
6
|
+
static NEXT: AtomicUsize = AtomicUsize::new(0);
|
|
7
|
+
let path = std::env::temp_dir().join(format!(
|
|
8
|
+
"watch-188-{}-{}",
|
|
9
|
+
std::process::id(),
|
|
10
|
+
NEXT.fetch_add(1, Ordering::Relaxed)
|
|
11
|
+
));
|
|
12
|
+
std::fs::create_dir_all(path.join(".team/runtime")).unwrap();
|
|
13
|
+
path
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
#[test]
|
|
17
|
+
fn complete_empty_roster_prunes_but_missing_roster_and_paused_members_survive() {
|
|
18
|
+
let mut state = json!({
|
|
19
|
+
"agents": {},
|
|
20
|
+
"teams": {
|
|
21
|
+
"empty": {"agents": {}, "coordinator": {"abnormal_exit_watch": {"old": {}}}},
|
|
22
|
+
"unknown": {"coordinator": {"abnormal_exit_watch": {"old": {}}}},
|
|
23
|
+
"paused": {"agents": {"p": {"status": "paused"}}, "coordinator": {
|
|
24
|
+
"abnormal_exit_watch": {"p": {"last_notified_key": "keep"}, "gone": {}}
|
|
25
|
+
}}
|
|
26
|
+
},
|
|
27
|
+
"coordinator": {"abnormal_exit_watch": {"unknown": {}}}
|
|
28
|
+
});
|
|
29
|
+
compact_after_merge(&mut state, None, &[]);
|
|
30
|
+
assert_eq!(
|
|
31
|
+
state["teams"]["empty"]["coordinator"]["abnormal_exit_watch"],
|
|
32
|
+
json!({})
|
|
33
|
+
);
|
|
34
|
+
assert_eq!(
|
|
35
|
+
state["teams"]["unknown"]["coordinator"]["abnormal_exit_watch"],
|
|
36
|
+
json!({"old": {}})
|
|
37
|
+
);
|
|
38
|
+
assert_eq!(
|
|
39
|
+
state["teams"]["paused"]["coordinator"]["abnormal_exit_watch"],
|
|
40
|
+
json!({"p": {"last_notified_key": "keep"}})
|
|
41
|
+
);
|
|
42
|
+
assert_eq!(
|
|
43
|
+
state["coordinator"]["abnormal_exit_watch"],
|
|
44
|
+
json!({"unknown": {}})
|
|
45
|
+
);
|
|
46
|
+
assert!(!needs_compaction(&state));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
#[test]
|
|
50
|
+
fn root_legacy_union_is_not_migrated_by_bare_agent_name() {
|
|
51
|
+
let mut state = json!({"agents": {}, "teams": {
|
|
52
|
+
"a": {"agents": {"w": {}}}, "b": {"agents": {"w": {}}}
|
|
53
|
+
}, "coordinator": {"abnormal_exit_watch": {"w": {"last_notified_key": "legacy"}, "gone": {}}}});
|
|
54
|
+
compact_after_merge(&mut state, None, &[]);
|
|
55
|
+
assert_eq!(
|
|
56
|
+
state["coordinator"]["abnormal_exit_watch"]
|
|
57
|
+
.as_object()
|
|
58
|
+
.unwrap()
|
|
59
|
+
.len(),
|
|
60
|
+
1
|
|
61
|
+
);
|
|
62
|
+
for team in ["a", "b"] {
|
|
63
|
+
let projection = crate::state::projection::project_top_level_view(&state, team);
|
|
64
|
+
assert!(projection["coordinator"]
|
|
65
|
+
.get("abnormal_exit_watch")
|
|
66
|
+
.is_none());
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
#[test]
|
|
71
|
+
fn fallback_preserves_other_coordinator_fields_and_owned_watch() {
|
|
72
|
+
let state = json!({"coordinator": {"tick": 9, "abnormal_exit_watch": {"w": {}}}, "teams": {
|
|
73
|
+
"a": {"agents": {}},
|
|
74
|
+
"b": {"agents": {}, "coordinator": {"abnormal_exit_watch": {"owned": {}}, "tick": 2}}
|
|
75
|
+
}});
|
|
76
|
+
let a = crate::state::projection::project_top_level_view(&state, "a");
|
|
77
|
+
let b = crate::state::projection::project_top_level_view(&state, "b");
|
|
78
|
+
assert_eq!(a["coordinator"], json!({"tick": 9}));
|
|
79
|
+
assert_eq!(b["coordinator"], state["teams"]["b"]["coordinator"]);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
fn observed() -> Value {
|
|
83
|
+
let mut state = json!({"active_team_key": "a", "agents": {"w": {
|
|
84
|
+
"provider": "codex", "rollout_path": "/fixture/w.jsonl", "spawn_epoch": 1
|
|
85
|
+
}}, "coordinator": {"abnormal_exit_watch": {"w": {"last_notified_key": "keep"}}}});
|
|
86
|
+
state["coordinator"]["abnormal_exit_watch"]["w"]["watch_identity"] =
|
|
87
|
+
observation_identity(&state, "w").unwrap();
|
|
88
|
+
state
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
#[test]
|
|
92
|
+
fn observation_identity_separates_same_name_team_path_and_spawn_generation() {
|
|
93
|
+
let initial = observed();
|
|
94
|
+
let mut unchanged = initial.clone();
|
|
95
|
+
begin_observation(&mut unchanged, "w");
|
|
96
|
+
begin_observation(&mut unchanged, "w");
|
|
97
|
+
assert_eq!(
|
|
98
|
+
unchanged, initial,
|
|
99
|
+
"same observation retains notification dedupe"
|
|
100
|
+
);
|
|
101
|
+
for (pointer, value) in [
|
|
102
|
+
("/active_team_key", json!("b")),
|
|
103
|
+
("/agents/w/rollout_path", json!("/fixture/other.jsonl")),
|
|
104
|
+
("/agents/w/spawn_epoch", json!(2)),
|
|
105
|
+
("/agents/w/provider", json!("claude")),
|
|
106
|
+
("/agents/w/spawned_at", json!("new-incarnation")),
|
|
107
|
+
] {
|
|
108
|
+
let mut changed = initial.clone();
|
|
109
|
+
if pointer.ends_with("spawned_at") {
|
|
110
|
+
changed["agents"]["w"]["spawned_at"] = value;
|
|
111
|
+
} else {
|
|
112
|
+
*changed.pointer_mut(pointer).unwrap() = value;
|
|
113
|
+
}
|
|
114
|
+
begin_observation(&mut changed, "w");
|
|
115
|
+
assert!(changed["coordinator"]["abnormal_exit_watch"]
|
|
116
|
+
.get("w")
|
|
117
|
+
.is_none());
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
#[test]
|
|
122
|
+
fn legacy_unknown_identity_is_retained_until_member_is_observed() {
|
|
123
|
+
let mut state = observed();
|
|
124
|
+
state["coordinator"]["abnormal_exit_watch"]["w"]
|
|
125
|
+
.as_object_mut()
|
|
126
|
+
.unwrap()
|
|
127
|
+
.remove("watch_identity");
|
|
128
|
+
let legacy = state.clone();
|
|
129
|
+
compact_after_merge(&mut state, None, &[]);
|
|
130
|
+
assert_eq!(
|
|
131
|
+
state, legacy,
|
|
132
|
+
"membership compaction cannot guess legacy identity"
|
|
133
|
+
);
|
|
134
|
+
begin_observation(&mut state, "w");
|
|
135
|
+
assert!(state["coordinator"]["abnormal_exit_watch"]
|
|
136
|
+
.get("w")
|
|
137
|
+
.is_none());
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
#[test]
|
|
141
|
+
fn unchanged_normal_save_compacts_large_old_snapshots_and_cannot_resurrect_them() {
|
|
142
|
+
let ws = workspace();
|
|
143
|
+
let watch: serde_json::Map<String, Value> = (0..100)
|
|
144
|
+
.map(|n| {
|
|
145
|
+
(
|
|
146
|
+
format!("old-{n}"),
|
|
147
|
+
json!({"last_notified_key": "x".repeat(100)}),
|
|
148
|
+
)
|
|
149
|
+
})
|
|
150
|
+
.collect();
|
|
151
|
+
let mut state =
|
|
152
|
+
json!({"agents": {}, "teams": {}, "coordinator": {"abnormal_exit_watch": watch}});
|
|
153
|
+
for n in 0..8 {
|
|
154
|
+
state["teams"][format!("team-{n}")] =
|
|
155
|
+
json!({"agents": {}, "coordinator": state["coordinator"].clone()});
|
|
156
|
+
}
|
|
157
|
+
let original = serde_json::to_vec(&state).unwrap().len();
|
|
158
|
+
// Warm the old cache without invoking the load-time migrations, which
|
|
159
|
+
// could otherwise compact before the no-change save under test.
|
|
160
|
+
crate::state::persist::save_runtime_state_without_migrations(&ws, &state).unwrap();
|
|
161
|
+
let stale = state;
|
|
162
|
+
assert_eq!(
|
|
163
|
+
serde_json::from_slice::<Value>(&std::fs::read(runtime_state_path(&ws)).unwrap())
|
|
164
|
+
.unwrap(),
|
|
165
|
+
stale
|
|
166
|
+
);
|
|
167
|
+
save_runtime_state(&ws, &stale).unwrap();
|
|
168
|
+
let first = std::fs::read(runtime_state_path(&ws)).unwrap();
|
|
169
|
+
assert!(
|
|
170
|
+
first.len() < original / 10,
|
|
171
|
+
"size must depend on actual members, not watch x teams"
|
|
172
|
+
);
|
|
173
|
+
save_runtime_state(&ws, &stale).unwrap();
|
|
174
|
+
assert_eq!(std::fs::read(runtime_state_path(&ws)).unwrap(), first);
|
|
175
|
+
std::fs::remove_dir_all(ws).unwrap();
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
#[test]
|
|
179
|
+
fn latest_roster_merge_precedes_pruning_and_explicit_deletion_removes_watch() {
|
|
180
|
+
let ws = workspace();
|
|
181
|
+
let latest = json!({"active_team_key": "a", "session_name": "team-a", "agents": {
|
|
182
|
+
"w": {"agent_id": "w", "provider": "codex", "status": "paused"}
|
|
183
|
+
}, "coordinator": {"abnormal_exit_watch": {"w": {"last_notified_key": "keep"}}}});
|
|
184
|
+
save_runtime_state(&ws, &latest).unwrap();
|
|
185
|
+
let mut stale = load_runtime_state(&ws).unwrap();
|
|
186
|
+
stale["agents"] = json!({});
|
|
187
|
+
save_runtime_state(&ws, &stale).unwrap();
|
|
188
|
+
let merged = load_runtime_state(&ws).unwrap();
|
|
189
|
+
assert!(merged["agents"].get("w").is_some());
|
|
190
|
+
assert_eq!(
|
|
191
|
+
merged["coordinator"]["abnormal_exit_watch"]["w"]["last_notified_key"],
|
|
192
|
+
"keep"
|
|
193
|
+
);
|
|
194
|
+
crate::state::persist::save_runtime_state_with_deleted_agents(&ws, &stale, &["w"]).unwrap();
|
|
195
|
+
let removed = load_runtime_state(&ws).unwrap();
|
|
196
|
+
assert!(removed["agents"].get("w").is_none());
|
|
197
|
+
assert_eq!(removed["coordinator"]["abnormal_exit_watch"], json!({}));
|
|
198
|
+
std::fs::remove_dir_all(ws).unwrap();
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
#[test]
|
|
202
|
+
fn unknown_team_retains_history_until_observation_but_never_authorizes_reuse() {
|
|
203
|
+
let mut state = observed();
|
|
204
|
+
state["active_team_key"] = Value::Null;
|
|
205
|
+
let before = state.clone();
|
|
206
|
+
assert!(observation_identity(&state, "w").is_none());
|
|
207
|
+
compact_after_merge(&mut state, None, &[]);
|
|
208
|
+
assert_eq!(state, before);
|
|
209
|
+
assert!(!matches_observation(
|
|
210
|
+
&state,
|
|
211
|
+
"w",
|
|
212
|
+
&state["coordinator"]["abnormal_exit_watch"]["w"]
|
|
213
|
+
));
|
|
214
|
+
begin_observation(&mut state, "w");
|
|
215
|
+
assert!(state["coordinator"]["abnormal_exit_watch"]
|
|
216
|
+
.get("w")
|
|
217
|
+
.is_none());
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
#[test]
|
|
221
|
+
fn malformed_team_catalog_does_not_authorize_root_pruning() {
|
|
222
|
+
let mut state =
|
|
223
|
+
json!({"agents": {}, "teams": [], "coordinator": {"abnormal_exit_watch": {"w": {}}}});
|
|
224
|
+
let before = state.clone();
|
|
225
|
+
compact_after_merge(&mut state, None, &[]);
|
|
226
|
+
assert_eq!(state, before);
|
|
227
|
+
assert!(!needs_compaction(&state));
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
#[test]
|
|
231
|
+
fn lock_held_latest_membership_protects_rows_without_roster_stub_fields() {
|
|
232
|
+
let latest = json!({"agents": {"unknown-row": {}}, "teams": {
|
|
233
|
+
"new-sibling": {"agents": {"sibling": {}}}
|
|
234
|
+
}});
|
|
235
|
+
let mut incoming = json!({"agents": {}, "coordinator": {"abnormal_exit_watch": {
|
|
236
|
+
"unknown-row": {}, "sibling": {}, "gone": {}
|
|
237
|
+
}}});
|
|
238
|
+
compact_after_merge(&mut incoming, Some(&latest), &[]);
|
|
239
|
+
assert_eq!(
|
|
240
|
+
incoming["coordinator"]["abnormal_exit_watch"],
|
|
241
|
+
json!({"unknown-row": {}, "sibling": {}})
|
|
242
|
+
);
|
|
243
|
+
compact_after_merge(&mut incoming, Some(&latest), &["unknown-row"]);
|
|
244
|
+
assert_eq!(
|
|
245
|
+
incoming["coordinator"]["abnormal_exit_watch"],
|
|
246
|
+
json!({"sibling": {}})
|
|
247
|
+
);
|
|
248
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
//! Scope and lifecycle boundaries for the abnormal-exit observation cache.
|
|
2
|
+
use std::collections::BTreeSet;
|
|
3
|
+
|
|
4
|
+
use serde_json::{json, Value};
|
|
5
|
+
|
|
6
|
+
fn members(state: &Value) -> Option<BTreeSet<String>> {
|
|
7
|
+
Some(state.get("agents")?.as_object()?.keys().cloned().collect())
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
// A missing roster is unknown. Only a complete, lock-merged roster authorizes
|
|
11
|
+
// discarding absent members; liveness, pause status and record age do not.
|
|
12
|
+
fn scopes(state: &Value) -> Vec<(Option<String>, Option<BTreeSet<String>>)> {
|
|
13
|
+
let mut root = members(state);
|
|
14
|
+
if state.get("teams").is_some_and(|teams| !teams.is_object()) {
|
|
15
|
+
root = None;
|
|
16
|
+
}
|
|
17
|
+
let mut scopes = Vec::new();
|
|
18
|
+
if let Some(teams) = state.get("teams").and_then(Value::as_object) {
|
|
19
|
+
for (team, entry) in teams {
|
|
20
|
+
let roster = members(entry);
|
|
21
|
+
root = match (root, &roster) {
|
|
22
|
+
(Some(mut union), Some(keys)) => {
|
|
23
|
+
union.extend(keys.iter().cloned());
|
|
24
|
+
Some(union)
|
|
25
|
+
}
|
|
26
|
+
_ => None,
|
|
27
|
+
};
|
|
28
|
+
scopes.push((Some(team.clone()), roster));
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
scopes.push((None, root));
|
|
32
|
+
scopes
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
pub(crate) fn needs_compaction(state: &Value) -> bool {
|
|
36
|
+
scopes(state).into_iter().any(|(team, roster)| {
|
|
37
|
+
let entry = team.as_ref().map_or(state, |key| &state["teams"][key]);
|
|
38
|
+
roster.is_some_and(|keys| {
|
|
39
|
+
entry
|
|
40
|
+
.pointer("/coordinator/abnormal_exit_watch")
|
|
41
|
+
.and_then(Value::as_object)
|
|
42
|
+
.is_some_and(|watch| watch.keys().any(|key| !keys.contains(key)))
|
|
43
|
+
})
|
|
44
|
+
})
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/// Call only after the persistence lock's latest-roster merge.
|
|
48
|
+
pub(crate) fn compact_after_merge(state: &mut Value, latest: Option<&Value>, deleted: &[&str]) {
|
|
49
|
+
let latest_scopes = latest.map(scopes).unwrap_or_default();
|
|
50
|
+
for (team, mut roster) in scopes(state) {
|
|
51
|
+
// The normal merge may omit a roster row with no allowlisted stub
|
|
52
|
+
// fields, or a whole sibling absent from a stale input. Membership in
|
|
53
|
+
// lock-held latest still forbids pruning its observation. Only explicit
|
|
54
|
+
// deletion authority can subtract it from this retention union.
|
|
55
|
+
if let Some((_, latest_roster)) = latest_scopes.iter().find(|(key, _)| key == &team) {
|
|
56
|
+
roster = match (roster, latest_roster) {
|
|
57
|
+
(Some(mut keys), Some(latest_keys)) => {
|
|
58
|
+
keys.extend(
|
|
59
|
+
latest_keys
|
|
60
|
+
.iter()
|
|
61
|
+
.filter(|key| !deleted.contains(&key.as_str()))
|
|
62
|
+
.cloned(),
|
|
63
|
+
);
|
|
64
|
+
Some(keys)
|
|
65
|
+
}
|
|
66
|
+
_ => None,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
let Some(keys) = roster else { continue };
|
|
70
|
+
let entry = match team {
|
|
71
|
+
Some(team) => &mut state["teams"][&team],
|
|
72
|
+
None => &mut *state,
|
|
73
|
+
};
|
|
74
|
+
if let Some(watch) = entry
|
|
75
|
+
.pointer_mut("/coordinator/abnormal_exit_watch")
|
|
76
|
+
.and_then(Value::as_object_mut)
|
|
77
|
+
{
|
|
78
|
+
watch.retain(|key, _| keys.contains(key));
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/// New observations carry their team and cohort, never just a bare agent id.
|
|
84
|
+
pub(crate) fn observation_identity(state: &Value, agent_id: &str) -> Option<Value> {
|
|
85
|
+
let agent = state.get("agents")?.get(agent_id)?;
|
|
86
|
+
let provider = crate::provider::wire::parse_provider(agent.get("provider")?.as_str()?)?;
|
|
87
|
+
let path = ["rollout_path", "transcript_path", "session_log_path"]
|
|
88
|
+
.into_iter()
|
|
89
|
+
.find_map(|key| agent.get(key).and_then(Value::as_str))
|
|
90
|
+
.filter(|path| !path.is_empty())?;
|
|
91
|
+
let spawn_epoch = agent.get("spawn_epoch").and_then(Value::as_u64);
|
|
92
|
+
let spawned_at = agent
|
|
93
|
+
.get("spawned_at")
|
|
94
|
+
.and_then(Value::as_str)
|
|
95
|
+
.filter(|value| !value.is_empty());
|
|
96
|
+
// Preserve the watcher's existing legacy cohort: missing spawn fields
|
|
97
|
+
// remain explicit nulls, never a fabricated generation. A later known
|
|
98
|
+
// cohort changes this identity and establishes a new baseline.
|
|
99
|
+
let team = state
|
|
100
|
+
.get("active_team_key")
|
|
101
|
+
.and_then(Value::as_str)
|
|
102
|
+
.filter(|team| !team.is_empty())
|
|
103
|
+
.map(str::to_string)
|
|
104
|
+
.or_else(|| {
|
|
105
|
+
let team = super::projection::team_state_key(state);
|
|
106
|
+
(team != super::projection::CURRENT_TEAM_ALIAS).then_some(team)
|
|
107
|
+
})?;
|
|
108
|
+
Some(json!({
|
|
109
|
+
"team_key": team,
|
|
110
|
+
"agent_id": agent_id,
|
|
111
|
+
"provider": crate::provider::wire::provider_wire(provider),
|
|
112
|
+
"path": path,
|
|
113
|
+
"spawn_epoch": spawn_epoch,
|
|
114
|
+
"spawned_at": spawned_at,
|
|
115
|
+
}))
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
pub(crate) fn matches_observation(state: &Value, agent_id: &str, row: &Value) -> bool {
|
|
119
|
+
observation_identity(state, agent_id)
|
|
120
|
+
.is_some_and(|identity| row.get("watch_identity") == Some(&identity))
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/// Legacy/foreign-cohort facts cannot be reused as current observation evidence.
|
|
124
|
+
/// This is invoked only when actually observing the member, not for paused or
|
|
125
|
+
/// otherwise ineligible members. The next scan establishes the ordinary baseline.
|
|
126
|
+
pub(crate) fn begin_observation(state: &mut Value, agent_id: &str) {
|
|
127
|
+
let identity = observation_identity(state, agent_id);
|
|
128
|
+
if let Some(watch) = state
|
|
129
|
+
.pointer_mut("/coordinator/abnormal_exit_watch")
|
|
130
|
+
.and_then(Value::as_object_mut)
|
|
131
|
+
{
|
|
132
|
+
if watch.get(agent_id).is_some_and(|row| {
|
|
133
|
+
identity
|
|
134
|
+
.as_ref()
|
|
135
|
+
.is_none_or(|identity| row.get("watch_identity") != Some(identity))
|
|
136
|
+
}) {
|
|
137
|
+
watch.remove(agent_id);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
#[cfg(test)]
|
|
143
|
+
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
|
144
|
+
mod tests;
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
//! 绝不 in-place truncate,绝不让审计/重试失败拖垮可见的原 state。
|
|
16
16
|
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
|
17
17
|
|
|
18
|
+
pub(crate) mod abnormal_watch;
|
|
18
19
|
pub mod identity;
|
|
19
20
|
// Stage 0 of identity-boundary unified plan (architect direction 2026-06-23):
|
|
20
21
|
// shared identity primitives. Pure additive — Stage 1+ consume these.
|
|
@@ -492,7 +492,8 @@ fn persist_runtime_state_with_merge_options_and_expected(
|
|
|
492
492
|
// Exact-seed cleanup must observe lock-held disk, not a pre-lock cache or
|
|
493
493
|
// byte-equality hit. A matching in-memory document would otherwise no-op
|
|
494
494
|
// and leave the seed, or skip copying a concurrent owner.
|
|
495
|
-
let skip_pre_lock_fast_path =
|
|
495
|
+
let skip_pre_lock_fast_path = super::abnormal_watch::needs_compaction(&migrated)
|
|
496
|
+
|| exact_owner_seed_to_clear.is_some()
|
|
496
497
|
|| expected_owner_receiver.is_some()
|
|
497
498
|
|| exact_owner_receiver_to_clear.is_some()
|
|
498
499
|
|| exact_owner_receiver_to_restore.is_some();
|
|
@@ -542,7 +543,7 @@ fn persist_runtime_state_with_merge_options_and_expected(
|
|
|
542
543
|
));
|
|
543
544
|
}
|
|
544
545
|
}
|
|
545
|
-
if let Some(latest) = latest {
|
|
546
|
+
if let Some(latest) = latest.as_ref() {
|
|
546
547
|
let deleted = deleted_agent_ids
|
|
547
548
|
.iter()
|
|
548
549
|
.copied()
|
|
@@ -572,7 +573,7 @@ fn persist_runtime_state_with_merge_options_and_expected(
|
|
|
572
573
|
.collect::<BTreeSet<_>>();
|
|
573
574
|
apply_persist_merge_contract(
|
|
574
575
|
&mut migrated,
|
|
575
|
-
|
|
576
|
+
latest,
|
|
576
577
|
&deleted,
|
|
577
578
|
skip_capture_backfill_team_key,
|
|
578
579
|
&skip_capture_backfill,
|
|
@@ -583,14 +584,55 @@ fn persist_runtime_state_with_merge_options_and_expected(
|
|
|
583
584
|
exact_owner_receiver_to_restore,
|
|
584
585
|
)?;
|
|
585
586
|
}
|
|
587
|
+
super::abnormal_watch::compact_after_merge(&mut migrated, latest.as_ref(), deleted_agent_ids);
|
|
586
588
|
// Stage 3 save-output strip second pass (defence-in-depth): after the
|
|
587
589
|
// `preserve_latest_roster_entries` lock-held merge, a future addition
|
|
588
590
|
// to the preserve path could re-introduce root owner fields from the
|
|
589
591
|
// on-disk latest. Re-strip here so the serialized payload is always
|
|
590
592
|
// canonical-only when a canonical teams.<key>.team_owner is present.
|
|
591
593
|
crate::state::ownership::strip_top_level_ownership_if_canonical_present(&mut migrated);
|
|
592
|
-
|
|
593
|
-
|
|
594
|
+
write_state_under_lock(workspace, &path, &migrated)
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
/// Apply a bounded writer to disk under state-save, without a cache shortcut.
|
|
598
|
+
pub(super) fn update_runtime_state(
|
|
599
|
+
workspace: &Path,
|
|
600
|
+
update: impl FnOnce(&Value) -> Result<Value, StateError>,
|
|
601
|
+
) -> Result<Value, StateError> {
|
|
602
|
+
let path = runtime_state_path(workspace);
|
|
603
|
+
let _lock = RuntimeLock::acquire(workspace, "state-save", 2.0)?;
|
|
604
|
+
// Malformed/unreadable state is an error, never an empty store.
|
|
605
|
+
let mut latest = match std::fs::read_to_string(&path) {
|
|
606
|
+
Ok(text) => serde_json::from_str(&text)?,
|
|
607
|
+
Err(error) if error.kind() == io::ErrorKind::NotFound => json!({}),
|
|
608
|
+
Err(error) => return Err(error.into()),
|
|
609
|
+
};
|
|
610
|
+
normalize_agent_session_state(&mut latest);
|
|
611
|
+
migrate_state_identity(&mut latest, &SystemEnv, workspace)?;
|
|
612
|
+
let _ = migrate_active_team_key(&mut latest);
|
|
613
|
+
let mut committed = update(&latest)?;
|
|
614
|
+
merge_ordinary_state(&mut committed, &latest)?;
|
|
615
|
+
migrate_state_identity(&mut committed, &SystemEnv, workspace)?;
|
|
616
|
+
crate::state::ownership::strip_top_level_ownership_if_canonical_present(&mut committed);
|
|
617
|
+
if committed == latest {
|
|
618
|
+
cache_set(&path, &committed);
|
|
619
|
+
return Ok(committed);
|
|
620
|
+
}
|
|
621
|
+
write_state_under_lock(workspace, &path, &committed)?;
|
|
622
|
+
Ok(committed)
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/// Reuse ordinary roster, topology, ownership, capture and endpoint guards.
|
|
626
|
+
pub(super) fn merge_ordinary_state(incoming: &mut Value, latest: &Value) -> Result<(), StateError> {
|
|
627
|
+
apply_persist_merge_contract(
|
|
628
|
+
incoming, latest, &BTreeSet::new(), None, &BTreeSet::new(),
|
|
629
|
+
&BTreeSet::new(), &BTreeSet::new(), None, None, None,
|
|
630
|
+
)
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
fn write_state_under_lock(workspace: &Path, path: &Path, migrated: &Value) -> Result<(), StateError> {
|
|
634
|
+
// Shared retry/self-heal/atomic replacement path; caller owns state-save.
|
|
635
|
+
let payload = serde_json::to_string_pretty(migrated)?;
|
|
594
636
|
let delays = [0.05_f64, 0.2, 0.5];
|
|
595
637
|
|
|
596
638
|
for attempt in 0..=delays.len() {
|
|
@@ -486,7 +486,13 @@ pub fn project_top_level_view(state: &Value, team_key: &str) -> Value {
|
|
|
486
486
|
.is_some_and(|o| o.contains_key("coordinator"))
|
|
487
487
|
&& !p.contains_key("coordinator")
|
|
488
488
|
{
|
|
489
|
-
|
|
489
|
+
let mut coordinator = state["coordinator"].clone();
|
|
490
|
+
// Workspace legacy watch facts have no proven owner in this team.
|
|
491
|
+
// Preserve other coordinator fields, but never broadcast this cache.
|
|
492
|
+
if let Some(coordinator) = coordinator.as_object_mut() {
|
|
493
|
+
coordinator.remove("abnormal_exit_watch");
|
|
494
|
+
}
|
|
495
|
+
p.insert("coordinator".to_string(), coordinator);
|
|
490
496
|
}
|
|
491
497
|
Value::Object(p)
|
|
492
498
|
}
|
|
@@ -769,6 +775,30 @@ pub fn save_team_scoped_state(workspace: &Path, team_state: &Value) -> Result<()
|
|
|
769
775
|
save_team_scoped_state_with_deleted_agents(workspace, team_state, &[])
|
|
770
776
|
}
|
|
771
777
|
|
|
778
|
+
/// Materialize a bounded Team update from the lock-held workspace. Siblings
|
|
779
|
+
/// come only from latest; retain the existing primary/legacy projection rules.
|
|
780
|
+
pub(super) fn merge_committed_team(existing: &Value, team_state: &Value, target_key: &str) -> Value {
|
|
781
|
+
let mut primary = existing.get("session_name").and_then(Value::as_str)
|
|
782
|
+
.filter(|name| !name.is_empty()).map(|_| team_state_key(existing));
|
|
783
|
+
if primary.as_deref().is_some_and(|key| key != target_key)
|
|
784
|
+
&& existing.get("session_name") == team_state.get("session_name")
|
|
785
|
+
{
|
|
786
|
+
primary = Some(target_key.to_string());
|
|
787
|
+
}
|
|
788
|
+
let mut teams = take_object(existing.get("teams"));
|
|
789
|
+
if teams.is_empty() && primary.as_deref() == Some(target_key) {
|
|
790
|
+
return compact_team_state(team_state);
|
|
791
|
+
}
|
|
792
|
+
teams.insert(target_key.to_string(), compact_team_state(team_state));
|
|
793
|
+
let mut merged = if primary.is_none() || primary.as_deref() == Some(target_key) {
|
|
794
|
+
to_object(team_state)
|
|
795
|
+
} else {
|
|
796
|
+
to_object(existing)
|
|
797
|
+
};
|
|
798
|
+
merged.insert("teams".to_string(), Value::Object(teams));
|
|
799
|
+
Value::Object(merged)
|
|
800
|
+
}
|
|
801
|
+
|
|
772
802
|
pub(crate) fn save_team_scoped_state_reapplying_after_conflict<F>(
|
|
773
803
|
workspace: &Path,
|
|
774
804
|
team_state: &Value,
|
|
@@ -46,3 +46,84 @@ fn optional_unmigrated_read_preserves_missing_corrupt_and_valid_shapes(
|
|
|
46
46
|
std::fs::remove_dir_all(workspace)?;
|
|
47
47
|
Ok(())
|
|
48
48
|
}
|
|
49
|
+
|
|
50
|
+
#[test]
|
|
51
|
+
fn coordinator_observations_keep_legacy_identity_and_shape()
|
|
52
|
+
-> Result<(), Box<dyn std::error::Error>> {
|
|
53
|
+
let nonce = std::time::SystemTime::now()
|
|
54
|
+
.duration_since(std::time::UNIX_EPOCH)?
|
|
55
|
+
.as_nanos();
|
|
56
|
+
let workspace =
|
|
57
|
+
std::env::temp_dir().join(format!("state-repository-legacy-observation-{nonce}"));
|
|
58
|
+
let initial = serde_json::json!({
|
|
59
|
+
"active_team_key": "team",
|
|
60
|
+
"agents": {"w1": {"agent_id": "w1", "provider": "codex", "status": "running"}}
|
|
61
|
+
});
|
|
62
|
+
let path = super::helper_workspace_path(&workspace);
|
|
63
|
+
if let Some(parent) = path.parent() {
|
|
64
|
+
std::fs::create_dir_all(parent)?;
|
|
65
|
+
}
|
|
66
|
+
std::fs::write(&path, serde_json::to_vec(&initial)?)?;
|
|
67
|
+
let repository = super::StateRepository::new(&workspace);
|
|
68
|
+
let before = repository.load_workspace()?;
|
|
69
|
+
|
|
70
|
+
let mut observed = before.clone();
|
|
71
|
+
observed["coordinator"] = serde_json::json!({
|
|
72
|
+
"abnormal_exit_watch": {"w1": {"last_error_observation_key": "error-1"}}
|
|
73
|
+
});
|
|
74
|
+
repository.commit_observations(
|
|
75
|
+
StateWriteIntent::CoordinatorTick {
|
|
76
|
+
team_key: "current",
|
|
77
|
+
},
|
|
78
|
+
&before,
|
|
79
|
+
&observed,
|
|
80
|
+
)?;
|
|
81
|
+
let first = repository.load_workspace()?;
|
|
82
|
+
assert!(
|
|
83
|
+
first.get("teams").is_none(),
|
|
84
|
+
"legacy observation must not synthesize teams.current"
|
|
85
|
+
);
|
|
86
|
+
assert_eq!(first["active_team_key"], serde_json::json!("team"));
|
|
87
|
+
assert_eq!(
|
|
88
|
+
first["coordinator"]["abnormal_exit_watch"]["w1"]["last_error_observation_key"],
|
|
89
|
+
serde_json::json!("error-1")
|
|
90
|
+
);
|
|
91
|
+
|
|
92
|
+
let mut observed = first.clone();
|
|
93
|
+
observed["coordinator"]["abnormal_exit_watch"]["w1"]["last_error_observation_key"] =
|
|
94
|
+
serde_json::json!("error-2");
|
|
95
|
+
repository.commit_observations(
|
|
96
|
+
StateWriteIntent::CoordinatorTick {
|
|
97
|
+
team_key: "current",
|
|
98
|
+
},
|
|
99
|
+
&first,
|
|
100
|
+
&observed,
|
|
101
|
+
)?;
|
|
102
|
+
let second = repository.load_workspace()?;
|
|
103
|
+
assert!(
|
|
104
|
+
second.get("teams").is_none(),
|
|
105
|
+
"legacy shape must remain root-only"
|
|
106
|
+
);
|
|
107
|
+
assert_eq!(second["active_team_key"], serde_json::json!("team"));
|
|
108
|
+
assert_eq!(
|
|
109
|
+
second["coordinator"]["abnormal_exit_watch"]["w1"]["last_error_observation_key"],
|
|
110
|
+
serde_json::json!("error-2")
|
|
111
|
+
);
|
|
112
|
+
|
|
113
|
+
let mut changed_latest = second.clone();
|
|
114
|
+
changed_latest["active_team_key"] = serde_json::json!("other");
|
|
115
|
+
std::fs::write(&path, serde_json::to_vec(&changed_latest)?)?;
|
|
116
|
+
let err = repository
|
|
117
|
+
.commit_observations(
|
|
118
|
+
StateWriteIntent::CoordinatorTick {
|
|
119
|
+
team_key: "current",
|
|
120
|
+
},
|
|
121
|
+
&second,
|
|
122
|
+
&observed,
|
|
123
|
+
)
|
|
124
|
+
.expect_err("identity drift must refuse the stale legacy observation");
|
|
125
|
+
assert!(matches!(err, super::StateError::SaveConflict(_)));
|
|
126
|
+
|
|
127
|
+
std::fs::remove_dir_all(workspace)?;
|
|
128
|
+
Ok(())
|
|
129
|
+
}
|