@team-agent/installer 0.5.84 → 0.5.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -1
- package/Cargo.toml +1 -1
- package/crates/team-agent/src/cli/profile.rs +4 -12
- package/crates/team-agent/src/cli/tests/verb_profile.rs +66 -0
- package/crates/team-agent/src/lifecycle/launch/mcp_config.rs +1 -31
- package/crates/team-agent/src/lifecycle/profile_launch.rs +1 -1
- package/crates/team-agent/src/lifecycle/restart/common.rs +2 -1
- package/crates/team-agent/src/lifecycle/tests/copilot_provider_red.rs +179 -8
- package/crates/team-agent/src/messaging/results/collection_tests.rs +719 -0
- package/crates/team-agent/src/messaging/results.rs +104 -27
- package/crates/team-agent/src/provider/adapters/copilot.rs +1 -1
- package/crates/team-agent/src/state/persist.rs +15 -1
- package/package.json +4 -4
|
@@ -0,0 +1,719 @@
|
|
|
1
|
+
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
|
2
|
+
|
|
3
|
+
use super::*;
|
|
4
|
+
use serde_json::{json, Value};
|
|
5
|
+
use std::io::{BufRead, Write};
|
|
6
|
+
use std::process::{Child, Command, Stdio};
|
|
7
|
+
|
|
8
|
+
type BoundaryHook = Box<dyn FnMut(&str) -> Result<(), MessagingError>>;
|
|
9
|
+
thread_local! {
|
|
10
|
+
static HOOK: std::cell::RefCell<Option<BoundaryHook>> = const { std::cell::RefCell::new(None) };
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
pub(super) fn boundary(point: &str) -> Result<(), MessagingError> {
|
|
14
|
+
HOOK.with(|hook| match hook.borrow_mut().as_mut() {
|
|
15
|
+
Some(hook) => hook(point),
|
|
16
|
+
None => Ok(()),
|
|
17
|
+
})
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const WORKER: &str = "messaging::results::collection_tests::s2_process_collector";
|
|
21
|
+
|
|
22
|
+
fn isolated(name: &str, check: impl FnOnce(&Path)) {
|
|
23
|
+
if std::env::var("TA_S2_CASE").as_deref() == Ok(name) {
|
|
24
|
+
check(&std::env::current_dir().unwrap());
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
let dir = std::env::temp_dir().join(format!("ta-s2-{name}-{}", std::process::id()));
|
|
28
|
+
std::fs::create_dir_all(&dir).unwrap();
|
|
29
|
+
let output = Command::new(std::env::current_exe().unwrap())
|
|
30
|
+
.args([
|
|
31
|
+
"--exact",
|
|
32
|
+
&format!("messaging::results::collection_tests::{name}"),
|
|
33
|
+
"--nocapture",
|
|
34
|
+
])
|
|
35
|
+
.env_clear()
|
|
36
|
+
.env("HOME", dir.join("home"))
|
|
37
|
+
.env("TA_S2_CASE", name)
|
|
38
|
+
.current_dir(&dir)
|
|
39
|
+
.output()
|
|
40
|
+
.unwrap();
|
|
41
|
+
print!("{}", String::from_utf8_lossy(&output.stdout));
|
|
42
|
+
assert!(
|
|
43
|
+
output.status.success(),
|
|
44
|
+
"{}",
|
|
45
|
+
String::from_utf8_lossy(&output.stderr)
|
|
46
|
+
);
|
|
47
|
+
std::fs::remove_dir_all(dir).unwrap();
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
fn envelope(result: &str, task: &str) -> Value {
|
|
51
|
+
json!({"schema_version": "result_envelope_v1", "result_id": result,
|
|
52
|
+
"task_id": task, "agent_id": "w1", "status": "success", "summary": "done",
|
|
53
|
+
"artifacts": [], "changes": [], "tests": [], "risks": [], "next_actions": []})
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
fn write_fixture_state(dir: &Path, state: &Value) {
|
|
57
|
+
let path = crate::state::persist::runtime_state_path(dir);
|
|
58
|
+
if let Some(parent) = path.parent() {
|
|
59
|
+
std::fs::create_dir_all(parent).unwrap();
|
|
60
|
+
}
|
|
61
|
+
std::fs::write(&path, serde_json::to_vec_pretty(state).unwrap()).unwrap();
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
fn seed(dir: &Path) {
|
|
65
|
+
std::fs::create_dir_all(dir).unwrap();
|
|
66
|
+
let mut teams = serde_json::Map::new();
|
|
67
|
+
for key in ["T1", "T2"] {
|
|
68
|
+
let spec = dir.join(".team/runtime").join(key).join("team.spec.yaml");
|
|
69
|
+
std::fs::create_dir_all(spec.parent().unwrap()).unwrap();
|
|
70
|
+
std::fs::write(&spec, format!("name: {key}\ntasks: []\nagents: []\n")).unwrap();
|
|
71
|
+
teams.insert(
|
|
72
|
+
key.to_string(),
|
|
73
|
+
json!({"session_name": key, "spec_path": spec,
|
|
74
|
+
"team_dir": dir.join(".team").join(key), "agents": {},
|
|
75
|
+
"tasks": [{"id": "task", "title": "Task", "status": "pending"}], "notes": []}),
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
let mut state = teams["T1"].clone();
|
|
79
|
+
state["active_team_key"] = json!("T1");
|
|
80
|
+
state["teams"] = Value::Object(teams);
|
|
81
|
+
write_fixture_state(dir, &state);
|
|
82
|
+
MessageStore::open(dir).unwrap();
|
|
83
|
+
std::fs::write(
|
|
84
|
+
dir.join("result.json"),
|
|
85
|
+
envelope("result", "task").to_string(),
|
|
86
|
+
)
|
|
87
|
+
.unwrap();
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
fn seed_batch(dir: &Path) {
|
|
91
|
+
seed(dir);
|
|
92
|
+
let mut state = disk(dir);
|
|
93
|
+
state["teams"]["T1"]["tasks"]
|
|
94
|
+
.as_array_mut()
|
|
95
|
+
.unwrap()
|
|
96
|
+
.push(json!({"id": "task-b", "title": "Task B", "status": "pending"}));
|
|
97
|
+
write_fixture_state(dir, &state);
|
|
98
|
+
let conn = db(dir);
|
|
99
|
+
for (id, task_id, created_at) in [
|
|
100
|
+
("a1", "task", "1"),
|
|
101
|
+
("a2", "task", "2"),
|
|
102
|
+
("b1", "task-b", "3"),
|
|
103
|
+
] {
|
|
104
|
+
insert_result_if_absent(
|
|
105
|
+
&conn,
|
|
106
|
+
id,
|
|
107
|
+
task_id,
|
|
108
|
+
"w1",
|
|
109
|
+
&envelope(id, task_id).to_string(),
|
|
110
|
+
"success",
|
|
111
|
+
Some("T1"),
|
|
112
|
+
)
|
|
113
|
+
.unwrap();
|
|
114
|
+
conn.execute(
|
|
115
|
+
"update results set created_at = ?1 where result_id = ?2",
|
|
116
|
+
rusqlite::params![created_at, id],
|
|
117
|
+
)
|
|
118
|
+
.unwrap();
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
fn disk(dir: &Path) -> Value {
|
|
123
|
+
serde_json::from_str(
|
|
124
|
+
&std::fs::read_to_string(crate::state::persist::runtime_state_path(dir)).unwrap(),
|
|
125
|
+
)
|
|
126
|
+
.unwrap()
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
fn db(dir: &Path) -> rusqlite::Connection {
|
|
130
|
+
crate::db::schema::open_db(MessageStore::open(dir).unwrap().db_path()).unwrap()
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
fn db_status(dir: &Path) -> Option<String> {
|
|
134
|
+
db_status_for(dir, "result")
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
fn db_status_for(dir: &Path, result_id: &str) -> Option<String> {
|
|
138
|
+
use rusqlite::OptionalExtension;
|
|
139
|
+
db(dir)
|
|
140
|
+
.query_row(
|
|
141
|
+
"select status from results where result_id = ?1",
|
|
142
|
+
[result_id],
|
|
143
|
+
|row| row.get(0),
|
|
144
|
+
)
|
|
145
|
+
.optional()
|
|
146
|
+
.unwrap()
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
fn collected_ids(response: &Value) -> Vec<String> {
|
|
150
|
+
response["response"]["collected_results"]
|
|
151
|
+
.as_array()
|
|
152
|
+
.unwrap()
|
|
153
|
+
.iter()
|
|
154
|
+
.map(|result| result["result_id"].as_str().unwrap().to_string())
|
|
155
|
+
.collect()
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
fn remove_task_from_state(dir: &Path, task_id: &str) {
|
|
159
|
+
let mut state = disk(dir);
|
|
160
|
+
let tasks = state
|
|
161
|
+
.pointer_mut("/teams/T1/tasks")
|
|
162
|
+
.and_then(Value::as_array_mut)
|
|
163
|
+
.unwrap();
|
|
164
|
+
tasks.retain(|task| task.get("id").and_then(Value::as_str) != Some(task_id));
|
|
165
|
+
write_fixture_state(dir, &state);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
fn collect_event_count(dir: &Path) -> usize {
|
|
169
|
+
let path = dir.join(".team/logs/events.jsonl");
|
|
170
|
+
std::fs::read_to_string(path)
|
|
171
|
+
.unwrap_or_default()
|
|
172
|
+
.lines()
|
|
173
|
+
.filter(|line| line.contains("collect.result"))
|
|
174
|
+
.count()
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
fn install_finalize_execute_error(dir: &Path) {
|
|
178
|
+
db(dir)
|
|
179
|
+
.execute(
|
|
180
|
+
"create trigger s2_test_finalize_execute_error
|
|
181
|
+
before update of status on results
|
|
182
|
+
when new.status = 'collected'
|
|
183
|
+
begin
|
|
184
|
+
select raise(abort, 's2 injected finalize execute error');
|
|
185
|
+
end",
|
|
186
|
+
[],
|
|
187
|
+
)
|
|
188
|
+
.unwrap();
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
fn remove_finalize_execute_error(dir: &Path) {
|
|
192
|
+
db(dir)
|
|
193
|
+
.execute("drop trigger s2_test_finalize_execute_error", [])
|
|
194
|
+
.unwrap();
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
fn start(dir: &Path, label: &str, point: &str, pause: bool, file: bool) -> Child {
|
|
198
|
+
start_with_options(dir, label, point, pause, file, false)
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
fn start_with_task_removal(dir: &Path, label: &str, point: &str, pause: bool, file: bool) -> Child {
|
|
202
|
+
start_with_options(dir, label, point, pause, file, true)
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
fn start_with_options(
|
|
206
|
+
dir: &Path,
|
|
207
|
+
label: &str,
|
|
208
|
+
point: &str,
|
|
209
|
+
pause: bool,
|
|
210
|
+
file: bool,
|
|
211
|
+
remove_task: bool,
|
|
212
|
+
) -> Child {
|
|
213
|
+
let mut command = Command::new(std::env::current_exe().unwrap());
|
|
214
|
+
command
|
|
215
|
+
.args(["--exact", WORKER, "--ignored", "--nocapture"])
|
|
216
|
+
.env_clear()
|
|
217
|
+
.env("HOME", dir.join("home"))
|
|
218
|
+
.env("TA_S2_LABEL", label)
|
|
219
|
+
.env("TA_S2_POINT", point)
|
|
220
|
+
.env("TA_S2_PAUSE", if pause { "1" } else { "0" })
|
|
221
|
+
.env("TA_S2_FILE", if file { "1" } else { "0" });
|
|
222
|
+
if remove_task {
|
|
223
|
+
command.env("TA_S2_REMOVE_TASK", "task");
|
|
224
|
+
}
|
|
225
|
+
let mut child = command
|
|
226
|
+
.current_dir(dir)
|
|
227
|
+
.stdin(Stdio::piped())
|
|
228
|
+
.stdout(Stdio::piped())
|
|
229
|
+
.stderr(Stdio::piped())
|
|
230
|
+
.spawn()
|
|
231
|
+
.unwrap();
|
|
232
|
+
if !point.is_empty() {
|
|
233
|
+
let mut reader = std::io::BufReader::new(child.stdout.take().unwrap());
|
|
234
|
+
loop {
|
|
235
|
+
let mut line = String::new();
|
|
236
|
+
assert!(
|
|
237
|
+
reader.read_line(&mut line).unwrap() > 0,
|
|
238
|
+
"collector exited before {point}"
|
|
239
|
+
);
|
|
240
|
+
print!("{line}");
|
|
241
|
+
if line.contains("S2_READY") {
|
|
242
|
+
break;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
child.stdout = Some(reader.into_inner());
|
|
246
|
+
}
|
|
247
|
+
child
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
fn finish(mut child: Child, paused: bool) {
|
|
251
|
+
if paused {
|
|
252
|
+
write!(child.stdin.as_mut().unwrap(), "continue\n").unwrap();
|
|
253
|
+
}
|
|
254
|
+
let output = child.wait_with_output().unwrap();
|
|
255
|
+
print!("{}", String::from_utf8_lossy(&output.stdout));
|
|
256
|
+
assert!(
|
|
257
|
+
output.status.success(),
|
|
258
|
+
"{}",
|
|
259
|
+
String::from_utf8_lossy(&output.stderr)
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
fn output(dir: &Path, label: &str) -> Value {
|
|
264
|
+
serde_json::from_str(&std::fs::read_to_string(dir.join(format!("{label}.json"))).unwrap())
|
|
265
|
+
.unwrap()
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
fn collect_fresh(dir: &Path, label: &str, file: bool) -> Value {
|
|
269
|
+
finish(start(dir, label, "", false, file), false);
|
|
270
|
+
output(dir, label)
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
fn recovered(dir: &Path, file: bool, expected_output: usize) {
|
|
274
|
+
let result = collect_fresh(dir, "recovered", file);
|
|
275
|
+
assert_eq!(result["response"]["ok"], true, "{result}");
|
|
276
|
+
assert_eq!(
|
|
277
|
+
result["response"]["collected_results"]
|
|
278
|
+
.as_array()
|
|
279
|
+
.unwrap()
|
|
280
|
+
.len(),
|
|
281
|
+
expected_output,
|
|
282
|
+
"{result}"
|
|
283
|
+
);
|
|
284
|
+
let state = disk(dir);
|
|
285
|
+
assert_eq!(
|
|
286
|
+
state["teams"]["T1"]["tasks"][0]["status"], "done",
|
|
287
|
+
"{state}"
|
|
288
|
+
);
|
|
289
|
+
assert_eq!(
|
|
290
|
+
state["teams"]["T1"]["tasks"][0]["accepted_result_id"],
|
|
291
|
+
"result"
|
|
292
|
+
);
|
|
293
|
+
assert_eq!(state["teams"]["T2"]["tasks"][0]["status"], "pending");
|
|
294
|
+
assert_eq!(db_status(dir).as_deref(), Some("collected"));
|
|
295
|
+
assert_eq!(result["response"]["results"]["collected"], 1);
|
|
296
|
+
let repeated = collect_fresh(dir, "repeated", false);
|
|
297
|
+
assert_eq!(repeated["response"]["collected"], json!([]));
|
|
298
|
+
assert_eq!(repeated["response"]["results"]["collected"], 1);
|
|
299
|
+
println!(
|
|
300
|
+
"S2 recovered state={} db={:?} output={result}",
|
|
301
|
+
state["teams"],
|
|
302
|
+
db_status(dir)
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
#[test]
|
|
307
|
+
#[ignore = "hermetic subprocess entry; parent contracts invoke this exact test"]
|
|
308
|
+
fn s2_process_collector() {
|
|
309
|
+
let dir = std::env::current_dir().unwrap();
|
|
310
|
+
let label = std::env::var("TA_S2_LABEL").unwrap();
|
|
311
|
+
let expected = std::env::var("TA_S2_POINT").unwrap();
|
|
312
|
+
let pause = std::env::var("TA_S2_PAUSE").as_deref() == Ok("1");
|
|
313
|
+
let file = std::env::var("TA_S2_FILE").as_deref() == Ok("1");
|
|
314
|
+
let remove_task = std::env::var("TA_S2_REMOVE_TASK").ok();
|
|
315
|
+
let hook_dir = dir.clone();
|
|
316
|
+
let mut fired = false;
|
|
317
|
+
HOOK.with(|slot| {
|
|
318
|
+
*slot.borrow_mut() = Some(Box::new(move |point| {
|
|
319
|
+
if !fired && point == expected {
|
|
320
|
+
fired = true;
|
|
321
|
+
if let Some(task_id) = remove_task.as_deref() {
|
|
322
|
+
remove_task_from_state(&hook_dir, task_id);
|
|
323
|
+
}
|
|
324
|
+
println!("S2_READY {point}");
|
|
325
|
+
std::io::stdout().flush().unwrap();
|
|
326
|
+
if pause {
|
|
327
|
+
let mut line = String::new();
|
|
328
|
+
std::io::stdin().read_line(&mut line).unwrap();
|
|
329
|
+
assert_eq!(line.trim(), "continue");
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
Ok(())
|
|
333
|
+
}))
|
|
334
|
+
});
|
|
335
|
+
let path = dir.join("result.json");
|
|
336
|
+
let result = match collect_for_team(&dir, file.then_some(path.as_path()), false, Some("T1")) {
|
|
337
|
+
Ok(value) => json!({"response": value}),
|
|
338
|
+
Err(error) => json!({"error": error.to_string()}),
|
|
339
|
+
};
|
|
340
|
+
std::fs::write(dir.join(format!("{label}.json")), result.to_string()).unwrap();
|
|
341
|
+
println!("S2_RESULT {label} {result}");
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
#[test]
|
|
345
|
+
fn s2_original_finalize_window_recovers() {
|
|
346
|
+
isolated("s2_original_finalize_window_recovers", |dir| {
|
|
347
|
+
seed(dir);
|
|
348
|
+
let mut child = start(dir, "interrupted", "after_finalize", true, true);
|
|
349
|
+
assert_eq!(db_status(dir).as_deref(), Some("collected"));
|
|
350
|
+
println!(
|
|
351
|
+
"S2 after durable DB finalize: task={} db={:?}",
|
|
352
|
+
disk(dir)["teams"]["T1"]["tasks"],
|
|
353
|
+
db_status(dir)
|
|
354
|
+
);
|
|
355
|
+
child.kill().unwrap();
|
|
356
|
+
assert!(!child.wait().unwrap().success());
|
|
357
|
+
recovered(dir, false, 0);
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
#[test]
|
|
362
|
+
fn s2_all_persistence_windows_recover_in_fresh_processes() {
|
|
363
|
+
isolated(
|
|
364
|
+
"s2_all_persistence_windows_recover_in_fresh_processes",
|
|
365
|
+
|root| {
|
|
366
|
+
for point in [
|
|
367
|
+
"before_ingest",
|
|
368
|
+
"after_ingest",
|
|
369
|
+
"before_state",
|
|
370
|
+
"after_state",
|
|
371
|
+
"before_event",
|
|
372
|
+
"after_event",
|
|
373
|
+
"before_finalize",
|
|
374
|
+
"after_finalize",
|
|
375
|
+
] {
|
|
376
|
+
let dir = root.join(point);
|
|
377
|
+
seed(&dir);
|
|
378
|
+
let mut child = start(&dir, "interrupted", point, true, true);
|
|
379
|
+
println!(
|
|
380
|
+
"S2 crash at={point} task={} db={:?}",
|
|
381
|
+
disk(&dir)["teams"]["T1"]["tasks"],
|
|
382
|
+
db_status(&dir)
|
|
383
|
+
);
|
|
384
|
+
child.kill().unwrap();
|
|
385
|
+
assert!(!child.wait().unwrap().success());
|
|
386
|
+
let expected_output = usize::from(point != "after_finalize");
|
|
387
|
+
let recovery_file = point == "before_ingest";
|
|
388
|
+
recovered(&dir, recovery_file, expected_output);
|
|
389
|
+
}
|
|
390
|
+
},
|
|
391
|
+
);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
#[test]
|
|
395
|
+
fn s2_batch_partial_finalize_recovers_only_the_unfinalized_suffix() {
|
|
396
|
+
isolated(
|
|
397
|
+
"s2_batch_partial_finalize_recovers_only_the_unfinalized_suffix",
|
|
398
|
+
|dir| {
|
|
399
|
+
seed_batch(dir);
|
|
400
|
+
let mut child = start(dir, "interrupted", "after_finalize", true, false);
|
|
401
|
+
assert_eq!(db_status_for(dir, "a1").as_deref(), Some("collected"));
|
|
402
|
+
assert_eq!(db_status_for(dir, "a2").as_deref(), Some("success"));
|
|
403
|
+
assert_eq!(db_status_for(dir, "b1").as_deref(), Some("success"));
|
|
404
|
+
child.kill().unwrap();
|
|
405
|
+
assert!(!child.wait().unwrap().success());
|
|
406
|
+
|
|
407
|
+
let recovered = collect_fresh(dir, "recovered", false);
|
|
408
|
+
assert_eq!(
|
|
409
|
+
collected_ids(&recovered),
|
|
410
|
+
vec!["a2".to_string(), "b1".to_string()]
|
|
411
|
+
);
|
|
412
|
+
let state = disk(dir);
|
|
413
|
+
assert_eq!(state["teams"]["T1"]["tasks"][0]["status"], "done");
|
|
414
|
+
assert_eq!(state["teams"]["T1"]["tasks"][0]["accepted_result_id"], "a2");
|
|
415
|
+
assert_eq!(state["teams"]["T1"]["tasks"][1]["status"], "done");
|
|
416
|
+
assert_eq!(state["teams"]["T1"]["tasks"][1]["accepted_result_id"], "b1");
|
|
417
|
+
assert_eq!(db_status_for(dir, "a1").as_deref(), Some("collected"));
|
|
418
|
+
assert_eq!(db_status_for(dir, "a2").as_deref(), Some("collected"));
|
|
419
|
+
assert_eq!(db_status_for(dir, "b1").as_deref(), Some("collected"));
|
|
420
|
+
assert_eq!(recovered["response"]["results"]["collected"], 3);
|
|
421
|
+
|
|
422
|
+
let repeated = collect_fresh(dir, "repeated", false);
|
|
423
|
+
assert_eq!(repeated["response"]["collected"], json!([]));
|
|
424
|
+
assert_eq!(repeated["response"]["results"]["collected"], 3);
|
|
425
|
+
},
|
|
426
|
+
);
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
#[test]
|
|
430
|
+
fn s2_event_and_state_io_failures_remain_recoverable() {
|
|
431
|
+
isolated(
|
|
432
|
+
"s2_event_and_state_io_failures_remain_recoverable",
|
|
433
|
+
|root| {
|
|
434
|
+
let event_dir = root.join("before_event");
|
|
435
|
+
seed(&event_dir);
|
|
436
|
+
let event_child = start(&event_dir, "failed", "before_event", true, true);
|
|
437
|
+
let event_log = event_dir.join(".team/logs/events.jsonl");
|
|
438
|
+
let event_backup = event_dir.join(".team/logs/events.jsonl.fault-original");
|
|
439
|
+
std::fs::create_dir_all(event_log.parent().unwrap()).unwrap();
|
|
440
|
+
if event_log.exists() {
|
|
441
|
+
std::fs::rename(&event_log, &event_backup).unwrap();
|
|
442
|
+
}
|
|
443
|
+
std::fs::create_dir(&event_log).unwrap();
|
|
444
|
+
finish(event_child, true);
|
|
445
|
+
let failure = output(&event_dir, "failed");
|
|
446
|
+
assert!(
|
|
447
|
+
failure["error"]
|
|
448
|
+
.as_str()
|
|
449
|
+
.is_some_and(|error| error.contains("event log")),
|
|
450
|
+
"{failure}"
|
|
451
|
+
);
|
|
452
|
+
assert_eq!(db_status(&event_dir).as_deref(), Some("success"));
|
|
453
|
+
assert_eq!(
|
|
454
|
+
disk(&event_dir)["teams"]["T1"]["tasks"][0]["status"],
|
|
455
|
+
"done"
|
|
456
|
+
);
|
|
457
|
+
assert_eq!(collect_event_count(&event_dir), 0, "{failure}");
|
|
458
|
+
std::fs::remove_dir(&event_log).unwrap();
|
|
459
|
+
if event_backup.exists() {
|
|
460
|
+
std::fs::rename(event_backup, event_log).unwrap();
|
|
461
|
+
}
|
|
462
|
+
recovered(&event_dir, false, 1);
|
|
463
|
+
|
|
464
|
+
let state_dir = root.join("before_state");
|
|
465
|
+
seed(&state_dir);
|
|
466
|
+
let state_child = start(&state_dir, "failed", "before_state", true, true);
|
|
467
|
+
let runtime_dir = state_dir.join(".team/runtime");
|
|
468
|
+
let pid = state_child.id();
|
|
469
|
+
let state_tmp_obstructions: Vec<_> = (0..32)
|
|
470
|
+
.map(|seq| runtime_dir.join(format!("state.json.{pid}.{seq}.tmp")))
|
|
471
|
+
.collect();
|
|
472
|
+
for path in &state_tmp_obstructions {
|
|
473
|
+
std::fs::create_dir(path).unwrap();
|
|
474
|
+
}
|
|
475
|
+
finish(state_child, true);
|
|
476
|
+
let failure = output(&state_dir, "failed");
|
|
477
|
+
assert!(
|
|
478
|
+
failure["error"]
|
|
479
|
+
.as_str()
|
|
480
|
+
.is_some_and(|error| error.contains("state")),
|
|
481
|
+
"{failure}"
|
|
482
|
+
);
|
|
483
|
+
assert_eq!(db_status(&state_dir).as_deref(), Some("success"));
|
|
484
|
+
assert_eq!(
|
|
485
|
+
disk(&state_dir)["teams"]["T1"]["tasks"][0]["status"],
|
|
486
|
+
"pending"
|
|
487
|
+
);
|
|
488
|
+
assert_eq!(collect_event_count(&state_dir), 0, "{failure}");
|
|
489
|
+
for path in state_tmp_obstructions {
|
|
490
|
+
std::fs::remove_dir(path).unwrap();
|
|
491
|
+
}
|
|
492
|
+
recovered(&state_dir, false, 1);
|
|
493
|
+
},
|
|
494
|
+
);
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
#[test]
|
|
498
|
+
fn s2_state_lock_timeout_keeps_result_eligible_for_recovery() {
|
|
499
|
+
isolated(
|
|
500
|
+
"s2_state_lock_timeout_keeps_result_eligible_for_recovery",
|
|
501
|
+
|dir| {
|
|
502
|
+
seed(dir);
|
|
503
|
+
let child = start(dir, "blocked", "before_state", true, true);
|
|
504
|
+
let lock = crate::state::persist::acquire_state_save_lock(dir, 2.0).unwrap();
|
|
505
|
+
finish(child, true);
|
|
506
|
+
let failure = output(dir, "blocked");
|
|
507
|
+
assert!(
|
|
508
|
+
failure["error"]
|
|
509
|
+
.as_str()
|
|
510
|
+
.is_some_and(|error| error.contains("state-save")),
|
|
511
|
+
"{failure}"
|
|
512
|
+
);
|
|
513
|
+
assert_eq!(db_status(dir).as_deref(), Some("success"));
|
|
514
|
+
assert_eq!(disk(dir)["teams"]["T1"]["tasks"][0]["status"], "pending");
|
|
515
|
+
drop(lock);
|
|
516
|
+
|
|
517
|
+
let recovered = collect_fresh(dir, "recovered", false);
|
|
518
|
+
assert_eq!(collected_ids(&recovered), vec!["result".to_string()]);
|
|
519
|
+
assert_eq!(db_status(dir).as_deref(), Some("collected"));
|
|
520
|
+
assert_eq!(disk(dir)["teams"]["T1"]["tasks"][0]["status"], "done");
|
|
521
|
+
},
|
|
522
|
+
);
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
#[test]
|
|
526
|
+
fn s2_finalize_execute_error_keeps_result_eligible_for_recovery() {
|
|
527
|
+
isolated(
|
|
528
|
+
"s2_finalize_execute_error_keeps_result_eligible_for_recovery",
|
|
529
|
+
|dir| {
|
|
530
|
+
seed(dir);
|
|
531
|
+
let child = start(dir, "failed", "before_finalize", true, true);
|
|
532
|
+
let state = disk(dir);
|
|
533
|
+
assert_eq!(
|
|
534
|
+
state["teams"]["T1"]["tasks"][0]["status"], "done",
|
|
535
|
+
"{state}"
|
|
536
|
+
);
|
|
537
|
+
assert_eq!(
|
|
538
|
+
state["teams"]["T1"]["tasks"][0]["accepted_result_id"], "result",
|
|
539
|
+
"{state}"
|
|
540
|
+
);
|
|
541
|
+
assert_eq!(db_status(dir).as_deref(), Some("success"));
|
|
542
|
+
|
|
543
|
+
install_finalize_execute_error(dir);
|
|
544
|
+
finish(child, true);
|
|
545
|
+
let failure = output(dir, "failed");
|
|
546
|
+
let error = failure["error"].as_str().unwrap_or_default();
|
|
547
|
+
assert!(
|
|
548
|
+
error.contains("s2 injected finalize execute error"),
|
|
549
|
+
"{failure}"
|
|
550
|
+
);
|
|
551
|
+
|
|
552
|
+
let state = disk(dir);
|
|
553
|
+
assert_eq!(
|
|
554
|
+
state["teams"]["T1"]["tasks"][0]["status"], "done",
|
|
555
|
+
"{state}"
|
|
556
|
+
);
|
|
557
|
+
assert_eq!(
|
|
558
|
+
state["teams"]["T1"]["tasks"][0]["accepted_result_id"], "result",
|
|
559
|
+
"{state}"
|
|
560
|
+
);
|
|
561
|
+
assert_eq!(db_status(dir).as_deref(), Some("success"));
|
|
562
|
+
|
|
563
|
+
remove_finalize_execute_error(dir);
|
|
564
|
+
let recovered = collect_fresh(dir, "recovered", false);
|
|
565
|
+
assert_eq!(recovered["response"]["ok"], true, "{recovered}");
|
|
566
|
+
assert_eq!(
|
|
567
|
+
recovered["response"]["collected"].as_array().unwrap().len(),
|
|
568
|
+
1,
|
|
569
|
+
"{recovered}"
|
|
570
|
+
);
|
|
571
|
+
assert_eq!(
|
|
572
|
+
recovered["response"]["collected_results"]
|
|
573
|
+
.as_array()
|
|
574
|
+
.unwrap()
|
|
575
|
+
.len(),
|
|
576
|
+
1,
|
|
577
|
+
"{recovered}"
|
|
578
|
+
);
|
|
579
|
+
assert_eq!(
|
|
580
|
+
recovered["response"]["results"]["collected"], 1,
|
|
581
|
+
"{recovered}"
|
|
582
|
+
);
|
|
583
|
+
assert_eq!(db_status(dir).as_deref(), Some("collected"));
|
|
584
|
+
|
|
585
|
+
let repeated = collect_fresh(dir, "repeated", false);
|
|
586
|
+
assert_eq!(repeated["response"]["collected"], json!([]), "{repeated}");
|
|
587
|
+
assert_eq!(
|
|
588
|
+
repeated["response"]["results"]["collected"], 1,
|
|
589
|
+
"{repeated}"
|
|
590
|
+
);
|
|
591
|
+
},
|
|
592
|
+
);
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
#[test]
|
|
596
|
+
fn s2_task_disappears_after_read_fails_closed_before_finalize() {
|
|
597
|
+
isolated(
|
|
598
|
+
"s2_task_disappears_after_read_fails_closed_before_finalize",
|
|
599
|
+
|dir| {
|
|
600
|
+
seed(dir);
|
|
601
|
+
let child = start_with_task_removal(dir, "vanished", "before_state", true, true);
|
|
602
|
+
finish(child, true);
|
|
603
|
+
let failure = output(dir, "vanished");
|
|
604
|
+
assert!(
|
|
605
|
+
failure["error"].as_str().is_some_and(|error| {
|
|
606
|
+
error.contains("task result projection missing after commit")
|
|
607
|
+
}),
|
|
608
|
+
"{failure}"
|
|
609
|
+
);
|
|
610
|
+
assert_eq!(db_status(dir).as_deref(), Some("success"));
|
|
611
|
+
assert_eq!(collect_event_count(dir), 0);
|
|
612
|
+
assert!(disk(dir)["teams"]["T1"]["tasks"]
|
|
613
|
+
.as_array()
|
|
614
|
+
.unwrap()
|
|
615
|
+
.is_empty());
|
|
616
|
+
},
|
|
617
|
+
);
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
#[test]
|
|
621
|
+
fn s2_concurrent_collectors_emit_each_result_once() {
|
|
622
|
+
isolated("s2_concurrent_collectors_emit_each_result_once", |dir| {
|
|
623
|
+
seed(dir);
|
|
624
|
+
let first = start(dir, "first", "after_state", true, true);
|
|
625
|
+
let second = start(dir, "second", "before_lock", false, true);
|
|
626
|
+
finish(first, true);
|
|
627
|
+
finish(second, false);
|
|
628
|
+
let first = output(dir, "first");
|
|
629
|
+
let second = output(dir, "second");
|
|
630
|
+
assert!(
|
|
631
|
+
first.get("error").is_none() && second.get("error").is_none(),
|
|
632
|
+
"{first} {second}"
|
|
633
|
+
);
|
|
634
|
+
assert_eq!(
|
|
635
|
+
first["response"]["collected"].as_array().unwrap().len()
|
|
636
|
+
+ second["response"]["collected"].as_array().unwrap().len(),
|
|
637
|
+
1
|
|
638
|
+
);
|
|
639
|
+
recovered(dir, false, 0);
|
|
640
|
+
let notifications: i64 = db(dir)
|
|
641
|
+
.query_row("select count(*) from leader_notification_log", [], |row| {
|
|
642
|
+
row.get(0)
|
|
643
|
+
})
|
|
644
|
+
.unwrap();
|
|
645
|
+
assert_eq!(
|
|
646
|
+
notifications, 0,
|
|
647
|
+
"collect must not create notification claims"
|
|
648
|
+
);
|
|
649
|
+
});
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
#[test]
|
|
653
|
+
fn s2_scope_order_and_legacy_collected_rows_stay_compatible() {
|
|
654
|
+
isolated(
|
|
655
|
+
"s2_scope_order_and_legacy_collected_rows_stay_compatible",
|
|
656
|
+
|dir| {
|
|
657
|
+
seed(dir);
|
|
658
|
+
let conn = db(dir);
|
|
659
|
+
conn.execute("insert into messages(message_id,owner_team_id,sender,recipient,content,status,created_at) values ('msg_scope','T1','leader','w1','fixture','accepted','0')", []).unwrap();
|
|
660
|
+
for (id, task, team) in [
|
|
661
|
+
("a", "task", "T1"),
|
|
662
|
+
("b", "task", "T1"),
|
|
663
|
+
("message", "msg_scope", "T1"),
|
|
664
|
+
("foreign", "task", "T2"),
|
|
665
|
+
] {
|
|
666
|
+
insert_result_if_absent(
|
|
667
|
+
&conn,
|
|
668
|
+
id,
|
|
669
|
+
task,
|
|
670
|
+
"w1",
|
|
671
|
+
&envelope(id, task).to_string(),
|
|
672
|
+
"success",
|
|
673
|
+
Some(team),
|
|
674
|
+
)
|
|
675
|
+
.unwrap();
|
|
676
|
+
}
|
|
677
|
+
insert_result_if_absent(&conn, "invalid", "task", "w1", "{}", "success", Some("T1"))
|
|
678
|
+
.unwrap();
|
|
679
|
+
insert_result_if_absent(
|
|
680
|
+
&conn,
|
|
681
|
+
"legacy-collected",
|
|
682
|
+
"task",
|
|
683
|
+
"w1",
|
|
684
|
+
&envelope("legacy-collected", "task").to_string(),
|
|
685
|
+
"collected",
|
|
686
|
+
Some("T1"),
|
|
687
|
+
)
|
|
688
|
+
.unwrap();
|
|
689
|
+
drop(conn);
|
|
690
|
+
let result = collect_fresh(dir, "scoped", false);
|
|
691
|
+
let response = &result["response"];
|
|
692
|
+
assert_eq!(
|
|
693
|
+
response["ok"], false,
|
|
694
|
+
"invalid envelope is reported, not hidden"
|
|
695
|
+
);
|
|
696
|
+
assert_eq!(
|
|
697
|
+
response["collected_results"]
|
|
698
|
+
.as_array()
|
|
699
|
+
.unwrap()
|
|
700
|
+
.iter()
|
|
701
|
+
.map(|row| row["result_id"].as_str().unwrap())
|
|
702
|
+
.collect::<Vec<_>>(),
|
|
703
|
+
vec!["a", "b", "message"]
|
|
704
|
+
);
|
|
705
|
+
assert_eq!(response["collected_results"][2]["scope"], "message");
|
|
706
|
+
assert_eq!(
|
|
707
|
+
response["results"],
|
|
708
|
+
json!({"total": 5, "collected": 4, "invalid": 1, "uncollected": 0, "by_status": {}})
|
|
709
|
+
);
|
|
710
|
+
assert_eq!(db_status_for(dir, "foreign").as_deref(), Some("success"));
|
|
711
|
+
let state = disk(dir);
|
|
712
|
+
assert_eq!(state["teams"]["T1"]["tasks"].as_array().unwrap().len(), 1);
|
|
713
|
+
assert_eq!(state["teams"]["T1"]["tasks"][0]["accepted_result_id"], "b");
|
|
714
|
+
assert_eq!(state["teams"]["T2"]["tasks"][0]["status"], "pending");
|
|
715
|
+
let repeated = collect_fresh(dir, "scope-repeated", false);
|
|
716
|
+
assert_eq!(repeated["response"]["collected"], json!([]));
|
|
717
|
+
},
|
|
718
|
+
);
|
|
719
|
+
}
|