@team-agent/installer 0.5.49 → 0.5.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -1
- package/Cargo.toml +1 -1
- package/crates/team-agent/src/cli/adapters.rs +6 -4
- package/crates/team-agent/src/cli/emit.rs +91 -39
- package/crates/team-agent/src/cli/mod.rs +33 -15
- package/crates/team-agent/src/cli/named_address.rs +82 -53
- package/crates/team-agent/src/cli/send/coordinator.rs +163 -0
- package/crates/team-agent/src/cli/send/mailbox.rs +99 -0
- package/crates/team-agent/src/cli/send/persist.rs +154 -0
- package/crates/team-agent/src/cli/send/presentation.rs +333 -0
- package/crates/team-agent/src/cli/send/resolve.rs +361 -0
- package/crates/team-agent/src/cli/send.rs +103 -1308
- package/crates/team-agent/src/cli/spec.rs +2 -2
- package/crates/team-agent/src/cli/status_port/agents.rs +358 -0
- package/crates/team-agent/src/cli/status_port/approvals.rs +79 -0
- package/crates/team-agent/src/cli/status_port/compact.rs +207 -0
- package/crates/team-agent/src/cli/status_port/format.rs +145 -0
- package/crates/team-agent/src/cli/status_port/inbox.rs +36 -0
- package/crates/team-agent/src/cli/status_port/runtime.rs +195 -0
- package/crates/team-agent/src/cli/status_port/snapshot.rs +181 -0
- package/crates/team-agent/src/cli/status_port/store.rs +412 -0
- package/crates/team-agent/src/cli/status_port/tests.rs +54 -0
- package/crates/team-agent/src/cli/status_port.rs +47 -1548
- package/crates/team-agent/src/cli/tests/leader_watch.rs +1 -1
- package/crates/team-agent/src/cli/tests/named_address.rs +9 -7
- package/crates/team-agent/src/cli/tests/run_delegation.rs +2 -3
- package/crates/team-agent/src/cli/tests/status_send.rs +17 -33
- package/crates/team-agent/src/cli/types.rs +5 -8
- package/crates/team-agent/src/coordinator/conpty_shim.rs +34 -30
- package/crates/team-agent/src/coordinator/steps/abnormal.rs +135 -13
- package/crates/team-agent/src/coordinator/tick.rs +37 -0
- package/crates/team-agent/src/db/agent_health_capture.rs +18 -13
- package/crates/team-agent/src/db/message_store.rs +154 -44
- package/crates/team-agent/src/event_log.rs +73 -0
- package/crates/team-agent/src/leader/start.rs +28 -4
- package/crates/team-agent/src/lifecycle/launch/add_agent.rs +424 -0
- package/crates/team-agent/src/lifecycle/launch/add_agent_state.rs +297 -0
- package/crates/team-agent/src/lifecycle/launch/agent_state.rs +160 -0
- package/crates/team-agent/src/lifecycle/launch/approval.rs +134 -0
- package/crates/team-agent/src/lifecycle/launch/fork_agent.rs +492 -0
- package/crates/team-agent/src/lifecycle/launch/fork_state.rs +297 -0
- package/crates/team-agent/src/lifecycle/launch/identity.rs +372 -0
- package/crates/team-agent/src/lifecycle/launch/layout.rs +313 -0
- package/crates/team-agent/src/lifecycle/launch/leader_context.rs +478 -0
- package/crates/team-agent/src/lifecycle/launch/mcp_config.rs +201 -0
- package/crates/team-agent/src/lifecycle/launch/ownership.rs +66 -0
- package/crates/team-agent/src/lifecycle/launch/quick_start.rs +477 -0
- package/crates/team-agent/src/lifecycle/launch/quick_start_transport.rs +278 -0
- package/crates/team-agent/src/lifecycle/launch/readiness.rs +123 -0
- package/crates/team-agent/src/lifecycle/launch/spawn.rs +377 -0
- package/crates/team-agent/src/lifecycle/launch/spec_state.rs +434 -0
- package/crates/team-agent/src/lifecycle/launch/state_projection.rs +499 -0
- package/crates/team-agent/src/lifecycle/launch/worker_env.rs +438 -0
- package/crates/team-agent/src/lifecycle/launch.rs +119 -5351
- package/crates/team-agent/src/lifecycle/restart/agent.rs +44 -26
- package/crates/team-agent/src/lifecycle/restart/common.rs +53 -27
- package/crates/team-agent/src/lifecycle/restart/rebuild.rs +67 -26
- package/crates/team-agent/src/lifecycle/restart/remove.rs +435 -72
- package/crates/team-agent/src/lifecycle/restart.rs +1 -1
- package/crates/team-agent/src/lifecycle/tests/lane_ops.rs +575 -17
- package/crates/team-agent/src/lifecycle/tests/launch_spawn.rs +55 -2
- package/crates/team-agent/src/lifecycle/tests/lifecycle_lock.rs +24 -1
- package/crates/team-agent/src/lifecycle/tests/phase_golden.rs +1 -1
- package/crates/team-agent/src/mcp_server/lifecycle_tools/agent_ops.rs +30 -7
- package/crates/team-agent/src/mcp_server/lifecycle_tools/state_status.rs +8 -4
- package/crates/team-agent/src/mcp_server/mod.rs +2 -2
- package/crates/team-agent/src/mcp_server/tests/send.rs +22 -15
- package/crates/team-agent/src/mcp_server/tests/wire.rs +6 -0
- package/crates/team-agent/src/mcp_server/tools.rs +26 -15
- package/crates/team-agent/src/mcp_server/wire.rs +2 -18
- package/crates/team-agent/src/messaging/activity.rs +4 -2
- package/crates/team-agent/src/messaging/address.rs +86 -0
- package/crates/team-agent/src/messaging/delivery.rs +165 -39
- package/crates/team-agent/src/messaging/helpers.rs +17 -13
- package/crates/team-agent/src/messaging/leader_receiver.rs +60 -35
- package/crates/team-agent/src/messaging/mod.rs +11 -2
- package/crates/team-agent/src/messaging/persist.rs +309 -0
- package/crates/team-agent/src/messaging/results.rs +16 -24
- package/crates/team-agent/src/messaging/scheduler.rs +4 -2
- package/crates/team-agent/src/messaging/selftest.rs +19 -12
- package/crates/team-agent/src/messaging/send.rs +133 -58
- package/crates/team-agent/src/messaging/tests/leader_inject_acceptance.rs +305 -0
- package/crates/team-agent/src/messaging/tests/mod.rs +1 -0
- package/crates/team-agent/src/messaging/tests/runtime.rs +38 -17
- package/crates/team-agent/src/messaging/watchers.rs +13 -3
- package/crates/team-agent/src/redaction.rs +72 -2
- package/crates/team-agent/src/state/persist.rs +2 -1
- package/crates/team-agent/src/state/repository/tests.rs +47 -0
- package/crates/team-agent/src/state/repository.rs +59 -16
- package/package.json +4 -4
|
@@ -149,7 +149,7 @@ fn named_send_args(
|
|
|
149
149
|
workspace: ws.to_path_buf(),
|
|
150
150
|
team: None,
|
|
151
151
|
task: None,
|
|
152
|
-
sender:
|
|
152
|
+
sender: TrustedSender::leader(),
|
|
153
153
|
no_ack: false,
|
|
154
154
|
no_wait: false,
|
|
155
155
|
watch_result: false,
|
|
@@ -459,7 +459,7 @@ fn resolve_app_server_leader_name_fails_closed_on_transport_conflict() {
|
|
|
459
459
|
|
|
460
460
|
#[cfg(unix)]
|
|
461
461
|
#[test]
|
|
462
|
-
fn
|
|
462
|
+
fn send_to_name_app_server_leader_persists_before_async_delivery() {
|
|
463
463
|
let ws = named_ws("appserver-leader-send");
|
|
464
464
|
let fake = crate::app_server_test_support::FakeAppServer::start(
|
|
465
465
|
"named-send",
|
|
@@ -484,11 +484,13 @@ fn send_to_name_app_server_leader_submits_turn_without_tmux_pane() {
|
|
|
484
484
|
};
|
|
485
485
|
|
|
486
486
|
assert_eq!(value["ok"], json!(true));
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
487
|
+
let message_id = value["message_id"].as_str().expect("persisted message id");
|
|
488
|
+
assert!(message_id.starts_with("msg_"));
|
|
489
|
+
let store = crate::message_store::MessageStore::open(&ws).unwrap();
|
|
490
|
+
assert!(store.message_exists(message_id).unwrap());
|
|
491
|
+
assert!(
|
|
492
|
+
fake.received_turns().is_empty(),
|
|
493
|
+
"send returns after persistence; coordinator owns physical delivery"
|
|
492
494
|
);
|
|
493
495
|
let _ = std::fs::remove_dir_all(&ws);
|
|
494
496
|
}
|
|
@@ -90,7 +90,7 @@ fn run_dispatches_status_to_handler_returns_ok() {
|
|
|
90
90
|
|
|
91
91
|
#[test]
|
|
92
92
|
fn run_dispatches_send_to_handler_returns_ok() {
|
|
93
|
-
// run([send w1 hello --
|
|
93
|
+
// run([send w1 hello --workspace <seeded>]) -> cmd_send -> messaging::send_message
|
|
94
94
|
// -> ok (w1 is an in-team agent) -> ExitCode::Ok. w1 must be seeded: golden refuses non-team targets.
|
|
95
95
|
//
|
|
96
96
|
// OLD seed: flat `{"agents": {"w1": ...}}`.
|
|
@@ -112,8 +112,6 @@ fn run_dispatches_send_to_handler_returns_ok() {
|
|
|
112
112
|
"send".to_string(),
|
|
113
113
|
"w1".to_string(),
|
|
114
114
|
"hello".to_string(),
|
|
115
|
-
"--sender".to_string(),
|
|
116
|
-
"leader".to_string(),
|
|
117
115
|
"--workspace".to_string(),
|
|
118
116
|
ws.to_string_lossy().to_string(),
|
|
119
117
|
];
|
|
@@ -322,6 +320,7 @@ fn cli_add_agent_duplicate_id_surfaces_real_error() {
|
|
|
322
320
|
workspace: team,
|
|
323
321
|
team: None,
|
|
324
322
|
role_file: dup_role.to_string_lossy().to_string(),
|
|
323
|
+
force: false,
|
|
325
324
|
no_display: false,
|
|
326
325
|
json: true,
|
|
327
326
|
};
|
|
@@ -460,7 +460,7 @@ fn send_args_fixture() -> SendArgs {
|
|
|
460
460
|
workspace: PathBuf::from("."),
|
|
461
461
|
team: Some("teamA".into()),
|
|
462
462
|
task: Some("t-1".into()),
|
|
463
|
-
sender:
|
|
463
|
+
sender: TrustedSender::leader(),
|
|
464
464
|
no_ack: true,
|
|
465
465
|
no_wait: true,
|
|
466
466
|
watch_result: true,
|
|
@@ -515,7 +515,7 @@ fn send_options_negates_no_ack_and_no_wait_and_carries_watch() {
|
|
|
515
515
|
"watch_result flag MUST pass through into SendOptions"
|
|
516
516
|
);
|
|
517
517
|
assert!(!opts.confirm_human);
|
|
518
|
-
assert_eq!(opts.sender, "leader");
|
|
518
|
+
assert_eq!(opts.sender.as_str(), "leader");
|
|
519
519
|
assert_eq!(opts.timeout, 12.5);
|
|
520
520
|
}
|
|
521
521
|
|
|
@@ -866,15 +866,8 @@ fn cmd_send_unknown_task_surfaces_golden_error_envelope_not_silent() {
|
|
|
866
866
|
let _ = std::fs::remove_dir_all(&ws);
|
|
867
867
|
}
|
|
868
868
|
|
|
869
|
-
// P0 (b') — the SWALLOW guard: `run()` (the CLI process entry) MUST RENDER the send error, not
|
|
870
|
-
// discard Err(CliError) via unwrap_or (advisor %7 root cause). Proxy: emit_cli_error WRITES a
|
|
871
|
-
// `.team/logs/cli-error-*.log` (and prints the compact envelope) — if run() swallowed, neither
|
|
872
|
-
// happens. So a cli-error log containing the BARE "unknown task id: <id>" (no "validation:" prefix)
|
|
873
|
-
// + ExitCode::Error proves run() rendered. Drives the real argv→(exit,render) path.
|
|
874
|
-
// OLD/NEW: same Bug 1/2 seed sync as cmd_send_unknown_task_*; the render-vs-swallow
|
|
875
|
-
// behavior under test is unchanged.
|
|
876
869
|
#[test]
|
|
877
|
-
fn
|
|
870
|
+
fn run_send_legacy_task_flag_is_internalized_before_persistence() {
|
|
878
871
|
let ws = std::env::temp_dir().join(format!(
|
|
879
872
|
"ta-run-sendunk-{}-{}",
|
|
880
873
|
std::process::id(),
|
|
@@ -906,30 +899,21 @@ fn run_send_unknown_task_renders_error_not_silent_swallow() {
|
|
|
906
899
|
let code = run(&argv, &ws);
|
|
907
900
|
assert_eq!(
|
|
908
901
|
code,
|
|
909
|
-
ExitCode::
|
|
910
|
-
"
|
|
911
|
-
);
|
|
912
|
-
// run() must have RENDERED (emit_cli_error wrote the cli-error log); a swallow leaves none.
|
|
913
|
-
let logs_dir = ws.join(".team").join("logs");
|
|
914
|
-
let mut found = String::new();
|
|
915
|
-
if let Ok(entries) = std::fs::read_dir(&logs_dir) {
|
|
916
|
-
for entry in entries.flatten() {
|
|
917
|
-
let name = entry.file_name().to_string_lossy().to_string();
|
|
918
|
-
if name.starts_with("cli-error-") {
|
|
919
|
-
found = std::fs::read_to_string(entry.path()).unwrap_or_default();
|
|
920
|
-
break;
|
|
921
|
-
}
|
|
922
|
-
}
|
|
923
|
-
}
|
|
924
|
-
assert!(
|
|
925
|
-
found.contains("unknown task id: t-unknown"),
|
|
926
|
-
"run() must RENDER the send error (cli-error log written with the bare message) — a silent \
|
|
927
|
-
swallow (unwrap_or discards Err) leaves no log. got log body: {found:?}"
|
|
928
|
-
);
|
|
929
|
-
assert!(
|
|
930
|
-
!found.contains("validation:"),
|
|
931
|
-
"rendered error must be the bare golden message, NO 'validation:' prefix; got {found:?}"
|
|
902
|
+
ExitCode::Ok,
|
|
903
|
+
"legacy --task is sunset-noticed and ignored; public send persists without caller binding"
|
|
932
904
|
);
|
|
905
|
+
let connection = rusqlite::Connection::open(
|
|
906
|
+
ws.join(".team").join("runtime").join("team.db"),
|
|
907
|
+
)
|
|
908
|
+
.unwrap();
|
|
909
|
+
let task_id: Option<String> = connection
|
|
910
|
+
.query_row(
|
|
911
|
+
"SELECT task_id FROM messages WHERE content = 'go' ORDER BY rowid DESC LIMIT 1",
|
|
912
|
+
[],
|
|
913
|
+
|row| row.get(0),
|
|
914
|
+
)
|
|
915
|
+
.unwrap();
|
|
916
|
+
assert_eq!(task_id, None, "caller-supplied task ids must not reach storage");
|
|
933
917
|
let _ = std::fs::remove_dir_all(&ws);
|
|
934
918
|
}
|
|
935
919
|
|
|
@@ -312,7 +312,7 @@ pub struct SendArgs {
|
|
|
312
312
|
pub workspace: PathBuf,
|
|
313
313
|
pub team: Option<String>,
|
|
314
314
|
pub task: Option<String>,
|
|
315
|
-
pub sender:
|
|
315
|
+
pub sender: TrustedSender,
|
|
316
316
|
pub no_ack: bool,
|
|
317
317
|
pub no_wait: bool,
|
|
318
318
|
pub watch_result: bool,
|
|
@@ -323,16 +323,12 @@ pub struct SendArgs {
|
|
|
323
323
|
/// When set, the store insert uses this id verbatim; a repeat with the same
|
|
324
324
|
/// id returns a `Duplicate` refusal instead of creating a second row.
|
|
325
325
|
pub message_id: Option<String>,
|
|
326
|
-
///
|
|
327
|
-
///
|
|
328
|
-
/// directly into the specified tmux pane via `transport.inject`, bypassing
|
|
329
|
-
/// the agent-name → pane-id resolution + team-membership check. This is
|
|
330
|
-
/// the **cross-team communication** primitive: the target pane does not
|
|
331
|
-
/// need to be in the sender's team.
|
|
326
|
+
/// Deprecated compatibility input. Public send refuses pane identity and
|
|
327
|
+
/// requires a logical recipient so persistence always precedes delivery.
|
|
332
328
|
pub pane: Option<String>,
|
|
333
329
|
/// `--to-name <name>` — stable named addressing. Mutually exclusive with
|
|
334
330
|
/// `target` / `targets` / `--pane`; resolves workspace/team/agent or leader
|
|
335
|
-
/// name to the
|
|
331
|
+
/// name before delegating to the persisted message primitive.
|
|
336
332
|
pub to_name: Option<String>,
|
|
337
333
|
/// E7 (0.5.9 host-leader-registry-design): `--to-leader NAME` resolves
|
|
338
334
|
/// NAME through `~/.team-agent/leaders`, canonical-validates, and then
|
|
@@ -501,6 +497,7 @@ pub struct AddAgentArgs {
|
|
|
501
497
|
pub workspace: PathBuf,
|
|
502
498
|
pub team: Option<String>,
|
|
503
499
|
pub role_file: String,
|
|
500
|
+
pub force: bool,
|
|
504
501
|
pub no_display: bool,
|
|
505
502
|
pub json: bool,
|
|
506
503
|
}
|
|
@@ -41,7 +41,7 @@ use std::time::Duration;
|
|
|
41
41
|
|
|
42
42
|
use conpty_transport::{pipe_name_for, NamedPipeClient};
|
|
43
43
|
|
|
44
|
-
use crate::state::persist::
|
|
44
|
+
use crate::state::persist::runtime_state_path;
|
|
45
45
|
use crate::state::StateError;
|
|
46
46
|
|
|
47
47
|
/// Maximum shim connect+hello attempts before giving up. Each attempt
|
|
@@ -397,15 +397,11 @@ fn finalize(
|
|
|
397
397
|
client: NamedPipeClient,
|
|
398
398
|
workspace: &Path,
|
|
399
399
|
) -> Result<ShimHandle, ShimError> {
|
|
400
|
-
let
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
}
|
|
406
|
-
} else {
|
|
407
|
-
json!({})
|
|
408
|
-
};
|
|
400
|
+
let mut state = crate::state::repository::StateRepository::new(workspace)
|
|
401
|
+
.load_workspace_if_exists_without_migrations()
|
|
402
|
+
.ok()
|
|
403
|
+
.flatten()
|
|
404
|
+
.unwrap_or_else(|| json!({}));
|
|
409
405
|
// CR C-1: token NOT stored. Only pid/pipe_name/pipe_ready.
|
|
410
406
|
let obj = state
|
|
411
407
|
.as_object_mut()
|
|
@@ -431,7 +427,14 @@ fn finalize(
|
|
|
431
427
|
"pipe_ready": true,
|
|
432
428
|
}),
|
|
433
429
|
);
|
|
434
|
-
|
|
430
|
+
crate::state::repository::StateRepository::new(workspace)
|
|
431
|
+
.save(
|
|
432
|
+
crate::state::repository::StateWriteIntent::CoordinatorConptyShim {
|
|
433
|
+
team_key: state.get("active_team_key").and_then(Value::as_str),
|
|
434
|
+
},
|
|
435
|
+
&state,
|
|
436
|
+
)
|
|
437
|
+
.map_err(|e| ShimError::StatePersist { source: e })?;
|
|
435
438
|
// 0.5.x Windows portability Batch 7 F6: state merge now preserves
|
|
436
439
|
// the `transport.shim` block through downstream saves (see
|
|
437
440
|
// `state::persist::preserve_transport_shim`), so the Batch 6
|
|
@@ -450,12 +453,9 @@ fn finalize(
|
|
|
450
453
|
/// Windows worker). Callers use `platform::process::terminate_pid`
|
|
451
454
|
/// on the returned pid.
|
|
452
455
|
pub fn recorded_shim_pid(workspace: &Path) -> Option<u32> {
|
|
453
|
-
let
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
}
|
|
457
|
-
let text = std::fs::read_to_string(&state_path).ok()?;
|
|
458
|
-
let state: Value = serde_json::from_str(&text).ok()?;
|
|
456
|
+
let state = crate::state::repository::StateRepository::new(workspace)
|
|
457
|
+
.load_workspace_if_exists_without_migrations()
|
|
458
|
+
.ok()??;
|
|
459
459
|
state
|
|
460
460
|
.get("transport")?
|
|
461
461
|
.get("shim")?
|
|
@@ -466,12 +466,9 @@ pub fn recorded_shim_pid(workspace: &Path) -> Option<u32> {
|
|
|
466
466
|
|
|
467
467
|
/// Read `state.transport.shim.pipe_name` for reconnect routing.
|
|
468
468
|
pub fn recorded_shim_pipe_name(workspace: &Path) -> Option<String> {
|
|
469
|
-
let
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
}
|
|
473
|
-
let text = std::fs::read_to_string(&state_path).ok()?;
|
|
474
|
-
let state: Value = serde_json::from_str(&text).ok()?;
|
|
469
|
+
let state = crate::state::repository::StateRepository::new(workspace)
|
|
470
|
+
.load_workspace_if_exists_without_migrations()
|
|
471
|
+
.ok()??;
|
|
475
472
|
state
|
|
476
473
|
.get("transport")?
|
|
477
474
|
.get("shim")?
|
|
@@ -647,19 +644,26 @@ pub fn mark_transport_unavailable(workspace: &Path, reason: &str) -> Result<(),
|
|
|
647
644
|
// Clear the shim block from state so subsequent factory
|
|
648
645
|
// `conpty_pipe_ready` checks return false and the operator sees
|
|
649
646
|
// an honest "no live shim" via status --detail.
|
|
650
|
-
let
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
647
|
+
let mut state = match crate::state::repository::StateRepository::new(workspace)
|
|
648
|
+
.load_workspace_if_exists_without_migrations()
|
|
649
|
+
{
|
|
650
|
+
Ok(Some(state)) => state,
|
|
651
|
+
Ok(None) => return Ok(()),
|
|
652
|
+
Err(StateError::Json(_)) => serde_json::json!({}),
|
|
653
|
+
Err(error) => return Err(error),
|
|
654
|
+
};
|
|
656
655
|
if let Some(transport) = state.get_mut("transport").and_then(|t| t.as_object_mut()) {
|
|
657
656
|
if let Some(shim) = transport.get_mut("shim").and_then(|s| s.as_object_mut()) {
|
|
658
657
|
shim.insert("pipe_ready".to_string(), serde_json::json!(false));
|
|
659
658
|
shim.insert("unavailable_reason".to_string(), serde_json::json!(reason));
|
|
660
659
|
}
|
|
661
660
|
}
|
|
662
|
-
|
|
661
|
+
crate::state::repository::StateRepository::new(workspace).save(
|
|
662
|
+
crate::state::repository::StateWriteIntent::CoordinatorConptyShim {
|
|
663
|
+
team_key: state.get("active_team_key").and_then(Value::as_str),
|
|
664
|
+
},
|
|
665
|
+
&state,
|
|
666
|
+
)
|
|
663
667
|
}
|
|
664
668
|
|
|
665
669
|
#[cfg(test)]
|
|
@@ -32,6 +32,10 @@ pub(crate) fn detect_abnormal_exits(
|
|
|
32
32
|
let team = crate::state::projection::team_state_key(&snapshot);
|
|
33
33
|
let session_name = snapshot.get("session_name").and_then(Value::as_str);
|
|
34
34
|
for agent in abnormal_watch_agents(&snapshot) {
|
|
35
|
+
// Pane/process liveness is independent of transcript content. A
|
|
36
|
+
// frozen rollout is common after pane death, so probe before the
|
|
37
|
+
// metadata dedupe gate; only the expensive tail scan stays deduped.
|
|
38
|
+
let liveness = agent_process_liveness(&agent, session_name, targets, transport);
|
|
35
39
|
let rollout_path = resolve_agent_rollout_path(workspace, &agent.rollout_path);
|
|
36
40
|
let metadata = match std::fs::metadata(&rollout_path) {
|
|
37
41
|
Ok(metadata) => metadata,
|
|
@@ -67,6 +71,7 @@ pub(crate) fn detect_abnormal_exits(
|
|
|
67
71
|
abnormal_watch_stored_metadata(&snapshot, &agent.agent_id),
|
|
68
72
|
) {
|
|
69
73
|
if stored == (size, mtime) {
|
|
74
|
+
refresh_abnormal_watch_liveness(state, &agent.agent_id, &liveness);
|
|
70
75
|
continue;
|
|
71
76
|
}
|
|
72
77
|
}
|
|
@@ -97,7 +102,6 @@ pub(crate) fn detect_abnormal_exits(
|
|
|
97
102
|
continue;
|
|
98
103
|
}
|
|
99
104
|
};
|
|
100
|
-
let liveness = agent_process_liveness(&agent, session_name, targets, transport);
|
|
101
105
|
let fact = crate::provider::latest_explicit_error_fact(agent.provider, &text);
|
|
102
106
|
let error_observation_key = fact
|
|
103
107
|
.as_ref()
|
|
@@ -218,6 +222,7 @@ pub(crate) fn detect_abnormal_exits(
|
|
|
218
222
|
} else {
|
|
219
223
|
"refused"
|
|
220
224
|
};
|
|
225
|
+
let provider_process_dead = provider_process_dead_fact(&liveness);
|
|
221
226
|
event_log.write(
|
|
222
227
|
"worker.abnormal_exit",
|
|
223
228
|
serde_json::json!({
|
|
@@ -227,7 +232,7 @@ pub(crate) fn detect_abnormal_exits(
|
|
|
227
232
|
"path": agent.rollout_path_display.as_str(),
|
|
228
233
|
"dead_process": liveness.state == ProcessLiveness::Dead,
|
|
229
234
|
"process_dead": liveness.state == ProcessLiveness::Dead,
|
|
230
|
-
"provider_process_dead":
|
|
235
|
+
"provider_process_dead": provider_process_dead,
|
|
231
236
|
"latest_error": true,
|
|
232
237
|
"latest_explicit_error": true,
|
|
233
238
|
"error_recency": error_recency.as_str(),
|
|
@@ -235,7 +240,7 @@ pub(crate) fn detect_abnormal_exits(
|
|
|
235
240
|
"dead_process_and_latest_error": liveness.state == ProcessLiveness::Dead,
|
|
236
241
|
"dead_process_and_latest_explicit_error": liveness.state == ProcessLiveness::Dead,
|
|
237
242
|
"process_dead_and_latest_explicit_error": liveness.state == ProcessLiveness::Dead,
|
|
238
|
-
"provider_process_dead_and_latest_explicit_error":
|
|
243
|
+
"provider_process_dead_and_latest_explicit_error": provider_process_dead,
|
|
239
244
|
"signature": fact.signature.as_str(),
|
|
240
245
|
"turn_id": fact.turn_id.as_ref().map(|id| id.as_str()),
|
|
241
246
|
"apiErrorStatus": fact.api_error_status,
|
|
@@ -256,6 +261,50 @@ pub(crate) fn detect_abnormal_exits(
|
|
|
256
261
|
Ok(())
|
|
257
262
|
}
|
|
258
263
|
|
|
264
|
+
fn provider_process_dead_fact(liveness: &ProcessCheck) -> bool {
|
|
265
|
+
liveness.state == ProcessLiveness::Dead
|
|
266
|
+
&& !liveness.detail.starts_with("pane_dead:")
|
|
267
|
+
&& !liveness.detail.starts_with("window_missing:")
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
fn worker_provider_exited_fact(liveness: &ProcessCheck) -> bool {
|
|
271
|
+
provider_process_dead_fact(liveness) && liveness.detail.starts_with("worker_provider_exited:")
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
fn refresh_abnormal_watch_liveness(state: &mut Value, agent_id: &str, liveness: &ProcessCheck) {
|
|
275
|
+
let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch")
|
|
276
|
+
.and_then(|watch| watch.get_mut(agent_id))
|
|
277
|
+
.and_then(Value::as_object_mut)
|
|
278
|
+
else {
|
|
279
|
+
return;
|
|
280
|
+
};
|
|
281
|
+
let dead_process = liveness.state == ProcessLiveness::Dead;
|
|
282
|
+
let provider_process_dead = provider_process_dead_fact(liveness);
|
|
283
|
+
let latest_explicit_error = watch
|
|
284
|
+
.get("latest_explicit_error")
|
|
285
|
+
.and_then(Value::as_bool)
|
|
286
|
+
.unwrap_or(false);
|
|
287
|
+
// Persist fact transitions, not the tick's observation time; otherwise an
|
|
288
|
+
// unchanged transcript makes every steady tick rewrite state.json.
|
|
289
|
+
let patch = serde_json::json!({
|
|
290
|
+
"last_liveness": process_liveness_wire(liveness.state),
|
|
291
|
+
"last_liveness_detail": liveness.detail.as_str(),
|
|
292
|
+
"dead_process": dead_process,
|
|
293
|
+
"process_dead": dead_process,
|
|
294
|
+
"provider_process_dead": provider_process_dead,
|
|
295
|
+
"worker_provider_exited": worker_provider_exited_fact(liveness),
|
|
296
|
+
"provider_process_dead_and_latest_explicit_error": provider_process_dead && latest_explicit_error,
|
|
297
|
+
});
|
|
298
|
+
if let Some(patch) = patch.as_object() {
|
|
299
|
+
if patch
|
|
300
|
+
.iter()
|
|
301
|
+
.any(|(key, value)| watch.get(key) != Some(value))
|
|
302
|
+
{
|
|
303
|
+
watch.extend(patch.clone());
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
259
308
|
#[derive(Debug, Clone)]
|
|
260
309
|
struct AbnormalWatchAgent {
|
|
261
310
|
agent_id: String,
|
|
@@ -816,8 +865,8 @@ fn abnormal_watch_payload(
|
|
|
816
865
|
// marker (detail prefix `worker_provider_exited:`). status_port's
|
|
817
866
|
// RuntimeFreshness collector reads this field to downgrade the
|
|
818
867
|
// corresponding agent row.
|
|
819
|
-
let
|
|
820
|
-
|
|
868
|
+
let provider_process_dead = provider_process_dead_fact(&liveness);
|
|
869
|
+
let worker_provider_exited = worker_provider_exited_fact(&liveness);
|
|
821
870
|
serde_json::json!({
|
|
822
871
|
"path": agent.rollout_path_display.as_str(),
|
|
823
872
|
"provider": provider_wire(agent.provider),
|
|
@@ -829,7 +878,7 @@ fn abnormal_watch_payload(
|
|
|
829
878
|
"last_liveness_detail": liveness.detail,
|
|
830
879
|
"dead_process": dead_process,
|
|
831
880
|
"process_dead": dead_process,
|
|
832
|
-
"provider_process_dead":
|
|
881
|
+
"provider_process_dead": provider_process_dead,
|
|
833
882
|
"worker_provider_exited": worker_provider_exited,
|
|
834
883
|
"latest_error": latest_explicit_error,
|
|
835
884
|
"latest_explicit_error": latest_explicit_error,
|
|
@@ -839,11 +888,10 @@ fn abnormal_watch_payload(
|
|
|
839
888
|
"dead_process_and_latest_error": dead_process && latest_explicit_error,
|
|
840
889
|
"dead_process_and_latest_explicit_error": dead_process && latest_explicit_error,
|
|
841
890
|
"process_dead_and_latest_explicit_error": dead_process && latest_explicit_error,
|
|
842
|
-
"provider_process_dead_and_latest_explicit_error":
|
|
891
|
+
"provider_process_dead_and_latest_explicit_error": provider_process_dead && latest_explicit_error,
|
|
843
892
|
"suppressed_reason": suppressed_reason,
|
|
844
893
|
"notification": notify,
|
|
845
894
|
"last_error": error,
|
|
846
|
-
"last_checked_at": chrono::Utc::now().to_rfc3339(),
|
|
847
895
|
})
|
|
848
896
|
}
|
|
849
897
|
|
|
@@ -1060,6 +1108,7 @@ fn write_abnormal_check(
|
|
|
1060
1108
|
mtime_ns: Option<u64>,
|
|
1061
1109
|
) -> Result<(), TickError> {
|
|
1062
1110
|
let dead_process = liveness.state == ProcessLiveness::Dead;
|
|
1111
|
+
let provider_process_dead = provider_process_dead_fact(liveness);
|
|
1063
1112
|
let latest_explicit_error = fact.is_some();
|
|
1064
1113
|
event_log.write(
|
|
1065
1114
|
"worker.abnormal_exit.check",
|
|
@@ -1073,7 +1122,7 @@ fn write_abnormal_check(
|
|
|
1073
1122
|
"mtime_ns": mtime_ns,
|
|
1074
1123
|
"dead_process": dead_process,
|
|
1075
1124
|
"process_dead": dead_process,
|
|
1076
|
-
"provider_process_dead":
|
|
1125
|
+
"provider_process_dead": provider_process_dead,
|
|
1077
1126
|
"latest_error": latest_explicit_error,
|
|
1078
1127
|
"latest_explicit_error": latest_explicit_error,
|
|
1079
1128
|
"error_recency": error_recency.as_str(),
|
|
@@ -1081,7 +1130,7 @@ fn write_abnormal_check(
|
|
|
1081
1130
|
"dead_process_and_latest_error": dead_process && latest_explicit_error,
|
|
1082
1131
|
"dead_process_and_latest_explicit_error": dead_process && latest_explicit_error,
|
|
1083
1132
|
"process_dead_and_latest_explicit_error": dead_process && latest_explicit_error,
|
|
1084
|
-
"provider_process_dead_and_latest_explicit_error":
|
|
1133
|
+
"provider_process_dead_and_latest_explicit_error": provider_process_dead && latest_explicit_error,
|
|
1085
1134
|
"notification": matches!(decision, AbnormalExitDecision::Notify),
|
|
1086
1135
|
"suppressed_reason": match decision {
|
|
1087
1136
|
AbnormalExitDecision::Suppress(reason) => Some(reason),
|
|
@@ -1107,6 +1156,7 @@ fn write_abnormal_suppressed(
|
|
|
1107
1156
|
liveness: &ProcessCheck,
|
|
1108
1157
|
reason: &str,
|
|
1109
1158
|
) -> Result<(), TickError> {
|
|
1159
|
+
let provider_process_dead = provider_process_dead_fact(liveness);
|
|
1110
1160
|
event_log.write(
|
|
1111
1161
|
"abnormal_exit.single_signal_suppressed",
|
|
1112
1162
|
serde_json::json!({
|
|
@@ -1118,7 +1168,7 @@ fn write_abnormal_suppressed(
|
|
|
1118
1168
|
"notification": false,
|
|
1119
1169
|
"dead_process": liveness.state == ProcessLiveness::Dead,
|
|
1120
1170
|
"process_dead": liveness.state == ProcessLiveness::Dead,
|
|
1121
|
-
"provider_process_dead":
|
|
1171
|
+
"provider_process_dead": provider_process_dead,
|
|
1122
1172
|
"latest_error": false,
|
|
1123
1173
|
"latest_explicit_error": false,
|
|
1124
1174
|
"error_recency": ErrorRecency::None.as_str(),
|
|
@@ -1845,7 +1895,13 @@ pub(crate) fn attempt_due_recoveries(
|
|
|
1845
1895
|
return;
|
|
1846
1896
|
};
|
|
1847
1897
|
if clear_stale_terminal_next_retry_at(&mut state) {
|
|
1848
|
-
let _ = crate::state::
|
|
1898
|
+
let _ = crate::state::repository::StateRepository::new(workspace).save(
|
|
1899
|
+
crate::state::repository::StateWriteIntent::CoordinatorApiErrorRecovery {
|
|
1900
|
+
team_key: state.get("active_team_key").and_then(Value::as_str),
|
|
1901
|
+
agent_id: None,
|
|
1902
|
+
},
|
|
1903
|
+
&state,
|
|
1904
|
+
);
|
|
1849
1905
|
}
|
|
1850
1906
|
let due_agents = collect_due_recovery_agents(&state);
|
|
1851
1907
|
for agent_id in due_agents {
|
|
@@ -2098,7 +2154,13 @@ fn write_recovery_intent_result(workspace: &Path, agent_id: &str, update: Recove
|
|
|
2098
2154
|
}
|
|
2099
2155
|
}
|
|
2100
2156
|
}
|
|
2101
|
-
let _ = crate::state::
|
|
2157
|
+
let _ = crate::state::repository::StateRepository::new(workspace).save(
|
|
2158
|
+
crate::state::repository::StateWriteIntent::CoordinatorApiErrorRecovery {
|
|
2159
|
+
team_key: state.get("active_team_key").and_then(Value::as_str),
|
|
2160
|
+
agent_id: Some(agent_id),
|
|
2161
|
+
},
|
|
2162
|
+
&state,
|
|
2163
|
+
);
|
|
2102
2164
|
}
|
|
2103
2165
|
|
|
2104
2166
|
#[cfg(test)]
|
|
@@ -2510,6 +2572,66 @@ mod tests {
|
|
|
2510
2572
|
assert_eq!(suppressed["fresh_error"], serde_json::json!(false));
|
|
2511
2573
|
}
|
|
2512
2574
|
|
|
2575
|
+
#[test]
|
|
2576
|
+
fn abnormal_unchanged_transcript_keeps_provider_death_out_of_pane_projection() {
|
|
2577
|
+
let dir = temp_abnormal_dir("unchanged-provider-dead");
|
|
2578
|
+
let rollout = dir.join("rollout-w1.jsonl");
|
|
2579
|
+
std::fs::write(
|
|
2580
|
+
&rollout,
|
|
2581
|
+
"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"completed\"}}}\n",
|
|
2582
|
+
)
|
|
2583
|
+
.unwrap();
|
|
2584
|
+
seed_abnormal_state(&dir, &rollout, "alive", 1);
|
|
2585
|
+
let coordinator = abnormal_test_coordinator(&dir);
|
|
2586
|
+
coordinator.tick().unwrap();
|
|
2587
|
+
|
|
2588
|
+
let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
|
|
2589
|
+
state["agents"]["w1"]["process_liveness"] = serde_json::json!("dead");
|
|
2590
|
+
crate::state::persist::save_runtime_state(&dir, &state).unwrap();
|
|
2591
|
+
coordinator.tick().unwrap();
|
|
2592
|
+
|
|
2593
|
+
let state = crate::state::persist::load_runtime_state(&dir).unwrap();
|
|
2594
|
+
let agent = &state["agents"]["w1"];
|
|
2595
|
+
assert!(
|
|
2596
|
+
agent.get("provider_process_dead").is_none(),
|
|
2597
|
+
"provider death belongs to abnormal watch, not the pane-dead seat projection: {agent}"
|
|
2598
|
+
);
|
|
2599
|
+
assert!(
|
|
2600
|
+
agent.get("stale_reason").is_none(),
|
|
2601
|
+
"provider death must not be mislabeled pane_dead: {agent}"
|
|
2602
|
+
);
|
|
2603
|
+
let watch = &state["coordinator"]["abnormal_exit_watch"]["w1"];
|
|
2604
|
+
assert_eq!(watch["provider_process_dead"], serde_json::json!(true));
|
|
2605
|
+
assert_eq!(watch["worker_provider_exited"], serde_json::json!(false));
|
|
2606
|
+
assert_eq!(
|
|
2607
|
+
watch["last_liveness_detail"],
|
|
2608
|
+
serde_json::json!("explicit:dead")
|
|
2609
|
+
);
|
|
2610
|
+
}
|
|
2611
|
+
|
|
2612
|
+
#[test]
|
|
2613
|
+
fn abnormal_pane_absence_is_not_provider_process_death() {
|
|
2614
|
+
let agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(1), None);
|
|
2615
|
+
for detail in ["pane_dead:%1", "window_missing:w1"] {
|
|
2616
|
+
let payload = abnormal_watch_payload(
|
|
2617
|
+
&agent,
|
|
2618
|
+
Some(1),
|
|
2619
|
+
Some(2),
|
|
2620
|
+
process_check(ProcessLiveness::Dead, detail.to_string()),
|
|
2621
|
+
None,
|
|
2622
|
+
ErrorRecency::None,
|
|
2623
|
+
None,
|
|
2624
|
+
None,
|
|
2625
|
+
);
|
|
2626
|
+
assert_eq!(
|
|
2627
|
+
payload["provider_process_dead"],
|
|
2628
|
+
serde_json::json!(false),
|
|
2629
|
+
"pane absence and provider death are distinct facts: {payload}"
|
|
2630
|
+
);
|
|
2631
|
+
assert_eq!(payload["worker_provider_exited"], serde_json::json!(false));
|
|
2632
|
+
}
|
|
2633
|
+
}
|
|
2634
|
+
|
|
2513
2635
|
#[test]
|
|
2514
2636
|
fn abnormal_recency_treats_cohort_change_as_stale() {
|
|
2515
2637
|
let agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(2), None);
|
|
@@ -752,6 +752,43 @@ impl Coordinator {
|
|
|
752
752
|
}
|
|
753
753
|
};
|
|
754
754
|
if !windows.iter().any(|known| known == &window) {
|
|
755
|
+
// Missing is proof only inside a non-empty same-session
|
|
756
|
+
// inventory (a live peer exists). An empty snapshot is an
|
|
757
|
+
// unavailable transport, not proof every worker died.
|
|
758
|
+
if windows.is_empty() {
|
|
759
|
+
continue;
|
|
760
|
+
}
|
|
761
|
+
if let Some(agent_obj) = agent.as_object_mut() {
|
|
762
|
+
agent_obj.insert("status".to_string(), serde_json::json!("stopped"));
|
|
763
|
+
agent_obj.insert("worker_state".to_string(), serde_json::json!("DEAD"));
|
|
764
|
+
agent_obj.insert("stale".to_string(), serde_json::json!(true));
|
|
765
|
+
agent_obj.insert("stale_reason".to_string(), serde_json::json!("pane_dead"));
|
|
766
|
+
}
|
|
767
|
+
let conn = crate::db::schema::open_db(store.db_path()).map_err(|error| {
|
|
768
|
+
TickError::MessageStore(crate::message_store::MessageStoreError::Db(error))
|
|
769
|
+
})?;
|
|
770
|
+
conn.execute(
|
|
771
|
+
"insert into agent_health(owner_team_id, agent_id, status, updated_at) \
|
|
772
|
+
values (?1, ?2, 'DEAD', ?3) \
|
|
773
|
+
on conflict(owner_team_id, agent_id) do update set \
|
|
774
|
+
status='DEAD', updated_at=excluded.updated_at",
|
|
775
|
+
rusqlite::params![
|
|
776
|
+
team_key.as_ref().map(|key| key.as_str()),
|
|
777
|
+
agent_id,
|
|
778
|
+
chrono::Utc::now().to_rfc3339()
|
|
779
|
+
],
|
|
780
|
+
)
|
|
781
|
+
.map_err(|error| {
|
|
782
|
+
TickError::MessageStore(crate::message_store::MessageStoreError::Sqlite(error))
|
|
783
|
+
})?;
|
|
784
|
+
let _ = event_log.write(
|
|
785
|
+
"coordinator.agent_pane_dead",
|
|
786
|
+
serde_json::json!({
|
|
787
|
+
"agent_id": agent_id,
|
|
788
|
+
"target": format!("{target:?}"),
|
|
789
|
+
"stale_reason": "pane_dead",
|
|
790
|
+
}),
|
|
791
|
+
);
|
|
755
792
|
continue;
|
|
756
793
|
}
|
|
757
794
|
// Warm-idle suppression still gates pane fallback ONLY. When
|