@team-agent/installer 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +3 -1
- package/Cargo.toml +1 -1
- package/crates/team-agent/Cargo.toml +20 -0
- package/crates/team-agent/src/cli/diagnose.rs +191 -31
- package/crates/team-agent/src/cli/emit.rs +5 -0
- package/crates/team-agent/src/cli/mod.rs +213 -89
- package/crates/team-agent/src/cli/tests/named_address.rs +4 -0
- package/crates/team-agent/src/codex_app_server.rs +62 -1
- package/crates/team-agent/src/conpty/backend.rs +120 -8
- package/crates/team-agent/src/coordinator/backoff.rs +88 -2
- package/crates/team-agent/src/coordinator/conpty_shim.rs +730 -0
- package/crates/team-agent/src/coordinator/health.rs +97 -11
- package/crates/team-agent/src/coordinator/mod.rs +8 -0
- package/crates/team-agent/src/coordinator/tests/daemon.rs +2 -1
- package/crates/team-agent/src/coordinator/tests/tick_core.rs +6 -0
- package/crates/team-agent/src/coordinator/tick.rs +13 -0
- package/crates/team-agent/src/diagnose/orphans.rs +18 -7
- package/crates/team-agent/src/leader/provider_attribution.rs +19 -34
- package/crates/team-agent/src/leader/tests/lease_api.rs +7 -0
- package/crates/team-agent/src/lib.rs +14 -1
- package/crates/team-agent/src/lifecycle/launch.rs +89 -92
- package/crates/team-agent/src/lifecycle/lock.rs +40 -33
- package/crates/team-agent/src/lifecycle/restart/agent.rs +25 -19
- package/crates/team-agent/src/lifecycle/restart/common.rs +53 -2
- package/crates/team-agent/src/lifecycle/restart/rebuild.rs +108 -6
- package/crates/team-agent/src/lifecycle/restart/selection.rs +47 -18
- package/crates/team-agent/src/lifecycle/restart.rs +16 -6
- package/crates/team-agent/src/lifecycle/tests/agent_ops.rs +7 -0
- package/crates/team-agent/src/lifecycle/tests/launch_spawn.rs +287 -53
- package/crates/team-agent/src/lifecycle/tests/restart.rs +144 -5
- package/crates/team-agent/src/lifecycle/types.rs +1 -0
- package/crates/team-agent/src/mcp_server/wire.rs +6 -7
- package/crates/team-agent/src/messaging/tests/runtime.rs +5 -0
- package/crates/team-agent/src/packaging/tests.rs +41 -6
- package/crates/team-agent/src/packaging/types.rs +31 -3
- package/crates/team-agent/src/platform/argv.rs +324 -0
- package/crates/team-agent/src/platform/errors.rs +95 -0
- package/crates/team-agent/src/platform/file_lock.rs +418 -0
- package/crates/team-agent/src/platform/mod.rs +66 -0
- package/crates/team-agent/src/platform/process.rs +555 -0
- package/crates/team-agent/src/provider/adapter.rs +103 -41
- package/crates/team-agent/src/provider/session/capture.rs +328 -66
- package/crates/team-agent/src/provider/session/resume.rs +30 -28
- package/crates/team-agent/src/provider/session_scan/common.rs +117 -1
- package/crates/team-agent/src/provider/session_scan/copilot.rs +1 -0
- package/crates/team-agent/src/provider/session_scan.rs +1 -0
- package/crates/team-agent/src/state/persist.rs +222 -25
- package/crates/team-agent/src/state/projection.rs +59 -4
- package/crates/team-agent/src/tmux_backend.rs +63 -13
- package/crates/team-agent/src/transport_factory.rs +124 -5
- package/package.json +4 -4
|
@@ -85,6 +85,16 @@ fn acquire_agent_lifecycle_lock_with_deadlines(
|
|
|
85
85
|
timeout: Duration,
|
|
86
86
|
held_long: Duration,
|
|
87
87
|
) -> Result<LifecycleLockGuard, LifecycleError> {
|
|
88
|
+
// 0.5.x Windows portability Batch 2: migrated to
|
|
89
|
+
// `crate::platform::file_lock::{try_lock_once_nonblocking, unlock}`
|
|
90
|
+
// so the same polling loop + waiter file + 5s `lock_held_long`
|
|
91
|
+
// event + 30s N38 timeout error shape works on both Unix (`flock`)
|
|
92
|
+
// and Windows (`LockFileEx`). The Batch 0 non-Unix stub that
|
|
93
|
+
// returned `lock_timeout_error` unconditionally is now gone —
|
|
94
|
+
// Windows callers get real lock behavior. Byte-preserving on Unix:
|
|
95
|
+
// the polling cadence, waiter file writes, `lock_held_long_event`
|
|
96
|
+
// emission, and error shape are all unchanged relative to the
|
|
97
|
+
// pre-Batch-2 unix branch.
|
|
88
98
|
let lock_path = agent_lifecycle_lock_path(request.workspace);
|
|
89
99
|
if let Some(parent) = lock_path.parent() {
|
|
90
100
|
std::fs::create_dir_all(parent)
|
|
@@ -100,42 +110,38 @@ fn acquire_agent_lifecycle_lock_with_deadlines(
|
|
|
100
110
|
let started = Instant::now();
|
|
101
111
|
let mut long_event_written = false;
|
|
102
112
|
let mut waiter = None;
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
let fd = file.as_raw_fd();
|
|
107
|
-
loop {
|
|
108
|
-
let rc = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) };
|
|
109
|
-
if rc == 0 {
|
|
113
|
+
loop {
|
|
114
|
+
match crate::platform::file_lock::try_lock_once_nonblocking(&file) {
|
|
115
|
+
Ok(true) => {
|
|
110
116
|
write_lock_metadata(&mut file, &request, &lock_path)?;
|
|
111
117
|
return Ok(LifecycleLockGuard { file });
|
|
112
118
|
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
119
|
+
Ok(false) => {}
|
|
120
|
+
Err(e) => {
|
|
121
|
+
return Err(LifecycleError::StatePersist(format!(
|
|
122
|
+
"lifecycle lock acquire io error at {}: {e}",
|
|
123
|
+
lock_path.display()
|
|
124
|
+
)));
|
|
116
125
|
}
|
|
117
|
-
if let Some(waiter) = waiter.as_ref() {
|
|
118
|
-
let _ = waiter.write(&request, elapsed);
|
|
119
|
-
}
|
|
120
|
-
if !long_event_written && elapsed >= held_long {
|
|
121
|
-
let _ =
|
|
122
|
-
write_lock_held_long_event(&request, &lock_path, elapsed, held_long, timeout);
|
|
123
|
-
long_event_written = true;
|
|
124
|
-
}
|
|
125
|
-
if elapsed >= timeout {
|
|
126
|
-
return Err(lock_timeout_error(&request, &lock_path, elapsed));
|
|
127
|
-
}
|
|
128
|
-
std::thread::sleep(std::cmp::min(
|
|
129
|
-
Duration::from_millis(50),
|
|
130
|
-
timeout.saturating_sub(elapsed),
|
|
131
|
-
));
|
|
132
126
|
}
|
|
133
|
-
}
|
|
134
|
-
#[cfg(not(unix))]
|
|
135
|
-
{
|
|
136
|
-
let _ = file;
|
|
137
127
|
let elapsed = started.elapsed();
|
|
138
|
-
|
|
128
|
+
if waiter.is_none() {
|
|
129
|
+
waiter = WaiterFile::create(&request, &lock_path).ok();
|
|
130
|
+
}
|
|
131
|
+
if let Some(waiter) = waiter.as_ref() {
|
|
132
|
+
let _ = waiter.write(&request, elapsed);
|
|
133
|
+
}
|
|
134
|
+
if !long_event_written && elapsed >= held_long {
|
|
135
|
+
let _ = write_lock_held_long_event(&request, &lock_path, elapsed, held_long, timeout);
|
|
136
|
+
long_event_written = true;
|
|
137
|
+
}
|
|
138
|
+
if elapsed >= timeout {
|
|
139
|
+
return Err(lock_timeout_error(&request, &lock_path, elapsed));
|
|
140
|
+
}
|
|
141
|
+
std::thread::sleep(std::cmp::min(
|
|
142
|
+
Duration::from_millis(50),
|
|
143
|
+
timeout.saturating_sub(elapsed),
|
|
144
|
+
));
|
|
139
145
|
}
|
|
140
146
|
}
|
|
141
147
|
|
|
@@ -293,10 +299,11 @@ impl Drop for WaiterFile {
|
|
|
293
299
|
}
|
|
294
300
|
}
|
|
295
301
|
|
|
296
|
-
#[cfg(unix)]
|
|
297
302
|
impl Drop for LifecycleLockGuard {
|
|
298
303
|
fn drop(&mut self) {
|
|
299
|
-
|
|
300
|
-
|
|
304
|
+
// Batch 2: unlock via platform primitive. Best-effort — OS
|
|
305
|
+
// releases when the file handle closes anyway. Uniform on
|
|
306
|
+
// both `flock(LOCK_UN)` (unix) and `UnlockFileEx` (windows).
|
|
307
|
+
let _ = crate::platform::file_lock::unlock(&self.file);
|
|
301
308
|
}
|
|
302
309
|
}
|
|
@@ -256,7 +256,21 @@ pub(crate) fn start_agent_at_paths(
|
|
|
256
256
|
// after spawn" race with coordinator and no double source of truth.
|
|
257
257
|
crate::lifecycle::launch::annotate_runtime_tmux_endpoint(&mut state, transport, workspace);
|
|
258
258
|
let team_key = restart_projection_team_key(&state, team);
|
|
259
|
-
|
|
259
|
+
let skip_capture_backfill = if matches!(
|
|
260
|
+
start_mode,
|
|
261
|
+
StartMode::Fresh | StartMode::FreshAfterMissingRollout
|
|
262
|
+
) {
|
|
263
|
+
vec![agent_id.as_str()]
|
|
264
|
+
} else {
|
|
265
|
+
Vec::new()
|
|
266
|
+
};
|
|
267
|
+
save_restart_projected_state_with_capture_backfill_skip(
|
|
268
|
+
workspace,
|
|
269
|
+
&mut state,
|
|
270
|
+
&team_key,
|
|
271
|
+
&skip_capture_backfill,
|
|
272
|
+
&[agent_id.as_str()],
|
|
273
|
+
)?;
|
|
260
274
|
write_start_agent_start_event(
|
|
261
275
|
workspace,
|
|
262
276
|
agent_id,
|
|
@@ -559,25 +573,17 @@ pub(super) fn drain_old_pane_and_pid(
|
|
|
559
573
|
})
|
|
560
574
|
}
|
|
561
575
|
|
|
562
|
-
/// Best-effort liveness probe for a pid.
|
|
563
|
-
///
|
|
576
|
+
/// Best-effort liveness probe for a pid.
|
|
577
|
+
///
|
|
578
|
+
/// 0.5.x Windows portability Batch 3: routes through
|
|
579
|
+
/// `crate::platform::process::pid_is_alive`. Unix uses `kill(pid, 0)`
|
|
580
|
+
/// with the `EPERM = Live` branch preserved byte-for-byte; Windows
|
|
581
|
+
/// uses `OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION)` +
|
|
582
|
+
/// `GetExitCodeProcess` (STILL_ACTIVE = Live). The legacy non-Unix
|
|
583
|
+
/// `true` fallback (which broke drain by reporting every pid alive)
|
|
584
|
+
/// is gone.
|
|
564
585
|
pub(super) fn pid_is_alive(pid: u32) -> bool {
|
|
565
|
-
|
|
566
|
-
{
|
|
567
|
-
// SAFETY: kill(pid, 0) checks if pid exists without sending a signal.
|
|
568
|
-
let ret = unsafe { libc::kill(pid as i32, 0) };
|
|
569
|
-
if ret == 0 {
|
|
570
|
-
return true;
|
|
571
|
-
}
|
|
572
|
-
// EPERM also means the process exists (we just can't signal it).
|
|
573
|
-
let err = std::io::Error::last_os_error();
|
|
574
|
-
matches!(err.raw_os_error(), Some(libc::EPERM))
|
|
575
|
-
}
|
|
576
|
-
#[cfg(not(unix))]
|
|
577
|
-
{
|
|
578
|
-
let _ = pid;
|
|
579
|
-
true
|
|
580
|
-
}
|
|
586
|
+
crate::platform::process::pid_is_alive(pid)
|
|
581
587
|
}
|
|
582
588
|
|
|
583
589
|
/// Read state.agents[agent_id].pane_pid (u32) from the runtime state.
|
|
@@ -507,7 +507,10 @@ pub(crate) fn lifecycle_worker_tmux_backend_for_selected_state(
|
|
|
507
507
|
// are expected to migrate to `lifecycle_worker_transport_for_selected_state`
|
|
508
508
|
// during Batch 2/3.
|
|
509
509
|
if let Some(state_ref) = state.as_ref() {
|
|
510
|
-
if let Some(kind) = state_ref
|
|
510
|
+
if let Some(kind) = state_ref
|
|
511
|
+
.pointer("/transport/kind")
|
|
512
|
+
.and_then(|v| v.as_str())
|
|
513
|
+
{
|
|
511
514
|
if kind.eq_ignore_ascii_case("conpty") {
|
|
512
515
|
return Err(LifecycleError::TeamSelect(format!(
|
|
513
516
|
"backend_kind_mismatch: state.transport.kind={kind:?} but the legacy \
|
|
@@ -581,11 +584,28 @@ pub(super) fn save_restart_projected_state(
|
|
|
581
584
|
state: &mut serde_json::Value,
|
|
582
585
|
team_key: &str,
|
|
583
586
|
topology_authority_agent_ids: &[&str],
|
|
587
|
+
) -> Result<(), LifecycleError> {
|
|
588
|
+
save_restart_projected_state_with_capture_backfill_skip(
|
|
589
|
+
workspace,
|
|
590
|
+
state,
|
|
591
|
+
team_key,
|
|
592
|
+
&[],
|
|
593
|
+
topology_authority_agent_ids,
|
|
594
|
+
)
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
pub(super) fn save_restart_projected_state_with_capture_backfill_skip(
|
|
598
|
+
workspace: &Path,
|
|
599
|
+
state: &mut serde_json::Value,
|
|
600
|
+
team_key: &str,
|
|
601
|
+
skip_capture_backfill_agent_ids: &[&str],
|
|
602
|
+
topology_authority_agent_ids: &[&str],
|
|
584
603
|
) -> Result<(), LifecycleError> {
|
|
585
604
|
sync_restart_team_projections(state, team_key);
|
|
586
|
-
crate::state::projection::
|
|
605
|
+
crate::state::projection::save_team_scoped_state_with_lifecycle_topology_authority_and_capture_backfill_skip(
|
|
587
606
|
workspace,
|
|
588
607
|
state,
|
|
608
|
+
skip_capture_backfill_agent_ids,
|
|
589
609
|
topology_authority_agent_ids,
|
|
590
610
|
)
|
|
591
611
|
.map_err(|e| LifecycleError::StatePersist(e.to_string()))
|
|
@@ -739,6 +759,37 @@ pub(super) struct BackingProbeResult {
|
|
|
739
759
|
pub checked_paths: Vec<PathBuf>,
|
|
740
760
|
}
|
|
741
761
|
|
|
762
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
763
|
+
pub(crate) struct SessionIdentityProbeResult {
|
|
764
|
+
pub identity_ok: Option<bool>,
|
|
765
|
+
pub embedded_agent_id: Option<String>,
|
|
766
|
+
pub rollout_path: Option<PathBuf>,
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
pub(crate) fn session_identity_probe_for_agent(
|
|
770
|
+
agent_id: &AgentId,
|
|
771
|
+
_provider: Provider,
|
|
772
|
+
rollout_path: Option<&RolloutPath>,
|
|
773
|
+
) -> SessionIdentityProbeResult {
|
|
774
|
+
let Some(path) = rollout_path.map(RolloutPath::as_path) else {
|
|
775
|
+
return SessionIdentityProbeResult {
|
|
776
|
+
identity_ok: None,
|
|
777
|
+
embedded_agent_id: None,
|
|
778
|
+
rollout_path: None,
|
|
779
|
+
};
|
|
780
|
+
};
|
|
781
|
+
let embedded_agent_id =
|
|
782
|
+
crate::provider::session_scan::common::rollout_path_embedded_team_agent_worker_id(path);
|
|
783
|
+
let identity_ok = embedded_agent_id
|
|
784
|
+
.as_deref()
|
|
785
|
+
.map(|embedded| embedded == agent_id.as_str());
|
|
786
|
+
SessionIdentityProbeResult {
|
|
787
|
+
identity_ok,
|
|
788
|
+
embedded_agent_id,
|
|
789
|
+
rollout_path: Some(path.to_path_buf()),
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
|
|
742
793
|
pub(super) fn resume_backing_probe_for_agent(
|
|
743
794
|
workspace: &Path,
|
|
744
795
|
agent_id: &AgentId,
|
|
@@ -467,10 +467,21 @@ pub fn restart_with_transport_with_session_convergence_deadline(
|
|
|
467
467
|
.iter()
|
|
468
468
|
.map(|agent| agent.agent_id.as_str().to_string()),
|
|
469
469
|
);
|
|
470
|
-
|
|
470
|
+
let capture_backfill_skip_agent_ids = successful_agents
|
|
471
|
+
.iter()
|
|
472
|
+
.filter(|agent| {
|
|
473
|
+
matches!(
|
|
474
|
+
agent.restart_mode,
|
|
475
|
+
StartMode::Fresh | StartMode::FreshAfterMissingRollout
|
|
476
|
+
)
|
|
477
|
+
})
|
|
478
|
+
.map(|agent| agent.agent_id.as_str().to_string())
|
|
479
|
+
.collect::<Vec<_>>();
|
|
480
|
+
save_restart_state_with_lifecycle_topology_authority_and_capture_backfill_skip(
|
|
471
481
|
&selected.run_workspace,
|
|
472
482
|
&mut state,
|
|
473
483
|
&selected.team_key,
|
|
484
|
+
&capture_backfill_skip_agent_ids,
|
|
474
485
|
&topology_authority_agent_ids,
|
|
475
486
|
)?;
|
|
476
487
|
if fatal_resume_failure {
|
|
@@ -572,6 +583,11 @@ pub fn restart_with_transport_with_session_convergence_deadline(
|
|
|
572
583
|
&session_id,
|
|
573
584
|
agent_rollout_path(&agent).as_ref(),
|
|
574
585
|
);
|
|
586
|
+
let identity_probe = session_identity_probe_for_agent(
|
|
587
|
+
&decision.agent_id,
|
|
588
|
+
provider,
|
|
589
|
+
agent_rollout_path(&agent).as_ref(),
|
|
590
|
+
);
|
|
575
591
|
write_restart_resume_postflight_event(
|
|
576
592
|
&selected.run_workspace,
|
|
577
593
|
&decision.agent_id,
|
|
@@ -579,11 +595,26 @@ pub fn restart_with_transport_with_session_convergence_deadline(
|
|
|
579
595
|
probe.exists,
|
|
580
596
|
&probe.checked_paths,
|
|
581
597
|
/* recaptured = */ false,
|
|
598
|
+
identity_probe.identity_ok,
|
|
599
|
+
identity_probe.embedded_agent_id.as_deref(),
|
|
582
600
|
)?;
|
|
583
|
-
if probe.exists {
|
|
601
|
+
if probe.exists && identity_probe.identity_ok != Some(false) {
|
|
584
602
|
survivors.push(decision);
|
|
585
603
|
continue;
|
|
586
604
|
}
|
|
605
|
+
if identity_probe.identity_ok == Some(false) {
|
|
606
|
+
let phase = "resume_postflight";
|
|
607
|
+
let error = "session_identity_mismatch_after_restart".to_string();
|
|
608
|
+
mark_agent_restart_failed(&mut state, &decision, &error);
|
|
609
|
+
let _ = write_restart_agent_failed_event(
|
|
610
|
+
&selected.run_workspace,
|
|
611
|
+
&decision,
|
|
612
|
+
phase,
|
|
613
|
+
&error,
|
|
614
|
+
);
|
|
615
|
+
postflight_failed.push(restart_failed_agent(&decision, phase, error));
|
|
616
|
+
continue;
|
|
617
|
+
}
|
|
587
618
|
// 0.4.6 tuple-atomic contract: backing went missing between
|
|
588
619
|
// preflight and post-spawn. Clear the FULL authoritative tuple
|
|
589
620
|
// (including session_id) and persist the old provider id only
|
|
@@ -640,6 +671,11 @@ pub fn restart_with_transport_with_session_convergence_deadline(
|
|
|
640
671
|
&session_id,
|
|
641
672
|
agent_rollout_path(&agent_after).as_ref(),
|
|
642
673
|
);
|
|
674
|
+
let identity_probe_after = session_identity_probe_for_agent(
|
|
675
|
+
&decision.agent_id,
|
|
676
|
+
provider,
|
|
677
|
+
agent_rollout_path(&agent_after).as_ref(),
|
|
678
|
+
);
|
|
643
679
|
write_restart_resume_postflight_event(
|
|
644
680
|
&selected.run_workspace,
|
|
645
681
|
&decision.agent_id,
|
|
@@ -647,14 +683,20 @@ pub fn restart_with_transport_with_session_convergence_deadline(
|
|
|
647
683
|
probe_after.exists,
|
|
648
684
|
&probe_after.checked_paths,
|
|
649
685
|
/* recaptured = */ true,
|
|
686
|
+
identity_probe_after.identity_ok,
|
|
687
|
+
identity_probe_after.embedded_agent_id.as_deref(),
|
|
650
688
|
)?;
|
|
651
|
-
if probe_after.exists {
|
|
689
|
+
if probe_after.exists && identity_probe_after.identity_ok != Some(false) {
|
|
652
690
|
survivors.push(decision);
|
|
653
691
|
continue;
|
|
654
692
|
}
|
|
655
693
|
// Still missing. Demote the agent.
|
|
656
694
|
let phase = "resume_postflight";
|
|
657
|
-
let error =
|
|
695
|
+
let error = if identity_probe_after.identity_ok == Some(false) {
|
|
696
|
+
"session_identity_mismatch_after_restart".to_string()
|
|
697
|
+
} else {
|
|
698
|
+
"session_backing_store_missing_after_restart".to_string()
|
|
699
|
+
};
|
|
658
700
|
mark_agent_restart_failed(&mut state, &decision, &error);
|
|
659
701
|
let _ =
|
|
660
702
|
write_restart_agent_failed_event(&selected.run_workspace, &decision, phase, &error);
|
|
@@ -1327,8 +1369,38 @@ fn save_restart_state_with_lifecycle_topology_authority(
|
|
|
1327
1369
|
team_key: &str,
|
|
1328
1370
|
agent_ids: &[String],
|
|
1329
1371
|
) -> Result<(), LifecycleError> {
|
|
1330
|
-
|
|
1331
|
-
|
|
1372
|
+
save_restart_state_with_lifecycle_topology_authority_and_capture_backfill_skip(
|
|
1373
|
+
workspace,
|
|
1374
|
+
state,
|
|
1375
|
+
team_key,
|
|
1376
|
+
&[],
|
|
1377
|
+
agent_ids,
|
|
1378
|
+
)
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
fn save_restart_state_with_lifecycle_topology_authority_and_capture_backfill_skip(
|
|
1382
|
+
workspace: &Path,
|
|
1383
|
+
state: &mut serde_json::Value,
|
|
1384
|
+
team_key: &str,
|
|
1385
|
+
skip_capture_backfill_agent_ids: &[String],
|
|
1386
|
+
topology_agent_ids: &[String],
|
|
1387
|
+
) -> Result<(), LifecycleError> {
|
|
1388
|
+
let skip_capture_backfill_agent_ids = skip_capture_backfill_agent_ids
|
|
1389
|
+
.iter()
|
|
1390
|
+
.map(String::as_str)
|
|
1391
|
+
.collect::<Vec<_>>();
|
|
1392
|
+
let topology_agent_ids = topology_agent_ids
|
|
1393
|
+
.iter()
|
|
1394
|
+
.map(String::as_str)
|
|
1395
|
+
.collect::<Vec<_>>();
|
|
1396
|
+
sync_restart_team_projections(state, team_key);
|
|
1397
|
+
crate::state::projection::save_team_scoped_state_with_lifecycle_topology_authority_and_capture_backfill_skip(
|
|
1398
|
+
workspace,
|
|
1399
|
+
state,
|
|
1400
|
+
&skip_capture_backfill_agent_ids,
|
|
1401
|
+
&topology_agent_ids,
|
|
1402
|
+
)
|
|
1403
|
+
.map_err(|e| LifecycleError::StatePersist(e.to_string()))
|
|
1332
1404
|
}
|
|
1333
1405
|
|
|
1334
1406
|
fn save_restart_session_repairs(
|
|
@@ -1676,6 +1748,8 @@ fn write_restart_resume_postflight_event(
|
|
|
1676
1748
|
exists: bool,
|
|
1677
1749
|
checked_paths: &[std::path::PathBuf],
|
|
1678
1750
|
recaptured: bool,
|
|
1751
|
+
identity_ok: Option<bool>,
|
|
1752
|
+
embedded_agent_id: Option<&str>,
|
|
1679
1753
|
) -> Result<(), LifecycleError> {
|
|
1680
1754
|
crate::event_log::EventLog::new(workspace)
|
|
1681
1755
|
.write(
|
|
@@ -1689,6 +1763,8 @@ fn write_restart_resume_postflight_event(
|
|
|
1689
1763
|
.map(|p| p.to_string_lossy().into_owned())
|
|
1690
1764
|
.collect::<Vec<_>>(),
|
|
1691
1765
|
"recaptured": recaptured,
|
|
1766
|
+
"identity_ok": identity_ok,
|
|
1767
|
+
"embedded_agent_id": embedded_agent_id,
|
|
1692
1768
|
}),
|
|
1693
1769
|
)
|
|
1694
1770
|
.map(|_| ())
|
|
@@ -1878,6 +1954,32 @@ fn write_restart_resume_decision_event(
|
|
|
1878
1954
|
);
|
|
1879
1955
|
}
|
|
1880
1956
|
}
|
|
1957
|
+
if let crate::provider::session::ResumeRefusalReason::SessionIdentityMismatch {
|
|
1958
|
+
expected_agent_id,
|
|
1959
|
+
embedded_agent_id,
|
|
1960
|
+
session_id,
|
|
1961
|
+
rollout_path,
|
|
1962
|
+
} = structured
|
|
1963
|
+
{
|
|
1964
|
+
obj.insert(
|
|
1965
|
+
"expected_agent_id".to_string(),
|
|
1966
|
+
serde_json::json!(expected_agent_id),
|
|
1967
|
+
);
|
|
1968
|
+
obj.insert(
|
|
1969
|
+
"embedded_agent_id".to_string(),
|
|
1970
|
+
serde_json::json!(embedded_agent_id),
|
|
1971
|
+
);
|
|
1972
|
+
obj.insert(
|
|
1973
|
+
"poisoned_session_id".to_string(),
|
|
1974
|
+
serde_json::json!(session_id),
|
|
1975
|
+
);
|
|
1976
|
+
if let Some(path) = rollout_path {
|
|
1977
|
+
obj.insert(
|
|
1978
|
+
"rollout_path".to_string(),
|
|
1979
|
+
serde_json::json!(path.to_string_lossy()),
|
|
1980
|
+
);
|
|
1981
|
+
}
|
|
1982
|
+
}
|
|
1881
1983
|
}
|
|
1882
1984
|
}
|
|
1883
1985
|
}
|
|
@@ -67,7 +67,11 @@ pub fn classify_first_send_at(raw: &serde_json::Value) -> FirstSendAtState {
|
|
|
67
67
|
/// * lifecycle/restart/common.rs::restart_required_missing_session_agent_ids
|
|
68
68
|
pub(crate) fn restart_agent_has_context_to_preserve(agent: &serde_json::Value) -> bool {
|
|
69
69
|
let has_valid_first_send_at = matches!(
|
|
70
|
-
classify_first_send_at(
|
|
70
|
+
classify_first_send_at(
|
|
71
|
+
agent
|
|
72
|
+
.get("first_send_at")
|
|
73
|
+
.unwrap_or(&serde_json::Value::Null)
|
|
74
|
+
),
|
|
71
75
|
FirstSendAtState::Valid
|
|
72
76
|
);
|
|
73
77
|
if has_valid_first_send_at {
|
|
@@ -242,6 +246,7 @@ pub(crate) fn classify_restart_plan_with_resume_validation(
|
|
|
242
246
|
let provider = agent_provider(agent);
|
|
243
247
|
let provider_wire = provider_wire(provider);
|
|
244
248
|
let provider_can_resume = provider_supports_resume(provider);
|
|
249
|
+
let rollout_path = agent_rollout_path(agent);
|
|
245
250
|
// Layer 2 self-healing (leader follow-up 2026-06-22): use the
|
|
246
251
|
// structured probe so we can carry the list of paths the runtime
|
|
247
252
|
// actually checked into the refusal — operators need to see WHICH
|
|
@@ -256,16 +261,15 @@ pub(crate) fn classify_restart_plan_with_resume_validation(
|
|
|
256
261
|
agent,
|
|
257
262
|
provider,
|
|
258
263
|
session,
|
|
259
|
-
|
|
264
|
+
rollout_path.as_ref(),
|
|
260
265
|
);
|
|
261
266
|
(probe.exists, probe.checked_paths)
|
|
262
267
|
}
|
|
263
268
|
(None, Some(_), true) if resumable_provider_requires_backing(provider_wire) => {
|
|
264
|
-
let
|
|
265
|
-
let exists = path_opt
|
|
269
|
+
let exists = rollout_path
|
|
266
270
|
.as_ref()
|
|
267
271
|
.is_some_and(|path| path.as_path().exists());
|
|
268
|
-
let checked =
|
|
272
|
+
let checked = rollout_path
|
|
269
273
|
.as_ref()
|
|
270
274
|
.map(|p| vec![p.as_path().to_path_buf()])
|
|
271
275
|
.unwrap_or_default();
|
|
@@ -273,6 +277,12 @@ pub(crate) fn classify_restart_plan_with_resume_validation(
|
|
|
273
277
|
}
|
|
274
278
|
_ => (true, Vec::new()),
|
|
275
279
|
};
|
|
280
|
+
let identity_probe =
|
|
281
|
+
session_identity_probe_for_agent(&agent_id, provider, rollout_path.as_ref());
|
|
282
|
+
let session_identity_mismatch = session_id.is_some()
|
|
283
|
+
&& provider_can_resume
|
|
284
|
+
&& resume_backing_exists
|
|
285
|
+
&& identity_probe.identity_ok == Some(false);
|
|
276
286
|
// 0.4.7 partial-resume + RESTART-RESUME-001 (0.4.8): when a worker
|
|
277
287
|
// has NEVER been captured (no session_id AND no context-bearing
|
|
278
288
|
// signal at all), it is structurally non-resumable — there is no
|
|
@@ -292,7 +302,11 @@ pub(crate) fn classify_restart_plan_with_resume_validation(
|
|
|
292
302
|
let _ = first_send_at_state; // retained for the Corrupt branch above
|
|
293
303
|
let never_captured =
|
|
294
304
|
restart_agent_never_captured(agent, session_id.as_ref().map(|s| s.as_str()));
|
|
295
|
-
let decision = if session_id.is_some()
|
|
305
|
+
let decision = if session_id.is_some()
|
|
306
|
+
&& provider_can_resume
|
|
307
|
+
&& resume_backing_exists
|
|
308
|
+
&& !session_identity_mismatch
|
|
309
|
+
{
|
|
296
310
|
ResumeDecision::Resume
|
|
297
311
|
} else if session_id.is_some() && allow_fresh {
|
|
298
312
|
ResumeDecision::FreshStart
|
|
@@ -313,7 +327,24 @@ pub(crate) fn classify_restart_plan_with_resume_validation(
|
|
|
313
327
|
// (round-tripped through ResumeRefusalReason::wire) so the
|
|
314
328
|
// CLI/JSON contract does not change.
|
|
315
329
|
let (reason_str, structured) = if session_id.is_some() {
|
|
316
|
-
if
|
|
330
|
+
if session_identity_mismatch {
|
|
331
|
+
let session = session_id
|
|
332
|
+
.as_ref()
|
|
333
|
+
.map(|session| session.as_str().to_string())
|
|
334
|
+
.unwrap_or_default();
|
|
335
|
+
(
|
|
336
|
+
"session_identity_mismatch".to_string(),
|
|
337
|
+
crate::provider::session::ResumeRefusalReason::SessionIdentityMismatch {
|
|
338
|
+
expected_agent_id: agent_id.as_str().to_string(),
|
|
339
|
+
embedded_agent_id: identity_probe
|
|
340
|
+
.embedded_agent_id
|
|
341
|
+
.clone()
|
|
342
|
+
.unwrap_or_default(),
|
|
343
|
+
session_id: session,
|
|
344
|
+
rollout_path: identity_probe.rollout_path.clone(),
|
|
345
|
+
},
|
|
346
|
+
)
|
|
347
|
+
} else if !provider_can_resume {
|
|
317
348
|
(
|
|
318
349
|
"session_unresumable".to_string(),
|
|
319
350
|
crate::provider::session::ResumeRefusalReason::ProviderResumeUnsupported {
|
|
@@ -330,17 +361,15 @@ pub(crate) fn classify_restart_plan_with_resume_validation(
|
|
|
330
361
|
// a recovery hint pointing at the agent_id (used as
|
|
331
362
|
// launch-time `--name`) and spawn_cwd. Operator-facing
|
|
332
363
|
// diagnostic only — no auto-resume off the hint.
|
|
333
|
-
let recovery_hint = Some(
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
spawn_cwd
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
},
|
|
343
|
-
);
|
|
364
|
+
let recovery_hint = Some(crate::provider::session::RecoveryHint {
|
|
365
|
+
provider_session_name_hint: Some(agent_id.as_str().to_string()),
|
|
366
|
+
spawn_cwd: agent
|
|
367
|
+
.get("spawn_cwd")
|
|
368
|
+
.and_then(|v| v.as_str())
|
|
369
|
+
.filter(|s| !s.is_empty())
|
|
370
|
+
.map(std::path::PathBuf::from),
|
|
371
|
+
provider: provider_wire.to_string(),
|
|
372
|
+
});
|
|
344
373
|
(
|
|
345
374
|
"session_unresumable".to_string(),
|
|
346
375
|
crate::provider::session::ResumeRefusalReason::SessionBackingStoreMissing {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
//! lifecycle::restart —— 单 worker 起/停/重置/删 + 整队 Route B 重建 + plan halt/status。
|
|
2
2
|
|
|
3
|
-
use std::path::Path;
|
|
4
3
|
use std::collections::BTreeMap;
|
|
4
|
+
use std::path::Path;
|
|
5
5
|
|
|
6
6
|
use crate::model::enums::{AuthMode, Provider};
|
|
7
7
|
use crate::model::ids::AgentId;
|
|
@@ -32,22 +32,28 @@ mod remove;
|
|
|
32
32
|
mod selection;
|
|
33
33
|
mod team_state;
|
|
34
34
|
|
|
35
|
-
pub use agent::{reset_agent, reset_agent_with_transport, start_agent, start_agent_with_transport, stop_agent, stop_agent_with_transport};
|
|
36
35
|
pub(crate) use agent::start_agent_at_paths;
|
|
36
|
+
pub use agent::{
|
|
37
|
+
reset_agent, reset_agent_with_transport, start_agent, start_agent_with_transport, stop_agent,
|
|
38
|
+
stop_agent_with_transport,
|
|
39
|
+
};
|
|
37
40
|
pub(crate) use common::refresh_missing_provider_sessions;
|
|
38
41
|
pub(crate) use common::restart_required_missing_session_agent_ids;
|
|
42
|
+
pub(crate) use common::session_identity_probe_for_agent;
|
|
39
43
|
// 0.3.24 add-agent socket drift fix: state-aware tmux resolver shared with
|
|
40
44
|
// `lifecycle::launch::add_agent` / `fork_agent` so all three (restart / add / fork)
|
|
41
45
|
// route to the SAME tmux socket the live team uses.
|
|
42
46
|
pub(crate) use common::lifecycle_worker_tmux_backend_for_selected_state;
|
|
43
47
|
pub use orchestrator::{halt_plan, plan_status};
|
|
48
|
+
pub(crate) use rebuild::restart_with_transport_with_session_convergence_deadline;
|
|
44
49
|
pub use rebuild::{
|
|
45
50
|
restart, restart_candidates, restart_with_session_convergence_deadline, restart_with_transport,
|
|
46
51
|
restart_with_transport_with_readiness_deadline, select_restart_state,
|
|
47
52
|
};
|
|
48
|
-
pub(crate) use rebuild::restart_with_transport_with_session_convergence_deadline;
|
|
49
53
|
pub use remove::{remove_agent, remove_agent_with_transport};
|
|
50
|
-
pub use selection::{
|
|
54
|
+
pub use selection::{
|
|
55
|
+
classify_first_send_at, classify_restart_plan, decide_start_mode, python_type_name,
|
|
56
|
+
};
|
|
51
57
|
// Layer 2 (leader follow-up 2026-06-22): test-visible workspace-aware
|
|
52
58
|
// classification so lifecycle/tests/restart.rs can exercise the
|
|
53
59
|
// SessionBackingStoreMissing + checked_paths + RecoveryHint path
|
|
@@ -55,7 +61,9 @@ pub use selection::{classify_first_send_at, classify_restart_plan, decide_start_
|
|
|
55
61
|
pub(crate) use selection::classify_restart_plan_with_resume_validation;
|
|
56
62
|
pub(crate) use team_state::write_team_state;
|
|
57
63
|
|
|
58
|
-
pub(crate) fn lifecycle_run_workspace(
|
|
64
|
+
pub(crate) fn lifecycle_run_workspace(
|
|
65
|
+
workspace: &Path,
|
|
66
|
+
) -> Result<std::path::PathBuf, LifecycleError> {
|
|
59
67
|
crate::model::paths::canonical_run_workspace(workspace)
|
|
60
68
|
.map_err(|e| LifecycleError::StatePersist(e.to_string()))
|
|
61
69
|
}
|
|
@@ -93,7 +101,9 @@ fn lifecycle_paths(workspace: &Path, team: Option<&str>) -> Result<LifecyclePath
|
|
|
93
101
|
.spec_workspace
|
|
94
102
|
.clone()
|
|
95
103
|
.or_else(|| selected_state_spec_workspace(&selected.state))
|
|
96
|
-
.ok_or_else(||
|
|
104
|
+
.ok_or_else(|| {
|
|
105
|
+
LifecycleError::TeamSelect("active team spec workspace not found".to_string())
|
|
106
|
+
})?;
|
|
97
107
|
Ok(LifecyclePaths {
|
|
98
108
|
run_workspace: selected.run_workspace,
|
|
99
109
|
spec_workspace,
|
|
@@ -287,6 +287,7 @@ impl Drop for EnvVarGuard {
|
|
|
287
287
|
}
|
|
288
288
|
}
|
|
289
289
|
|
|
290
|
+
#[cfg(unix)]
|
|
290
291
|
struct TmuxShim {
|
|
291
292
|
log: std::path::PathBuf,
|
|
292
293
|
_path: EnvVarGuard,
|
|
@@ -297,6 +298,7 @@ struct TmuxShim {
|
|
|
297
298
|
_real_tmux: EnvVarGuard,
|
|
298
299
|
}
|
|
299
300
|
|
|
301
|
+
#[cfg(unix)]
|
|
300
302
|
fn install_e27_tmux_shim(expected_endpoint: &str, session_name: &str) -> TmuxShim {
|
|
301
303
|
use std::os::unix::fs::PermissionsExt;
|
|
302
304
|
|
|
@@ -441,6 +443,9 @@ fn assert_only_expected_socket_used(log: &std::path::Path, expected_endpoint: &s
|
|
|
441
443
|
}
|
|
442
444
|
}
|
|
443
445
|
|
|
446
|
+
// 0.5.x Windows portability Batch 5: E27 tests use a shell-script
|
|
447
|
+
// tmux shim + Unix socket paths. Unix-only.
|
|
448
|
+
#[cfg(unix)]
|
|
444
449
|
#[test]
|
|
445
450
|
#[serial_test::serial(env)]
|
|
446
451
|
fn e27_stop_agent_uses_attached_explicit_state_socket() {
|
|
@@ -461,6 +466,7 @@ fn e27_stop_agent_uses_attached_explicit_state_socket() {
|
|
|
461
466
|
);
|
|
462
467
|
}
|
|
463
468
|
|
|
469
|
+
#[cfg(unix)]
|
|
464
470
|
#[test]
|
|
465
471
|
#[serial_test::serial(env)]
|
|
466
472
|
fn e27_reset_agent_uses_attached_explicit_state_socket_for_stop_and_spawn() {
|
|
@@ -491,6 +497,7 @@ fn e27_reset_agent_uses_attached_explicit_state_socket_for_stop_and_spawn() {
|
|
|
491
497
|
);
|
|
492
498
|
}
|
|
493
499
|
|
|
500
|
+
#[cfg(unix)]
|
|
494
501
|
#[test]
|
|
495
502
|
#[serial_test::serial(env)]
|
|
496
503
|
fn e27_stop_agent_expands_short_state_socket_name() {
|