@team-agent/installer 0.5.61 → 0.5.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -1
- package/Cargo.toml +1 -1
- package/crates/team-agent/src/cli/adapters.rs +5 -2
- package/crates/team-agent/src/cli/attach_app_server_leader.rs +1 -0
- package/crates/team-agent/src/cli/diagnose.rs +1 -0
- package/crates/team-agent/src/cli/emit.rs +41 -7
- package/crates/team-agent/src/cli/helpers.rs +1 -0
- package/crates/team-agent/src/cli/leader.rs +6 -1
- package/crates/team-agent/src/cli/leaders.rs +2 -0
- package/crates/team-agent/src/cli/mod.rs +33 -3
- package/crates/team-agent/src/cli/named_address.rs +8 -1
- package/crates/team-agent/src/cli/profile.rs +2 -1
- package/crates/team-agent/src/cli/send/coordinator.rs +1 -0
- package/crates/team-agent/src/cli/send/mailbox.rs +3 -0
- package/crates/team-agent/src/cli/send/persist.rs +1 -0
- package/crates/team-agent/src/cli/send/presentation.rs +1 -0
- package/crates/team-agent/src/cli/send/resolve.rs +206 -1
- package/crates/team-agent/src/cli/send.rs +2 -1
- package/crates/team-agent/src/cli/spec.rs +1 -0
- package/crates/team-agent/src/cli/status.rs +2 -1
- package/crates/team-agent/src/cli/status_port/compact.rs +1 -0
- package/crates/team-agent/src/cli/status_port/format.rs +1 -0
- package/crates/team-agent/src/cli/status_port/inbox.rs +1 -0
- package/crates/team-agent/src/cli/tests/verb_profile.rs +38 -0
- package/crates/team-agent/src/cli/types.rs +3 -2
- package/crates/team-agent/src/communication_mode/mod.rs +1 -0
- package/crates/team-agent/src/compiler.rs +1 -0
- package/crates/team-agent/src/conpty/backend.rs +1 -0
- package/crates/team-agent/src/conpty/mod.rs +1 -0
- package/crates/team-agent/src/coordinator/backoff.rs +5 -8
- package/crates/team-agent/src/coordinator/conpty_shim.rs +3 -2
- package/crates/team-agent/src/coordinator/health.rs +1 -0
- package/crates/team-agent/src/coordinator/mod.rs +2 -2
- package/crates/team-agent/src/coordinator/orphan.rs +4 -116
- package/crates/team-agent/src/coordinator/runtime_detectors.rs +1 -0
- package/crates/team-agent/src/coordinator/runtime_observation.rs +1 -0
- package/crates/team-agent/src/coordinator/steps/abnormal.rs +6 -5
- package/crates/team-agent/src/coordinator/steps/delivery.rs +1 -0
- package/crates/team-agent/src/coordinator/steps/health_sync.rs +1 -0
- package/crates/team-agent/src/coordinator/steps/mod.rs +2 -0
- package/crates/team-agent/src/coordinator/steps/persist.rs +1 -0
- package/crates/team-agent/src/coordinator/steps/runtime_prompts.rs +1 -0
- package/crates/team-agent/src/coordinator/steps/session_gate.rs +1 -0
- package/crates/team-agent/src/coordinator/tests/abnormal.rs +0 -174
- package/crates/team-agent/src/coordinator/tests/api_error_recovery.rs +0 -9
- package/crates/team-agent/src/coordinator/tests/basics.rs +0 -184
- package/crates/team-agent/src/coordinator/tests/mod.rs +2 -17
- package/crates/team-agent/src/coordinator/tests/tick_core.rs +0 -42
- package/crates/team-agent/src/coordinator/tick.rs +24 -28
- package/crates/team-agent/src/coordinator/types.rs +3 -176
- package/crates/team-agent/src/db/agent_health_capture.rs +1 -0
- package/crates/team-agent/src/db/message_store.rs +2 -1
- package/crates/team-agent/src/db/migration.rs +5 -4
- package/crates/team-agent/src/db/mod.rs +1 -0
- package/crates/team-agent/src/db/schema.rs +1 -0
- package/crates/team-agent/src/diagnose/comms.rs +2 -1
- package/crates/team-agent/src/diagnose/mod.rs +1 -0
- package/crates/team-agent/src/diagnose/orphans.rs +1 -0
- package/crates/team-agent/src/layout/manager.rs +3 -7
- package/crates/team-agent/src/layout/mod.rs +2 -2
- package/crates/team-agent/src/layout/overlay.rs +6 -1
- package/crates/team-agent/src/layout/placement.rs +1 -0
- package/crates/team-agent/src/layout/recovery.rs +3 -0
- package/crates/team-agent/src/layout/runtime_sessions.rs +8 -3
- package/crates/team-agent/src/layout/sessions.rs +8 -1
- package/crates/team-agent/src/layout/tmux_endpoint.rs +3 -0
- package/crates/team-agent/src/layout/worker_env.rs +3 -0
- package/crates/team-agent/src/layout/worker_window_helpers.rs +2 -0
- package/crates/team-agent/src/leader/helpers.rs +1 -0
- package/crates/team-agent/src/leader/incident.rs +1 -0
- package/crates/team-agent/src/leader/lease.rs +27 -22
- package/crates/team-agent/src/leader/mod.rs +2 -3
- package/crates/team-agent/src/leader/owner_bind.rs +13 -3
- package/crates/team-agent/src/leader/provider_attribution.rs +1 -0
- package/crates/team-agent/src/leader/rediscover.rs +4 -17
- package/crates/team-agent/src/leader/registry.rs +3 -15
- package/crates/team-agent/src/leader/start.rs +373 -125
- package/crates/team-agent/src/leader/takeover.rs +10 -232
- package/crates/team-agent/src/leader/tests/claim_leader_mailbox_flush_contract.rs +544 -0
- package/crates/team-agent/src/leader/tests/historical_inventory_upgrade_gate_red.rs +624 -0
- package/crates/team-agent/src/leader/tests/identity.rs +7 -2
- package/crates/team-agent/src/leader/tests/idle.rs +0 -193
- package/crates/team-agent/src/leader/tests/lease_api.rs +0 -45
- package/crates/team-agent/src/leader/tests/mod.rs +4 -41
- package/crates/team-agent/src/leader/tests/rehomed_env.rs +83 -0
- package/crates/team-agent/src/leader/tests/team_in_team_state_scope_red.rs +1273 -0
- package/crates/team-agent/src/leader/tests/terminal_and_replay_vs_rerender_invariants_red.rs +417 -0
- package/crates/team-agent/src/leader/tests/wake_start_owner.rs +22 -103
- package/crates/team-agent/src/leader/types.rs +1 -28
- package/crates/team-agent/src/lib.rs +3 -0
- package/crates/team-agent/src/lifecycle/display.rs +7 -1
- package/crates/team-agent/src/lifecycle/helpers.rs +3 -1
- package/crates/team-agent/src/lifecycle/launch/add_agent.rs +2 -2
- package/crates/team-agent/src/lifecycle/launch/fork_agent/completion.rs +1 -0
- package/crates/team-agent/src/lifecycle/launch/fork_entry.rs +1 -0
- package/crates/team-agent/src/lifecycle/launch/fork_finalize.rs +1 -0
- package/crates/team-agent/src/lifecycle/launch/plan.rs +2 -2
- package/crates/team-agent/src/lifecycle/launch/quick_start.rs +4 -26
- package/crates/team-agent/src/lifecycle/launch/readiness.rs +0 -27
- package/crates/team-agent/src/lifecycle/launch/spawn.rs +1 -27
- package/crates/team-agent/src/lifecycle/launch/spec_state.rs +4 -32
- package/crates/team-agent/src/lifecycle/launch/state_projection.rs +1 -0
- package/crates/team-agent/src/lifecycle/launch.rs +11 -12
- package/crates/team-agent/src/lifecycle/mod.rs +2 -1
- package/crates/team-agent/src/lifecycle/restart/agent.rs +3 -3
- package/crates/team-agent/src/lifecycle/restart/orchestrator.rs +2 -2
- package/crates/team-agent/src/lifecycle/restart/rebuild.rs +13 -5
- package/crates/team-agent/src/lifecycle/restart/remove.rs +1 -1
- package/crates/team-agent/src/lifecycle/restart/selection.rs +3 -3
- package/crates/team-agent/src/lifecycle/restart.rs +13 -11
- package/crates/team-agent/src/lifecycle/tests/acceptance_b_batch_red.rs +1007 -0
- package/crates/team-agent/src/lifecycle/tests/b0_legacy_snapshot_nonauthority_contract.rs +265 -0
- package/crates/team-agent/src/lifecycle/tests/b0_reader_hideone_audit_contract.rs +459 -0
- package/crates/team-agent/src/lifecycle/tests/behavioral_diff_264_red.rs +849 -0
- package/crates/team-agent/src/lifecycle/tests/claude_compatible_config_red.rs +323 -0
- package/crates/team-agent/src/lifecycle/tests/claude_profile_launch_red.rs +423 -0
- package/crates/team-agent/src/lifecycle/tests/clone_fork_copilot_perms_red.rs +737 -0
- package/crates/team-agent/src/lifecycle/tests/codex_mcp_approval_inheritance_red.rs +1187 -0
- package/crates/team-agent/src/lifecycle/tests/codex_weak_window_attribution_red.rs +683 -0
- package/crates/team-agent/src/lifecycle/tests/communication_mode_runtime_contract_red.rs +394 -0
- package/crates/team-agent/src/lifecycle/tests/copilot_provider_red.rs +1856 -0
- package/crates/team-agent/src/lifecycle/tests/core_034_real_red.rs +917 -0
- package/crates/team-agent/src/lifecycle/tests/display_adaptive_red.rs +481 -0
- package/crates/team-agent/src/lifecycle/tests/f032_startup_prompt_best_effort_red.rs +301 -0
- package/crates/team-agent/src/lifecycle/tests/harvest2_a_batch_red.rs +516 -0
- package/crates/team-agent/src/lifecycle/tests/host_cotenant_death_p0_contract.rs +986 -0
- package/crates/team-agent/src/lifecycle/tests/lifecycle_rollback_red.rs +651 -0
- package/crates/team-agent/src/lifecycle/tests/main_preserved.rs +2 -0
- package/crates/team-agent/src/lifecycle/tests/quick_start_worker_readiness_red.rs +332 -0
- package/crates/team-agent/src/lifecycle/tests/realmachine_clusters_1_6_red.rs +846 -0
- package/crates/team-agent/src/lifecycle/tests/realmachine_residual_g1_g4_red.rs +648 -0
- package/crates/team-agent/src/lifecycle/tests/restart_build_before_destroy_0540_contract.rs +910 -0
- package/crates/team-agent/src/lifecycle/tests/restart_liveness_red.rs +630 -0
- package/crates/team-agent/src/lifecycle/tests/restart_rebind_hotfix_252_red.rs +1083 -0
- package/crates/team-agent/src/lifecycle/tests/restart_session_capture_red.rs +1677 -0
- package/crates/team-agent/src/lifecycle/tests/resume_recover_red.rs +410 -0
- package/crates/team-agent/src/lifecycle/tests/stale_team_saveconflict_contract.rs +428 -0
- package/crates/team-agent/src/lifecycle/tests/startup_latency_contract.rs +1032 -0
- package/crates/team-agent/src/lifecycle/tests/status_credential_redaction_contract.rs +869 -0
- package/crates/team-agent/src/lifecycle/tests/subprep_codex_trust_roundtrip_red.rs +1249 -0
- package/crates/team-agent/src/lifecycle/tests/swallow_batch4_semantics_red.rs +342 -0
- package/crates/team-agent/src/lifecycle/tests/team_in_team_identity_scope_red.rs +542 -0
- package/crates/team-agent/src/lifecycle/tests/team_in_team_sibling_quick_start_red.rs +1103 -0
- package/crates/team-agent/src/lifecycle/tests/team_in_team_state_scope_red.rs +1276 -0
- package/crates/team-agent/src/lifecycle/tests/team_key_retirement_roster_red.rs +145 -0
- package/crates/team-agent/src/lifecycle/tests/team_key_retirement_txn_red.rs +603 -0
- package/crates/team-agent/src/lifecycle/tests/test_isolation_escape_contract.rs +837 -0
- package/crates/team-agent/src/lifecycle/tests/upgrade_compat_0211_red.rs +928 -0
- package/crates/team-agent/src/lifecycle/tests/verify_rs031_window_consistency_red.rs +951 -0
- package/crates/team-agent/src/lifecycle/tests/worker_dangerous_bypass_argv_red.rs +327 -0
- package/crates/team-agent/src/lifecycle/tests/worker_spawn_env_red.rs +760 -0
- package/crates/team-agent/src/lifecycle/tests.rs +59 -0
- package/crates/team-agent/src/lifecycle/types.rs +0 -9
- package/crates/team-agent/src/mcp_server/helpers.rs +1 -0
- package/crates/team-agent/src/mcp_server/lifecycle_tools/mod.rs +1 -0
- package/crates/team-agent/src/mcp_server/lifecycle_tools/state_status.rs +1 -0
- package/crates/team-agent/src/mcp_server/mod.rs +1 -0
- package/crates/team-agent/src/mcp_server/normalize.rs +4 -3
- package/crates/team-agent/src/mcp_server/tools.rs +1 -0
- package/crates/team-agent/src/mcp_server/types.rs +1 -18
- package/crates/team-agent/src/mcp_server/wire.rs +5 -5
- package/crates/team-agent/src/messaging/activity.rs +1 -0
- package/crates/team-agent/src/messaging/address.rs +1 -0
- package/crates/team-agent/src/messaging/delivery.rs +4 -3
- package/crates/team-agent/src/messaging/helpers.rs +2 -1
- package/crates/team-agent/src/messaging/leader_channel.rs +1 -0
- package/crates/team-agent/src/messaging/leader_receiver.rs +2 -1
- package/crates/team-agent/src/messaging/mod.rs +10 -14
- package/crates/team-agent/src/messaging/peers.rs +2 -0
- package/crates/team-agent/src/messaging/persist.rs +3 -1
- package/crates/team-agent/src/messaging/presentation.rs +2 -1
- package/crates/team-agent/src/messaging/results.rs +1 -0
- package/crates/team-agent/src/messaging/scheduler.rs +1 -0
- package/crates/team-agent/src/messaging/selftest.rs +1 -1
- package/crates/team-agent/src/messaging/send.rs +41 -8
- package/crates/team-agent/src/messaging/tests/mod.rs +5 -0
- package/crates/team-agent/src/messaging/types.rs +2 -2
- package/crates/team-agent/src/messaging/watchers.rs +19 -13
- package/crates/team-agent/src/model/enums.rs +1 -0
- package/crates/team-agent/src/model/errors.rs +1 -0
- package/crates/team-agent/src/model/ids.rs +1 -0
- package/crates/team-agent/src/model/mod.rs +1 -0
- package/crates/team-agent/src/model/name_similarity.rs +1 -0
- package/crates/team-agent/src/model/pane_authority_refusal.rs +1 -0
- package/crates/team-agent/src/model/paths.rs +1 -0
- package/crates/team-agent/src/model/permissions.rs +3 -2
- package/crates/team-agent/src/model/routing.rs +1 -0
- package/crates/team-agent/src/model/spec.rs +1 -0
- package/crates/team-agent/src/model/task_graph.rs +1 -0
- package/crates/team-agent/src/model/yaml/tests.rs +1 -0
- package/crates/team-agent/src/model/yaml.rs +1 -0
- package/crates/team-agent/src/packaging/install.rs +9 -147
- package/crates/team-agent/src/packaging/migrate.rs +2 -0
- package/crates/team-agent/src/packaging/mod.rs +1 -0
- package/crates/team-agent/src/packaging/repair.rs +2 -0
- package/crates/team-agent/src/packaging/tests.rs +29 -218
- package/crates/team-agent/src/packaging/types.rs +8 -15
- package/crates/team-agent/src/platform/argv.rs +4 -0
- package/crates/team-agent/src/platform/errors.rs +10 -1
- package/crates/team-agent/src/platform/file_lock.rs +4 -157
- package/crates/team-agent/src/platform/mod.rs +0 -61
- package/crates/team-agent/src/platform/process.rs +1 -0
- package/crates/team-agent/src/provider/adapter.rs +1 -0
- package/crates/team-agent/src/provider/adapters/claude.rs +1 -0
- package/crates/team-agent/src/provider/adapters/claude_fork.rs +1 -0
- package/crates/team-agent/src/provider/adapters/codex.rs +1 -0
- package/crates/team-agent/src/provider/adapters/copilot.rs +1 -0
- package/crates/team-agent/src/provider/adapters/copilot_fork.rs +1 -0
- package/crates/team-agent/src/provider/adapters/fake.rs +1 -0
- package/crates/team-agent/src/provider/adapters/mod.rs +1 -0
- package/crates/team-agent/src/provider/approvals/mod.rs +1 -0
- package/crates/team-agent/src/provider/approvals/parsing.rs +2 -5
- package/crates/team-agent/src/provider/approvals/runtime_prompts.rs +6 -5
- package/crates/team-agent/src/provider/classify.rs +1 -0
- package/crates/team-agent/src/provider/faults.rs +1 -26
- package/crates/team-agent/src/provider/helpers.rs +1 -0
- package/crates/team-agent/src/provider/mod.rs +1 -1
- package/crates/team-agent/src/provider/session/capture.rs +1 -4
- package/crates/team-agent/src/provider/session/context_fork/claude.rs +1 -0
- package/crates/team-agent/src/provider/session/context_fork/codex.rs +1 -0
- package/crates/team-agent/src/provider/session/context_fork/outcome.rs +1 -0
- package/crates/team-agent/src/provider/session/context_fork.rs +1 -0
- package/crates/team-agent/src/provider/session/mod.rs +2 -4
- package/crates/team-agent/src/provider/session/resume.rs +2 -209
- package/crates/team-agent/src/provider/session_scan/claude.rs +1 -0
- package/crates/team-agent/src/provider/session_scan/codex.rs +1 -0
- package/crates/team-agent/src/provider/session_scan/common.rs +1 -0
- package/crates/team-agent/src/provider/session_scan/copilot.rs +1 -0
- package/crates/team-agent/src/provider/session_scan.rs +1 -42
- package/crates/team-agent/src/provider/startup_prompt.rs +1 -0
- package/crates/team-agent/src/provider/types.rs +1 -0
- package/crates/team-agent/src/provider/wire.rs +1 -0
- package/crates/team-agent/src/state/identity.rs +16 -5
- package/crates/team-agent/src/state/identity_keys.rs +1 -0
- package/crates/team-agent/src/state/mod.rs +2 -0
- package/crates/team-agent/src/state/owner_gate.rs +5 -0
- package/crates/team-agent/src/state/ownership.rs +10 -4
- package/crates/team-agent/src/state/paths.rs +12 -0
- package/crates/team-agent/src/state/persist.rs +9 -3
- package/crates/team-agent/src/state/projection.rs +15 -2
- package/crates/team-agent/src/state/repository/intent.rs +1 -0
- package/crates/team-agent/src/state/repository/tests.rs +1 -0
- package/crates/team-agent/src/state/repository.rs +7 -0
- package/crates/team-agent/src/state/selector.rs +1 -0
- package/crates/team-agent/src/tmux_backend/tests.rs +39 -0
- package/crates/team-agent/src/tmux_backend.rs +63 -0
- package/crates/team-agent/src/transport.rs +15 -0
- package/package.json +4 -4
- package/crates/team-agent/src/leader/inject.rs +0 -33
- package/crates/team-agent/src/provider/command.rs +0 -80
|
@@ -0,0 +1,1677 @@
|
|
|
1
|
+
//! Restart must preserve provider session context before destructive teardown.
|
|
2
|
+
//!
|
|
3
|
+
//! #240 root cause: `restart`/`shutdown` decide or kill from runtime state before
|
|
4
|
+
//! force-refreshing missing provider `session_id`s. A real provider session can exist under the
|
|
5
|
+
//! provider home while `state.agents[*].session_id` is still null; destructive lifecycle must capture
|
|
6
|
+
//! it first and resume from that durable id.
|
|
7
|
+
|
|
8
|
+
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
|
9
|
+
|
|
10
|
+
#[path = "../../../tests/support/hermetic.rs"]
|
|
11
|
+
mod hermetic_guard;
|
|
12
|
+
#[allow(dead_code)]
|
|
13
|
+
fn _hermetic_boundary_marker(_: &hermetic_guard::HermeticTestEnv) {}
|
|
14
|
+
|
|
15
|
+
#[path = "../../../tests/support/composite_source.rs"]
|
|
16
|
+
mod composite_source;
|
|
17
|
+
use std::collections::BTreeMap;
|
|
18
|
+
use std::path::{Path, PathBuf};
|
|
19
|
+
use std::sync::atomic::{AtomicU64, Ordering};
|
|
20
|
+
use std::sync::Mutex;
|
|
21
|
+
|
|
22
|
+
use serde_json::{json, Value};
|
|
23
|
+
use serial_test::serial;
|
|
24
|
+
use team_agent::cli::{cmd_wait_ready, CmdOutput, WaitReadyArgs};
|
|
25
|
+
use team_agent::lifecycle::{
|
|
26
|
+
classify_restart_plan, restart_with_transport, RestartReport, ResumeDecision, StartMode,
|
|
27
|
+
};
|
|
28
|
+
use team_agent::model::enums::Provider;
|
|
29
|
+
use team_agent::provider::get_adapter;
|
|
30
|
+
use team_agent::state::persist::{load_runtime_state, save_runtime_state};
|
|
31
|
+
use team_agent::transport::{
|
|
32
|
+
AttachOutcome, BackendKind, CaptureRange, CapturedText, InjectPayload, InjectReport,
|
|
33
|
+
InjectStage, InjectVerification, Key, PaneField, PaneId, PaneInfo, SessionName, SetEnvOutcome,
|
|
34
|
+
SpawnResult, SubmitVerification, Target, Transport, TransportError, TurnVerification,
|
|
35
|
+
WindowName,
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
#[test]
|
|
39
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
40
|
+
#[serial(env)]
|
|
41
|
+
fn provider_home_codex_rollout_session_meta_is_captured_for_restart_resume() {
|
|
42
|
+
let fixture = RestartFixture::new("codex-home-capture", "codex");
|
|
43
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
44
|
+
let codex_session = "codex-sess-240";
|
|
45
|
+
seed_codex_home_rollout(&fixture.home(), &fixture.spawn_cwd, codex_session);
|
|
46
|
+
seed_running_state_without_session(&fixture.team, "codex", &fixture.spawn_cwd);
|
|
47
|
+
|
|
48
|
+
let adapter = get_adapter(Provider::Codex);
|
|
49
|
+
let captured = adapter
|
|
50
|
+
.capture_session_id("worker_a", &fixture.spawn_cwd, 0)
|
|
51
|
+
.expect("Codex provider-home capture should not error")
|
|
52
|
+
.expect("Codex rollout under HOME/.codex/sessions must be discoverable before restart");
|
|
53
|
+
|
|
54
|
+
assert_eq!(
|
|
55
|
+
captured.session_id.as_ref().map(|id| id.as_str()),
|
|
56
|
+
Some(codex_session),
|
|
57
|
+
"Codex capture must extract session_meta.payload.id from provider-home rollout JSONL"
|
|
58
|
+
);
|
|
59
|
+
assert!(
|
|
60
|
+
captured.rollout_path.as_ref().is_some_and(|path| path
|
|
61
|
+
.as_path()
|
|
62
|
+
.starts_with(fixture.home().join(".codex/sessions"))),
|
|
63
|
+
"Codex capture must scan provider home, not only spawn_cwd; captured={captured:?}"
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
#[test]
|
|
68
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
69
|
+
#[serial(env)]
|
|
70
|
+
fn provider_home_codex_rollout_matches_workspace_spawn_cwd_to_team_subdir_across_symlink() {
|
|
71
|
+
let fixture = RestartFixture::new_symlinked_workspace("codex-workspace-team-cwd", "codex");
|
|
72
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
73
|
+
let codex_session = "codex-sess-workspace-team-subdir";
|
|
74
|
+
seed_codex_home_rollout_with_cwd(&fixture.home(), &fixture.team, codex_session);
|
|
75
|
+
seed_running_state_without_session(&fixture.team, "codex", &fixture.spawn_cwd);
|
|
76
|
+
|
|
77
|
+
let adapter = get_adapter(Provider::Codex);
|
|
78
|
+
let captured = adapter
|
|
79
|
+
.capture_session_id("worker_a", &fixture.spawn_cwd, 0)
|
|
80
|
+
.expect("Codex provider-home capture should not error")
|
|
81
|
+
.expect(
|
|
82
|
+
"Codex rollout cwd at the team subdir must match a worker spawn_cwd at the parent workspace, even through symlink-equivalent paths",
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
assert_eq!(
|
|
86
|
+
captured.session_id.as_ref().map(|id| id.as_str()),
|
|
87
|
+
Some(codex_session),
|
|
88
|
+
"Codex capture must accept workspace↔team-subdir cwd granularity when the team dir belongs to this worker workspace"
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
#[test]
|
|
93
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
94
|
+
#[serial(env)]
|
|
95
|
+
fn codex_session_capture_assigns_complete_unique_rollouts_to_each_live_same_team_agent() {
|
|
96
|
+
let fixture = RestartFixture::new("codex-session-complete-unique-per-agent", "codex");
|
|
97
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
98
|
+
seed_codex_home_rollout_with_cwd(&fixture.home(), &fixture.spawn_cwd, "codex-sess-alpha");
|
|
99
|
+
seed_codex_home_rollout_with_cwd(&fixture.home(), &fixture.spawn_cwd, "codex-sess-beta");
|
|
100
|
+
seed_two_running_codex_agents_without_sessions(&fixture.team, &fixture.spawn_cwd);
|
|
101
|
+
let transport = RecordingTransport::new().with_session_present(true);
|
|
102
|
+
|
|
103
|
+
let report = restart_with_transport(&fixture.team, false, None, &transport)
|
|
104
|
+
.expect("restart should refresh missing provider sessions for both live agents");
|
|
105
|
+
|
|
106
|
+
let state = load_runtime_state(&fixture.team).unwrap();
|
|
107
|
+
assert_all_live_codex_agents_have_complete_unique_sessions(&state, ["worker_a", "worker_b"]);
|
|
108
|
+
assert_restart_resumed_all_agents(&report, ["worker_a", "worker_b"]);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
#[test]
|
|
112
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
113
|
+
#[serial(env)]
|
|
114
|
+
fn session_attribution_prefers_positive_pid_env_over_path_order() {
|
|
115
|
+
let fixture = RestartFixture::new("codex-session-positive-source-order", "codex");
|
|
116
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
117
|
+
seed_codex_home_rollout_with_agent_hint(
|
|
118
|
+
&fixture.home(),
|
|
119
|
+
&fixture.spawn_cwd,
|
|
120
|
+
"000-path-first-worker-b.jsonl",
|
|
121
|
+
"codex-sess-worker-b",
|
|
122
|
+
"worker_b",
|
|
123
|
+
41002,
|
|
124
|
+
);
|
|
125
|
+
seed_codex_home_rollout_with_agent_hint(
|
|
126
|
+
&fixture.home(),
|
|
127
|
+
&fixture.spawn_cwd,
|
|
128
|
+
"999-path-last-worker-a.jsonl",
|
|
129
|
+
"codex-sess-worker-a",
|
|
130
|
+
"worker_a",
|
|
131
|
+
41001,
|
|
132
|
+
);
|
|
133
|
+
seed_two_running_codex_agents_without_sessions(&fixture.team, &fixture.spawn_cwd);
|
|
134
|
+
let transport = RecordingTransport::new().with_session_present(true);
|
|
135
|
+
|
|
136
|
+
restart_with_transport(&fixture.team, false, None, &transport)
|
|
137
|
+
.expect("restart should run the real session refresh path");
|
|
138
|
+
|
|
139
|
+
let state = load_runtime_state(&fixture.team).unwrap();
|
|
140
|
+
assert_eq!(
|
|
141
|
+
agent_session(&state, "worker_a"),
|
|
142
|
+
Some("codex-sess-worker-a"),
|
|
143
|
+
"MUST-11: positive pane_pid/process-env attribution must beat rollout path ordering for worker_a; state={state}"
|
|
144
|
+
);
|
|
145
|
+
assert_eq!(
|
|
146
|
+
agent_session(&state, "worker_b"),
|
|
147
|
+
Some("codex-sess-worker-b"),
|
|
148
|
+
"MUST-11: positive pane_pid/process-env attribution must beat rollout path ordering for worker_b; state={state}"
|
|
149
|
+
);
|
|
150
|
+
assert_all_live_codex_agents_have_complete_unique_sessions(&state, ["worker_a", "worker_b"]);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
#[test]
|
|
154
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
155
|
+
#[serial(env)]
|
|
156
|
+
fn session_attribution_leaves_ambiguous_when_no_unique_match() {
|
|
157
|
+
let fixture = RestartFixture::new("codex-session-ambiguous-honest", "codex");
|
|
158
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
159
|
+
seed_codex_home_rollout_with_cwd(&fixture.home(), &fixture.spawn_cwd, "codex-sess-only-one");
|
|
160
|
+
seed_two_running_codex_agents_without_sessions(&fixture.team, &fixture.spawn_cwd);
|
|
161
|
+
let transport = RecordingTransport::new().with_session_present(true);
|
|
162
|
+
|
|
163
|
+
restart_with_transport(&fixture.team, false, None, &transport)
|
|
164
|
+
.expect("restart should run the real session refresh path");
|
|
165
|
+
|
|
166
|
+
let state = load_runtime_state(&fixture.team).unwrap();
|
|
167
|
+
let nonempty = ["worker_a", "worker_b"]
|
|
168
|
+
.into_iter()
|
|
169
|
+
.filter(|agent_id| agent_session(&state, agent_id).is_some())
|
|
170
|
+
.count();
|
|
171
|
+
assert_eq!(
|
|
172
|
+
nonempty, 1,
|
|
173
|
+
"with only one matching rollout, exactly one live agent may be captured; state={state}"
|
|
174
|
+
);
|
|
175
|
+
let ambiguous_agent = ["worker_a", "worker_b"]
|
|
176
|
+
.into_iter()
|
|
177
|
+
.find(|agent_id| agent_session(&state, agent_id).is_none())
|
|
178
|
+
.expect("one agent should remain uncaptured");
|
|
179
|
+
let agent = state
|
|
180
|
+
.get("agents")
|
|
181
|
+
.and_then(|agents| agents.get(ambiguous_agent))
|
|
182
|
+
.unwrap();
|
|
183
|
+
assert_eq!(
|
|
184
|
+
agent.get("attribution_ambiguous").and_then(serde_json::Value::as_bool),
|
|
185
|
+
Some(true),
|
|
186
|
+
"ambiguous session attribution must be explicit and null, not silently duplicated or path-order guessed; agent={agent} state={state}"
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
#[test]
|
|
191
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
192
|
+
#[serial(env)]
|
|
193
|
+
fn restart_waits_for_delayed_rollout_before_writing_resume_decisions() {
|
|
194
|
+
let fixture = RestartFixture::new("codex-delayed-rollout-converges", "codex");
|
|
195
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
196
|
+
seed_codex_home_rollout_with_agent_hint(
|
|
197
|
+
&fixture.home(),
|
|
198
|
+
&fixture.spawn_cwd,
|
|
199
|
+
"rollout-worker-a-visible-first.jsonl",
|
|
200
|
+
"codex-sess-delayed-a",
|
|
201
|
+
"worker_a",
|
|
202
|
+
41001,
|
|
203
|
+
);
|
|
204
|
+
seed_two_interacted_running_codex_agents_without_sessions(&fixture.team, &fixture.spawn_cwd);
|
|
205
|
+
let delayed_home = fixture.home();
|
|
206
|
+
let delayed_cwd = fixture.spawn_cwd.clone();
|
|
207
|
+
let delayed = std::thread::spawn(move || {
|
|
208
|
+
std::thread::sleep(std::time::Duration::from_millis(750));
|
|
209
|
+
seed_codex_home_rollout_with_agent_hint(
|
|
210
|
+
&delayed_home,
|
|
211
|
+
&delayed_cwd,
|
|
212
|
+
"rollout-worker-b-arrives-after-first-poll.jsonl",
|
|
213
|
+
"codex-sess-delayed-b",
|
|
214
|
+
"worker_b",
|
|
215
|
+
41002,
|
|
216
|
+
);
|
|
217
|
+
});
|
|
218
|
+
let transport = RecordingTransport::new().with_session_present(true);
|
|
219
|
+
|
|
220
|
+
let report = restart_with_transport(&fixture.team, false, None, &transport);
|
|
221
|
+
delayed
|
|
222
|
+
.join()
|
|
223
|
+
.expect("delayed rollout writer should finish");
|
|
224
|
+
let report = report
|
|
225
|
+
.expect("restart should wait for delayed rollout convergence instead of refusing early");
|
|
226
|
+
let state = load_runtime_state(&fixture.team).unwrap();
|
|
227
|
+
let events = read_events(&fixture.team);
|
|
228
|
+
|
|
229
|
+
assert_no_agent_attribution_ambiguous(&state, ["worker_a", "worker_b"]);
|
|
230
|
+
assert_events_have_converging_progress(&events, ["worker_b"]);
|
|
231
|
+
assert_all_live_codex_agents_have_complete_unique_sessions(&state, ["worker_a", "worker_b"]);
|
|
232
|
+
assert_restart_resumed_all_agents(&report, ["worker_a", "worker_b"]);
|
|
233
|
+
assert_all_resume_decisions_are_session_backed(&events, ["worker_a", "worker_b"]);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
#[test]
|
|
237
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
238
|
+
#[serial(env)]
|
|
239
|
+
fn restart_refuses_resume_not_ready_without_teardown_when_delayed_rollout_never_arrives() {
|
|
240
|
+
let fixture = RestartFixture::new("codex-delayed-rollout-never-arrives", "codex");
|
|
241
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
242
|
+
seed_codex_home_rollout_with_agent_hint(
|
|
243
|
+
&fixture.home(),
|
|
244
|
+
&fixture.spawn_cwd,
|
|
245
|
+
"rollout-worker-a-only.jsonl",
|
|
246
|
+
"codex-sess-only-a",
|
|
247
|
+
"worker_a",
|
|
248
|
+
41001,
|
|
249
|
+
);
|
|
250
|
+
seed_two_interacted_running_codex_agents_without_sessions(&fixture.team, &fixture.spawn_cwd);
|
|
251
|
+
let state_before = load_runtime_state(&fixture.team).unwrap();
|
|
252
|
+
let transport = RecordingTransport::new().with_session_present(true);
|
|
253
|
+
|
|
254
|
+
let report = restart_with_transport(&fixture.team, false, None, &transport)
|
|
255
|
+
.expect("restart should return a structured refusal, not throw, when session convergence misses its deadline");
|
|
256
|
+
|
|
257
|
+
match report {
|
|
258
|
+
RestartReport::RefusedResumeAtomicity { error, .. } => {
|
|
259
|
+
assert!(
|
|
260
|
+
error.contains("resume_not_ready") || error.contains("session convergence"),
|
|
261
|
+
"missing delayed rollout should be reported as resume_not_ready/session convergence, not ordinary resume atomicity; error={error}"
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
other => panic!(
|
|
265
|
+
"without --allow-fresh, missing delayed rollout must refuse before teardown, not restart/fresh-start; got {other:?}"
|
|
266
|
+
),
|
|
267
|
+
}
|
|
268
|
+
assert_eq!(
|
|
269
|
+
transport.kill_count(),
|
|
270
|
+
0,
|
|
271
|
+
"resume_not_ready refusal must not teardown the existing session"
|
|
272
|
+
);
|
|
273
|
+
assert_eq!(
|
|
274
|
+
transport.spawn_count(),
|
|
275
|
+
0,
|
|
276
|
+
"resume_not_ready refusal must not spawn fresh workers"
|
|
277
|
+
);
|
|
278
|
+
let state_after = load_runtime_state(&fixture.team).unwrap();
|
|
279
|
+
assert_eq!(
|
|
280
|
+
state_after, state_before,
|
|
281
|
+
"resume_not_ready must be atomic: deadline refusal leaves session_id/rollout_path/attribution fields unchanged; before={state_before} after={state_after}"
|
|
282
|
+
);
|
|
283
|
+
let events = read_events(&fixture.team);
|
|
284
|
+
assert!(
|
|
285
|
+
!events.contains("\"event\":\"restart.resume_decision\"")
|
|
286
|
+
&& !events.contains("\"event\": \"restart.resume_decision\""),
|
|
287
|
+
"restart.resume_decision must not be written before all running provider sessions converge; events={events}"
|
|
288
|
+
);
|
|
289
|
+
assert!(
|
|
290
|
+
!events.contains("degraded") && !events.contains(r#""decision":"fresh_start""#),
|
|
291
|
+
"without --allow-fresh, restart must hard-refuse resume_not_ready; it must not choose silent/degraded fresh. events={events}"
|
|
292
|
+
);
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
#[test]
|
|
296
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
297
|
+
#[serial(env)]
|
|
298
|
+
fn restart_with_allow_fresh_audit_marks_forced_fresh_true() {
|
|
299
|
+
let fixture = RestartFixture::new("codex-delayed-rollout-allow-fresh", "codex");
|
|
300
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
301
|
+
seed_codex_home_rollout_with_agent_hint(
|
|
302
|
+
&fixture.home(),
|
|
303
|
+
&fixture.spawn_cwd,
|
|
304
|
+
"rollout-worker-a-only.jsonl",
|
|
305
|
+
"codex-sess-only-a",
|
|
306
|
+
"worker_a",
|
|
307
|
+
41001,
|
|
308
|
+
);
|
|
309
|
+
seed_two_interacted_running_codex_agents_without_sessions(&fixture.team, &fixture.spawn_cwd);
|
|
310
|
+
let transport = RecordingTransport::new().with_session_present(true);
|
|
311
|
+
|
|
312
|
+
let report = restart_with_transport(&fixture.team, true, None, &transport)
|
|
313
|
+
.expect("allow_fresh may proceed only after auditing delayed rollout convergence failure");
|
|
314
|
+
let RestartReport::Restarted { agents, .. } = report else {
|
|
315
|
+
panic!("--allow-fresh should permit explicit forced fresh path after deadline");
|
|
316
|
+
};
|
|
317
|
+
assert!(
|
|
318
|
+
agents
|
|
319
|
+
.iter()
|
|
320
|
+
.any(|agent| agent.agent_id.as_str() == "worker_b" && agent.decision == ResumeDecision::FreshStart),
|
|
321
|
+
"fixture expects worker_b to be the only delayed rollout miss and explicit forced-fresh candidate; agents={agents:?}"
|
|
322
|
+
);
|
|
323
|
+
let events = read_events(&fixture.team);
|
|
324
|
+
assert!(
|
|
325
|
+
events
|
|
326
|
+
.lines()
|
|
327
|
+
.any(|line| line.contains("restart.resume_decision")
|
|
328
|
+
&& line.contains(r#""worker_id":"worker_b""#)
|
|
329
|
+
&& line.contains(r#""decision":"fresh_start""#)
|
|
330
|
+
&& line.contains(r#""allow_fresh":true"#)
|
|
331
|
+
&& line.contains(r#""forced_fresh":true"#)
|
|
332
|
+
&& line.contains("resume_not_ready")),
|
|
333
|
+
"--allow-fresh after session convergence miss must audit worker_b as forced_fresh=true with resume_not_ready context, not silent fresh_start; events={events}"
|
|
334
|
+
);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
#[test]
|
|
338
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
339
|
+
#[serial(env)]
|
|
340
|
+
fn restart_waits_for_running_resumable_session_incomplete_agent_even_when_first_send_at_null() {
|
|
341
|
+
let fixture = RestartFixture::new("codex-first-send-null-delayed-rollout", "codex");
|
|
342
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
343
|
+
seed_running_codex_agent_without_session_and_first_send_at(&fixture.team, &fixture.spawn_cwd);
|
|
344
|
+
let delayed_home = fixture.home();
|
|
345
|
+
let delayed_cwd = fixture.spawn_cwd.clone();
|
|
346
|
+
let delayed = std::thread::spawn(move || {
|
|
347
|
+
std::thread::sleep(std::time::Duration::from_millis(1_500));
|
|
348
|
+
seed_codex_home_rollout_with_agent_hint(
|
|
349
|
+
&delayed_home,
|
|
350
|
+
&delayed_cwd,
|
|
351
|
+
"rollout-worker-a-first-send-null-arrives-after-first-poll.jsonl",
|
|
352
|
+
"codex-sess-first-send-null-delayed",
|
|
353
|
+
"worker_a",
|
|
354
|
+
41001,
|
|
355
|
+
);
|
|
356
|
+
});
|
|
357
|
+
let transport = RecordingTransport::new().with_session_present(true);
|
|
358
|
+
|
|
359
|
+
let report = restart_with_transport(&fixture.team, false, None, &transport);
|
|
360
|
+
delayed
|
|
361
|
+
.join()
|
|
362
|
+
.expect("delayed rollout writer should finish");
|
|
363
|
+
let report =
|
|
364
|
+
report.expect("restart should wait for first_send_at=null running resumable worker");
|
|
365
|
+
let events = read_events(&fixture.team);
|
|
366
|
+
|
|
367
|
+
assert_convergence_required_missing_contains(&events, "worker_a");
|
|
368
|
+
assert_restarted_with_resume(&report, "codex-sess-first-send-null-delayed");
|
|
369
|
+
assert_events_contain_resume_decision(
|
|
370
|
+
&fixture.team,
|
|
371
|
+
"resume",
|
|
372
|
+
"codex-sess-first-send-null-delayed",
|
|
373
|
+
);
|
|
374
|
+
assert_eq!(
|
|
375
|
+
transport.kill_count(),
|
|
376
|
+
1,
|
|
377
|
+
"restart may teardown only after session convergence captures the delayed rollout; events={events}"
|
|
378
|
+
);
|
|
379
|
+
assert_eq!(
|
|
380
|
+
transport.spawn_count(),
|
|
381
|
+
1,
|
|
382
|
+
"restart must respawn worker_a as resume, not fresh_start; events={events}"
|
|
383
|
+
);
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
#[test]
|
|
387
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
388
|
+
#[serial(env)]
|
|
389
|
+
fn restart_refuses_first_send_at_null_running_resumable_session_incomplete_when_rollout_never_arrives(
|
|
390
|
+
) {
|
|
391
|
+
let fixture = RestartFixture::new("codex-first-send-null-never-arrives", "codex");
|
|
392
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
393
|
+
let _deadline = EnvGuard::set("TEAM_AGENT_RESTART_SESSION_CONVERGENCE_DEADLINE_MS", "0");
|
|
394
|
+
seed_running_codex_agent_without_session_and_first_send_at(&fixture.team, &fixture.spawn_cwd);
|
|
395
|
+
let state_before = load_runtime_state(&fixture.team).unwrap();
|
|
396
|
+
let transport = RecordingTransport::new().with_session_present(true);
|
|
397
|
+
|
|
398
|
+
let report = restart_with_transport(&fixture.team, false, None, &transport)
|
|
399
|
+
.expect("restart should return structured resume_not_ready refusal");
|
|
400
|
+
|
|
401
|
+
match report {
|
|
402
|
+
RestartReport::RefusedResumeAtomicity {
|
|
403
|
+
unresumable: missing,
|
|
404
|
+
allow_fresh,
|
|
405
|
+
error,
|
|
406
|
+
} => {
|
|
407
|
+
assert!(!allow_fresh, "fixture covers hard refusal without --allow-fresh");
|
|
408
|
+
assert!(
|
|
409
|
+
missing.iter().any(|worker| worker.agent_id.as_str() == "worker_a"),
|
|
410
|
+
"running resumable session-incomplete worker_a must be required missing even when first_send_at=null; missing={missing:?}"
|
|
411
|
+
);
|
|
412
|
+
assert!(
|
|
413
|
+
error.contains("resume_not_ready") || error.contains("session_capture_incomplete"),
|
|
414
|
+
"deadline miss must be named resume_not_ready/session_capture_incomplete; error={error}"
|
|
415
|
+
);
|
|
416
|
+
}
|
|
417
|
+
other => panic!(
|
|
418
|
+
"first_send_at=null must not let allow_fresh=false choose silent fresh_start; expected RefusedResumeNotReady, got {other:?}; events={}",
|
|
419
|
+
read_events(&fixture.team)
|
|
420
|
+
),
|
|
421
|
+
}
|
|
422
|
+
assert_eq!(
|
|
423
|
+
transport.kill_count(),
|
|
424
|
+
0,
|
|
425
|
+
"resume_not_ready must not teardown existing session"
|
|
426
|
+
);
|
|
427
|
+
assert_eq!(
|
|
428
|
+
transport.spawn_count(),
|
|
429
|
+
0,
|
|
430
|
+
"resume_not_ready must not spawn fresh worker"
|
|
431
|
+
);
|
|
432
|
+
assert_eq!(
|
|
433
|
+
load_runtime_state(&fixture.team).unwrap(),
|
|
434
|
+
state_before,
|
|
435
|
+
"resume_not_ready hard refusal must leave state unchanged when first_send_at=null"
|
|
436
|
+
);
|
|
437
|
+
let events = read_events(&fixture.team);
|
|
438
|
+
assert_convergence_required_missing_contains(&events, "worker_a");
|
|
439
|
+
assert!(
|
|
440
|
+
!events
|
|
441
|
+
.lines()
|
|
442
|
+
.any(|line| line.contains("restart.resume_decision")
|
|
443
|
+
&& line.contains(r#""worker_id":"worker_a""#)
|
|
444
|
+
&& line.contains(r#""decision":"fresh_start""#)
|
|
445
|
+
&& line.contains(r#""allow_fresh":false"#)),
|
|
446
|
+
"event invariant: pending first_send_at=null worker may not get fresh_start with allow_fresh=false; events={events}"
|
|
447
|
+
);
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
#[test]
|
|
451
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
452
|
+
#[serial(env)]
|
|
453
|
+
fn wait_ready_reports_session_capture_incomplete_for_running_resumable_worker_without_session() {
|
|
454
|
+
let fixture = RestartFixture::new("wait-ready-session-incomplete", "codex");
|
|
455
|
+
seed_running_state_without_session(&fixture.team, "codex", &fixture.spawn_cwd);
|
|
456
|
+
let mut state = load_runtime_state(&fixture.team).unwrap();
|
|
457
|
+
let mcp_config = fixture.team.join(".team/runtime/worker_a.mcp.json");
|
|
458
|
+
std::fs::create_dir_all(mcp_config.parent().unwrap()).unwrap();
|
|
459
|
+
std::fs::write(&mcp_config, "{}").unwrap();
|
|
460
|
+
state["agents"]["worker_a"]["pane_id"] = json!("%1");
|
|
461
|
+
state["agents"]["worker_a"]["mcp_config"] = json!(mcp_config.to_string_lossy().to_string());
|
|
462
|
+
save_runtime_state(&fixture.team, &state).unwrap();
|
|
463
|
+
|
|
464
|
+
let out = json_result(
|
|
465
|
+
cmd_wait_ready(&WaitReadyArgs {
|
|
466
|
+
workspace: fixture.team.clone(),
|
|
467
|
+
timeout: 0.0,
|
|
468
|
+
json: true,
|
|
469
|
+
team: None,
|
|
470
|
+
})
|
|
471
|
+
.expect("wait-ready should return structured JSON"),
|
|
472
|
+
);
|
|
473
|
+
|
|
474
|
+
assert_eq!(
|
|
475
|
+
out["ok"],
|
|
476
|
+
json!(false),
|
|
477
|
+
"wait-ready cannot report ready while a running Codex/Claude worker is missing captured session context; out={out}"
|
|
478
|
+
);
|
|
479
|
+
assert_eq!(
|
|
480
|
+
out["reason"],
|
|
481
|
+
json!("session_capture_incomplete"),
|
|
482
|
+
"readiness reason must distinguish missing resumable session from generic workers_not_ready; out={out}"
|
|
483
|
+
);
|
|
484
|
+
assert!(
|
|
485
|
+
out["readiness"]["session_capture_incomplete"].as_bool() == Some(true)
|
|
486
|
+
&& out["readiness"]["all_resumable_have_session"].as_bool() == Some(false),
|
|
487
|
+
"readiness truth table must include all_resumable_have_session and session_capture_incomplete; out={out}"
|
|
488
|
+
);
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
#[test]
|
|
492
|
+
fn status_quick_start_and_wait_ready_surfaces_include_session_capture_completeness() {
|
|
493
|
+
let diagnose =
|
|
494
|
+
std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/src/cli/diagnose.rs"))
|
|
495
|
+
.unwrap();
|
|
496
|
+
let status = composite_source::composite_source("src/cli/status_port.rs");
|
|
497
|
+
let launch = composite_source::composite_source("src/lifecycle/launch.rs");
|
|
498
|
+
let adapters =
|
|
499
|
+
std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/src/cli/adapters.rs"))
|
|
500
|
+
.unwrap();
|
|
501
|
+
|
|
502
|
+
for (name, source) in [
|
|
503
|
+
("wait-ready", diagnose.as_str()),
|
|
504
|
+
("status --json", status.as_str()),
|
|
505
|
+
("quick-start readiness", launch.as_str()),
|
|
506
|
+
("quick-start --json", adapters.as_str()),
|
|
507
|
+
] {
|
|
508
|
+
assert!(
|
|
509
|
+
source.contains("session_capture_incomplete")
|
|
510
|
+
&& source.contains("all_resumable_have_session"),
|
|
511
|
+
"C6-C8: {name} must expose readiness = all spawned + attached receiver + all resumable agents have sessions; source lacks session completeness guard"
|
|
512
|
+
);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
#[test]
|
|
517
|
+
fn restart_session_convergence_deadline_is_configurable_and_bounded() {
|
|
518
|
+
let restart = std::fs::read_to_string(concat!(
|
|
519
|
+
env!("CARGO_MANIFEST_DIR"),
|
|
520
|
+
"/src/lifecycle/restart/rebuild.rs"
|
|
521
|
+
))
|
|
522
|
+
.unwrap();
|
|
523
|
+
let types =
|
|
524
|
+
std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/src/cli/types.rs")).unwrap();
|
|
525
|
+
let emit =
|
|
526
|
+
std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/src/cli/emit.rs")).unwrap();
|
|
527
|
+
|
|
528
|
+
assert!(
|
|
529
|
+
restart.contains("session_converge_deadline")
|
|
530
|
+
|| restart.contains("SessionConvergenceDeadline"),
|
|
531
|
+
"C14: restart must have a bounded session convergence deadline, not one opportunistic pass or an unbounded wait"
|
|
532
|
+
);
|
|
533
|
+
assert!(
|
|
534
|
+
restart.contains("10") || restart.contains("15"),
|
|
535
|
+
"C14: default convergence deadline should be a bounded 10-15s class value; restart={restart}"
|
|
536
|
+
);
|
|
537
|
+
assert!(
|
|
538
|
+
types.contains("session_converge_deadline")
|
|
539
|
+
&& emit.contains("--session-converge-deadline")
|
|
540
|
+
&& emit.contains("deadline"),
|
|
541
|
+
"C14: tests/scenarios need an explicit deadline knob; CLI types/parser must expose --session-converge-deadline for restart"
|
|
542
|
+
);
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
#[test]
|
|
546
|
+
fn attribution_ambiguous_is_final_only_after_convergence_deadline() {
|
|
547
|
+
let common = std::fs::read_to_string(concat!(
|
|
548
|
+
env!("CARGO_MANIFEST_DIR"),
|
|
549
|
+
"/src/lifecycle/restart/common.rs"
|
|
550
|
+
))
|
|
551
|
+
.unwrap();
|
|
552
|
+
let tick = std::fs::read_to_string(concat!(
|
|
553
|
+
env!("CARGO_MANIFEST_DIR"),
|
|
554
|
+
"/src/coordinator/tick.rs"
|
|
555
|
+
))
|
|
556
|
+
.unwrap();
|
|
557
|
+
|
|
558
|
+
assert!(
|
|
559
|
+
!common.contains(r#""attribution_ambiguous""#)
|
|
560
|
+
|| common.contains("deadline_expired")
|
|
561
|
+
|| common.contains("final_ambiguous"),
|
|
562
|
+
"C9-C11/C16: restart capture must not durably write attribution_ambiguous during an in-window poll; final ambiguous is allowed only after deadline with unallocatable candidates"
|
|
563
|
+
);
|
|
564
|
+
assert!(
|
|
565
|
+
tick.contains("provider.session.attribution_ambiguous"),
|
|
566
|
+
"C16 guard: opportunistic coordinator tick may still record final attribution ambiguity when it is not the restart convergence gate; tick capture must not be deleted"
|
|
567
|
+
);
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
#[test]
|
|
571
|
+
fn coordinator_tick_session_capture_stays_opportunistic_not_restart_barrier() {
|
|
572
|
+
let tick = std::fs::read_to_string(concat!(
|
|
573
|
+
env!("CARGO_MANIFEST_DIR"),
|
|
574
|
+
"/src/coordinator/tick.rs"
|
|
575
|
+
))
|
|
576
|
+
.unwrap();
|
|
577
|
+
|
|
578
|
+
assert!(
|
|
579
|
+
!tick.contains("session_converge_deadline")
|
|
580
|
+
&& !tick.contains("provider.session.converging")
|
|
581
|
+
&& !tick.contains("resume_not_ready"),
|
|
582
|
+
"C15/C16: only destructive lifecycle restart uses the convergence barrier; coordinator tick capture remains opportunistic and must not block on restart resume readiness"
|
|
583
|
+
);
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
#[test]
|
|
587
|
+
fn session_capture_enumerates_candidates_and_carries_per_agent_context_grep_guard() {
|
|
588
|
+
let adapter = std::fs::read_to_string(concat!(
|
|
589
|
+
env!("CARGO_MANIFEST_DIR"),
|
|
590
|
+
"/src/provider/adapter.rs"
|
|
591
|
+
))
|
|
592
|
+
.unwrap();
|
|
593
|
+
let tick = std::fs::read_to_string(concat!(
|
|
594
|
+
env!("CARGO_MANIFEST_DIR"),
|
|
595
|
+
"/src/coordinator/tick.rs"
|
|
596
|
+
))
|
|
597
|
+
.unwrap();
|
|
598
|
+
let restart = std::fs::read_to_string(concat!(
|
|
599
|
+
env!("CARGO_MANIFEST_DIR"),
|
|
600
|
+
"/src/lifecycle/restart/common.rs"
|
|
601
|
+
))
|
|
602
|
+
.unwrap();
|
|
603
|
+
let launch = composite_source::composite_source("src/lifecycle/launch.rs");
|
|
604
|
+
|
|
605
|
+
assert!(
|
|
606
|
+
adapter.contains("capture_session_candidates")
|
|
607
|
+
|| adapter.contains("matching_session_candidates")
|
|
608
|
+
|| adapter.contains("SessionAttributionContext"),
|
|
609
|
+
"C9: provider capture must expose/enumerate all cwd-matching session candidates instead of returning only the first path-ordered candidate"
|
|
610
|
+
);
|
|
611
|
+
assert!(
|
|
612
|
+
!tick.contains(".capture_session_id(") && !restart.contains(".capture_session_id("),
|
|
613
|
+
"C9/C10: coordinator tick and restart refresh must allocate across all candidates with per-agent context, not call capture_session_id(agent,cwd) one agent at a time"
|
|
614
|
+
);
|
|
615
|
+
assert!(
|
|
616
|
+
adapter.contains("pane_pid") && adapter.contains("spawned_at") && adapter.contains("TEAM_AGENT_ID"),
|
|
617
|
+
"C10/C11: attribution context/decision must include pane_pid, per-agent spawned_at/process time, and positive MCP env TEAM_AGENT_ID before bounded fallback"
|
|
618
|
+
);
|
|
619
|
+
assert!(
|
|
620
|
+
launch.contains("spawned_at") && !launch.contains("let spawned_at = spawn_timestamp();"),
|
|
621
|
+
"C10: launch must persist a per-agent spawned_at/process time, not one shared timestamp for the whole launch"
|
|
622
|
+
);
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
#[test]
|
|
626
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
627
|
+
#[serial(env)]
|
|
628
|
+
fn restart_force_capture_matches_workspace_spawn_cwd_to_team_subdir_across_symlink() {
|
|
629
|
+
let fixture = RestartFixture::new_symlinked_workspace("restart-workspace-team-cwd", "codex");
|
|
630
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
631
|
+
let codex_session = "codex-sess-resume-team-subdir";
|
|
632
|
+
seed_codex_home_rollout_with_cwd(&fixture.home(), &fixture.team, codex_session);
|
|
633
|
+
seed_running_state_without_session(&fixture.team, "codex", &fixture.spawn_cwd);
|
|
634
|
+
let transport = RecordingTransport::new().with_session_present(true);
|
|
635
|
+
|
|
636
|
+
let report = restart_with_transport(&fixture.team, false, None, &transport)
|
|
637
|
+
.expect("restart should succeed after forced session capture from team-subdir rollout cwd");
|
|
638
|
+
|
|
639
|
+
assert_restarted_with_resume(&report, codex_session);
|
|
640
|
+
assert_state_session_id(&fixture.team, codex_session);
|
|
641
|
+
assert_events_contain_resume_decision(&fixture.team, "resume", codex_session);
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
#[test]
|
|
645
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
646
|
+
#[serial(env)]
|
|
647
|
+
fn provider_home_codex_rollout_from_foreign_workspace_is_not_captured() {
|
|
648
|
+
let fixture = RestartFixture::new_symlinked_workspace("codex-foreign-workspace", "codex");
|
|
649
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
650
|
+
let foreign = fixture.root.join("foreign-workspace").join("teamdir");
|
|
651
|
+
std::fs::create_dir_all(&foreign).unwrap();
|
|
652
|
+
seed_codex_home_rollout_with_cwd(&fixture.home(), &foreign, "codex-sess-foreign");
|
|
653
|
+
seed_running_state_without_session(&fixture.team, "codex", &fixture.spawn_cwd);
|
|
654
|
+
|
|
655
|
+
let adapter = get_adapter(Provider::Codex);
|
|
656
|
+
let captured = adapter
|
|
657
|
+
.capture_session_id("worker_a", &fixture.spawn_cwd, 0)
|
|
658
|
+
.expect("Codex provider-home capture should not error");
|
|
659
|
+
|
|
660
|
+
assert!(
|
|
661
|
+
captured.is_none(),
|
|
662
|
+
"workspace↔team-subdir matching must stay scoped to this worker workspace; foreign rollout cwd must not be captured: {captured:?}"
|
|
663
|
+
);
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
#[test]
|
|
667
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
668
|
+
#[serial(env)]
|
|
669
|
+
fn provider_home_claude_session_is_captured_for_restart_resume() {
|
|
670
|
+
let fixture = RestartFixture::new("claude-home-capture", "claude");
|
|
671
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
672
|
+
let claude_session = "claude-sess-240";
|
|
673
|
+
seed_claude_home_session(&fixture.home(), &fixture.spawn_cwd, claude_session);
|
|
674
|
+
seed_running_state_without_session(&fixture.team, "claude", &fixture.spawn_cwd);
|
|
675
|
+
|
|
676
|
+
let adapter = get_adapter(Provider::Claude);
|
|
677
|
+
let captured = adapter
|
|
678
|
+
.capture_session_id("worker_a", &fixture.spawn_cwd, 0)
|
|
679
|
+
.expect("Claude provider-home capture should not error")
|
|
680
|
+
.expect("Claude session under HOME/.claude/sessions must be discoverable before restart");
|
|
681
|
+
|
|
682
|
+
assert_eq!(
|
|
683
|
+
captured.session_id.as_ref().map(|id| id.as_str()),
|
|
684
|
+
Some(claude_session),
|
|
685
|
+
"Claude capture must recover sessionId from provider-home session JSON"
|
|
686
|
+
);
|
|
687
|
+
assert!(
|
|
688
|
+
captured.rollout_path.as_ref().is_some_and(|path| path
|
|
689
|
+
.as_path()
|
|
690
|
+
.starts_with(fixture.home().join(".claude/sessions"))),
|
|
691
|
+
"Claude capture must scan provider home, not only spawn_cwd; captured={captured:?}"
|
|
692
|
+
);
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
#[test]
|
|
696
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
697
|
+
#[serial(env)]
|
|
698
|
+
fn restart_force_captures_real_codex_session_meta_before_teardown() {
|
|
699
|
+
let fixture = RestartFixture::new("restart-codex-force-capture", "codex");
|
|
700
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
701
|
+
let codex_session = "codex-sess-before-kill";
|
|
702
|
+
seed_codex_home_rollout(&fixture.home(), &fixture.spawn_cwd, codex_session);
|
|
703
|
+
seed_running_state_without_session(&fixture.team, "codex", &fixture.spawn_cwd);
|
|
704
|
+
let transport = RecordingTransport::new().with_session_present(true);
|
|
705
|
+
|
|
706
|
+
let report = restart_with_transport(&fixture.team, false, None, &transport)
|
|
707
|
+
.expect("restart should succeed after forced session capture");
|
|
708
|
+
|
|
709
|
+
assert_restarted_with_resume(&report, codex_session);
|
|
710
|
+
assert_spawn_contains_ordered(
|
|
711
|
+
&transport.single_spawn_argv(),
|
|
712
|
+
&["resume", codex_session],
|
|
713
|
+
"Codex restart must spawn `codex ... resume <session_id>`",
|
|
714
|
+
);
|
|
715
|
+
assert_state_session_id(&fixture.team, codex_session);
|
|
716
|
+
assert_events_contain_resume_decision(&fixture.team, "resume", codex_session);
|
|
717
|
+
assert!(
|
|
718
|
+
transport.kill_count() <= 1,
|
|
719
|
+
"transport should not need multiple destructive teardowns; kill_count={}",
|
|
720
|
+
transport.kill_count()
|
|
721
|
+
);
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
#[test]
|
|
725
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
726
|
+
#[serial(env)]
|
|
727
|
+
fn restart_force_captures_claude_session_before_teardown() {
|
|
728
|
+
let fixture = RestartFixture::new("restart-claude-force-capture", "claude");
|
|
729
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
730
|
+
let claude_session = "claude-sess-before-kill";
|
|
731
|
+
seed_claude_home_session(&fixture.home(), &fixture.spawn_cwd, claude_session);
|
|
732
|
+
seed_running_state_without_session(&fixture.team, "claude", &fixture.spawn_cwd);
|
|
733
|
+
let transport = RecordingTransport::new().with_session_present(true);
|
|
734
|
+
|
|
735
|
+
let report = restart_with_transport(&fixture.team, false, None, &transport)
|
|
736
|
+
.expect("restart should succeed after forced session capture");
|
|
737
|
+
|
|
738
|
+
assert_restarted_with_resume(&report, claude_session);
|
|
739
|
+
assert_spawn_contains_adjacent(
|
|
740
|
+
&transport.single_spawn_argv(),
|
|
741
|
+
&["--resume", claude_session],
|
|
742
|
+
"Claude restart must spawn `claude ... --resume <session_id>`",
|
|
743
|
+
);
|
|
744
|
+
assert_state_session_id(&fixture.team, claude_session);
|
|
745
|
+
assert_events_contain_resume_decision(&fixture.team, "resume", claude_session);
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
#[test]
|
|
749
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
750
|
+
#[serial(env)]
|
|
751
|
+
fn shutdown_force_captures_missing_provider_session_before_kill() {
|
|
752
|
+
let fixture = RestartFixture::new("shutdown-force-capture", "codex");
|
|
753
|
+
let _home = HomeGuard::with_home(fixture.root.join("home"));
|
|
754
|
+
let codex_session = "codex-sess-before-shutdown";
|
|
755
|
+
seed_codex_home_rollout(&fixture.home(), &fixture.spawn_cwd, codex_session);
|
|
756
|
+
seed_running_state_without_session_no_coordinator(&fixture.team, "codex", &fixture.spawn_cwd);
|
|
757
|
+
let transport = RecordingTransport::new().with_session_present(true);
|
|
758
|
+
|
|
759
|
+
let result = team_agent::cli::lifecycle_port::shutdown_with_transport(
|
|
760
|
+
&fixture.team,
|
|
761
|
+
true,
|
|
762
|
+
None,
|
|
763
|
+
&transport,
|
|
764
|
+
)
|
|
765
|
+
.expect("shutdown should complete after pre-kill session capture");
|
|
766
|
+
|
|
767
|
+
assert_eq!(result.get("ok").and_then(|v| v.as_bool()), Some(true));
|
|
768
|
+
assert_eq!(
|
|
769
|
+
transport.kill_count(),
|
|
770
|
+
1,
|
|
771
|
+
"shutdown fixture should kill the team session once"
|
|
772
|
+
);
|
|
773
|
+
assert_state_session_id(&fixture.team, codex_session);
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
#[test]
|
|
777
|
+
#[ignore = "real-machine: provider session filesystem + restart/shutdown lifecycle convergence"]
|
|
778
|
+
fn persisted_session_id_is_the_resume_authority_for_restart_plan_and_spawn_argv() {
|
|
779
|
+
let codex = RestartFixture::new("codex-resume-authority", "codex");
|
|
780
|
+
seed_running_state_with_session(
|
|
781
|
+
&codex.team,
|
|
782
|
+
"codex",
|
|
783
|
+
&codex.spawn_cwd,
|
|
784
|
+
"codex-known-session",
|
|
785
|
+
);
|
|
786
|
+
let codex_state = load_runtime_state(&codex.team).unwrap();
|
|
787
|
+
let codex_plan = classify_restart_plan(&codex_state, false).unwrap();
|
|
788
|
+
assert_eq!(codex_plan.decisions.len(), 1);
|
|
789
|
+
assert_eq!(codex_plan.decisions[0].decision, ResumeDecision::Resume);
|
|
790
|
+
assert_eq!(codex_plan.decisions[0].restart_mode, StartMode::Resumed);
|
|
791
|
+
|
|
792
|
+
let codex_transport = RecordingTransport::new().with_session_present(true);
|
|
793
|
+
let codex_report = restart_with_transport(&codex.team, false, None, &codex_transport)
|
|
794
|
+
.expect("persisted Codex session_id must be resumable");
|
|
795
|
+
assert_restarted_with_resume(&codex_report, "codex-known-session");
|
|
796
|
+
assert_spawn_contains_ordered(
|
|
797
|
+
&codex_transport.single_spawn_argv(),
|
|
798
|
+
&["resume", "codex-known-session"],
|
|
799
|
+
"Codex restart must use the persisted session id",
|
|
800
|
+
);
|
|
801
|
+
|
|
802
|
+
let claude = RestartFixture::new("claude-resume-authority", "claude");
|
|
803
|
+
seed_running_state_with_session(
|
|
804
|
+
&claude.team,
|
|
805
|
+
"claude",
|
|
806
|
+
&claude.spawn_cwd,
|
|
807
|
+
"claude-known-session",
|
|
808
|
+
);
|
|
809
|
+
let claude_transport = RecordingTransport::new().with_session_present(true);
|
|
810
|
+
let claude_report = restart_with_transport(&claude.team, false, None, &claude_transport)
|
|
811
|
+
.expect("persisted Claude session_id must be resumable");
|
|
812
|
+
assert_restarted_with_resume(&claude_report, "claude-known-session");
|
|
813
|
+
assert_spawn_contains_adjacent(
|
|
814
|
+
&claude_transport.single_spawn_argv(),
|
|
815
|
+
&["--resume", "claude-known-session"],
|
|
816
|
+
"Claude restart must use the persisted session id",
|
|
817
|
+
);
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
fn assert_restarted_with_resume(report: &RestartReport, expected_session: &str) {
|
|
821
|
+
match report {
|
|
822
|
+
RestartReport::Restarted { agents, .. } => {
|
|
823
|
+
assert_eq!(
|
|
824
|
+
agents.len(),
|
|
825
|
+
1,
|
|
826
|
+
"fixture has exactly one worker; agents={agents:?}"
|
|
827
|
+
);
|
|
828
|
+
assert_eq!(agents[0].decision, ResumeDecision::Resume);
|
|
829
|
+
assert_eq!(agents[0].restart_mode, StartMode::Resumed);
|
|
830
|
+
assert_eq!(
|
|
831
|
+
agents[0].session_id.as_ref().map(|id| id.as_str()),
|
|
832
|
+
Some(expected_session)
|
|
833
|
+
);
|
|
834
|
+
}
|
|
835
|
+
other => panic!("restart must resume after pre-kill capture, got {other:?}"),
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
fn assert_restart_resumed_all_agents<const N: usize>(
|
|
840
|
+
report: &RestartReport,
|
|
841
|
+
expected_agents: [&str; N],
|
|
842
|
+
) {
|
|
843
|
+
match report {
|
|
844
|
+
RestartReport::Restarted { agents, .. } => {
|
|
845
|
+
let resumed = agents
|
|
846
|
+
.iter()
|
|
847
|
+
.map(|agent| {
|
|
848
|
+
(
|
|
849
|
+
agent.agent_id.as_str(),
|
|
850
|
+
(
|
|
851
|
+
agent.decision,
|
|
852
|
+
agent.restart_mode,
|
|
853
|
+
agent.session_id.as_ref().map(|id| id.as_str()),
|
|
854
|
+
),
|
|
855
|
+
)
|
|
856
|
+
})
|
|
857
|
+
.collect::<std::collections::BTreeMap<_, _>>();
|
|
858
|
+
for agent_id in expected_agents {
|
|
859
|
+
let Some((decision, mode, session_id)) = resumed.get(agent_id).copied() else {
|
|
860
|
+
panic!("restart must report a decision for live Codex agent {agent_id}; resumed={resumed:?}");
|
|
861
|
+
};
|
|
862
|
+
assert_eq!(
|
|
863
|
+
decision,
|
|
864
|
+
ResumeDecision::Resume,
|
|
865
|
+
"restart must not fresh-start live Codex agent {agent_id} after session capture; resumed={resumed:?}"
|
|
866
|
+
);
|
|
867
|
+
assert_eq!(
|
|
868
|
+
mode,
|
|
869
|
+
StartMode::Resumed,
|
|
870
|
+
"restart spawn mode must be resumed for live Codex agent {agent_id}; resumed={resumed:?}"
|
|
871
|
+
);
|
|
872
|
+
assert!(
|
|
873
|
+
session_id.is_some_and(|id| !id.is_empty()),
|
|
874
|
+
"restart resume decision for {agent_id} must carry the captured session_id; resumed={resumed:?}"
|
|
875
|
+
);
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
other => panic!("restart must resume every live Codex agent after capture, got {other:?}"),
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
fn assert_all_live_codex_agents_have_complete_unique_sessions<const N: usize>(
|
|
883
|
+
state: &serde_json::Value,
|
|
884
|
+
expected_agents: [&str; N],
|
|
885
|
+
) {
|
|
886
|
+
let mut pairs = std::collections::BTreeSet::new();
|
|
887
|
+
for agent_id in expected_agents {
|
|
888
|
+
let agent = state
|
|
889
|
+
.get("agents")
|
|
890
|
+
.and_then(|agents| agents.get(agent_id))
|
|
891
|
+
.unwrap_or_else(|| {
|
|
892
|
+
panic!("state must contain live Codex agent {agent_id}; state={state}")
|
|
893
|
+
});
|
|
894
|
+
let session_id = agent
|
|
895
|
+
.get("session_id")
|
|
896
|
+
.and_then(serde_json::Value::as_str)
|
|
897
|
+
.filter(|value| !value.is_empty())
|
|
898
|
+
.unwrap_or_else(|| {
|
|
899
|
+
panic!("live Codex agent {agent_id} must have a nonempty captured session_id before restart teardown; state={state}")
|
|
900
|
+
});
|
|
901
|
+
let rollout_path = agent
|
|
902
|
+
.get("rollout_path")
|
|
903
|
+
.and_then(serde_json::Value::as_str)
|
|
904
|
+
.filter(|value| !value.is_empty())
|
|
905
|
+
.unwrap_or_else(|| {
|
|
906
|
+
panic!("live Codex agent {agent_id} must have a nonempty captured rollout_path before restart teardown; state={state}")
|
|
907
|
+
});
|
|
908
|
+
assert!(
|
|
909
|
+
pairs.insert((session_id.to_string(), rollout_path.to_string())),
|
|
910
|
+
"Codex session capture must allocate a unique nonempty (session_id, rollout_path) pair per live agent; duplicate for {agent_id}; state={state}"
|
|
911
|
+
);
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
fn assert_all_resume_decisions_are_session_backed<const N: usize>(
|
|
916
|
+
events: &str,
|
|
917
|
+
expected_agents: [&str; N],
|
|
918
|
+
) {
|
|
919
|
+
for agent_id in expected_agents {
|
|
920
|
+
let matching = events
|
|
921
|
+
.lines()
|
|
922
|
+
.filter(|line| {
|
|
923
|
+
line.contains("restart.resume_decision")
|
|
924
|
+
&& line.contains(&format!(r#""worker_id":"{agent_id}""#))
|
|
925
|
+
})
|
|
926
|
+
.collect::<Vec<_>>();
|
|
927
|
+
assert!(
|
|
928
|
+
!matching.is_empty(),
|
|
929
|
+
"restart must write a resume decision for {agent_id} only after session convergence; events={events}"
|
|
930
|
+
);
|
|
931
|
+
for line in matching {
|
|
932
|
+
assert!(
|
|
933
|
+
line.contains(r#""decision":"resume""#) && line.contains(r#""has_session_id":true"#),
|
|
934
|
+
"restart.resume_decision for {agent_id} must be session-backed resume, never fresh/refuse before delayed rollout convergence; line={line}"
|
|
935
|
+
);
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
assert!(
|
|
939
|
+
!events.contains(r#""decision":"fresh_start""#) && !events.contains(r#""decision":"refuse""#),
|
|
940
|
+
"restart must not emit fresh/refuse resume decisions while delayed rollout convergence is still possible; events={events}"
|
|
941
|
+
);
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
fn read_events(team: &Path) -> String {
|
|
945
|
+
std::fs::read_to_string(team.join(".team/logs/events.jsonl")).unwrap_or_default()
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
fn json_result(result: team_agent::cli::CmdResult) -> Value {
|
|
949
|
+
match result.output {
|
|
950
|
+
CmdOutput::Json(value) => value,
|
|
951
|
+
other => panic!("expected JSON command output, got {other:?}"),
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
fn assert_no_agent_attribution_ambiguous<const N: usize>(
|
|
956
|
+
state: &Value,
|
|
957
|
+
expected_agents: [&str; N],
|
|
958
|
+
) {
|
|
959
|
+
for agent_id in expected_agents {
|
|
960
|
+
let agent = state
|
|
961
|
+
.get("agents")
|
|
962
|
+
.and_then(|agents| agents.get(agent_id))
|
|
963
|
+
.unwrap_or_else(|| panic!("state must contain agent {agent_id}; state={state}"));
|
|
964
|
+
assert!(
|
|
965
|
+
agent.get("attribution_ambiguous").and_then(Value::as_bool) != Some(true),
|
|
966
|
+
"C9-C11: delayed rollout convergence must not write durable attribution_ambiguous during the transient window; agent={agent} state={state}"
|
|
967
|
+
);
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
fn assert_events_have_converging_progress<const N: usize>(events: &str, pending_agents: [&str; N]) {
|
|
972
|
+
let progress = events
|
|
973
|
+
.lines()
|
|
974
|
+
.filter(|line| line.contains("provider.session.converging"))
|
|
975
|
+
.collect::<Vec<_>>();
|
|
976
|
+
assert!(
|
|
977
|
+
!progress.is_empty(),
|
|
978
|
+
"C12: every delayed capture poll must emit provider.session.converging progress instead of silently waiting; events={events}"
|
|
979
|
+
);
|
|
980
|
+
for line in progress {
|
|
981
|
+
let event: serde_json::Value = serde_json::from_str(line)
|
|
982
|
+
.unwrap_or_else(|error| panic!("convergence event must be JSON: {error}; line={line}"));
|
|
983
|
+
assert!(
|
|
984
|
+
line.contains("elapsed_ms")
|
|
985
|
+
&& line.contains("deadline_ms")
|
|
986
|
+
&& line.contains("pending_agent_ids"),
|
|
987
|
+
"C12/C14: converging progress event must expose elapsed_ms, deadline_ms, and pending_agent_ids; line={line}"
|
|
988
|
+
);
|
|
989
|
+
assert!(
|
|
990
|
+
event.get("required_missing").is_none() && event.get("pending").is_none(),
|
|
991
|
+
"convergence event must emit canonical field names only; line={line}"
|
|
992
|
+
);
|
|
993
|
+
}
|
|
994
|
+
for agent_id in pending_agents {
|
|
995
|
+
assert!(
|
|
996
|
+
events
|
|
997
|
+
.lines()
|
|
998
|
+
.any(|line| line.contains("provider.session.converging") && line.contains(agent_id)),
|
|
999
|
+
"C12: pending agent {agent_id} must be named in convergence progress events; events={events}"
|
|
1000
|
+
);
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
fn assert_convergence_required_missing_contains(events: &str, agent_id: &str) {
|
|
1005
|
+
let matching = events
|
|
1006
|
+
.lines()
|
|
1007
|
+
.filter(|line| line.contains("provider.session.converging"))
|
|
1008
|
+
.collect::<Vec<_>>();
|
|
1009
|
+
assert!(
|
|
1010
|
+
!matching.is_empty(),
|
|
1011
|
+
"restart must emit convergence progress before deciding resume/fresh/refuse; events={events}"
|
|
1012
|
+
);
|
|
1013
|
+
assert!(
|
|
1014
|
+
matching.iter().any(|line| {
|
|
1015
|
+
let expected_missing = format!(r#""missing":["{agent_id}"]"#);
|
|
1016
|
+
let expected_required = format!(r#""required_missing_agent_ids":["{agent_id}"]"#);
|
|
1017
|
+
line.contains(&expected_missing) || line.contains(&expected_required)
|
|
1018
|
+
}),
|
|
1019
|
+
"running resumable session-incomplete agent {agent_id} must be in required missing even when first_send_at=null; convergence_events={matching:?}"
|
|
1020
|
+
);
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
fn agent_session<'a>(state: &'a serde_json::Value, agent_id: &str) -> Option<&'a str> {
|
|
1024
|
+
state
|
|
1025
|
+
.get("agents")
|
|
1026
|
+
.and_then(|agents| agents.get(agent_id))
|
|
1027
|
+
.and_then(|agent| agent.get("session_id"))
|
|
1028
|
+
.and_then(serde_json::Value::as_str)
|
|
1029
|
+
.filter(|value| !value.is_empty())
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
fn assert_spawn_contains_adjacent(argv: &[String], expected: &[&str], message: &str) {
|
|
1033
|
+
assert!(
|
|
1034
|
+
argv.windows(expected.len()).any(|window| window
|
|
1035
|
+
.iter()
|
|
1036
|
+
.map(String::as_str)
|
|
1037
|
+
.eq(expected.iter().copied())),
|
|
1038
|
+
"{message}; argv={argv:?}"
|
|
1039
|
+
);
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
fn assert_spawn_contains_ordered(argv: &[String], expected: &[&str], message: &str) {
|
|
1043
|
+
let mut index = 0;
|
|
1044
|
+
for arg in argv {
|
|
1045
|
+
if expected.get(index).is_some_and(|needle| arg == needle) {
|
|
1046
|
+
index += 1;
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
assert_eq!(
|
|
1050
|
+
index,
|
|
1051
|
+
expected.len(),
|
|
1052
|
+
"{message}; expected ordered tokens {expected:?}; argv={argv:?}"
|
|
1053
|
+
);
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
fn assert_state_session_id(team: &Path, expected_session: &str) {
|
|
1057
|
+
let state = load_runtime_state(team).unwrap();
|
|
1058
|
+
assert_eq!(
|
|
1059
|
+
state
|
|
1060
|
+
.get("agents")
|
|
1061
|
+
.and_then(|agents| agents.get("worker_a"))
|
|
1062
|
+
.and_then(|agent| agent.get("session_id"))
|
|
1063
|
+
.and_then(|value| value.as_str()),
|
|
1064
|
+
Some(expected_session),
|
|
1065
|
+
"destructive lifecycle must persist captured session_id as resume authority before teardown; state={state}"
|
|
1066
|
+
);
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
fn assert_events_contain_resume_decision(
|
|
1070
|
+
team: &Path,
|
|
1071
|
+
expected_decision: &str,
|
|
1072
|
+
expected_session: &str,
|
|
1073
|
+
) {
|
|
1074
|
+
let events = std::fs::read_to_string(team.join(".team/logs/events.jsonl")).unwrap();
|
|
1075
|
+
let needle = format!(
|
|
1076
|
+
r#""event":"restart.resume_decision".*"decision":"{expected_decision}".*"has_session_id":true.*"session_id":"{expected_session}""#
|
|
1077
|
+
);
|
|
1078
|
+
assert!(
|
|
1079
|
+
events.lines().any(|line| {
|
|
1080
|
+
line.contains(r#""event":"restart.resume_decision""#)
|
|
1081
|
+
&& line.contains(&format!(r#""decision":"{expected_decision}""#))
|
|
1082
|
+
&& line.contains(r#""has_session_id":true"#)
|
|
1083
|
+
&& line.contains(&format!(r#""session_id":"{expected_session}""#))
|
|
1084
|
+
}),
|
|
1085
|
+
"restart must audit a resume decision with captured session_id; missing pattern {needle}; events={events}"
|
|
1086
|
+
);
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
fn seed_codex_home_rollout(home: &Path, spawn_cwd: &Path, session_id: &str) {
|
|
1090
|
+
seed_codex_home_rollout_with_cwd(home, spawn_cwd, session_id);
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
fn seed_codex_home_rollout_with_cwd(home: &Path, rollout_cwd: &Path, session_id: &str) {
|
|
1094
|
+
let dir = home.join(".codex/sessions/2026/06/07");
|
|
1095
|
+
std::fs::create_dir_all(&dir).unwrap();
|
|
1096
|
+
std::fs::write(
|
|
1097
|
+
dir.join(format!("rollout-worker-a-{session_id}.jsonl")),
|
|
1098
|
+
format!(
|
|
1099
|
+
"{}\n{}\n",
|
|
1100
|
+
json!({
|
|
1101
|
+
"type": "session_meta",
|
|
1102
|
+
"payload": {
|
|
1103
|
+
"id": session_id,
|
|
1104
|
+
"cwd": rollout_cwd.to_string_lossy().to_string()
|
|
1105
|
+
}
|
|
1106
|
+
}),
|
|
1107
|
+
json!({"type":"event","payload":{"message":"later record"}})
|
|
1108
|
+
),
|
|
1109
|
+
)
|
|
1110
|
+
.unwrap();
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
fn seed_codex_home_rollout_with_agent_hint(
|
|
1114
|
+
home: &Path,
|
|
1115
|
+
rollout_cwd: &Path,
|
|
1116
|
+
file_name: &str,
|
|
1117
|
+
session_id: &str,
|
|
1118
|
+
agent_id: &str,
|
|
1119
|
+
pane_pid: u32,
|
|
1120
|
+
) {
|
|
1121
|
+
let dir = home.join(".codex/sessions/2026/06/07");
|
|
1122
|
+
std::fs::create_dir_all(&dir).unwrap();
|
|
1123
|
+
std::fs::write(
|
|
1124
|
+
dir.join(file_name),
|
|
1125
|
+
format!(
|
|
1126
|
+
"{}\n{}\n",
|
|
1127
|
+
json!({
|
|
1128
|
+
"type": "session_meta",
|
|
1129
|
+
"payload": {
|
|
1130
|
+
"id": session_id,
|
|
1131
|
+
"cwd": rollout_cwd.to_string_lossy().to_string()
|
|
1132
|
+
}
|
|
1133
|
+
}),
|
|
1134
|
+
json!({
|
|
1135
|
+
"type": "event",
|
|
1136
|
+
"payload": {
|
|
1137
|
+
"pane_pid": pane_pid,
|
|
1138
|
+
"mcp_servers": {
|
|
1139
|
+
"team_orchestrator": {
|
|
1140
|
+
"env": {
|
|
1141
|
+
"TEAM_AGENT_ID": agent_id
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
})
|
|
1147
|
+
),
|
|
1148
|
+
)
|
|
1149
|
+
.unwrap();
|
|
1150
|
+
}
|
|
1151
|
+
|
|
1152
|
+
fn seed_claude_home_session(home: &Path, spawn_cwd: &Path, session_id: &str) {
|
|
1153
|
+
let dir = home.join(".claude/sessions");
|
|
1154
|
+
std::fs::create_dir_all(&dir).unwrap();
|
|
1155
|
+
std::fs::write(
|
|
1156
|
+
dir.join("worker-a-session.json"),
|
|
1157
|
+
serde_json::to_string(&json!({
|
|
1158
|
+
"sessionId": session_id,
|
|
1159
|
+
"cwd": spawn_cwd.to_string_lossy().to_string()
|
|
1160
|
+
}))
|
|
1161
|
+
.unwrap(),
|
|
1162
|
+
)
|
|
1163
|
+
.unwrap();
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
fn seed_running_state_without_session(team: &Path, provider: &str, spawn_cwd: &Path) {
|
|
1167
|
+
seed_running_state(team, provider, spawn_cwd, None);
|
|
1168
|
+
seed_healthy_coordinator(team);
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
fn seed_running_state_without_session_no_coordinator(
|
|
1172
|
+
team: &Path,
|
|
1173
|
+
provider: &str,
|
|
1174
|
+
spawn_cwd: &Path,
|
|
1175
|
+
) {
|
|
1176
|
+
seed_running_state(team, provider, spawn_cwd, None);
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
fn seed_running_state_with_session(
|
|
1180
|
+
team: &Path,
|
|
1181
|
+
provider: &str,
|
|
1182
|
+
spawn_cwd: &Path,
|
|
1183
|
+
session_id: &str,
|
|
1184
|
+
) {
|
|
1185
|
+
seed_running_state(team, provider, spawn_cwd, Some(session_id));
|
|
1186
|
+
seed_healthy_coordinator(team);
|
|
1187
|
+
}
|
|
1188
|
+
|
|
1189
|
+
fn seed_two_running_codex_agents_without_sessions(team: &Path, spawn_cwd: &Path) {
|
|
1190
|
+
save_runtime_state(
|
|
1191
|
+
team,
|
|
1192
|
+
&json!({
|
|
1193
|
+
"active_team_key": "ctxteam",
|
|
1194
|
+
"team_dir": team.to_string_lossy().to_string(),
|
|
1195
|
+
"spec_path": team.join("team.spec.yaml").to_string_lossy().to_string(),
|
|
1196
|
+
"session_name": "team-ctxteam",
|
|
1197
|
+
"agents": {
|
|
1198
|
+
"worker_a": {
|
|
1199
|
+
"status": "running",
|
|
1200
|
+
"provider": "codex",
|
|
1201
|
+
"role": "Worker",
|
|
1202
|
+
"tools": ["mcp_team"],
|
|
1203
|
+
"window": "worker_a",
|
|
1204
|
+
"owner_team_id": "ctxteam",
|
|
1205
|
+
"session_id": null,
|
|
1206
|
+
"rollout_path": null,
|
|
1207
|
+
"spawn_cwd": spawn_cwd.to_string_lossy().to_string(),
|
|
1208
|
+
"spawned_at": "2026-06-07T00:00:00+00:00",
|
|
1209
|
+
"pane_pid": 41001
|
|
1210
|
+
},
|
|
1211
|
+
"worker_b": {
|
|
1212
|
+
"status": "running",
|
|
1213
|
+
"provider": "codex",
|
|
1214
|
+
"role": "Worker",
|
|
1215
|
+
"tools": ["mcp_team"],
|
|
1216
|
+
"window": "worker_b",
|
|
1217
|
+
"owner_team_id": "ctxteam",
|
|
1218
|
+
"session_id": null,
|
|
1219
|
+
"rollout_path": null,
|
|
1220
|
+
"spawn_cwd": spawn_cwd.to_string_lossy().to_string(),
|
|
1221
|
+
"spawned_at": "2026-06-07T00:00:07+00:00",
|
|
1222
|
+
"pane_pid": 41002
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
}),
|
|
1226
|
+
)
|
|
1227
|
+
.unwrap();
|
|
1228
|
+
seed_healthy_coordinator(team);
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
fn seed_two_interacted_running_codex_agents_without_sessions(team: &Path, spawn_cwd: &Path) {
|
|
1232
|
+
seed_two_running_codex_agents_without_sessions(team, spawn_cwd);
|
|
1233
|
+
let mut state = load_runtime_state(team).unwrap();
|
|
1234
|
+
for (agent_id, first_send_at) in [
|
|
1235
|
+
("worker_a", "2026-06-07T00:02:00+00:00"),
|
|
1236
|
+
("worker_b", "2026-06-07T00:02:07+00:00"),
|
|
1237
|
+
] {
|
|
1238
|
+
if let Some(agent) = state
|
|
1239
|
+
.get_mut("agents")
|
|
1240
|
+
.and_then(serde_json::Value::as_object_mut)
|
|
1241
|
+
.and_then(|agents| agents.get_mut(agent_id))
|
|
1242
|
+
.and_then(serde_json::Value::as_object_mut)
|
|
1243
|
+
{
|
|
1244
|
+
agent.insert(
|
|
1245
|
+
"first_send_at".to_string(),
|
|
1246
|
+
serde_json::json!(first_send_at),
|
|
1247
|
+
);
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
save_runtime_state(team, &state).unwrap();
|
|
1251
|
+
}
|
|
1252
|
+
|
|
1253
|
+
fn seed_running_codex_agent_without_session_and_first_send_at(team: &Path, spawn_cwd: &Path) {
|
|
1254
|
+
seed_running_state(team, "codex", spawn_cwd, None);
|
|
1255
|
+
let mut state = load_runtime_state(team).unwrap();
|
|
1256
|
+
if let Some(agent) = state
|
|
1257
|
+
.get_mut("agents")
|
|
1258
|
+
.and_then(serde_json::Value::as_object_mut)
|
|
1259
|
+
.and_then(|agents| agents.get_mut("worker_a"))
|
|
1260
|
+
.and_then(serde_json::Value::as_object_mut)
|
|
1261
|
+
{
|
|
1262
|
+
agent.insert("first_send_at".to_string(), serde_json::Value::Null);
|
|
1263
|
+
agent.insert("rollout_path".to_string(), serde_json::Value::Null);
|
|
1264
|
+
agent.insert("pane_id".to_string(), serde_json::json!("%1"));
|
|
1265
|
+
agent.insert("pane_pid".to_string(), serde_json::json!(41001));
|
|
1266
|
+
agent.insert("mcp_ready".to_string(), serde_json::json!(true));
|
|
1267
|
+
agent.insert("startup_prompts".to_string(), serde_json::json!("complete"));
|
|
1268
|
+
}
|
|
1269
|
+
save_runtime_state(team, &state).unwrap();
|
|
1270
|
+
seed_true_machine_interaction_evidence(team);
|
|
1271
|
+
}
|
|
1272
|
+
|
|
1273
|
+
fn seed_true_machine_interaction_evidence(team: &Path) {
|
|
1274
|
+
let path = team.join(".team/logs/events.jsonl");
|
|
1275
|
+
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
|
1276
|
+
std::fs::write(
|
|
1277
|
+
path,
|
|
1278
|
+
[
|
|
1279
|
+
serde_json::json!({
|
|
1280
|
+
"event": "send.delivered",
|
|
1281
|
+
"message_id": "msg_true_machine_leader_to_worker",
|
|
1282
|
+
"from": "leader",
|
|
1283
|
+
"to": "worker_a",
|
|
1284
|
+
"owner_team_id": "ctxteam",
|
|
1285
|
+
"status": "delivered"
|
|
1286
|
+
})
|
|
1287
|
+
.to_string(),
|
|
1288
|
+
serde_json::json!({
|
|
1289
|
+
"event": "turn_open.armed_after_delivery",
|
|
1290
|
+
"agent_id": "worker_a",
|
|
1291
|
+
"owner_team_id": "ctxteam",
|
|
1292
|
+
"pane_id": "%1"
|
|
1293
|
+
})
|
|
1294
|
+
.to_string(),
|
|
1295
|
+
serde_json::json!({
|
|
1296
|
+
"event": "mcp.report_result",
|
|
1297
|
+
"agent_id": "worker_a",
|
|
1298
|
+
"owner_team_id": "ctxteam",
|
|
1299
|
+
"status": "delivered"
|
|
1300
|
+
})
|
|
1301
|
+
.to_string(),
|
|
1302
|
+
]
|
|
1303
|
+
.join("\n")
|
|
1304
|
+
+ "\n",
|
|
1305
|
+
)
|
|
1306
|
+
.unwrap();
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
fn seed_running_state(team: &Path, provider: &str, spawn_cwd: &Path, session_id: Option<&str>) {
|
|
1310
|
+
save_runtime_state(
|
|
1311
|
+
team,
|
|
1312
|
+
&json!({
|
|
1313
|
+
"active_team_key": "ctxteam",
|
|
1314
|
+
"team_dir": team.to_string_lossy().to_string(),
|
|
1315
|
+
"spec_path": team.join("team.spec.yaml").to_string_lossy().to_string(),
|
|
1316
|
+
"session_name": "team-ctxteam",
|
|
1317
|
+
"agents": {
|
|
1318
|
+
"worker_a": {
|
|
1319
|
+
"status": "running",
|
|
1320
|
+
"provider": provider,
|
|
1321
|
+
"role": "Worker",
|
|
1322
|
+
"tools": ["mcp_team"],
|
|
1323
|
+
"window": "worker_a",
|
|
1324
|
+
"owner_team_id": "ctxteam",
|
|
1325
|
+
"session_id": session_id,
|
|
1326
|
+
"rollout_path": null,
|
|
1327
|
+
"spawn_cwd": spawn_cwd.to_string_lossy().to_string(),
|
|
1328
|
+
"spawned_at": "2026-06-07T00:00:00+00:00",
|
|
1329
|
+
"first_send_at": "2026-06-07T00:01:00+00:00"
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
}),
|
|
1333
|
+
)
|
|
1334
|
+
.unwrap();
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
fn seed_healthy_coordinator(workspace: &Path) {
|
|
1338
|
+
let workspace = team_agent::coordinator::WorkspacePath::new(workspace.to_path_buf());
|
|
1339
|
+
std::fs::create_dir_all(team_agent::model::paths::runtime_dir(workspace.as_path())).unwrap();
|
|
1340
|
+
let _ = team_agent::message_store::MessageStore::open(workspace.as_path()).unwrap();
|
|
1341
|
+
let pid = team_agent::coordinator::Pid::new(std::process::id());
|
|
1342
|
+
team_agent::coordinator::write_coordinator_metadata(
|
|
1343
|
+
&workspace,
|
|
1344
|
+
pid,
|
|
1345
|
+
team_agent::coordinator::MetadataSource::Boot,
|
|
1346
|
+
)
|
|
1347
|
+
.unwrap();
|
|
1348
|
+
std::fs::write(
|
|
1349
|
+
team_agent::coordinator::coordinator_pid_path(&workspace),
|
|
1350
|
+
pid.to_string(),
|
|
1351
|
+
)
|
|
1352
|
+
.unwrap();
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
struct RestartFixture {
|
|
1356
|
+
root: PathBuf,
|
|
1357
|
+
team: PathBuf,
|
|
1358
|
+
spawn_cwd: PathBuf,
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
impl RestartFixture {
|
|
1362
|
+
fn new(label: &str, provider: &str) -> Self {
|
|
1363
|
+
let root = tmp_dir(label);
|
|
1364
|
+
let team = root.join("teamdir");
|
|
1365
|
+
std::fs::create_dir_all(team.join("agents")).unwrap();
|
|
1366
|
+
std::fs::write(
|
|
1367
|
+
team.join("TEAM.md"),
|
|
1368
|
+
"---\nname: ctxteam\nobjective: Restart session context contract.\nprovider: codex\n---\n\nTeam.\n",
|
|
1369
|
+
)
|
|
1370
|
+
.unwrap();
|
|
1371
|
+
std::fs::write(
|
|
1372
|
+
team.join("agents/worker_a.md"),
|
|
1373
|
+
format!(
|
|
1374
|
+
"---\nname: worker_a\nrole: Worker\nprovider: {provider}\nmodel: {}\nauth_mode: subscription\ntools:\n - mcp_team\n---\n\nWorker.\n",
|
|
1375
|
+
if provider == "claude" {
|
|
1376
|
+
"claude-sonnet-4-6"
|
|
1377
|
+
} else {
|
|
1378
|
+
"gpt-5.5"
|
|
1379
|
+
}
|
|
1380
|
+
),
|
|
1381
|
+
)
|
|
1382
|
+
.unwrap();
|
|
1383
|
+
let spec = team_agent::compiler::compile_team(&team).unwrap();
|
|
1384
|
+
std::fs::write(
|
|
1385
|
+
team.join("team.spec.yaml"),
|
|
1386
|
+
team_agent::model::yaml::dumps(&spec),
|
|
1387
|
+
)
|
|
1388
|
+
.unwrap();
|
|
1389
|
+
let spawn_cwd = root.join("spawn-cwd");
|
|
1390
|
+
std::fs::create_dir_all(&spawn_cwd).unwrap();
|
|
1391
|
+
Self {
|
|
1392
|
+
root,
|
|
1393
|
+
team,
|
|
1394
|
+
spawn_cwd,
|
|
1395
|
+
}
|
|
1396
|
+
}
|
|
1397
|
+
|
|
1398
|
+
fn new_symlinked_workspace(label: &str, provider: &str) -> Self {
|
|
1399
|
+
let root = tmp_dir(label);
|
|
1400
|
+
let workspace = root.join("real-workspace");
|
|
1401
|
+
std::fs::create_dir_all(&workspace).unwrap();
|
|
1402
|
+
let team = workspace.join("teamdir");
|
|
1403
|
+
std::fs::create_dir_all(team.join("agents")).unwrap();
|
|
1404
|
+
std::fs::write(
|
|
1405
|
+
team.join("TEAM.md"),
|
|
1406
|
+
"---\nname: ctxteam\nobjective: Restart session context contract.\nprovider: codex\n---\n\nTeam.\n",
|
|
1407
|
+
)
|
|
1408
|
+
.unwrap();
|
|
1409
|
+
std::fs::write(
|
|
1410
|
+
team.join("agents/worker_a.md"),
|
|
1411
|
+
format!(
|
|
1412
|
+
"---\nname: worker_a\nrole: Worker\nprovider: {provider}\nmodel: {}\nauth_mode: subscription\ntools:\n - mcp_team\n---\n\nWorker.\n",
|
|
1413
|
+
if provider == "claude" {
|
|
1414
|
+
"claude-sonnet-4-6"
|
|
1415
|
+
} else {
|
|
1416
|
+
"gpt-5.5"
|
|
1417
|
+
}
|
|
1418
|
+
),
|
|
1419
|
+
)
|
|
1420
|
+
.unwrap();
|
|
1421
|
+
let spec = team_agent::compiler::compile_team(&team).unwrap();
|
|
1422
|
+
std::fs::write(
|
|
1423
|
+
team.join("team.spec.yaml"),
|
|
1424
|
+
team_agent::model::yaml::dumps(&spec),
|
|
1425
|
+
)
|
|
1426
|
+
.unwrap();
|
|
1427
|
+
let spawn_cwd = root.join("workspace-symlink");
|
|
1428
|
+
#[cfg(unix)]
|
|
1429
|
+
std::os::unix::fs::symlink(&workspace, &spawn_cwd).unwrap();
|
|
1430
|
+
#[cfg(not(unix))]
|
|
1431
|
+
{
|
|
1432
|
+
std::fs::create_dir_all(&spawn_cwd).unwrap();
|
|
1433
|
+
}
|
|
1434
|
+
Self {
|
|
1435
|
+
root,
|
|
1436
|
+
team,
|
|
1437
|
+
spawn_cwd,
|
|
1438
|
+
}
|
|
1439
|
+
}
|
|
1440
|
+
|
|
1441
|
+
fn home(&self) -> PathBuf {
|
|
1442
|
+
self.root.join("home")
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
fn tmp_dir(tag: &str) -> PathBuf {
|
|
1447
|
+
static N: AtomicU64 = AtomicU64::new(0);
|
|
1448
|
+
let dir = std::env::temp_dir().join(format!(
|
|
1449
|
+
"ta-rs-restart-session-capture-{tag}-{}-{}",
|
|
1450
|
+
std::process::id(),
|
|
1451
|
+
N.fetch_add(1, Ordering::Relaxed)
|
|
1452
|
+
));
|
|
1453
|
+
let _ = std::fs::remove_dir_all(&dir);
|
|
1454
|
+
std::fs::create_dir_all(&dir).unwrap();
|
|
1455
|
+
std::fs::canonicalize(dir).unwrap()
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
struct HomeGuard {
|
|
1459
|
+
previous_home: Option<String>,
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
impl HomeGuard {
|
|
1463
|
+
fn with_home(home: PathBuf) -> Self {
|
|
1464
|
+
std::fs::create_dir_all(&home).unwrap();
|
|
1465
|
+
let previous_home = std::env::var("HOME").ok();
|
|
1466
|
+
unsafe {
|
|
1467
|
+
std::env::set_var("HOME", home);
|
|
1468
|
+
}
|
|
1469
|
+
Self { previous_home }
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
impl Drop for HomeGuard {
|
|
1474
|
+
fn drop(&mut self) {
|
|
1475
|
+
unsafe {
|
|
1476
|
+
if let Some(value) = self.previous_home.take() {
|
|
1477
|
+
std::env::set_var("HOME", value);
|
|
1478
|
+
} else {
|
|
1479
|
+
std::env::remove_var("HOME");
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
struct EnvGuard {
|
|
1486
|
+
key: &'static str,
|
|
1487
|
+
previous: Option<String>,
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
impl EnvGuard {
|
|
1491
|
+
fn set(key: &'static str, value: &str) -> Self {
|
|
1492
|
+
let previous = std::env::var(key).ok();
|
|
1493
|
+
unsafe {
|
|
1494
|
+
std::env::set_var(key, value);
|
|
1495
|
+
}
|
|
1496
|
+
Self { key, previous }
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1500
|
+
impl Drop for EnvGuard {
|
|
1501
|
+
fn drop(&mut self) {
|
|
1502
|
+
unsafe {
|
|
1503
|
+
if let Some(value) = self.previous.take() {
|
|
1504
|
+
std::env::set_var(self.key, value);
|
|
1505
|
+
} else {
|
|
1506
|
+
std::env::remove_var(self.key);
|
|
1507
|
+
}
|
|
1508
|
+
}
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1511
|
+
|
|
1512
|
+
#[derive(Debug, Clone)]
|
|
1513
|
+
struct RecordedSpawn {
|
|
1514
|
+
argv: Vec<String>,
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
#[derive(Debug, Default)]
|
|
1518
|
+
struct RecordingTransport {
|
|
1519
|
+
spawns: Mutex<Vec<RecordedSpawn>>,
|
|
1520
|
+
kills: Mutex<u32>,
|
|
1521
|
+
session_present: bool,
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1524
|
+
impl RecordingTransport {
|
|
1525
|
+
fn new() -> Self {
|
|
1526
|
+
Self::default()
|
|
1527
|
+
}
|
|
1528
|
+
|
|
1529
|
+
fn with_session_present(mut self, present: bool) -> Self {
|
|
1530
|
+
self.session_present = present;
|
|
1531
|
+
self
|
|
1532
|
+
}
|
|
1533
|
+
|
|
1534
|
+
fn single_spawn_argv(&self) -> Vec<String> {
|
|
1535
|
+
let spawns = self.spawns.lock().unwrap();
|
|
1536
|
+
assert_eq!(
|
|
1537
|
+
spawns.len(),
|
|
1538
|
+
1,
|
|
1539
|
+
"fixture should record exactly one worker spawn; spawns={spawns:?}"
|
|
1540
|
+
);
|
|
1541
|
+
spawns[0].argv.clone()
|
|
1542
|
+
}
|
|
1543
|
+
|
|
1544
|
+
fn kill_count(&self) -> u32 {
|
|
1545
|
+
*self.kills.lock().unwrap()
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
fn spawn_count(&self) -> usize {
|
|
1549
|
+
self.spawns.lock().unwrap().len()
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1552
|
+
fn spawn_result(
|
|
1553
|
+
&self,
|
|
1554
|
+
session: &SessionName,
|
|
1555
|
+
window: &WindowName,
|
|
1556
|
+
argv: &[String],
|
|
1557
|
+
) -> SpawnResult {
|
|
1558
|
+
let mut spawns = self.spawns.lock().unwrap();
|
|
1559
|
+
spawns.push(RecordedSpawn {
|
|
1560
|
+
argv: argv.to_vec(),
|
|
1561
|
+
});
|
|
1562
|
+
SpawnResult {
|
|
1563
|
+
pane_id: PaneId::new(format!("%{}", spawns.len())),
|
|
1564
|
+
session: session.clone(),
|
|
1565
|
+
window: window.clone(),
|
|
1566
|
+
child_pid: None,
|
|
1567
|
+
}
|
|
1568
|
+
}
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
impl Transport for RecordingTransport {
|
|
1572
|
+
fn kind(&self) -> BackendKind {
|
|
1573
|
+
BackendKind::Tmux
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
fn spawn_first(
|
|
1577
|
+
&self,
|
|
1578
|
+
session: &SessionName,
|
|
1579
|
+
window: &WindowName,
|
|
1580
|
+
argv: &[String],
|
|
1581
|
+
_cwd: &Path,
|
|
1582
|
+
_env: &BTreeMap<String, String>,
|
|
1583
|
+
) -> Result<SpawnResult, TransportError> {
|
|
1584
|
+
Ok(self.spawn_result(session, window, argv))
|
|
1585
|
+
}
|
|
1586
|
+
|
|
1587
|
+
fn spawn_into(
|
|
1588
|
+
&self,
|
|
1589
|
+
session: &SessionName,
|
|
1590
|
+
window: &WindowName,
|
|
1591
|
+
argv: &[String],
|
|
1592
|
+
_cwd: &Path,
|
|
1593
|
+
_env: &BTreeMap<String, String>,
|
|
1594
|
+
) -> Result<SpawnResult, TransportError> {
|
|
1595
|
+
Ok(self.spawn_result(session, window, argv))
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
fn inject(
|
|
1599
|
+
&self,
|
|
1600
|
+
_target: &Target,
|
|
1601
|
+
_payload: &InjectPayload,
|
|
1602
|
+
_submit: Key,
|
|
1603
|
+
_bracketed: bool,
|
|
1604
|
+
) -> Result<InjectReport, TransportError> {
|
|
1605
|
+
Ok(InjectReport {
|
|
1606
|
+
stage_reached: InjectStage::Submit,
|
|
1607
|
+
inject_verification: InjectVerification::CaptureContainsToken,
|
|
1608
|
+
submit_verification: SubmitVerification::EnterSentWithoutPlaceholderCheck,
|
|
1609
|
+
turn_verification: TurnVerification::NotYetObserved,
|
|
1610
|
+
attempts: 1,
|
|
1611
|
+
submit_diagnostics: None,
|
|
1612
|
+
})
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1615
|
+
fn send_keys(&self, _target: &Target, _keys: &[Key]) -> Result<(), TransportError> {
|
|
1616
|
+
Ok(())
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
fn capture(
|
|
1620
|
+
&self,
|
|
1621
|
+
_target: &Target,
|
|
1622
|
+
range: CaptureRange,
|
|
1623
|
+
) -> Result<CapturedText, TransportError> {
|
|
1624
|
+
Ok(CapturedText {
|
|
1625
|
+
text: String::new(),
|
|
1626
|
+
range,
|
|
1627
|
+
})
|
|
1628
|
+
}
|
|
1629
|
+
|
|
1630
|
+
fn query(&self, _target: &Target, field: PaneField) -> Result<Option<String>, TransportError> {
|
|
1631
|
+
match field {
|
|
1632
|
+
PaneField::PaneWidth => Ok(Some("120".to_string())),
|
|
1633
|
+
_ => Ok(None),
|
|
1634
|
+
}
|
|
1635
|
+
}
|
|
1636
|
+
|
|
1637
|
+
fn liveness(
|
|
1638
|
+
&self,
|
|
1639
|
+
_pane: &PaneId,
|
|
1640
|
+
) -> Result<team_agent::transport::PaneLiveness, TransportError> {
|
|
1641
|
+
Ok(team_agent::transport::PaneLiveness::Live)
|
|
1642
|
+
}
|
|
1643
|
+
|
|
1644
|
+
fn list_targets(&self) -> Result<Vec<PaneInfo>, TransportError> {
|
|
1645
|
+
Ok(Vec::new())
|
|
1646
|
+
}
|
|
1647
|
+
|
|
1648
|
+
fn has_session(&self, _session: &SessionName) -> Result<bool, TransportError> {
|
|
1649
|
+
Ok(self.session_present)
|
|
1650
|
+
}
|
|
1651
|
+
|
|
1652
|
+
fn list_windows(&self, _session: &SessionName) -> Result<Vec<WindowName>, TransportError> {
|
|
1653
|
+
Ok(Vec::new())
|
|
1654
|
+
}
|
|
1655
|
+
|
|
1656
|
+
fn set_session_env(
|
|
1657
|
+
&self,
|
|
1658
|
+
_session: &SessionName,
|
|
1659
|
+
_key: &str,
|
|
1660
|
+
_value: &str,
|
|
1661
|
+
) -> Result<SetEnvOutcome, TransportError> {
|
|
1662
|
+
Ok(SetEnvOutcome::Applied)
|
|
1663
|
+
}
|
|
1664
|
+
|
|
1665
|
+
fn kill_session(&self, _session: &SessionName) -> Result<(), TransportError> {
|
|
1666
|
+
*self.kills.lock().unwrap() += 1;
|
|
1667
|
+
Ok(())
|
|
1668
|
+
}
|
|
1669
|
+
|
|
1670
|
+
fn kill_window(&self, _target: &Target) -> Result<(), TransportError> {
|
|
1671
|
+
Ok(())
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
fn attach_session(&self, _session: &SessionName) -> Result<AttachOutcome, TransportError> {
|
|
1675
|
+
Ok(AttachOutcome::Attached)
|
|
1676
|
+
}
|
|
1677
|
+
}
|