@team-agent/installer 0.5.3 → 0.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -1
- package/Cargo.toml +1 -1
- package/crates/team-agent/src/cli/adapters.rs +9 -1
- package/crates/team-agent/src/cli/diagnose.rs +191 -31
- package/crates/team-agent/src/cli/mod.rs +93 -34
- package/crates/team-agent/src/cli/send.rs +69 -2
- package/crates/team-agent/src/cli/status_port.rs +62 -0
- package/crates/team-agent/src/cli/tests/status_send.rs +12 -19
- package/crates/team-agent/src/coordinator/tick.rs +13 -0
- package/crates/team-agent/src/lifecycle/launch.rs +14 -6
- package/crates/team-agent/src/lifecycle/restart/agent.rs +44 -1
- package/crates/team-agent/src/lifecycle/restart/common.rs +53 -2
- package/crates/team-agent/src/lifecycle/restart/rebuild.rs +108 -6
- package/crates/team-agent/src/lifecycle/restart/selection.rs +47 -18
- package/crates/team-agent/src/lifecycle/restart.rs +16 -6
- package/crates/team-agent/src/lifecycle/tests/launch_spawn.rs +287 -53
- package/crates/team-agent/src/lifecycle/tests/restart.rs +144 -5
- package/crates/team-agent/src/lifecycle/types.rs +1 -0
- package/crates/team-agent/src/messaging/delivery.rs +209 -30
- package/crates/team-agent/src/messaging/leader_receiver.rs +60 -25
- package/crates/team-agent/src/messaging/tests/runtime.rs +446 -0
- package/crates/team-agent/src/messaging/watchers.rs +36 -19
- package/crates/team-agent/src/provider/adapter.rs +103 -41
- package/crates/team-agent/src/provider/session/capture.rs +328 -66
- package/crates/team-agent/src/provider/session/resume.rs +30 -28
- package/crates/team-agent/src/provider/session_scan/common.rs +117 -1
- package/crates/team-agent/src/provider/session_scan/copilot.rs +1 -0
- package/crates/team-agent/src/provider/session_scan.rs +1 -0
- package/crates/team-agent/src/state/persist.rs +17 -0
- package/crates/team-agent/src/state/projection.rs +59 -4
- package/package.json +4 -4
|
@@ -90,9 +90,12 @@ pub fn cmd_send(args: &SendArgs) -> Result<CmdResult, CliError> {
|
|
|
90
90
|
{
|
|
91
91
|
return Ok(CmdResult::from_json(amb, args.json));
|
|
92
92
|
}
|
|
93
|
-
let outcome = messaging::send_message(&selected.run_workspace, &target, &content, &opts)?;
|
|
94
|
-
let mut value = delivery_outcome_json(&outcome, &target, &content, &opts);
|
|
93
|
+
let mut outcome = messaging::send_message(&selected.run_workspace, &target, &content, &opts)?;
|
|
95
94
|
if opts.watch_result {
|
|
95
|
+
outcome = observe_initial_delivery_for_watch(&selected, &target, &outcome, &opts)?;
|
|
96
|
+
}
|
|
97
|
+
let mut value = delivery_outcome_json(&outcome, &target, &content, &opts);
|
|
98
|
+
if opts.watch_result && initial_delivery_allows_watch(outcome.status) {
|
|
96
99
|
if let Some(obj) = value.as_object_mut() {
|
|
97
100
|
obj.insert("watch".to_string(), watch_notice_json(&target, &opts));
|
|
98
101
|
}
|
|
@@ -575,6 +578,48 @@ fn primary_delivery_succeeded(status: DeliveryStatus) -> bool {
|
|
|
575
578
|
)
|
|
576
579
|
}
|
|
577
580
|
|
|
581
|
+
fn initial_delivery_allows_watch(status: DeliveryStatus) -> bool {
|
|
582
|
+
matches!(
|
|
583
|
+
status,
|
|
584
|
+
DeliveryStatus::Delivered | DeliveryStatus::AlreadyDelivered
|
|
585
|
+
)
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
fn observe_initial_delivery_for_watch(
|
|
589
|
+
selected: &crate::state::selector::SelectedTeam,
|
|
590
|
+
target: &MessageTarget,
|
|
591
|
+
outcome: &DeliveryOutcome,
|
|
592
|
+
opts: &SendOptions,
|
|
593
|
+
) -> Result<DeliveryOutcome, CliError> {
|
|
594
|
+
if !matches!(target, MessageTarget::Single(agent) if !agent.is_empty()) {
|
|
595
|
+
return Ok(outcome.clone());
|
|
596
|
+
}
|
|
597
|
+
if !matches!(outcome.status, DeliveryStatus::Queued) {
|
|
598
|
+
return Ok(outcome.clone());
|
|
599
|
+
}
|
|
600
|
+
let Some(message_id) = outcome.message_id.as_deref() else {
|
|
601
|
+
return Ok(outcome.clone());
|
|
602
|
+
};
|
|
603
|
+
let store = crate::message_store::MessageStore::open(&selected.run_workspace)
|
|
604
|
+
.map_err(|e| CliError::Runtime(e.to_string()))?;
|
|
605
|
+
let transport = crate::lifecycle::restart::lifecycle_worker_tmux_backend_for_selected_state(
|
|
606
|
+
&selected.run_workspace,
|
|
607
|
+
opts.team.as_ref().map(TeamKey::as_str),
|
|
608
|
+
)
|
|
609
|
+
.map_err(|e| CliError::Runtime(e.to_string()))?;
|
|
610
|
+
let event_log = crate::event_log::EventLog::new(&selected.run_workspace);
|
|
611
|
+
let state = selected_state_with_active_key(selected);
|
|
612
|
+
crate::messaging::delivery::deliver_pending_message(
|
|
613
|
+
&selected.run_workspace,
|
|
614
|
+
&store,
|
|
615
|
+
&transport,
|
|
616
|
+
message_id,
|
|
617
|
+
&event_log,
|
|
618
|
+
&state,
|
|
619
|
+
)
|
|
620
|
+
.map_err(CliError::from)
|
|
621
|
+
}
|
|
622
|
+
|
|
578
623
|
fn is_business_reject_text(error: &str) -> bool {
|
|
579
624
|
let lower = error.to_ascii_lowercase();
|
|
580
625
|
[
|
|
@@ -668,6 +713,8 @@ fn delivery_outcome_json(
|
|
|
668
713
|
json!({
|
|
669
714
|
"ok": outcome.ok,
|
|
670
715
|
"status": delivery_status_wire(outcome.status),
|
|
716
|
+
"delivery_status": api_delivery_status(outcome),
|
|
717
|
+
"delivered": delivery_proven(outcome.status),
|
|
671
718
|
"target": target_wire,
|
|
672
719
|
"agent_id": first_target(target),
|
|
673
720
|
"content_length_bytes": content.len(),
|
|
@@ -681,6 +728,26 @@ fn delivery_outcome_json(
|
|
|
681
728
|
})
|
|
682
729
|
}
|
|
683
730
|
|
|
731
|
+
fn api_delivery_status(outcome: &DeliveryOutcome) -> &'static str {
|
|
732
|
+
if delivery_proven(outcome.status) {
|
|
733
|
+
return "delivered";
|
|
734
|
+
}
|
|
735
|
+
if matches!(outcome.status, DeliveryStatus::Queued) && outcome.message_status.0 == "accepted" {
|
|
736
|
+
return "pending";
|
|
737
|
+
}
|
|
738
|
+
delivery_status_wire(outcome.status)
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
fn delivery_proven(status: DeliveryStatus) -> bool {
|
|
742
|
+
matches!(
|
|
743
|
+
status,
|
|
744
|
+
DeliveryStatus::Delivered
|
|
745
|
+
| DeliveryStatus::AlreadyDelivered
|
|
746
|
+
| DeliveryStatus::BroadcastDelivered
|
|
747
|
+
| DeliveryStatus::FanoutDelivered
|
|
748
|
+
)
|
|
749
|
+
}
|
|
750
|
+
|
|
684
751
|
fn add_send_reminder_if_ok(value: &mut Value) {
|
|
685
752
|
if value.get("ok").and_then(Value::as_bool) != Some(true) {
|
|
686
753
|
return;
|
|
@@ -99,6 +99,7 @@ use rusqlite::params;
|
|
|
99
99
|
"tasks": tasks,
|
|
100
100
|
"messages": message_counts(&conn, owner_team_id)?,
|
|
101
101
|
"queued_messages": queued_messages(&conn, owner_team_id, 8)?,
|
|
102
|
+
"pending_leader_notifications": pending_leader_notifications(&conn, owner_team_id, 8)?,
|
|
102
103
|
"results": result_counts(&conn, owner_team_id)?,
|
|
103
104
|
"latest_results": latest_result_summaries(&store, owner_team_id)?,
|
|
104
105
|
"readiness": readiness,
|
|
@@ -639,6 +640,67 @@ use rusqlite::params;
|
|
|
639
640
|
Ok(Value::Array(values))
|
|
640
641
|
}
|
|
641
642
|
|
|
643
|
+
/// 0.5.5 gate054 round-2: leader notifications that were refused with
|
|
644
|
+
/// `rebind_required` (status=failed, error=leader_not_attached) sit as
|
|
645
|
+
/// failed rows in the store; without a dedicated status field the
|
|
646
|
+
/// operator sees only `messages.failed=N` and cannot tell that the
|
|
647
|
+
/// notifications are waiting for a rebind. This field surfaces them
|
|
648
|
+
/// alongside `queued_messages` so `attach-leader` / `takeover` is
|
|
649
|
+
/// visibly the fix. Once the pane is rebound the requeue path flips
|
|
650
|
+
/// each row back to `status=accepted` and it drops out of this list.
|
|
651
|
+
fn pending_leader_notifications(
|
|
652
|
+
conn: &rusqlite::Connection,
|
|
653
|
+
owner_team_id: Option<&str>,
|
|
654
|
+
limit: usize,
|
|
655
|
+
) -> Result<Value, CliError> {
|
|
656
|
+
let limit = i64::try_from(limit).unwrap_or(i64::MAX);
|
|
657
|
+
let sql = match owner_team_id {
|
|
658
|
+
Some(_) => {
|
|
659
|
+
"select message_id, sender, status, error, created_at, delivery_attempts
|
|
660
|
+
from messages
|
|
661
|
+
where recipient = 'leader'
|
|
662
|
+
and status = 'failed'
|
|
663
|
+
and error = 'leader_not_attached'
|
|
664
|
+
and owner_team_id = ?1
|
|
665
|
+
order by created_at desc
|
|
666
|
+
limit ?2"
|
|
667
|
+
}
|
|
668
|
+
None => {
|
|
669
|
+
"select message_id, sender, status, error, created_at, delivery_attempts
|
|
670
|
+
from messages
|
|
671
|
+
where recipient = 'leader'
|
|
672
|
+
and status = 'failed'
|
|
673
|
+
and error = 'leader_not_attached'
|
|
674
|
+
order by created_at desc
|
|
675
|
+
limit ?1"
|
|
676
|
+
}
|
|
677
|
+
};
|
|
678
|
+
let mut stmt = conn
|
|
679
|
+
.prepare(sql)
|
|
680
|
+
.map_err(|e| CliError::Runtime(e.to_string()))?;
|
|
681
|
+
let map_row = |row: &rusqlite::Row<'_>| {
|
|
682
|
+
Ok(json!({
|
|
683
|
+
"message_id": row.get::<_, String>(0)?,
|
|
684
|
+
"sender": row.get::<_, Option<String>>(1)?,
|
|
685
|
+
"status": row.get::<_, String>(2)?,
|
|
686
|
+
"error": row.get::<_, Option<String>>(3)?,
|
|
687
|
+
"created_at": row.get::<_, Option<String>>(4)?,
|
|
688
|
+
"delivery_attempts": row.get::<_, i64>(5)?,
|
|
689
|
+
"channel": "rebind_required",
|
|
690
|
+
"action": "run team-agent attach-leader or team-agent takeover",
|
|
691
|
+
}))
|
|
692
|
+
};
|
|
693
|
+
let rows = match owner_team_id {
|
|
694
|
+
Some(team) => stmt.query_map(params![team, limit], map_row),
|
|
695
|
+
None => stmt.query_map(params![limit], map_row),
|
|
696
|
+
}
|
|
697
|
+
.map_err(|e| CliError::Runtime(e.to_string()))?;
|
|
698
|
+
let values = rows
|
|
699
|
+
.collect::<Result<Vec<_>, _>>()
|
|
700
|
+
.map_err(|e| CliError::Runtime(e.to_string()))?;
|
|
701
|
+
Ok(Value::Array(values))
|
|
702
|
+
}
|
|
703
|
+
|
|
642
704
|
/// 0.4.x: slim default compact payload — exactly 7 top-level fields.
|
|
643
705
|
/// Diagnostic detail moves to `--detail`. Plan:
|
|
644
706
|
/// /Users/alauda/Documents/code/team-agent-public/.team/artifacts/status-compact-plan.md
|
|
@@ -511,33 +511,26 @@ use super::*;
|
|
|
511
511
|
}
|
|
512
512
|
|
|
513
513
|
#[test]
|
|
514
|
-
fn
|
|
515
|
-
//
|
|
516
|
-
//
|
|
517
|
-
// (send.py:322-337). That dict MUST survive verbatim into CmdResult Json output.
|
|
514
|
+
fn cmd_send_watch_result_does_not_register_before_delivery() {
|
|
515
|
+
// 0.5.x send contract: --watch-result may only advertise a watcher after
|
|
516
|
+
// initial worker delivery is physically proven.
|
|
518
517
|
let r = cmd_send(&send_args_fixture()).expect("cmd_send returns CmdResult");
|
|
519
518
|
let v = match r.output {
|
|
520
519
|
CmdOutput::Json(v) => v,
|
|
521
520
|
other => panic!("expected Json, got {other:?}"),
|
|
522
521
|
};
|
|
523
|
-
|
|
524
|
-
.get("
|
|
525
|
-
|
|
526
|
-
assert_eq!(
|
|
527
|
-
watch.get("status").and_then(|s| s.as_str()),
|
|
528
|
-
Some("registered"),
|
|
529
|
-
"registered-watcher notice status must be exactly 'registered'"
|
|
522
|
+
assert!(
|
|
523
|
+
v.get("delivery_status").and_then(|s| s.as_str()).is_some(),
|
|
524
|
+
"send output must expose delivery_status; got {v}"
|
|
530
525
|
);
|
|
531
|
-
assert!(watch.get("watcher_id").is_some(), "watch notice carries watcher_id");
|
|
532
526
|
assert_eq!(
|
|
533
|
-
|
|
534
|
-
Some(
|
|
535
|
-
"
|
|
527
|
+
v.get("delivered").and_then(|s| s.as_bool()),
|
|
528
|
+
Some(false),
|
|
529
|
+
"undelivered send outcome must not look delivered"
|
|
536
530
|
);
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
watch
|
|
540
|
-
Some("Team Agent will collect the result and notify the leader when this task reports completion.")
|
|
531
|
+
assert!(
|
|
532
|
+
!v.as_object().unwrap().contains_key("watch"),
|
|
533
|
+
"watch_result:true must not attach result['watch'] before delivery; got {v}"
|
|
541
534
|
);
|
|
542
535
|
}
|
|
543
536
|
|
|
@@ -553,6 +553,19 @@ impl Coordinator {
|
|
|
553
553
|
}),
|
|
554
554
|
)?;
|
|
555
555
|
}
|
|
556
|
+
for mismatch in &report.identity_mismatches {
|
|
557
|
+
event_log.write(
|
|
558
|
+
"provider.session.identity_mismatch",
|
|
559
|
+
serde_json::json!({
|
|
560
|
+
"agent_id": mismatch.agent_id,
|
|
561
|
+
"expected_agent_id": mismatch.expected_agent_id,
|
|
562
|
+
"embedded_agent_id": mismatch.embedded_agent_id,
|
|
563
|
+
"session_id": mismatch.session_id,
|
|
564
|
+
"rollout_path": mismatch.rollout_path,
|
|
565
|
+
"spawn_cwd": mismatch.spawn_cwd,
|
|
566
|
+
}),
|
|
567
|
+
)?;
|
|
568
|
+
}
|
|
556
569
|
for ambiguous in report.ambiguous {
|
|
557
570
|
let candidate_count = report
|
|
558
571
|
.candidate_count_by_agent
|
|
@@ -374,6 +374,8 @@ fn spawn_agents(
|
|
|
374
374
|
);
|
|
375
375
|
}
|
|
376
376
|
}
|
|
377
|
+
let spawn_epoch = u64::try_from(started.len()).unwrap_or(u64::MAX);
|
|
378
|
+
let spawned_at = spawn_timestamp_for_agent(u32::try_from(spawn_epoch).unwrap_or(u32::MAX));
|
|
377
379
|
// E6 层1 实证3 + 诊断留痕:落最终 worker argv(spawn 前的真实形态)。
|
|
378
380
|
// 任何"--session-id 预定 UUID 没生效"必须能从 events.jsonl 回答:argv 里到底有没有它。
|
|
379
381
|
// 抽出 --session-id 值单列,方便和盘上 ~/.claude/projects/<cwd> 实际落的 UUID 对账。
|
|
@@ -393,6 +395,10 @@ fn spawn_agents(
|
|
|
393
395
|
"argv": plan.argv,
|
|
394
396
|
"session_id_in_argv": session_id_in_argv,
|
|
395
397
|
"expected_session_id": plan.expected_session_id.as_ref().map(|s| s.as_str()),
|
|
398
|
+
"spawn_cwd": workspace.to_string_lossy(),
|
|
399
|
+
"spawned_at": spawned_at.as_str(),
|
|
400
|
+
"source": "launch",
|
|
401
|
+
"spawn_epoch": spawn_epoch,
|
|
396
402
|
}),
|
|
397
403
|
);
|
|
398
404
|
}
|
|
@@ -473,6 +479,7 @@ fn spawn_agents(
|
|
|
473
479
|
agent_id,
|
|
474
480
|
start_mode: StartMode::Fresh,
|
|
475
481
|
target: spawn.pane_id.as_str().to_string(),
|
|
482
|
+
spawned_at,
|
|
476
483
|
session_id: None,
|
|
477
484
|
rollout_path: None,
|
|
478
485
|
pending_session_id: plan.expected_session_id.clone(),
|
|
@@ -872,7 +879,9 @@ fn persist_spawn_agent_state(
|
|
|
872
879
|
continue;
|
|
873
880
|
}
|
|
874
881
|
let pane_pid = pane_pids_by_agent.get(id).copied();
|
|
875
|
-
let spawned_at =
|
|
882
|
+
let spawned_at = started_agent
|
|
883
|
+
.map(|started| started.spawned_at.clone())
|
|
884
|
+
.unwrap_or_else(|| spawn_timestamp_for_agent(spawn_index));
|
|
876
885
|
spawn_index = spawn_index.saturating_add(1);
|
|
877
886
|
agents.insert(
|
|
878
887
|
id.to_string(),
|
|
@@ -2970,11 +2979,10 @@ pub fn quick_start_in_workspace_with_display_and_backend(
|
|
|
2970
2979
|
// `state.transport.shim.pipe_ready = true` marker so its
|
|
2971
2980
|
// `conpty_pipe_ready` gate opens.
|
|
2972
2981
|
#[cfg(windows)]
|
|
2973
|
-
let state_value =
|
|
2974
|
-
&workspace
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
.and_then(|t| serde_json::from_str::<serde_json::Value>(&t).ok());
|
|
2982
|
+
let state_value =
|
|
2983
|
+
std::fs::read_to_string(crate::state::persist::runtime_state_path(&workspace))
|
|
2984
|
+
.ok()
|
|
2985
|
+
.and_then(|t| serde_json::from_str::<serde_json::Value>(&t).ok());
|
|
2978
2986
|
#[cfg(windows)]
|
|
2979
2987
|
let input = match state_value.as_ref() {
|
|
2980
2988
|
Some(v) => input.with_state(Some(v)),
|
|
@@ -143,6 +143,7 @@ pub(crate) fn start_agent_at_paths(
|
|
|
143
143
|
if let Ok(spec) = load_team_spec(spec_workspace) {
|
|
144
144
|
write_team_state(spec_workspace, &spec, &state)?;
|
|
145
145
|
}
|
|
146
|
+
replay_worker_target_missing_messages(workspace, agent_id, &team_key, &state, transport)?;
|
|
146
147
|
let coordinator_started = start_coordinator_for_workspace(workspace)?;
|
|
147
148
|
let target = format!("{}:{window}", session_name.as_str());
|
|
148
149
|
write_start_agent_noop_event(workspace, agent_id, &target, coordinator_started)?;
|
|
@@ -256,7 +257,21 @@ pub(crate) fn start_agent_at_paths(
|
|
|
256
257
|
// after spawn" race with coordinator and no double source of truth.
|
|
257
258
|
crate::lifecycle::launch::annotate_runtime_tmux_endpoint(&mut state, transport, workspace);
|
|
258
259
|
let team_key = restart_projection_team_key(&state, team);
|
|
259
|
-
|
|
260
|
+
let skip_capture_backfill = if matches!(
|
|
261
|
+
start_mode,
|
|
262
|
+
StartMode::Fresh | StartMode::FreshAfterMissingRollout
|
|
263
|
+
) {
|
|
264
|
+
vec![agent_id.as_str()]
|
|
265
|
+
} else {
|
|
266
|
+
Vec::new()
|
|
267
|
+
};
|
|
268
|
+
save_restart_projected_state_with_capture_backfill_skip(
|
|
269
|
+
workspace,
|
|
270
|
+
&mut state,
|
|
271
|
+
&team_key,
|
|
272
|
+
&skip_capture_backfill,
|
|
273
|
+
&[agent_id.as_str()],
|
|
274
|
+
)?;
|
|
260
275
|
write_start_agent_start_event(
|
|
261
276
|
workspace,
|
|
262
277
|
agent_id,
|
|
@@ -268,6 +283,7 @@ pub(crate) fn start_agent_at_paths(
|
|
|
268
283
|
spawn_session_id,
|
|
269
284
|
tmux_start_mode_for_spawn(&spawn, into_existing_session),
|
|
270
285
|
)?;
|
|
286
|
+
replay_worker_target_missing_messages(workspace, agent_id, &team_key, &state, transport)?;
|
|
271
287
|
let coordinator_started = start_coordinator_for_workspace(workspace)?;
|
|
272
288
|
Ok(StartAgentOutcome::Running {
|
|
273
289
|
env: AgentActionEnvelope {
|
|
@@ -283,6 +299,33 @@ pub(crate) fn start_agent_at_paths(
|
|
|
283
299
|
})
|
|
284
300
|
}
|
|
285
301
|
|
|
302
|
+
fn replay_worker_target_missing_messages(
|
|
303
|
+
workspace: &Path,
|
|
304
|
+
agent_id: &AgentId,
|
|
305
|
+
team_key: &str,
|
|
306
|
+
state: &serde_json::Value,
|
|
307
|
+
transport: &dyn crate::transport::Transport,
|
|
308
|
+
) -> Result<(), LifecycleError> {
|
|
309
|
+
let store = crate::message_store::MessageStore::open(workspace)
|
|
310
|
+
.map_err(|e| LifecycleError::StatePersist(e.to_string()))?;
|
|
311
|
+
let event_log = crate::event_log::EventLog::new(workspace);
|
|
312
|
+
let ids = crate::messaging::delivery::requeue_worker_target_missing_messages(
|
|
313
|
+
workspace,
|
|
314
|
+
&store,
|
|
315
|
+
&event_log,
|
|
316
|
+
agent_id.as_str(),
|
|
317
|
+
Some(team_key),
|
|
318
|
+
)
|
|
319
|
+
.map_err(|e| LifecycleError::StatePersist(e.to_string()))?;
|
|
320
|
+
if !ids.is_empty() {
|
|
321
|
+
crate::messaging::delivery::deliver_pending_messages(
|
|
322
|
+
workspace, state, transport, &event_log,
|
|
323
|
+
)
|
|
324
|
+
.map_err(|e| LifecycleError::StatePersist(e.to_string()))?;
|
|
325
|
+
}
|
|
326
|
+
Ok(())
|
|
327
|
+
}
|
|
328
|
+
|
|
286
329
|
fn verify_spawned_pane_matches_target(
|
|
287
330
|
transport: &dyn crate::transport::Transport,
|
|
288
331
|
pane: &crate::transport::PaneId,
|
|
@@ -507,7 +507,10 @@ pub(crate) fn lifecycle_worker_tmux_backend_for_selected_state(
|
|
|
507
507
|
// are expected to migrate to `lifecycle_worker_transport_for_selected_state`
|
|
508
508
|
// during Batch 2/3.
|
|
509
509
|
if let Some(state_ref) = state.as_ref() {
|
|
510
|
-
if let Some(kind) = state_ref
|
|
510
|
+
if let Some(kind) = state_ref
|
|
511
|
+
.pointer("/transport/kind")
|
|
512
|
+
.and_then(|v| v.as_str())
|
|
513
|
+
{
|
|
511
514
|
if kind.eq_ignore_ascii_case("conpty") {
|
|
512
515
|
return Err(LifecycleError::TeamSelect(format!(
|
|
513
516
|
"backend_kind_mismatch: state.transport.kind={kind:?} but the legacy \
|
|
@@ -581,11 +584,28 @@ pub(super) fn save_restart_projected_state(
|
|
|
581
584
|
state: &mut serde_json::Value,
|
|
582
585
|
team_key: &str,
|
|
583
586
|
topology_authority_agent_ids: &[&str],
|
|
587
|
+
) -> Result<(), LifecycleError> {
|
|
588
|
+
save_restart_projected_state_with_capture_backfill_skip(
|
|
589
|
+
workspace,
|
|
590
|
+
state,
|
|
591
|
+
team_key,
|
|
592
|
+
&[],
|
|
593
|
+
topology_authority_agent_ids,
|
|
594
|
+
)
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
pub(super) fn save_restart_projected_state_with_capture_backfill_skip(
|
|
598
|
+
workspace: &Path,
|
|
599
|
+
state: &mut serde_json::Value,
|
|
600
|
+
team_key: &str,
|
|
601
|
+
skip_capture_backfill_agent_ids: &[&str],
|
|
602
|
+
topology_authority_agent_ids: &[&str],
|
|
584
603
|
) -> Result<(), LifecycleError> {
|
|
585
604
|
sync_restart_team_projections(state, team_key);
|
|
586
|
-
crate::state::projection::
|
|
605
|
+
crate::state::projection::save_team_scoped_state_with_lifecycle_topology_authority_and_capture_backfill_skip(
|
|
587
606
|
workspace,
|
|
588
607
|
state,
|
|
608
|
+
skip_capture_backfill_agent_ids,
|
|
589
609
|
topology_authority_agent_ids,
|
|
590
610
|
)
|
|
591
611
|
.map_err(|e| LifecycleError::StatePersist(e.to_string()))
|
|
@@ -739,6 +759,37 @@ pub(super) struct BackingProbeResult {
|
|
|
739
759
|
pub checked_paths: Vec<PathBuf>,
|
|
740
760
|
}
|
|
741
761
|
|
|
762
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
763
|
+
pub(crate) struct SessionIdentityProbeResult {
|
|
764
|
+
pub identity_ok: Option<bool>,
|
|
765
|
+
pub embedded_agent_id: Option<String>,
|
|
766
|
+
pub rollout_path: Option<PathBuf>,
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
pub(crate) fn session_identity_probe_for_agent(
|
|
770
|
+
agent_id: &AgentId,
|
|
771
|
+
_provider: Provider,
|
|
772
|
+
rollout_path: Option<&RolloutPath>,
|
|
773
|
+
) -> SessionIdentityProbeResult {
|
|
774
|
+
let Some(path) = rollout_path.map(RolloutPath::as_path) else {
|
|
775
|
+
return SessionIdentityProbeResult {
|
|
776
|
+
identity_ok: None,
|
|
777
|
+
embedded_agent_id: None,
|
|
778
|
+
rollout_path: None,
|
|
779
|
+
};
|
|
780
|
+
};
|
|
781
|
+
let embedded_agent_id =
|
|
782
|
+
crate::provider::session_scan::common::rollout_path_embedded_team_agent_worker_id(path);
|
|
783
|
+
let identity_ok = embedded_agent_id
|
|
784
|
+
.as_deref()
|
|
785
|
+
.map(|embedded| embedded == agent_id.as_str());
|
|
786
|
+
SessionIdentityProbeResult {
|
|
787
|
+
identity_ok,
|
|
788
|
+
embedded_agent_id,
|
|
789
|
+
rollout_path: Some(path.to_path_buf()),
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
|
|
742
793
|
pub(super) fn resume_backing_probe_for_agent(
|
|
743
794
|
workspace: &Path,
|
|
744
795
|
agent_id: &AgentId,
|
|
@@ -467,10 +467,21 @@ pub fn restart_with_transport_with_session_convergence_deadline(
|
|
|
467
467
|
.iter()
|
|
468
468
|
.map(|agent| agent.agent_id.as_str().to_string()),
|
|
469
469
|
);
|
|
470
|
-
|
|
470
|
+
let capture_backfill_skip_agent_ids = successful_agents
|
|
471
|
+
.iter()
|
|
472
|
+
.filter(|agent| {
|
|
473
|
+
matches!(
|
|
474
|
+
agent.restart_mode,
|
|
475
|
+
StartMode::Fresh | StartMode::FreshAfterMissingRollout
|
|
476
|
+
)
|
|
477
|
+
})
|
|
478
|
+
.map(|agent| agent.agent_id.as_str().to_string())
|
|
479
|
+
.collect::<Vec<_>>();
|
|
480
|
+
save_restart_state_with_lifecycle_topology_authority_and_capture_backfill_skip(
|
|
471
481
|
&selected.run_workspace,
|
|
472
482
|
&mut state,
|
|
473
483
|
&selected.team_key,
|
|
484
|
+
&capture_backfill_skip_agent_ids,
|
|
474
485
|
&topology_authority_agent_ids,
|
|
475
486
|
)?;
|
|
476
487
|
if fatal_resume_failure {
|
|
@@ -572,6 +583,11 @@ pub fn restart_with_transport_with_session_convergence_deadline(
|
|
|
572
583
|
&session_id,
|
|
573
584
|
agent_rollout_path(&agent).as_ref(),
|
|
574
585
|
);
|
|
586
|
+
let identity_probe = session_identity_probe_for_agent(
|
|
587
|
+
&decision.agent_id,
|
|
588
|
+
provider,
|
|
589
|
+
agent_rollout_path(&agent).as_ref(),
|
|
590
|
+
);
|
|
575
591
|
write_restart_resume_postflight_event(
|
|
576
592
|
&selected.run_workspace,
|
|
577
593
|
&decision.agent_id,
|
|
@@ -579,11 +595,26 @@ pub fn restart_with_transport_with_session_convergence_deadline(
|
|
|
579
595
|
probe.exists,
|
|
580
596
|
&probe.checked_paths,
|
|
581
597
|
/* recaptured = */ false,
|
|
598
|
+
identity_probe.identity_ok,
|
|
599
|
+
identity_probe.embedded_agent_id.as_deref(),
|
|
582
600
|
)?;
|
|
583
|
-
if probe.exists {
|
|
601
|
+
if probe.exists && identity_probe.identity_ok != Some(false) {
|
|
584
602
|
survivors.push(decision);
|
|
585
603
|
continue;
|
|
586
604
|
}
|
|
605
|
+
if identity_probe.identity_ok == Some(false) {
|
|
606
|
+
let phase = "resume_postflight";
|
|
607
|
+
let error = "session_identity_mismatch_after_restart".to_string();
|
|
608
|
+
mark_agent_restart_failed(&mut state, &decision, &error);
|
|
609
|
+
let _ = write_restart_agent_failed_event(
|
|
610
|
+
&selected.run_workspace,
|
|
611
|
+
&decision,
|
|
612
|
+
phase,
|
|
613
|
+
&error,
|
|
614
|
+
);
|
|
615
|
+
postflight_failed.push(restart_failed_agent(&decision, phase, error));
|
|
616
|
+
continue;
|
|
617
|
+
}
|
|
587
618
|
// 0.4.6 tuple-atomic contract: backing went missing between
|
|
588
619
|
// preflight and post-spawn. Clear the FULL authoritative tuple
|
|
589
620
|
// (including session_id) and persist the old provider id only
|
|
@@ -640,6 +671,11 @@ pub fn restart_with_transport_with_session_convergence_deadline(
|
|
|
640
671
|
&session_id,
|
|
641
672
|
agent_rollout_path(&agent_after).as_ref(),
|
|
642
673
|
);
|
|
674
|
+
let identity_probe_after = session_identity_probe_for_agent(
|
|
675
|
+
&decision.agent_id,
|
|
676
|
+
provider,
|
|
677
|
+
agent_rollout_path(&agent_after).as_ref(),
|
|
678
|
+
);
|
|
643
679
|
write_restart_resume_postflight_event(
|
|
644
680
|
&selected.run_workspace,
|
|
645
681
|
&decision.agent_id,
|
|
@@ -647,14 +683,20 @@ pub fn restart_with_transport_with_session_convergence_deadline(
|
|
|
647
683
|
probe_after.exists,
|
|
648
684
|
&probe_after.checked_paths,
|
|
649
685
|
/* recaptured = */ true,
|
|
686
|
+
identity_probe_after.identity_ok,
|
|
687
|
+
identity_probe_after.embedded_agent_id.as_deref(),
|
|
650
688
|
)?;
|
|
651
|
-
if probe_after.exists {
|
|
689
|
+
if probe_after.exists && identity_probe_after.identity_ok != Some(false) {
|
|
652
690
|
survivors.push(decision);
|
|
653
691
|
continue;
|
|
654
692
|
}
|
|
655
693
|
// Still missing. Demote the agent.
|
|
656
694
|
let phase = "resume_postflight";
|
|
657
|
-
let error =
|
|
695
|
+
let error = if identity_probe_after.identity_ok == Some(false) {
|
|
696
|
+
"session_identity_mismatch_after_restart".to_string()
|
|
697
|
+
} else {
|
|
698
|
+
"session_backing_store_missing_after_restart".to_string()
|
|
699
|
+
};
|
|
658
700
|
mark_agent_restart_failed(&mut state, &decision, &error);
|
|
659
701
|
let _ =
|
|
660
702
|
write_restart_agent_failed_event(&selected.run_workspace, &decision, phase, &error);
|
|
@@ -1327,8 +1369,38 @@ fn save_restart_state_with_lifecycle_topology_authority(
|
|
|
1327
1369
|
team_key: &str,
|
|
1328
1370
|
agent_ids: &[String],
|
|
1329
1371
|
) -> Result<(), LifecycleError> {
|
|
1330
|
-
|
|
1331
|
-
|
|
1372
|
+
save_restart_state_with_lifecycle_topology_authority_and_capture_backfill_skip(
|
|
1373
|
+
workspace,
|
|
1374
|
+
state,
|
|
1375
|
+
team_key,
|
|
1376
|
+
&[],
|
|
1377
|
+
agent_ids,
|
|
1378
|
+
)
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
fn save_restart_state_with_lifecycle_topology_authority_and_capture_backfill_skip(
|
|
1382
|
+
workspace: &Path,
|
|
1383
|
+
state: &mut serde_json::Value,
|
|
1384
|
+
team_key: &str,
|
|
1385
|
+
skip_capture_backfill_agent_ids: &[String],
|
|
1386
|
+
topology_agent_ids: &[String],
|
|
1387
|
+
) -> Result<(), LifecycleError> {
|
|
1388
|
+
let skip_capture_backfill_agent_ids = skip_capture_backfill_agent_ids
|
|
1389
|
+
.iter()
|
|
1390
|
+
.map(String::as_str)
|
|
1391
|
+
.collect::<Vec<_>>();
|
|
1392
|
+
let topology_agent_ids = topology_agent_ids
|
|
1393
|
+
.iter()
|
|
1394
|
+
.map(String::as_str)
|
|
1395
|
+
.collect::<Vec<_>>();
|
|
1396
|
+
sync_restart_team_projections(state, team_key);
|
|
1397
|
+
crate::state::projection::save_team_scoped_state_with_lifecycle_topology_authority_and_capture_backfill_skip(
|
|
1398
|
+
workspace,
|
|
1399
|
+
state,
|
|
1400
|
+
&skip_capture_backfill_agent_ids,
|
|
1401
|
+
&topology_agent_ids,
|
|
1402
|
+
)
|
|
1403
|
+
.map_err(|e| LifecycleError::StatePersist(e.to_string()))
|
|
1332
1404
|
}
|
|
1333
1405
|
|
|
1334
1406
|
fn save_restart_session_repairs(
|
|
@@ -1676,6 +1748,8 @@ fn write_restart_resume_postflight_event(
|
|
|
1676
1748
|
exists: bool,
|
|
1677
1749
|
checked_paths: &[std::path::PathBuf],
|
|
1678
1750
|
recaptured: bool,
|
|
1751
|
+
identity_ok: Option<bool>,
|
|
1752
|
+
embedded_agent_id: Option<&str>,
|
|
1679
1753
|
) -> Result<(), LifecycleError> {
|
|
1680
1754
|
crate::event_log::EventLog::new(workspace)
|
|
1681
1755
|
.write(
|
|
@@ -1689,6 +1763,8 @@ fn write_restart_resume_postflight_event(
|
|
|
1689
1763
|
.map(|p| p.to_string_lossy().into_owned())
|
|
1690
1764
|
.collect::<Vec<_>>(),
|
|
1691
1765
|
"recaptured": recaptured,
|
|
1766
|
+
"identity_ok": identity_ok,
|
|
1767
|
+
"embedded_agent_id": embedded_agent_id,
|
|
1692
1768
|
}),
|
|
1693
1769
|
)
|
|
1694
1770
|
.map(|_| ())
|
|
@@ -1878,6 +1954,32 @@ fn write_restart_resume_decision_event(
|
|
|
1878
1954
|
);
|
|
1879
1955
|
}
|
|
1880
1956
|
}
|
|
1957
|
+
if let crate::provider::session::ResumeRefusalReason::SessionIdentityMismatch {
|
|
1958
|
+
expected_agent_id,
|
|
1959
|
+
embedded_agent_id,
|
|
1960
|
+
session_id,
|
|
1961
|
+
rollout_path,
|
|
1962
|
+
} = structured
|
|
1963
|
+
{
|
|
1964
|
+
obj.insert(
|
|
1965
|
+
"expected_agent_id".to_string(),
|
|
1966
|
+
serde_json::json!(expected_agent_id),
|
|
1967
|
+
);
|
|
1968
|
+
obj.insert(
|
|
1969
|
+
"embedded_agent_id".to_string(),
|
|
1970
|
+
serde_json::json!(embedded_agent_id),
|
|
1971
|
+
);
|
|
1972
|
+
obj.insert(
|
|
1973
|
+
"poisoned_session_id".to_string(),
|
|
1974
|
+
serde_json::json!(session_id),
|
|
1975
|
+
);
|
|
1976
|
+
if let Some(path) = rollout_path {
|
|
1977
|
+
obj.insert(
|
|
1978
|
+
"rollout_path".to_string(),
|
|
1979
|
+
serde_json::json!(path.to_string_lossy()),
|
|
1980
|
+
);
|
|
1981
|
+
}
|
|
1982
|
+
}
|
|
1881
1983
|
}
|
|
1882
1984
|
}
|
|
1883
1985
|
}
|