@team-agent/installer 0.5.74 → 0.5.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -1
- package/Cargo.toml +1 -1
- package/crates/team-agent/src/layout/worker_env.rs +366 -0
- package/crates/team-agent/src/leader/lease.rs +228 -32
- package/crates/team-agent/src/leader/registry.rs +177 -19
- package/crates/team-agent/src/lifecycle/launch/pi_mcp.rs +12 -1
- package/crates/team-agent/src/lifecycle/launch/quick_start.rs +1820 -60
- package/crates/team-agent/src/lifecycle/launch/spawn.rs +4 -0
- package/crates/team-agent/src/lifecycle/launch/spec_state.rs +138 -13
- package/crates/team-agent/src/lifecycle/restart/common.rs +3 -0
- package/crates/team-agent/src/lifecycle/tests/worker_spawn_env_red.rs +110 -31
- package/crates/team-agent/src/messaging/delivery.rs +200 -29
- package/crates/team-agent/src/messaging/leader_channel.rs +12 -3
- package/crates/team-agent/src/state/persist.rs +166 -6
- package/crates/team-agent/src/state/repository.rs +36 -0
- package/crates/team-agent/src/tmux_backend/tests.rs +350 -7
- package/crates/team-agent/src/tmux_backend.rs +217 -26
- package/crates/team-agent/src/transport/test_support.rs +18 -3
- package/crates/team-agent/src/transport.rs +99 -0
- package/package.json +4 -4
|
@@ -31,9 +31,9 @@ use crate::provider::wire::{
|
|
|
31
31
|
is_claude_family, parse_canonical_provider, parse_provider, provider_wire,
|
|
32
32
|
};
|
|
33
33
|
use crate::transport::{
|
|
34
|
-
submit_verification_wire, turn_verification_wire, CaptureRange,
|
|
35
|
-
|
|
36
|
-
Target, Transport, WindowName,
|
|
34
|
+
submit_verification_wire, turn_verification_wire, CaptureRange, CaptureSampleOutcome,
|
|
35
|
+
InjectPayload, InjectReport, InjectVerification, InputSurfaceProbe, Key, PaneId, PaneInfo,
|
|
36
|
+
QueryOutcome, SessionName, SubmitObserver, SubmitVerification, Target, Transport, WindowName,
|
|
37
37
|
};
|
|
38
38
|
|
|
39
39
|
use super::helpers::{message_exists, MessageStatusShadow};
|
|
@@ -749,16 +749,20 @@ pub fn deliver_pending_message(
|
|
|
749
749
|
// are preserved byte-for-byte for grep compatibility; new keys are
|
|
750
750
|
// ADDITIONAL.
|
|
751
751
|
let submit_attempts_detail = render_submit_diagnostics(&inject_report);
|
|
752
|
-
|
|
753
|
-
"
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
752
|
+
let mut unverified = serde_json::json!({
|
|
753
|
+
"message_id": message_id,
|
|
754
|
+
"recipient": message.recipient,
|
|
755
|
+
"reason": reason,
|
|
756
|
+
"attempts": inject_report.attempts,
|
|
757
|
+
"submit_attempts_detail": submit_attempts_detail,
|
|
758
|
+
});
|
|
759
|
+
merge_inject_readback_fields(
|
|
760
|
+
&mut unverified,
|
|
761
|
+
&inject_report,
|
|
762
|
+
Some(&target),
|
|
763
|
+
resolved.metadata.as_ref(),
|
|
764
|
+
);
|
|
765
|
+
event_log.write("send.unverified", unverified)?;
|
|
762
766
|
if inject_report.attempts >= u32::from(SEND_RETRY_MAX_ATTEMPTS) {
|
|
763
767
|
store.mark(message_id, "failed", Some("send_unverified_exhausted"))?;
|
|
764
768
|
emit_send_failed_exhausted(
|
|
@@ -1070,7 +1074,7 @@ fn persist_submit_verification(workspace: &Path, message_id: &str, report: &Inje
|
|
|
1070
1074
|
if std::fs::create_dir_all(&dir).is_err() {
|
|
1071
1075
|
return;
|
|
1072
1076
|
}
|
|
1073
|
-
let body = serde_json::json!({
|
|
1077
|
+
let mut body = serde_json::json!({
|
|
1074
1078
|
"message_id": message_id,
|
|
1075
1079
|
"submit_verification": format!("{:?}", report.submit_verification),
|
|
1076
1080
|
"inject_verification": format!("{:?}", report.inject_verification),
|
|
@@ -1078,6 +1082,7 @@ fn persist_submit_verification(workspace: &Path, message_id: &str, report: &Inje
|
|
|
1078
1082
|
"submit_verified": inject_submit_verified(report),
|
|
1079
1083
|
"attempts": report.attempts,
|
|
1080
1084
|
});
|
|
1085
|
+
merge_inject_readback_fields(&mut body, report, None, None);
|
|
1081
1086
|
let _ = std::fs::write(dir.join("submit-verification.json"), format!("{body}\n"));
|
|
1082
1087
|
}
|
|
1083
1088
|
|
|
@@ -1914,6 +1919,116 @@ fn render_submit_diagnostics(report: &InjectReport) -> serde_json::Value {
|
|
|
1914
1919
|
)
|
|
1915
1920
|
}
|
|
1916
1921
|
|
|
1922
|
+
fn json_opt_bool(value: Option<bool>) -> serde_json::Value {
|
|
1923
|
+
match value {
|
|
1924
|
+
Some(flag) => serde_json::Value::Bool(flag),
|
|
1925
|
+
None => serde_json::Value::Null,
|
|
1926
|
+
}
|
|
1927
|
+
}
|
|
1928
|
+
|
|
1929
|
+
fn inject_pane_id(target: &Target) -> Option<&str> {
|
|
1930
|
+
match target {
|
|
1931
|
+
Target::Pane(pane) => Some(pane.as_str()),
|
|
1932
|
+
Target::SessionWindow { .. } => None,
|
|
1933
|
+
}
|
|
1934
|
+
}
|
|
1935
|
+
|
|
1936
|
+
fn merge_inject_readback_fields(
|
|
1937
|
+
event: &mut serde_json::Value,
|
|
1938
|
+
report: &InjectReport,
|
|
1939
|
+
target: Option<&Target>,
|
|
1940
|
+
metadata: Option<&TargetMetadata>,
|
|
1941
|
+
) {
|
|
1942
|
+
if let Some(metadata) = metadata {
|
|
1943
|
+
append_target_metadata(event, metadata);
|
|
1944
|
+
}
|
|
1945
|
+
let Some(obj) = event.as_object_mut() else {
|
|
1946
|
+
return;
|
|
1947
|
+
};
|
|
1948
|
+
let inject_pane = target.and_then(inject_pane_id);
|
|
1949
|
+
if let Some(pane) = inject_pane {
|
|
1950
|
+
obj.insert(
|
|
1951
|
+
"inject_target_pane_id".to_string(),
|
|
1952
|
+
serde_json::json!(pane),
|
|
1953
|
+
);
|
|
1954
|
+
}
|
|
1955
|
+
let resolved_pane = metadata.map(|meta| meta.target_pane_id.as_str());
|
|
1956
|
+
obj.insert(
|
|
1957
|
+
"target_pane_matched".to_string(),
|
|
1958
|
+
json_opt_bool(match (inject_pane, resolved_pane) {
|
|
1959
|
+
(Some(left), Some(right)) => Some(left == right),
|
|
1960
|
+
_ => None,
|
|
1961
|
+
}),
|
|
1962
|
+
);
|
|
1963
|
+
let inject_endpoint = metadata.and_then(|meta| meta.tmux_endpoint.as_deref());
|
|
1964
|
+
// metadata.tmux_endpoint is the transport used to resolve; compare to
|
|
1965
|
+
// itself is tautological. Equality vs inject target id is the pane check
|
|
1966
|
+
// above. Socket equality is recorded when both sides exist on metadata.
|
|
1967
|
+
obj.insert(
|
|
1968
|
+
"resolved_tmux_endpoint_present".to_string(),
|
|
1969
|
+
serde_json::Value::Bool(inject_endpoint.is_some()),
|
|
1970
|
+
);
|
|
1971
|
+
let diag = report.submit_diagnostics.as_ref();
|
|
1972
|
+
obj.insert(
|
|
1973
|
+
"token_seen_before_paste".to_string(),
|
|
1974
|
+
json_opt_bool(diag.and_then(|d| d.token_seen_before_paste)),
|
|
1975
|
+
);
|
|
1976
|
+
obj.insert(
|
|
1977
|
+
"token_seen_after_paste".to_string(),
|
|
1978
|
+
json_opt_bool(diag.and_then(|d| d.token_seen_after_paste)),
|
|
1979
|
+
);
|
|
1980
|
+
obj.insert(
|
|
1981
|
+
"token_seen_after_enter".to_string(),
|
|
1982
|
+
json_opt_bool(diag.and_then(|d| d.token_seen_after_enter)),
|
|
1983
|
+
);
|
|
1984
|
+
obj.insert(
|
|
1985
|
+
"capture_ok_count".to_string(),
|
|
1986
|
+
serde_json::json!(diag.map(|d| d.capture_ok_count).unwrap_or(0)),
|
|
1987
|
+
);
|
|
1988
|
+
obj.insert(
|
|
1989
|
+
"capture_err_count".to_string(),
|
|
1990
|
+
serde_json::json!(diag.map(|d| d.capture_err_count).unwrap_or(0)),
|
|
1991
|
+
);
|
|
1992
|
+
obj.insert(
|
|
1993
|
+
"capture_sample_count".to_string(),
|
|
1994
|
+
serde_json::json!(diag.map(|d| d.capture_sample_count).unwrap_or(0)),
|
|
1995
|
+
);
|
|
1996
|
+
obj.insert(
|
|
1997
|
+
"last_capture_outcome".to_string(),
|
|
1998
|
+
serde_json::json!(diag
|
|
1999
|
+
.map(|d| d.last_capture_outcome.as_str())
|
|
2000
|
+
.unwrap_or(CaptureSampleOutcome::NotAttempted.as_str())),
|
|
2001
|
+
);
|
|
2002
|
+
obj.insert(
|
|
2003
|
+
"pane_command_basename".to_string(),
|
|
2004
|
+
diag.and_then(|d| d.pane_command_basename.as_deref())
|
|
2005
|
+
.map(|name| serde_json::json!(name))
|
|
2006
|
+
.unwrap_or(serde_json::Value::Null),
|
|
2007
|
+
);
|
|
2008
|
+
obj.insert(
|
|
2009
|
+
"pane_command_query".to_string(),
|
|
2010
|
+
serde_json::json!(diag
|
|
2011
|
+
.map(|d| d.pane_command_query.as_str())
|
|
2012
|
+
.unwrap_or(QueryOutcome::Unavailable.as_str())),
|
|
2013
|
+
);
|
|
2014
|
+
obj.insert(
|
|
2015
|
+
"input_surface".to_string(),
|
|
2016
|
+
serde_json::json!(diag
|
|
2017
|
+
.map(|d| d.input_surface.as_str())
|
|
2018
|
+
.unwrap_or(InputSurfaceProbe::Unavailable.as_str())),
|
|
2019
|
+
);
|
|
2020
|
+
obj.insert(
|
|
2021
|
+
"diag_target_pane_id".to_string(),
|
|
2022
|
+
diag.and_then(|d| d.target_pane_id.as_deref())
|
|
2023
|
+
.map(|pane| serde_json::json!(pane))
|
|
2024
|
+
.unwrap_or(serde_json::Value::Null),
|
|
2025
|
+
);
|
|
2026
|
+
obj.insert(
|
|
2027
|
+
"diag_target_pane_query_matched".to_string(),
|
|
2028
|
+
json_opt_bool(diag.and_then(|d| d.target_pane_query_matched)),
|
|
2029
|
+
);
|
|
2030
|
+
}
|
|
2031
|
+
|
|
1917
2032
|
#[allow(clippy::too_many_arguments)]
|
|
1918
2033
|
fn emit_send_failed_exhausted(
|
|
1919
2034
|
workspace: &Path,
|
|
@@ -1944,21 +2059,22 @@ fn emit_send_failed_exhausted(
|
|
|
1944
2059
|
.and_then(|r| r.submit_diagnostics.as_ref())
|
|
1945
2060
|
.and_then(|d| d.attempts_detail.last())
|
|
1946
2061
|
.map(|a| a.pane_tail_excerpt.clone());
|
|
1947
|
-
|
|
1948
|
-
"
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
2062
|
+
let mut failed = serde_json::json!({
|
|
2063
|
+
"message_id": message_id,
|
|
2064
|
+
"recipient": recipient,
|
|
2065
|
+
"attempts": attempts,
|
|
2066
|
+
"max_attempts": SEND_RETRY_MAX_ATTEMPTS,
|
|
2067
|
+
"reason": failure_reason,
|
|
2068
|
+
"verification": verification,
|
|
2069
|
+
"submit_attempts_detail": submit_attempts_detail,
|
|
2070
|
+
"total_elapsed_ms": total_elapsed_ms,
|
|
2071
|
+
"last_matched_literal": last_matched_literal,
|
|
2072
|
+
"last_pane_tail_excerpt": last_pane_tail_excerpt,
|
|
2073
|
+
});
|
|
2074
|
+
if let Some(report) = inject_report {
|
|
2075
|
+
merge_inject_readback_fields(&mut failed, report, None, None);
|
|
2076
|
+
}
|
|
2077
|
+
event_log.write("send.failed", failed)?;
|
|
1962
2078
|
let content = format!(
|
|
1963
2079
|
"send.failed\nerror: send to {recipient} failed with {failure_reason} after {attempts}/{SEND_RETRY_MAX_ATTEMPTS} attempts\naction: inspect the target pane and retry the send\nlog: .team/logs/events.jsonl"
|
|
1964
2080
|
);
|
|
@@ -3266,4 +3382,59 @@ mod paste_floor_tests {
|
|
|
3266
3382
|
Duration::ZERO
|
|
3267
3383
|
);
|
|
3268
3384
|
}
|
|
3385
|
+
|
|
3386
|
+
fn empty_report() -> InjectReport {
|
|
3387
|
+
InjectReport {
|
|
3388
|
+
stage_reached: crate::transport::InjectStage::Submit,
|
|
3389
|
+
inject_verification: InjectVerification::CaptureMissingToken,
|
|
3390
|
+
submit_verification: SubmitVerification::SubmitConsumptionUnverified,
|
|
3391
|
+
turn_verification: crate::transport::TurnVerification::NotYetObserved,
|
|
3392
|
+
attempts: 1,
|
|
3393
|
+
submit_diagnostics: Some(crate::transport::SubmitDiagnostics {
|
|
3394
|
+
last_capture_outcome: CaptureSampleOutcome::Failed,
|
|
3395
|
+
capture_err_count: 1,
|
|
3396
|
+
capture_sample_count: 1,
|
|
3397
|
+
pane_command_basename: Some("grok".to_string()),
|
|
3398
|
+
pane_command_query: QueryOutcome::Observed,
|
|
3399
|
+
input_surface: InputSurfaceProbe::Input,
|
|
3400
|
+
target_pane_id: Some("%9".to_string()),
|
|
3401
|
+
target_pane_query_matched: Some(true),
|
|
3402
|
+
token_seen_after_paste: None,
|
|
3403
|
+
..crate::transport::SubmitDiagnostics::default()
|
|
3404
|
+
}),
|
|
3405
|
+
}
|
|
3406
|
+
}
|
|
3407
|
+
|
|
3408
|
+
#[test]
|
|
3409
|
+
fn inject_readback_fields_are_safe_scalars_and_keep_attempts_array() {
|
|
3410
|
+
let report = empty_report();
|
|
3411
|
+
let mut event = serde_json::json!({
|
|
3412
|
+
"message_id": "msg_x",
|
|
3413
|
+
"submit_attempts_detail": [{"matched": false}],
|
|
3414
|
+
});
|
|
3415
|
+
merge_inject_readback_fields(
|
|
3416
|
+
&mut event,
|
|
3417
|
+
&report,
|
|
3418
|
+
Some(&Target::Pane(PaneId::new("%9"))),
|
|
3419
|
+
None,
|
|
3420
|
+
);
|
|
3421
|
+
assert_eq!(event["last_capture_outcome"], serde_json::json!("failed"));
|
|
3422
|
+
assert_eq!(event["token_seen_after_paste"], serde_json::Value::Null);
|
|
3423
|
+
assert_eq!(event["pane_command_basename"], serde_json::json!("grok"));
|
|
3424
|
+
assert_eq!(event["input_surface"], serde_json::json!("input"));
|
|
3425
|
+
assert_eq!(event["inject_target_pane_id"], serde_json::json!("%9"));
|
|
3426
|
+
assert!(event["submit_attempts_detail"].is_array());
|
|
3427
|
+
assert!(event.get("pane_tail_excerpt").is_none());
|
|
3428
|
+
let dump = event.to_string();
|
|
3429
|
+
assert!(
|
|
3430
|
+
!dump.contains("--help") && !dump.contains("argv"),
|
|
3431
|
+
"readback fields must not leak argv: {dump}"
|
|
3432
|
+
);
|
|
3433
|
+
assert_eq!(
|
|
3434
|
+
report.inject_verification,
|
|
3435
|
+
InjectVerification::CaptureMissingToken
|
|
3436
|
+
);
|
|
3437
|
+
assert!(!inject_submit_verified(&report));
|
|
3438
|
+
assert!(!pane_readback_verified(&report));
|
|
3439
|
+
}
|
|
3269
3440
|
}
|
|
@@ -7,6 +7,11 @@ use crate::codex_app_server::AppServerBinding;
|
|
|
7
7
|
use crate::model::pane_authority_refusal::PaneWorkspaceMismatchFacts;
|
|
8
8
|
use crate::transport::{PaneInfo, Transport};
|
|
9
9
|
|
|
10
|
+
/// Scope authority written by a fresh quick-start caller binding. It is distinct
|
|
11
|
+
/// from an operator-initiated explicit claim, but uses the same pane-instance
|
|
12
|
+
/// nonce and exact workspace checks at delivery.
|
|
13
|
+
pub(crate) const FRESH_CALLER_SCOPE_AUTHORITY: &str = "fresh_caller";
|
|
14
|
+
|
|
10
15
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
11
16
|
pub enum LiveLeaderChannel {
|
|
12
17
|
DirectTmux(DirectTmuxLeaderChannel),
|
|
@@ -117,7 +122,7 @@ pub fn resolve_live_leader_channel(
|
|
|
117
122
|
};
|
|
118
123
|
if let Some(observed_pane_workspace) = observed.current_path.clone() {
|
|
119
124
|
if !path_is_in_workspace(&observed_pane_workspace, workspace)
|
|
120
|
-
&& !
|
|
125
|
+
&& !scoped_authority_matches(workspace, receiver, &observed)
|
|
121
126
|
{
|
|
122
127
|
return LeaderChannelResolution::Unbound(LeaderChannelUnbound::PaneWorkspaceMismatch(
|
|
123
128
|
PaneWorkspaceMismatchFacts {
|
|
@@ -138,12 +143,16 @@ pub fn resolve_live_leader_channel(
|
|
|
138
143
|
}))
|
|
139
144
|
}
|
|
140
145
|
|
|
141
|
-
fn
|
|
146
|
+
fn scoped_authority_matches(
|
|
142
147
|
workspace: &Path,
|
|
143
148
|
receiver: &Value,
|
|
144
149
|
observed: &PaneInfo,
|
|
145
150
|
) -> bool {
|
|
146
|
-
|
|
151
|
+
let authority = string_field(receiver, "scope_authority");
|
|
152
|
+
if !matches!(
|
|
153
|
+
authority,
|
|
154
|
+
Some("explicit_claim") | Some(FRESH_CALLER_SCOPE_AUTHORITY)
|
|
155
|
+
) {
|
|
147
156
|
return false;
|
|
148
157
|
}
|
|
149
158
|
let Some(authorized_workspace) = string_field(receiver, "authorized_team_workspace") else {
|
|
@@ -364,6 +364,78 @@ fn save_runtime_state_with_merge_options(
|
|
|
364
364
|
topology_update_agent_ids: &[&str],
|
|
365
365
|
receiver_update_team_key: Option<&str>,
|
|
366
366
|
exact_owner_seed_to_clear: Option<&Value>,
|
|
367
|
+
) -> Result<(), StateError> {
|
|
368
|
+
persist_runtime_state_with_merge_options_and_expected(
|
|
369
|
+
workspace,
|
|
370
|
+
state,
|
|
371
|
+
deleted_agent_ids,
|
|
372
|
+
skip_capture_backfill_team_key,
|
|
373
|
+
skip_capture_backfill_agent_ids,
|
|
374
|
+
topology_update_team_key,
|
|
375
|
+
topology_update_agent_ids,
|
|
376
|
+
receiver_update_team_key,
|
|
377
|
+
exact_owner_seed_to_clear,
|
|
378
|
+
None,
|
|
379
|
+
None,
|
|
380
|
+
)
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
pub(crate) fn save_runtime_state_with_receiver_authority_and_expected(
|
|
384
|
+
workspace: &Path,
|
|
385
|
+
state: &Value,
|
|
386
|
+
receiver_team_key: &str,
|
|
387
|
+
expected_owner: &Value,
|
|
388
|
+
expected_receiver: &Value,
|
|
389
|
+
) -> Result<(), StateError> {
|
|
390
|
+
persist_runtime_state_with_merge_options_and_expected(
|
|
391
|
+
workspace,
|
|
392
|
+
state,
|
|
393
|
+
&[],
|
|
394
|
+
None,
|
|
395
|
+
&[],
|
|
396
|
+
None,
|
|
397
|
+
&[],
|
|
398
|
+
Some(receiver_team_key),
|
|
399
|
+
None,
|
|
400
|
+
Some((expected_owner, expected_receiver)),
|
|
401
|
+
None,
|
|
402
|
+
)
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
pub(crate) fn save_runtime_state_with_exact_owner_receiver_cleanup(
|
|
406
|
+
workspace: &Path,
|
|
407
|
+
state: &Value,
|
|
408
|
+
team_key: &str,
|
|
409
|
+
expected_owner: &Value,
|
|
410
|
+
expected_receiver: &Value,
|
|
411
|
+
) -> Result<(), StateError> {
|
|
412
|
+
persist_runtime_state_with_merge_options_and_expected(
|
|
413
|
+
workspace,
|
|
414
|
+
state,
|
|
415
|
+
&[],
|
|
416
|
+
None,
|
|
417
|
+
&[],
|
|
418
|
+
None,
|
|
419
|
+
&[],
|
|
420
|
+
Some(team_key),
|
|
421
|
+
None,
|
|
422
|
+
None,
|
|
423
|
+
Some((team_key, expected_owner, expected_receiver)),
|
|
424
|
+
)
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
fn persist_runtime_state_with_merge_options_and_expected(
|
|
428
|
+
workspace: &Path,
|
|
429
|
+
state: &Value,
|
|
430
|
+
deleted_agent_ids: &[&str],
|
|
431
|
+
skip_capture_backfill_team_key: Option<&str>,
|
|
432
|
+
skip_capture_backfill_agent_ids: &[&str],
|
|
433
|
+
topology_update_team_key: Option<&str>,
|
|
434
|
+
topology_update_agent_ids: &[&str],
|
|
435
|
+
receiver_update_team_key: Option<&str>,
|
|
436
|
+
exact_owner_seed_to_clear: Option<&Value>,
|
|
437
|
+
expected_owner_receiver: Option<(&Value, &Value)>,
|
|
438
|
+
exact_owner_receiver_to_clear: Option<(&str, &Value, &Value)>,
|
|
367
439
|
) -> Result<(), StateError> {
|
|
368
440
|
let path = runtime_state_path(workspace);
|
|
369
441
|
// Python `state.py:497`:先对入参 state 跑 `_migrate_state_identity`(就地填缺失 leader uuid)。
|
|
@@ -385,7 +457,9 @@ fn save_runtime_state_with_merge_options(
|
|
|
385
457
|
// Exact-seed cleanup must observe lock-held disk, not a pre-lock cache or
|
|
386
458
|
// byte-equality hit. A matching in-memory document would otherwise no-op
|
|
387
459
|
// and leave the seed, or skip copying a concurrent owner.
|
|
388
|
-
let skip_pre_lock_fast_path = exact_owner_seed_to_clear.is_some()
|
|
460
|
+
let skip_pre_lock_fast_path = exact_owner_seed_to_clear.is_some()
|
|
461
|
+
|| expected_owner_receiver.is_some()
|
|
462
|
+
|| exact_owner_receiver_to_clear.is_some();
|
|
389
463
|
if !skip_pre_lock_fast_path && cache_equals(&path, &migrated) {
|
|
390
464
|
return Ok(());
|
|
391
465
|
}
|
|
@@ -418,7 +492,21 @@ fn save_runtime_state_with_merge_options(
|
|
|
418
492
|
if let Some(parent) = path.parent() {
|
|
419
493
|
std::fs::create_dir_all(parent)?;
|
|
420
494
|
}
|
|
421
|
-
|
|
495
|
+
let latest = read_latest_state_under_lock(workspace, &path);
|
|
496
|
+
if let Some((expected_owner, expected_receiver)) = expected_owner_receiver {
|
|
497
|
+
let expected_matches = latest.as_ref().is_some_and(|latest| {
|
|
498
|
+
latest_team_owner_receiver(latest, receiver_update_team_key)
|
|
499
|
+
.is_some_and(|(owner, receiver)| {
|
|
500
|
+
owner == expected_owner && receiver == expected_receiver
|
|
501
|
+
})
|
|
502
|
+
});
|
|
503
|
+
if !expected_matches {
|
|
504
|
+
return Err(StateError::SaveConflict(
|
|
505
|
+
"fresh caller owner/receiver changed before commit".to_string(),
|
|
506
|
+
));
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
if let Some(latest) = latest {
|
|
422
510
|
let deleted = deleted_agent_ids
|
|
423
511
|
.iter()
|
|
424
512
|
.copied()
|
|
@@ -455,6 +543,7 @@ fn save_runtime_state_with_merge_options(
|
|
|
455
543
|
&topology_updates,
|
|
456
544
|
&receiver_updates,
|
|
457
545
|
receiver_update_team_key.zip(exact_owner_seed_to_clear),
|
|
546
|
+
exact_owner_receiver_to_clear,
|
|
458
547
|
)?;
|
|
459
548
|
}
|
|
460
549
|
// Stage 3 save-output strip second pass (defence-in-depth): after the
|
|
@@ -523,6 +612,7 @@ fn apply_persist_merge_contract(
|
|
|
523
612
|
topology_updates: &BTreeSet<(String, String)>,
|
|
524
613
|
receiver_updates: &BTreeSet<String>,
|
|
525
614
|
exact_owner_seed_to_clear: Option<(&str, &Value)>,
|
|
615
|
+
exact_owner_receiver_to_clear: Option<(&str, &Value, &Value)>,
|
|
526
616
|
) -> Result<(), StateError> {
|
|
527
617
|
// A0/R1: the projection gate only guards the TOP-LEVEL passes (top-level agents and
|
|
528
618
|
// the top-level<->active-team cross projections depend on which team is active); the
|
|
@@ -645,6 +735,16 @@ fn apply_persist_merge_contract(
|
|
|
645
735
|
apply_lock_held_exact_owner_cleanup(incoming_entry, latest_entry, seed);
|
|
646
736
|
}
|
|
647
737
|
}
|
|
738
|
+
if let Some((clear_team, seed, expected_receiver)) = exact_owner_receiver_to_clear {
|
|
739
|
+
if team == clear_team {
|
|
740
|
+
apply_lock_held_exact_owner_receiver_cleanup(
|
|
741
|
+
incoming_entry,
|
|
742
|
+
latest_entry,
|
|
743
|
+
seed,
|
|
744
|
+
Some(expected_receiver),
|
|
745
|
+
);
|
|
746
|
+
}
|
|
747
|
+
}
|
|
648
748
|
preserve_latest_endpoint_convergence_fields(incoming_entry, latest_entry);
|
|
649
749
|
}
|
|
650
750
|
}
|
|
@@ -776,14 +876,26 @@ fn preserve_latest_ownership_fields(
|
|
|
776
876
|
}
|
|
777
877
|
|
|
778
878
|
fn apply_lock_held_exact_owner_cleanup(incoming: &mut Value, latest: &Value, seed: &Value) {
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
879
|
+
apply_lock_held_exact_owner_receiver_cleanup(incoming, latest, seed, None);
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
fn apply_lock_held_exact_owner_receiver_cleanup(
|
|
883
|
+
incoming: &mut Value,
|
|
884
|
+
latest: &Value,
|
|
885
|
+
seed: &Value,
|
|
886
|
+
expected_receiver: Option<&Value>,
|
|
887
|
+
) {
|
|
888
|
+
// Compare the lock-held disk owner and, when supplied, receiver to the
|
|
889
|
+
// attempt-local grant. A same-owner receiver update is a concurrent
|
|
890
|
+
// winner and must survive cleanup just like a different owner/epoch.
|
|
782
891
|
let latest_owner = latest.get("team_owner");
|
|
892
|
+
let receiver_matches = expected_receiver.is_none_or(|expected| {
|
|
893
|
+
latest.get("leader_receiver") == Some(expected)
|
|
894
|
+
});
|
|
783
895
|
let Some(incoming_obj) = incoming.as_object_mut() else {
|
|
784
896
|
return;
|
|
785
897
|
};
|
|
786
|
-
if latest_owner == Some(seed) {
|
|
898
|
+
if latest_owner == Some(seed) && receiver_matches {
|
|
787
899
|
incoming_obj.insert("team_owner".to_string(), Value::Null);
|
|
788
900
|
incoming_obj.insert("leader_receiver".to_string(), Value::Null);
|
|
789
901
|
if let Some(epoch) = latest
|
|
@@ -873,6 +985,21 @@ fn latest_has_preferable_ownership(
|
|
|
873
985
|
&& (ownership_attached(latest) || !ownership_attached(incoming))
|
|
874
986
|
}
|
|
875
987
|
|
|
988
|
+
fn latest_team_owner_receiver<'a>(
|
|
989
|
+
latest: &'a Value,
|
|
990
|
+
team_key: Option<&str>,
|
|
991
|
+
) -> Option<(&'a Value, &'a Value)> {
|
|
992
|
+
let entry = if let Some(team_key) = team_key {
|
|
993
|
+
latest
|
|
994
|
+
.get("teams")
|
|
995
|
+
.and_then(Value::as_object)
|
|
996
|
+
.and_then(|teams| teams.get(team_key))?
|
|
997
|
+
} else {
|
|
998
|
+
latest
|
|
999
|
+
};
|
|
1000
|
+
Some((entry.get("team_owner")?, entry.get("leader_receiver")?))
|
|
1001
|
+
}
|
|
1002
|
+
|
|
876
1003
|
fn ownership_epoch(state: &Value) -> u64 {
|
|
877
1004
|
state
|
|
878
1005
|
.get("owner_epoch")
|
|
@@ -2985,6 +3112,7 @@ mod tests {
|
|
|
2985
3112
|
&BTreeSet::new(),
|
|
2986
3113
|
&BTreeSet::new(),
|
|
2987
3114
|
None,
|
|
3115
|
+
None,
|
|
2988
3116
|
)
|
|
2989
3117
|
.unwrap();
|
|
2990
3118
|
|
|
@@ -3020,6 +3148,7 @@ mod tests {
|
|
|
3020
3148
|
&BTreeSet::new(),
|
|
3021
3149
|
&receiver_updates,
|
|
3022
3150
|
None,
|
|
3151
|
+
None,
|
|
3023
3152
|
)
|
|
3024
3153
|
.unwrap();
|
|
3025
3154
|
|
|
@@ -3219,6 +3348,37 @@ mod tests {
|
|
|
3219
3348
|
);
|
|
3220
3349
|
}
|
|
3221
3350
|
|
|
3351
|
+
#[test]
|
|
3352
|
+
fn lock_held_exact_seed_receiver_cleanup_preserves_same_owner_receiver_update() {
|
|
3353
|
+
let ws = temp_ws();
|
|
3354
|
+
save_runtime_state(
|
|
3355
|
+
&ws,
|
|
3356
|
+
&team_state_with_binding(&ws, seed_owner(1), seed_receiver(1), 1),
|
|
3357
|
+
)
|
|
3358
|
+
.unwrap();
|
|
3359
|
+
let seed = persisted_fresh_owner(&ws);
|
|
3360
|
+
let seeded_receiver = load_runtime_state(&ws).unwrap()["teams"]["fresh"]
|
|
3361
|
+
["leader_receiver"]
|
|
3362
|
+
.clone();
|
|
3363
|
+
let mut winner = team_state_with_binding(&ws, seed_owner(1), takeover_receiver(1), 1);
|
|
3364
|
+
winner["teams"]["fresh"]["team_owner"] = seed.clone();
|
|
3365
|
+
save_runtime_state_with_receiver_authority(&ws, &winner, "fresh", None).unwrap();
|
|
3366
|
+
save_runtime_state_with_exact_owner_receiver_cleanup(
|
|
3367
|
+
&ws,
|
|
3368
|
+
&tombstone_incoming(&ws, 1),
|
|
3369
|
+
"fresh",
|
|
3370
|
+
&seed,
|
|
3371
|
+
&seeded_receiver,
|
|
3372
|
+
)
|
|
3373
|
+
.unwrap();
|
|
3374
|
+
assert_fresh_binding(
|
|
3375
|
+
&load_runtime_state(&ws).unwrap(),
|
|
3376
|
+
&seed,
|
|
3377
|
+
&takeover_receiver(1),
|
|
3378
|
+
1,
|
|
3379
|
+
);
|
|
3380
|
+
}
|
|
3381
|
+
|
|
3222
3382
|
#[test]
|
|
3223
3383
|
fn lock_held_exact_seed_cleanup_preserves_epoch2_takeover_binding() {
|
|
3224
3384
|
let ws = temp_ws();
|
|
@@ -48,6 +48,10 @@ use super::persist::{
|
|
|
48
48
|
save_runtime_state_with_lifecycle_topology_authority as helper_write_root_with_lifecycle_topology_authority,
|
|
49
49
|
save_runtime_state_with_lifecycle_topology_authority_and_capture_backfill_skip as helper_write_root_with_lifecycle_topology_authority_and_capture_backfill_skip,
|
|
50
50
|
save_runtime_state_with_receiver_authority as helper_write_root_with_receiver_authority,
|
|
51
|
+
save_runtime_state_with_receiver_authority_and_expected
|
|
52
|
+
as helper_write_root_with_receiver_authority_and_expected,
|
|
53
|
+
save_runtime_state_with_exact_owner_receiver_cleanup
|
|
54
|
+
as helper_write_root_with_exact_owner_receiver_cleanup,
|
|
51
55
|
save_runtime_state_with_team_tombstone_lifecycle_topology_authority as helper_write_root_with_team_tombstone_lifecycle_topology_authority,
|
|
52
56
|
save_runtime_state_with_team_tombstoned_agents as helper_write_root_with_team_tombstoned_agents,
|
|
53
57
|
save_runtime_state_without_migrations as helper_write_root_without_migrations,
|
|
@@ -206,10 +210,20 @@ pub enum StateWriteIntent<'a> {
|
|
|
206
210
|
ClaimLeader {
|
|
207
211
|
team_key: &'a str,
|
|
208
212
|
},
|
|
213
|
+
ClaimLeaderFreshCaller {
|
|
214
|
+
team_key: &'a str,
|
|
215
|
+
expected_owner: &'a Value,
|
|
216
|
+
expected_receiver: &'a Value,
|
|
217
|
+
},
|
|
209
218
|
ClearExactTeamOwner {
|
|
210
219
|
team_key: &'a str,
|
|
211
220
|
seed: &'a Value,
|
|
212
221
|
},
|
|
222
|
+
ClearExactTeamOwnerAndReceiver {
|
|
223
|
+
team_key: &'a str,
|
|
224
|
+
seed: &'a Value,
|
|
225
|
+
expected_receiver: &'a Value,
|
|
226
|
+
},
|
|
213
227
|
LeaderBindingRestoreNonTargetTeams {
|
|
214
228
|
target_team_key: &'a str,
|
|
215
229
|
},
|
|
@@ -389,9 +403,31 @@ fn route_direct(
|
|
|
389
403
|
StateWriteIntent::ClaimLeader { team_key } => {
|
|
390
404
|
helper_write_root_with_receiver_authority(workspace, state, team_key, None)
|
|
391
405
|
}
|
|
406
|
+
StateWriteIntent::ClaimLeaderFreshCaller {
|
|
407
|
+
team_key,
|
|
408
|
+
expected_owner,
|
|
409
|
+
expected_receiver,
|
|
410
|
+
} => helper_write_root_with_receiver_authority_and_expected(
|
|
411
|
+
workspace,
|
|
412
|
+
state,
|
|
413
|
+
team_key,
|
|
414
|
+
expected_owner,
|
|
415
|
+
expected_receiver,
|
|
416
|
+
),
|
|
392
417
|
StateWriteIntent::ClearExactTeamOwner { team_key, seed } => {
|
|
393
418
|
helper_write_root_with_receiver_authority(workspace, state, team_key, Some(seed))
|
|
394
419
|
}
|
|
420
|
+
StateWriteIntent::ClearExactTeamOwnerAndReceiver {
|
|
421
|
+
team_key,
|
|
422
|
+
seed,
|
|
423
|
+
expected_receiver,
|
|
424
|
+
} => helper_write_root_with_exact_owner_receiver_cleanup(
|
|
425
|
+
workspace,
|
|
426
|
+
state,
|
|
427
|
+
team_key,
|
|
428
|
+
seed,
|
|
429
|
+
expected_receiver,
|
|
430
|
+
),
|
|
395
431
|
StateWriteIntent::LeaderBindingRestoreNonTargetTeams { .. } => {
|
|
396
432
|
helper_write_root_without_migrations(workspace, state)
|
|
397
433
|
}
|