@team-agent/installer 0.5.18 → 0.5.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -1
- package/Cargo.toml +1 -1
- package/crates/team-agent/src/cli/adapters.rs +4 -3
- package/crates/team-agent/src/cli/diagnose.rs +185 -0
- package/crates/team-agent/src/cli/mod.rs +4 -4
- package/crates/team-agent/src/cli/tests/lane_c.rs +5 -2
- package/crates/team-agent/src/coordinator/backoff.rs +228 -6
- package/crates/team-agent/src/coordinator/tick.rs +80 -13
- package/crates/team-agent/src/diagnose/mod.rs +2 -2
- package/crates/team-agent/src/diagnose/orphans.rs +132 -40
- package/crates/team-agent/src/lifecycle/types.rs +6 -0
- package/package.json +4 -4
package/Cargo.lock
CHANGED
package/Cargo.toml
CHANGED
|
@@ -732,7 +732,8 @@ pub fn cmd_diagnose(args: &DiagnoseArgs) -> Result<CmdResult, CliError> {
|
|
|
732
732
|
&selected.run_workspace,
|
|
733
733
|
)),
|
|
734
734
|
};
|
|
735
|
-
let (issues, suggested_repairs) =
|
|
735
|
+
let (issues, suggested_repairs) =
|
|
736
|
+
diagnose_runtime_for_workspace(&selected.run_workspace, &state, backend.as_ref());
|
|
736
737
|
let ok = issues.as_array().is_some_and(Vec::is_empty);
|
|
737
738
|
Ok(CmdResult::from_json(
|
|
738
739
|
json!({
|
|
@@ -1701,9 +1702,9 @@ pub fn cmd_doctor(args: &DoctorArgs) -> Result<CmdResult, CliError> {
|
|
|
1701
1702
|
return Ok(CmdResult::from_json(value, true));
|
|
1702
1703
|
}
|
|
1703
1704
|
let value = if matches!(args.gate, Some(DoctorGate::Orphans)) {
|
|
1704
|
-
crate::diagnose::orphans::orphan_gate_json(args.fix, args.confirm)?
|
|
1705
|
+
crate::diagnose::orphans::orphan_gate_json(&args.workspace, args.fix, args.confirm)?
|
|
1705
1706
|
} else if args.cleanup_orphans {
|
|
1706
|
-
crate::diagnose::orphans::cleanup_orphans_json(args.confirm)?
|
|
1707
|
+
crate::diagnose::orphans::cleanup_orphans_json(&args.workspace, args.confirm)?
|
|
1707
1708
|
} else if args.fix_schema {
|
|
1708
1709
|
diagnose_port::fix_schema(&args.workspace)?
|
|
1709
1710
|
} else {
|
|
@@ -165,6 +165,125 @@ pub(crate) fn diagnose_runtime(state: &Value, backend: &dyn Transport) -> (Value
|
|
|
165
165
|
(Value::Array(issues), Value::Array(repairs))
|
|
166
166
|
}
|
|
167
167
|
|
|
168
|
+
pub(crate) fn diagnose_runtime_for_workspace(
|
|
169
|
+
workspace: &std::path::Path,
|
|
170
|
+
state: &Value,
|
|
171
|
+
backend: &dyn Transport,
|
|
172
|
+
) -> (Value, Value) {
|
|
173
|
+
let (mut issues, mut repairs) = diagnose_runtime(state, backend);
|
|
174
|
+
append_coordinator_health_issue(workspace, state, &mut issues, &mut repairs);
|
|
175
|
+
(issues, repairs)
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
fn append_coordinator_health_issue(
|
|
179
|
+
workspace: &std::path::Path,
|
|
180
|
+
state: &Value,
|
|
181
|
+
issues: &mut Value,
|
|
182
|
+
repairs: &mut Value,
|
|
183
|
+
) {
|
|
184
|
+
let workspace = crate::coordinator::WorkspacePath::new(workspace.to_path_buf());
|
|
185
|
+
let health = crate::coordinator::coordinator_health(&workspace);
|
|
186
|
+
let Some(id) = coordinator_issue_id(state, &health) else {
|
|
187
|
+
return;
|
|
188
|
+
};
|
|
189
|
+
if let Some(items) = issues.as_array_mut() {
|
|
190
|
+
items.push(coordinator_issue_value(id, &health, workspace.as_path()));
|
|
191
|
+
}
|
|
192
|
+
if let Some(items) = repairs.as_array_mut() {
|
|
193
|
+
items.push(coordinator_repair_hint(id, &health));
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
fn coordinator_issue_id(
|
|
198
|
+
state: &Value,
|
|
199
|
+
health: &crate::coordinator::HealthReport,
|
|
200
|
+
) -> Option<&'static str> {
|
|
201
|
+
match health.status {
|
|
202
|
+
crate::coordinator::CoordinatorHealthStatus::Stale
|
|
203
|
+
| crate::coordinator::CoordinatorHealthStatus::InvalidPid => {
|
|
204
|
+
Some("coordinator_unavailable")
|
|
205
|
+
}
|
|
206
|
+
crate::coordinator::CoordinatorHealthStatus::Running => {
|
|
207
|
+
if !health.metadata_ok {
|
|
208
|
+
return match health.metadata_mismatch_reason.as_deref() {
|
|
209
|
+
Some(
|
|
210
|
+
"binary_identity_missing"
|
|
211
|
+
| "binary_version_mismatch"
|
|
212
|
+
| "binary_path_mismatch",
|
|
213
|
+
) => Some("coordinator_stale_identity"),
|
|
214
|
+
Some(_) | None => Some("coordinator_unavailable"),
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
if !health.schema.ok {
|
|
218
|
+
Some("coordinator_schema_incompatible")
|
|
219
|
+
} else {
|
|
220
|
+
None
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
crate::coordinator::CoordinatorHealthStatus::Missing => {
|
|
224
|
+
coordinator_expected(state).then_some("coordinator_unavailable")
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
fn coordinator_issue_value(
|
|
230
|
+
id: &str,
|
|
231
|
+
health: &crate::coordinator::HealthReport,
|
|
232
|
+
workspace: &std::path::Path,
|
|
233
|
+
) -> Value {
|
|
234
|
+
json!({
|
|
235
|
+
"id": id,
|
|
236
|
+
"status": coordinator_status_wire(health.status),
|
|
237
|
+
"pid": health.pid.map(|pid| pid.get()),
|
|
238
|
+
"metadata_ok": health.metadata_ok,
|
|
239
|
+
"metadata_mismatch_reason": health.metadata_mismatch_reason.clone(),
|
|
240
|
+
"binary_path": health.current_binary_identity.binary_path.clone(),
|
|
241
|
+
"binary_version": health.current_binary_identity.binary_version.clone(),
|
|
242
|
+
"schema_ok": health.schema.ok,
|
|
243
|
+
"coordinator_log": crate::coordinator::coordinator_log_path(
|
|
244
|
+
&crate::coordinator::WorkspacePath::new(workspace.to_path_buf())
|
|
245
|
+
)
|
|
246
|
+
.to_string_lossy()
|
|
247
|
+
.to_string(),
|
|
248
|
+
})
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
fn coordinator_repair_hint(id: &str, _health: &crate::coordinator::HealthReport) -> Value {
|
|
252
|
+
let hint_action = match id {
|
|
253
|
+
"coordinator_schema_incompatible" => "team-agent repair-state --schema",
|
|
254
|
+
_ => "team-agent restart",
|
|
255
|
+
};
|
|
256
|
+
json!({
|
|
257
|
+
"issue": id,
|
|
258
|
+
"action_required": true,
|
|
259
|
+
"advisory": true,
|
|
260
|
+
"broken_class": id,
|
|
261
|
+
"hint_action": hint_action,
|
|
262
|
+
"dedupe_key": id,
|
|
263
|
+
"action": format!("{hint_action} # diagnose only reports coordinator health; it does not start, stop, or rotate the coordinator"),
|
|
264
|
+
})
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
fn coordinator_expected(state: &Value) -> bool {
|
|
268
|
+
state
|
|
269
|
+
.get("session_name")
|
|
270
|
+
.and_then(Value::as_str)
|
|
271
|
+
.is_some_and(|session| !session.is_empty())
|
|
272
|
+
|| state
|
|
273
|
+
.get("agents")
|
|
274
|
+
.and_then(Value::as_object)
|
|
275
|
+
.is_some_and(|agents| !agents.is_empty())
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
fn coordinator_status_wire(status: crate::coordinator::CoordinatorHealthStatus) -> &'static str {
|
|
279
|
+
match status {
|
|
280
|
+
crate::coordinator::CoordinatorHealthStatus::Missing => "missing",
|
|
281
|
+
crate::coordinator::CoordinatorHealthStatus::InvalidPid => "invalid_pid",
|
|
282
|
+
crate::coordinator::CoordinatorHealthStatus::Running => "running",
|
|
283
|
+
crate::coordinator::CoordinatorHealthStatus::Stale => "stale",
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
168
287
|
fn topology_repair_hint(issue: &str) -> Value {
|
|
169
288
|
json!({
|
|
170
289
|
"issue": issue,
|
|
@@ -246,6 +365,72 @@ mod tests {
|
|
|
246
365
|
path
|
|
247
366
|
}
|
|
248
367
|
|
|
368
|
+
fn coordinator_health_fixture(
|
|
369
|
+
status: crate::coordinator::CoordinatorHealthStatus,
|
|
370
|
+
metadata_mismatch_reason: Option<&str>,
|
|
371
|
+
schema_ok: bool,
|
|
372
|
+
) -> crate::coordinator::HealthReport {
|
|
373
|
+
crate::coordinator::HealthReport {
|
|
374
|
+
ok: matches!(status, crate::coordinator::CoordinatorHealthStatus::Running)
|
|
375
|
+
&& metadata_mismatch_reason.is_none()
|
|
376
|
+
&& schema_ok,
|
|
377
|
+
status,
|
|
378
|
+
pid: Some(crate::coordinator::Pid::new(std::process::id())),
|
|
379
|
+
metadata: None,
|
|
380
|
+
metadata_ok: metadata_mismatch_reason.is_none(),
|
|
381
|
+
metadata_mismatch_reason: metadata_mismatch_reason.map(ToString::to_string),
|
|
382
|
+
current_binary_identity: crate::coordinator::CoordinatorBinaryIdentity {
|
|
383
|
+
binary_path: "/current/team-agent".to_string(),
|
|
384
|
+
binary_version: env!("CARGO_PKG_VERSION").to_string(),
|
|
385
|
+
},
|
|
386
|
+
schema: crate::coordinator::SchemaHealth {
|
|
387
|
+
ok: schema_ok,
|
|
388
|
+
schema_version: crate::db::schema::SCHEMA_VERSION,
|
|
389
|
+
error: None,
|
|
390
|
+
action: None,
|
|
391
|
+
},
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
#[test]
|
|
396
|
+
fn coordinator_issue_id_maps_stale_pid_to_unavailable() {
|
|
397
|
+
let health = coordinator_health_fixture(
|
|
398
|
+
crate::coordinator::CoordinatorHealthStatus::Stale,
|
|
399
|
+
None,
|
|
400
|
+
true,
|
|
401
|
+
);
|
|
402
|
+
assert_eq!(
|
|
403
|
+
coordinator_issue_id(&json!({"session_name": "team-fixture"}), &health),
|
|
404
|
+
Some("coordinator_unavailable")
|
|
405
|
+
);
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
#[test]
|
|
409
|
+
fn coordinator_issue_id_maps_binary_identity_to_stale_identity() {
|
|
410
|
+
let health = coordinator_health_fixture(
|
|
411
|
+
crate::coordinator::CoordinatorHealthStatus::Running,
|
|
412
|
+
Some("binary_version_mismatch"),
|
|
413
|
+
true,
|
|
414
|
+
);
|
|
415
|
+
assert_eq!(
|
|
416
|
+
coordinator_issue_id(&json!({"session_name": "team-fixture"}), &health),
|
|
417
|
+
Some("coordinator_stale_identity")
|
|
418
|
+
);
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
#[test]
|
|
422
|
+
fn coordinator_issue_id_maps_schema_failure_to_schema_incompatible() {
|
|
423
|
+
let health = coordinator_health_fixture(
|
|
424
|
+
crate::coordinator::CoordinatorHealthStatus::Running,
|
|
425
|
+
None,
|
|
426
|
+
false,
|
|
427
|
+
);
|
|
428
|
+
assert_eq!(
|
|
429
|
+
coordinator_issue_id(&json!({"session_name": "team-fixture"}), &health),
|
|
430
|
+
Some("coordinator_schema_incompatible")
|
|
431
|
+
);
|
|
432
|
+
}
|
|
433
|
+
|
|
249
434
|
#[test]
|
|
250
435
|
fn diagnose_surfaces_codex_session_identity_mismatch() {
|
|
251
436
|
let workspace =
|
|
@@ -3820,12 +3820,12 @@ pub mod diagnose_port {
|
|
|
3820
3820
|
}
|
|
3821
3821
|
|
|
3822
3822
|
/// `orphan_gate(fix, confirm)`(`--gate orphans`)。CI gate。
|
|
3823
|
-
pub fn orphan_gate(fix: bool, confirm: bool) -> Result<Value, CliError> {
|
|
3824
|
-
crate::diagnose::orphans::orphan_gate_json(fix, confirm)
|
|
3823
|
+
pub fn orphan_gate(workspace: &Path, fix: bool, confirm: bool) -> Result<Value, CliError> {
|
|
3824
|
+
crate::diagnose::orphans::orphan_gate_json(workspace, fix, confirm)
|
|
3825
3825
|
}
|
|
3826
3826
|
/// `cleanup_orphan_coordinators(confirm)`(`--cleanup-orphans`;dry-run unless `--confirm`)。
|
|
3827
|
-
pub fn cleanup_orphans(confirm: bool) -> Result<Value, CliError> {
|
|
3828
|
-
crate::diagnose::orphans::cleanup_orphans_json(confirm)
|
|
3827
|
+
pub fn cleanup_orphans(workspace: &Path, confirm: bool) -> Result<Value, CliError> {
|
|
3828
|
+
crate::diagnose::orphans::cleanup_orphans_json(workspace, confirm)
|
|
3829
3829
|
}
|
|
3830
3830
|
/// `fix_schema_layout`(`--fix-schema`)/`schema_diagnosis`。
|
|
3831
3831
|
pub fn fix_schema(workspace: &Path) -> Result<Value, CliError> {
|
|
@@ -269,7 +269,9 @@ use super::*;
|
|
|
269
269
|
// 530-531 stub {ok:true,fix,confirm,orphans:[]}. Fully deterministic (no subprocess). RED. ─────────
|
|
270
270
|
#[test]
|
|
271
271
|
fn orphan_gate_fix_without_confirm_is_refused_envelope() {
|
|
272
|
-
let
|
|
272
|
+
let ws = tmp_workspace();
|
|
273
|
+
let v = diagnose_port::orphan_gate(&ws, /*fix=*/ true, /*confirm=*/ false)
|
|
274
|
+
.expect("orphan_gate");
|
|
273
275
|
assert_eq!(
|
|
274
276
|
v,
|
|
275
277
|
json!({
|
|
@@ -292,7 +294,8 @@ use super::*;
|
|
|
292
294
|
#[ignore = "real-machine: cleanup_orphans scans machine-wide ta-* tmux/process residue"]
|
|
293
295
|
#[serial_test::file_serial(tmux)]
|
|
294
296
|
fn cleanup_orphans_dryrun_golden_envelope() {
|
|
295
|
-
let
|
|
297
|
+
let ws = tmp_workspace();
|
|
298
|
+
let v = diagnose_port::cleanup_orphans(&ws, /*confirm=*/ false).expect("cleanup_orphans");
|
|
296
299
|
let obj = v.as_object().expect("cleanup dict");
|
|
297
300
|
assert_eq!(v["dry_run"], json!(true), "no --confirm => dry_run:true (cleanup_orphan_coordinators)");
|
|
298
301
|
assert_eq!(v["orphans"], json!([]), "golden lists orphans (empty when none)");
|
|
@@ -8,7 +8,7 @@ use crate::model::enums::Provider;
|
|
|
8
8
|
use crate::provider::ProviderAdapter;
|
|
9
9
|
|
|
10
10
|
use super::health::{coordinator_pid_path, write_coordinator_metadata};
|
|
11
|
-
use super::tick::{TickError, TickReport};
|
|
11
|
+
use super::tick::{write_coordinator_heartbeat, TickError, TickReport, HEARTBEAT_STATUS_PANIC};
|
|
12
12
|
use super::types::{
|
|
13
13
|
ErrorLists, MetadataSource, Pid, ProviderRegistry, WorkspacePath, BACKOFF_MAX_SEC,
|
|
14
14
|
DEFAULT_TICK_INTERVAL_SEC,
|
|
@@ -86,7 +86,7 @@ pub fn run_daemon(args: DaemonArgs) -> Result<(), DaemonError> {
|
|
|
86
86
|
// event log shows the reason. This preserves the daemon
|
|
87
87
|
// liveness path while making the failure explicit.
|
|
88
88
|
eprintln!(
|
|
89
|
-
"coordinator
|
|
89
|
+
"[coordinator] transport_factory refused ({e}); falling back to tmux workspace for daemon liveness"
|
|
90
90
|
);
|
|
91
91
|
let sel = crate::tmux_backend::tmux_backend_for_runtime_state_or_workspace(
|
|
92
92
|
args.workspace.as_path(),
|
|
@@ -139,13 +139,87 @@ fn run_daemon_with_coordinator_and_boot_tmux(
|
|
|
139
139
|
args: &DaemonArgs,
|
|
140
140
|
coordinator: &Coordinator,
|
|
141
141
|
tmux_metadata: Option<DaemonTmuxEndpointMetadata>,
|
|
142
|
+
) -> Result<(), DaemonError> {
|
|
143
|
+
let pid = Pid::new(std::process::id());
|
|
144
|
+
let boot_id = daemon_boot_id(pid);
|
|
145
|
+
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
|
146
|
+
run_daemon_body_with_panic_marker(args, coordinator, tmux_metadata, pid, &boot_id)
|
|
147
|
+
})) {
|
|
148
|
+
Ok(Ok(())) => Ok(()),
|
|
149
|
+
Ok(Err(error)) => {
|
|
150
|
+
let reason = if matches!(error, DaemonError::Tick(_)) {
|
|
151
|
+
"fatal_error"
|
|
152
|
+
} else {
|
|
153
|
+
"startup_error"
|
|
154
|
+
};
|
|
155
|
+
let _ = write_coordinator_heartbeat(
|
|
156
|
+
&args.workspace,
|
|
157
|
+
pid,
|
|
158
|
+
Some(&boot_id),
|
|
159
|
+
reason,
|
|
160
|
+
Some("error"),
|
|
161
|
+
Some(&error.to_string()),
|
|
162
|
+
false,
|
|
163
|
+
);
|
|
164
|
+
let _ = write_daemon_exit(
|
|
165
|
+
&EventLog::new(args.workspace.as_path()),
|
|
166
|
+
&args.workspace,
|
|
167
|
+
pid,
|
|
168
|
+
&boot_id,
|
|
169
|
+
reason,
|
|
170
|
+
serde_json::json!({"error": error.to_string()}),
|
|
171
|
+
);
|
|
172
|
+
Err(error)
|
|
173
|
+
}
|
|
174
|
+
Err(payload) => {
|
|
175
|
+
let panic_payload = panic_payload_message(payload.as_ref());
|
|
176
|
+
let backtrace = std::backtrace::Backtrace::force_capture().to_string();
|
|
177
|
+
let _ = write_coordinator_heartbeat(
|
|
178
|
+
&args.workspace,
|
|
179
|
+
pid,
|
|
180
|
+
Some(&boot_id),
|
|
181
|
+
"panic",
|
|
182
|
+
Some(HEARTBEAT_STATUS_PANIC),
|
|
183
|
+
Some(&panic_payload),
|
|
184
|
+
false,
|
|
185
|
+
);
|
|
186
|
+
let _ = write_daemon_exit(
|
|
187
|
+
&EventLog::new(args.workspace.as_path()),
|
|
188
|
+
&args.workspace,
|
|
189
|
+
pid,
|
|
190
|
+
&boot_id,
|
|
191
|
+
"panic",
|
|
192
|
+
serde_json::json!({
|
|
193
|
+
"panic_payload": panic_payload.clone(),
|
|
194
|
+
"backtrace": backtrace,
|
|
195
|
+
}),
|
|
196
|
+
);
|
|
197
|
+
Err(DaemonError::Panic(panic_payload))
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
fn run_daemon_body_with_panic_marker(
|
|
203
|
+
args: &DaemonArgs,
|
|
204
|
+
coordinator: &Coordinator,
|
|
205
|
+
tmux_metadata: Option<DaemonTmuxEndpointMetadata>,
|
|
206
|
+
pid: Pid,
|
|
207
|
+
boot_id: &str,
|
|
142
208
|
) -> Result<(), DaemonError> {
|
|
143
209
|
let runtime_dir = crate::model::paths::runtime_dir(args.workspace.as_path());
|
|
144
210
|
std::fs::create_dir_all(&runtime_dir)?;
|
|
145
|
-
let pid = Pid::new(std::process::id());
|
|
146
211
|
std::fs::write(coordinator_pid_path(&args.workspace), pid.to_string())?;
|
|
147
212
|
write_coordinator_metadata(&args.workspace, pid, MetadataSource::Boot)?;
|
|
148
213
|
let binary_identity = crate::coordinator::current_coordinator_binary_identity();
|
|
214
|
+
let _ = write_coordinator_heartbeat(
|
|
215
|
+
&args.workspace,
|
|
216
|
+
pid,
|
|
217
|
+
Some(boot_id),
|
|
218
|
+
"boot",
|
|
219
|
+
None,
|
|
220
|
+
None,
|
|
221
|
+
false,
|
|
222
|
+
);
|
|
149
223
|
|
|
150
224
|
let event_log = EventLog::new(args.workspace.as_path());
|
|
151
225
|
let mut boot_event = serde_json::json!({
|
|
@@ -204,14 +278,14 @@ fn run_daemon_with_coordinator_and_boot_tmux(
|
|
|
204
278
|
// Detach so the shim survives beyond this
|
|
205
279
|
// coordinator instance (F7 core invariant).
|
|
206
280
|
let _shim_pid = handle.detach();
|
|
207
|
-
eprintln!("coordinator
|
|
281
|
+
eprintln!("[coordinator] shim ensured (pid={_shim_pid})");
|
|
208
282
|
}
|
|
209
283
|
Err(e) => {
|
|
210
284
|
// Honest degradation: emit the stale-family
|
|
211
285
|
// event, coord continues without a shim. Tick
|
|
212
286
|
// loop will surface `mux_unavailable` on any
|
|
213
287
|
// transport call.
|
|
214
|
-
eprintln!("coordinator
|
|
288
|
+
eprintln!("[coordinator] shim ensure failed: {e}");
|
|
215
289
|
let _ = crate::coordinator::conpty_shim::mark_transport_unavailable(
|
|
216
290
|
args.workspace.as_path(),
|
|
217
291
|
&format!("boot_ensure_failed: {e}"),
|
|
@@ -231,7 +305,28 @@ fn run_daemon_with_coordinator_and_boot_tmux(
|
|
|
231
305
|
let initial_ppid = current_ppid();
|
|
232
306
|
let mut consecutive_failures = 0_u32;
|
|
233
307
|
let mut last_failure_signature: Option<String> = None;
|
|
308
|
+
install_signal_handlers();
|
|
234
309
|
loop {
|
|
310
|
+
if let Some(signal) = take_signal_stop_request() {
|
|
311
|
+
let _ = write_coordinator_heartbeat(
|
|
312
|
+
&args.workspace,
|
|
313
|
+
pid,
|
|
314
|
+
Some(boot_id),
|
|
315
|
+
"signal",
|
|
316
|
+
Some("stop_requested"),
|
|
317
|
+
Some(signal),
|
|
318
|
+
false,
|
|
319
|
+
);
|
|
320
|
+
write_daemon_exit(
|
|
321
|
+
&event_log,
|
|
322
|
+
&args.workspace,
|
|
323
|
+
pid,
|
|
324
|
+
boot_id,
|
|
325
|
+
"signal",
|
|
326
|
+
serde_json::json!({"signal": signal}),
|
|
327
|
+
)?;
|
|
328
|
+
return Ok(());
|
|
329
|
+
}
|
|
235
330
|
let ppid_now = current_ppid();
|
|
236
331
|
if super::should_orphan_self_terminate(initial_ppid, ppid_now, &args.workspace) {
|
|
237
332
|
let _ = event_log.write(
|
|
@@ -244,8 +339,31 @@ fn run_daemon_with_coordinator_and_boot_tmux(
|
|
|
244
339
|
);
|
|
245
340
|
break;
|
|
246
341
|
}
|
|
342
|
+
let _ = write_coordinator_heartbeat(
|
|
343
|
+
&args.workspace,
|
|
344
|
+
pid,
|
|
345
|
+
Some(boot_id),
|
|
346
|
+
"tick_running",
|
|
347
|
+
Some("running"),
|
|
348
|
+
None,
|
|
349
|
+
false,
|
|
350
|
+
);
|
|
247
351
|
match run_tick_with_panic_marker(&event_log, || coordinator.tick()) {
|
|
248
352
|
Ok(report) => {
|
|
353
|
+
let status = if report.stop {
|
|
354
|
+
"stop_requested"
|
|
355
|
+
} else {
|
|
356
|
+
"ok"
|
|
357
|
+
};
|
|
358
|
+
let _ = write_coordinator_heartbeat(
|
|
359
|
+
&args.workspace,
|
|
360
|
+
pid,
|
|
361
|
+
Some(boot_id),
|
|
362
|
+
"tick_finished",
|
|
363
|
+
Some(status),
|
|
364
|
+
None,
|
|
365
|
+
false,
|
|
366
|
+
);
|
|
249
367
|
if consecutive_failures > 0 {
|
|
250
368
|
event_log.write(
|
|
251
369
|
"coordinator.tick_recovered",
|
|
@@ -260,6 +378,20 @@ fn run_daemon_with_coordinator_and_boot_tmux(
|
|
|
260
378
|
sleep_seconds(tick_interval);
|
|
261
379
|
}
|
|
262
380
|
Err(err) => {
|
|
381
|
+
let status = if matches!(err, TickError::Panic(_)) {
|
|
382
|
+
HEARTBEAT_STATUS_PANIC
|
|
383
|
+
} else {
|
|
384
|
+
"error"
|
|
385
|
+
};
|
|
386
|
+
let _ = write_coordinator_heartbeat(
|
|
387
|
+
&args.workspace,
|
|
388
|
+
pid,
|
|
389
|
+
Some(boot_id),
|
|
390
|
+
"tick_finished",
|
|
391
|
+
Some(status),
|
|
392
|
+
Some(&err.to_string()),
|
|
393
|
+
false,
|
|
394
|
+
);
|
|
263
395
|
consecutive_failures = consecutive_failures.saturating_add(1);
|
|
264
396
|
let next_sleep_sec = backoff_sleep_sec(tick_interval, consecutive_failures);
|
|
265
397
|
// P7-F2 (Python __main__.py:66-89): identical-signature failures emit
|
|
@@ -303,10 +435,54 @@ fn run_daemon_with_coordinator_and_boot_tmux(
|
|
|
303
435
|
}
|
|
304
436
|
}
|
|
305
437
|
}
|
|
306
|
-
|
|
438
|
+
let _ = write_coordinator_heartbeat(
|
|
439
|
+
&args.workspace,
|
|
440
|
+
pid,
|
|
441
|
+
Some(boot_id),
|
|
442
|
+
"exit",
|
|
443
|
+
Some("stop_requested"),
|
|
444
|
+
None,
|
|
445
|
+
false,
|
|
446
|
+
);
|
|
447
|
+
write_daemon_exit(
|
|
448
|
+
&event_log,
|
|
449
|
+
&args.workspace,
|
|
450
|
+
pid,
|
|
451
|
+
boot_id,
|
|
452
|
+
"stop",
|
|
453
|
+
serde_json::json!({"stop": true}),
|
|
454
|
+
)?;
|
|
307
455
|
Ok(())
|
|
308
456
|
}
|
|
309
457
|
|
|
458
|
+
fn write_daemon_exit(
|
|
459
|
+
event_log: &EventLog,
|
|
460
|
+
workspace: &WorkspacePath,
|
|
461
|
+
pid: Pid,
|
|
462
|
+
boot_id: &str,
|
|
463
|
+
reason: &str,
|
|
464
|
+
mut extra: serde_json::Value,
|
|
465
|
+
) -> Result<(), crate::event_log::EventLogError> {
|
|
466
|
+
let mut event = serde_json::json!({
|
|
467
|
+
"reason": reason,
|
|
468
|
+
"pid": pid.get(),
|
|
469
|
+
"boot_id": boot_id,
|
|
470
|
+
"workspace": workspace.as_path().to_string_lossy(),
|
|
471
|
+
});
|
|
472
|
+
if let (Some(target), Some(extra_obj)) = (event.as_object_mut(), extra.as_object_mut()) {
|
|
473
|
+
target.extend(std::mem::take(extra_obj));
|
|
474
|
+
}
|
|
475
|
+
event_log.write("coordinator.exit", event).map(|_| ())
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
fn daemon_boot_id(pid: Pid) -> String {
|
|
479
|
+
format!(
|
|
480
|
+
"coord_{}_{}",
|
|
481
|
+
chrono::Utc::now().format("%Y%m%dT%H%M%SZ"),
|
|
482
|
+
pid.get()
|
|
483
|
+
)
|
|
484
|
+
}
|
|
485
|
+
|
|
310
486
|
fn run_tick_with_panic_marker<F>(event_log: &EventLog, tick: F) -> Result<TickReport, TickError>
|
|
311
487
|
where
|
|
312
488
|
F: FnOnce() -> Result<TickReport, TickError>,
|
|
@@ -388,6 +564,52 @@ pub enum DaemonError {
|
|
|
388
564
|
EventLog(#[from] crate::event_log::EventLogError),
|
|
389
565
|
#[error("tick: {0}")]
|
|
390
566
|
Tick(#[from] TickError),
|
|
567
|
+
#[error("panic: {0}")]
|
|
568
|
+
Panic(String),
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
#[cfg(unix)]
|
|
572
|
+
static SIGNAL_STOP_REQUESTED: std::sync::atomic::AtomicBool =
|
|
573
|
+
std::sync::atomic::AtomicBool::new(false);
|
|
574
|
+
#[cfg(unix)]
|
|
575
|
+
static SIGNAL_STOP_NUMBER: std::sync::atomic::AtomicI32 =
|
|
576
|
+
std::sync::atomic::AtomicI32::new(0);
|
|
577
|
+
|
|
578
|
+
#[cfg(unix)]
|
|
579
|
+
extern "C" fn coordinator_signal_handler(signal: libc::c_int) {
|
|
580
|
+
SIGNAL_STOP_NUMBER.store(signal, std::sync::atomic::Ordering::SeqCst);
|
|
581
|
+
SIGNAL_STOP_REQUESTED.store(true, std::sync::atomic::Ordering::SeqCst);
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
#[cfg(unix)]
|
|
585
|
+
fn install_signal_handlers() {
|
|
586
|
+
unsafe {
|
|
587
|
+
let handler = coordinator_signal_handler as *const () as libc::sighandler_t;
|
|
588
|
+
libc::signal(libc::SIGTERM, handler);
|
|
589
|
+
libc::signal(libc::SIGHUP, handler);
|
|
590
|
+
libc::signal(libc::SIGINT, handler);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
#[cfg(not(unix))]
|
|
595
|
+
fn install_signal_handlers() {}
|
|
596
|
+
|
|
597
|
+
#[cfg(unix)]
|
|
598
|
+
fn take_signal_stop_request() -> Option<&'static str> {
|
|
599
|
+
if !SIGNAL_STOP_REQUESTED.swap(false, std::sync::atomic::Ordering::SeqCst) {
|
|
600
|
+
return None;
|
|
601
|
+
}
|
|
602
|
+
match SIGNAL_STOP_NUMBER.load(std::sync::atomic::Ordering::SeqCst) {
|
|
603
|
+
libc::SIGTERM => Some("SIGTERM"),
|
|
604
|
+
libc::SIGHUP => Some("SIGHUP"),
|
|
605
|
+
libc::SIGINT => Some("SIGINT"),
|
|
606
|
+
_ => Some("UNKNOWN"),
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
#[cfg(not(unix))]
|
|
611
|
+
fn take_signal_stop_request() -> Option<&'static str> {
|
|
612
|
+
None
|
|
391
613
|
}
|
|
392
614
|
|
|
393
615
|
#[cfg(test)]
|
|
@@ -33,6 +33,7 @@ const STARTUP_PROMPT_GRACE_SECS: i64 = 120;
|
|
|
33
33
|
const RUNTIME_APPROVAL_INITIAL_BACKOFF_SECS: i64 = 30;
|
|
34
34
|
const RUNTIME_APPROVAL_MAX_BACKOFF_SECS: i64 = 300;
|
|
35
35
|
const IDLE_HEALTH_CAPTURE_INTERVAL_SECS: i64 = 60;
|
|
36
|
+
pub(crate) const HEARTBEAT_STATUS_PANIC: &str = "panic";
|
|
36
37
|
|
|
37
38
|
// ===========================================================================
|
|
38
39
|
// TickReport / TickError(§10:tick(..) -> Result<TickReport, TickError>)
|
|
@@ -1420,21 +1421,87 @@ impl TurnStateClassifier for ProviderTurnClassifier {
|
|
|
1420
1421
|
/// `coordinator.coordinator_tick_iteration_count` load fine (read-compat, C-P3-3) —
|
|
1421
1422
|
/// new versions simply stop writing it.
|
|
1422
1423
|
fn increment_coordinator_tick_iteration_count(workspace: &WorkspacePath) {
|
|
1423
|
-
let
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1424
|
+
let _ = write_coordinator_heartbeat(
|
|
1425
|
+
workspace,
|
|
1426
|
+
Pid::new(std::process::id()),
|
|
1427
|
+
None,
|
|
1428
|
+
"tick_running",
|
|
1429
|
+
Some("running"),
|
|
1430
|
+
None,
|
|
1431
|
+
true,
|
|
1432
|
+
);
|
|
1433
|
+
}
|
|
1434
|
+
|
|
1435
|
+
pub(crate) fn write_coordinator_heartbeat(
|
|
1436
|
+
workspace: &WorkspacePath,
|
|
1437
|
+
pid: Pid,
|
|
1438
|
+
boot_id: Option<&str>,
|
|
1439
|
+
last_phase: &str,
|
|
1440
|
+
last_tick_status: Option<&str>,
|
|
1441
|
+
last_error: Option<&str>,
|
|
1442
|
+
increment_count: bool,
|
|
1443
|
+
) -> std::io::Result<()> {
|
|
1444
|
+
let path = coordinator_heartbeat_path(workspace);
|
|
1445
|
+
let value = read_coordinator_heartbeat_value(&path);
|
|
1446
|
+
let next_count = value
|
|
1447
|
+
.get("coordinator_tick_iteration_count")
|
|
1448
|
+
.and_then(Value::as_u64)
|
|
1449
|
+
.unwrap_or(0)
|
|
1450
|
+
.saturating_add(u64::from(increment_count));
|
|
1451
|
+
let now = chrono::Utc::now().to_rfc3339();
|
|
1452
|
+
let identity = crate::coordinator::current_coordinator_binary_identity();
|
|
1453
|
+
let boot_id = boot_id
|
|
1454
|
+
.map(ToString::to_string)
|
|
1455
|
+
.or_else(|| {
|
|
1428
1456
|
value
|
|
1429
|
-
.get("
|
|
1430
|
-
.and_then(Value::
|
|
1457
|
+
.get("boot_id")
|
|
1458
|
+
.and_then(Value::as_str)
|
|
1459
|
+
.map(ToString::to_string)
|
|
1431
1460
|
})
|
|
1432
|
-
.
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1461
|
+
.unwrap_or_else(|| format!("coord_unknown_{}", pid.get()));
|
|
1462
|
+
let mut next = serde_json::json!({
|
|
1463
|
+
"coordinator_tick_iteration_count": next_count,
|
|
1464
|
+
"pid": pid.get(),
|
|
1465
|
+
"boot_id": boot_id,
|
|
1466
|
+
"binary_path": identity.binary_path,
|
|
1467
|
+
"binary_version": identity.binary_version,
|
|
1468
|
+
"last_phase": last_phase,
|
|
1469
|
+
"last_tick_status": last_tick_status,
|
|
1470
|
+
"last_error": last_error,
|
|
1471
|
+
"updated_at": now,
|
|
1472
|
+
});
|
|
1473
|
+
if last_phase == "tick_running" {
|
|
1474
|
+
next["last_tick_started_at"] = serde_json::Value::String(now.clone());
|
|
1475
|
+
if let Some(finished) = value.get("last_tick_finished_at").cloned() {
|
|
1476
|
+
next["last_tick_finished_at"] = finished;
|
|
1477
|
+
}
|
|
1478
|
+
} else {
|
|
1479
|
+
if let Some(started) = value.get("last_tick_started_at").cloned() {
|
|
1480
|
+
next["last_tick_started_at"] = started;
|
|
1481
|
+
}
|
|
1482
|
+
next["last_tick_finished_at"] = serde_json::Value::String(chrono::Utc::now().to_rfc3339());
|
|
1483
|
+
}
|
|
1484
|
+
write_coordinator_heartbeat_value(&path, &next)
|
|
1485
|
+
}
|
|
1486
|
+
|
|
1487
|
+
fn coordinator_heartbeat_path(workspace: &WorkspacePath) -> PathBuf {
|
|
1488
|
+
crate::model::paths::runtime_dir(workspace.as_path()).join("coordinator_tick.json")
|
|
1489
|
+
}
|
|
1490
|
+
|
|
1491
|
+
fn read_coordinator_heartbeat_value(path: &Path) -> Value {
|
|
1492
|
+
std::fs::read_to_string(path)
|
|
1493
|
+
.ok()
|
|
1494
|
+
.and_then(|text| serde_json::from_str::<Value>(&text).ok())
|
|
1495
|
+
.unwrap_or_else(|| serde_json::json!({}))
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
fn write_coordinator_heartbeat_value(path: &Path, value: &Value) -> std::io::Result<()> {
|
|
1499
|
+
if let Some(parent) = path.parent() {
|
|
1500
|
+
std::fs::create_dir_all(parent)?;
|
|
1501
|
+
}
|
|
1502
|
+
let tmp = path.with_extension("json.tmp");
|
|
1503
|
+
std::fs::write(&tmp, value.to_string())?;
|
|
1504
|
+
std::fs::rename(tmp, path)
|
|
1438
1505
|
}
|
|
1439
1506
|
|
|
1440
1507
|
fn idle_node_value(node: &crate::leader::IdleNode) -> Value {
|
|
@@ -18,10 +18,10 @@ pub fn doctor_gate_blockers(
|
|
|
18
18
|
let _ = (fix, confirm);
|
|
19
19
|
match gate {
|
|
20
20
|
Some(DoctorGate::Orphans) => {
|
|
21
|
-
if orphans::has_orphan_residue() {
|
|
21
|
+
if orphans::has_orphan_residue(workspace) {
|
|
22
22
|
Ok(vec![Blocker {
|
|
23
23
|
source: BlockerSource::OrphanCoordinator,
|
|
24
|
-
detail: orphans::orphan_blocker_detail(),
|
|
24
|
+
detail: orphans::orphan_blocker_detail(workspace),
|
|
25
25
|
}])
|
|
26
26
|
} else {
|
|
27
27
|
Ok(Vec::new())
|
|
@@ -32,9 +32,57 @@ struct OrphanRecord {
|
|
|
32
32
|
struct ScanReport {
|
|
33
33
|
scanned: usize,
|
|
34
34
|
orphans: Vec<OrphanRecord>,
|
|
35
|
+
ignored_foreign: Vec<OrphanRecord>,
|
|
35
36
|
}
|
|
36
37
|
|
|
37
|
-
|
|
38
|
+
#[derive(Debug, Clone)]
|
|
39
|
+
pub(crate) enum OrphanScanScope {
|
|
40
|
+
CurrentWorkspace {
|
|
41
|
+
canonical_current_workspace: PathBuf,
|
|
42
|
+
},
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
impl OrphanScanScope {
|
|
46
|
+
fn current_workspace(workspace: &Path) -> Self {
|
|
47
|
+
Self::CurrentWorkspace {
|
|
48
|
+
canonical_current_workspace: canonicalize_workspace(workspace),
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
fn canonicalize_workspace(workspace: &Path) -> PathBuf {
|
|
54
|
+
std::fs::canonicalize(workspace).unwrap_or_else(|_| workspace.to_path_buf())
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
fn workspace_matches_scope(scope: &OrphanScanScope, workspace: &Path) -> bool {
|
|
58
|
+
match scope {
|
|
59
|
+
OrphanScanScope::CurrentWorkspace {
|
|
60
|
+
canonical_current_workspace,
|
|
61
|
+
} => canonicalize_workspace(workspace) == *canonical_current_workspace,
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
fn assert_current_workspace_orphan(orphan: &OrphanRecord, scope: &OrphanScanScope) -> bool {
|
|
66
|
+
orphan
|
|
67
|
+
.workspace
|
|
68
|
+
.as_deref()
|
|
69
|
+
.is_some_and(|workspace| workspace_matches_scope(scope, workspace))
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
fn foreign_workspace_record(process: ProcessRow, workspace: PathBuf) -> OrphanRecord {
|
|
73
|
+
OrphanRecord {
|
|
74
|
+
kind: "coordinator_process",
|
|
75
|
+
pid: Some(process.pid),
|
|
76
|
+
session: None,
|
|
77
|
+
tmux_socket: None,
|
|
78
|
+
workspace: Some(workspace),
|
|
79
|
+
reason: OrphanReason::WorkspaceAlive,
|
|
80
|
+
command: Some(process.command),
|
|
81
|
+
action: "none",
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
pub fn orphan_gate_json(workspace: &Path, fix: bool, confirm: bool) -> Result<Value, CliError> {
|
|
38
86
|
if fix && !confirm {
|
|
39
87
|
return Ok(json!({
|
|
40
88
|
"ok": false,
|
|
@@ -44,9 +92,10 @@ pub fn orphan_gate_json(fix: bool, confirm: bool) -> Result<Value, CliError> {
|
|
|
44
92
|
"action": "re-run with --gate orphans --fix --confirm",
|
|
45
93
|
}));
|
|
46
94
|
}
|
|
47
|
-
let
|
|
95
|
+
let scope = OrphanScanScope::current_workspace(workspace);
|
|
96
|
+
let report = scan_orphans_bounded(&scope, false);
|
|
48
97
|
if report.orphans.is_empty() {
|
|
49
|
-
return Ok(json!({
|
|
98
|
+
return Ok(with_ignored_foreign(json!({
|
|
50
99
|
"ok": true,
|
|
51
100
|
"gate": "orphans",
|
|
52
101
|
"status": "passed",
|
|
@@ -56,12 +105,12 @@ pub fn orphan_gate_json(fix: bool, confirm: bool) -> Result<Value, CliError> {
|
|
|
56
105
|
"action_required": false,
|
|
57
106
|
"fix": fix,
|
|
58
107
|
"orphans": [],
|
|
59
|
-
}));
|
|
108
|
+
}), &report));
|
|
60
109
|
}
|
|
61
110
|
if fix {
|
|
62
|
-
return fix_orphans(report);
|
|
111
|
+
return fix_orphans(&scope, report);
|
|
63
112
|
}
|
|
64
|
-
Ok(json!({
|
|
113
|
+
Ok(with_ignored_foreign(json!({
|
|
65
114
|
"ok": false,
|
|
66
115
|
"gate": "orphans",
|
|
67
116
|
"status": "failed",
|
|
@@ -71,14 +120,15 @@ pub fn orphan_gate_json(fix: bool, confirm: bool) -> Result<Value, CliError> {
|
|
|
71
120
|
"action_required": true,
|
|
72
121
|
"fix": false,
|
|
73
122
|
"orphans": orphan_values(&report.orphans),
|
|
74
|
-
}))
|
|
123
|
+
}), &report))
|
|
75
124
|
}
|
|
76
125
|
|
|
77
|
-
pub fn cleanup_orphans_json(confirm: bool) -> Result<Value, CliError> {
|
|
78
|
-
let
|
|
126
|
+
pub fn cleanup_orphans_json(workspace: &Path, confirm: bool) -> Result<Value, CliError> {
|
|
127
|
+
let scope = OrphanScanScope::current_workspace(workspace);
|
|
128
|
+
let report = scan_orphans_bounded(&scope, false);
|
|
79
129
|
if confirm {
|
|
80
130
|
if report.orphans.is_empty() {
|
|
81
|
-
return Ok(json!({
|
|
131
|
+
return Ok(with_ignored_foreign(json!({
|
|
82
132
|
"ok": true,
|
|
83
133
|
"scanned": report.scanned,
|
|
84
134
|
"orphans": [],
|
|
@@ -86,26 +136,28 @@ pub fn cleanup_orphans_json(confirm: bool) -> Result<Value, CliError> {
|
|
|
86
136
|
"scanned_at": chrono::Utc::now().to_rfc3339(),
|
|
87
137
|
"killed": [],
|
|
88
138
|
"failed": [],
|
|
89
|
-
}));
|
|
139
|
+
}), &report));
|
|
90
140
|
}
|
|
91
|
-
return cleanup_confirmed(report);
|
|
141
|
+
return cleanup_confirmed(&scope, report);
|
|
92
142
|
}
|
|
93
|
-
Ok(json!({
|
|
143
|
+
Ok(with_ignored_foreign(json!({
|
|
94
144
|
"ok": true,
|
|
95
145
|
"scanned": report.scanned,
|
|
96
146
|
"orphans": orphan_values(&report.orphans),
|
|
97
147
|
"dry_run": true,
|
|
98
148
|
"scanned_at": chrono::Utc::now().to_rfc3339(),
|
|
99
149
|
"action_required": "re-run with --confirm to send SIGTERM",
|
|
100
|
-
}))
|
|
150
|
+
}), &report))
|
|
101
151
|
}
|
|
102
152
|
|
|
103
|
-
pub fn has_orphan_residue() -> bool {
|
|
104
|
-
|
|
153
|
+
pub fn has_orphan_residue(workspace: &Path) -> bool {
|
|
154
|
+
let scope = OrphanScanScope::current_workspace(workspace);
|
|
155
|
+
!scan_orphans_bounded(&scope, false).orphans.is_empty()
|
|
105
156
|
}
|
|
106
157
|
|
|
107
|
-
pub fn orphan_blocker_detail() -> String {
|
|
108
|
-
let
|
|
158
|
+
pub fn orphan_blocker_detail(workspace: &Path) -> String {
|
|
159
|
+
let scope = OrphanScanScope::current_workspace(workspace);
|
|
160
|
+
let report = scan_orphans_bounded(&scope, false);
|
|
109
161
|
if report.orphans.is_empty() {
|
|
110
162
|
return "no orphan coordinator residue detected".to_string();
|
|
111
163
|
}
|
|
@@ -133,10 +185,10 @@ pub fn orphan_blocker_detail() -> String {
|
|
|
133
185
|
.join("; ")
|
|
134
186
|
}
|
|
135
187
|
|
|
136
|
-
fn fix_orphans(report: ScanReport) -> Result<Value, CliError> {
|
|
137
|
-
let cleanup = cleanup_report(report);
|
|
138
|
-
let residual = scan_orphans(false);
|
|
139
|
-
Ok(json!({
|
|
188
|
+
fn fix_orphans(scope: &OrphanScanScope, report: ScanReport) -> Result<Value, CliError> {
|
|
189
|
+
let cleanup = cleanup_report(report, scope);
|
|
190
|
+
let residual = scan_orphans(scope, false);
|
|
191
|
+
Ok(with_ignored_foreign(json!({
|
|
140
192
|
"ok": residual.orphans.is_empty() && cleanup.failed.is_empty(),
|
|
141
193
|
"gate": "orphans",
|
|
142
194
|
"status": if residual.orphans.is_empty() && cleanup.failed.is_empty() { "fixed" } else { "failed" },
|
|
@@ -148,13 +200,13 @@ fn fix_orphans(report: ScanReport) -> Result<Value, CliError> {
|
|
|
148
200
|
"orphans": orphan_values(&residual.orphans),
|
|
149
201
|
"killed": cleanup.killed,
|
|
150
202
|
"failed": cleanup.failed,
|
|
151
|
-
}))
|
|
203
|
+
}), &residual))
|
|
152
204
|
}
|
|
153
205
|
|
|
154
|
-
fn cleanup_confirmed(report: ScanReport) -> Result<Value, CliError> {
|
|
155
|
-
let cleanup = cleanup_report(report);
|
|
156
|
-
let residual = scan_orphans(false);
|
|
157
|
-
Ok(json!({
|
|
206
|
+
fn cleanup_confirmed(scope: &OrphanScanScope, report: ScanReport) -> Result<Value, CliError> {
|
|
207
|
+
let cleanup = cleanup_report(report, scope);
|
|
208
|
+
let residual = scan_orphans(scope, false);
|
|
209
|
+
Ok(with_ignored_foreign(json!({
|
|
158
210
|
"ok": residual.orphans.is_empty() && cleanup.failed.is_empty(),
|
|
159
211
|
"scanned": cleanup.scanned,
|
|
160
212
|
"orphans": orphan_values(&residual.orphans),
|
|
@@ -162,7 +214,7 @@ fn cleanup_confirmed(report: ScanReport) -> Result<Value, CliError> {
|
|
|
162
214
|
"scanned_at": chrono::Utc::now().to_rfc3339(),
|
|
163
215
|
"killed": cleanup.killed,
|
|
164
216
|
"failed": cleanup.failed,
|
|
165
|
-
}))
|
|
217
|
+
}), &residual))
|
|
166
218
|
}
|
|
167
219
|
|
|
168
220
|
struct CleanupReport {
|
|
@@ -171,11 +223,15 @@ struct CleanupReport {
|
|
|
171
223
|
failed: Vec<Value>,
|
|
172
224
|
}
|
|
173
225
|
|
|
174
|
-
fn cleanup_report(report: ScanReport) -> CleanupReport {
|
|
226
|
+
fn cleanup_report(report: ScanReport, scope: &OrphanScanScope) -> CleanupReport {
|
|
175
227
|
let protected = protected_pids();
|
|
176
228
|
let mut killed = Vec::new();
|
|
177
229
|
let mut failed = Vec::new();
|
|
178
230
|
for orphan in &report.orphans {
|
|
231
|
+
if !assert_current_workspace_orphan(orphan, scope) {
|
|
232
|
+
failed.push(orphan_value(orphan, "skipped"));
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
179
235
|
if let Some(pid) = orphan.pid {
|
|
180
236
|
if protected.contains(&pid.get()) {
|
|
181
237
|
failed.push(orphan_value(orphan, "skipped"));
|
|
@@ -201,10 +257,11 @@ fn cleanup_report(report: ScanReport) -> CleanupReport {
|
|
|
201
257
|
}
|
|
202
258
|
}
|
|
203
259
|
|
|
204
|
-
fn scan_orphans(include_unparsed: bool) -> ScanReport {
|
|
260
|
+
fn scan_orphans(scope: &OrphanScanScope, include_unparsed: bool) -> ScanReport {
|
|
205
261
|
let protected = protected_pids();
|
|
206
262
|
let mut scanned = 0;
|
|
207
263
|
let mut orphans = Vec::new();
|
|
264
|
+
let mut ignored_foreign = Vec::new();
|
|
208
265
|
for process in coordinator_processes() {
|
|
209
266
|
if protected.contains(&process.pid.get()) {
|
|
210
267
|
continue;
|
|
@@ -225,6 +282,10 @@ fn scan_orphans(include_unparsed: bool) -> ScanReport {
|
|
|
225
282
|
}
|
|
226
283
|
continue;
|
|
227
284
|
};
|
|
285
|
+
if !workspace_matches_scope(scope, &workspace) {
|
|
286
|
+
ignored_foreign.push(foreign_workspace_record(process, workspace));
|
|
287
|
+
continue;
|
|
288
|
+
}
|
|
228
289
|
if let Some(reason) = classify_workspace_orphan(&workspace, process.pid) {
|
|
229
290
|
orphans.push(OrphanRecord {
|
|
230
291
|
kind: "coordinator_process",
|
|
@@ -238,22 +299,26 @@ fn scan_orphans(include_unparsed: bool) -> ScanReport {
|
|
|
238
299
|
});
|
|
239
300
|
}
|
|
240
301
|
}
|
|
241
|
-
for orphan in coordinator_pid_file_orphans() {
|
|
302
|
+
for orphan in coordinator_pid_file_orphans(scope) {
|
|
242
303
|
scanned += 1;
|
|
243
304
|
orphans.push(orphan);
|
|
244
305
|
}
|
|
245
|
-
for orphan in tmux_session_orphans() {
|
|
306
|
+
for orphan in tmux_session_orphans(scope) {
|
|
246
307
|
scanned += 1;
|
|
247
308
|
orphans.push(orphan);
|
|
248
309
|
}
|
|
249
|
-
for orphan in provider_mcp_process_orphans() {
|
|
310
|
+
for orphan in provider_mcp_process_orphans(scope) {
|
|
250
311
|
scanned += 1;
|
|
251
312
|
orphans.push(orphan);
|
|
252
313
|
}
|
|
253
|
-
ScanReport {
|
|
314
|
+
ScanReport {
|
|
315
|
+
scanned,
|
|
316
|
+
orphans,
|
|
317
|
+
ignored_foreign,
|
|
318
|
+
}
|
|
254
319
|
}
|
|
255
320
|
|
|
256
|
-
fn coordinator_pid_file_orphans() -> Vec<OrphanRecord> {
|
|
321
|
+
fn coordinator_pid_file_orphans(scope: &OrphanScanScope) -> Vec<OrphanRecord> {
|
|
257
322
|
temp_scan_roots()
|
|
258
323
|
.into_iter()
|
|
259
324
|
.flat_map(|root| match std::fs::read_dir(root) {
|
|
@@ -265,6 +330,9 @@ fn coordinator_pid_file_orphans() -> Vec<OrphanRecord> {
|
|
|
265
330
|
if !workspace.is_dir() || ephemeral_workspace_hint(&workspace).is_none() {
|
|
266
331
|
return None;
|
|
267
332
|
}
|
|
333
|
+
if !workspace_matches_scope(scope, &workspace) {
|
|
334
|
+
return None;
|
|
335
|
+
}
|
|
268
336
|
let pid_path = crate::model::paths::runtime_dir(&workspace).join("coordinator.pid");
|
|
269
337
|
let pid = read_pid_file(&pid_path)?;
|
|
270
338
|
let workspace_path = WorkspacePath::new(workspace.clone());
|
|
@@ -293,7 +361,7 @@ fn coordinator_pid_file_orphans() -> Vec<OrphanRecord> {
|
|
|
293
361
|
.collect()
|
|
294
362
|
}
|
|
295
363
|
|
|
296
|
-
fn tmux_session_orphans() -> Vec<OrphanRecord> {
|
|
364
|
+
fn tmux_session_orphans(scope: &OrphanScanScope) -> Vec<OrphanRecord> {
|
|
297
365
|
tmux_socket_names()
|
|
298
366
|
.into_iter()
|
|
299
367
|
.flat_map(|socket| {
|
|
@@ -304,6 +372,9 @@ fn tmux_session_orphans() -> Vec<OrphanRecord> {
|
|
|
304
372
|
if !is_orphan_marker_workspace(&workspace) {
|
|
305
373
|
return None;
|
|
306
374
|
}
|
|
375
|
+
if !workspace_matches_scope(scope, &workspace) {
|
|
376
|
+
return None;
|
|
377
|
+
}
|
|
307
378
|
let reason = classify_workspace_without_pid(&workspace)?;
|
|
308
379
|
Some(OrphanRecord {
|
|
309
380
|
kind: "tmux_session",
|
|
@@ -382,7 +453,7 @@ fn tmux_list_panes(socket: &str) -> Vec<TmuxPaneRow> {
|
|
|
382
453
|
.collect()
|
|
383
454
|
}
|
|
384
455
|
|
|
385
|
-
fn provider_mcp_process_orphans() -> Vec<OrphanRecord> {
|
|
456
|
+
fn provider_mcp_process_orphans(scope: &OrphanScanScope) -> Vec<OrphanRecord> {
|
|
386
457
|
ps_command_rows()
|
|
387
458
|
.into_iter()
|
|
388
459
|
.filter(|row| is_provider_or_mcp_workspace_command(&row.command))
|
|
@@ -391,6 +462,9 @@ fn provider_mcp_process_orphans() -> Vec<OrphanRecord> {
|
|
|
391
462
|
if !is_orphan_marker_workspace(&workspace) {
|
|
392
463
|
return None;
|
|
393
464
|
}
|
|
465
|
+
if !workspace_matches_scope(scope, &workspace) {
|
|
466
|
+
return None;
|
|
467
|
+
}
|
|
394
468
|
let reason = classify_workspace_without_pid(&workspace)?;
|
|
395
469
|
Some(OrphanRecord {
|
|
396
470
|
kind: if process.command.contains("mcp-server") {
|
|
@@ -427,16 +501,20 @@ fn read_pid_file(path: &Path) -> Option<Pid> {
|
|
|
427
501
|
Some(Pid::new(pid))
|
|
428
502
|
}
|
|
429
503
|
|
|
430
|
-
fn scan_orphans_bounded(include_unparsed: bool) -> ScanReport {
|
|
504
|
+
fn scan_orphans_bounded(scope: &OrphanScanScope, include_unparsed: bool) -> ScanReport {
|
|
431
505
|
let deadline = Instant::now() + Duration::from_millis(800);
|
|
432
506
|
let mut scanned = 0;
|
|
433
507
|
let mut by_key = BTreeMap::new();
|
|
508
|
+
let mut foreign_by_key = BTreeMap::new();
|
|
434
509
|
loop {
|
|
435
|
-
let report = scan_orphans(include_unparsed);
|
|
510
|
+
let report = scan_orphans(scope, include_unparsed);
|
|
436
511
|
scanned = scanned.max(report.scanned);
|
|
437
512
|
for orphan in report.orphans {
|
|
438
513
|
by_key.insert(orphan_key(&orphan), orphan);
|
|
439
514
|
}
|
|
515
|
+
for orphan in report.ignored_foreign {
|
|
516
|
+
foreign_by_key.insert(orphan_key(&orphan), orphan);
|
|
517
|
+
}
|
|
440
518
|
if !by_key.is_empty() || Instant::now() >= deadline {
|
|
441
519
|
break;
|
|
442
520
|
}
|
|
@@ -445,6 +523,7 @@ fn scan_orphans_bounded(include_unparsed: bool) -> ScanReport {
|
|
|
445
523
|
ScanReport {
|
|
446
524
|
scanned,
|
|
447
525
|
orphans: by_key.into_values().collect(),
|
|
526
|
+
ignored_foreign: foreign_by_key.into_values().collect(),
|
|
448
527
|
}
|
|
449
528
|
}
|
|
450
529
|
|
|
@@ -653,10 +732,23 @@ fn orphan_values(orphans: &[OrphanRecord]) -> Vec<Value> {
|
|
|
653
732
|
.collect()
|
|
654
733
|
}
|
|
655
734
|
|
|
735
|
+
fn with_ignored_foreign(mut value: Value, report: &ScanReport) -> Value {
|
|
736
|
+
if !report.ignored_foreign.is_empty() {
|
|
737
|
+
value["ignored_foreign"] = Value::Array(
|
|
738
|
+
report
|
|
739
|
+
.ignored_foreign
|
|
740
|
+
.iter()
|
|
741
|
+
.map(|orphan| orphan_value(orphan, "none"))
|
|
742
|
+
.collect(),
|
|
743
|
+
);
|
|
744
|
+
}
|
|
745
|
+
value
|
|
746
|
+
}
|
|
747
|
+
|
|
656
748
|
fn orphan_value(orphan: &OrphanRecord, action: &str) -> Value {
|
|
657
749
|
let mut value = json!({
|
|
658
750
|
"kind": orphan.kind,
|
|
659
|
-
"reason": reason_key(&orphan.reason),
|
|
751
|
+
"reason": if action == "none" { "foreign_workspace" } else { reason_key(&orphan.reason) },
|
|
660
752
|
"action": action,
|
|
661
753
|
});
|
|
662
754
|
if let Some(pid) = orphan.pid {
|
|
@@ -695,6 +695,9 @@ pub enum RestartReport {
|
|
|
695
695
|
Restarted {
|
|
696
696
|
session_name: SessionName,
|
|
697
697
|
agents: Vec<RestartedAgent>,
|
|
698
|
+
/// Compatibility field: true means the coordinator requirement is
|
|
699
|
+
/// satisfied. The structured `coordinator` summary carries started vs
|
|
700
|
+
/// already-running vs rotated detail.
|
|
698
701
|
coordinator_started: bool,
|
|
699
702
|
coordinator: CoordinatorStartSummary,
|
|
700
703
|
next_actions: Vec<String>,
|
|
@@ -706,6 +709,9 @@ pub enum RestartReport {
|
|
|
706
709
|
session_name: SessionName,
|
|
707
710
|
agents: Vec<RestartedAgent>,
|
|
708
711
|
failed_agents: Vec<RestartFailedAgent>,
|
|
712
|
+
/// Compatibility field: true means the coordinator requirement is
|
|
713
|
+
/// satisfied. The structured `coordinator` summary carries started vs
|
|
714
|
+
/// already-running vs rotated detail.
|
|
709
715
|
coordinator_started: bool,
|
|
710
716
|
coordinator: CoordinatorStartSummary,
|
|
711
717
|
next_actions: Vec<String>,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@team-agent/installer",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.20",
|
|
4
4
|
"description": "npx installer for Team Agent",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"codex",
|
|
@@ -20,9 +20,9 @@
|
|
|
20
20
|
"team-agent-installer": "npm/install.mjs"
|
|
21
21
|
},
|
|
22
22
|
"optionalDependencies": {
|
|
23
|
-
"@team-agent/cli-darwin-arm64": "0.5.
|
|
24
|
-
"@team-agent/cli-darwin-x64": "0.5.
|
|
25
|
-
"@team-agent/cli-linux-x64": "0.5.
|
|
23
|
+
"@team-agent/cli-darwin-arm64": "0.5.20",
|
|
24
|
+
"@team-agent/cli-darwin-x64": "0.5.20",
|
|
25
|
+
"@team-agent/cli-linux-x64": "0.5.20"
|
|
26
26
|
},
|
|
27
27
|
"scripts": {
|
|
28
28
|
"postinstall": "node npm/bincheck.mjs",
|